From: Krisztian Lengyel <krisztian.lengyel@nokia.com>
Date: Fri, 7 Jun 2019 10:55:29 +0000 (+0200)
Subject: Make CPU reservations based on CPU allocation
X-Git-Url: https://gerrit.akraino.org/r/gitweb?p=ta%2Fcaas-kubernetes.git;a=commitdiff_plain;h=df3fea70cdc2661a7cfd5fbadcbc5a857b7c6413

Make CPU reservations based on CPU allocation

Kubelet CPU reservations set to configure proper node allocatable resources:
- kube-reserved currently set to 0
- system-reserved is calculated by the following formula:
  system-reserved = total cpu count - kube-reserved count - default cpu pool

Change-Id: Ibc937a533c881d55e2933c55eb3c208200c12a4f
Signed-off-by: Krisztian Lengyel <krisztian.lengyel@nokia.com>
---

diff --git a/ansible/roles/kubelet/defaults/main.yaml b/ansible/roles/kubelet/defaults/main.yaml
index 1bf3c9a..c0c071b 100644
--- a/ansible/roles/kubelet/defaults/main.yaml
+++ b/ansible/roles/kubelet/defaults/main.yaml
@@ -34,3 +34,9 @@ kubectl_configs:
   - path: "/home/{{ users.admin_user_name }}/.kube/config"
     owner: "{{ users.admin_user_name }}"
     group: "{{ users.admin_user_name }}"
+
+kube_reserved_cpu: 0
+total_cpus: "{{ ansible_local['cpu_allocation'].values() | map(attribute='set') | map('length') | sum }}"
+default_allocation: "{{ caas.cpu_allocation_mapping | get_mapped_key('pool_name', 'default') }}"
+default_cpu: "{{ ansible_local['cpu_allocation'].get(default_allocation, {}).get('set', []) | length }}"
+system_reserved_cpu: "{{ total_cpus|int - kube_reserved_cpu|int - default_cpu|int  }}"
diff --git a/ansible/roles/kubelet/templates/common-config.yaml.j2 b/ansible/roles/kubelet/templates/common-config.yaml.j2
new file mode 100644
index 0000000..7065aeb
--- /dev/null
+++ b/ansible/roles/kubelet/templates/common-config.yaml.j2
@@ -0,0 +1,28 @@
+apiVersion: kubelet.config.k8s.io/v1beta1
+kind: KubeletConfiguration
+address: "{{ networking.infra_internal.ip }}"
+authentication:
+  x509:
+    clientCAFile: "/etc/openssl/ca.pem"
+  webhook:
+    enabled: true
+  anonymous:
+    enabled: false
+authorization:
+  mode: "AlwaysAllow"
+cgroupsPerQOS: true
+cgroupRoot: "/"
+cgroupDriver: cgroupfs
+clusterDNS:
+- {{ caas.dns_svc_ip }}
+clusterDomain: {{ caas.dns_domain }}
+kubeReserved:
+  cpu: "{{ kube_reserved_cpu }}"
+rotateCertificates: true
+runtimeRequestTimeout: 5m0s
+staticPodPath: "/etc/kubernetes/manifests"
+streamingConnectionIdleTimeout: 5m0s
+systemReserved:
+  cpu: "{{ system_reserved_cpu }}"
+tlsCertFile: "/etc/kubernetes/ssl/kubelet-server.pem"
+tlsPrivateKeyFile: "/etc/kubernetes/ssl/kubelet-server-key.pem"
diff --git a/ansible/roles/kubelet/templates/master-config.yaml.j2 b/ansible/roles/kubelet/templates/master-config.yaml.j2
index 453bb02..95b0d7b 100644
--- a/ansible/roles/kubelet/templates/master-config.yaml.j2
+++ b/ansible/roles/kubelet/templates/master-config.yaml.j2
@@ -1,3 +1,4 @@
+#jinja2:trim_blocks:False
 {#
 Copyright 2019 Nokia
 
@@ -14,30 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 #}
 ---
-apiVersion: kubelet.config.k8s.io/v1beta1
-kind: KubeletConfiguration
-address: "{{ networking.infra_internal.ip }}"
-authentication:
-  x509:
-    clientCAFile: "/etc/openssl/ca.pem"
-  webhook:
-    enabled: true
-  anonymous:
-    enabled: false
-authorization:
-  mode: "AlwaysAllow"
-cgroupsPerQOS: true
-cgroupRoot: "/"
-cgroupDriver: cgroupfs
-clusterDNS:
-- {{ caas.dns_svc_ip }}
-clusterDomain: {{ caas.dns_domain }}
-staticPodPath: "/etc/kubernetes/manifests"
-rotateCertificates: true
-runtimeRequestTimeout: 5m0s
-streamingConnectionIdleTimeout: 5m0s
-tlsCertFile: "/etc/kubernetes/ssl/kubelet-server.pem"
-tlsPrivateKeyFile: "/etc/kubernetes/ssl/kubelet-server-key.pem"
+{% include 'common-config.yaml.j2' %}
 featureGates:
   CPUManager: false
   DevicePlugins: true
diff --git a/ansible/roles/kubelet/templates/worker-config.yaml.j2 b/ansible/roles/kubelet/templates/worker-config.yaml.j2
index 0d86736..756feb6 100644
--- a/ansible/roles/kubelet/templates/worker-config.yaml.j2
+++ b/ansible/roles/kubelet/templates/worker-config.yaml.j2
@@ -1,3 +1,4 @@
+#jinja2:trim_blocks:False
 {#
 Copyright 2019 Nokia
 
@@ -14,30 +15,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 #}
 ---
-apiVersion: kubelet.config.k8s.io/v1beta1
-kind: KubeletConfiguration
-address: "{{ networking.infra_internal.ip }}"
-authentication:
-  x509:
-    clientCAFile: "/etc/openssl/ca.pem"
-  webhook:
-    enabled: true
-  anonymous:
-    enabled: false
-authorization:
-  mode: "AlwaysAllow"
-cgroupsPerQOS: true
-cgroupRoot: "/"
-cgroupDriver: cgroupfs
-clusterDNS:
-- {{ caas.dns_svc_ip }}
-clusterDomain: {{ caas.dns_domain }}
-staticPodPath: "/etc/kubernetes/manifests"
-rotateCertificates: true
-runtimeRequestTimeout: 5m0s
-streamingConnectionIdleTimeout: 5m0s
-tlsCertFile: "/etc/kubernetes/ssl/kubelet-server.pem"
-tlsPrivateKeyFile: "/etc/kubernetes/ssl/kubelet-server-key.pem"
+{% include 'common-config.yaml.j2' %}
 featureGates:
   CPUManager: false
   DevicePlugins: true
diff --git a/caas-kubernetes.spec b/caas-kubernetes.spec
index beee7b5..4323fa5 100644
--- a/caas-kubernetes.spec
+++ b/caas-kubernetes.spec
@@ -15,7 +15,7 @@
 %define COMPONENT kubernetes
 %define RPM_NAME caas-%{COMPONENT}
 %define RPM_MAJOR_VERSION 1.14.0
-%define RPM_MINOR_VERSION 1
+%define RPM_MINOR_VERSION 3
 %define IMAGE_TAG %{RPM_MAJOR_VERSION}-%{RPM_MINOR_VERSION}
 %define KUBERNETESPAUSE_VERSION 3.1