kubernetes role for controller nodes

roles/kubernetes/files/audit-policy.yml

@@ -0,0 +1,68 @@
+apiVersion: audit.k8s.io/v1 # This is required.
+kind: Policy
+# Don't generate audit events for all requests in RequestReceived stage.
+omitStages:
+  - "RequestReceived"
+rules:
+  # Log pod changes at RequestResponse level
+  - level: RequestResponse
+    resources:
+    - group: ""
+      # Resource "pods" doesn't match requests to any subresource of pods,
+      # which is consistent with the RBAC policy.
+      resources: ["pods"]
+  # Log "pods/log", "pods/status" at Metadata level
+  - level: Metadata
+    resources:
+    - group: ""
+      resources: ["pods/log", "pods/status"]
+
+  # Don't log requests to a configmap called "controller-leader"
+  - level: None
+    resources:
+    - group: ""
+      resources: ["configmaps"]
+      resourceNames: ["controller-leader"]
+
+  # Don't log watch requests by the "system:kube-proxy" on endpoints or services
+  - level: None
+    users: ["system:kube-proxy"]
+    verbs: ["watch"]
+    resources:
+    - group: "" # core API group
+      resources: ["endpoints", "services"]
+
+  # Don't log authenticated requests to certain non-resource URL paths.
+  - level: None
+    userGroups: ["system:authenticated"]
+    nonResourceURLs:
+    - "/api*" # Wildcard matching.
+    - "/version"
+
+  # Log the request body of configmap changes in kube-system.
+  - level: Request
+    resources:
+    - group: "" # core API group
+      resources: ["configmaps"]
+    # This rule only applies to resources in the "kube-system" namespace.
+    # The empty string "" can be used to select non-namespaced resources.
+    namespaces: ["kube-system"]
+
+  # Log configmap and secret changes in all other namespaces at the Metadata level.
+  - level: Metadata
+    resources:
+    - group: "" # core API group
+      resources: ["secrets", "configmaps"]
+
+  # Log all other resources in core and extensions at the Request level.
+  - level: Request
+    resources:
+    - group: "" # core API group
+    - group: "extensions" # Version of group should NOT be included.
+
+  # A catch-all rule to log all other requests at the Metadata level.
+  - level: Metadata
+    # Long-running requests like watches that fall under this rule will not
+    # generate an audit event in RequestReceived.
+    omitStages:
+      - "RequestReceived"

roles/kubernetes/files/kube-scheduler.yml

@@ -0,0 +1,6 @@
+apiVersion: kubescheduler.config.k8s.io/v1beta2
+kind: KubeSchedulerConfiguration
+clientConnection:
+  kubeconfig: "/var/lib/kubernetes/kube-scheduler.kubeconfig"
+leaderElection:
+  leaderElect: true

roles/kubernetes/tasks/main.yml

@@ -0,0 +1,142 @@
+---
+- name: Download Kuberneters controller binaries
+  get_url:
+    url: "{{ kubernetes_download_path }}/{{ item }}"
+    dest: /usr/local/bin
+    owner: root
+    group: root
+    mode: 0755
+    # TODO Add hash check
+  with_items:
+    - kube-apiserver
+    - kube-controller-manager
+    - kube-scheduler
+    - kubectl
+  become: true
+
+- name: Create kubernetes var dir
+  file: path=/var/lib/kubernetes state=directory
+  become: true
+
+- name: Create kubernetes etc dir
+  file: path=/etc/kubernetes/config state=directory
+  become: true
+
+- name: Copy Authorisation files
+  copy:
+    src: "{{ playbook_dir }}/../../kazan-ssl/data-encryption/{{ item }}"
+    dest: /var/lib/kubernetes
+    owner: root
+    group: root
+    mode: 0644
+  with_items:
+    - encryption-config.yaml
+  become: true
+
+- name: Copy cert files
+  copy:
+    src: "{{ playbook_dir }}/../../kazan-ssl/pki/{{ item }}"
+    dest: /var/lib/kubernetes
+    owner: root
+    group: root
+    mode: 0644
+  with_items:
+    - ca/ca.pem
+    - ca/ca-key.pem
+    - api/kubernetes-key.pem
+    - api/kubernetes.pem
+    - service-account/service-account-key.pem
+    - service-account/service-account.pem
+    - front-proxy/front-proxy-key.pem
+    - front-proxy/front-proxy.pem
+  become: true
+
+- name: Copy kube-* kubeconfig files
+  copy:
+    src: "{{ playbook_dir }}/../../kazan-ssl/configs/{{ item }}"
+    dest: /var/lib/kubernetes
+    owner: root
+    group: root
+    mode: 0644
+  with_items:
+    - controller/kube-controller-manager.kubeconfig
+    - scheduler/kube-scheduler.kubeconfig
+  become: true
+
+- name: Copy kube-* config files
+  copy:
+    src: "{{ item }}"
+    dest: /etc/kubernetes/config
+    owner: root
+    group: root
+    mode: 0644
+  with_items:
+    - kube-scheduler.yml
+  become: true
+
+- name: Copy kube audit policy file
+  copy:
+    src: audit-policy.yml
+    dest: /etc/kubernetes
+    owner: root
+    group: root
+    mode: 0644
+  become: true
+
+- name: Copy admin kube config
+  copy:
+    src: "{{ playbook_dir }}/../../kazan-ssl/configs/admin/admin.kubeconfig"
+    dest: /opt/kubernetes/admin.kubeconfig
+    owner: root
+    group: root
+    mode: 0644
+    directory_mode: false
+  become: true
+
+- name: Add kube-* systemd unit
+  template:
+    src: "{{ item }}.service.j2"
+    dest: /etc/systemd/system/{{ item }}.service
+    mode: 700
+  with_items:
+    - kube-controller-manager
+    - kube-apiserver
+    - kube-scheduler
+  become: true
+
+- name: Reload systemd
+  command: systemctl daemon-reload
+  become: true
+
+- name: Enable kube-* services
+  command: "systemctl enable {{ item }}"
+  with_items:
+    - kube-apiserver
+    - kube-controller-manager
+    - kube-scheduler
+  become: true
+
+- name: Restart kube-* services
+  service:
+    name: "{{ item }}"
+    state: restarted
+    enabled: yes
+  with_items:
+    - kube-apiserver
+    - kube-controller-manager
+    - kube-scheduler
+  become: true
+
+- name: Verify Kubernetes status
+  shell: kubectl get componentstatuses --kubeconfig /opt/kubernetes/admin.kubeconfig
+  register: cmd_result
+  retries: 5
+  delay: 10
+
+- assert:
+    that:
+      - "'scheduler            Healthy' in cmd_result.stdout"
+      - "'controller-manager   Healthy' in cmd_result.stdout"
+      - "'etcd-0               Healthy' in cmd_result.stdout"
+      - "'etcd-1               Healthy' in cmd_result.stdout"
+      - "'etcd-2               Healthy' in cmd_result.stdout"

roles/kubernetes/templates/kube-apiserver.service.j2

@@ -0,0 +1,51 @@
+[Unit]
+Description=Kubernetes API Server
+Documentation=https://github.com/kubernetes/kubernetes
+
+[Service]
+ExecStart=/usr/local/bin/kube-apiserver \
+  --advertise-address={{ ansible_default_ipv4.address }} \
+  --allow-privileged=true \
+  --apiserver-count=3 \
+  --audit-policy-file=/etc/kubernetes/audit-policy.yml \
+  --audit-log-maxage=30 \
+  --audit-log-maxbackup=3 \
+  --audit-log-maxsize=100 \
+  --audit-log-path=/var/log/audit.log \
+  --authorization-mode=Node,RBAC \
+  --bind-address=0.0.0.0 \
+  --client-ca-file=/var/lib/kubernetes/ca.pem \
+  --enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \
+  --etcd-cafile=/var/lib/kubernetes/ca.pem \
+  --etcd-certfile=/var/lib/kubernetes/kubernetes.pem \
+  --etcd-keyfile=/var/lib/kubernetes/kubernetes-key.pem \
+  --etcd-servers=https://10.0.0.141:2379,https://10.0.0.142:2379,https://10.0.0.143:2379 \
+  --event-ttl=1h \
+  --encryption-provider-config=/var/lib/kubernetes/encryption-config.yaml \
+  --kubelet-certificate-authority=/var/lib/kubernetes/ca.pem \
+  --kubelet-client-certificate=/var/lib/kubernetes/kubernetes.pem \
+  --kubelet-client-key=/var/lib/kubernetes/kubernetes-key.pem \
+  --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \
+  --proxy-client-cert-file=/var/lib/kubernetes/front-proxy.pem \
+  --proxy-client-key-file=/var/lib/kubernetes/front-proxy-key.pem \
+  --requestheader-allowed-names=front-proxy-client \
+  --requestheader-client-ca-file=/var/lib/kubernetes/ca.pem\
+  --requestheader-extra-headers-prefix=X-Remote-Extra- \
+  --requestheader-group-headers=X-Remote-Group \
+  --requestheader-username-headers=X-Remote-User \
+  --runtime-config='api/all=true' \
+  --secure-port=6443 \
+  --service-account-issuer=https://10.0.0.140:6443 \
+  --service-account-key-file=/var/lib/kubernetes/service-account.pem \
+  --service-account-signing-key-file=/var/lib/kubernetes/service-account-key.pem \
+  --service-cluster-ip-range=10.32.0.0/24 \
+  --service-node-port-range=30000-32767 \
+  --tls-cert-file=/var/lib/kubernetes/kubernetes.pem \
+  --tls-private-key-file=/var/lib/kubernetes/kubernetes-key.pem \
+  --v=2
+
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

roles/kubernetes/templates/kube-controller-manager.service.j2

@@ -0,0 +1,24 @@
+[Unit]
+Description=Kubernetes Controller Manager
+Documentation=https://github.com/kubernetes/kubernetes
+
+[Service]
+ExecStart=/usr/local/bin/kube-controller-manager \
+  --allocate-node-cidrs=true \
+  --bind-address=0.0.0.0 \
+  --cluster-cidr=10.200.0.0/16 \
+  --cluster-name=kubernetes \
+  --cluster-signing-cert-file=/var/lib/kubernetes/ca.pem \
+  --cluster-signing-key-file=/var/lib/kubernetes/ca-key.pem \
+  --kubeconfig=/var/lib/kubernetes/kube-controller-manager.kubeconfig \
+  --leader-elect=true \
+  --root-ca-file=/var/lib/kubernetes/ca.pem \
+  --service-account-private-key-file=/var/lib/kubernetes/service-account-key.pem \
+  --service-cluster-ip-range=10.32.0.0/24 \
+  --use-service-account-credentials=true \
+  --v=2
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

roles/kubernetes/templates/kube-scheduler.service.j2

@@ -0,0 +1,13 @@
+[Unit]
+Description=Kubernetes Scheduler
+Documentation=https://github.com/kubernetes/kubernetes
+
+[Service]
+ExecStart=/usr/local/bin/kube-scheduler \
+  --config=/etc/kubernetes/config/kube-scheduler.yml \
+  --v=2
+Restart=on-failure
+RestartSec=5
+
+[Install]
+WantedBy=multi-user.target

roles/kubernetes/vars/main.yml

@@ -0,0 +1,4 @@
+---
+
+kubernetes_version: "v1.26.0"
+kubernetes_download_path: "https://dl.k8s.io/{{ kubernetes_version }}/bin/linux/amd64"