diff --git a/roles/kubernetes/files/audit-policy.yml b/roles/kubernetes/files/audit-policy.yml new file mode 100644 index 0000000..705a3ed --- /dev/null +++ b/roles/kubernetes/files/audit-policy.yml @@ -0,0 +1,68 @@ +apiVersion: audit.k8s.io/v1 # This is required. +kind: Policy +# Don't generate audit events for all requests in RequestReceived stage. +omitStages: + - "RequestReceived" +rules: + # Log pod changes at RequestResponse level + - level: RequestResponse + resources: + - group: "" + # Resource "pods" doesn't match requests to any subresource of pods, + # which is consistent with the RBAC policy. + resources: ["pods"] + # Log "pods/log", "pods/status" at Metadata level + - level: Metadata + resources: + - group: "" + resources: ["pods/log", "pods/status"] + + # Don't log requests to a configmap called "controller-leader" + - level: None + resources: + - group: "" + resources: ["configmaps"] + resourceNames: ["controller-leader"] + + # Don't log watch requests by the "system:kube-proxy" on endpoints or services + - level: None + users: ["system:kube-proxy"] + verbs: ["watch"] + resources: + - group: "" # core API group + resources: ["endpoints", "services"] + + # Don't log authenticated requests to certain non-resource URL paths. + - level: None + userGroups: ["system:authenticated"] + nonResourceURLs: + - "/api*" # Wildcard matching. + - "/version" + + # Log the request body of configmap changes in kube-system. + - level: Request + resources: + - group: "" # core API group + resources: ["configmaps"] + # This rule only applies to resources in the "kube-system" namespace. + # The empty string "" can be used to select non-namespaced resources. + namespaces: ["kube-system"] + + # Log configmap and secret changes in all other namespaces at the Metadata level. + - level: Metadata + resources: + - group: "" # core API group + resources: ["secrets", "configmaps"] + + # Log all other resources in core and extensions at the Request level. + - level: Request + resources: + - group: "" # core API group + - group: "extensions" # Version of group should NOT be included. + + # A catch-all rule to log all other requests at the Metadata level. + - level: Metadata + # Long-running requests like watches that fall under this rule will not + # generate an audit event in RequestReceived. + omitStages: + - "RequestReceived" \ No newline at end of file diff --git a/roles/kubernetes/files/kube-scheduler.yml b/roles/kubernetes/files/kube-scheduler.yml new file mode 100644 index 0000000..4fc71e8 --- /dev/null +++ b/roles/kubernetes/files/kube-scheduler.yml @@ -0,0 +1,6 @@ +apiVersion: kubescheduler.config.k8s.io/v1beta2 +kind: KubeSchedulerConfiguration +clientConnection: + kubeconfig: "/var/lib/kubernetes/kube-scheduler.kubeconfig" +leaderElection: + leaderElect: true \ No newline at end of file diff --git a/roles/kubernetes/tasks/main.yml b/roles/kubernetes/tasks/main.yml new file mode 100644 index 0000000..e25a90e --- /dev/null +++ b/roles/kubernetes/tasks/main.yml @@ -0,0 +1,142 @@ +--- +- name: Download Kuberneters controller binaries + get_url: + url: "{{ kubernetes_download_path }}/{{ item }}" + dest: /usr/local/bin + owner: root + group: root + mode: 0755 + # TODO Add hash check + with_items: + - kube-apiserver + - kube-controller-manager + - kube-scheduler + - kubectl + become: true + +- name: Create kubernetes var dir + file: path=/var/lib/kubernetes state=directory + become: true + +- name: Create kubernetes etc dir + file: path=/etc/kubernetes/config state=directory + become: true + +- name: Copy Authorisation files + copy: + src: "{{ playbook_dir }}/../../kazan-ssl/data-encryption/{{ item }}" + dest: /var/lib/kubernetes + owner: root + group: root + mode: 0644 + with_items: + - encryption-config.yaml + become: true + +- name: Copy cert files + copy: + src: "{{ playbook_dir }}/../../kazan-ssl/pki/{{ item }}" + dest: /var/lib/kubernetes + owner: root + group: root + mode: 0644 + with_items: + - ca/ca.pem + - ca/ca-key.pem + - api/kubernetes-key.pem + - api/kubernetes.pem + - service-account/service-account-key.pem + - service-account/service-account.pem + - front-proxy/front-proxy-key.pem + - front-proxy/front-proxy.pem + become: true + +- name: Copy kube-* kubeconfig files + copy: + src: "{{ playbook_dir }}/../../kazan-ssl/configs/{{ item }}" + dest: /var/lib/kubernetes + owner: root + group: root + mode: 0644 + with_items: + - controller/kube-controller-manager.kubeconfig + - scheduler/kube-scheduler.kubeconfig + become: true + +- name: Copy kube-* config files + copy: + src: "{{ item }}" + dest: /etc/kubernetes/config + owner: root + group: root + mode: 0644 + with_items: + - kube-scheduler.yml + become: true + +- name: Copy kube audit policy file + copy: + src: audit-policy.yml + dest: /etc/kubernetes + owner: root + group: root + mode: 0644 + become: true + +- name: Copy admin kube config + copy: + src: "{{ playbook_dir }}/../../kazan-ssl/configs/admin/admin.kubeconfig" + dest: /opt/kubernetes/admin.kubeconfig + owner: root + group: root + mode: 0644 + directory_mode: false + become: true + +- name: Add kube-* systemd unit + template: + src: "{{ item }}.service.j2" + dest: /etc/systemd/system/{{ item }}.service + mode: 700 + with_items: + - kube-controller-manager + - kube-apiserver + - kube-scheduler + become: true + +- name: Reload systemd + command: systemctl daemon-reload + become: true + +- name: Enable kube-* services + command: "systemctl enable {{ item }}" + with_items: + - kube-apiserver + - kube-controller-manager + - kube-scheduler + become: true + +- name: Restart kube-* services + service: + name: "{{ item }}" + state: restarted + enabled: yes + with_items: + - kube-apiserver + - kube-controller-manager + - kube-scheduler + become: true + +- name: Verify Kubernetes status + shell: kubectl get componentstatuses --kubeconfig /opt/kubernetes/admin.kubeconfig + register: cmd_result + retries: 5 + delay: 10 + +- assert: + that: + - "'scheduler Healthy' in cmd_result.stdout" + - "'controller-manager Healthy' in cmd_result.stdout" + - "'etcd-0 Healthy' in cmd_result.stdout" + - "'etcd-1 Healthy' in cmd_result.stdout" + - "'etcd-2 Healthy' in cmd_result.stdout" \ No newline at end of file diff --git a/roles/kubernetes/templates/kube-apiserver.service.j2 b/roles/kubernetes/templates/kube-apiserver.service.j2 new file mode 100644 index 0000000..8cfcd2b --- /dev/null +++ b/roles/kubernetes/templates/kube-apiserver.service.j2 @@ -0,0 +1,51 @@ +[Unit] +Description=Kubernetes API Server +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +ExecStart=/usr/local/bin/kube-apiserver \ + --advertise-address={{ ansible_default_ipv4.address }} \ + --allow-privileged=true \ + --apiserver-count=3 \ + --audit-policy-file=/etc/kubernetes/audit-policy.yml \ + --audit-log-maxage=30 \ + --audit-log-maxbackup=3 \ + --audit-log-maxsize=100 \ + --audit-log-path=/var/log/audit.log \ + --authorization-mode=Node,RBAC \ + --bind-address=0.0.0.0 \ + --client-ca-file=/var/lib/kubernetes/ca.pem \ + --enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota \ + --etcd-cafile=/var/lib/kubernetes/ca.pem \ + --etcd-certfile=/var/lib/kubernetes/kubernetes.pem \ + --etcd-keyfile=/var/lib/kubernetes/kubernetes-key.pem \ + --etcd-servers=https://10.0.0.141:2379,https://10.0.0.142:2379,https://10.0.0.143:2379 \ + --event-ttl=1h \ + --encryption-provider-config=/var/lib/kubernetes/encryption-config.yaml \ + --kubelet-certificate-authority=/var/lib/kubernetes/ca.pem \ + --kubelet-client-certificate=/var/lib/kubernetes/kubernetes.pem \ + --kubelet-client-key=/var/lib/kubernetes/kubernetes-key.pem \ + --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname \ + --proxy-client-cert-file=/var/lib/kubernetes/front-proxy.pem \ + --proxy-client-key-file=/var/lib/kubernetes/front-proxy-key.pem \ + --requestheader-allowed-names=front-proxy-client \ + --requestheader-client-ca-file=/var/lib/kubernetes/ca.pem\ + --requestheader-extra-headers-prefix=X-Remote-Extra- \ + --requestheader-group-headers=X-Remote-Group \ + --requestheader-username-headers=X-Remote-User \ + --runtime-config='api/all=true' \ + --secure-port=6443 \ + --service-account-issuer=https://10.0.0.140:6443 \ + --service-account-key-file=/var/lib/kubernetes/service-account.pem \ + --service-account-signing-key-file=/var/lib/kubernetes/service-account-key.pem \ + --service-cluster-ip-range=10.32.0.0/24 \ + --service-node-port-range=30000-32767 \ + --tls-cert-file=/var/lib/kubernetes/kubernetes.pem \ + --tls-private-key-file=/var/lib/kubernetes/kubernetes-key.pem \ + --v=2 + +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/roles/kubernetes/templates/kube-controller-manager.service.j2 b/roles/kubernetes/templates/kube-controller-manager.service.j2 new file mode 100644 index 0000000..7386676 --- /dev/null +++ b/roles/kubernetes/templates/kube-controller-manager.service.j2 @@ -0,0 +1,24 @@ +[Unit] +Description=Kubernetes Controller Manager +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +ExecStart=/usr/local/bin/kube-controller-manager \ + --allocate-node-cidrs=true \ + --bind-address=0.0.0.0 \ + --cluster-cidr=10.200.0.0/16 \ + --cluster-name=kubernetes \ + --cluster-signing-cert-file=/var/lib/kubernetes/ca.pem \ + --cluster-signing-key-file=/var/lib/kubernetes/ca-key.pem \ + --kubeconfig=/var/lib/kubernetes/kube-controller-manager.kubeconfig \ + --leader-elect=true \ + --root-ca-file=/var/lib/kubernetes/ca.pem \ + --service-account-private-key-file=/var/lib/kubernetes/service-account-key.pem \ + --service-cluster-ip-range=10.32.0.0/24 \ + --use-service-account-credentials=true \ + --v=2 +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/roles/kubernetes/templates/kube-scheduler.service.j2 b/roles/kubernetes/templates/kube-scheduler.service.j2 new file mode 100644 index 0000000..c8047e9 --- /dev/null +++ b/roles/kubernetes/templates/kube-scheduler.service.j2 @@ -0,0 +1,13 @@ +[Unit] +Description=Kubernetes Scheduler +Documentation=https://github.com/kubernetes/kubernetes + +[Service] +ExecStart=/usr/local/bin/kube-scheduler \ + --config=/etc/kubernetes/config/kube-scheduler.yml \ + --v=2 +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target \ No newline at end of file diff --git a/roles/kubernetes/vars/main.yml b/roles/kubernetes/vars/main.yml new file mode 100644 index 0000000..6a1d7a1 --- /dev/null +++ b/roles/kubernetes/vars/main.yml @@ -0,0 +1,4 @@ +--- + +kubernetes_version: "v1.26.0" +kubernetes_download_path: "https://dl.k8s.io/{{ kubernetes_version }}/bin/linux/amd64"