部署控制面组件

准备所有组件的 kubeconfig

kube-proxy

  1. $ kubectl config set-cluster openeuler-k8s --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://192.168.122.154:6443 --kubeconfig=kube-proxy.kubeconfig
  2. $ kubectl config set-credentials system:kube-proxy --client-certificate=/etc/kubernetes/pki/kube-proxy.pem --client-key=/etc/kubernetes/pki/kube-proxy-key.pem --embed-certs=true --kubeconfig=kube-proxy.kubeconfig
  3. $ kubectl config set-context default --cluster=openeuler-k8s --user=system:kube-proxy --kubeconfig=kube-proxy.kubeconfig
  4. $ kubectl config use-context default --kubeconfig=kube-proxy.kubeconfig

kube-controller-manager

  1. $ kubectl config set-cluster openeuler-k8s --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://127.0.0.1:6443 --kubeconfig=kube-controller-manager.kubeconfig
  2. $ kubectl config set-credentials system:kube-controller-manager --client-certificate=/etc/kubernetes/pki/kube-controller-manager.pem --client-key=/etc/kubernetes/pki/kube-controller-manager-key.pem --embed-certs=true --kubeconfig=kube-controller-manager.kubeconfig
  3. $ kubectl config set-context default --cluster=openeuler-k8s --user=system:kube-controller-manager --kubeconfig=kube-controller-manager.kubeconfig
  4. $ kubectl config use-context default --kubeconfig=kube-controller-manager.kubeconfig

kube-scheduler

  1. $ kubectl config set-cluster openeuler-k8s --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://127.0.0.1:6443 --kubeconfig=kube-scheduler.kubeconfig
  2. $ kubectl config set-credentials system:kube-scheduler --client-certificate=/etc/kubernetes/pki/kube-scheduler.pem --client-key=/etc/kubernetes/pki/kube-scheduler-key.pem --embed-certs=true --kubeconfig=kube-scheduler.kubeconfig
  3. $ kubectl config set-context default --cluster=openeuler-k8s --user=system:kube-scheduler --kubeconfig=kube-scheduler.kubeconfig
  4. $ kubectl config use-context default --kubeconfig=kube-scheduler.kubeconfig

admin

  1. $ kubectl config set-cluster openeuler-k8s --certificate-authority=/etc/kubernetes/pki/ca.pem --embed-certs=true --server=https://127.0.0.1:6443 --kubeconfig=admin.kubeconfig
  2. $ kubectl config set-credentials admin --client-certificate=/etc/kubernetes/pki/admin.pem --client-key=/etc/kubernetes/pki/admin-key.pem --embed-certs=true --kubeconfig=admin.kubeconfig
  3. $ kubectl config set-context default --cluster=openeuler-k8s --user=admin --kubeconfig=admin.kubeconfig
  4. $ kubectl config use-context default --kubeconfig=admin.kubeconfig

获得相关 kubeconfig 配置文件

  1. admin.kubeconfig kube-proxy.kubeconfig kube-controller-manager.kubeconfig kube-scheduler.kubeconfig

生成密钥提供者的配置

api-server 启动时需要提供一个密钥对--encryption-provider-config=/etc/kubernetes/pki/encryption-config.yaml,本文通过 urandom 生成一个:

  1. $ cat generate.bash
  2. #!/bin/bash
  3. ENCRYPTION_KEY=$(head -c 32 /dev/urandom | base64)
  4. cat > encryption-config.yaml <<EOF
  5. kind: EncryptionConfig
  6. apiVersion: v1
  7. resources:
  8. - resources:
  9. - secrets
  10. providers:
  11. - aescbc:
  12. keys:
  13. - name: key1
  14. secret: ${ENCRYPTION_KEY}
  15. - identity: {}
  16. EOF
  17. # api-server启动配置 --encryption-provider-config=/etc/kubernetes/pki/encryption-config.yaml

拷贝证书

本文把所有组件使用的证书、密钥以及配置统一放到/etc/kubernetes/pki/目录下。

  1. # 准备证书目录
  2. $ mkdir -p /etc/kubernetes/pki/
  3. $ ls /etc/kubernetes/pki/
  4. admin-key.pem encryption-config.yaml kube-proxy-key.pem kubernetes.pem service-account-key.pem
  5. admin.pem kube-controller-manager-key.pem kube-proxy.kubeconfig kube-scheduler-key.pem service-account.pem
  6. ca-key.pem kube-controller-manager.kubeconfig kube-proxy.pem kube-scheduler.kubeconfig
  7. ca.pem kube-controller-manager.pem kubernetes-key.pem kube-scheduler.pem

部署 admin 角色的 RBAC

使能 admin role

  1. $ cat admin_cluster_role.yaml
  2. apiVersion: rbac.authorization.k8s.io/v1
  3. kind: ClusterRole
  4. metadata:
  5. annotations:
  6. rbac.authorization.kubernetes.io/autoupdate: "true"
  7. labels:
  8. kubernetes.io/bootstrapping: rbac-defaults
  9. name: system:kube-apiserver-to-kubelet
  10. rules:
  11. - apiGroups:
  12. - ""
  13. resources:
  14. - nodes/proxy
  15. - nodes/stats
  16. - nodes/log
  17. - nodes/spec
  18. - nodes/metrics
  19. verbs:
  20. - "*"
  21. # 使能admin role
  22. $ kubectl apply --kubeconfig admin.kubeconfig -f admin_cluster_role.yaml

绑定 admin role

  1. $ cat admin_cluster_rolebind.yaml
  2. apiVersion: rbac.authorization.k8s.io/v1
  3. kind: ClusterRoleBinding
  4. metadata:
  5. name: system:kube-apiserver
  6. namespace: ""
  7. roleRef:
  8. apiGroup: rbac.authorization.k8s.io
  9. kind: ClusterRole
  10. name: system:kube-apiserver-to-kubelet
  11. subjects:
  12. - apiGroup: rbac.authorization.k8s.io
  13. kind: User
  14. name: kubernetes
  15. # 绑定admin role
  16. $ kubectl apply --kubeconfig admin.kubeconfig -f admin_cluster_rolebind.yaml

部署 api server 服务

修改 apiserver 的 etc 配置文件:

  1. $ cat /etc/kubernetes/apiserver
  2. KUBE_ADVERTIS_ADDRESS="--advertise-address=192.168.122.154"
  3. KUBE_ALLOW_PRIVILEGED="--allow-privileged=true"
  4. KUBE_AUTHORIZATION_MODE="--authorization-mode=Node,RBAC"
  5. KUBE_ENABLE_ADMISSION_PLUGINS="--enable-admission-plugins=NamespaceLifecycle,NodeRestriction,LimitRanger,ServiceAccount,DefaultStorageClass,ResourceQuota"
  6. KUBE_SECURE_PORT="--secure-port=6443"
  7. KUBE_ENABLE_BOOTSTRAP_TOKEN_AUTH="--enable-bootstrap-token-auth=true"
  8. KUBE_ETCD_CAFILE="--etcd-cafile=/etc/kubernetes/pki/ca.pem"
  9. KUBE_ETCD_CERTFILE="--etcd-certfile=/etc/kubernetes/pki/kubernetes.pem"
  10. KUBE_ETCD_KEYFILE="--etcd-keyfile=/etc/kubernetes/pki/kubernetes-key.pem"
  11. KUBE_ETCD_SERVERS="--etcd-servers=https://192.168.122.154:2379,https://192.168.122.155:2379,https://192.168.122.156:2379"
  12. KUBE_CLIENT_CA_FILE="--client-ca-file=/etc/kubernetes/pki/ca.pem"
  13. KUBE_KUBELET_CERT_AUTH="--kubelet-certificate-authority=/etc/kubernetes/pki/ca.pem"
  14. KUBE_KUBELET_CLIENT_CERT="--kubelet-client-certificate=/etc/kubernetes/pki/kubernetes.pem"
  15. KUBE_KUBELET_CLIENT_KEY="--kubelet-client-key=/etc/kubernetes/pki/kubernetes-key.pem"
  16. KUBE_KUBELET_HTTPS="--kubelet-https=true"
  17. KUBE_PROXY_CLIENT_CERT_FILE="--proxy-client-cert-file=/etc/kubernetes/pki/kube-proxy.pem"
  18. KUBE_PROXY_CLIENT_KEY_FILE="--proxy-client-key-file=/etc/kubernetes/pki/kube-proxy-key.pem"
  19. KUBE_TLS_CERT_FILE="--tls-cert-file=/etc/kubernetes/pki/kubernetes.pem"
  20. KUBE_TLS_PRIVATE_KEY_FILE="--tls-private-key-file=/etc/kubernetes/pki/kubernetes-key.pem"
  21. KUBE_SERVICE_CLUSTER_IP_RANGE="--service-cluster-ip-range=10.32.0.0/16"
  22. KUBE_SERVICE_ACCOUNT_ISSUER="--service-account-issuer=https://kubernetes.default.svc.cluster.local"
  23. KUBE_SERVICE_ACCOUNT_KEY_FILE="--service-account-key-file=/etc/kubernetes/pki/service-account.pem"
  24. KUBE_SERVICE_ACCOUNT_SIGN_KEY_FILE="--service-account-signing-key-file=/etc/kubernetes/pki/service-account-key.pem"
  25. KUBE_SERVICE_NODE_PORT_RANGE="--service-node-port-range=30000-32767"
  26. KUB_ENCRYPTION_PROVIDER_CONF="--encryption-provider-config=/etc/kubernetes/pki/encryption-config.yaml"
  27. KUBE_REQUEST_HEADER_ALLOWED_NAME="--requestheader-allowed-names=front-proxy-client"
  28. KUBE_REQUEST_HEADER_EXTRA_HEADER_PREF="--requestheader-extra-headers-prefix=X-Remote-Extra-"
  29. KUBE_REQUEST_HEADER_GROUP_HEADER="--requestheader-group-headers=X-Remote-Group"
  30. KUBE_REQUEST_HEADER_USERNAME_HEADER="--requestheader-username-headers=X-Remote-User"
  31. KUBE_API_ARGS=""

所有apiserver的配置都/etc/kubernetes/config文件中定义,然后在后面的service文件中直接使用即可。

大部分配置都是比较固定的,部分需要注意:

  • --service-cluster-ip-range该地址需要和后面的设置的clusterDNS需要一致;

编写 apiserver 的 systemd 配置

  1. cat /usr/lib/systemd/system/kube-apiserver.service
  2. [Unit]
  3. Description=Kubernetes API Server
  4. Documentation=https://kubernetes.io/docs/reference/generated/kube-apiserver/
  5. After=network.target
  6. After=etcd.service
  7. [Service]
  8. EnvironmentFile=-/etc/kubernetes/config
  9. EnvironmentFile=-/etc/kubernetes/apiserver
  10. ExecStart=/usr/bin/kube-apiserver \
  11. $KUBE_ADVERTIS_ADDRESS \
  12. $KUBE_ALLOW_PRIVILEGED \
  13. $KUBE_AUTHORIZATION_MODE \
  14. $KUBE_ENABLE_ADMISSION_PLUGINS \
  15. $KUBE_SECURE_PORT \
  16. $KUBE_ENABLE_BOOTSTRAP_TOKEN_AUTH \
  17. $KUBE_ETCD_CAFILE \
  18. $KUBE_ETCD_CERTFILE \
  19. $KUBE_ETCD_KEYFILE \
  20. $KUBE_ETCD_SERVERS \
  21. $KUBE_CLIENT_CA_FILE \
  22. $KUBE_KUBELET_CERT_AUTH \
  23. $KUBE_KUBELET_CLIENT_CERT \
  24. $KUBE_KUBELET_CLIENT_KEY \
  25. $KUBE_PROXY_CLIENT_CERT_FILE \
  26. $KUBE_PROXY_CLIENT_KEY_FILE \
  27. $KUBE_TLS_CERT_FILE \
  28. $KUBE_TLS_PRIVATE_KEY_FILE \
  29. $KUBE_SERVICE_CLUSTER_IP_RANGE \
  30. $KUBE_SERVICE_ACCOUNT_ISSUER \
  31. $KUBE_SERVICE_ACCOUNT_KEY_FILE \
  32. $KUBE_SERVICE_ACCOUNT_SIGN_KEY_FILE \
  33. $KUBE_SERVICE_NODE_PORT_RANGE \
  34. $KUBE_LOGTOSTDERR \
  35. $KUBE_LOG_LEVEL \
  36. $KUBE_API_PORT \
  37. $KUBELET_PORT \
  38. $KUBE_ALLOW_PRIV \
  39. $KUBE_SERVICE_ADDRESSES \
  40. $KUBE_ADMISSION_CONTROL \
  41. $KUB_ENCRYPTION_PROVIDER_CONF \
  42. $KUBE_REQUEST_HEADER_ALLOWED_NAME \
  43. $KUBE_REQUEST_HEADER_EXTRA_HEADER_PREF \
  44. $KUBE_REQUEST_HEADER_GROUP_HEADER \
  45. $KUBE_REQUEST_HEADER_USERNAME_HEADER \
  46. $KUBE_API_ARGS
  47. Restart=on-failure
  48. Type=notify
  49. LimitNOFILE=65536
  50. [Install]
  51. WantedBy=multi-user.target

部署 controller-manager 服务

修改 controller-manager 配置文件:

  1. $ cat /etc/kubernetes/controller-manager
  2. KUBE_BIND_ADDRESS="--bind-address=127.0.0.1"
  3. KUBE_CLUSTER_CIDR="--cluster-cidr=10.200.0.0/16"
  4. KUBE_CLUSTER_NAME="--cluster-name=kubernetes"
  5. KUBE_CLUSTER_SIGNING_CERT_FILE="--cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem"
  6. KUBE_CLUSTER_SIGNING_KEY_FILE="--cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem"
  7. KUBE_KUBECONFIG="--kubeconfig=/etc/kubernetes/pki/kube-controller-manager.kubeconfig"
  8. KUBE_LEADER_ELECT="--leader-elect=true"
  9. KUBE_ROOT_CA_FILE="--root-ca-file=/etc/kubernetes/pki/ca.pem"
  10. KUBE_SERVICE_ACCOUNT_PRIVATE_KEY_FILE="--service-account-private-key-file=/etc/kubernetes/pki/service-account-key.pem"
  11. KUBE_SERVICE_CLUSTER_IP_RANGE="--service-cluster-ip-range=10.32.0.0/24"
  12. KUBE_USE_SERVICE_ACCOUNT_CRED="--use-service-account-credentials=true"
  13. KUBE_CONTROLLER_MANAGER_ARGS="--v=2"

编写 controller-manager 的 systemd 配置文件

  1. $ cat /usr/lib/systemd/system/kube-controller-manager.service
  2. [Unit]
  3. Description=Kubernetes Controller Manager
  4. Documentation=https://kubernetes.io/docs/reference/generated/kube-controller-manager/
  5. [Service]
  6. EnvironmentFile=-/etc/kubernetes/config
  7. EnvironmentFile=-/etc/kubernetes/controller-manager
  8. ExecStart=/usr/bin/kube-controller-manager \
  9. $KUBE_BIND_ADDRESS \
  10. $KUBE_LOGTOSTDERR \
  11. $KUBE_LOG_LEVEL \
  12. $KUBE_CLUSTER_CIDR \
  13. $KUBE_CLUSTER_NAME \
  14. $KUBE_CLUSTER_SIGNING_CERT_FILE \
  15. $KUBE_CLUSTER_SIGNING_KEY_FILE \
  16. $KUBE_KUBECONFIG \
  17. $KUBE_LEADER_ELECT \
  18. $KUBE_ROOT_CA_FILE \
  19. $KUBE_SERVICE_ACCOUNT_PRIVATE_KEY_FILE \
  20. $KUBE_SERVICE_CLUSTER_IP_RANGE \
  21. $KUBE_USE_SERVICE_ACCOUNT_CRED \
  22. $KUBE_CONTROLLER_MANAGER_ARGS
  23. Restart=on-failure
  24. LimitNOFILE=65536
  25. [Install]
  26. WantedBy=multi-user.target

部署 scheduler 服务

修改 scheduler 配置文件:

  1. $ cat /etc/kubernetes/scheduler
  2. KUBE_CONFIG="--kubeconfig=/etc/kubernetes/pki/kube-scheduler.kubeconfig"
  3. KUBE_AUTHENTICATION_KUBE_CONF="--authentication-kubeconfig=/etc/kubernetes/pki/kube-scheduler.kubeconfig"
  4. KUBE_AUTHORIZATION_KUBE_CONF="--authorization-kubeconfig=/etc/kubernetes/pki/kube-scheduler.kubeconfig"
  5. KUBE_BIND_ADDR="--bind-address=127.0.0.1"
  6. KUBE_LEADER_ELECT="--leader-elect=true"
  7. KUBE_SCHEDULER_ARGS=""

编写 scheduler 的 systemd 配置文件

  1. $ cat /usr/lib/systemd/system/kube-scheduler.service
  2. [Unit]
  3. Description=Kubernetes Scheduler Plugin
  4. Documentation=https://kubernetes.io/docs/reference/generated/kube-scheduler/
  5. [Service]
  6. EnvironmentFile=-/etc/kubernetes/config
  7. EnvironmentFile=-/etc/kubernetes/scheduler
  8. ExecStart=/usr/bin/kube-scheduler \
  9. $KUBE_LOGTOSTDERR \
  10. $KUBE_LOG_LEVEL \
  11. $KUBE_CONFIG \
  12. $KUBE_AUTHENTICATION_KUBE_CONF \
  13. $KUBE_AUTHORIZATION_KUBE_CONF \
  14. $KUBE_BIND_ADDR \
  15. $KUBE_LEADER_ELECT \
  16. $KUBE_SCHEDULER_ARGS
  17. Restart=on-failure
  18. LimitNOFILE=65536
  19. [Install]
  20. WantedBy=multi-user.target

使能各组件

  1. $ systemctl enable kube-controller-manager kube-scheduler kube-proxy
  2. $ systemctl restart kube-controller-manager kube-scheduler kube-proxy

基本功能验证

  1. $ curl --cacert /etc/kubernetes/pki/ca.pem https://192.168.122.154:6443/version
  2. {
  3. "major": "1",
  4. "minor": "20",
  5. "gitVersion": "v1.20.2",
  6. "gitCommit": "faecb196815e248d3ecfb03c680a4507229c2a56",
  7. "gitTreeState": "archive",
  8. "buildDate": "2021-03-02T07:26:14Z",
  9. "goVersion": "go1.15.7",
  10. "compiler": "gc",
  11. "platform": "linux/arm64"
  12. }