anolis8.9-k8s1.32-master集群-二进制部署

发布于：2025-03-06 ⋅ 阅读:(13) ⋅ 点赞:(0)

一、系统信息

# cat /etc/anolis-release 
Anolis OS release 8.9

二、节点规划

主机名 ip 应用 备注
pro-k8s-master1-7200 192.168.7.200 Kube-apiserver,kube-controller-manager,kube-scheduler
pro-k8s-master2-7201 192.168.7.201 Kube-apiserver,kube-controller-manager,kube-scheduler
pro-k8s-master3-7202 192.168.7.202 Kube-apiserver,kube-controller-manager,kube-scheduler
pro-anol89-k8sma-ng-7209 192.168.7.208 nginx、keepalived
vip 192.168.7.209

三、nginx 部署

# cat /etc/yum.repos.d/nginx.repo 
[nginx-stable]
name=nginx stable repo
baseurl=http://nginx.org/packages/centos/$releasever/$basearch/
gpgcheck=1
enabled=1
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true

[nginx-mainline]
name=nginx mainline repo
baseurl=http://nginx.org/packages/mainline/centos/$releasever/$basearch/
gpgcheck=1
enabled=0
gpgkey=https://nginx.org/keys/nginx_signing.key
module_hotfixes=true


# dnf install nginx keepalived


# 配置：
# egrep -v "^$|^#|^ *#" /etc/nginx/nginx.conf 
user  nginx;
worker_processes  auto;
error_log  /var/log/nginx/error.log notice;
pid        /var/run/nginx.pid;
events {
    worker_connections  1024;
}
stream {
    upstream k8s_masters {
        least_conn;  # 最小连接负载均衡算法 
        server 192.168.7.200:6443 max_fails=3 fail_timeout=30s;
        server 192.168.7.201:6443 max_fails=3 fail_timeout=30s;
        server 192.168.7.202:6443 max_fails=3 fail_timeout=30s;
    }
 
    server {
        listen  6443;  # 对外暴露的 API 端口 
        proxy_pass k8s_masters;
        proxy_connect_timeout 5s;
        proxy_timeout  300s;  # 适配长连接操作 
        proxy_buffer_size  16k;
    }
}
http {
    include       /etc/nginx/mime.types;
    default_type  application/octet-stream;
    log_format  main  '$remote_addr - $remote_user [$time_local] "$request" '
                      '$status $body_bytes_sent "$http_referer" '
                      '"$http_user_agent" "$http_x_forwarded_for"';
    access_log  /var/log/nginx/access.log  main;
    sendfile        on;
    keepalive_timeout  65;
    include /etc/nginx/conf.d/*.conf;
}



# 启动
# nginx -t
# systemctl enable --now nginx
# systemctl status nginx


# keepalived 配置
# cat /etc/keepalived/keepalived.conf
! Configuration File for keepalived

global_defs {
   notification_email {
     acassen@firewall.loc
     failover@firewall.loc
     sysadmin@firewall.loc
   }
   notification_email_from Alexandre.Cassen@firewall.loc
   smtp_server 127.0.0.1
   smtp_connect_timeout 30
   router_id LVS_DEVEL
   vrrp_skip_check_adv_addr
   #vrrp_strict
   vrrp_garp_interval 0
   vrrp_gna_interval 0
}

vrrp_script chk-apiserver{
   script "/opt/keepalived/check_apiserver.sh"
   interval 5
   weight -5
   fall 2
rise 1
}

vrrp_instance VI_1 {
    state MASTER
    interface ens18
    virtual_router_id 51
    priority 100
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass 1111
    }
    virtual_ipaddress {
        192.168.7.209/24
    }
    track_script {
      chk_apiserver
    }
}


# 监控脚本
# cat /opt/keepalived/check_apiserver.sh 
#!/bin/bash
count=$(ss -antp|grep 6443|egrep -cv "grep|$$")

if [ "$count" -eqq 0 ];then
    exit 1
else
    exit 0
fi

# chmod +x /opt/keepalived/check_apiserver.sh

# systemctl enable --now keepalived
# systemctl status keepalived

四、下载kubernetes二进制文件

# https://github.com/kubernetes/kubernetes/blob/master/CHANGELOG/CHANGELOG-1.32.md

解压包：

# tar -zxf kubernetes-server-linux-amd64.tar.gz

# mkdir /usr/local/kubernetes/{bin,cfg,ssl} -p

# cp kubernetes/server/bin/{kube-apiserver,kube-scheduler,kube-controller-manager} /usr/local/kubernetes/bin/

# cp kubernetes/server/bin/kubectl /usr/bin/

五、自签证书机构

5.1 自签证书颁发机构（CA）

生成证书<会生成ca.pem和ca-key.pem证书>

# cat ca-config.json 
{
  "signing": {
    "default": {
      "expiry": "87600h"
    },
    "profiles": {
      "kubernetes": {
        "expiry": "87600h",
         "usages": [
           "signing",
           "key encipherment",
           "server auth",
           "client auth"
         ]
      }
    }
  }
}


# cat ca-csr.json 
{
  "CN": "kubernetes",
    "key": {
      "algo": "rsa",
      "size": 2048
    },
    "names": [
       {
         "C": "CN",
         "L": "Beijing",
         "ST": "Beijing",
         "O": "k8s",
         "OU": "System"
       }
    ]
}


# cfssl gencert -initca ca-csr.json | cfssljson -bare ca -

六、第一台master部署

6.1  使用自签CA签发kube-apiserver HTTPS证书

生成证书<生成server.pem和server-key.pem两个文件>

# cat > server-csr.json << EOF
{
  "CN": "kubernetes",
  "hosts": [
    "10.96.0.1",
    "127.0.0.1",
    "192.168.7.200",
    "192.168.7.201",
    "192.168.7.202",
    "192.168.7.203",
    "192.168.7.204",
    "192.168.7.205",
    "192.168.7.206",
    "192.168.7.207",
    "192.168.7.208",
    "192.168.7.209",
    "192.168.7.210",
    "kubernetes",
    "kubernetes.default",
    "kubernetes.default.svc",
    "kubernetes.default.svc.cluster",
    "kubernetes.default.svc.cluster.local"
  ],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "k8s",
      "OU": "System"
    }
  ]
}
EOF


# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes server-csr.json | cfssljson -bare server

#上面server-csr.json文件中hosts字段中IP为所有Master/LB/VIP IP,一个都不能少！为了方便后期扩容可以多写几个预留的IP

6.2 部署kube-apiserver

拷贝etcd证书：

拷贝api-server证书

创建配置文件

# cat kube-apiserver.conf 
KUBE_APISERVER_OPTS="--logging-format=json \
--v=4 \
--etcd-servers=https://192.168.8.115:2379,https://192.168.8.116:2379,https://192.168.8.117:2379 \
--bind-address=192.168.7.200 \
--secure-port=6443 \
--advertise-address=192.168.7.200 \
--allow-privileged=true \
--service-cluster-ip-range=10.96.0.0/16 \
--enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,ResourceQuota,NodeRestriction \
--authorization-mode=RBAC,Node \
--enable-bootstrap-token-auth=true \
--token-auth-file=/usr/local/kubernetes/cfg/token.csv \
--service-node-port-range=30000-32767 \
--kubelet-client-certificate=/usr/local/kubernetes/ssl/server.pem \
--kubelet-client-key=/usr/local/kubernetes/ssl/server-key.pem \
--tls-cert-file=/usr/local/kubernetes/ssl/server.pem \
--tls-private-key-file=/usr/local/kubernetes/ssl/server-key.pem \
--client-ca-file=/usr/local/kubernetes/ssl/ca.pem \
--service-account-key-file=/usr/local/kubernetes/ssl/ca-key.pem \
--service-account-issuer=api \
--service-account-signing-key-file=/usr/local/kubernetes/ssl/server-key.pem \
--etcd-cafile=/usr/local/kubernetes/etcd/ssl/ca.pem \
--etcd-certfile=/usr/local/kubernetes/etcd/ssl/server.pem \
--etcd-keyfile=/usr/local/kubernetes/etcd/ssl/server-key.pem \
--requestheader-client-ca-file=/usr/local/kubernetes/ssl/ca.pem \
--proxy-client-cert-file=/usr/local/kubernetes/ssl/server.pem \
--proxy-client-key-file=/usr/local/kubernetes/ssl/server-key.pem \
--requestheader-allowed-names=kubernetes \
--requestheader-extra-headers-prefix=X-Remote-Extra- \
--requestheader-group-headers=X-Remote-Group \
--requestheader-username-headers=X-Remote-User \
--enable-aggregator-routing=true \
--audit-log-maxage=30 \
--audit-log-maxbackup=3 \
--audit-log-maxsize=100 \
--audit-log-path=/var/log/kubernetes/logs/k8s-audit.log"

配置文件注释说明：

注：上面文档中两个\\,第一个是转义符，第二个是换行符，使用转义符是为了使用EOF保留换行符。

  1. --v: 日志登记
  2. --etcd-servers：etcd集群地址
  3. --bind-address: 监听地址
  4. --secure-port： https安全端口
  5. --advertise-address: 集群通告地址
  6. --allow-privileged: 启用授权
  7. --service-cluster-ip-range: Service虚拟ip地址段
  8. --enable-admission-plugins：准入控制模块
  9. --authorization-mode：认证授权，启用RBAC授权和节点自管理
  10. --enable-bootstrap-token-auth：启用TLS bootstrap机制
  11. --token-auth-file：bootstrap token文件
  12. --service-node-port-range： service nodepcrt类型默认分配端口范围
  13. --kubelet-client-xxx: apiserver访问kubelet客户端证书
  14. --tls-xxx-file: apiserver https证书
  15. 1.20版本必须加的参数：--service-account-issuer,--service-account-signing-key-file
  16. --etct-xxxfile: 连接etcd集群证书
  17. --audit-log-xxx: 审计日志
  18. 启动聚合层相关配置：
  • --requestheader-client-ca-file,--proxy-client-cert-file,--prosy-client-key-file,--requestheader-allowed-names,--requestheader-extra-headers-prefix,--requesheader-group-headers,--requestheader-username-headers,--enable-aggregator-routin

启用TLS Bootstrapping机制

TLSBootstraping： Master apiserver启用TLS认证后，Node节点kubelet和kube-proxy要与kube-apiserver进行通信，必须使用ca签发的有效证书才可以，当node节点很多时，这种客户端证书颁发需要大量工作，同样也会增加集群扩展复杂度。为了简化流程，kubernetes引入了TLS bootstraping机制来自动颁发客户端证书，kubelet会以一个低权限用户自动向apiserver申请证书，kubelet的证书有apiserver动态签署。所以最好在node上使用这种方式，目前主要用于kubelet，kube-proxy还是有我们同一颁发一个证书。

TLS bootstraping工作流程：

创建上述配置文件kube-apiserver.conf中的token：

格式：

token,用户名,UID,用户组

token也可自行生成替换：

# head -c 16 /dev/urandom | od -An -t x | tr -d ' '

# head -c 16 /dev/urandom | od -An -t x | tr -d ' '
3ae3609cfd216a2bc7e3db97e039c9da

# cat /usr/local/kubernetes/cfg/token.csv
3ae3609cfd216a2bc7e3db97e039c9da,kubelet-bootstrap,10001,"system:node-bootstrapper"

或者使用下面命令：

cat > token.csv << EOF
$(head -c 16 /dev/urandom | od -An -t x | tr -d ' '),kubelet-bootstrap,10001,"system:kubelet-bootstrap"
EOF

systemd管理apiserver

# cat /usr/lib/systemd/system/kube-apiserver.service 
[Unit]
Desription=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/usr/local/kubernetes/cfg/kube-apiserver.conf
ExecStart=/usr/local/kubernetes/bin/kube-apiserver $KUBE_APISERVER_OPTS
StandardOutput=file:/var/log/kubernetes/logs/kube-apiserver.log
StandardError=file:/var/log/kubernetes/logs/kube-apiserver.log
Restart=on-failure

[Install]
WantedBy=multi-user.target



# chmod 755 /usr/lib/systemd/system/kube-apiserver.service



# 日志分割
# cat /etc/logrotate.d/kube-apiserver
/var/log/kubernetes/logs/kube-apiserver.log {
  daily
  rotate 7
  compress
  missingok
}


# 启动：
# systemctl daemon-reload
# systemctl start kube-apiserver.service 
# systemctl sttus kube-apiserver.service 
# systemctl enable kube-apiserver.service

注：启动失败使用： journalctl -u kube-apiserver 命令查看

6.3 生成kube-controller-manager证书

# kube-controller-manager.kubeconfig 文件
# cat /usr/local/kubernetes/cfg/kube-controller-manager.conf
KUBE_CONTROLLER_MANAGER_OPTS=" --logging-format=json \
  --v=4 \
  --leader-elect=true \
  --kubeconfig=/usr/local/kubernetes/cfg/kube-controller-manager.kubeconfig \
  --bind-address=127.0.0.1 \
  --allocate-node-cidrs=true \
  --cluster-cidr=10.244.0.0/16 \
  --service-cluster-ip-range=10.96.0.0/16 \
  --cluster-signing-cert-file=/usr/local/kubernetes/ssl/ca.pem \
  --cluster-signing-key-file=/usr/local/kubernetes/ssl/ca-key.pem \
  --root-ca-file=/usr/local/kubernetes/ssl/ca.pem \
  --service-account-private-key-file=/usr/local/kubernetes/ssl/ca-key.pem \
  --cluster-name=kubernetes \
  --feature-gates=RotateKubeletServerCertificate=true \
  --controllers=*,bootstrapsigner,tokencleaner \
  --tls-cert-file=/usr/local/kubernetes/ssl/kube-controller-manager.pem \
  --tls-private-key-file=/usr/local/kubernetes/ssl/kube-controller-manager-key.pem \
  --use-service-account-credentials=true \
  --cluster-signing-duration=87600h0m0s"

注：
• --kubeconfig: 连接apiserver配置文件
• --leader-elect： 当该组件启动多个是，自动选举(HA)
• --cluster-signing-cert-file/--cluster-signing-key-file:自动为kubelet颁发证书的CA，与apiserver保持一致





# cat kube-controller-manager-csr.json 
{
  "CN": "system:kube-controller-manager",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "system:kube-controller-manager",
      "OU": "system"
    }
  ]
}




# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-controller-manager-csr.json | cfssljson -bare kube-controller-manager

生成kubeconfig文件

KUBE_CONFIG="/usr/local/kubernetes/cfg/kube-controller-manager.kubeconfig"
# 这里使用的master集群的vip IP地址
KUBE_APISERVER="https://192.168.7.209:6443"

kubectl config set-cluster kubernetes \
--certificate-authority=/usr/local/kubernetes/ssl/ca.pem \
--embed-certs=true \
--server=${KUBE_APISERVER} \
--kubeconfig=${KUBE_CONFIG}

kubectl config set-credentials kube-controller-manager \
--client-certificate=/usr/local/kubernetes/ssl/kube-controller-manager.pem \
--client-key=/usr/local/kubernetes/ssl/kube-controller-manager-key.pem \
--embed-certs=true \
--kubeconfig=${KUBE_CONFIG}

kubectl config set-context default \
--cluster=kubernetes \
--user=kube-controller-manager \
--kubeconfig=${KUBE_CONFIG}

kubectl config use-context default --kubeconfig=${KUBE_CONFIG}

    systemd管理controller-manager

    # cat /usr/lib/systemd/system/kube-controller-manager.service
[Unit]
Desription=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/usr/local/kubernetes/cfg/kube-controller-manager.conf
ExecStart=/usr/local/kubernetes//bin/kube-controller-manager $KUBE_CONTROLLER_MANAGER_OPTS
StandardOutput=file:/var/log/kubernetes/logs/kube-controller-manager.log
StandardError=file:/var/log/kubernetes/logs/kube-controller-manager.log
Restart=on-failure

[Install]
WantedBy=multi-user.target



# 日志切割
# cat /etc/logrotate.d/kube-controller-manager 
/var/log/kubernetes/logs/kube-controller-manager.log {
  daily
  rotate 7
  compress
  missingok
}




# systemctl daemon-reload
# systemctl start kube-controller-manager
# systemctl status kube-controller-manager
# systemctl enable kube-controller-manager

    6.4  生成kube-scheduler证书

    # 生成配置文件
# cat /usr/local/kubernetes/cfg/kube-scheduler.conf 
KUBE_SCHEDULER_OPTS=" --logging-format=json \
--v=4 \
--kubeconfig=/usr/local/kubernetes/cfg/kube-scheduler.kubeconfig \
--leader-elect=true \
--bind-address=127.0.0.1"



# cat kube-scheduler-csr.json
{
  "CN": "system:kube-scheduler",
  "host": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "system:kube-scheduler",
      "OU": "system"
    }
  ]
}


# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes kube-scheduler-csr.json | cfssljson -bare kube-scheduler

    生成kube-scheduler文件

    # KUBE_CONFIG="/usr/local/kubernetes/cfg/kube-scheduler.kubeconfig"
# KUBE_APISERVER="https://192.168.7.209:6443"

# kubectl config set-cluster kubernetes \
> --certificate-authority=/usr/local/kubernetes/ssl/ca.pem \
> --embed-certs=true \
> --server=${KUBE_APISERVER} \
> --kubeconfig=${KUBE_CONFIG}

# mv kube-scheduler.pem kube-scheduler-key.pem /usr/local/kubernetes/ssl/

# kubectl config set-credentials kube-scheduler \
> --client-certificate=/usr/local/kubernetes/ssl/kube-scheduler.pem \
> --client-key=/usr/local/kubernetes/ssl/kube-scheduler-key.pem \
> --embed-certs=true \
> --kubeconfig=${KUBE_CONFIG}

# kubectl config set-context default \
> --cluster=kubernetes \
> --user=kube-scheduler \
> --kubeconfig=${KUBE_CONFIG}

# kubectl config set-context default \
> --cluster=kubernetes \
> --user=kube-scheduler \
> --kubeconfig=${KUBE_CONFIG}

    systemd管理scheduler

    # cat /usr/lib/systemd/system/kube-scheduler.service 
[Unit]
Desription=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes

[Service]
EnvironmentFile=/usr/local/kubernetes/cfg/kube-scheduler.conf
ExecStart=/usr/local/kubernetes/bin/kube-scheduler $KUBE_SCHEDULER_OPTS
StandardOutput=file:/var/log/kubernetes/logs/kube-scheduler.log
StandardError=file:/var/log/kubernetes/logs/kube-scheduler.log
Restart=on-failure

[Install]
WantedBy=multi-user.target


# 日志分割
# cat /etc/logrotate.d/kube-scheduler 
/var/log/kubernetes/logs/kube-scheduler.log {
  daily
  rotate 7
  compress
  missingok
}

# 启动
# systemctl daemon-reload
# systemctl start kube-scheduler.service 
# systemctl status kube-scheduler.service
# systemctl enable kube-scheduler.service

    6.5  生成kubectl连接集群的证书

    # cat admin-csr.json 
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "system:masters",
      "OU": "System"
    }
  ]
}


# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

    6.6 集群状态查看

    # cat admin-csr.json
{
  "CN": "admin",
  "hosts": [],
  "key": {
    "algo": "rsa",
    "size": 2048
  },
  "names": [
    {
      "C": "CN",
      "L": "BeiJing",
      "ST": "BeiJing",
      "O": "system:masters",
      "OU": "system"
    }
  ]
}


# cfssl gencert -ca=ca.pem -ca-key=ca-key.pem -config=ca-config.json -profile=kubernetes admin-csr.json | cfssljson -bare admin

    生成kubeconfig文件

    # mkdir /root/.kube

# KUBE_CONFIG="/root/.kube/config"
# KUBE_APISERVER="https://192.168.7.209:6443"

# kubectl config set-cluster kubernetes \
> --certificate-authority=/usr/local/kubernetes/ssl/ca.pem \
> --embed-certs=true \
> --server=${KUBE_APISERVER} \
> --kubeconfig=${KUBE_CONFIG}

# mv TLS/k8s/admin.pem /usr/local/kubernetes/ssl/
# mv TLS/k8s/admin-key.pem /usr/local/kubernetes/ssl/

# kubectl config set-credentials cluster-admin \
> --client-certificate=/usr/local/kubernetes/ssl/admin.pem \
> --client-key=/usr/local/kubernetes/ssl/admin-key.pem \
> --embed-certs=true \
> --kubeconfig=${KUBE_CONFIG}

# kubectl config set-context default \
> --cluster=kubernetes \
> --user=cluster-admin \
> --kubeconfig=${KUBE_CONFIG}



# kubectl config use-context default --kubeconfig=${KUBE_CONFIG}

    6.6 授权kubelet-bootstrap用户允许请求证书

    # kubectl create clusterrolebinding kubelet-bootstrap --clusterrole=system:node-bootstrapper --user=kubelet-bootstrap

    七、第二台master部署

    7.1 从第一台master拷贝相关文件

    # scp -r root@192.168.7.200:/usr/local/kubernetes /usr/local/

# scp -r root@192.168.7.200:/usr/bin/kubectl /usr/bin/

# scp -r root@192.168.7.200:/usr/lib/systemd/system/kube-*.service /usr/lib/systemd/system/

# scp -r root@192.168.7.200:/etc/logrotate.d/kube-* /etc/logrotate.d/

# scp -r root@192.168.7.200:/root/.kube /root/

# mkdir -p /var/log/kubernetes/logs

    7.2 修改api-server配置

    # systemctl daemon-reload
    # systemctl enable --now kube-apiserver.service

    7.3 kube-controller-manager

    # systemctl daemon-reload
    # systemctl enable --now kube-controller-manager.service

    # systemctl status kube-controller-manager.service 

    7.4 kube-scheduler

    # systemctl daemon-reload

    # systemctl enable --now kube-scheduler.service

    # systemctl status kube-scheduler.service

    网站公告

    今日签到

    点亮在社区的每一天
    去签到

    热门文章

    最新发布