K8s 1.32.6版本部署文档

主机配置

设置IP

方式一：
nmcli connection modify ens160 ipv4.addresses 172.16.1.30/24 ipv4.gateway 172.16.1.1 ipv4.method manual
nmcli connection modify ens160 ipv4.addresses 172.16.1.31/24 ipv4.gateway 172.16.1.1 ipv4.method manual
nmcli connection modify ens160 ipv4.addresses 172.16.1.32/24 ipv4.gateway 172.16.1.1 ipv4.method manual
nmcli connection modify ens160 ipv4.addresses 172.16.1.33/24 ipv4.gateway 172.16.1.1 ipv4.method manual

nmcli connection up ens160

方式二：
vi /etc/NetworkManager/system-connections/ens160.nmconnection
method=manual## 在IPV4下面修改如下内容
address1=192.168.0.5/24,192.168.0.1## 修改IP，子网掩码（24是子网掩码的24位，对应255.255.255.0）， 网关
dns=119.29.29.29;114.114.114.114## 设置DNS服务
may-fail=false
重新加载配置文件
nmcli connection reload ens160.nmconnection
激活配置文件
nmcli connection up ens160

配置YUM源

1. 配置yum源

（1）确认文件是否存在且可读
sudo cat /etc/yum.repos.d/rocky.repo
如果文件不存在或内容为空，重新创建它。

（2）重新下载正确的阿里云源文件
sudo rm -f /etc/yum.repos.d/rocky.repo  # 删除旧文件（如果有）
sudo curl -o /etc/yum.repos.d/rocky.repo https://mirrors.aliyun.com/rockylinux/rocky.repo?repo=rocky-9
（3）手动编辑文件（如果下载失败）
sudo vi /etc/yum.repos.d/rocky.repo
粘贴以下内容（阿里云 Rocky Linux 9 镜像源）：
[baseos]
name=Rocky Linux $releasever - BaseOS - Aliyun
baseurl=https://mirrors.aliyun.com/rockylinux/$releasever/BaseOS/$basearch/os/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial

[appstream]
name=Rocky Linux $releasever - AppStream - Aliyun
baseurl=https://mirrors.aliyun.com/rockylinux/$releasever/AppStream/$basearch/os/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial

[extras]
name=Rocky Linux $releasever - Extras - Aliyun
baseurl=https://mirrors.aliyun.com/rockylinux/$releasever/extras/$basearch/os/
gpgcheck=1
enabled=1
gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-rockyofficial


（4）也可以直接替换yum源里的地址
sed -e 's|^mirrorlist=|#mirrorlist=|g' \
    -e 's|^#baseurl=http://dl.rockylinux.org/$contentdir|baseurl=https://mirrors.aliyun.com/rockylinux|g' \
    -i.bak \
    /etc/yum.repos.d/rocky*.repo

2. 强制替换变量为 Rocky Linux 9
   确保 $releasever 和 $basearch 被正确解析：

sudo sed -i 's/$releasever/9/g' /etc/yum.repos.d/rocky.repo
sudo sed -i 's/$basearch/x86_64/g' /etc/yum.repos.d/rocky.repo  # 如果是 x86_64 架构

3. 导入 GPG 密钥

sudo rpm --import https://mirrors.aliyun.com/rockylinux/RPM-GPG-KEY-rockyofficial

4. 检查文件权限和格式
   （1）确保文件权限正确

   sudo chmod 644 /etc/yum.repos.d/rocky.repo
   

（2）检查文件格式（避免 UTF-8 BOM 或 Windows 换行符）

```bash
sudo dos2unix /etc/yum.repos.d/rocky.repo  # 如果是从 Windows 复制的文件
```

清除缓存并重新加载

sudo dnf clean all
sudo dnf makecache

6. 验证仓库是否启用

sudo dnf repolist
正常输出应类似：
text
repo id                          repo name
baseos                           Rocky Linux 9 - BaseOS - Aliyun
appstream                        Rocky Linux 9 - AppStream - Aliyun
extras                           Rocky Linux 9 - Extras - Aliyun

在 Rocky Linux 9 中启用并安装 EPEL Repo。

dnf install epel-release

备份(如有配置其他epel源)并替换为国内镜像
注意最后这个库，阿里云没有对应的镜像，不要修改它，如果误改恢复原版源即可

cp /etc/yum.repos.d/epel.repo  /etc/yum.repos.d/epel.repo.backup 
cp /etc/yum.repos.d/epel-testing.repo  /etc/yum.repos.d/epel-testing.repo.backup
cp /etc/yum.repos.d/epel-cisco-openh264.repo  /etc/yum.repos.d/epel-cisco-openh264.repo.backup

将 repo 配置中的地址替换为阿里云镜像站地址

执行下面语句，它会替换epel.repo、eple-testing.repo中的网址，不会修改epel-cisco-openh264.repo，可以正常使用。

sed -e 's!^metalink=!#metalink=!g' \
    -e 's!^#baseurl=!baseurl=!g' \
    -e 's!https\?://download\.fedoraproject\.org/pub/epel!https://mirrors.aliyun.com/epel!g' \
    -e 's!https\?://download\.example/pub/epel!https://mirrors.aliyun.com/epel!g' \
    -i /etc/yum.repos.d/epel{,-testing}.repo

更新仓库缓存

dnf clean all 
dnf makecache   ---生成缓存，安装软件更快

每台机器单独做

hostnamectl set-hostname k8s-master
hostnamectl set-hostname k8s-node1
hostnamectl set-hostname k8s-node2
hostnamectl set-hostname k8s-node3

设置hosts

cat >> /etc/hosts << EOF
172.16.1.30 k8s-master
172.16.1.31 k8s-node1
172.16.1.32 k8s-node2
172.16.1.33 k8s-node3
EOF

配置免密登录，只在k8s-master上操作

[root@k8s-master ~]# ssh-keygen -f ~/.ssh/id_rsa -N '' -q

拷贝密钥到其他3 台节点

[root@k8s-master ~]# ssh-copy-id k8s-node1
[root@k8s-master ~]# ssh-copy-id k8s-node2
[root@k8s-master ~]# ssh-copy-id k8s-node3

防火墙和SELinux

# 关闭防火墙
systemctl disable --now firewalld
# 禁用SELinux
sed -i '/^SELINUX=/ c SELINUX=disabled' /etc/selinux/config
# 重启生效所以临时设置为宽容模式
setenforce 0

时间同步配置

# 安装时间服务器软件包
dnf install -y chrony
# 修改同步服务器
sed -i '/^pool/ c pool ntp1.aliyun.com  iburst' /etc/chrony.conf
systemctl restart chronyd
systemctl enable chronyd
chronyc sources

配置内核转发及网桥过滤

# 添加网桥过滤及内核转发配置文件
cat > /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
vm.swappiness = 0
EOF

# 加载br_netfilter模块
modprobe br_netfilter

使用新添加配置文件生效

sysctl -p /etc/sysctl.d/k8s.conf

关闭swap

查看交换分区情况

# 临时关闭
swapoff -a
# 永远关闭swap分区
sed -i 's/.*swap.*/#&/' /etc/fstab

启用ipvs

cat >> /etc/modules-load.d/ipvs.conf << EOF
br_netfilter
ip_conntrack
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF


# 安装依赖
dnf install ipvsadm ipset sysstat conntrack libseccomp -y

重启服务

systemctl restart systemd-modules-load.service

查看模块内容

lsmod | grep -e ip_vs -e nf_conntrack

句柄数最大

# 设置为最大
ulimit -SHn 65535

cat >> /etc/security/limits.conf <<EOF
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* seft memlock unlimited
* hard memlock unlimitedd
EOF

      
# 查看修改结果
ulimit -a

系统优化

cat > /etc/sysctl.d/k8s_better.conf << EOF
net.bridge.bridge-nf-call-iptables=1
net.bridge.bridge-nf-call-ip6tables=1
net.ipv4.ip_forward=1
vm.swappiness=0
vm.overcommit_memory=1
vm.panic_on_oom=0
fs.inotify.max_user_instances=8192
fs.inotify.max_user_watches=1048576
fs.file-max=52706963
fs.nr_open=52706963
net.ipv6.conf.all.disable_ipv6=1
net.netfilter.nf_conntrack_max=2310720
EOF


modprobe br_netfilter
lsmod |grep conntrack
modprobe ip_conntrack
sysctl -p /etc/sysctl.d/k8s_better.conf

安装docker

# Step 1: 安装依赖
yum install -y yum-utils device-mapper-persistent-data lvm2
 
# Step 2: 添加软件源信息
yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/rhel/docker-ce.repo

# Step 3: 安装Docker-CE
yum -y install docker-ce
 
# docker -v
Docker version 27.5.1, build 9f9e405
 
# 设置国内镜像加速
mkdir -p /etc/docker/ 
cat  >> /etc/docker/daemon.json << EOF
{
   "registry-mirrors":["https://p3kgr6db.mirror.aliyuncs.com",
   "https://docker.m.daocloud.io",
   "https://your_id.mirror.aliyuncs.com",
   "https://docker.nju.edu.cn/",
    "https://docker.anyhub.us.kg",
    "https://dockerhub.jobcher.com",
    "https://dockerhub.icu",
    "https://docker.ckyl.me",
       "https://cr.console.aliyun.com"
   ],
"exec-opts": ["native.cgroupdriver=systemd"]
}
EOF
 
 
# 设置docker开机启动并启动
systemctl enable --now docker

# 查看docker版本
docker version

安装cri-dockerd

下载地址：Releases · Mirantis/cri-dockerd (github.com)。

https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.16/cri-dockerd-0.3.16-3.fc35.x86_64.rpm

安装cri-docker

# 下载rpm包
wget -c https://github.com/Mirantis/cri-dockerd/releases/download/v0.3.16/cri-dockerd-0.3.16-3.fc35.x86_64.rpm
wget -c https://rpmfind.net/linux/almalinux/8.10/BaseOS/x86_64/os/Packages/libcgroup-0.41-19.el8.x86_64.rpm

# 安装rpm包
yum install libcgroup-0.41-19.el8.x86_64.rpm
yum install cri-dockerd-0.3.16-3.fc35.x86_64.rpm

设置cri-docker服务开机自启

systemctl enable cri-docker

cri-docke设置国内镜像加速

# 编辑service文件
vim /usr/lib/systemd/system/cri-docker.service文件
修改第10行内容
------------------
ExecStart=/usr/bin/cri-dockerd  --pod-infra-container-image=registry.aliyuncs.com/google_containers/pause:3.9  --container-runtime-endpoint  fd://
-----------------------------------
 
# 重启Docker组件
systemctl daemon-reload && systemctl restart docker cri-docker.socket cri-docker 

# 检查Docker组件状态
systemctl status docker cir-docker.socket cri-docker

K8S软件安装

# 1、配置kubernetes源
cat <<EOF | tee /etc/yum.repos.d/kubernetes.repo
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.32/rpm/
enabled=1
gpgcheck=1
gpgkey=https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.32/rpm/repodata/repomd.xml.key
EOF
 
# 2、查看所有可用的版本
yum list kubelet --showduplicates | sort -r |grep 1.32
 
# 3、安装kubelet、kubeadm、kubectl、kubernetes-cni
yum install -y kubelet kubeadm kubectl kubernetes-cni
 
# 4、配置cgroup
为了实现docker使用的cgroupdriver与kubelet使用的cgroup的一致性，建议修改如下文件内容。
vim /etc/sysconfig/kubelet  [3台全部设置下]
---------------------
KUBELET_EXTRA_ARGS="--cgroup-driver=systemd"
---------------------

# 5、设置kubelet为开机自启动即可，由于没有生成配置文件，集群初始化后自动启动
systemctl enable kubelet

K8S集群初始化

# 只在k8s-master节点上操作
[root@localhost ~]#  kubeadm config print init-defaults > kubeadm-init.yaml

# 编辑kubeadm-init.yaml修改如下配置：
- advertiseAddress：为控制平面地址，（Master主机IP）
	advertiseAddress: 1.2.3.4
修改为 advertiseAddress: 172.16.1.30

- criSocket：为 containerd 的socket 文件地址
	criSocket: unix:///var/run/containerd/containerd.sock
修改为 criSocket: unix:///var/run/cri-dockerd.sock

- name: node 修改node为k8s-master
	name: node
修改为 name: k8s-master
 
- imageRepository：阿里云镜像代理地址，否则拉取镜像会失败
	imageRepository: registry.k8s.io
修改为：imageRepository: registry.aliyuncs.com/google_containers

- kubernetesVersion：为k8s版本
	kubernetesVersion: 1.32.0
修改为：kubernetesVersion: 1.32.6


# 文件末尾增加启用ipvs功能
---
apiVersion: kubeproxy.config.k8s.io/v1alpha1
kind: KubeProxyConfiguration
mode: ipvs



# 根据配置文件启动kubeadm初始化k8s
$ kubeadm init --config=kubeadm-init.yaml --upload-certs --v=6
输出结果：
Your Kubernetes control-plane has initialized successfully!
 
To start using your cluster, you need to run the following as a regular user:
 
  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config
 
Alternatively, if you are the root user, you can run:
 
  export KUBECONFIG=/etc/kubernetes/admin.conf
 
You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/
 
Then you can join any number of worker nodes by running the following on each as root:
 
kubeadm join 172.16.1.30:6443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:9d25c16abfec6ff6832ed2260c6c998d3fa6fedef61529d88520d3038bdbdde5

K8S集群工作节点加入

# 注意：加入集群时需要添加 --cri-socket unix:///var/run/cri-dockerd.sock
kubeadm join 172.16.1.30:6443 --token abcdef.0123456789abcdef \
	--discovery-token-ca-cert-hash sha256:9d25c16abfec6ff6832ed2260c6c998d3fa6fedef61529d88520d3038bdbdde5 \
        --cri-socket unix:///var/run/cri-dockerd.sock

K8S集群网络插件使用

# 下载calico资源清单
wget  --no-check-certificate   https://raw.githubusercontent.com/projectcalico/calico/v3.28.0/manifests/calico.yaml

# 修改calico文件
vim calico.yaml
- name: CALICO_IPV4POOL_CIDR
  value: "10.244.0.0/16"
  
# 可以将镜像提前拉取下来,如果官网仓库不可达，可以尝试手动从quay.io下载镜像，quay.io是一个公共镜像仓库。
docker pull calico/cni:v3.28.0
docker pull calico/node:v3.28.0
docker pull calico/kube-controllers:v3.28.0

# 应用calico资源清单
kubectl apply -f calico.yaml

Kubectl命令自动补全

yum -y install bash-completion
source /usr/share/bash-completion/bash_completion
source <(kubectl completion bash)
echo "source <(kubectl completion bash)" >> ~/.bashrc

安装helm v3.16.3

wget https://get.helm.sh/helm-v3.16.3-linux-amd64.tar.gz
tar xf helm-v3.16.3-linux-amd64.tar.gz
cd linux-amd64/
mv helm /usr/local/bin
helm version

部署动态sc存储

# k8s-master节点上执行
yum -y install nfs-utils
echo "/nfs/data/ *(insecure,rw,sync,no_root_squash)" > /etc/exports
mkdir -p /nfs/data/
chmod 777 -R /nfs/data/
systemctl enable rpcbind
systemctl enable nfs-server
systemctl start rpcbind
systemctl start nfs-server
exportfs  -v

创建nfs-provisioner

apiVersion: v1
kind: ServiceAccount
metadata:
  name: nfs-client-provisioner  # sa名字，nfs-provisioner-deploy里的要对应
  namespace: kube-system  # 命名空间
---
kind: ClusterRole
apiVersion: rbac.authorization.k8s.io/v1  # 创建集群规则
metadata:
  name: nfs-client-provisioner-runner
rules:
  - apiGroups: [""]
    resources: ["persistentvolumes"]
    verbs: ["get", "list", "watch", "create", "delete"]
  - apiGroups: [""]
    resources: ["persistentvolumeclaims"]
    verbs: ["get", "list", "watch", "update"]
  - apiGroups: ["storage.k8s.io"]
    resources: ["storageclasses"]
    verbs: ["get", "list", "watch"]
  - apiGroups: [""]
    resources: ["events"]
    verbs: ["create", "update", "patch"]
---
kind: ClusterRoleBinding   # 将服务认证用户与集群规则进行绑定
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: run-nfs-client-provisioner
subjects:
  - kind: ServiceAccount   # 类型为sa
    name: nfs-client-provisioner   # sa的名字一致
    namespace: kube-system  # 和nfs provisioner安装的namespace一致
roleRef:
  kind: ClusterRole
  name: nfs-client-provisioner-runner
  apiGroup: rbac.authorization.k8s.io
---
kind: Role
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  namespace: kube-system # 和nfs provisioner安装的namespace一致
rules:
  - apiGroups: [""]
    resources: ["endpoints"]
    verbs: ["get", "list", "watch", "create", "update", "patch"]
---
kind: RoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: leader-locking-nfs-client-provisioner
  namespace: kube-system  # 和nfs provisioner安装的namespace一致
subjects:
  - kind: ServiceAccount  # 类型为sa
    name: nfs-client-provisioner  # sa的名字一致
    namespace: kube-system # 和nfs provisioner安装的namespace一致
roleRef:
  kind: Role
  name: leader-locking-nfs-client-provisioner
  apiGroup: rbac.authorization.k8s.io
---
apiVersion: apps/v1
kind: Deployment
metadata:
  name: nfs-client-provisioner
  labels:
    app: nfs-client-provisioner
  namespace: kube-system  # 部署在指定ns下
spec:
  replicas: 1   # 副本数，建议为奇数[1，3，5，7，9]
  strategy:
    type: Recreate   # 使用重建的升级策略
  selector:
    matchLabels:
      app: nfs-client-provisioner
  template:
    metadata:
      labels:
        app: nfs-client-provisioner
    spec:
      serviceAccountName: nfs-client-provisioner  # sa名字，这个是在nfs-rbac.yaml里定义
      containers:
        - name: nfs-client-provisioner  # 容器名字
          image: k8s.m.daocloud.io/sig-storage/nfs-subdir-external-provisioner:v4.0.2  # 镜像地址，这里采用私有仓库。
          volumeMounts:
            - name: nfs-client-root
              mountPath: /persistentvolumes  # 指定容器内挂载的目录
          env:
            - name: PROVISIONER_NAME  # 容器内的变量用于指定提供存储的名称
              value: nfsnas     # nfs-provisioner的名称，以后设置的storage class要和这个保持一致
            - name: NFS_SERVER      # 容器内的变量指定nfs服务器对应的目录
              value: 172.16.1.254  # NFS服务器的地址
            - name: NFS_PATH           # 容器内的变量指定nfs服务器对应的目录
              value: /volume1/服务/K8s-NFS   # NFS服务的挂载目录，如果采用这个nfs动态申请PV,所创建的文件在这个目录里，一定要给权限，直接777。
      volumes:
        - name: nfs-client-root  # 赋值卷名字
          nfs:
            server: 172.16.1.254  # NFS服务器的地址
            path: /volume1/服务/K8s-NFS   # NFS服务的挂载目录，一定要给权限，直接777，不服就是干
---
apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: nfsnas
  annotations:
    storageclass.kubernetes.io/is-default-class: "true"  # 设为默认存储类
provisioner: nfsnas  # 必须与 Deployment 中 PROVISIONER_NAME 一致
parameters:
  archiveOnDelete: "false"   # "true" 表示删除 PVC 时归档数据（重命名目录）