k8s1.27版本集群部署minio分布式

需求：

1.创建4个pv，一个pv一个minio-pod。使用sts动态分配pvc(根据存储类找到pv)。----持久化

2.暴露minio的9001端口。（nodeport）----管理界面

镜像：minio/minio:RELEASE.2023-03-20T20-16-18Z--->换国内源

说明：本环境仅测试，两个minio的pod放在master节点，在master加了污点容忍，其他两个miniopod放在工作节点，上生产环境需要调整，官方建议：4个minio需要4块硬盘，2个完好可读，3个完好可读写，可以挂一块磁盘到pv目录。

159m  2个minio+/data/minio-pv-0,1

160n   1个minio+/data/minio-pv-2

161n   1个minio+/data/minio-pv-3

159m# mkdir /data/minio-pv-{0..1} && chmod 777 -R /data/minio-pv-{0..1}

160n#mkdir /data/minio-pv-2 && chmod 777 -R /data/minio-pv-2

161n#mkdir /data/minio-pv-3 && chmod 777 -R /data/minio-pv-3

#159master上操作

mkdir minio-kind && cd minio-kind

vim minio-ns.yaml  #官方建议独立分配minio命名空间

apiVersion: v1
kind: Namespace
metadata:
  name: minio

vim minio-pv.yaml

apiVersion: storage.k8s.io/v1
kind: StorageClass
metadata:
  name: minio-storage   #创建类
provisioner: kubernetes.io/no-provisioner
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: minio-pv-0
  labels:
    type: local
spec:
  capacity:
    storage: 1Gi  #生产环境需要调整扩大
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: minio-storage
  local:
    path: /data/minio-pv-0
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - 159m     #kubectl get nodes查看node名称
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: minio-pv-1
  labels:
    type: local
spec:
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: minio-storage
  local:
    path: /data/minio-pv-1
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - 159m
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: minio-pv-2
  labels:
    type: local
spec:
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: minio-storage
  local:
    path: /data/minio-pv-2
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - 160n
---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: minio-pv-3
  labels:
    type: local
spec:
  capacity:
    storage: 1Gi
  accessModes:
    - ReadWriteOnce
  persistentVolumeReclaimPolicy: Retain
  storageClassName: minio-storage
  local:
    path: /data/minio-pv-3
  nodeAffinity:
    required:
      nodeSelectorTerms:
      - matchExpressions:
        - key: kubernetes.io/hostname
          operator: In
          values:
          - 161n

vim minio-sts.yaml  #svc,secret一起写了，注意要保证集群的coredns正常工作

apiVersion: apps/v1
kind: StatefulSet
metadata:
  name: minio
  namespace: minio
spec:
  serviceName: "minio"
  replicas: 4  # 集群节点数量
  selector:
    matchLabels:
      app: minio
  template:
    metadata:
      labels:
        app: minio
    spec:
      tolerations:
      - key: "node-role.kubernetes.io/control-plane"
        operator: "Exists"
        effect: "NoSchedule"
      containers:
      - name: minio
        #image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/quay.io/minio/minio:latest
        image: swr.cn-north-4.myhuaweicloud.com/ddn-k8s/docker.io/minio/minio:RELEASE.2023-03-20T20-16-18Z
        args:
        - server
        - http://minio-0.minio:9000/data
        - http://minio-1.minio:9000/data
        - http://minio-2.minio:9000/data
        - http://minio-3.minio:9000/data
        - --console-address
        - ":9001"  # 控制台地址参数
        - --address
        - ":9000"  # API 地址参数
        ports:
        - containerPort: 9000
          name: minio
        - containerPort: 9001
          name: minio-console
        env:
        - name: MINIO_ROOT_PASSWORD
          valueFrom:
            secretKeyRef:
              name: minio-secret
              key: MINIO_ROOT_PASSWORD
        - name: MINIO_ROOT_USER
          valueFrom:
            secretKeyRef:
              name: minio-secret
              key: MINIO_ROOT_USER
              #- name: MINIO_BROWSER
              #value: "on"
        volumeMounts:
        - name: minio-data
          mountPath: /data
  volumeClaimTemplates:
  - metadata:
      name: minio-data
    spec:
      accessModes: [ "ReadWriteOnce" ]
      storageClassName: minio-storage  # 使用 minio-storage 存储类
      resources:
        requests:
          storage: 1Gi  # 每个节点的存储大小
---
apiVersion: v1
kind: Service
metadata:
  name: minio
  namespace: minio
spec:
        #clusterIP: None
  ports:
  - port: 9000
    targetPort: 9000
    protocol: TCP
    name: minio
    nodePort: 30001
  - port: 9001
    targetPort: 9001
    protocol: TCP
    name: minio-console
    nodePort: 30002
  selector:
    app: minio
  type: NodePort
---
apiVersion: v1
kind: Secret
metadata:
  name: minio-secret
  namespace: minio
type: Opaque
data: 
  MINIO_ROOT_USER: cXdlYXNkMTE=  #用户名base64，这里的参数根据版本不同可能需要调整为access
  MINIO_ROOT_PASSWORD: cXdlYXNkMTE=  #密码base64

running起来了过一会再用mc(linux系统需要安装)连接测试，初始化时间较长

#测试连接

kubectl port-forward svc/minio 9000:9000 -n minio #临时测试
mc alias set myminio http://127.0.0.1:9000 用户名 密码

#防火墙开放30001和30002端口，提供外部访问(nodeport)

ufw allow 30001/TCP

ufw allow 30002/TCP

#访问管理页面

10.10.10.159:30002 

报错：

ERROR Unable to initialize backend: format.json file: expected format-type: fs, found: xl

delete -f .资源后，需要清理干净pv残留的数据，隐藏目录需要指定删除，不然会一直引用之前的数据。这里删除的是我的pv环境目录，根据场景删除。

rm -fr /data/minio-pv-0/.minio.sys/

rm -fr /data/minio-pv-1/.minio.sys/

rm -fr /data/minio-pv-2/.minio.sys/

rm -fr /data/minio-pv-3/.minio.sys/