背景

服务器配置

节点 内网IP 公网IP 配置
ds-2 10.80.239.78 无 4C16G
ds-8 10.80.239.84 无 4C16G
ds-9 10.80.239.85 无 4C16G
ds-10 10.80.239.86 无 4C16G
ds-11 10.80.239.87 无 4C16G

软件版本

软件 版本
centos 7.6
docker 20.10.7
kubelet 1.20.9
kubeadm 1.20.9
kubectl 1.20.9

镜像版本

镜像 版本
k8s.gcr.io/kube-apiserver 1.20.9
k8s.gcr.io/kube-controller-manager 1.20.9
k8s.gcr.io/kube-scheduler 1.20.9
k8s.gcr.io/kube-proxy 1.20.9
k8s.gcr.io/pause 3.2
k8s.gcr.io/etcd 3.4.13-0
k8s.gcr.io/coredns 1.7.0

创建初始文件夹

  1. #/Users/keyboardone/同步空间/software
  2. mkdir -p /opt/software
  3. cd /opt/software/k8s/
  4. chmod 755 /opt/software/k8s/*.sh

配置ssh免密

ds-2

ds-2.sh

  1. cd /opt/software/k8s/
  2. vi ds-2.sh
  1. #修改主机名
  2. sudo hostnamectl set-hostname ds-2
  3. sudo hostnamectl set-hostname "ds-2" --pretty
  4. sudo hostnamectl set-hostname ds-2 --static
  5. sudo hostnamectl set-hostname ds-2 --transient
  6. #标识其他主机名
  7. cat > /etc/hosts <<EOF
  8. 127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
  9. ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
  10. 10.80.239.78 ds-2
  11. 10.80.239.84 ds-8
  12. 10.80.239.85 ds-9
  13. 10.80.239.86 ds-10
  14. 10.80.239.87 ds-11
  15. EOF
  16. #清空密钥
  17. cd ~/.ssh/
  18. rm -rf *
  19. #用户目录下生成公钥、私钥文件
  20. ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
  21. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
  22. chmod 600 ~/.ssh/authorized_keys
  23. #分发公钥到其他主机
  24. for ip in ds-8 ds-9 ds-10 ds-11;     # 请将此处主机名替换为自己要部署的机器的 hostname
  25. do
  26. ssh-copy-id  $ip   # 该操作执行过程中需要手动输入用户的密码
  27. done

ds-8

ds-8.sh

  1. cd /opt/software/k8s/
  2. vi ds-8.sh
  1. #修改主机名
  2. sudo hostnamectl set-hostname ds-8
  3. sudo hostnamectl set-hostname "ds-8" --pretty
  4. sudo hostnamectl set-hostname ds-8 --static
  5. sudo hostnamectl set-hostname ds-8 --transient
  6. #标识其他主机名
  7. cat > /etc/hosts <<EOF
  8. 127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
  9. ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
  10. 10.80.239.78 ds-2
  11. 10.80.239.84 ds-8
  12. 10.80.239.85 ds-9
  13. 10.80.239.86 ds-10
  14. 10.80.239.87 ds-11
  15. EOF
  16. #清空密钥
  17. cd ~/.ssh/
  18. rm -rf *
  19. #用户目录下生成公钥、私钥文件
  20. ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
  21. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
  22. chmod 600 ~/.ssh/authorized_keys
  23. #分发公钥到其他主机
  24. for ip in ds-2 ds-9 ds-10 ds-11;     # 请将此处主机名替换为自己要部署的机器的 hostname
  25. do
  26. ssh-copy-id  $ip   # 该操作执行过程中需要手动输入用户的密码
  27. done

ds-9

ds-9.sh

  1. cd /opt/software/k8s/
  2. vi ds-9.sh
  1. #修改主机名
  2. sudo hostnamectl set-hostname ds-9
  3. sudo hostnamectl set-hostname "ds-9" --pretty
  4. sudo hostnamectl set-hostname ds-9 --static
  5. sudo hostnamectl set-hostname ds-9 --transient
  6. #标识其他主机名
  7. cat > /etc/hosts <<EOF
  8. 127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
  9. ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
  10. 10.80.239.78 ds-2
  11. 10.80.239.84 ds-8
  12. 10.80.239.85 ds-9
  13. 10.80.239.86 ds-10
  14. 10.80.239.87 ds-11
  15. EOF
  16. #清空密钥
  17. cd ~/.ssh/
  18. rm -rf *
  19. #用户目录下生成公钥、私钥文件
  20. ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
  21. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
  22. chmod 600 ~/.ssh/authorized_keys
  23. #分发公钥到其他主机
  24. for ip in ds-2 ds-8 ds-10 ds-11;     # 请将此处主机名替换为自己要部署的机器的 hostname
  25. do
  26. ssh-copy-id  $ip   # 该操作执行过程中需要手动输入用户的密码
  27. done

ds-10

ds-10.sh

  1. cd /opt/software/k8s/
  2. vi ds-10.sh
  1. #修改主机名
  2. sudo hostnamectl set-hostname ds-10
  3. sudo hostnamectl set-hostname "ds-10" --pretty
  4. sudo hostnamectl set-hostname ds-10 --static
  5. sudo hostnamectl set-hostname ds-10 --transient
  6. #标识其他主机名
  7. cat > /etc/hosts <<EOF
  8. 127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
  9. ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
  10. 10.80.239.78 ds-2
  11. 10.80.239.84 ds-8
  12. 10.80.239.85 ds-9
  13. 10.80.239.86 ds-10
  14. 10.80.239.87 ds-11
  15. EOF
  16. #清空密钥
  17. cd ~/.ssh/
  18. rm -rf *
  19. #用户目录下生成公钥、私钥文件
  20. ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
  21. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
  22. chmod 600 ~/.ssh/authorized_keys
  23. #分发公钥到其他主机
  24. for ip in ds-2 ds-8 ds-9 ds-11;     # 请将此处主机名替换为自己要部署的机器的 hostname
  25. do
  26. ssh-copy-id  $ip   # 该操作执行过程中需要手动输入用户的密码
  27. done

ds-11

ds-11.sh

  1. cd /opt/software/k8s/
  2. vi ds-11.sh
  1. #修改主机名
  2. sudo hostnamectl set-hostname ds-11
  3. sudo hostnamectl set-hostname "ds-11" --pretty
  4. sudo hostnamectl set-hostname ds-11 --static
  5. sudo hostnamectl set-hostname ds-11 --transient
  6. #标识其他主机名
  7. cat > /etc/hosts <<EOF
  8. 127.0.0.1   localhost localhost.localdomain localhost4 localhost4.localdomain4
  9. ::1         localhost localhost.localdomain localhost6 localhost6.localdomain6
  10. 10.80.239.78 ds-2
  11. 10.80.239.84 ds-8
  12. 10.80.239.85 ds-9
  13. 10.80.239.86 ds-10
  14. 10.80.239.87 ds-11
  15. EOF
  16. #清空密钥
  17. cd ~/.ssh/
  18. rm -rf *
  19. #用户目录下生成公钥、私钥文件
  20. ssh-keygen -t rsa -P '' -f ~/.ssh/id_rsa
  21. cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
  22. chmod 600 ~/.ssh/authorized_keys
  23. #分发公钥到其他主机
  24. for ip in ds-2 ds-8 ds-9 ds-10;     # 请将此处主机名替换为自己要部署的机器的 hostname
  25. do
  26. ssh-copy-id  $ip   # 该操作执行过程中需要手动输入用户的密码
  27. done

文件准备

上传相关文件到 /opt/software/
image.png
分发

  1. scp -r /opt/software/ ds-8:/opt/
  2. scp -r /opt/software/ ds-9:/opt/
  3. scp -r /opt/software/ ds-10:/opt/
  4. scp -r /opt/software/ ds-11:/opt/

安装docker

rpm版(依赖epel)

  1. #安装yum存储库
  2. yum install -y epel-release.noarch
  3. #安装Docker Yum源
  4. yum -y install yum-utils
  5. yum-config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
  6. #rpm离线安装
  7. cd /opt/software/offline_package/docker
  8. rpm -ivh --replacefiles --replacepkgs *.rpm
  9. #启动Docker
  10. systemctl enable docker.service
  11. systemctl start docker.service

tar版

  1. #安装docker
  2. cd /opt/software/docker
  3. tar xzvf docker-20.10.7.tgz
  4. chmod +x docker/*
  5. mv docker/* /usr/local/bin/
  7. #创建docker配置文件
  8. echo '[Unit]
  9. Description=Docker Application Container Engine
  10. Documentation=http://docs.docker.io
  11. After=network.target
  12. [Service]
  13. Environment="PATH=/usr/local/bin:/bin:/sbin:/usr/bin:/usr/sbin"
  14. ExecStart=/usr/local/bin/dockerd -H unix:///var/run/docker.sock -H tcp://0.0.0.0:2375
  15. ExecReload=/bin/kill -s HUP $MAINPID
  16. Restart=always
  17. RestartSec=5
  18. TimeoutSec=0
  19. LimitNOFILE=infinity
  20. LimitNPROC=infinity
  21. LimitCORE=infinity
  22. Delegate=yes
  23. KillMode=process
  24. [Install]
  25. WantedBy=multi-user.target
  26. ' >> /etc/systemd/system/docker.service
  28. #重新加载docker配置文件
  29. cd /usr/local/bin
  30. #重新加载配置文件
  31. systemctl daemon-reload
  32. #设置开机启动
  33. systemctl enable docker.service
  34. #启动
  35. systemctl start docker.service
  36. #重启
  37. systemctl daemon-reload
  38. systemctl restart docker
  40. #等待
  41. sleep 30s
  43. #添加docker源
  44. mkdir -p /etc/docker/
  45. touch /etc/docker/daemon.json
  46. cat > /etc/docker/daemon.json <<EOF
  47. {
  48. "registry-mirrors":["https://docker.mirrors.ustc.edu.cn/"],
  49. "exec-opts": ["native.cgroupdriver=systemd"],
  50. "insecure-registries": ["http://ren:8088"]
  51. }
  52. EOF
  54. #重启docker
  55. systemctl daemon-reload
  56. systemctl restart docker
  58. #等待
  59. sleep 1m
  61. #查看验证docker
  62. docker info

安装Kubernetes

加载Kubernetes镜像

  1. #加载Kubernetes镜像
  2. docker load -i /opt/software/images/k8simages.tar
  4. #恢复镜像名称及tag
  5. while read REPOSITORY TAG IMAGE_ID
  6. do
  7.         echo "== Tagging $REPOSITORY $TAG $IMAGE_ID =="
  8.         docker tag "$IMAGE_ID" "$REPOSITORY:$TAG"
  9. done < /opt/software/images/k8simages.list

设置Kubernetes对应内核参数

  1. #设置Kubernetes对应内核参数
  2. cat > /etc/sysctl.d/kubernetes.conf << EOF
  3. net.ipv4.ip_forward = 1
  4. net.bridge.bridge-nf-call-ip6tables = 1
  5. net.bridge.bridge-nf-call-iptables = 1
  6. EOF

重新加载参数

  1. #重新加载参数
  2. modprobe br_netfilter
  3. sysctl --system

关闭swap区

  1. #关闭swap区
  2. swapoff -a
  3. sed -e '/swap/s/^/#/g' -i /etc/fstab

关闭SELinux

  1. #关闭SELinux
  2. getenforce
  3. sestatus
  4. setenforce 0
  5. getenforce
  6. sestatus

rpm离线安装

  1. #添加yum源
  2. cat <<EOF | sudo tee /etc/yum.repos.d/kubernetes.repo
  3. [kubernetes]
  4. name=Kubernetes
  5. baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
  6. enabled=1
  7. gpgcheck=0
  8. repo_gpgcheck=0
  9. gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
  10.    http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
  11. exclude=kubelet kubeadm kubectl
  12. EOF
  14. #安装离线包
  15. systemctl stop firewalld
  16. cd /opt/software/offline_package/k8s
  17. rpm -ivh --replacefiles --replacepkgs /opt/software/offline_package/k8s/*.rpm

bash中启用kubectl

  1. #bash中启用kubectl
  2. kubectl completion bash > /etc/bash_completion.d/kubectl

确认主节点的kubelet服务状态

  1. #初始化Kubernetes主节点
  2. systemctl status kubelet
  3. systemctl daemon-reload
  4. sudo systemctl enable --now kubelet
  5. systemctl status kubelet

使用 kubeadm 引导集群

所有节点执行

  1. #所有机器添加master域名映射，以下需要修改为自己的
  2. echo "10.80.239.78  cluster-endpoint" >> /etc/hosts

主节点执行

  1. kubeadm init \
  2. --apiserver-advertise-address=10.80.239.78 \
  3. --control-plane-endpoint=cluster-endpoint \
  4. --image-repository k8s.gcr.io \
  5. --kubernetes-version v1.20.9 \
  6. --service-cidr=10.96.0.0/16 \
  7. --pod-network-cidr=192.168.0.0/16

得到

  1. Your Kubernetes control-plane has initialized successfully!
  3. To start using your cluster, you need to run the following as a regular user:
  5.   mkdir -p $HOME/.kube
  6.   sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  7.   sudo chown $(id -u):$(id -g) $HOME/.kube/config
  9. Alternatively, if you are the root user, you can run:
  11.   export KUBECONFIG=/etc/kubernetes/admin.conf
  13. You should now deploy a pod network to the cluster.
  14. Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  15.   https://kubernetes.io/docs/concepts/cluster-administration/addons/
  17. You can now join any number of control-plane nodes by copying certificate authorities
  18. and service account keys on each node and then running the following as root:
  20.   kubeadm join cluster-endpoint:6443 --token yee1e9.8nrqge5yc5xsftbi \
  21.     --discovery-token-ca-cert-hash sha256:2767e227965196fef7ac8831c12b67ac6d9babbc6e25f41b6560f48e5651eab4 \
  22.     --control-plane 
  24. Then you can join any number of worker nodes by running the following on each as root:
  26. kubeadm join cluster-endpoint:6443 --token yee1e9.8nrqge5yc5xsftbi \
  27.     --discovery-token-ca-cert-hash sha256:2767e227965196fef7ac8831c12b67ac6d9babbc6e25f41b6560f48e5651eab4

主节点执行建议脚本

  1. #执行建议脚本
  2. mkdir -p $HOME/.kube
  3. cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  4. chown $(id -u):$(id -g) $HOME/.kube/config

确认从节点的kubelet服务状态

  1. #初始化Kubernetes从节点
  2. systemctl status kubelet
  3. systemctl daemon-reload
  4. sudo systemctl enable --now kubelet
  5. systemctl status kubelet

从节点执行命令加入k8s集群

  1. kubeadm join cluster-endpoint:6443 --token yee1e9.8nrqge5yc5xsftbi \
  2.     --discovery-token-ca-cert-hash sha256:2767e227965196fef7ac8831c12b67ac6d9babbc6e25f41b6560f48e5651eab4

安装calico

上传文件

  1. kubectl apply -f calico.yaml

image.png

查看初始化进度

  1. #监听应用启动情况
  2. kubectl get pod -A -w
  3. #或者
  4. watch -n 1 kubectl get pod -A
  5. #检查各节点连接状态
  6. kubectl get pods -o wide --all-namespaces
  7. #或者
  8. watch -n 1 kubectl get pods -o wide --all-namespaces

安装完成