kubernetes集群环境搭建

kubernetes集群环境搭建环境准备准备三台虚拟机节点名称系统环境配置ipk8s-master01Centos7.82Cpu 4G内存 20G硬盘10.0.0.210k8s-node01Centos7.82Cpu 4G内存 20G硬盘10.0.0.211k8s-node02Centos7.82Cpu 4G内存 20G硬盘10.0.0.212基础配置（三台虚拟机都需要安装的环境）修改hostna

辛勤汗水@

2114人浏览 · 2022-05-18 21:19:18

辛勤汗水@ · 2022-05-18 21:19:18 发布

kubernetes集群环境搭建

环境准备

准备三台虚拟机

节点名称	系统环境	配置	ip
k8s-master01	Centos7.8	2Cpu 4G内存 20G硬盘	10.0.0.210
k8s-node01	Centos7.8	2Cpu 4G内存 20G硬盘	10.0.0.211
k8s-node02	Centos7.8	2Cpu 4G内存 20G硬盘	10.0.0.212

基础配置（三台虚拟机都需要安装的环境）

修改hostname

### k8s-master01虚拟机
hostnamectl set-hostname k8s-master01

### k8s-node01虚拟机
hostnamectl set-hostname k8s-node01

### k8s-node02虚拟机
hostnamectl set-hostname k8s-node02

主机名解析

cat >> /etc/sysctl.d/k8s.conf << EOF
10.0.0.210 k8s-master01
10.0.0.211 k8s-node02
10.0.0.212 k8s-node02
EOF

安装基础软件

yum install -y  lrzsz sl wget bash-completion

关闭防火墙

sed -i '/^SELINUX=/c SELINUX=disable' /etc/selinux/config
systemctl disable firewalld
systemctl stop firewalld

关闭swap

swapoff -a
sed -i 's/.*swap.*/#&/' /etc/fstab
### 内核（禁用swap）参数
cat >> /etc/sysctl.d/k8s.conf << EOF
vm.swappiness=0
EOF

网络参数

cat >> /etc/sysctl.d/k8s.conf << EOF
net.bridge.bridge-nf-call-ip6tables = 1
net.bridge.bridge-nf-call-iptables = 1
net.ipv4.ip_forward = 1
EOF

配置生效

modprobe br_netfilter
modprobe overlay
sysctl -p /etc/sysctl.d/k8s.conf

免密登录（可做可不做）

## 生产秘钥
ssh-keygen -t rsa
## 安装
yum install expect -y
## 执行脚本
for i in {210..212}
do
	expect -c "
	spawn ssh-copy-id root@10.0.0.$i
	expect {
	"yes/no" { send \"yes\n\"; exp_continue }
	"password:" { send \"idriverplus\n\"; exp_continue } 
	}"
done

安装docker环境

curl -o /etc/yum.repos.d/Centos-7.repo http://mirrors.aliyun.com/repo/Centos-7.repo
curl -o /etc/yum.repos.d/docker-ce.repo http://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo

cat >>/etc/yum.repos.d/kubernetes.repo <<EOF 
[kubernetes]
name=Kubernetes
baseurl=http://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=http://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg
	http://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

yum clean all && yum makecache
yum list docker-ce --showduplicates | sort -r
yum install docker-ce -y
mkdir -p /etc/docker

cat >>/etc/docker/daemon.json <<EOF
{
	 "registry-mirrors": ["https://docker.mirrors.ustc.edu.cn/","https://hub-mirror.c.163.com","https://registry.docker-cn.com"],
	 "insecure-registries" : ["如果有harbor仓库的话可以添加harbor仓库地址"],
	 "exec-opts": ["native.cgroupdriver=systemd"]
}
EOF

systemctl enable docker && systemctl restart docker

修改docker的启动项，内核

yum -y install kernel*
cat >>/etc/default/grub <<EOF 
GRUB_CMDLINE_LINUX=" cgroup_enable=memory swapaccount=1"
EOF
## 重启
reboot

安装kubernetes环境

k8s-master01节点

yum install -y kubelet-1.23.6-0 kubeadm-1.23.6-0 kubectl-1.23.6-0 --disableexcludes=kubernetes
kubeadm version
kubelet version
kubectl version
systemctl enable kubelet
echo "source <(kubectl completion bash)" >> ~/.bashrc
source ~/.bashrc

k8s-node01节点

yum install -y kubelet-1.23.6-0  kubectl-1.23.6-0 --disableexcludes=kubernetes
kubeadm version
kubelet version
kubectl version
systemctl enable kubelet
echo "source <(kubectl completion bash)" >> ~/.bashrc
source ~/.bashrc

k8s-node02节点

yum install -y kubelet-1.23.6-0  kubectl-1.23.6-0 --disableexcludes=kubernetes
kubeadm version
kubelet version
kubectl version
systemctl enable kubelet
echo "source <(kubectl completion bash)" >> ~/.bashrc
source ~/.bashrc

k8s-master01节点初始化

使用阿里云镜像（registry.aliyuncs.com/google_containers）
查看1.23.6版本所用镜像

kubeadm config images list --kubernetes-version=v1.23.6
## 需要的镜像，可修改成阿里云镜像，避免由于网络原因下载失败
k8s.gcr.io/kube-apiserver:v1.23.6
k8s.gcr.io/kube-controller-manager:v1.23.6
k8s.gcr.io/kube-scheduler:v1.23.6
k8s.gcr.io/kube-proxy:v1.23.6
k8s.gcr.io/pause:3.6
k8s.gcr.io/etcd:3.5.1-0
k8s.gcr.io/coredns/coredns:v1.8.6

执行初始化

[root@k8s-master01 opt]# kubeadm init --kubernetes-version=1.23.6 \
> --apiserver-advertise-address=10.0.0.210 \
> --image-repository registry.aliyuncs.com/google_containers \
> --service-cidr=10.96.0.0/12 \
> --pod-network-cidr=10.244.0.0/16 \
> --ignore-preflight-errors=Swap
[init] Using Kubernetes version: v1.23.6
[preflight] Running pre-flight checks
[preflight] Pulling images required for setting up a Kubernetes cluster
[preflight] This might take a minute or two, depending on the speed of your internet connection
[preflight] You can also perform this action in beforehand using 'kubeadm config images pull'
[certs] Using certificateDir folder "/etc/kubernetes/pki"
[certs] Generating "ca" certificate and key
[certs] Generating "apiserver" certificate and key
[certs] apiserver serving cert is signed for DNS names [k8s-master kubernetes kubernetes.default kubernetes.default.svc kubernetes.default.svc.cluster.local] and IPs [10.96.0.1 10.0.0.215]
[certs] Generating "apiserver-kubelet-client" certificate and key
[certs] Generating "front-proxy-ca" certificate and key
[certs] Generating "front-proxy-client" certificate and key
[certs] Generating "etcd/ca" certificate and key
[certs] Generating "etcd/server" certificate and key
[certs] etcd/server serving cert is signed for DNS names [k8s-master localhost] and IPs [10.0.0.215 127.0.0.1 ::1]
[certs] Generating "etcd/peer" certificate and key
[certs] etcd/peer serving cert is signed for DNS names [k8s-master localhost] and IPs [10.0.0.215 127.0.0.1 ::1]
[certs] Generating "etcd/healthcheck-client" certificate and key
[certs] Generating "apiserver-etcd-client" certificate and key
[certs] Generating "sa" key and public key
[kubeconfig] Using kubeconfig folder "/etc/kubernetes"
[kubeconfig] Writing "admin.conf" kubeconfig file
[kubeconfig] Writing "kubelet.conf" kubeconfig file
[kubeconfig] Writing "controller-manager.conf" kubeconfig file
[kubeconfig] Writing "scheduler.conf" kubeconfig file
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Starting the kubelet
[control-plane] Using manifest folder "/etc/kubernetes/manifests"
[control-plane] Creating static Pod manifest for "kube-apiserver"
[control-plane] Creating static Pod manifest for "kube-controller-manager"
[control-plane] Creating static Pod manifest for "kube-scheduler"
[etcd] Creating static Pod manifest for local etcd in "/etc/kubernetes/manifests"
[wait-control-plane] Waiting for the kubelet to boot up the control plane as static Pods from directory "/etc/kubernetes/manifests". This can take up to 4m0s
[apiclient] All control plane components are healthy after 10.504413 seconds
[upload-config] Storing the configuration used in ConfigMap "kubeadm-config" in the "kube-system" Namespace
[kubelet] Creating a ConfigMap "kubelet-config-1.23" in namespace kube-system with the configuration for the kubelets in the cluster
NOTE: The "kubelet-config-1.23" naming of the kubelet ConfigMap is deprecated. Once the UnversionedKubeletConfigMap feature gate graduates to Beta the default name will become just "kubelet-config". Kubeadm upgrade will handle this transition transparently.
[upload-certs] Skipping phase. Please see --upload-certs
[mark-control-plane] Marking the node k8s-master as control-plane by adding the labels: [node-role.kubernetes.io/master(deprecated) node-role.kubernetes.io/control-plane node.kubernetes.io/exclude-from-external-load-balancers]
[mark-control-plane] Marking the node k8s-master as control-plane by adding the taints [node-role.kubernetes.io/master:NoSchedule]
[bootstrap-token] Using token: vftfc4.awgkdg93fs4q57k5
[bootstrap-token] Configuring bootstrap tokens, cluster-info ConfigMap, RBAC Roles
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to get nodes
[bootstrap-token] configured RBAC rules to allow Node Bootstrap tokens to post CSRs in order for nodes to get long term certificate credentials
[bootstrap-token] configured RBAC rules to allow the csrapprover controller automatically approve CSRs from a Node Bootstrap Token
[bootstrap-token] configured RBAC rules to allow certificate rotation for all node client certificates in the cluster
[bootstrap-token] Creating the "cluster-info" ConfigMap in the "kube-public" namespace
[kubelet-finalize] Updating "/etc/kubernetes/kubelet.conf" to point to a rotatable kubelet client certificate and key
[addons] Applied essential addon: CoreDNS
[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

Alternatively, if you are the root user, you can run:

  export KUBECONFIG=/etc/kubernetes/admin.conf

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 10.0.0.215:6443 --token vftfc4.awgkdg93fs4q57k5 \
        --discovery-token-ca-cert-hash sha256:7d3cebfeb5ef628609d999ee043f88e8f7846516c72e573ab70fc4615a79d91a

创建配置文件

[root@k8s-master01 opt]#  mkdir -p $HOME/.kube
[root@k8s-master01 opt]#  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@k8s-master01 opt]#  sudo chown $(id -u):$(id -g) $HOME/.kube/config

查看cs是否正常

[root@k8s-master01 opt]# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE                         ERROR
controller-manager   Healthy   ok                              
scheduler            Healthy   ok                              
etcd-0               Healthy   {"health":"true","reason":""}

如果不正常需要屏蔽端口

[root@k8s-master opt]# vim /etc/kubernetes/manifests/kube-scheduler.yaml
##屏蔽随机端口
			   # - --port=0
## 保存后，等一会就正常了

查看node节点是否正常

[root@k8s-master01 opt]# kubectl get nodes
NAME         STATUS   ROLES                  AGE     VERSION
k8s-master   NoReady    control-plane,master   6h25m   v1.23.6
[root@k8s-master01 opt]#

发现不正常NoReady，是网络的问题需要安装flannel

[root@k8s-master opt]# kubectl apply -f https://raw.githubusercontent.com/coreos/flannel/master/Documentation/kube-flannel.yml

在过一会查看

[root@k8s-master01 opt]# kubectl get nodes
NAME         STATUS   ROLES                  AGE     VERSION
k8s-master   Ready    control-plane,master   6h25m   v1.23.6
[root@k8s-master01 opt]#

添加k8s-node01，k8s-node02节点到master节点中（上述中kubeadm init 命令执行成功后生成的最后一条为加入从节点命令）

k8s-node01节点

[root@k8s-node01 opt]#  kubeadm join 10.0.0.215:6443 --token vftfc4.awgkdg93fs4q57k5 \
        --discovery-token-ca-cert-hash sha256:7d3cebfeb5ef628609d999ee043f88e8f7846516c72e573ab70fc4615a79d91a 
## 远程拷贝k8s-master01节点的配置文件到本地
[root@k8s-node01 opt]#  scp k8s-master01:/etc/kubernetes/admin.conf /etc/kubernetes/
[root@k8s-node01 opt]#  echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> ~/.bash_profile
[root@k8s-node01 opt]#  source ~/.bash_profile

k8s-node02节点

[root@k8s-node02 opt]# kubeadm join 10.0.0.215:6443 --token vftfc4.awgkdg93fs4q57k5 \
        --discovery-token-ca-cert-hash sha256:7d3cebfeb5ef628609d999ee043f88e8f7846516c72e573ab70fc4615a79d91a 
[root@k8s-node01 opt]#  scp k8s-master01:/etc/kubernetes/admin.conf /etc/kubernetes/
[root@k8s-node01 opt]#  echo "export KUBECONFIG=/etc/kubernetes/admin.conf" >> ~/.bash_profile
[root@k8s-node01 opt]#  source ~/.bash_profile

k8s-master01节点
查看所有节点状态

[root@k8s-master01 opt]# kubectl get nodes
NAME           STATUS   ROLES                  AGE     VERSION
k8s-master01   Ready    control-plane,master   4h41m   v1.23.6
k8s-node01     Ready    <none>                 4h33m   v1.23.6
k8s-node02     Ready    <none>                 4h33m   v1.23.6

此时搭建集群完毕！！！！！！！！！！！！！！！！！

搭建dashboard

[root@k8s-master01 opt]#  wget https://raw.githubusercontent.com/kubernetes/dashboard/v2.5.1/aio/deploy/recommended.yaml
[root@k8s-master01 opt]# vim recommended.yaml
### 需要修改添加NodePort
kind: Service
apiVersion: v1
metadata:
  labels:
    k8s-app: kubernetes-dashboard
  name: kubernetes-dashboard
  namespace: kubernetes-dashboard
spec:
  ports:
    - port: 443
      targetPort: 8443
      nodePort: 30924   ## 添加此行
  type: NodePort        ## 添加此行
  selector:
    k8s-app: kubernetes-dashboard

[root@k8s-master01 opt]#  kubectl apply -f recommended.yaml
### 生成永久秘钥
[root@k8s-master opt]# kubeadm token create --ttl 0 --print-join-command 
kubeadm join 10.0.0.215:6443 --token dtgkok.08suwpwrwnko8d3a --discovery-token-ca-cert-hash sha256:7d3cebfeb5ef628609d999ee043f88e8f7846516c72e573ab70fc4615a79d91a 
[root@k8s-master opt]# kubectl create serviceaccount dashboard-admin -n kube-system
serviceaccount/dashboard-admin created
[root@k8s-master opt]# kubectl create clusterrolebinding dashboard-admin --clusterrole=cluster-admin --serviceaccount=kube-system:dashboard-admin
clusterrolebinding.rbac.authorization.k8s.io/dashboard-admin created
[root@k8s-master opt]# kubectl -n kube-system get secret | grep dashboard-admin
dashboard-admin-token-nc82q                      kubernetes.io/service-account-token   3      18s
[root@k8s-master opt]# kubectl describe secrets -n kube-system $(kubectl -n kube-system get secret | awk '/dashboard-admin/{print $1}')
Name:         dashboard-admin-token-nc82q
Namespace:    kube-system
Labels:       <none>
Annotations:  kubernetes.io/service-account.name: dashboard-admin
              kubernetes.io/service-account.uid: 4f559e22-5153-4562-b296-b823d91b146e

Type:  kubernetes.io/service-account-token

Data
====
ca.crt:     1099 bytes
namespace:  11 bytes
token:      eyJhbGciOiJSUzI1NiIsImtpZCI6ImpwQ3AybEQxWHMwbmlDc1hmUVo5MVl4SVBSVGl1SjU3U3hJRWVJUk5HV3MifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJrdWJlLXN5c3RlbSIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VjcmV0Lm5hbWUiOiJkYXNoYm9hcmQtYWRtaW4tdG9rZW4tbmM4MnEiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGFzaGJvYXJkLWFkbWluIiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZXJ2aWNlLWFjY291bnQudWlkIjoiNGY1NTllMjItNTE1My00NTYyLWIyOTYtYjgyM2Q5MWIxNDZlIiwic3ViIjoic3lzdGVtOnNlcnZpY2VhY2NvdW50Omt1YmUtc3lzdGVtOmRhc2hib2FyZC1hZG1pbiJ9.FiXPo_YrUew0cMewhBZ8vHfLFKOOsG1zTY-2ZsYcLR1THeI8PHqwVxyzSp0BQ5VDfAzAJGtmt8Gvfs5w9RZdDOBQYO8VYWe_xDzXjpk8GTHe76BEw8sWUps9Qk0zF5z22Mc_D2bwx4FWZR7D_wRjpLgfd8JPQEN2CZH9KU_jkWvdP_wlC_t0kVFyzf1hOhpDjyuqaCmJVVwXmOShGsChyKkLg5wHuBbnt-aVLea6ngz6VCeXNoe-Qv2x8fHhCtovW17cdjbSvjpVK7L99DyErd4NypxaHYOcXJvVqFetUWm-7ihYrs3txfYsWcBNaKPXqgEXeW3-0eP6fHTy9Fw-Zg