---

准备机器

所有机器配置至少2核2G
192.168.1.22 kuber-master01
192.168.1.21 kuber-node01
192.168.1.24 kuber-node02

机器初始化工作（所有机器都执行）

1、更改国内源

curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
wget -O /etc/yum.repos.d/epel.repo http://mirrors.aliyun.com/repo/epel-7.repo

2、全部机器上做域名解析

vim /etc/hosts
192.168.1.22 kuber-master01 
192.168.1.21 kuber-node01 
192.168.1.24 kuber-node02

3、关闭防火墙及selinux

systemctl disable --now firewalld
sed -ri 's/SELINUX=enforcing/SELINUX=disabled/' /etc/selinux/config

4、更新系统内rpm软件包(除内核外)

yum -y update --exclude=kernel*

5、安装必要的依赖软件包

yum -y install yum-utils device-mapper-persistent-data lvm2 wget jq psmisc vim net-tools telnet git ntpdate
  ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
  echo "Asia/Shanghai" >/etc/timezone
  echo '*/5 * * * *    ntpdate -b ntp.aliyun.com' >>/var/spool/cron/${USER}

7、配置系统docker安装源

yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
  sed -i 's+download.docker.com+mirrors.ustc.edu.cn/docker-ce+' /etc/yum.repos.d/docker-ce.repo

8、关闭swap功能

swapoff -a && sysctl -w vm.swappiness=0
sed -i '/swap/d' /etc/fstab

9、设置最大文件打开数

cat <<-EOF >>/etc/security/limits.conf
* soft nofile 655360
* hard nofile 131072
* soft nproc 655350
* hard nproc 655350
* soft memlock unlimited
* hard memlock unlimited
EOF

10、安装ipvs并生成内核配置

yum -y install ipvsadm ipset sysstat conntrack libseccomp
if [ $? -eq 0 ];then
cat <<-EOF >>/etc/modules-load.d/ipvs.conf
ip_vs
ip_vs_lc
ip_vs_wlc
ip_vs_rr
ip_vs_wrr
ip_vs_lblc
ip_vs_lblcr
ip_vs_dh
ip_vs_sh
ip_vs_fo
ip_vs_nq
ip_vs_sed
ip_vs_ftp
ip_vs_sh
nf_conntrack
ip_tables
ip_set
xt_set
ipt_set
ipt_rpfilter
ipt_REJECT
ipip
EOF
fi

11、k8s内核配置项

cat <<-EOF >>/etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.may_detach_mounts = 1
vm.overcommit_memory = 1
vm.panic_on_oom = 0
fs.inotify.max_user_watches = 89100
fs.file-max = 52706963
fs.nr_open = 52706963
net.netfilter.nf_conntrack_max = 2310720
net.ipv4.tcp_keepalive_time = 600
net.ipv4.tcp_keepalive_probes = 3
net.ipv4.tcp_keepalive_intvl = 15
net.ipv4.tcp_max_tw_buckets = 36000
net.ipv4.tcp_tw_reuse = 1
net.ipv4.tcp_max_orphans = 327680
net.ipv4.tcp_orphan_retries = 3
net.ipv4.tcp_syncookies = 1
net.ipv4.tcp_max_syn_backlog = 16384
net.ipv4.ip_conntrack_max = 65536
net.ipv4.tcp_timestamps = 0
net.core.somaxconn = 16384
EOF

12、安装docker引擎

yum -y install docker-ce-19.03.15-3.el7 docker-ce-cli-19.03.15-3.el7

systemctl enable docker && systemctl start docker && docker version
cat <<-EOF >/etc/docker/daemon.json
{
  "registry-mirrors": ["https://registry.docker-cn.com"],
  "exec-opts": ["native.cgroupdriver=systemd"],
  "insecure-registries": ["harbor.hiops.icu"]
}
EOF
systemctl daemon-reload && systemctl restart docker

13、更新内核

#>>> 载入公钥
rpm --import https://www.elrepo.org/RPM-GPG-KEY-elrepo.org
#>>> 升级安装ELRepo
rpm -Uvh http://www.elrepo.org/elrepo-release-7.0-3.el7.elrepo.noarch.rpm
#>>> 载入elrepo-kernel元数据
yum --disablerepo=\* --enablerepo=elrepo-kernel repolist
#>>> 安装最新版本的kernel
yum --disablerepo=\* --enablerepo=elrepo-kernel install  kernel-ml.x86_64  -y
#>>> 删除旧版本工具包
yum remove kernel-tools-libs.x86_64 kernel-tools.x86_64  -y
#>>> 安装新版本工具包
yum --disablerepo=\* --enablerepo=elrepo-kernel install kernel-ml-tools.x86_64  -y
#>>> 设置默认启动
grub2-set-default 0 && grub2-mkconfig -o /etc/grub2.cfg
grubby --args="user_namespace.enable=1" --update-kernel="$(grubby --default-kernel)"

14、重新启动系统

reboot

在kuber-master01操作：

在kuber-master01 机器上做免密登录node节点机器

ssh-keygen -t rsa -N "" -f ~/.ssh/id_rsa

同步其他机器

for role in kuber-master02 kuber-node01 kuber-node02;do ssh-copy-id ${role};done

批量传输文件至其他机器

for role in kuber-master02 kuber-node01 kuber-node02;do scp init.sh root@${role}:/root;done

添加kubernetes YUM软件源

cat > /etc/yum.repos.d/kubernetes.repo << EOF
[kubernetes]
name=Kubernetes
baseurl=https://mirrors.aliyun.com/kubernetes/yum/repos/kubernetes-el7-x86_64
enabled=1
gpgcheck=0
repo_gpgcheck=0
gpgkey=https://mirrors.aliyun.com/kubernetes/yum/doc/yum-key.gpg https://mirrors.aliyun.com/kubernetes/yum/doc/rpm-package-key.gpg
EOF

安装kubeadm,kubelet和kubectl

所有主机都需要操作，由于版本更新频繁，这里指定版本号部署

 yum install -y kubelet-1.15.0 kubeadm-1.15.0 kubectl-1.15.0
 systemctl enable kubelet

部署Kubernetes Master

只需要在Master 节点执行，这里的apiserve需要修改成自己的master地址

kubeadm init \
--apiserver-advertise-address=192.168.1.22 \
--image-repository registry.aliyuncs.com/google_containers \
--kubernetes-version v1.15.0 \
--service-cidr=10.1.0.0/16 \
--pod-network-cidr=10.244.0.0/16

之后提示如下，则表示成功初始化

[addons] Applied essential addon: kube-proxy

Your Kubernetes control-plane has initialized successfully!

To start using your cluster, you need to run the following as a regular user:

  mkdir -p $HOME/.kube
  sudo cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
  sudo chown $(id -u):$(id -g) $HOME/.kube/config

You should now deploy a pod network to the cluster.
Run "kubectl apply -f [podnetwork].yaml" with one of the options listed at:
  https://kubernetes.io/docs/concepts/cluster-administration/addons/

Then you can join any number of worker nodes by running the following on each as root:

kubeadm join 192.168.1.22:6443 --token akgucj.ihold4ci10opz52z \
    --discovery-token-ca-cert-hash sha256:40cbcaf21f10ffc9b131865aa7808456a215ba26a395d202256c6b0185284690
You have new mail in /var/spool/mail/root

根据提示做如下操作：

[root@kuber-master01 ~]# mkdir -p $HOME/.kube
You have new mail in /var/spool/mail/root
[root@kuber-master01 ~]# cp -i /etc/kubernetes/admin.conf $HOME/.kube/config
[root@kuber-master01 ~]# chown $(id -u):$(id -g) $HOME/.kube/config

在node节点操作

加入集群：

kubeadm join 192.168.1.22:6443 --token akgucj.ihold4ci10opz52z --discovery-token-ca-cert-hash sha256:40cbcaf21f10ffc9b131865aa7808456a215ba26a395d202256c6b0185284690 --skip-preflight-chec

输出如下内容表述成功加入：

[preflight] Running pre-flight checks
    [WARNING IsDockerSystemdCheck]: detected "cgroupfs" as the Docker cgroup driver. The recommended driver is "systemd". Please follow the guide at https://kubernetes.io/docs/setup/cri/
[preflight] Reading configuration from the cluster...
[preflight] FYI: You can look at this config file with 'kubectl -n kube-system get cm kubeadm-config -oyaml'
[kubelet-start] Downloading configuration for the kubelet from the "kubelet-config-1.15" ConfigMap in the kube-system namespace
[kubelet-start] Writing kubelet configuration to file "/var/lib/kubelet/config.yaml"
[kubelet-start] Writing kubelet environment file with flags to file "/var/lib/kubelet/kubeadm-flags.env"
[kubelet-start] Activating the kubelet service
[kubelet-start] Waiting for the kubelet to perform the TLS Bootstrap...

This node has joined the cluster:
* Certificate signing request was sent to apiserver and a response was received.
* The Kubelet was informed of the new secure connection details.

Run 'kubectl get nodes' on the control-plane to see this node join the cluster.

安装网络组件，实现master和node节点网络互通（master节点操作）

wget https://raw.githubusercontent.com/coreos/flannel/a70459be0084506e4ec919aa1c114638878db11b/Documentation/kube-flannel.yml

若无法wget，在/etc/hosts添加如下解析

199.232.68.133 raw.githubusercontent.com

成功wget后，执行如下命令：

kubectl apply -f kube-flannel.yml

查看flannel进程是否启动成功，输出如下结果，表示成功启动

[root@kuber-master01 ~]# ps -ef | grep flannel
root      90313  90296  0 Apr06 ?        00:01:13 /opt/bin/flanneld --ip-masq --kube-subnet-mgr

这时查看集群各节点状态

[root@kuber-master01 ~]# kubectl get node
NAME             STATUS   ROLES    AGE   VERSION
kuber-master01   Ready    master   23h   v1.15.0
kuber-node01     Ready    <none>   23h   v1.15.0
kuber-node02     Ready    <none>   23h   v1.15.0

只有全部都是running状态才算成功

[root@kuber-master01 ~]# kubectl get pod -n kube-system
NAME                                     READY   STATUS    RESTARTS   AGE
coredns-bccdc95cf-h4pt9                  1/1     Running   0          23h
coredns-bccdc95cf-v6glf                  1/1     Running   0          23h
etcd-kuber-master01                      1/1     Running   0          23h
kube-apiserver-kuber-master01            1/1     Running   0          23h
kube-controller-manager-kuber-master01   1/1     Running   0          23h
kube-flannel-ds-amd64-jmqlt              1/1     Running   0          22h
kube-flannel-ds-amd64-kfvst              1/1     Running   0          22h
kube-flannel-ds-amd64-l2dlw              1/1     Running   0          22h
kube-proxy-9lqv7                         1/1     Running   0          23h
kube-proxy-f8m92                         1/1     Running   0          23h
kube-proxy-q5jv9                         1/1     Running   0          23h
kube-scheduler-kuber-master01            1/1     Running   0          23h

搭建遇到错误：

之前node节点加入集群不成功，重新初始化报错：

[root@kuber-master01 .ssh]# kubeadm init --apiserver-advertise-address=192.168.43.165 --image-repository registry.aliyuncs.com/google_containers --kubernetes-version v1.15.0 --service-cidr=10.1.0.0/16
[init] Using Kubernetes version: v1.15.0
[preflight] Running pre-flight checks
        [WARNING SystemVerification]: this Docker version is not on the list of validated versions: 19.03.15. Latest validated version: 18.09
error execution phase preflight: [preflight] Some fatal errors occurred:
        [ERROR Port-6443]: Port 6443 is in use
        [ERROR Port-10251]: Port 10251 is in use
        [ERROR Port-10252]: Port 10252 is in use
        [ERROR FileAvailable--etc-kubernetes-manifests-kube-apiserver.yaml]: /etc/kubernetes/manifests/kube-apiserver.yaml already exists
        [ERROR FileAvailable--etc-kubernetes-manifests-kube-controller-manager.yaml]: /etc/kubernetes/manifests/kube-controller-manager.yaml already exists
        [ERROR FileAvailable--etc-kubernetes-manifests-kube-scheduler.yaml]: /etc/kubernetes/manifests/kube-scheduler.yaml already exists
        [ERROR FileAvailable--etc-kubernetes-manifests-etcd.yaml]: /etc/kubernetes/manifests/etcd.yaml already exists
        [ERROR Port-10250]: Port 10250 is in use
        [ERROR DirAvailable--var-lib-etcd]: /var/lib/etcd is not empty
[preflight] If you know what you are doing, you can make a check non-fatal with `--ignore-preflight-errors=...`

解决方法：
重置，执行如下命令：

kubeadm reset

重新初始化：

kubeadm init --apiserver-advertise-address=10.2.158.57 --image-repository registry.aliyuncs.com/google_containers --kubernetes-version v1.15.0 --service-cidr=10.1.0.0/16 --pod-network-cidr=10.244.0.0/16

kubectl命令自动补全

    yum install -y bash-completion
    source /usr/share/bash-completion/bash_completion
    source <(kubectl completion bash)
    echo "source <(kubectl completion bash)" >> ~/.bashrc