Kubernetes: k8s-安装篇-二进制-1.37

root@jasper-server:~# cat /etc/os-release
PRETTY_NAME="Ubuntu 26.04.1 LTS"
NAME="Ubuntu"
VERSION_ID="26.04"
VERSION="26.04.1 LTS (Resolute Raccoon)"
VERSION_CODENAME=resolute
ID=ubuntu
ID_LIKE=debian
HOME_URL="https://www.ubuntu.com/"
SUPPORT_URL="https://help.ubuntu.com/"
BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/"
PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy"
UBUNTU_CODENAME=resolute
LOGO=ubuntu-logo

MASTER_IPS="10.103.236.201 10.103.236.202 10.103.236.203"
NODE_IPS="10.103.236.204 10.103.236.205"

K8S_VERSION=v1.37.0
ETCD_VERSION=v3.7.0

CONTAINERD_VERSION=2.3.5
#CRI_DOCKER_VERSION=0.4.2

POD_CIDR=172.16.0.0/12
SERVICE_CIDR=192.168.0.0/16
#POD_CIDR=10.244.0.0/16
#SERVICE_CIDR=10.96.0.0/16

IFACE=enp2s0
#IFACE=eth0

ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa
for i in master01.jasper.org master02.jasper.org master03.jasper.org node01.jasper.org node02.jasper.org;do
    ssh-copy-id -i ~/.ssh/id_rsa.pub $i;
done

#或
for i in $MASTER_IPS  $NODE_IPS;do
    ssh-copy-id -i ~/.ssh/id_rsa.pub $i;
done

# 1. 设置主机名（各自节点执行对应命令）
# Master 节点
hostnamectl set-hostname master01.jasper.org && exit

# Node 节点
hostnamectl set-hostname node01.jasper.org && exit

# 2. 配置  /etc/hosts，添加所有节点的 IP 与主机名解析
cat >> /etc/hosts << EOF
10.103.236.201 master01.jasper.org master01
10.103.236.202 master02.jasper.org master02
10.103.236.203 master03.jasper.org master03
10.103.236.236 kubeapi.jasper.org kubeapi # VIP 虚IP不占用机器资源 # 如果不是高可用集群，该 IP 为 Master01 的 IP
10.103.236.204 node01.jasper.org node01
10.103.236.205 node02.jasper.org node02
EOF

# CentOS 7 安装yum源如下(作废)
curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-7.repo
yum install -y yum-utils device-mapper-persistent-data lvm2

yum-config-manager --add-repo https://mirrors.aliyun.com/docker-ce/linux/centos/docker-ce.repo
sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo

# 必备工具安装
yum install wget jq psmisc vim net-tools telnet yum-utils device-mapper-persistent-data lvm2 git -y

swapoff -a && sysctl -w vm.swappiness=0
sed -ri '/^[^#]*swap/s@^@#@' /etc/fstab

# k8s 要求禁用 swap 以确保调试和资源管理的准确性。开启 swap
# 时 kubelet 默认拒绝启动， k8s-1.22 以后版本可以
# kubectl --fail-swap-on=false 允许开启 swap

# 安装
apt update
apt install  -y chrony
systemctl enable --now  chrony
systemctl status chrony

#配置
cat > /etc/chrony/sources.d/ubuntu-ntp-pools.sources << EOF
server ntp1.aliyun.com iburst minpoll 4 maxpoll 10
server cn.pool.ntp.org iburst minpoll 4 maxpoll 10
server s1b.time.edu.cn iburst minpoll 4 maxpoll 10
EOF

# 把原来 makestep 1.0 3 修改，去掉次数限制，偏差>1s就允许跳
cp /etc/chrony/chrony.conf{,.bak}
sed -i 's/^makestep .*/makestep 1 -1/g' /etc/chrony/chrony.conf

systemctl restart chrony.service
timedatectl set-timezone Asia/Shanghai # 修改时区
chronyc -a makestep # 强制同步下系统时钟
timedatectl set-ntp yes # 启用NTP服务
timedatectl
# 查看时间同步源：
chronyc sources -v
# 查看时间同步源状态：
chronyc sourcestats -v
# 查看同步偏移量，看System time     : 偏移值，越小越好
chronyc tracking

apt update
apt install systemd-timesyncd
# 开机自启
systemctl enable --now systemd-timesyncd

timedatectl status

# ubuntu 默认安装的轻量时间同步 systemd-timesyncd 可以支持时间同步，但不如 chronyd 精度高
systemctl status systemd-timesyncd

# 优化为国内时间同步配置
cp /etc/systemd/timesyncd.conf{,.bak}
cat > /etc/systemd/timesyncd.conf << EOF
[Time]
NTP=ntp.aliyun.com ntp.tencent.com ntp.ntsc.ac.cn
FallbackNTP=cn.pool.ntp.org 0.cn.pool.ntp.org 1.cn.pool.ntp.org
RootDistanceMaxSec=5
PollIntervalMinSec=32
PollIntervalMaxSec=2048
ConnectionRetrySec=30
SaveIntervalSec=60
EOF
systemctl restart systemd-timesyncd
timedatectl timesync-status

# 安装
rpm -ivh http://mirrors.wlnmp.com/centos/wlnmp-release-centos.noarch.rpm
yum install ntpdate -y

# 所有节点同步时间。时间同步配置如下
ln -sf /usr/share/zoneinfo/Asia/Shanghai /etc/localtime
echo 'Asia/Shanghai' >/etc/timezone
ntpdate time2.aliyun.com

# 加入到crontab
*/5 * * * * /usr/sbin/ntpdate time2.aliyun.com &>/dev/null

# Ubuntu 默认使用 ufw，不是 firewalld
ufw disable
ufw status

# CentOS
systemctl disable --now firewalld 
systemctl disable --now dnsmasq
systemctl disable --now NetworkManager

setenforce 0
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/sysconfig/selinux
sed -i 's#SELINUX=enforcing#SELINUX=disabled#g' /etc/selinux/config

ulimit -SHn 65535

cat >/etc/security/limits.d/k8s.conf<<EOF
* soft nofile 1000000
* hard nofile 1000000
* soft nproc 1000000
* hard nproc 1000000
* soft memlock unlimited
* hard memlock unlimited

root soft nofile 1000000
root hard nofile 1000000
root soft nproc unlimited
root hard nproc unlimited
root soft memlock unlimited
root hard memlock unlimited
EOF

# 1.安装 nftalbes
apt update
apt install -y nftables
# 启用nftables服务（kube‑proxy本身不需要依赖这个服务开机加载自定义规则，只是提供nft工具）
systemctl enable --now nftables
# 验证
nft --version

#kube‑proxy 会自动维护 table ip kube-proxy，不要在 /etc/nftables.conf 里操作这张表，否则冲突。

# 2.加载必要内核模块
modinfo  br_netfilter
modprobe br_netfilter
lsmod |grep br_netfilter

cat>/etc/modules-load.d/k8s.conf <<'EOF'
br_netfilter
overlay
EOF
#生效
systemctl enable --now systemd-modules-load.service

lsmod | grep -E "br_netfilter|overlay|nf_tables"

# 3. 优化内核参数

cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.file-max=10485760
fs.nr_open=10485760
EOF
# 使用内核配置生效
sysctl --system

# 说明
# net.ipv4.ip_forward = 1                  # 启用 IP 转发，这是容器网络通信的基础
# net.bridge.bridge-nf-call-iptables = 1   # 让网桥上的流量经过 iptables规则，用于 service 和网络策略
# net.bridge.bridge-nf-call-ip6tables = 1  # 让网桥上的流量经过 iptables规则，用于 service 和网络策略

# 1.安装 IPVS 模式所依赖的基础工具，否则 IPVS 模式下的 kube-proxy.service 服务无法启动
apt update && apt install -y ipvsadm ipset # (sysstat conntrack)
#注释:
#ipvsadm: 用于 IPVS 负载均衡模式(kube-proxy的一种模式)。
#ipset,conntrack: 网络过滤和连接跟踪工具，为Service网络提供支持。

# 2.加载必要内核模块
cat>/etc/modules-load.d/k8s.conf <<'EOF'
br_netfilter
overlay
EOF

#生效
systemctl enable --now systemd-modules-load.service

# 3. 优化内核参数

cat <<EOF > /etc/sysctl.d/k8s.conf
net.ipv4.ip_forward = 1
net.bridge.bridge-nf-call-iptables = 1
net.bridge.bridge-nf-call-ip6tables = 1
fs.file-max=10485760
fs.nr_open=10485760
EOF
# 说明
# net.ipv4.ip_forward = 1                  # 启用 IP 转发，这是容器网络通信的基础
# net.bridge.bridge-nf-call-iptables = 1   # 让网桥上的流量经过 iptables规则，用于 service 和网络策略
# net.bridge.bridge-nf-call-ip6tables = 1  # 让网桥上的流量经过 iptables规则，用于 service 和网络策略

# 使用内核配置生效
sysctl --system

RUNC_VERSION=1.5.0
ARCH=$([ `arch`  = "aarch64" ] && echo arm64 || echo amd64)
wget https://github.com/opencontainers/runc/releases/download/v${RUNC_VERSION}/runc.${ARCH}
install -m 755 runc.${ARCH} /usr/local/bin/runc
runc --version


# 检查 containerd 配置，确认 runc 路径
cp /etc/containerd/config.toml{,.bak}
sed -i 's@BinaryName =.*@BinaryName = "/usr/local/bin/runc"@g' /etc/containerd/config.toml

systemctl restart containerd
systemctl status containerd

VERSION="v1.37.0"
ARCH=$([ `arch`  = "aarch64" ] && echo arm64 || echo amd64)

wget https://github.com/kubernetes-sigs/cri-tools/releases/download/${VERSION}/crictl-${VERSION}-linux-$ARCH.tar.gz

sudo tar zxvf crictl-${VERSION}-linux-$ARCH.tar.gz -C /usr/local/bin
#rm -f crictl-${VERSION}-linux-$ARCH.tar.gz

# 配置crictl客户端连接
cat > /etc/crictl.yaml <<EOF
runtime-endpoint: unix:///run/containerd/containerd.sock
image-endpoint: unix:///run/containerd/containerd.sock
timeout: 10
debug: false
EOF
# 验证
#crictl info

K8S_VERSION=v1.37.0
ARCH=$([ `arch`  = "aarch64" ] && echo arm64 || echo amd64)

wget https://dl.k8s.io/${K8S_VERSION}/kubernetes-server-linux-${ARCH}.tar.gz

# 解压文件到 /usr/local/bin 下。
# kubectl,kubelet, kube-apiserver,
# kube-controller-manager,kube-scheduler,kube-proxy
tar tf kubernetes-server-linux-${ARCH}.tar.gz
tar -xvf kubernetes-server-linux-${ARCH}.tar.gz \
    --strip-components=3 \
    -C /usr/local/bin \
    kubernetes/server/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy}

# kubectl 自动补全
apt update && apt install -y bash-completion
echo 'alias k=kubectl' >>~/.bashrc
echo 'source <(kubectl completion bash)' >>~/.bashrc && source ~/.bashrc

MasterNodes='master02.jasper.org master03.jasper.org'

for NODE in $MasterNodes; do
    echo $NODE;
    scp /usr/local/bin/kube{let,ctl,-apiserver,-controller-manager,-scheduler,-proxy} $NODE:/usr/local/bin/;
    #scp /usr/local/bin/etcd* $NODE:/usr/local/bin/;
done

kubelet --version
etcdctl version

WorkNodes='node01.jasper.org node02.jasper.org'

for NODE in $WorkNodes; do
    scp /usr/local/bin/kube{let,-proxy} $NODE:/usr/local/bin/ ;
done

openssl genrsa -out /etc/kubernetes/pki/sa.key 2048
openssl rsa -in /etc/kubernetes/pki/sa.key -pubout -out /etc/kubernetes/pki/sa.pub

# 查看最后生成所有文件如下

tree /etc/kubernetes/pki
/etc/kubernetes/pki
├── admin-key.pem
├── admin.csr
├── admin.pem
├── apiserver-key.pem
├── apiserver.csr
├── apiserver.pem
├── ca-key.pem
├── ca.csr
├── ca.pem
├── controller-manager-key.pem
├── controller-manager.csr
├── controller-manager.pem
├── etcd
│   ├── etcd-ca-key.pem
│   ├── etcd-ca.csr
│   ├── etcd-ca.pem
│   ├── etcd-key.pem
│   ├── etcd.csr
│   └── etcd.pem
├── front-proxy-ca-key.pem
├── front-proxy-ca.csr
├── front-proxy-ca.pem
├── front-proxy-client-key.pem
├── front-proxy-client.csr
├── front-proxy-client.pem
├── kube-proxy-key.pem
├── kube-proxy.csr
├── kube-proxy.pem
├── sa.key
├── sa.pub
├── scheduler-key.pem
├── scheduler.csr
└── scheduler.pem
2 directories, 32 files

# 注释:此文件将用于 kube-controller-manager 组件连接 API Server 时的身份认证。

# 注意，如果不是高可用集群，10.103.236.236:8443 改为
# master01 的地址，8443 改为 apiserver 的端口，默认是 6443


#创建配置文件，指定使用的 CA 证书和 api 地址
# set-cluster：设置一个集群项
kubectl config set-cluster kubernetes \
     --certificate-authority=/etc/kubernetes/pki/ca.pem \
     --embed-certs=true \
     --server=https://kubeapi.jasper.org:8443 \
     --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
# set-credentials 设置身份验证凭据
kubectl config set-credentials system:kube-controller-manager \
     --client-certificate=/etc/kubernetes/pki/controller-manager.pem \
     --client-key=/etc/kubernetes/pki/controller-manager-key.pem \
     --embed-certs=true \
     --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
# 为配置文件追加一个上下文，上下文关联了特定的集群和用户
kubectl config set-context system:kube-controller-manager@kubernetes \
    --cluster=kubernetes \
    --user=system:kube-controller-manager \
    --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig
# 使用某个环境当做默认环境
kubectl config use-context system:kube-controller-manager@kubernetes \
     --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig


ls /etc/kubernetes/
controller-manager.kubeconfig  pki

# 查看 kubeconfig 内容
root@master01:/data/k8s# kubectl config view --kubeconfig /etc/kubernetes/controller-manager.kubeconfig
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: DATA+OMITTED
    server: https://kubeapi.jasper.org:8443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: system:kube-controller-manager
  name: system:kube-controller-manager@kubernetes
current-context: system:kube-controller-manager@kubernetes
kind: Config
users:
- name: system:kube-controller-manager
  user:
    client-certificate-data: DATA+OMITTED
    client-key-data: DATA+OMITTED

# 注意，如果不是高可用集群，10.103.236.236:8443 改为
# master01 的地址，8443 改为 apiserver 的端口，默认是 6443

#创建配置文件，指定使用的 CA 证书和 api 地址
kubectl config set-cluster kubernetes \
     --certificate-authority=/etc/kubernetes/pki/ca.pem \
     --embed-certs=true \
     --server=https://kubeapi.jasper.org:8443 \
     --kubeconfig=/etc/kubernetes/scheduler.kubeconfig
# set-credentials 设置身份验证凭据
kubectl config set-credentials system:kube-scheduler \
     --client-certificate=/etc/kubernetes/pki/scheduler.pem \
     --client-key=/etc/kubernetes/pki/scheduler-key.pem \
     --embed-certs=true \
     --kubeconfig=/etc/kubernetes/scheduler.kubeconfig
# 为配置文件追加一个上下文，上下文关联了特定的集群和用户
kubectl config set-context system:kube-scheduler@kubernetes \
     --cluster=kubernetes \
     --user=system:kube-scheduler \
     --kubeconfig=/etc/kubernetes/scheduler.kubeconfig
# 使用某个环境当做默认环境
kubectl config use-context system:kube-scheduler@kubernetes \
     --kubeconfig=/etc/kubernetes/scheduler.kubeconfig


ls /etc/kubernetes/
controller-manager.kubeconfig  pki  scheduler.kubeconfig

# 查看 kubeconfig 内容
root@master01:/data/k8s# kubectl config view --kubeconfig /etc/kubernetes/scheduler.kubeconfig
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: DATA+OMITTED
    server: https://kubeapi.jasper.org:8443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: system:kube-scheduler
  name: system:kube-scheduler@kubernetes
current-context: system:kube-scheduler@kubernetes
kind: Config
users:
- name: system:kube-scheduler
  user:
    client-certificate-data: DATA+OMITTED
    client-key-data: DATA+OMITTED

# 注意，如果不是高可用集群，10.103.236.236:8443 改为
# master01 的地址，8443 改为 apiserver 的端口，默认是 6443

#创建配置文件，指定使用的 CA 证书和 api 地址
kubectl config set-cluster kubernetes \
        --certificate-authority=/etc/kubernetes/pki/ca.pem \
        --embed-certs=true \
        --server=https://kubeapi.jasper.org:8443 \
        --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
# set-credentials 设置身份验证凭据
kubectl config set-credentials kube-proxy \
        --client-certificate=/etc/kubernetes/pki/kube-proxy.pem  \
        --client-key=/etc/kubernetes/pki/kube-proxy-key.pem \
        --embed-certs=true \
        --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
# 为配置文件追加一个上下文，上下文关联了特定的集群和用户
kubectl config set-context kube-proxy@kubernetes \
        --cluster=kubernetes \
        --user=kube-proxy \
        --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig
# 使用某个环境当做默认环境
kubectl config use-context kube-proxy@kubernetes \
        --kubeconfig=/etc/kubernetes/kube-proxy.kubeconfig


ls /etc/kubernetes/
controller-manager.kubeconfig  kube-proxy.kubeconfig  pki  scheduler.kubeconfig

# 查看 kubeconfig 内容
root@master01:/data/k8s# kubectl config view --kubeconfig /etc/kubernetes/kube-proxy.kubeconfig
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: DATA+OMITTED
    server: https://kubeapi.jasper.org:8443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: kube-proxy
  name: kube-proxy@kubernetes
current-context: kube-proxy@kubernetes
kind: Config
users:
- name: kube-proxy
  user:
    client-certificate-data: DATA+OMITTED
    client-key-data: DATA+OMITTED

# 注意，如果不是高可用集群，10.103.236.236:8443改为master01的地址，8443改为apiserver的端口，默认是6443

BOOTSTRAP_TOKEN=$(head -c 6 /dev/urandom | md5sum | head -c 6).$(head -c 16 /dev/urandom | md5sum | head -c 16)

# 或者
#TOKEN_PUB=$(openssl rand -hex 3)
#TOKEN_SECRET=$(openssl rand -hex 8)
#BOOTSTRAP_TOKEN="${TOKEN_PUB}.${TOKEN_SECRET}"

# 保存此 TOKEN 后面会用到
echo $BOOTSTRAP_TOKEN
5ba17d.e9d5d01308883dab


BOOTSTRAP_TOKEN=5ba17d.e9d5d01308883dab

kubectl config set-cluster kubernetes \
        --certificate-authority=/etc/kubernetes/pki/ca.pem \
        --embed-certs=true \
        --server=https://kubeapi.jasper.org:8443 \
        --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig

kubectl config set-credentials tls-bootstrap-token-user \
        --token=${BOOTSTRAP_TOKEN} \
        --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig

kubectl config set-context tls-bootstrap-token-user@kubernetes \
        --cluster=kubernetes \
        --user=tls-bootstrap-token-user \
        --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig

kubectl config use-context tls-bootstrap-token-user@kubernetes \
        --kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig


ls /etc/kubernetes/
bootstrap-kubelet.kubeconfig  controller-manager.kubeconfig  kube-proxy.kubeconfig  pki  scheduler.kubeconfig

# 查看 kubeconfig 内容
root@master01:/data/k8s# kubectl config view --kubeconfig /etc/kubernetes/bootstrap-kubelet.kubeconfig
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: DATA+OMITTED
    server: https://kubeapi.jasper.org:8443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: tls-bootstrap-token-user
  name: tls-bootstrap-token-user@kubernetes
current-context: tls-bootstrap-token-user@kubernetes
kind: Config
users:
- name: tls-bootstrap-token-user
  user:
    token: REDACTED

# 注意，如果不是高可用集群，10.103.236.236:8443 改为
# master01 的地址，8443 改为 apiserver 的端口，默认是 6443

#创建配置文件，指定使用的 CA 证书和 api 地址
kubectl config set-cluster kubernetes \
        --certificate-authority=/etc/kubernetes/pki/ca.pem \
        --embed-certs=true \
        --server=https://kubeapi.jasper.org:8443 \
        --kubeconfig=/etc/kubernetes/admin.kubeconfig
# set-credentials 设置身份验证凭据
kubectl config set-credentials kubernetes-admin \
        --client-certificate=/etc/kubernetes/pki/admin.pem  \
        --client-key=/etc/kubernetes/pki/admin-key.pem \
        --embed-certs=true \
        --kubeconfig=/etc/kubernetes/admin.kubeconfig
# 为配置文件追加一个上下文，上下文关联了特定的集群和用户
kubectl config set-context kubernetes-admin@kubernetes \
        --cluster=kubernetes \
        --user=kubernetes-admin \
        --kubeconfig=/etc/kubernetes/admin.kubeconfig
# 使用某个环境当做默认环境
kubectl config use-context kubernetes-admin@kubernetes \
        --kubeconfig=/etc/kubernetes/admin.kubeconfig


ls /etc/kubernetes/
admin.kubeconfig  bootstrap-kubelet.kubeconfig  controller-manager.kubeconfig  kube-proxy.kubeconfig  pki  scheduler.kubeconfig

# 查看 kubeconfig 内容
root@master01:/data/k8s# kubectl config view --kubeconfig /etc/kubernetes/admin.kubeconfig
apiVersion: v1
clusters:
- cluster:
    certificate-authority-data: DATA+OMITTED
    server: https://kubeapi.jasper.org:8443
  name: kubernetes
contexts:
- context:
    cluster: kubernetes
    user: kubernetes-admin
  name: kubernetes-admin@kubernetes
current-context: kubernetes-admin@kubernetes
kind: Config
users:
- name: kubernetes-admin
  user:
    client-certificate-data: DATA+OMITTED
    client-key-data: DATA+OMITTED

mkdir -p /root/.kube
cp /etc/kubernetes/admin.kubeconfig  /root/.kube/config

# 37 个文件
root@master01:/data/k8s# tree /etc/kubernetes/
/etc/kubernetes/
├── admin.kubeconfig
├── bootstrap-kubelet.kubeconfig
├── controller-manager.kubeconfig
├── kube-proxy.kubeconfig
├── pki
│   ├── admin-key.pem
│   ├── admin.csr
│   ├── admin.pem
│   ├── apiserver-key.pem
│   ├── apiserver.csr
│   ├── apiserver.pem
│   ├── ca-key.pem
│   ├── ca.csr
│   ├── ca.pem
│   ├── controller-manager-key.pem
│   ├── controller-manager.csr
│   ├── controller-manager.pem
│   ├── etcd
│   │   ├── etcd-ca-key.pem
│   │   ├── etcd-ca.csr
│   │   ├── etcd-ca.pem
│   │   ├── etcd-key.pem
│   │   ├── etcd.csr
│   │   └── etcd.pem
│   ├── front-proxy-ca-key.pem
│   ├── front-proxy-ca.csr
│   ├── front-proxy-ca.pem
│   ├── front-proxy-client-key.pem
│   ├── front-proxy-client.csr
│   ├── front-proxy-client.pem
│   ├── kube-proxy-key.pem
│   ├── kube-proxy.csr
│   ├── kube-proxy.pem
│   ├── sa.key
│   ├── sa.pub
│   ├── scheduler-key.pem
│   ├── scheduler.csr
│   └── scheduler.pem
└── scheduler.kubeconfig

3 directories, 37 files

ETCD_VERSION=v3.7.1
ARCH=$([ `arch`  = "aarch64" ] && echo arm64 || echo amd64)

cd /data/k8s
wget https://github.com/etcd-io/etcd/releases/download/${ETCD_VERSION}/etcd-${ETCD_VERSION}-linux-${ARCH}.tar.gz

tar tf etcd-${ETCD_VERSION}-linux-${ARCH}.tar.gz
tar xvf etcd-${ETCD_VERSION}-linux-${ARCH}.tar.gz \
    --strip-components=1 \
    -C /usr/local/bin \
    etcd-${ETCD_VERSION}-linux-${ARCH}/etcd{,ctl}


etcdctl version
etcdctl version: 3.7.1
API version: 3.7

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1
}
vrrp_instance VI_1 {
    state MASTER
    interface enp2s0
    mcast_src_ip 10.103.236.201
    virtual_router_id 51
    priority 101
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        10.103.236.236
    }
    track_script {
      chk_apiserver 
} }
EOF

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5 
    weight -5
    fall 2
    rise 1
 
}
vrrp_instance VI_1 {
    state BACKUP
    interface enp2s0
    mcast_src_ip 10.103.236.202
    virtual_router_id 51
    priority 100
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        10.103.236.236
    }
    track_script {
      chk_apiserver 
} }
EOF

cat > /etc/keepalived/keepalived.conf << EOF
! Configuration File for keepalived
global_defs {
    router_id LVS_DEVEL
}
vrrp_script chk_apiserver {
    script "/etc/keepalived/check_apiserver.sh"
    interval 5
    weight -5
    fall 2  
    rise 1
}
vrrp_instance VI_1 {
    state BACKUP
    interface enp2s0
    mcast_src_ip 10.103.236.203
    virtual_router_id 51
    priority 100
    nopreempt
    advert_int 2
    authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {
        10.103.236.236
    }
    track_script {
      chk_apiserver 
} }
EOF

cat > /etc/keepalived/check_apiserver.sh <<\EOF
#!/bin/bash

err=0
for k in $(seq 1 3)
do
    check_code=$(pgrep haproxy)
    if [[ $check_code == "" ]]; then
        err=$(expr $err + 1)
        sleep 1
        continue
    else
        err=0
        break
    fi
done

if [[ $err != "0" ]]; then
    echo "systemctl stop keepalived"
    /usr/bin/systemctl stop keepalived
    exit 1
else
    exit 0
fi
EOF

chmod +x /etc/keepalived/check_apiserver.sh

systemctl daemon-reload
systemctl enable --now haproxy
systemctl enable --now keepalived
systemctl restart haproxy keepalived

[root@master01.jasper.org pki]# ping 10.103.236.236
PING 10.103.236.236 (10.103.236.236) 56(84) bytes of data.
64 bytes from 10.103.236.236: icmp_seq=1 ttl=64 time=1.39 ms
64 bytes from 10.103.236.236: icmp_seq=2 ttl=64 time=2.46 ms
64 bytes from 10.103.236.236: icmp_seq=3 ttl=64 time=1.68 ms
64 bytes from 10.103.236.236: icmp_seq=4 ttl=64 time=1.08 ms

nc -zv 10.103.236.236 8443
Connection to 10.103.236.236 8443 port [tcp/*] succeeded!

mkdir -p /etc/kubernetes/manifests/ \
      /etc/systemd/system/kubelet.service.d \
      /var/lib/kubelet \
      /var/log/kubernetes

LOCAL_IP=`hostname -I|awk '{print $1}'`
ETCD1_IP=10.103.236.201
ETCD2_IP=10.103.236.202
ETCD3_IP=10.103.236.203


cat > /lib/systemd/system/kube-apiserver.service << EOF
[Unit]
Description=Kubernetes API Server
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-apiserver \\
      --v=2  \\
      --allow-privileged=true  \\
      --bind-address=0.0.0.0  \\
      --secure-port=6443  \\
      --advertise-address=${LOCAL_IP} \\
      --service-cluster-ip-range=192.168.0.0/16  \\
      --service-node-port-range=30000-32767  \\
      --etcd-servers=https://${ETCD1_IP}:2379,https://${ETCD2_IP}:2379,https://${ETCD3_IP}:2379 \\
      --etcd-cafile=/etc/kubernetes/pki/etcd/etcd-ca.pem  \\
      --etcd-certfile=/etc/kubernetes/pki/etcd/etcd.pem  \\
      --etcd-keyfile=/etc/kubernetes/pki/etcd/etcd-key.pem  \\
      --client-ca-file=/etc/kubernetes/pki/ca.pem  \\
      --tls-cert-file=/etc/kubernetes/pki/apiserver.pem  \\
      --tls-private-key-file=/etc/kubernetes/pki/apiserver-key.pem  \\
      --kubelet-client-certificate=/etc/kubernetes/pki/apiserver.pem  \\
      --kubelet-client-key=/etc/kubernetes/pki/apiserver-key.pem  \\
      --service-account-key-file=/etc/kubernetes/pki/sa.pub  \\
      --service-account-signing-key-file=/etc/kubernetes/pki/sa.key  \\
      --service-account-issuer=https://kubernetes.default.svc.cluster.local \\
      --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname  \\
      --enable-admission-plugins=NamespaceLifecycle,LimitRanger,ServiceAccount,DefaultStorageClass,DefaultTolerationSeconds,NodeRestriction,ResourceQuota  \\
      --authorization-mode=Node,RBAC  \\
      --enable-bootstrap-token-auth=true  \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem  \\
      --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.pem  \\
      --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client-key.pem  \\
      --requestheader-allowed-names=aggregator  \\
      --requestheader-group-headers=X-Remote-Group  \\
      --requestheader-extra-headers-prefix=X-Remote-Extra-  \\
      --requestheader-username-headers=X-Remote-User \\
      --enable-aggregator-routing=true
      # --token-auth-file=/etc/kubernetes/token.csv

Restart=on-failure
RestartSec=10s
LimitNOFILE=65535

[Install]
WantedBy=multi-user.target
EOF


# 启动 apiserver
systemctl daemon-reload && systemctl enable --now kube-apiserver.service && systemctl status kube-apiserver

# 查看组件状态
root@master01:/data/k8s# kubectl  get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS      MESSAGE                                                                                        ERROR
scheduler            Unhealthy   Get "https://127.0.0.1:10259/healthz": dial tcp 127.0.0.1:10259: connect: connection refused
controller-manager   Unhealthy   Get "https://127.0.0.1:10257/healthz": dial tcp 127.0.0.1:10257: connect: connection refused
etcd-0               Healthy     ok

# 创建 kube-controller-manager service
cat > /usr/lib/systemd/system/kube-controller-manager.service << EOF
[Unit]
Description=Kubernetes Controller Manager
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-controller-manager \\
      --v=2 \\
      --bind-address=0.0.0.0 \\
      --root-ca-file=/etc/kubernetes/pki/ca.pem \\
      --cluster-signing-cert-file=/etc/kubernetes/pki/ca.pem \\
      --cluster-signing-key-file=/etc/kubernetes/pki/ca-key.pem \\
      --service-account-private-key-file=/etc/kubernetes/pki/sa.key \\
      --kubeconfig=/etc/kubernetes/controller-manager.kubeconfig \\
      --leader-elect=true \\
      --use-service-account-credentials=true \\
      --node-monitor-grace-period=40s \\
      --node-monitor-period=5s \\
      --controllers=*,bootstrapsigner,tokencleaner \\
      --allocate-node-cidrs=true \\
      --service-cluster-ip-range=192.168.0.0/16 \\
      --cluster-cidr=172.16.0.0/12 \\
      --node-cidr-mask-size=24 \\
      --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.pem

      
Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target
EOF

# 启动kube-controller-manager
systemctl daemon-reload && systemctl enable --now kube-controller-manager.service
systemctl status kube-controller-manager

# 查看启动状态 默认端口 10257
root@master01:/data/k8s# kubectl get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS      MESSAGE                                                                                        ERROR
scheduler            Unhealthy   Get "https://127.0.0.1:10259/healthz": dial tcp 127.0.0.1:10259: connect: connection refused
etcd-0               Healthy     ok
controller-manager   Healthy     ok

# 创建启动 kube-scheduler service
cat > /usr/lib/systemd/system/kube-scheduler.service << EOF
[Unit]
Description=Kubernetes Scheduler
Documentation=https://github.com/kubernetes/kubernetes
After=network.target

[Service]
ExecStart=/usr/local/bin/kube-scheduler \\
      --v=2 \\
      --bind-address=0.0.0.0 \\
      --leader-elect=true \\
      --kubeconfig=/etc/kubernetes/scheduler.kubeconfig

Restart=always
RestartSec=10s

[Install]
WantedBy=multi-user.target
EOF

# 启动服务
systemctl daemon-reload && systemctl enable --now kube-scheduler.service
systemctl status kube-scheduler.service

# 查看组件状态。默认端口 10259
root@master01:/data/k8s# kubectl  get cs
Warning: v1 ComponentStatus is deprecated in v1.19+
NAME                 STATUS    MESSAGE   ERROR
controller-manager   Healthy   ok
scheduler            Healthy   ok
etcd-0               Healthy   ok

cd /data/k8s

BOOTSTRAP_TOKEN=5ba17d.e9d5d01308883dab
TOKEN_ID=$(echo $BOOTSTRAP_TOKEN |cut -d. -f1)
TOKEN_SECRET=$(echo $BOOTSTRAP_TOKEN |cut -d. -f2)

cat > bootstrap.secret.yaml << EOF
apiVersion: v1
kind: Secret
metadata:
  name: bootstrap-token-$TOKEN_ID   # 使用前面的的变量 BOOTSTRAP_TOKEN=5ba17d.e9d5d01308883dab
  namespace: kube-system
type: bootstrap.kubernetes.io/token
stringData:
  description: "The default bootstrap token generated by 'kubelet '."
  token-id: $TOKEN_ID                # Token ID，使用前面的变量 BOOTSTRAP_TOKEN=5ba17d.e9d5d01308883dab
  token-secret: $TOKEN_SECRET        # Token secret，使用前面的变量 BOOTSTRAP_TOKEN=5ba17d.e9d5d01308883dab
  usage-bootstrap-authentication: "true"
  usage-bootstrap-signing: "true"
  auth-extra-groups:  system:bootstrappers:default-node-token,system:bootstrappers:worker,system:bootstrappers:ingress
 
---
# 授权 bootstrap token 创建证书签名请求(CSR)
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: kubelet-bootstrap
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:node-bootstrapper
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:bootstrappers:default-node-token
---
# 自动批准 kubelet 的客户端证书 CSR
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: node-autoapprove-bootstrap
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:nodeclient
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:bootstrappers:default-node-token
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: node-autoapprove-certificate-rotation
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:certificates.k8s.io:certificatesigningrequests:selfnodeclient
subjects:
- apiGroup: rbac.authorization.k8s.io
  kind: Group
  name: system:nodes
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
  labels:
    kubernetes.io/bootstrapping: rbac-defaults
  name: system:kube-apiserver-to-kubelet
rules:
  - apiGroups:
      - ""
    resources:
      - nodes/proxy
      - nodes/stats
      - nodes/log
      - nodes/spec
      - nodes/metrics
    verbs:
      - "*"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
  name: system:kube-apiserver
  namespace: ""
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: system:kube-apiserver-to-kubelet
subjects:
  - apiGroup: rbac.authorization.k8s.io
    kind: User
    name: kube-apiserver
EOF


kubectl apply -f bootstrap.secret.yaml

kubectl apply -f bootstrap.secret.yaml
secret/bootstrap-token-5ba17d created
clusterrolebinding.rbac.authorization.k8s.io/kubelet-bootstrap created
clusterrolebinding.rbac.authorization.k8s.io/node-autoapprove-bootstrap created
clusterrolebinding.rbac.authorization.k8s.io/node-autoapprove-certificate-rotation created
clusterrole.rbac.authorization.k8s.io/system:kube-apiserver-to-kubelet created
clusterrolebinding.rbac.authorization.k8s.io/system:kube-apiserver created

# 查看 secret
root@master01:/data/k8s# kubectl  get secret -n kube-system |grep bootstrap-token
bootstrap-token-5ba17d   bootstrap.kubernetes.io/token   6      4m37s

kubectl  get secret,clusterroles,clusterrolebindings -n kube-system |grep -E "bootstrap|node-auto|apiserver"

root@master01:/data/k8s# kubectl  get node
No resources found

# 确认在 work 节点 node01 上的 kubelet 和 kube-proxy 二进制文件已安装
root@node01:~# ls /usr/local/bin/kube*
/usr/local/bin/kube-proxy  /usr/local/bin/kubelet

mkdir -p /etc/kubernetes/manifests/ \
      /etc/systemd/system/kubelet.service.d \
      /var/lib/kubelet \
      /var/log/kubernetes

COREDNS_SERVICE_IP=`kubectl get svc | grep kubernetes | awk '{print $3}'`0
K8S_VERSION=v1.37.0

curl -O https://raw.githubusercontent.com/kubernetes/kubernetes/refs/tags/${K8S_VERSION}/cluster/addons/dns/coredns/coredns.yaml.base

# 修改配置文件中的相关配置
sed -e 's/__DNS__SERVER__/'"${COREDNS_SERVICE_IP}"'/g' \
    -e 's#image: registry.k8s.io/coredns#image: registry.aliyuncs.com/google_containers#g' \
    -e 's/__DNS__DOMAIN__/cluster.local/g' \
    -e 's/__DNS__MEMORY__LIMIT__/1024Mi/g' \
    coredns.yaml.base > coredns.yaml


root@master01:/data/k8s# kubectl  apply -f coredns.yaml
serviceaccount/coredns created
clusterrole.rbac.authorization.k8s.io/system:coredns created
clusterrolebinding.rbac.authorization.k8s.io/system:coredns created
configmap/coredns created
deployment.apps/coredns created
service/kube-dns created

root@master01:/data/k8s# kubectl  -n kube-system get pod
NAME                                      READY   STATUS             RESTARTS      AGE
coredns-544cdd5dd8-4495b                  0/1     CrashLoopBackOff   4 (44s ago)   2m16s

# 查看原因：出现 CoreDNS 发生了 DNS 递归死循环(DNS)
root@master01:/data/k8s# kubectl  -n kube-system logs -f coredns-544cdd5dd8-4495b
maxprocs: Leaving GOMAXPROCS=2: CPU quota undefined
.:53
[INFO] plugin/reload: Running configuration SHA512 = 591cf328cccc12bc490481273e738df59329c62c0b729d94e8b61db9961c2fa5f046dd37f1cf888b953814040d180f52594972691cd6ff41be96639138a43908
CoreDNS-1.14.6
linux/arm64, go1.26.5, 424d125
# 出现死循环。收到 dns 请求一直转给自己
[FATAL] plugin/loop: Loop (127.0.0.1:35907 -> :53) detected for zone ".", see https://coredns.io/plugins/loop#troubleshooting. Query: "HINFO 4322075668335749943.3134280878331088951."

# 解决方法：转给互联网上真空的 DNS
kubectl -n  kube-system edit cm coredns
....
        #forward . /etc/resolv.conf {
        forward . 223.6.6.6 180.76.76.76 {
            max_concurrent 1000
        }

# 重启 coredns
kubectl  rollout restart deploy -n kube-system coredn

curl -LO https://github.com/kubernetes-sigs/metrics-server/releases/latest/download/components.yaml

# 关闭 tls 检验，更换国内镜像。
vim components.yaml
    spec:
      containers:
      - args:
        - --cert-dir=/tmp
        - --secure-port=10250
        - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
        - --kubelet-use-node-status-port
        - --metric-resolution=15s
        - --kubelet-insecure-tls # HTTPS 加密，跳过证书校验；鉴权（token）仍然保留
        image: registry.cn-hangzhou.aliyuncs.com/google_containers/metrics-server:v0.9.0
        imagePullPolicy: IfNotPresent


kubectl  apply -f components.yaml

root@master01:/data/k8s# kubectl  get pod -n kube-system
NAME                                      READY   STATUS    RESTARTS   AGE
calico-kube-controllers-db57f7644-frdhp   1/1     Running   0          144m
calico-node-vvhzj                         1/1     Running   0          96m
calico-node-ww8d9                         1/1     Running   0          96m
coredns-54d449487f-gv78z                  1/1     Running   0          18m
metrics-server-74ccf797cb-4zm25           1/1     Running   0          47s

# kubectl  top node
NAME           CPU(cores)   CPU%   MEMORY(bytes)   MEMORY%   
master01.jasper.org   231m         5%     1620Mi          42%
master02.jasper.org   274m         6%     1203Mi          31%
master03.jasper.org   202m         5%     1251Mi          32%
node01.jasper.org     69m          1%     667Mi           17%
node02.jasper.org     73m          1%     650Mi           16%

MASTER_IPS="10.103.236.201 10.103.236.202 10.103.236.203"
NODE_IPS="10.103.236.206"

ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa

for i in $NODE_IPS;do
    ssh-copy-id -i ~/.ssh/id_rsa.pub $i;
done

echo 10.103.236.206 node03.jasper.org node03 >> /etc/hosts
scp /etc/hosts node03:/etc/hosts

# 设置主机名，在新的 Node 节点执行。
LOCAL_IP=`hostname -I|awk '{print $1}'`
HOST_NAME=$(awk -v ip=$LOCAL_IP '{if($1==ip && $2 !~ "kubeapi")print $2}' /etc/hosts)
hostnamectl set-hostname $HOST_NAME || { color "主机名配置失败，检查 /etc/hosts 文件" 1 ; exit 1; }

# 环境初始化和安装容器运行时的所有操作
略

# 拷贝 node01 上的文件
NODE_IPS="node03.jasper.org"

NODES=($NODE_IPS)
FILES=(
    "/etc/hosts"
    "/usr/local/bin/runc"
    "/etc/containerd/config.toml"
    "/usr/local/bin/kubelet"
    "/usr/local/bin/kube-proxy"
    "/lib/systemd/system/kubelet.service"
    "/lib/systemd/system/kube-proxy.service"
    "/etc/kubernetes"
    "/opt/cni"
)


for NODE in "${NODES[@]}"; do
    echo ====$NODE START====
    for FILE in "${FILES[@]}"; do
        scp -r "$FILE" "root@${NODE}:$FILE"
    done
    echo ====$NODE END====
done

systemctl daemon-reload && systemctl enable --now kubelet.service kube-proxy.service
systemctl status kubelet.service
systemctl status kube-proxy

# node 节点
root@node03:~# tree /etc/kubernetes/
/etc/kubernetes/
├── bootstrap-kubelet.kubeconfig
├── kube-proxy.kubeconfig
├── kube-proxy.yaml
├── kubelet-conf.yml
├── kubelet.kubeconfig
├── manifests
└── pki
    └── ca.pem

# kubectl 查看新节点
root@master01:/data/k8s# kubectl  get node
NAME                STATUS   ROLES    AGE     VERSION
node01.jasper.org   Ready    worker   23h     v1.37.0
node02.jasper.org   Ready    <none>   23h     v1.37.0
node03.jasper.org   Ready    <none>   5m49s   v1.37.0

# 验证 pod 运行
root@master01:/data/k8s# kubectl  get deploy
NAME    READY   UP-TO-DATE   AVAILABLE   AGE
nginx   1/1     1            1           16h

root@master01:/data/k8s# kubectl  scale deploy nginx --replicas 6

root@master01:/data/k8s# kubectl  get pod -owide
NAME                     READY   STATUS    RESTARTS   AGE   IP               NODE                NOMINATED NODE   READINESS GATES
nginx-75d67d6d8c-bj94f   1/1     Running   0          16s   172.20.7.142     node01.jasper.org   <none>           <none>
nginx-75d67d6d8c-ftjq2   1/1     Running   0          16s   172.18.37.1      node03.jasper.org   <none>           <none>
nginx-75d67d6d8c-knvq2   1/1     Running   0          16h   172.30.118.134   node02.jasper.org   <none>           <none>
nginx-75d67d6d8c-snh5z   1/1     Running   0          16s   172.20.7.143     node01.jasper.org   <none>           <none>
nginx-75d67d6d8c-x6wk8   1/1     Running   0          16s   172.18.37.2      node03.jasper.org   <none>           <none>
nginx-75d67d6d8c-xx6vt   1/1     Running   0          16s   172.30.118.135   node02.jasper.org   <none>           <none>

# 因为没有将 Master 节点加入集群，所以 Master 节点和 Pod 网络不能，造成无法查看性能信息
root@master01:/data/k8s# kubectl top node
error: Metrics API not available
root@master01:/data/k8s# kubectl top pod
error: Metrics API not available

MASTER_IPS="10.103.236.202 10.103.236.203"
NODES=(${MASTER_IPS})

# master01 可以免密登录所有 master 节点
#ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa
for i in ${NODES[@]};do
    ssh-copy-id -i ~/.ssh/id_rsa.pub $i;
done

# 复制 Master01 节点相关必要文件到所有 master 节点，执行下面命令。
FILES=(
    "/etc/kubernetes/kubelet-conf.yml"
    "/lib/systemd/system/kubelet.service"
    "/etc/kubernetes/kube-proxy.yaml"
    "/lib/systemd/system/kube-proxy.service"
    "/etc/kubernetes/kube-proxy.kubeconfig"
    "/etc/kubernetes/bootstrap-kubelet.kubeconfig"
    #"/opt/cni"
)

for NODE in "${NODES[@]}"; do
    echo ====$NODE START====
    for FILE in "${FILES[@]}"; do
        scp -r "$FILE" "root@${NODE}:$FILE"
    done
    scp -r /opt/cni root@${NODE}:/opt
    #ssh root@${NODE} "systemctl daemon-reload && systemctl enable --now kube-proxy kubelet"
    echo ====$NODE END====
done

for NODE in "${NODES[@]}"; do
    echo ====$NODE START====
    ssh root@${NODE} "systemctl daemon-reload && systemctl enable --now kube-proxy kubelet"
    echo ====$NODE END====
done

# 再次查看集群节点成员
root@master01:~# kubectl  get node
NAME                  STATUS   ROLES           AGE    VERSION
master01.jasper.org   Ready    control-plane   89m    v1.37.0
master02.jasper.org   Ready    <none>          2m9s   v1.37.0
master03.jasper.org   Ready    <none>          118s   v1.37.0
node01.jasper.org     Ready    worker          26h    v1.37.0
node02.jasper.org     Ready    worker          26h    v1.37.0
node03.jasper.org     Ready    worker          149m   v1.37.0

# 给节点添加 ROLES
kubectl label nodes master01.jasper.org node-role.kubernetes.io/control-plane=
kubectl label nodes node01.jasper.org node-role.kubernetes.io/worker=
kubectl label nodes node02.jasper.org node-role.kubernetes.io/worker=
kubectl label nodes node03.jasper.org node-role.kubernetes.io/worker=

root@master01:~# kubectl  get node
NAME                  STATUS   ROLES           AGE    VERSION
master01.jasper.org   Ready    control-plane   89m    v1.37.0
master02.jasper.org   Ready    <none>          2m9s   v1.37.0
master03.jasper.org   Ready    <none>          118s   v1.37.0
node01.jasper.org     Ready    worker          26h    v1.37.0
node02.jasper.org     Ready    worker          26h    v1.37.0
node03.jasper.org     Ready    worker          149m   v1.37.0


# 只看 node 节点
alias kgn='kubectl get node -l "!node-role.kubernetes.io/control-plane"'

root@master01:~# kgn
NAME                STATUS   ROLES    AGE   VERSION
node01.jasper.org   Ready    worker   2d    v1.37.0
node02.jasper.org   Ready    worker   2d    v1.37.0
node03.jasper.org   Ready    worker   24h   v1.37.0

# 重新生成 Pod，可以看到调试到 master01 节点
kubectl scale deployment nginx --replicas 0
kubectl scale deployment nginx --replicas 3

root@master01:/data/k8s# kubectl  get pod -owide
NAME                     READY   STATUS    RESTARTS   AGE   IP             NODE                  NOMINATED NODE   READINESS GATES
nginx-75d67d6d8c-4mnfv   1/1     Running   0          43s   172.20.7.144   node01.jasper.org     <none>           <none>
nginx-75d67d6d8c-x4r8v   1/1     Running   0          43s   172.18.37.3    node03.jasper.org     <none>           <none>
nginx-75d67d6d8c-zsld5   1/1     Running   0          43s   172.30.9.193   master01.jasper.org   <none>           <none>

# 本机可以通过 service 访问了
root@master01:/data/k8s# kubectl  get svc
NAME         TYPE        CLUSTER-IP       EXTERNAL-IP   PORT(S)   AGE
kubernetes   ClusterIP   192.168.0.1      <none>        443/TCP   27h
nginx        ClusterIP   192.168.192.68   <none>        80/TCP    17h
root@master01:/data/k8s# curl -I 192.168.192.68
HTTP/1.1 200 OK

# 1. 先看现状
kubectl get node -o custom-columns='NAME:.metadata.name,TAINTS:.spec.taints'
# 2. 补齐 master 的角色标签
kubectl label node master01.jasper.org master02.jasper.org master03.jasper.org \
    node-role.kubernetes.io/control-plane=
# 3. 打污点
kubectl taint node master01.jasper.org master02.jasper.org master03.jasper.org \
  node-role.kubernetes.io/control-plane=:NoSchedule

# 4. 清掉已经在 master 上的存量 Pod
## NoSchedule 只拦新调度的 Pod，已经跑在上面的不动。
## 别用 NoExecute 去做这件事——它会连带影响那些只容忍 NoSchedule 的系统组件。
kubectl drain master01.jasper.org --ignore-daemonsets --delete-emptydir-data
kubectl uncordon master02.jasper.org   # 只解除 unschedulable，污点保留

# 5. 让污点持久化（二进制部署容易踩的坑）
## kubectl taint 改的是 apiserver 里的 Node 对象。
## 如果哪天你删了 Node 对象让 kubelet 重新注册
## （重装、改 hostname、清 etcd 数据等），污点就没了。
## 建议同时写进 kubelet 启动参数：
## systemd unit 里加：
--register-with-taints=node-role.kubernetes.io/control-plane=:NoSchedule

## 或者 KubeletConfiguration yaml 里：
registerWithTaints:
  - key: node-role.kubernetes.io/control-plane
    effect: NoSchedule

# 方法1 yaml 文件
kubectl apply -f https://raw.githubusercontent.com/kubernetes-sigs/headlamp/main/kubernetes-headlamp.yaml

# 方法2 helm
helm repo add headlamp https://kubernetes-sigs.github.io/headlamp/
helm search repo -l  headlamp # 查看版本
helm pull headlamp/headlamp --version 0.45.0

tar xf headlamp-0.45.0.tgz
cd headlamp/

cat > my-values.yaml << EOF
service:
  type: NodePort
  nodePort: 30080

resources:
  requests: { cpu: 100m, memory: 128Mi }
  limits:   { cpu: 500m, memory: 512Mi }
EOF

helm install my-headlamp  . \
  -f my-values.yaml \
  --create-namespace -n kube-system 

root@master01:~/hdp/headlamp# kubectl  -n kube-system get svc  my-headlamp
NAME          TYPE       CLUSTER-IP       EXTERNAL-IP   PORT(S)        AGE
my-headlamp   NodePort   192.168.249.40   <none>        80:30080/TCP   2m51s

cat > admin.yaml <<EOF
apiVersion: v1
kind: ServiceAccount
metadata:
  name: admin-user
  namespace: kube-system
---
apiVersion: v1
kind: Secret
metadata:
  name: admin-user
  namespace: kube-system
  annotations:
    kubernetes.io/service-account.name: "admin-user"   
type: kubernetes.io/service-account-token  
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding 
metadata: 
  name: admin-user
  annotations:
    rbac.authorization.kubernetes.io/autoupdate: "true"
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-admin
subjects:
- kind: ServiceAccount
  name: admin-user
  namespace: kube-system
EOF

# 执行
kubectl apply -f admin.yaml

cat > /etc/docker/daemon.json <<EOF
{
 "registry-mirrors": [
    "https://registry.docker-cn.com",
    "http://hub-mirror.c.163.com",
    "https://docker.mirrors.ustc.edu.cn"
  ],
 "exec-opts": ["native.cgroupdriver=systemd"],
 "max-concurrent-downloads": 10,
 "max-concurrent-uploads": 5,
 "log-opts": {
   "max-size": "300m",
   "max-file": "2"
 },
 "live-restore": true
}
EOF

# 所有节点改完重启docker
systemctl daemon-reload && systemctl restart docker

# 参数说明
#max-concurrent-downloads # 下载并发数
#max-concurrent-uploads   # 上传并发数
#max-size                 # 日志文件最大到多少切割 （此处是300m）
#max-file                 # 日志文件保留个数 （此处是2个）
#live-restore             # 设置为 true，docker 重启是不会影响上面的容器的

# controller-manager 颁发的证书有效期是 1 年，指定证书过期时间

vim /usr/lib/systemd/system/kube-controller-manager.service
# --feature-gates=RotateKubeletClientCertificate=true,RotateKubeletServerCertificate=true \ # 新版是 true
--cluster-signing-duration=876000h0m0s \

# 改完重启
systemctl daemon-reload && systemctl restart kube-controller-manager

#vim /etc/systemd/system/kubelet.service.d/10-kubelet.conf
[Service]

Environment="KUBELET_KUBECONFIG_ARGS=--kubeconfig=/etc/kubernetes/kubelet.kubeconfig --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.kubeconfig"

Environment="KUBELET_SYSTEM_ARGS=--network-plugin=cni --cni-conf-dir=/etc/cni/net.d --cni-bin-dir=/opt/cni/bin"

Environment="KUBELET_CONFIG_ARGS=--config=/etc/kubernetes/kubelet-conf.yml  --pod-infra-container-image=registry.cn-hangzhou.aliyuncs.com/google_containers/pause:3.6"

Environment="KUBELET_EXTRA_ARGS=--tls-cipher-suites=TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384    --image-pull-progress-deadline=30m"

ExecStart=

ExecStart=/usr/local/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_CONFIG_ARGS $KUBELET_SYSTEM_ARGS $KUBELET_EXTRA_ARGS

vim /etc/kubernetes/kubelet-conf.yml

rotateServerCertificates: true
allowedUnsafeSysctls:
 - "net.core*"
 - "net.ipv4.*"
kubeReserved:  
  cpu: "1"
  memory: 1Gi
  ephemeral-storage: 10Gi
systemReserved:
  cpu: "1"
  memory: 1Gi
  ephemeral-storage: 10Gi
  
# 改完重启
systemctl daemon-reload && systemctl restart kubelet  

allowedUnsafeSysctl：根据安全，判定是否允许修改内核
kubeReserved：k8s 组件预留资源
systemReserved：系统预留资源

测试环境：
rotateServerCertificates: true
allowedUnsafeSysctls:
 - "net.core*"
 - "net.ipv4.*"
kubeReserved:
  cpu: "10m"
  memory: 10Mi
  ephemeral-storage: 10Mi
systemReserved:
  cpu: "1"
  memory: 20Mi
  ephemeral-storage: 1Gi