新k8s部署方案

Ethereal Lv4
  2024-09-27 22:55:06 2024-09-27 22:55 Created   2024-11-23 00:44:10 2024-11-23 00:44 Updated  

1. 配置ip

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
nmtui # 配置某个静态ip作为k8s连接地址
# 重启应用配置 nmcli connection up eno1
readonly cluster_float_ip="192.168.29.100"
vim /etc/hosts
添加自己定义的静态ip
# 127.0.0.1 localhost
# 127.0.1.1 Ethereal-Desktop
# 192.168.29.100 cluster.svc

# # The following lines are desirable for IPv6 capable hosts
# ::1     ip6-localhost ip6-loopback
# fe00::0 ip6-localnet
# ff00::0 ip6-mcastprefix
# ff02::1 ip6-allnodes
# ff02::2 ip6-allrouters

2. 初始化配置

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
swapoff -a # 暂时关闭swap
# sed -i 's/^SELINUX.*/SELINUX=disabled/g' /etc/selinux/config 关闭selinux
# 启用模块
echo "bridge" |  tee /etc/modules-load.d/bridge.conf
echo "br_netfilter" |  tee /etc/modules-load.d/br_netfilter.conf

chmod 755  /etc/modules-load.d/bridge.conf
chmod 755  /etc/modules-load.d/br_netfilter.conf

modprobe bridge
modprobe br_netfilter


# 配置 sysctl
# 注意：需要更换下列net.ipv4.conf.enp1s0.rp_filter=0中的网卡
cat >>  /etc/sysctl.conf << "EOF"
# sysctl settings are defined through files in
# /usr/lib/sysctl.d/, /run/sysctl.d/, and /etc/sysctl.d/.
#
# Vendors settings live in /usr/lib/sysctl.d/.
# To override a whole file, create a new file with the same in
# /etc/sysctl.d/ and put new settings there. To override
# only specific settings, add a file with a lexically later
# name in /etc/sysctl.d/ and put new settings there.
#
# For more information, see sysctl.conf(5) and sysctl.d(5).
kernel.sysrq=0
net.ipv4.ip_forward=0
net.ipv4.conf.all.send_redirects=0
net.ipv4.conf.default.send_redirects=0
net.ipv4.conf.all.accept_source_route=0
net.ipv4.conf.default.accept_source_route=0
net.ipv4.conf.all.accept_redirects=0
net.ipv4.conf.default.accept_redirects=0
net.ipv4.conf.all.secure_redirects=0
net.ipv4.conf.default.secure_redirects=0
net.ipv4.icmp_echo_ignore_broadcasts=1
net.ipv4.icmp_ignore_bogus_error_responses=1
net.ipv4.conf.all.rp_filter=1
net.ipv4.conf.default.rp_filter=1
net.ipv4.tcp_syncookies=1
kernel.dmesg_restrict=1
net.ipv6.conf.all.accept_redirects=0
net.ipv6.conf.default.accept_redirects=0

#         [ERROR FileContent--proc-sys-net-bridge-bridge-nf-call-iptables]: /proc/sys/net/bridge/bridge-nf-call-iptables does not exist
# [ERROR FileContent--proc-sys-net-bridge-bridge-nf-call-ip6tables]: /proc/sys/net/bridge/bridge-nf-call-ip6tables does not exist
net.bridge.bridge-nf-call-iptables=1
net.ipv4.ip_forward=1

# ip6
net.ipv6.conf.all.forwarding=1
net.bridge.bridge-nf-call-ip6tables=1

# fs inode
fs.inotify.max_user_watches = 524288
fs.inotify.max_user_instances = 512

#  many interface of linux node for spical vlan can connect config
net.ipv4.conf.default.rp_filter=0
net.ipv4.conf.all.rp_filter=0
net.ipv4.conf.lo.rp_filter=0
net.ipv4.conf.eno1.rp_filter=0
EOF

# 应用更改
sysctl -p

3. 获取依赖

注意，可能直接下载最新版本无法使用，可以尝试直接通过apt下载

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
# runc # https://github.com/opencontainers/runc
wget https://github.com/opencontainers/runc/releases/download/v1.1.14/runc.amd64
install -m 755 runc.amd64 /usr/local/sbin/runc
which runc # /usr/local/sbin/runc

# cni # https://github.com/containernetworking/plugins
wget https://github.com/containernetworking/plugins/releases/download/v1.5.1/cni-plugins-linux-amd64-v1.5.1.tgz
mkdir -p /opt/cni/bin
tar Cxzvf /opt/cni/bin cni-plugins-linux-amd64-v1.5.1.tgz
chown -R root:root /opt/cni

# containerd # https://github.com/containerd/containerd
wget https://github.com/containerd/containerd/releases/download/v1.7.22/containerd-1.7.22-linux-amd64.tar.gz
tar Cxzvf /usr/local containerd-1.7.22-linux-amd64.tar.gz
which containerd # /usr/local/bin/containerd

# config containerd
mkdir -p /etc/containerd/
containerd config default > /etc/containerd/config.toml
sed -i 's/SystemdCgroup = false/SystemdCgroup = true/g' /etc/containerd/config.toml
wget https://raw.githubusercontent.com/containerd/containerd/refs/heads/main/containerd.service
mkdir -p /usr/local/lib/systemd/system/
cp containerd.service /usr/local/lib/systemd/system/containerd.service

# restart containerd
systemctl daemon-reload
systemctl enable --now containerd
systemctl status containerd

3. 获取k8s

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# crictl https://github.com/kubernetes-sigs/cri-tools
wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.31.1/crictl-v1.31.1-linux-amd64.tar.gz
tar zxvf crictl-v1.31.1-linux-amd64.tar.gz -C /usr/local/bin
which crictl # /usr/local/bin/crictl
crictl config runtime-endpoint unix:///var/run/containerd/containerd.sock
crictl config image-endpoint unix:///run/containerd/containerd.sock

# k8s
apt-get update && apt-get install -y apt-transport-https
curl -fsSL https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.31/deb/Release.key | gpg --dearmor -o /etc/apt/keyrings/kubernetes-apt-keyring.gpg
echo "deb [signed-by=/etc/apt/keyrings/kubernetes-apt-keyring.gpg] https://mirrors.aliyun.com/kubernetes-new/core/stable/v1.31/deb/ /" | tee /etc/apt/sources.list.d/kubernetes.list
apt-get update
apt-get install -y kubelet kubeadm kubectl
systemctl enable kubelet && systemctl start kubelet
sudo apt-mark hold kubelet kubeadm kubectl

4. 配置k8s

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
mkdir -p /etc/sysconfig/
cat > /etc/sysconfig/kubelet <<"EOF"
Environment="KUBELET_EXTRA_ARGS=--fail-swap-on=false"
EOF
systemctl daemon-reload
systemctl enable --now kubelet

kubeadm config print init-defaults --component-configs KubeProxyConfiguration,KubeletConfiguration > kubeadm-init.yaml
sed -i 's/\(kubernetesVersion: \).*/\1 1.31.1/g' kubeadm-init.yaml
sed -i 's/\(advertiseAddress: \).*/\1 '${cluster_float_ip}'/g' kubeadm-init.yaml
sed -i 's/\( name: \).*//g' kubeadm-init.yaml
sed -i 's#\(serviceSubnet: \).*#\1 10.96.0.0/16,2001:db8:42:1::/112\n  podSubnet: 10.244.0.0/16,2001:db8:42:0::/56#g' kubeadm-init.yaml
sed -i 's/imageRepository: registry.k8s.io/imageRepository: registry.aliyuncs.com\/google_containers/g' kubeadm-init.yaml
sed -i '/scheduler: {}/ a\controlPlaneEndpoint: cluster.svc' kubeadm-init.yaml
sed -i '/memorySwap: {}/ a\failSwapOn: false' kubeadm-init.yaml

kubeadm config images list   --config=kubeadm-init.yaml
crictl pull registry.aliyuncs.com/google_containers/pause:3.8
ctr -n k8s.io i tag registry.aliyuncs.com/google_containers/pause:3.8 registry.k8s.io/pause:3.8

apt install conntrack bash-completion ipvsadm socat net-tools iproute2 -y
# init

# 万一 出现错误 重新 安装前， 请 清空环境
# kubeadm reset   -f

# rm -rf /etc/kubernetes/manifests
# rm -rf  /etc/kubernetes
# rm -rf  /etc/kubelet
# rm -rf  /var/lib/kublet
# rm -rf  /var/lib/etcd

# 验证配置文件
# kubeadm config validate --config kubeadm-init.yaml

kubeadm init --config=kubeadm-init.yaml --upload-certs


echo 'export KUBECONFIG=/etc/kubernetes/admin.conf' >> ~/.bashrc
source ~/.bashrc

echo 'source <(kubectl completion bash)' >>~/.bashrc

echo 'alias k=kubectl' >>~/.bashrc
echo 'complete -o default -F __start_kubectl k' >>~/.bashrc

source ~/.bashrc

kubectl taint node ethereal-desktop node-role.kubernetes.io/control-plane:NoSchedule-

5. 配置网络

1
2
3
4
5
6
7
8
9
10
11
12
13
# cilium # https://github.com/cilium/cilium-cli
wget https://github.com/cilium/cilium-cli/releases/download/v0.16.18/cilium-linux-amd64.tar.gz
tar xzvfC cilium-linux-amd64.tar.gz /usr/local/bin

cilium version --client
cilium install --version 1.16.1

cilium config set ipam kubernetes
cilium config set enable-ipv4 true
cilium config set cluster-pool-ipv4-cidr 10.96.0.0/16
cilium config set enable-ipv6 false

cilium status --wait

6. 配置本地存储

1
2
3
4
5
6
# local path-provisioner # https://github.com/rancher/local-path-provisioner
wget https://raw.githubusercontent.com/rancher/local-path-provisioner/v0.0.29/deploy/local-path-storage.yaml
sed -i 's#/opt/local-path-provisioner#/opt/k8s-local-path-storage#g' local-path-storage.yaml
crictl pull dhub.kubesre.xyz/rancher/local-path-provisioner:v0.0.29
ctr -n k8s.io i tag dhub.kubesre.xyz/rancher/local-path-provisioner:v0.0.29 docker.io/rancher/local-path-provisioner:v0.0.29
k apply -f local-path-storage.yaml

7. 修改网络代理

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
# https://github.com/DaoCloud/public-image-mirror
vim /etc/containerd/config.toml
# 修改[plugins.”io.containerd.grpc.v1.cri”.registry]段，在子段[plugins.io.containerd.grpc.v1.cri”.registry.mirrors]和[plugins.”io.containerd.grpc.v1.cri”.registry.configs]中添加配置即可。


      [plugins."io.containerd.grpc.v1.cri".registry.configs]
        [plugins."io.containerd.grpc.v1.cri".registry.configs."docker.io".tls]
          insecure_skip_verify = true

      [plugins."io.containerd.grpc.v1.cri".registry.headers]

      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
           endpoint = ["https://docker.m.daocloud.io"]
           # endpoint = ["http://10.0.0.101:500"]
           # endpoint = ["https://dhub.kubesre.xyz"]

### 私有部署
      [plugins."io.containerd.grpc.v1.cri".registry.configs]
        [plugins."io.containerd.grpc.v1.cri".registry.configs."registry.cn-shanghai.aliyuncs.com".tls]
                        insecure_skip_verify = true
        [plugins."io.containerd.grpc.v1.cri".registry.configs."registry.cn-shanghai.aliyuncs.com".auth]
                        username = "Ethereal@1608148795872129"
                        password = "SichaoMiaoA123456"

      [plugins."io.containerd.grpc.v1.cri".registry.headers]

      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."registry.cn-shanghai.aliyuncs.com"]
                endpoint = ["https://registry.cn-shanghai.aliyuncs.com"]

# 重启服务
systemctl daemon-reload
systemctl restart containerd
systemctl status containerd

8. 部署Metrics

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
# https://github.com/kubernetes-sigs/metrics-server/
wget https://github.com/kubernetes-sigs/metrics-server/releases/download/v0.7.2/components.yaml
# 修改文件中的deployment中参数：
containers:
- args:
  - --cert-dir=/tmp
  - --secure-port=443
  - --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname
  - --kubelet-use-node-status-port
  - --metric-resolution=15s
  - --kubelet-insecure-tls # 加上该启动参数 


# 部署
k apply -f components.yaml
## 需要注意，只有当部署的组件中包含hpa与deployment中指定resource.request时，才会在k get hpa中获得正确的占用情况

9. 参考

k8s集群部署时etcd容器不停重启问题及处理_6443: connect: connection refused-CSDN博客

nv-k8s - Ethereal’s Blog (ethereal-o.github.io)

k8s集群搭建 - Ethereal’s Blog (ethereal-o.github.io)

cby-chen/Kubernetes: kubernetes (k8s) 二进制高可用安装，Binary installation of kubernetes (k8s) — 开源不易，帮忙点个star，谢谢了🌹 (github.com)

Kubernetes/doc/kubeadm-install.md at main · cby-chen/Kubernetes (github.com)

DaoCloud/public-image-mirror: 很多镜像都在国外。比如 gcr 。国内下载很慢，需要加速。致力于提供连接全世界的稳定可靠安全的容器镜像服务。 (github.com)

GitHub 文件加速

nmtui修改静态IP地址，巨好用！_openeneuler系统中nmtui怎么设置ip地址-CSDN博客

Kubernetes上安装Metrics-Server - YOYOFx - 博客园

  • Title: 新k8s部署方案
  • Author: Ethereal
  • Created at: 2024-09-27 22:55:06
  • Updated at: 2024-11-23 00:44:10
  • Link: https://ethereal-o.github.io/2024/09/27/新k8s部署方案/
  • License: This work is licensed under CC BY-NC-SA 4.0.
 Comments
On this page
新k8s部署方案
  1. 1. 配置ip
  2. 2. 初始化配置
  3. 3. 获取依赖
  4. 3. 获取k8s
  5. 4. 配置k8s
  6. 5. 配置网络
  7. 6. 配置本地存储
  8. 7. 修改网络代理
  9. 8. 部署Metrics
  10. 9. 参考