虚拟机搭建k3s多节点
节点服务器安装
# 所有的用户名和密码
user01
123456
# 创建节点,不要用下划线
nodemaster
# 然后克隆出
node01
node02
# 内网测试需要这一步: 统一设置host,实体机最好也设置
networking.hosts={
"192.168.122.48" = [ "nodemaster" ];
"192.168.122.47" = [ "node01" ];
"192.168.122.46" = [ "node02" ];
};
# 编辑和生效
sudo vi /etc/nixos/configuration.nix
sudo nixos-rebuild switch --option substituters "https://mirror.nju.edu.cn/nix-channels/store/" --show-trace
# 测试是否可以ping
ping nodemaster
ping node01
ping node02
软件安装
# nodemaster安装k3s
ssh user01@nodemaster
services.k3s.enable=true;
services.k3s.role="server";
services.k3s.extraFlags="--node-name nodemaster";
# 检查报错
systemctl status k3s.service
sudo k3s kubectl get node
# master起来之后就直接安装 kube-dashboard
sudo k3s kubectl port-forward -n kubernetes-dashboard --address 0.0.0.0 service/kubernetes-dashboard 8080:443
# https://nodemaster:8080/#/login
# 最后配置节点 sudo cat /var/lib/rancher/k3s/server/node-token
services.k3s.enable=true;
services.k3s.role="agent";
services.k3s.extraFlags="--node-name node01";
services.k3s.token="K107af8350ca00739a455638dbe5acb190d6917a19f63e06dee591713edd13a43d6::server:27115c1e1d5473aaf61d2860b0f8a018";
services.k3s.serverAddr="https://nodemaster:6443";
# 查看节点
sudo k3s kubectl get node
# NAME STATUS ROLES AGE VERSION
# node01 Ready <none> 3m59s v1.23.10+k3s1
# nodemaster Ready control-plane,master 168m v1.23.10+k3s1
# node02 Ready <none> 118s v1.23.10+k3s1
sudo ls -lh /var/lib/libvirt/images/ 1 ↵
# 总用量 24G
# -rw------- 1 root root 2.6G 11月 14 20:18 nixos-node01.qcow2
# -rw------- 1 root root 2.7G 11月 14 20:18 nixos-node02.qcow2
# -rw------- 1 root root 21G 11月 14 20:20 nixos.qcow2
# 如果内网测试ingress不能获取ADDRESS(ingress也配上域名)
# 在Service的ports同级增加externalIPs指定为 内网ip/外网ip
externalIPs:
- 192.168.122.48
# 生效之后即使ADDRESS为空现在也可以访问 http://nodemaster/
# 在svc中:
# 通过CLUSTER-IP(对内)/EXTERNAL-IP(都兼容)都可以访问对应的服务
kubectl使用配置文件
# 优先级最高
sudo kubectl --kubeconfig=/etc/rancher/k3s/k3s.yaml get pods
# 优先级中
export KUBECONFIG=/etc/rancher/k3s/k3s.yaml
# 优先级低,但是可以不需要使用sudo
sudo cp /etc/rancher/k3s/k3s.yaml /home/jcleng/.kube/config
sudo chown jcleng /home/jcleng/.kube/config
kubectl get pods
80端口进行暴露服务
# A.traefik+Ingress: 默认的80端口被traefik服务占用了,类型是LoadBalancer,接管了80和443的入口,可以直接使用 kind: Ingress 进行部署
kubectl get service --all-namespaces
# NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
# default kubernetes ClusterIP 10.43.0.1 <none> 443/TCP 66m
# kube-system kube-dns ClusterIP 10.43.0.10 <none> 53/UDP,53/TCP,9153/TCP 66m
# kube-system metrics-server ClusterIP 10.43.67.252 <none> 443/TCP 66m
# kube-system traefik LoadBalancer 10.43.237.166 192.168.0.7 80:32733/TCP,443:30316/TCP 64m
# default service-testhold NodePort 10.43.48.89 <none> 9501:80/TCP 40m
# B.Ingressnginx+Ingress,自行安装Ingressnginx
# C.NodePort+nginx/caddy 自行操作入口进行负载均衡
C.自己配置caddy作为应用入口,替代traefik+ingress-nginx
www.leng2011.icu:443 {
reverse_proxy * {
to http://127.0.0.1:20826
}
tls /home/jcleng/www.leng2011.icu_bundle.crt /home/jcleng/www.leng2011.icu.key
}
www.leng2011.icu:80 {
redir https://www.leng2011.icu
}
# sudo caddy run -config ./Caddyfile
# 需要备案成功即可访问
# 实际测试如果LoadBalancer/caddy入口加了ssl,ingress就可以不加ssl即可
检查k3s默认启动时的基础服务
sudo k3s crictl images
# IMAGE TAG IMAGE ID SIZE
# docker.io/rancher/klipper-helm v0.7.3-build20220613 38b3b9ad736af 83MB
# docker.io/rancher/klipper-lb v0.3.5 dbd43b6716a08 3.33MB
# docker.io/rancher/local-path-provisioner v0.0.21 fb9b574e03c34 11.4MB
# docker.io/rancher/mirrored-coredns-coredns 1.9.1 99376d8f35e0a 14.1MB
# docker.io/rancher/mirrored-library-traefik 2.9.1 e6de8578b2384 33.4MB
# docker.io/rancher/mirrored-metrics-server v0.6.1 e57a417f15d36 28.1MB
# docker.io/rancher/mirrored-pause 3.6 6270bb605e12e 301kB
sudo k3s crictl ps
# CONTAINER IMAGE CREATED STATE NAME ATTEMPT POD ID POD
# 304fd62642a58 e6de8578b2384 11 hours ago Running traefik 0 7db66edaf6b6f traefik-9c6dc6686-dztj7
# 917af9586a598 dbd43b6716a08 11 hours ago Running lb-tcp-443 0 910ab3a0a8740 svclb-traefik-541f40a6-65lnc
# aa83ed171c590 dbd43b6716a08 11 hours ago Running lb-tcp-80 0 910ab3a0a8740 svclb-traefik-541f40a6-65lnc
# 316cc247cb033 e57a417f15d36 11 hours ago Running metrics-server 0 d57ecb95239a8 metrics-server-5c8978b444-hctsb
# 534a443df85b5 99376d8f35e0a 11 hours ago Running coredns 0 058b64e247624 coredns-75fc8f8fff-4wpzr
# 64778137d1346 fb9b574e03c34 11 hours ago Running local-path-provisioner 0 e7aff696ab4d7 local-path-provisioner-5b5579c644-jtj4x
# 通过ctr修改镜像tag
ctr --namespace=k8s.io image tag docker.io/library/httpd:latest k8s.gcr.io/httpd:latest
nerdctl -n k8s.io tag docker.io/library/httpd:latest k8s.gcr.io/httpd:latest
重启服务
kubectl -n kube-system rollout status deployments/traefik
关于traefik,默认自带可以直接使用(LoadBalancer)
kubectl -n kube-system get svc
#NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
#kube-dns ClusterIP 10.43.0.10 <none> 53/UDP,53/TCP,9153/TCP 3d23h
#metrics-server ClusterIP 10.43.250.225 <none> 443/TCP 3d23h
#traefik LoadBalancer 10.43.29.240 172.23.186.253 80:32528/TCP,443:30364/TCP 150m
# 主ingress,默认是traefik,不加ingressClassName
apiVersion: networking.k8s.io/v1
kind: Ingress
metadata:
name: stable
spec:
# ingressClassName: traefik
rules:
- host: test.leng2011.icu
http:
paths:
- backend:
service:
name: service-testhold
port:
number: 80
path: /
pathType: Prefix