docker-swarm使用
# 设置"live-restore": false
cat /nix/store/8hghs52v46dhvbgfm3l1dgs6n39cdqi1-daemon.json
{
"group": "docker",
"hosts": [
"0.0.0.0:39012",
"fd://"
],
"live-restore": false,
"log-driver": "journald"
}
# nix
virtualisation.docker.enable = true;
virtualisation.docker.enableOnBoot = true;
virtualisation.docker.daemon.settings = {
hosts=[
"0.0.0.0:39012"
];
live-restore=false;
};
初始化
# 端口占用
TCP 2377: 用于管理通信（如节点加入和更新）。
TCP 和 UDP 7946: 用于节点之间的通信。
UDP 4789: 用于覆盖网络（overlay network）通信。
# 初始话内网ip节点
docker swarm init --advertise-addr 116.204.106.129
# Swarm initialized: current node (qxlwqzkkhecvepwujl61na1no) is now a manager.
# To add a worker to this swarm, run the following command:
# docker swarm join --token SWMTKN-1-16hil64vsxjtj87vlkmd5znncdjoptkzf7x8d7not4qirsvwyt-d3bzjmc8p19a0ux8iqb7zbc5v 116.204.106.129:2377
# To add a manager to this swarm, run 'docker swarm join-token manager' and follow the instructions.
# 离开(服务全部都会清除),之后可以重新init
docker swarm leave --force
# 查看当前主节点join token,提供给其他节点加入, 注意ip以实际为准(如果ip和实际不一致会导致管理异常, 可能需要leave force)
docker swarm join-token worker
Deploy 通过stack使用compose-file进行部署,而不用安装docker-compose
docker stack ls
# NAME SERVICES ORCHESTRATOR
# 从文件创建 https://docs.docker.com/engine/reference/commandline/stack/
# v3详细文档: https://docs.docker.com/compose/compose-file/compose-file-v3/
# The deploy command supports compose file version 3.0 and above.
docker stack deploy --compose-file dep.yaml exadmin
# Creating network exadmin_default
# Creating service exadmin_exadmin
# dep.yaml这里的ports会直接映射到实体机,在物理机器访问
curl http://127.0.0.1:8080
# ps
docker stack ps exadmin
# 显示Running/和Shutdown 数据以及 \_开头删除的数据
# 删除
docker stack rm exadmin
# Removing service exadmin_exadmin
# Removing network exadmin_default
docker service ls
# ID NAME MODE REPLICAS IMAGE PORTS
# cyqmyd3ipiuy exadmin_exadmin replicated 2/2 registry.jihulab.com/jcleng/ex-admindemo:aa344af0 *:8080->8080/tcp
# https://docs.docker.com/engine/reference/commandline/stack_services/
docker stack services exadmin
# ID NAME MODE REPLICAS IMAGE PORTS
# cyqmyd3ipiuy exadmin_exadmin replicated 4/4 registry.jihulab.com/jcleng/ex-admindemo:aa344af0 *:8080->8080/tcp
# 设置数量,查看 docker stack ps exadmin
docker service update --replicas 2 exadmin_exadmin
# --health-cmd=命令，用于检查接口的命令。
# --health-interval=时间间隔 (默认: 30s)，它是每次执行healthcheck的时间间隔。
# --health-timeout=时间间隔 (默认: 30s)，如果在超时时间之内没有响应，则代表异常。
# --health-retries=N (默认: 3)，连续达到多少次异常之后退出。
# --health-start-period=时间间隔 (默认：0)，容器启动之后多久进行健康检查（服务启动预热），即运行health-cmd。
# 健康检查实例: https://github.com/docker-library/healthcheck
# 如果容器中没有curl命令,可以下载httpie来使用https://github.com/httpie/httpie/releases
# curl 参数用-f/--fail,httpie用--check-status
# wget用(检查文件是不是存在)wget --no-verbose --tries=1 --spider http://localhost:2019/metrics
# 注意需要容器存在的命令,否则inspect容器会提示: /bin/sh: 1: curl: not found
docker service update \
--health-cmd "/srv/http --check-status http://127.0.0.1:8080 || exit 1" \
--health-interval 5s \
--health-timeout 3s \
--health-retries 3 \
--health-start-period 45s \
--image registry.jihulab.com/jcleng/ex-admindemo:9699d0ba \
exadmin_exadmin
# docker ps 的时候状态会增加 (healthy) 显示
# 查看服务
docker service inspect exadmin_exadmin
stack部署之后可以查看服务
# 节点
docker node ls
# ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
# qxlwqzkkhecvepwujl61na1no * nixos Ready Active Leader 20.10.18
# 服务
docker service ls
# ID NAME MODE REPLICAS IMAGE PORTS
不使用stack,直接创建服务
# https://docs.docker.com/engine/reference/commandline/service_create/
docker service create --name admindemo_sv \
--replicas=3 \
-p 8088:8080 \
registry.jihulab.com/jcleng/ex-admindemo:aa344af0
# 查看,且8088进行访问
docker service ls
# ID NAME MODE REPLICAS IMAGE PORTS
# su6y8prthnzu admindemo_sv replicated 3/3 registry.jihulab.com/jcleng/ex-admindemo:aa344af0 *:8088->8080/tcp
# 查看service错误日志
docker service ps admindemo_sv
docker service logs 16oikw3t8f0c
dep.yaml 文档 https://docs.docker.com/compose/compose-file/deploy/
version: '3.3'
services:
exadmin:
image: registry.jihulab.com/jcleng/ex-admindemo:aa344af0
ports:
- "8080:8080"
deploy:
replicas: 4
update_config:
parallelism: 2
failure_action: rollback
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 3
window: 120s
# 更新
# docker service update --image registry.jihulab.com/jcleng/ex-admindemo:bfee2a3a exadmin_exadmin --with-registry-auth
# 因为有4个副本，parallelism设置同时更新2个。先更新两个，等待delay后在更新两个
api 文档 https://docs.docker.com/engine/api/v1.41/#section/Versioning
curl -X GET http://127.0.0.1:39012/version
curl -X GET http://127.0.0.1:39012/images/json?all=false
curl -X GET http://127.0.0.1:39012/v1.41/containers/json
# 私有仓库[更新镜像]需要请求头加上验证信息,就是base64_encode的json帐户授权信息
X-Registry-Auth: "xxxxxxxxxx"
<?php
$json = <<<JSON
{
"username": "jcleng",
"password": "password123456",
"email": "[email protected]",
"serveraddress": "registry.jihulab.com"
}
JSON;
var_export(base64_encode(json_encode(json_decode($json, true))));
官方提供仓库搭建registry仓库 自建仓库
# 文档: https://docs.docker.com/registry/
# https://distribution.github.io/distribution/
# 使用官方的registry镜像,映射到本地的5555端口,注意公网的5555端口不要开放出来
docker pull registry
docker run -v ~/registry:/var/lib/registry -itd -p 5000:5000 --name registry registry
# 设置Auth授权验证
https://distribution.github.io/distribution/about/deploying/#native-basic-auth
## 创建用户密码
docker run -itd \
--restart unless-stopped \
-v ./auth/:/auth \
-e "REGISTRY_AUTH=htpasswd" \
-e "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm" \
-e REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd \
-p 5000:5000 \
--name registry \
registry.cn-hangzhou.aliyuncs.com/jcleng/library-registry:3.0
# 新增用户和密码
htpasswd -Bb ./auth/htpasswd test test
# 登录
docker login --username=test -p test localhost:5000
# 查看状态
curl http://192.168.195.35:5000/v2/_catalog
# 配置自己的域名解析到公网服务器的内网ip,就可以使用5555端口
ndocker.leng2011.icu 解析到 192.168.0.7
# 配置ssl
ndocker.leng2011.icu:443 {
reverse_proxy * {
to http://127.0.0.1:5000
}
tls /home/jcleng/cert/ndocker.leng2011.icu_nginx/ndocker.leng2011.icu_bundle.crt /home/jcleng/cert/ndocker.leng2011.icu_nginx/ndocker.leng2011.icu.key
}
# 在公网服务器上进行测试,测试用已经存在的镜像增加一个tag
docker tag 06541a2a33fb ndocker.leng2011.icu/jcleng/ntfy
docker images|grep ntfy
# binwiederhier/ntfy latest 06541a2a33fb 2 months ago 27MB
# ndocker.leng2011.icu/jcleng/ntfy latest 06541a2a33fb 2 months ago 27MB
# 进行推送即可
docker push ndocker.leng2011.icu/jcleng/ntfy:latest
# 获取,内网是很快的
docker rmi ndocker.leng2011.icu/jcleng/ntfy:latest
docker pull ndocker.leng2011.icu/jcleng/ntfy:latest
# 镜像列表
## 文件查看
tree -L 2 ~/imagedata/docker/registry/v2/repositories
## api,获取用户和镜像,然后获取tag
curl https://ndocker.leng2011.icu/v2/_catalog
curl https://ndocker.leng2011.icu/v2/jcleng/ntfy/tags/list
# 删除,获取config的digest: https://github.com/distribution/distribution/blob/main/docs/spec/api.md#deleting-an-image
# 手动删除imagedata/docker/registry/v2/repositories的目录,文件在blobs目录里面
# 执行命令,清除blobs的数据
registry garbage-collect /etc/docker/registry/config.yml
# 比如自己的docker-runner就可以开设到本内网域名,无需登录即可上传
# 配置文件,可以配置密码等: https://docs.docker.com/registry/configuration/#auth
/etc/docker/registry/config.yml
# 使用htpasswd作为密码,加密方式是bcrypt https://docs.docker.com/registry/configuration/#htpasswd
# nix-shell -p apacheHttpd
# htpasswd -Bc htpasswd2 jcleng
# 用123456测试
# cat htpasswd2
# jcleng:$2y$05$8UMyFaiuS299X/mBZ2InA.nAn.Ni0QgV.QDrrjyGl4toOF/FQ7b0a
auth:
htpasswd:
realm: basic-realm
path: /var/lib/registry/docker/registry/htpasswd2
# 配合web端简易管理(不推荐),删除操作等,但是不支持内网操作,只是html静态页面
https://github.com/Joxit/docker-registry-ui
# gitlab的runner使用
- docker build --pull -t "ndocker.leng2011.icu/$CI_PROJECT_PATH_SLUG${tag}" .
- docker push "ndocker.leng2011.icu/$CI_PROJECT_PATH_SLUG${tag}"
# 项目地址: https://jihulab.com/jcleng1/first_demo/
# 生成的容器名称,$CI_PROJECT_PATH_SLUG就是项目地址,tag用$CI_COMMIT_REF_SLUG或者$CI_COMMIT_SHORT_SHA
jcleng1-first-demo:v21
# 安装器,记得先打开Docker Swarm
# 执行命令会初始化安装管理面板引导程序
docker run -it --rm \
--name swarmpit-installer \
--volume /var/run/docker.sock:/var/run/docker.sock \
swarmpit/install:1.9
# 或者手动安装
git clone --depth=1 http://github.com/swarmpit/swarmpit -b master
docker stack deploy -c swarmpit/docker-compose.yml swarmpit
# 初始密码是12345678
# 可能会提示失败,查看实际服务状态
docker service ls|grep swarmpit
# bcpsiqw2vz2z swarmpit_agent global 1/1 swarmpit/agent:2.2
# pvo9njyuoi65 swarmpit_app replicated 1/1 swarmpit/swarmpit:1.9 *:888->8080/tcp
# anrv0itajrur swarmpit_db replicated 1/1 couchdb:2.3.0
# 4pla8l9ihc8d swarmpit_influxdb replicated 1/1 influxdb:1.7
# 全部启动之后,访问端口即可
http://www.leng2011.icu:888/
# 配置Registry v2仓库地址的时候,不需要选[Use custom API]
# 其他面板
https://github.com/cuigh/swirl
例子
version: '3.9'
services:
myingsite:
image: ndocker.leng2011.icu/jcleng-imgsite:52ce37b8
ports:
- "7871:8080"
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:8080"]
interval: 5s
timeout: 6s
retries: 3
start_period: 40s
deploy:
replicas: 2
update_config:
parallelism: 1
delay: 30s
failure_action: rollback
monitor: 15s
max_failure_ratio: 0
order: start-first
rollback_config:
parallelism: 1
delay: 30s
failure_action: pause
monitor: 15s
max_failure_ratio: 0
order: start-first
restart_policy:
condition: on-failure
delay: 5s
max_attempts: 3
window: 120s
# 使用3.9支持update_config/order
# docker stack deploy --compose-file dep.yaml mainsite
# Creating network mainsite_default
# Creating service mainsite_myingsite
# docker service rm mainsite_myingsite
# docker network rm mainsite_default
使用docker的
socket文件进行调用api
sudo chmod 777 /var/run/docker.sock
curl --unix-socket /var/run/docker.sock http://localhost/version | jq
curl --unix-socket /var/run/docker.sock http://localhost/services/nginxs_nginx | jq
curl --unix-socket /var/run/docker.sock http://localhost/tasks | jq
curl --unix-socket /var/run/docker.sock http://localhost/v1.51/containers/json?filters=%7B%22health%22%3A%5B%22healthy%22%5D%2C%22label%22%3A%20%5B%22openresty.enable%3Dtrue%22%5D%7D | jq
主节点配置私有仓库
// cat ~/.docker/config.json
// auth的值是base64_encode("email:password");名称是域名
{
"auths": {
"registry.cn-hangzhou.aliyuncs.com": {
"auth": "ai5jLmxlbxxxxxxxxxxxxxxxxEyMzQ1Xg=="
}
}
}
// 创建服务时, 子节点需要带上X-Registry-Auth才能拉取镜像
高可用, 使用多个Leader节点, 然后配置上有负载均衡器到各个主节点
# 如果是单节点使用, 可以直接使用nginx(ssl)+work节点作为服务直接使用swarm进行流量分发(内部网络使用服务商提供的内网, 可能是不可靠的)
# 如果多节点高可用, 需要使用自己的组网工具组网(ztncui自建作为参看), 然后上层再使用负载均衡(ssl)再进行流量分发到多个主节点ip
# 当一个主节点Leader宕机, 可以切换到其他Leader节点进行管理
# 主节点, 获取manager token
docker swarm join-token manager
# 新节点执行join
docker node ls
# ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
# 0a0s2aj5co77qczb9xkwinflv * 0822c3de9560 Ready Active Leader 28.2.2
# imkg3c2t85bon75tdwt9pzpq6 f42e42c32252 Ready Active Reachable 28.2.2
# 或者, 主节点给work节点提升回管理节点角色
docker swarm join-token worker
# 查看当前节点名称
docker info|grep -C 2 NodeID
docker node promote tsel91769bax7l4efubrlrzy2
docker node ls
# ID HOSTNAME STATUS AVAILABILITY MANAGER STATUS ENGINE VERSION
# tsel91769bax7l4efubrlrzy2 4c020a54e77f Ready Active Reachable 28.2.2
# 0a0s2aj5co77qczb9xkwinflv * 0822c3de9560 Ready Active Leader 28.2.2
# imkg3c2t85bon75tdwt9pzpq6 f42e42c32252 Ready Active Reachable 28.2.2
# The swarm does not have a leader.
# 主节点全部异常, 找一个可以运行的主节点, 执行初始化重新作为主节点
docker swarm init --force-new-cluster
# 再删除宕机的节点
docker node rm xzswjzcvhj5n9febhvr4xn71c