使用nexus搭建了http的私有docker镜像proxy, 对外地址为http://11.0.2.132:5000
我的步骤如下：
mkdir -p /etc/rancher/rke2/
cat >/etc/rancher/rke2/config.yaml <<EOL
token: 123456
system-default-registry: "11.0.2.132:5000"
EOL
cat >/etc/rancher/rke2/registries.yaml<<EOL
mirrors:
docker.io:
endpoint:
- "http://11.0.2.132:5000"
11.0.2.132:
endpoint:
- "http://11.0.2.132:5000"
configs:
"11.0.2.132:5000":
tls:
insecure_skip_verify: true
EOL
systemctl start rke2-server.service
错误信息如下：
Oct 20 11:48:54 k8s-m3 rke2[628575]: time="2024-10-20T11:48:54+08:00" level=info msg="Checking local image archives in /var/lib/rancher/rke2/agent/images for index.docker.io/rancher/rke2-runtime:v1.30.5-rke2r1"
Oct 20 11:48:54 k8s-m3 rke2[628575]: time="2024-10-20T11:48:54+08:00" level=warning msg="Failed to load runtime image index.docker.io/rancher/rke2-runtime:v1.30.5-rke2r1 from tarball: no local image available for index.docker.io/rancher/rke>
Oct 20 11:48:54 k8s-m3 rke2[628575]: time="2024-10-20T11:48:54+08:00" level=info msg="Checking local image archives in /var/lib/rancher/rke2/agent/images for 11.0.2.132:5000/rancher/rke2-runtime:v1.30.5-rke2r1"
Oct 20 11:48:54 k8s-m3 rke2[628575]: time="2024-10-20T11:48:54+08:00" level=warning msg="Failed to load runtime image 11.0.2.132:5000/rancher/rke2-runtime:v1.30.5-rke2r1 from tarball: no local image available for 11.0.2.132:5000/rancher/rke>
Oct 20 11:48:54 k8s-m3 rke2[628575]: time="2024-10-20T11:48:54+08:00" level=info msg="Using private registry config file at /etc/rancher/rke2/registries.yaml"
Oct 20 11:48:54 k8s-m3 rke2[628575]: time="2024-10-20T11:48:54+08:00" level=info msg="Pulling runtime image 11.0.2.132:5000/rancher/rke2-runtime:v1.30.5-rke2r1"
Oct 20 11:48:54 k8s-m3 rke2[628575]: time="2024-10-20T11:48:54+08:00" level=warning msg="Failed to get image from endpoint: Get \"https://11.0.2.132:5000/v2/\": http: server gave HTTP response to HTTPS client"
Oct 20 11:48:54 k8s-m3 rke2[628575]: time="2024-10-20T11:48:54+08:00" level=fatal msg="failed to get runtime image 11.0.2.132:5000/rancher/rke2-runtime:v1.30.5-rke2r1: all endpoints failed: Get \"https://11.0.2.132:5000/v2/\": http: server >
Oct 20 11:48:54 k8s-m3 systemd[1]: rke2-server.service: Main process exited, code=exited, status=1/FAILURE
然后更改/etc/rancher/rke2/config.yaml ，去掉system-default-registry: “11.0.2.132:5000”。
mkdir -p /etc/rancher/rke2/
cat >/etc/rancher/rke2/config.yaml <<EOL
token: 123456
EOL
cat >/etc/rancher/rke2/registries.yaml<<EOL
mirrors:
docker.io:
endpoint:
- "http://11.0.2.132:5000"
11.0.2.132:
endpoint:
- "http://11.0.2.132:5000"
configs:
"11.0.2.132:5000":
tls:
insecure_skip_verify: true
EOL
systemctl start rke2-server.service
可以启动成功，并成功部署rke2集群:
/var/lib/rancher/rke2/bin/kubectl get pod -A
kube-system cloud-controller-manager-k8s-m3 1/1 Running 0 94s
kube-system etcd-k8s-m3 1/1 Running 0 90s
kube-system helm-install-rke2-canal-5sdx4 0/1 Completed 0 80s
kube-system helm-install-rke2-coredns-2hqmz 0/1 Completed 0 80s
kube-system helm-install-rke2-ingress-nginx-8twsr 0/1 Completed 0 80s
kube-system helm-install-rke2-metrics-server-rn8bp 0/1 Completed 0 80s
kube-system helm-install-rke2-snapshot-controller-crd-ngjjf 0/1 Completed 0 80s
kube-system helm-install-rke2-snapshot-controller-zvrz5 0/1 Completed 0 80s
kube-system helm-install-rke2-snapshot-validation-webhook-26llw 0/1 Completed 0 80s
kube-system kube-apiserver-k8s-m3 1/1 Running 0 95s
kube-system kube-controller-manager-k8s-m3 1/1 Running 0 96s
kube-system kube-proxy-k8s-m3 1/1 Running 0 89s
kube-system kube-scheduler-k8s-m3 1/1 Running 0 96s
kube-system rke2-canal-ff9xg 2/2 Running 0 76s
kube-system rke2-coredns-rke2-coredns-7d8f866c78-mntp6 1/1 Running 0 77s
kube-system rke2-coredns-rke2-coredns-autoscaler-75bc99ff8-m8xbp 1/1 Running 0 77s
kube-system rke2-ingress-nginx-controller-vk2p9 1/1 Running 0 42s
kube-system rke2-metrics-server-77fd97b84-2p6p5 1/1 Running 0 51s
kube-system rke2-snapshot-controller-7dcf5d5b46-xfwrg 1/1 Running 0 50s
kube-system rke2-snapshot-validation-webhook-bf7bbd6fc-9692p 1/1 Running 0 53s
但我的问题来了为什么cmd执行crictl的pull,却不使用http访问11.0.2.132:5000，而是使用https,比如
/var/lib/rancher/rke2/bin/crictl pull 11.0.2.132:5000/quay.io/jetstack/cert-manager-cainjector:v1.16.1
E1020 12:03:18.462538 638845 remote_image.go:180] "PullImage from image service failed" err="rpc error: code = Unknown desc = failed to pull and unpack image \"11.0.2.132:5000/quay.io/jetstack/cert-manager-cainjector:v1.16.1\": failed to resolve reference \"11.0.2.132:5000/quay.io/jetstack/cert-manager-cainjector:v1.16.1\": failed to do request: Head \"https://11.0.2.132:5000/v2/quay.io/jetstack/cert-manager-cainjector/manifests/v1.16.1\": http: server gave HTTP response to HTTPS client" image="11.0.2.132:5000/quay.io/jetstack/cert-manager-cainjector:v1.16.1"
FATA[0000] pulling image: failed to pull and unpack image "11.0.2.132:5000/quay.io/jetstack/cert-manager-cainjector:v1.16.1": failed to resolve reference "11.0.2.132:5000/quay.io/jetstack/cert-manager-cainjector:v1.16.1": failed to do request: Head "https://11.0.2.132:5000/v2/quay.io/jetstack/cert-manager-cainjector/manifests/v1.16.1": http: server gave HTTP response to HTTPS client
root@k8s-m3:~# cat /var/lib/rancher/rke2/agent/etc/containerd/config.toml
# File generated by rke2. DO NOT EDIT. Use config.toml.tmpl instead.
version = 2
[plugins."io.containerd.internal.v1.opt"]
path = "/var/lib/rancher/rke2/agent/containerd"
[plugins."io.containerd.grpc.v1.cri"]
stream_server_address = "127.0.0.1"
stream_server_port = "10010"
enable_selinux = false
enable_unprivileged_ports = true
enable_unprivileged_icmp = true
sandbox_image = "index.docker.io/rancher/mirrored-pause:3.6"
[plugins."io.containerd.grpc.v1.cri".containerd]
snapshotter = "overlayfs"
disable_snapshot_annotations = true
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
runtime_type = "io.containerd.runc.v2"
[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc.options]
SystemdCgroup = true
[plugins."io.containerd.grpc.v1.cri".registry]
config_path = "/var/lib/rancher/rke2/agent/etc/containerd/certs.d"
root@k8s-m3:~# cat /var/lib/rancher/rke2/agent/etc/containerd/certs.d/docker.io/hosts.toml
# File generated by rke2. DO NOT EDIT.
server = "https://registry-1.docker.io/v2"
capabilities = ["pull", "resolve", "push"]
[host]
[host."http://11.0.2.132:5000/v2"]
capabilities = ["pull", "resolve"]
skip_verify = true
skip_verify = true 出来了，但是为什么crictl pull的时候，依然使用https而不是http啊？
containerd的log，
cat /var/lib/rancher/rke2/agent/containerd/containerd.log | grep 11.0.2.132
time="2024-10-20T12:09:54.854016828+08:00" level=info msg="host will try HTTPS first since it is configured for HTTP with a TLS configuration, consider changing host to HTTPS or removing unused TLS configuration" host="11.0.2.132:5000"
time="2024-10-20T12:09:54.860286882+08:00" level=info msg="host will try HTTPS first since it is configured for HTTP with a TLS configuration, consider changing host to HTTPS or removing unused TLS configuration" host="11.0.2.132:5000"
time="2024-10-20T12:23:40.175514512+08:00" level=info msg="PullImage \"11.0.2.132:5000/quay.io/jetstack/cert-manager-cainjector:v1.16.1\""
time="2024-10-20T12:23:40.178225248+08:00" level=info msg="trying next host" error="failed to do request: Head \"https://11.0.2.132:5000/v2/quay.io/jetstack/cert-manager-cainjector/manifests/v1.16.1\": http: server gave HTTP response to HTTPS client" host="11.0.2.132:5000"
time="2024-10-20T12:23:40.179075980+08:00" level=error msg="PullImage \"11.0.2.132:5000/quay.io/jetstack/cert-manager-cainjector:v1.16.1\" failed" error="failed to pull and unpack image \"11.0.2.132:5000/quay.io/jetstack/cert-manager-cainjector:v1.16.1\": failed to resolve reference \"11.0.2.132:5000/quay.io/jetstack/cert-manager-cainjector:v1.16.1\": failed to do request: Head \"https://11.0.2.132:5000/v2/quay.io/jetstack/cert-manager-cainjector/manifests/v1.16.1\": http: server gave HTTP response to HTTPS client"
time="2024-10-20T12:23:40.179173263+08:00" level=info msg="stop pulling image 11.0.2.132:5000/quay.io/jetstack/cert-manager-cainjector:v1.16.1: active requests=0, bytes read=0"