下载confd 二进制文件
# 创建目录方便存放文件
mkdir confd
# 进入新创建的目录
cd confd
# 下载 confd
wget https://github.com/kelseyhightower/confd/releases/download/v0.16.0/confd-0.16.0-linux-amd64
# 重命名
mv confd-0.16.0-linux-amd64 confd
# 给confd 可执行权限
chmod +x confd生成confd 配置
# 创建confd 配置目录
mkdir -p ./conf.d
# 创建模版存放目录
mkdir -p ./templates
# 生成confd 配置文件
cat << EOF | tee ./conf.d/nginx.toml
[template]
src = "nginx.tmpl"
dest = "/etc/nginx/nginx.conf"
keys = [
"CP_HOSTS",
]
EOF
# 生成模版文件
cat << EOF | tee ./templates/nginx.tmpl
error_log stderr notice;
worker_processes auto;
events {
multi_accept on;
use epoll;
worker_connections 4096;
}
stream {
upstream kube_apiserver {
{{ \$servers := split (getenv "CP_HOSTS") "," }}{{range \$servers}}
server {{.}}:6443;
{{end}}
}
server {
listen 6443;
proxy_pass kube_apiserver;
proxy_timeout 30;
proxy_connect_timeout 2s;
}
}
EOF
# 生成启动文件
cat << EOF | tee ./nginx-proxy
#!/bin/sh
# Run confd
confd -onetime -backend env
# Start nginx
nginx -g ‘daemon off;‘
EOF
# 给启动文件执行权限
chmod +x ./nginx-proxyDockerfile
vim Dockerfile
# 基础镜像
FROM alpine
# 作者信息
MAINTAINER nginx 1.17.9 Docker Maintainers "[email protected]"
# 修改源
RUN sed -i ‘s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g‘ /etc/apk/repositories
# 安装ca 证书
RUN apk update && apk add --no-cache ca-certificates
# 设置环境变量
ENV NGINX_VERSION 1.17.9
ENV OPENSSL_VERSION 1.1.1e
# 编译安装NGINX
WORKDIR /tmp
RUN NGINX_CONFIG=" --prefix=/etc/nginx --sbin-path=/usr/sbin/nginx --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/log/nginx/error.log --http-log-path=/var/log/nginx/access.log --pid-path=/var/run/nginx.pid --lock-path=/var/run/nginx.lock --http-client-body-temp-path=/var/cache/nginx/client_temp --http-proxy-temp-path=/var/cache/nginx/proxy_temp --http-fastcgi-temp-path=/var/cache/nginx/fastcgi_temp --http-uwsgi-temp-path=/var/cache/nginx/uwsgi_temp --http-scgi-temp-path=/var/cache/nginx/scgi_temp --with-pcre --user=nginx --group=nginx --with-compat --with-file-aio --with-threads --with-http_addition_module --with-http_auth_request_module --with-http_dav_module --with-http_flv_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_mp4_module --with-http_random_index_module --with-http_realip_module --with-http_secure_link_module --with-http_slice_module --with-http_ssl_module --with-http_stub_status_module --with-http_sub_module --with-http_v2_module --with-ipv6 --with-openssl=../openssl-$OPENSSL_VERSION --with-openssl-opt=enable-tls1_3 --with-mail --with-mail_ssl_module --with-stream --with-stream_realip_module --with-stream_ssl_module --with-stream_ssl_preread_module --with-ld-opt=-Wl,--as-needed " && addgroup -S nginx && adduser -D -S -h /www -s /sbin/nologin -G nginx nginx && apk add --no-cache --virtual .build-deps gcc libc-dev make pcre-dev zlib-dev linux-headers curl gnupg libxslt-dev gd-dev geoip-dev libstdc++ wget libjpeg libpng libpng-dev freetype freetype-dev libxml2 libxml2-dev curl-dev libmcrypt libmcrypt-dev autoconf libjpeg-turbo-dev libmemcached libmemcached-dev gettext gettext-dev libzip git libzip-dev && curl -fSL https://www.openssl.org/source/openssl-$OPENSSL_VERSION.tar.gz -o /tmp/openssl-$OPENSSL_VERSION.tar.gz && curl -fSL https://nginx.org/download/nginx-$NGINX_VERSION.tar.gz -o /tmp/nginx-$NGINX_VERSION.tar.gz && cd /tmp && tar -xzf openssl-$OPENSSL_VERSION.tar.gz && tar -xzf nginx-$NGINX_VERSION.tar.gz && cd /tmp/nginx-$NGINX_VERSION && ./configure $NGINX_CONFIG && make -j$(getconf _NPROCESSORS_ONLN) && make install
# 构建confd nginx 镜像
FROM alpine
# 作者信息
MAINTAINER nginx 1.17.9 Docker Maintainers "[email protected]"
# 修改源
RUN sed -i ‘s/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g‘ /etc/apk/repositories
# 安装ca 证书
RUN apk update && apk add --no-cache ca-certificates
# 设置环境变量
ENV NGINX_VERSION 1.17.9
ENV OPENSSL_VERSION 1.1.1e
RUN mkdir -p /var/lib/nginx/cache && apk add --no-cache \
curl wget pcre && addgroup -S nginx && adduser -D -S -h /var/lib/nginx -s /sbin/nologin -G nginx nginx && chown -R nginx:nginx /var/lib/nginx && mkdir -p /var/log/nginx && rm -rf /var/cache/apk/* && mkdir -p /etc/confd && mkdir -p /var/cache/nginx/client_temp
#COPY 编译结果
COPY --from=0 /usr/sbin/nginx /usr/sbin/nginx
COPY --from=0 /etc/nginx /etc/nginx
ADD confd /usr/sbin/confd
ADD conf.d /etc/confd/conf.d
ADD templates /etc/confd/templates
ADD nginx-proxy /usr/bin/nginx-proxy
STOPSIGNAL SIGTERM
ENTRYPOINT ["/usr/bin/nginx-proxy"]
生成镜像
[[email protected] confd]# tree
.
|-- Dockerfile
|-- conf.d
| `-- nginx.toml
|-- confd
|-- nginx-proxy
`-- templates
`-- nginx.tmpl
2 directories, 5 files
# 生成镜像
docker build -t ha-tools:v1.17.9 . # 镜像名字自己修改 我这里以ng 版本为tag
# 给进行打新tag
docker tag ha-tools:v1.17.9 juestnow/ha-tools:v1.17.9
# 上传镜像
docker push juestnow/ha-tools:v1.17.9测试生成的镜像
# 单个IP
docker run -tid --network=host --name=ha-proxy -e "CP_HOSTS=192.168.2.175" juestnow/ha-tools:v1.17.9s CP_HOSTS=192.168.2.175
# 多个IP
docker run -tid --network=host --name=ha-proxy -e "CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177" juestnow/ha-tools:v1.17.9 CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177
# 进去容器查看是否正常
docker ps
docker exec -ti 27733e5f9a97 /bin/sh
/ # ps -ef
PID USER TIME COMMAND
1 root 0:00 {nginx-proxy} /bin/sh /usr/bin/nginx-proxy CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177
12 root 0:00 nginx: master process nginx -g daemon off;
13 nginx 0:00 nginx: worker process
14 nginx 0:00 nginx: worker process
15 nginx 0:00 nginx: worker process
16 nginx 0:00 nginx: worker process
17 root 0:00 /bin/sh
22 root 0:00 ps -ef
# 查看端口监听
/ # netstat -tnlp
Active Internet connections (only servers)
Proto Recv-Q Send-Q Local Address Foreign Address State PID/Program name
tcp 0 0 0.0.0.0:6443 0.0.0.0:* LISTEN 12/nginx: master pr
# 验证访问
/ # curl -k https://127.0.0.1:6443
{
"kind": "Status",
"apiVersion": "v1",
"metadata": {
},
"status": "Failure",
"message": "Unauthorized",
"reason": "Unauthorized",
"code": 401
}/ #
代理正常有数据返回k8s 使用 ha-tools
# kube-apiserver 节点不部署 ha-tools 只是node 节点部署
# 二进制部署kube-apiserver 证书签名时加上127.0.0.1 这个IP 以后整个集群访问都走127.0.0.1 这个IP+端口 同时kube-apiserver 改成0.0.0.0如果不修改master 安装kubelet 的时候记得修改IP
# kubeadm 安装时 请加入apiserver-cert-extra-sans=127.0.0.1 这样才能127.0.0.1 访问不然会一致报错
# 每个node 节点运行
docker run -tid --network=host --name=ha-proxy -e "CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177" juestnow/ha-tools:v1.17.9 CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177
# 还可以放到kubelet manifests 目录
[[email protected] manifests]# cat ha-tools.yaml
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
labels:
component: ha-tools
tier: control-plane
name: ha-tools
namespace: kube-system
spec:
containers:
- args:
- "CP_HOSTS=192.168.2.175,192.168.2.176,192.168.2.177"
image: juestnow/ha-tools:v1.17.9
imagePullPolicy: IfNotPresent
name: ha-tools
env:
- name: CP_HOSTS
value: "192.168.2.175,192.168.2.176,192.168.2.177"
hostNetwork: true
priorityClassName: system-cluster-critical
status: {}
# 二进制方式部署推荐使用以上的方式
[[email protected] ~]# kubectl get pod -A | grep ha-tools
kube-system ha-tools-nginx-1 1/1 Running 0 14h
# kubeadm 方式部署高可用修改kube-proxy 让它连接127.0.0.1
kubectl -n kube-system edit configmaps kube-proxy
# 二进制部署直接在 kubeconfig 添加就可以
原文地址：https://blog.51cto.com/juestnow/2479933
时间： 2024-07-31 08:47:59