Nginx - Sim Wiki

最后修改人	sim
最后修改时间	2026-06-02 10:01:46
创建人	sim
创建时间	2021-09-27 15:16:24

该文档由 sim 创建于 2021-09-27 15:16:24 ， sim 在 2026-06-02 10:01:46 修改了该文档。

[TOCM]

- **LVS与Nginx集群部署**

|HOSTNAME|物理IP|VIP|
|-|-|-|
|ngx-master1|192.168.2.71|192.168.2.5|
|ngx-master2|192.168.2.72|192.168.2.5|
|ngx-master3|192.168.2.73|192.168.2.5|

##### 准备工作：
```shell
#修改hostname
hostnamectl set-hostname "ngx-master1"

#安装基础软件
yum install -y wget net-tools openssl openssl-devel gcc-c++ pcre pcre-devel ntp

#关闭防火墙
systemctl stop firewalld
systemctl disable firewalld

#关闭selinux
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/sysconfig/selinux
setenforce 0

#同步时间（LVS集群中时间差距不能大于1秒，建议每隔一段时间同步一次）
/usr/sbin/ntpdate -s asia.pool.ntp.org
echo "*/20 * * * *     /usr/sbin/ntpdate -s asia.pool.ntp.org" >> /var/spool/cron/root
```

##### 安装nginx：
```shell
cd /usr/local/src
wget http://nginx.org/download/nginx-1.20.1.tar.gz
tar zxvf nginx-1.20.1.tar.gz
cd nginx-1.20.1
```
修改2个文件改变nginx显示的版本名称，然后再进行编译
```shell
vi src/http/ngx_http_header_filter_module.c
static u_char ngx_http_server_string[] = "Server: SimServer" CRLF;
和
vi src/core/nginx.h
#define NGINX_VERSION      "1.0"
#define NGINX_VER          "SimServer/" NGINX_VERSION
```
编译安装启动nginx：
```shell
./configure --user=nobody --group=nobody --prefix=/usr/local/nginx --with-poll_module --with-threads --with-file-aio --with-http_realip_module --with-stream --with-pcre --with-http_ssl_module
make && make install
echo "master1" > /usr/local/nginx/html/index.html	#修改默认主页内容，方便后面测试
/usr/local/nginx/sbin/nginx
```
nginx探活脚本：
```shell
#!/bin/bash

Count=$(netstat -lntup | egrep ":443|:80" | wc -l)

if [ "$Count" -ge 2 ]; then
    exit 0
else
    /usr/local/nginx/sbin/nginx && Count=$(netstat -lntup | egrep ":443|:80" | wc -l)
    if [ "$Count" -ge 2 ]; then
        exit 0
    else
        systemctl stop keepalived
        exit 1
    fi
fi

exit 0
```
nginx.conf配置文件：
```shell
user nobody;
worker_processes 4;
pid /run/nginx.pid;

events {
    worker_connections 65535;
}

stream {
log_format proxy '$remote_addr [$time_local] '
                 '$protocol $status $bytes_sent $bytes_received '
                 '$session_time "$upstream_addr" '
                 '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
access_log logs/stream-access.log proxy;

upstream ingress_backend {
        least_conn;
        server x.x.x.x:80 max_fails=3 fail_timeout=5s;
        server x.x.x.x:80 max_fails=3 fail_timeout=5s;
        server x.x.x.x:80 max_fails=3 fail_timeout=5s;
        }

upstream ingress_ssl_backend {
        least_conn;
        server x.x.x.x:443 max_fails=3 fail_timeout=5s;
        server x.x.x.x:443 max_fails=3 fail_timeout=5s;
        server x.x.x.x:443 max_fails=3 fail_timeout=5s;
        }

server {
        listen 80;
        proxy_connect_timeout 60s;
        proxy_timeout 600s;
        proxy_protocol on;
        proxy_pass ingress_backend;
        proxy_next_upstream_timeout 600s;
    }

server {
        listen 443;
        ssl_preread on;
        proxy_connect_timeout 60s;
        proxy_timeout 600s;
        proxy_protocol on;
        proxy_pass ingress_ssl_backend;
        proxy_next_upstream_timeout 600s;
    }
}
```
##### 安装LVS：
```shell
yum install -y keepalived
```
编辑LVS配置文件：
```shell
mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_bak
vi /etc/keepalived/keepalived.conf
```
```shell
# NAT模式
global_defs {
   router_id LVS_DEVEL
}

vrrp_instance VI_1 {
    state MASTER				 # 在另外2台服务器上配置改为 BACKUP
	nopreempt					# 只在BACKUP节点上生效，不抢占MASTER
    interface ens192			 # 根据服务器实际网卡名称填写
	mcast_src_ip 192.168.2.71	# 本机物理IP地址
    virtual_router_id 51		 # 三机保持一致
    priority 100				 # 分别配置100-102数字越大权限越高
    advert_int 2				 # 心跳监测间隔
    authentication {			 # LVS验证保持一致
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {		  # 虚拟共享IP的地址与绑定的网卡及生成的虚拟网卡
    192.168.2.5/24 dev ens192 label ens192:1
    }
	track_script {
    ngxchk
  }
}
```
```shell
# DR模式 该模式下Keepalived服务和Read节点服务，如Nginx等，不可以在同一个系统中，会出现交叉转发的死循环。
global_defs {
    router_id LVS_INGRESS_NEW 			#修改id
}

vrrp_instance VI_1 {
    state MASTER 						# (主MASTER|备SLAVE)
    interface ens192					 # 根据服务器实际网卡名称填写
    virtual_router_id 84 				# (routeid，每组lvs分开)
    priority 100 						# (值越小权重越低,主100，备90)
    advert_int 1						 # 心跳监测间隔
    authentication {					 # LVS验证保持一致
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {  				# 虚拟共享IP的地址与绑定的网卡及生成的虚拟网卡
    192.168.2.5/24 #(VIP地址)
    }
}

virtual_server 192.168.2.5 80 {		  # 高可用VIP，端口，负载算法和协议设置
    delay_loop 3
    lb_algo wlc
    lb_kind DR
    protocol TCP
    
    real_server 192.168.2.5 80 {		# VIP对应real server后端节点IP端口
        weight 1
        TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
        }
    }
    
    real_server 192.168.2.5 80 {
        weight 1
        TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
           connect_port 80
        }
    }
}
```
```shell
# DR模式设置arp及lo回包配置脚本（在Real节点，如Nginx上执行，不要在Keepalived节点执行）
#!/bin/bash

. /etc/rc.d/init.d/functions
VIP=192.168.2.5
host=`/bin/hostname`

case "$1" in
start)
# Start LVS-DR real server
/sbin/ifconfig lo down
/sbin/ifconfig lo up
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
/sbin/ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
/sbin/route add -host $VIP dev lo:0
echo "LVS-DR real server Running."
;;
stop)
# Stop LVS-DR real server
/sbin/ifconfig lo:0 down
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
echo "LVS-DR real server Stopped."
;;
status)
# Status of LVS-DR real server.
islothere=`/sbin/ifconfig lo:0 | grep $VIP`
isrothere=`netstat -rn | grep "lo:0" | grep $VIP`
if [ ! "$islothere" -o ! "isrothere" ];then
echo "LVS-DR real server Stopped."
else
echo "LVS-DR real server Running."
fi
;;
*)
# Invalid entry.
echo "$0: Usage: $0 {start|status|stop}"
exit 1
;;
esac

```
```shell
# 启动LVS
systemctl start keepalived
# 加入自启动
systemctl enable keepalived
```
```shell
# 启用rc.local启动文件，在Centos7中默认关闭了rc.local启动文件
chmod +x /etc/rc.d/rc.local && systemctl enable rc-local.service
```

##### 总结几点：
1. Keepalived主要有主备（只启用keepalived负责切换节点承担流量进出，不使用ipvs模块），DR模式（通过ipvs模块由后端RS承担流量进出，DR模式下Keepalived不可以和Real节点，如nginx等在一个服务器上）

1. Keepalived配置中对于主备的切换通过定义priority权重来确定MASTER，权重数越大的优先为MASTER比如priority100和priority150的2台机器在应用都存活的状态下必定是priority150的机器为MASTER，哪怕该机器宕机，由priority100的机器接手MASTER后，只要priority150的机器应用恢复了后就会自动抢占MASTER。所以，如不想使用抢占模式，可以在BACKUP节点上设置好对应顺序的priority值，并加上nopreempt（只在BACKUP上生效，所以为了完全不抢占可以将所有节点都设置成MASTER）参数来关闭抢占模式。

#### 自动化脚本
```shell
#!/bin/bash
# LVS自动部署脚本（主备模式），默认关闭防火墙，Keepalived和Nginx使用yum安装，保持网络通畅。
# By Sim

set -e

echo "========================"
echo "Keepalived + Nginx 安装配置脚本"
echo "========================"

# ===== 基础信息输入 =====
read -p "请输入 router_id (例如 LVS_INGRESS_OSS): " ROUTER_ID
read -p "请输入角色 (MASTER/SLAVE): " STATE
read -p "请输入网卡名称 (例如 ens192): " INTERFACE
read -p "请输入本机IP (例如 10.7.70.251): " LOCAL_IP
read -p "请输入 VIP (例如 10.7.70.13): " VIP
read -p "请输入 virtual_router_id (例如 87): " VRID
read -p "请输入 priority (主100，备90): " PRIORITY

# proxy_protocol
while true; do
    read -p "是否开启 proxy_protocol? (on/off): " PROXY_PROTOCOL
    [[ "$PROXY_PROTOCOL" == "on" || "$PROXY_PROTOCOL" == "off" ]] && break
    echo "请输入 on 或 off"
done

# ===== HTTP upstream =====
echo "请输入 HTTP 后端节点（格式 IP:PORT），输入完成后直接回车结束："
HTTP_SERVERS=()
while true; do
    read -p "HTTP节点: " SERVER
    [[ -z "$SERVER" ]] && break
    HTTP_SERVERS+=("$SERVER")
done

# ===== HTTPS upstream =====
echo "请输入 HTTPS 后端节点（格式 IP:PORT），输入完成后直接回车结束："
HTTPS_SERVERS=()
while true; do
    read -p "HTTPS节点: " SERVER
    [[ -z "$SERVER" ]] && break
    HTTPS_SERVERS+=("$SERVER")
done

# ===== 生成 upstream 配置 =====
HTTP_UPSTREAM=""
for s in "${HTTP_SERVERS[@]}"; do
    HTTP_UPSTREAM="${HTTP_UPSTREAM}        server ${s};\n"
done

HTTPS_UPSTREAM=""
for s in "${HTTPS_SERVERS[@]}"; do
    HTTPS_UPSTREAM="${HTTPS_UPSTREAM}        server ${s};\n"
done

# ===== 关闭防火墙 =====
echo "关闭 firewalld..."
systemctl stop firewalld || true
systemctl disable firewalld || true

# ===== 安装 keepalived 和 nginx =====
echo "安装 keepalived 和 nginx..."
yum install -y keepalived nginx

# ===== 执行 nginx 替换脚本 =====
echo "执行 nginx 替换脚本..."
if ! curl -sL http://10.9.25.25:88/nginx/sh/replace_nginx.sh | bash; then
    echo "ERROR: replace_nginx.sh 执行失败，脚本退出"
    exit 1
fi

# ===== 配置 nginx systemd 自动拉起 =====
echo "配置 nginx systemd 自动拉起..."
mkdir -p /etc/systemd/system/nginx.service.d

cat > /etc/systemd/system/nginx.service.d/restart.conf <<EOF
[Service]
Restart=always
RestartSec=1
EOF

if [ $? -ne 0 ]; then
    echo "ERROR: nginx systemd override 配置失败，脚本退出"
    exit 1
fi

systemctl daemon-reload

# ===== nginx 配置 =====
echo "生成 nginx 配置..."
cat > /etc/nginx/nginx.conf <<EOF
user nobody;
worker_processes auto;
pid /run/nginx.pid;

events {
    worker_connections 65535;
}

stream {
    log_format proxy '\$remote_addr [\$time_local] '
                     '\$protocol \$status \$bytes_sent \$bytes_received '
                     '\$session_time "\$upstream_addr" '
                     '"\$upstream_bytes_sent" "\$upstream_bytes_received" "\$upstream_connect_time"';

access_log logs/stream-access.log proxy;

upstream ingress_backend {
        least_conn;
$(echo -e "$HTTP_UPSTREAM")
        check interval=3000 rise=2 fall=3 timeout=3000 type=tcp;
    }

upstream ingress_ssl_backend {
        least_conn;
$(echo -e "$HTTPS_UPSTREAM")
        check interval=3000 rise=2 fall=3 timeout=3000 type=tcp;
    }

server {
        listen 80;
        proxy_connect_timeout 60s;
        proxy_timeout 600s;
        proxy_protocol ${PROXY_PROTOCOL};
        proxy_pass ingress_backend;
        proxy_next_upstream_timeout 600s;
    }

server {
        listen 443;
        ssl_preread on;
        proxy_connect_timeout 60s;
        proxy_timeout 600s;
        proxy_protocol ${PROXY_PROTOCOL};
        proxy_pass ingress_ssl_backend;
        proxy_next_upstream_timeout 600s;
    }
}

http {
    vhost_traffic_status_zone shared:vhost_traffic_status:10m;

include mime.types;
    default_type application/octet-stream;
    access_log off;
    sendfile on;
    keepalive_timeout 65;
    gzip on;

server {
        listen 9913;
        server_name localhost;

location /metrics {
            vhost_traffic_status_display;
            vhost_traffic_status_display_format prometheus;
        }

location /upstream_status {
            healthcheck_status prometheus;
        }
    }
}
EOF

# ===== 创建 nginx 健康检查脚本 =====
echo "创建 /etc/keepalived/chk_ngx.sh ..."

cat > /etc/keepalived/chk_ngx.sh <<EOF
#!/bin/bash
systemctl is-active nginx >/dev/null 2>&1
EOF

if [ $? -ne 0 ]; then
    echo "ERROR: 创建 chk_ngx.sh 失败，脚本退出"
    exit 1
fi

chmod +x /etc/keepalived/chk_ngx.sh

if [ $? -ne 0 ]; then
    echo "ERROR: chk_ngx.sh 赋予执行权限失败，脚本退出"
    exit 1
fi

# ===== keepalived 配置 =====
echo "生成 keepalived 配置..."
cat > /etc/keepalived/keepalived.conf <<EOF
global_defs {
    router_id ${ROUTER_ID}
    script_user root
}

vrrp_script ngxchk {
    script "/etc/keepalived/chk_ngx.sh"
    interval 1
    fall 3
    rise 2
}

vrrp_instance VI_1 {
    state ${STATE}
    interface ${INTERFACE}
    mcast_src_ip ${LOCAL_IP}
    virtual_router_id ${VRID}
    priority ${PRIORITY}
    advert_int 1

authentication {
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }

virtual_ipaddress {
        ${VIP}/24 dev ${INTERFACE} label ${INTERFACE}:1
    }

track_script {
        ngxchk
    }
}
EOF

# ===== 检查配置 =====
echo "检查 nginx 配置..."
if ! nginx -t; then
    echo "ERROR: nginx 配置检查失败，脚本退出"
    exit 1
fi

echo "检查 keepalived 配置..."
if ! keepalived -t; then
    echo "ERROR: keepalived 配置检查失败，脚本退出"
    exit 1
fi

# ===== 设置开机自启并启动服务 =====
echo "设置 nginx 开机自启..."
if ! systemctl enable nginx; then
    echo "ERROR: nginx 设置开机自启失败，脚本退出"
    exit 1
fi

echo "启动 nginx..."
if ! systemctl restart nginx; then
    echo "ERROR: nginx 启动失败，脚本退出"
    exit 1
fi

echo "设置 keepalived 开机自启..."
if ! systemctl enable keepalived; then
    echo "ERROR: keepalived 设置开机自启失败，脚本退出"
    exit 1
fi

echo "启动 keepalived..."
if ! systemctl restart keepalived; then
    echo "ERROR: keepalived 启动失败，脚本退出"
    exit 1
fi

echo "配置 nginx stream-access.log 每天 00:00 清空..."

CRON_JOB='00 00 * * * cat /dev/null > /etc/nginx/logs/stream-access.log'

if ! (crontab -l 2>/dev/null | grep -F "$CRON_JOB") >/dev/null; then
    (crontab -l 2>/dev/null; echo "$CRON_JOB") | crontab -
fi

if [ $? -ne 0 ]; then
    echo "ERROR: 配置 cron 定时任务失败，脚本退出"
    exit 1
fi

echo "========================"
echo "部署完成"
echo "VIP: ${VIP}"
echo "角色: ${STATE}"
echo "nginx / keepalived 已设置开机自启"
echo "cron 定时任务配置完成"
echo "========================"

exit 0
```