Nginx - Sim Wiki

最后修改人	sim
最后修改时间	2025-09-25 10:10:25
创建人	sim
创建时间	2021-09-27 15:16:24

该文档由 sim 创建于 2021-09-27 15:16:24 ， sim 在 2025-09-25 10:10:25 修改了该文档。

[TOCM]

- **LVS与Nginx集群部署**

|HOSTNAME|物理IP|VIP|
|-|-|-|
|ngx-master1|192.168.2.71|192.168.2.5|
|ngx-master2|192.168.2.72|192.168.2.5|
|ngx-master3|192.168.2.73|192.168.2.5|

##### 准备工作：
```shell
#修改hostname
hostnamectl set-hostname "ngx-master1"

#安装基础软件
yum install -y wget net-tools openssl openssl-devel gcc-c++ pcre pcre-devel ntp

#关闭防火墙
systemctl stop firewalld
systemctl disable firewalld

#关闭selinux
sed -i 's/^SELINUX=.*/SELINUX=disabled/' /etc/sysconfig/selinux
setenforce 0

#同步时间（LVS集群中时间差距不能大于1秒，建议每隔一段时间同步一次）
/usr/sbin/ntpdate -s asia.pool.ntp.org
echo "*/20 * * * *     /usr/sbin/ntpdate -s asia.pool.ntp.org" >> /var/spool/cron/root
```

##### 安装nginx：
```shell
cd /usr/local/src
wget http://nginx.org/download/nginx-1.20.1.tar.gz
tar zxvf nginx-1.20.1.tar.gz
cd nginx-1.20.1
```
修改2个文件改变nginx显示的版本名称，然后再进行编译
```shell
vi src/http/ngx_http_header_filter_module.c
static u_char ngx_http_server_string[] = "Server: SimServer" CRLF;
和
vi src/core/nginx.h
#define NGINX_VERSION      "1.0"
#define NGINX_VER          "SimServer/" NGINX_VERSION
```
编译安装启动nginx：
```shell
./configure --user=nobody --group=nobody --prefix=/usr/local/nginx --with-poll_module --with-threads --with-file-aio --with-http_realip_module --with-stream --with-pcre --with-http_ssl_module
make && make install
echo "master1" > /usr/local/nginx/html/index.html	#修改默认主页内容，方便后面测试
/usr/local/nginx/sbin/nginx
```
nginx探活脚本：
```shell
#!/bin/bash

Count=$(netstat -lntup | egrep ":443|:80" | wc -l)

if [ "$Count" -ge 2 ]; then
    exit 0
else
    /usr/local/nginx/sbin/nginx && Count=$(netstat -lntup | egrep ":443|:80" | wc -l)
    if [ "$Count" -ge 2 ]; then
        exit 0
    else
        systemctl stop keepalived
        exit 1
    fi
fi

exit 0
```
nginx.conf配置文件：
```shell
user nobody;
worker_processes 4;
pid /run/nginx.pid;

events {
    worker_connections 65535;
}

stream {
log_format proxy '$remote_addr [$time_local] '
                 '$protocol $status $bytes_sent $bytes_received '
                 '$session_time "$upstream_addr" '
                 '"$upstream_bytes_sent" "$upstream_bytes_received" "$upstream_connect_time"';
access_log logs/stream-access.log proxy;

upstream ingress_backend {
        least_conn;
        server x.x.x.x:80 max_fails=3 fail_timeout=5s;
        server x.x.x.x:80 max_fails=3 fail_timeout=5s;
        server x.x.x.x:80 max_fails=3 fail_timeout=5s;
        }

upstream ingress_ssl_backend {
        least_conn;
        server x.x.x.x:443 max_fails=3 fail_timeout=5s;
        server x.x.x.x:443 max_fails=3 fail_timeout=5s;
        server x.x.x.x:443 max_fails=3 fail_timeout=5s;
        }

server {
        listen 80;
        proxy_connect_timeout 60s;
        proxy_timeout 600s;
        proxy_protocol on;
        proxy_pass ingress_backend;
        proxy_next_upstream_timeout 600s;
    }

server {
        listen 443;
        ssl_preread on;
        proxy_connect_timeout 60s;
        proxy_timeout 600s;
        proxy_protocol on;
        proxy_pass ingress_ssl_backend;
        proxy_next_upstream_timeout 600s;
    }
}
```
##### 安装LVS：
```shell
yum install -y keepalived
```
编辑LVS配置文件：
```shell
mv /etc/keepalived/keepalived.conf /etc/keepalived/keepalived.conf_bak
vi /etc/keepalived/keepalived.conf
```
```shell
# NAT模式
global_defs {
   router_id LVS_DEVEL
}

vrrp_instance VI_1 {
    state MASTER				 # 在另外2台服务器上配置改为 BACKUP
	nopreempt					# 只在BACKUP节点上生效，不抢占MASTER
    interface ens192			 # 根据服务器实际网卡名称填写
	mcast_src_ip 192.168.2.71	# 本机物理IP地址
    virtual_router_id 51		 # 三机保持一致
    priority 100				 # 分别配置100-102数字越大权限越高
    advert_int 2				 # 心跳监测间隔
    authentication {			 # LVS验证保持一致
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {		  # 虚拟共享IP的地址与绑定的网卡及生成的虚拟网卡
    192.168.2.5/24 dev ens192 label ens192:1
    }
	track_script {
    ngxchk
  }
}
```
```shell
# DR模式
global_defs {
    router_id LVS_INGRESS_NEW 			#修改id
}

vrrp_instance VI_1 {
    state MASTER 						# (主MASTER|备SLAVE)
    interface ens192					 # 根据服务器实际网卡名称填写
    virtual_router_id 84 				# (routeid，每组lvs分开)
    priority 100 						# (值越小权重越低,主100，备90)
    advert_int 1						 # 心跳监测间隔
    authentication {					 # LVS验证保持一致
        auth_type PASS
        auth_pass K8SHA_KA_AUTH
    }
    virtual_ipaddress {  				# 虚拟共享IP的地址与绑定的网卡及生成的虚拟网卡
    192.168.2.5/24 #(VIP地址)
    }
}

virtual_server 192.168.2.5 80 {		  # 高可用VIP，端口，负载算法和协议设置
    delay_loop 3
    lb_algo wlc
    lb_kind DR
    protocol TCP
    
    real_server 192.168.2.5 80 {		# VIP对应real server后端节点IP端口
        weight 1
        TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
            connect_port 80
        }
    }
    
    real_server 192.168.2.5 80 {
        weight 1
        TCP_CHECK {
            connect_timeout 10
            nb_get_retry 3
            delay_before_retry 3
           connect_port 80
        }
    }
}
```
```shell
# DR模式设置arp及lo回包配置脚本
#!/bin/bash

. /etc/rc.d/init.d/functions
VIP=192.168.2.5
host=`/bin/hostname`

case "$1" in
start)
# Start LVS-DR real server
/sbin/ifconfig lo down
/sbin/ifconfig lo up
echo 1 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo 1 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 2 > /proc/sys/net/ipv4/conf/all/arp_announce
/sbin/ifconfig lo:0 $VIP broadcast $VIP netmask 255.255.255.255 up
/sbin/route add -host $VIP dev lo:0
echo "LVS-DR real server Running."
;;
stop)
# Stop LVS-DR real server
/sbin/ifconfig lo:0 down
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/lo/arp_announce
echo 0 > /proc/sys/net/ipv4/conf/all/arp_ignore
echo 0 > /proc/sys/net/ipv4/conf/all/arp_announce
echo "LVS-DR real server Stopped."
;;
status)
# Status of LVS-DR real server.
islothere=`/sbin/ifconfig lo:0 | grep $VIP`
isrothere=`netstat -rn | grep "lo:0" | grep $VIP`
if [ ! "$islothere" -o ! "isrothere" ];then
echo "LVS-DR real server Stopped."
else
echo "LVS-DR real server Running."
fi
;;
*)
# Invalid entry.
echo "$0: Usage: $0 {start|status|stop}"
exit 1
;;
esac

```
```shell
# 启动LVS
systemctl start keepalived
# 加入自启动
systemctl enable keepalived
```
```shell
# 启用rc.local启动文件，在Centos7中默认关闭了rc.local启动文件
chmod +x /etc/rc.d/rc.local && systemctl enable rc-local.service
```

##### 总结几点：
1. Keepalived主要有主备，主主；NAT模式（由LVS节点承担流量进出），DR模式（由后端RS承担流量进出）

1. Keepalived配置中对于主备的切换通过定义priority权重来确定MASTER，权重数越大的优先为MASTER比如priority100和priority150的2台机器在应用都存活的状态下必定是priority150的机器为MASTER，哪怕该机器宕机，由priority100的机器接手MASTER后，只要priority150的机器应用恢复了后就会自动抢占MASTER。所以，如不想使用抢占模式，可以在BACKUP节点上设置好对应顺序的priority值，并加上nopreempt（只在BACKUP上生效，所以为了完全不抢占可以将所有节点都设置成MASTER）参数来关闭抢占模式。