当前位置：首页 > news >正文

keepalived实现nginx负载均衡机高可用

news 来源：原创 2024/4/28 8:41:47

系统配置

系统	主机名	IP	服务
centos8	master	192.168.48.151	nginx keepalived
centos8	backup	192.168.48.152	nginx keepalived
centos8	apache	192.168.48.153	httpd
centos8	nginx	192.168.48.154	nginx

配置web界面

apache

//修改名字
[root@localhost ~]# hostnamectl set-hostname apache
[root@localhost ~]# bash
[root@apache ~]# 

//关闭防火墙和selinux
[root@apache ~]# setenforce 0
[root@apache ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
[root@apache ~]# systemctl disable --now firewalld
[root@apache ~]# reboot

//配置yum源
[root@apache ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo
[root@apache ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo

//安装apache,主页内容为apache
[root@apache ~]# yum -y install httpd
[root@apache ~]# echo "apache" > /var/www/html/index.html
[root@apache ~]# systemctl enable --now httpd
[root@apache ~]# ss -anlt
State        Recv-Q        Send-Q               Local Address:Port               Peer Address:Port       Process       
LISTEN       0             128                        0.0.0.0:22                      0.0.0.0:*                        
LISTEN       0             128                              *:80                            *:*                        
LISTEN       0             128                           [::]:22                         [::]:*                        
[root@apache ~]# curl 192.168.48.153
apache

nginx

//修改名字
[root@localhost ~]# hostnamectl set-hostname nginx
[root@localhost ~]# bash
[root@nginx ~]# 

//关闭防火墙和selinux
[root@nginx ~]# setenforce 0
[root@nginx ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
[root@nginx ~]# systemctl disable --now firewalld
[root@nginx ~]# reboot

//配置yum源
[root@nginx ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo
[root@nginx ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo

//安装nginx,主页内容为nginx
[root@nginx ~]# yum -y install nginx
[root@nginx ~]# echo "nginx" > /usr/share/nginx/html/index.html
[root@nginx ~]# systemctl enable --now nginx
[root@nginx ~]# ss -anlt
State        Recv-Q        Send-Q               Local Address:Port               Peer Address:Port       Process       
LISTEN       0             128                        0.0.0.0:80                      0.0.0.0:*                        
LISTEN       0             128                        0.0.0.0:22                      0.0.0.0:*                        
LISTEN       0             128                           [::]:80                         [::]:*                        
LISTEN       0             128                           [::]:22                         [::]:*                        
[root@nginx ~]# curl 192.168.48.154
nginx

配置nginx负载均衡

master

//修改名字
[root@localhost ~]# hostnamectl set-hostname master
[root@localhost ~]# bash
[root@master ~]# 

//关闭防火墙和selinux
[root@master ~]# setenforce 0
[root@master ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
[root@master ~]# systemctl disable --now firewalld
[root@master ~]# reboot

//配置yum源
[root@master ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo
[root@master ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo

//创建用户
[root@master ~]# useradd -rMs /sbin/nologin nginx

//安装依赖包
[root@master ~]# dnf -y install pcre-devel openssl openssl-devel gd-devel gcc gcc-c++ make wget vim

//编译安装nginx
[root@backup ~]# wget http://nginx.org/download/nginx-1.22.0.tar.gz
[root@master ~]# tar xf nginx-1.22.0.tar.gz 
[root@master ~]# cd nginx-1.22.0
[root@master nginx-1.22.0]# ./configure \
 --prefix=/usr/local/nginx \
 --user=nginx \
 --group=nginx \
 --with-debug \
 --with-http_ssl_module \
 --with-http_realip_module \
 --with-http_gunzip_module \
 --with-http_gzip_static_module \
 --with-http_stub_status_module
 [root@master nginx-1.22.0]# make -j $(grep 'processor' /proc/cpuinfo | wc -l) && make install

//配置环境变量
[root@master ~]# echo "export PATH=$PATH:/usr/local/nginx/sbin" > /etc/profile.d/nginx.sh
[root@master ~]# source /etc/profile.d/nginx.sh
                
//配置system启动服务
[root@master ~]# vim /usr/lib/systemd/system/nginx.service
[Unit]
Description=nginx server daemon
After=network.target 
 
[Service]
Type=forking
ExecStart=/usr/local/nginx/sbin/nginx
ExecStop=/usr/local/nginx/sbin/nginx -s stop
ExecReload=/bin/kill -HUP \$MAINPID
 
[Install]
WantedBy=multi-user.target
[root@nginx ~]# systemctl daemon-reload

//启动nginx
[root@master ~]# systemctl enable --now nginx
[root@master ~]# ss -anlt
State        Recv-Q       Send-Q               Local Address:Port               Peer Address:Port       Process       
LISTEN       0            128                        0.0.0.0:80                      0.0.0.0:*                        
LISTEN       0            128                        0.0.0.0:22                      0.0.0.0:*                        
LISTEN       0            128                           [::]:22                         [::]:*               

//修改配置文件,配置负载均衡
[root@master ~]# vim /usr/local/nginx/conf/nginx.conf
    upstream web {				//http字段内
    server 192.168.48.153;
    server 192.168.48.154;
    }
    .........
        location / {				//修改配置文件
        root html;
        proxy_pass   http://web;
        }
[root@master ~]# systemctl restart nginx

//查看负载均衡效果
[root@master ~]# curl 192.168.48.151
apache
[root@master ~]# curl 192.168.48.151
nginx
[root@master ~]# curl 192.168.48.151
apache
[root@master ~]# curl 192.168.48.151
nginx

backup

//修改名字
[root@localhost ~]# hostnamectl set-hostname backup
[root@localhost ~]# bash
[root@backup ~]# 

//关闭防火墙和selinux
[root@backup ~]# setenforce 0
[root@backup ~]# sed -ri 's/^(SELINUX=).*/\1disabled/g' /etc/selinux/config
[root@backup ~]# systemctl disable --now firewalld
[root@backup ~]# reboot

//配置yum源
[root@backup ~]# curl -o /etc/yum.repos.d/CentOS-Base.repo https://mirrors.aliyun.com/repo/Centos-vault-8.5.2111.repo
[root@backup ~]# sed -i -e '/mirrors.cloud.aliyuncs.com/d' -e '/mirrors.aliyuncs.com/d' /etc/yum.repos.d/CentOS-Base.repo

//创建用户
[root@backup ~]# useradd -rMs /sbin/nologin nginx

//安装依赖包
[root@backup ~]# dnf -y install pcre-devel openssl openssl-devel gd-devel gcc gcc-c++ make wget vim

//编译安装nginx
[root@backup ~]# wget http://nginx.org/download/nginx-1.22.0.tar.gz
[root@backup ~]# tar xf nginx-1.22.0.tar.gz 
[root@backup ~]# cd nginx-1.22.0
[root@backup nginx-1.22.0]# ./configure \
 --prefix=/usr/local/nginx \
 --user=nginx \
 --group=nginx \
 --with-debug \
 --with-http_ssl_module \
 --with-http_realip_module \
 --with-http_gunzip_module \
 --with-http_gzip_static_module \
 --with-http_stub_status_module
[root@backup nginx-1.22.0]# make -j $(grep 'processor' /proc/cpuinfo | wc -l) && make install

//配置环境变量
[root@backup ~]# echo "export PATH=$PATH:/usr/local/nginx/sbin" > /etc/profile.d/nginx.sh
[root@backup ~]# source /etc/profile.d/nginx.sh
                
//配置system启动服务
[root@backup ~]# vim /usr/lib/systemd/system/nginx.service
[Unit]
Description=nginx server daemon
After=network.target 
 
[Service]
Type=forking
ExecStart=/usr/local/nginx/sbin/nginx
ExecStop=/usr/local/nginx/sbin/nginx -s stop
ExecReload=/bin/kill -HUP \$MAINPID
 
[Install]
WantedBy=multi-user.target
[root@backup ~]# systemctl daemon-reload

//启动nginx
[root@backup ~]# systemctl enable --now nginx
[root@backup ~]# ss -anlt
State        Recv-Q        Send-Q               Local Address:Port               Peer Address:Port       Process       
LISTEN       0             128                        0.0.0.0:80                      0.0.0.0:*                        
LISTEN       0             128                        0.0.0.0:22                      0.0.0.0:*                        
LISTEN       0             128                           [::]:22                         [::]:*                        

//修改配置文件,配置负载均衡
[root@backup ~]# vim /usr/local/nginx/conf/nginx.conf
    upstream web {				//http字段内
    server 192.168.48.153;
    server 192.168.48.154;
    }
    .........
        location / {				//修改配置文件
        root html;
        proxy_pass   http://web;
        }
[root@backup ~]# systemctl restart nginx

//查看负载均衡效果
[root@backup ~]# curl 192.168.48.152
apache
[root@backup ~]# curl 192.168.48.152
nginx
[root@backup ~]# curl 192.168.48.152
apache
[root@backup ~]# curl 192.168.48.152
nginx

配置keepalived高可用

master

//首先安装keepalived
[root@master ~]# yum -y install keepalived
 
//编辑配置文件，并启动服务
[root@master ~]# mv /etc/keepalived/keepalived.conf{,.bak}
[root@master ~]# vim /etc/keepalived/keepalived.conf 
! Configuration File for keepalived
 
global_defs {
   router_id lb01
}
 
vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.111.250
    }
}
 
virtual_server 192.168.111.250 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP
 
    real_server 192.168.48.151 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
 
    real_server 192.168.48.152 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}
[root@master ~]# systemctl enable --now keepalived

//通过虚拟IP访问
[root@master ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:50:34:72 brd ff:ff:ff:ff:ff:ff
    inet 192.168.48.151/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33
       valid_lft 1500sec preferred_lft 1500sec
    inet 192.168.111.250/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe50:3472/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
 
//通过虚拟ip访问
//一定要把backup端的nginx服务关掉才能访问，

[root@master ~]# curl 192.168.111.250
apache
[root@master ~]# curl 192.168.111.250
nginx
[root@master ~]# curl 192.168.111.250
apache
[root@master ~]# curl 192.168.111.250
nginx

backup

//首先安装keepalived
[root@backup ~]# dnf -y install keepalived
 
//编辑配置文件，并启动服务
[root@backup ~]# mv /etc/keepalived/keepalived.conf{,.back}
[root@backup ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
 
global_defs {
   router_id lb02
}
 
vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 50
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.111.250
    }
}
 
virtual_server 192.168.111.250 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP
 
    real_server 192.168.48.151 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
 
    real_server 192.168.48.152 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}
[root@backup ~]# systemctl enable --now keepalived

编写脚本

master

[root@master ~]# mkdir /scripts
[root@master ~]# cd /scripts/
[root@master scripts]# vim check_nginx.sh
#!/bin/bash
nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)
if [ $nginx_status -lt 1 ];then
    systemctl stop keepalived
fi

[root@master scripts]# vim notify.sh
#!/bin/bash
VIP=$2
case "$1" in
  master)
        nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)
        if [ $nginx_status -lt 1 ];then
            systemctl start nginx
        fi
  ;;
  backup)
        nginx_status=$(ps -ef|grep -Ev "grep|$0"|grep '\bnginx\b'|wc -l)
        if [ $nginx_status -gt 0 ];then
            systemctl stop nginx
        fi
  ;;
  *)
        echo "Usage:$0 master|backup VIP"
  ;;
esac
[root@master scripts]# chmod +x check_haproxy.sh notify.sh

backup

[root@backup ~]# mkdir /scripts
[root@backup ~]# cd /scripts/
[root@backup scripts]# scp root@192.168.48.151:/scripts/notify.sh .

配置keepalived加入监控脚本的配置

master

[root@master ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
 
global_defs {
   router_id lb01
}
 
vrrp_script nginx_check {				//添加
    script "/scripts/check_nginx.sh"
    interval 1
    weight -20
}

vrrp_instance VI_1 {
    state MASTER
    interface ens33
    virtual_router_id 51
    priority 100
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.111.250
    }
    track_script {			//添加
        haproxy_check
    }
    notify_master "/scripts/notify.sh master 192.168.111.250"
}
 
virtual_server 192.168.111.250 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP
 
    real_server 192.168.48.151 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
 
    real_server 192.168.48.152 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}
[root@master scripts]# systemctl restart keepalived

backup

[root@backup ~]# vim /etc/keepalived/keepalived.conf
! Configuration File for keepalived
 
global_defs {
   router_id lb02
}
 
vrrp_instance VI_1 {
    state BACKUP
    interface ens33
    virtual_router_id 50
    priority 90
    advert_int 1
    authentication {
        auth_type PASS
        auth_pass 123456
    }
    virtual_ipaddress {
        192.168.111.250
    }
    notify_master "/scripts/notify.sh master 192.168.111.250"		//添加
    notify_backup "/scripts/notify.sh backup 192.168.111.250"
}
 
virtual_server 192.168.111.250 80 {
    delay_loop 6
    lb_algo rr
    lb_kind DR
    persistence_timeout 50
    protocol TCP
 
    real_server 192.168.48.151 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
 
    real_server 192.168.48.152 80 {
        weight 1
        TCP_CHECK {
            connect_port 80
            connect_timeout 3
            nb_get_retry 3
            delay_before_retry 3
        }
    }
}
[root@backup ~]# systemctl restart keepalived

测试

模拟nginx服务故障

//master端
[root@master ~]# curl 192.168.111.250
apache
[root@master ~]# curl 192.168.111.250
nginx
[root@master ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:50:34:72 brd ff:ff:ff:ff:ff:ff
    inet 192.168.48.151/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33
       valid_lft 1601sec preferred_lft 1601sec
    inet 192.168.111.250/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe50:3472/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@master ~]# systemctl stop nginx
[root@master ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:50:34:72 brd ff:ff:ff:ff:ff:ff
    inet 192.168.48.151/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33
       valid_lft 1591sec preferred_lft 1591sec
    inet6 fe80::20c:29ff:fe50:3472/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

//backup端
[root@backup ~]# systemctl start nginx    //前面把服务关了这里启动一下
[root@backup ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:07:42:65 brd ff:ff:ff:ff:ff:ff
    inet 192.168.48.152/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33
       valid_lft 947sec preferred_lft 947sec
    inet 192.168.111.250/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe07:4265/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever
[root@backup ~]# curl 192.168.111.250
apache
[root@backup ~]# curl 192.168.111.250
nginx
[root@backup ~]# curl 192.168.111.250
apache
[root@backup ~]# curl 192.168.111.250
nginx

启动master端的nginx服务

//master端
[root@master scripts]# systemctl start nginx
[root@master scripts]# systemctl restart keepalived
[root@master ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:50:34:72 brd ff:ff:ff:ff:ff:ff
    inet 192.168.48.151/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33
       valid_lft 1223sec preferred_lft 1223sec
    inet 192.168.111.250/32 scope global ens33
       valid_lft forever preferred_lft forever
    inet6 fe80::20c:29ff:fe50:3472/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever

//backup端
[root@backup ~]# ip a
1: lo: <LOOPBACK,UP,LOWER_UP> mtu 65536 qdisc noqueue state UNKNOWN group default qlen 1000
    link/loopback 00:00:00:00:00:00 brd 00:00:00:00:00:00
    inet 127.0.0.1/8 scope host lo
       valid_lft forever preferred_lft forever
    inet6 ::1/128 scope host 
       valid_lft forever preferred_lft forever
2: ens33: <BROADCAST,MULTICAST,UP,LOWER_UP> mtu 1500 qdisc fq_codel state UP group default qlen 1000
    link/ether 00:0c:29:07:42:65 brd ff:ff:ff:ff:ff:ff
    inet 192.168.48.152/24 brd 192.168.111.255 scope global dynamic noprefixroute ens33
       valid_lft 1632sec preferred_lft 1632sec
    inet6 fe80::20c:29ff:fe07:4265/64 scope link noprefixroute 
       valid_lft forever preferred_lft forever