1、实验环境的搭建
准备3台主机：
主机1
主机名：haproxy
ip：172.25.254.100
主机2
主机名：hp-RS1
ip：172.25.254.10
主机3
主机名：hp-RS2
ip：172.25.254.20
两台RS都安装nginx：
[haproxy-RS1]# dnf install nginx -y
[haproxy-RS2]# dnf install nginx -y
设置开机自启动（两个都设置）
[haproxy-RS1]systemctl enable --now nginx.service
关闭火墙（两个都设置）
[haproxy-RS1]systemctl disable --now firewalld.service
两台RS设置nginx的index.html内容
[haproxy-RS1]echo "RS1-172.25.254.10" > /usr/share/nginx/html/index.html
连通测试
[haproxy-haproxy]curl 172.25.254.10
2、haproxy的安装和frontend区
安装haproxy
[haproxy-haproxy]dnf install haproxy -y
启动
[haproxy-haproxy]systemctl enable --now haproxy.service
进入haproxy配置文件
[haproxy-haproxy]vim /etc/haproxy/haproxy.cfg
frontend webcluster
bind *:80
mode http
balance roundrobinuse
backend webserver
backend webserver
server webl 172.25.254.10:80
server web2 172.25.254.20:80
重启服务
[haproxy-haproxy]systemctl restart haproxy.service
3、global配置（多进程与多线程）
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
global
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 100000
user haproxy
group haproxy
daemon
# turn on stats unix socket
stats socket /var/lib/haproxy/haproxy.sock1 mode 600 level admin process 1 #
启用多个sock文件
stats socket /var/lib/haproxy/haproxy.sock2 mode 600 level admin process 2
nbproc 2 #启用多进程
cpu-map 1 0 #进程和cpu核心绑定防止cpu抖动从而减少系统资源消耗
cpu-map 2 1 #2 表示第二个进程，1表示第二个cpu核心
...下面内容省略 ...
查看多进程信息
haproxy haproxy]# pstree -p | grep haproxy
|-haproxy(4816)-+-haproxy(4820)
| `-haproxy(4821)
启用多线程
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
log 127.0.0.1 local2
chroot /var/lib/haproxy
pidfile /var/run/haproxy.pid
maxconn 100000
user haproxy
group haproxy
daemon
# turn on stats unix socket
stats socket /var/lib/haproxy/haproxy.sock1 mode 600 level admin process 1 #
启用多个sock文件
stats socket /var/lib/haproxy/haproxy.sock2 mode 600 level admin process 2
#nbproc 2
#cpu-map 1 0
#cpu-map 2 1
nbthread 2 #启用多线程
...下面内容省略...
多线程对比
未开启多线程
haproxy ~]# cat /proc/xxxx(haproxy子进程id)/status
...上面内容省略...
Threads: 1
...下面内容省略...
开启后
haproxy ~]# cat /proc/xxxx(haproxy子进程id)/status
...上面内容省略...
Threads: 2
...下面内容省略...
4、socat工具
#修改配置文件
[root@haproxy ~]# vim /etc/haproxy/haproxy.cfg
stats socket /var/lib/haproxy/stats mode 600 level admin
#查看haproxy状态
[root@haproxy ~]# echo "show info" | socat stdio /var/lib/haproxy/stats
Name: HAProxy
Version: 2.4.22-f8e3218
Release_date: 2023/02/14
Nbthread: 1
Nbproc: 1
Process_num: 1
Pid: 33542
Uptime: 0d 0h03m43s
Uptime_sec: 223
Memmax_MB: 0
PoolAlloc_MB: 0
#查看集群状态
[root@haproxy ~]# echo "show servers state" | socat stdio /var/lib/haproxy/stats
1
# be_id be_name srv_id srv_name srv_addr srv_op_state srv_admin_state srv_uweight
srv_iweight srv_time_since_last_change srv_check_status srv_check_result
srv_check_health srv_check_state srv_agent_state bk_f_forced_id srv_f_forced_id
srv_fqdn srv_port srvrecord srv_use_ssl srv_check_port srv_check_addr
srv_agent_addr srv_agent_port
2 webcluster 1 web1 172.25.254.20 2 0 2 2 188 6 3 7 6 0 0 0 - 80 - 0 0 - - 0
2 webcluster 2 web2 172.25.254.30 2 0 1 1 188 6 3 7 6 0 0 0 - 80 - 0 0 - - 0
4 static 1 static 127.0.0.1 0 0 1 1 187 8 2 0 6 0 0 0 - 4331 - 0 0 - - 0
5 app 1 app1 127.0.0.1 0 0 1 1 187 8 2 0 6 0 0 0 - 5001 - 0 0 - - 0
5 app 2 app2 127.0.0.1 0 0 1 1 187 8 2 0 6 0 0 0 - 5002 - 0 0 - - 0
5 app 3 app3 127.0.0.1 0 0 1 1 186 8 2 0 6 0 0 0 - 5003 - 0 0 - - 0
5 app 4 app4 127.0.0.1 0 0 1 1 186 8 2 0 6 0 0 0 - 5004 - 0 0 - - 0
#查看集群权重
[root@haproxy ~]# echo get weight webcluster/web1 | socat stdio
/var/lib/haproxy/stats
2 (initial 2)
[root@haproxy ~]# echo get weight webcluster/web2 | socat stdio
/var/lib/haproxy/stats
1 (initial 1)#设置权重
[root@haproxy ~]# echo "set weight webcluster/web1 1 " | socat stdio
/var/lib/haproxy/stats
[root@haproxy ~]# echo "set weight webcluster/web1 2 " | socat stdio
/var/lib/haproxy/stats
#下线后端服务器
[root@haproxy ~]# echo "disable server webserver_80/webserver1 " | socat stdio
/var/lib/haproxy/stats
#上线后端服务器
[root@haproxy ~]# echo "enable server webserver_80/webserver1 " | socat stdio
/var/lib/haproxy/stats
5、haproxy算法
静态算法
static-rr
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80
bind 172.25.254.100:80
mode http
balance static-rr
server webserver1 192.168.0.101:80 weight 2 check inter 3s fall 3 rise 5
server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...
first
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80
bind 172.25.254.100:80
mode http
balance first
server webserver1 192.168.0.101:80 maxconn 3 check inter 3s fall 3 rise 5
server webserver2 192.168.0.102:80 check inter 3s fall 3 rise 5
...上面内容省略...
#在两台主机上分别执行此循环，可以观察是否102被调度到
while true;do curl 172.25.254.100 ; sleep 0.1;done
动态算法
roundrobin
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80
bind 172.25.254.100:80
mode http
balance roundrobin
server webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5
server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...
动态调整权重
[root@haproxy ~]# echo "set weight webserver_80/webserver1 2" | socat stdio
/var/lib/haproxy/haproxy.sock
leastconn
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80
bind 172.25.254.100:80
mode http
balance leastconn
server webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5
server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...
source
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80
bind 172.25.254.100:80
mode http
balance source
server webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5
server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...
[root@node10 ~]# for N in {1..6}; do curl 172.25.254.100; done
RS1 server - 192.168.0.101
RS1 server - 192.168.0.101
RS1 server - 192.168.0.101
RS1 server - 192.168.0.101
RS1 server - 192.168.0.101
RS1 server - 192.168.0.101
uri
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80
bind 172.25.254.100:80
mode http
balance uri
server webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5
server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...
url_param
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80
bind 172.25.254.100:80
mode http
balance url_param name,userid #支持对多个url_param hash
server webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5
server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...
hdr
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80
bind 172.25.254.100:80
mode http
balance hdr(User-Agent)
hash-type consistent
server webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5
server webserver2 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...
[root@node10 ~]# curl -v 172.25.254.100
[root@node10 ~]# curl -vA "firefox" 172.25.254.100
[root@node10 ~]# curl -vA "sougou" 172.25.254.100
6、cookie
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80
bind 172.25.254.100:80
option forwardfor
mode http
balance roundrobin
cookie WEBCOOKIE insert nocache indirect
server webserver1 192.168.0.101:80 cookie web1 weight 1 check inter 3s fall
3 rise 5
server webserver2 192.168.0.102:80 cookie web2 weight 1 check inter 3s fall
3 rise 5
...上面内容省略...
[root@node10 ~]# curl -i 172.25.254.100
HTTP/1.1 200 OK
date: Wed, 10 Jul 2024 16:36:17 GMT
server: Apache/2.4.6 (Red Hat Enterprise Linux) OpenSSL/1.0.2k-fips
last-modified: Thu, 04 Jul 2024 11:18:39 GMT
etag: "1b-61c6a1bd2408d"
accept-ranges: bytes
content-length: 27
content-type: text/html; charset=UTF-8
set-cookie: WEBCOOKIE=web2; path=/
cache-control: private
RS2 server - 192.168.0.102
[Administrator.WIN-20240602BIS] ➤ curl -i 172.25.254.100
HTTP/1.1 200 OK
server: nginx/1.20.1
date: Wed, 10 Jul 2024 08:36:43 GMT
content-type: text/html
content-length: 18
last-modified: Wed, 10 Jul 2024 04:09:05 GMT
etag: "668e0961-12"
accept-ranges: bytes
set-cookie: WEBCOOKIE=web1; path=/
cache-control: private
RS1 192.168.0.101
#curl访问时指定cookie
[root@node10 ~]# curl -b WEBCOOKIE=web1 172.25.254.100
RS1 server - 192.168.0.101
[root@node10 ~]# curl -b WEBCOOKIE=web2 172.25.254.100
RS2 server - 192.168.0.102
[root@node10 ~]# curl -vb WEBCOOKIE=web1 172.25.254.100
* About to connect() to 172.25.254.100 port 80 (#0)
* Trying 172.25.254.100...
* Connected to 172.25.254.100 (172.25.254.100) port 80 (#0)
> GET / HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 172.25.254.100
> Accept: */*
> Cookie: WEBCOOKIE=web1
>
< HTTP/1.1 200 OK
< server: nginx/1.20.1
< date: Wed, 10 Jul 2024 08:38:25 GMT
< content-type: text/html
< content-length: 18
< last-modified: Wed, 10 Jul 2024 04:09:05 GMT
< etag: "668e0961-12"
< accept-ranges: bytes
<
RS1 192.168.0.101
* Connection #0 to host 172.25.254.100 left intact
7、HAProxy状态页
启用状态页
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen stats：
mode http
bind 0.0.0.0:8888
stats enable
log global
stats uri /status #自定义stats page uri
stats auth lee:lee #认证，此行可以出现多次
...上面内容省略...
测试：
浏览器访问：172.25.254.100:8888/status
登录状态页
#pid为当前pid号，process为当前进程号，nbproc和nbthread为一共多少进程和每个进程多少个线程
pid = 27134 (process #1, nbproc = 1, nbthread = 1)
#启动了多长时间
uptime = 0d 0h00m04s
#系统资源限制：内存/最大打开文件数/
system limits: memmax = unlimited; ulimit-n = 200029
#最大socket连接数/单进程最大连接数/最大管道数maxpipes
maxsock = 200029; maxconn = 100000; maxpipes = 0
#当前连接数/当前管道数/当前连接速率
current conns = 2; current pipes = 0/0; conn rate = 2/sec; bit rate = 0.000 kbps
#运行的任务/当前空闲率
Running tasks: 1/14; idle = 100 %
active UP： #在线服务器
backup UP： #标记为backup的服务器
active UP, going down： #监测未通过正在进入down过程
backup UP, going down： #备份服务器正在进入down过程
active DOWN, going up： #down的服务器正在进入up过程
backup DOWN, going up： #备份服务器正在进入up过程
active or backup DOWN： #在线的服务器或者是backup的服务器已经转换成了down状态
not checked： #标记为不监测的服务器
#active或者backup服务器人为下线的
active or backup DOWN for maintenance (MAINT)
#active或者backup被人为软下线(人为将weight改成0)
active or backup SOFT STOPPED for maintenance
8、IP透传
四层IP透传
#未开启透传的四层代理
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80
bind 172.25.254.100:80
mode tcp
balance roundrobin
server webserver1 192.168.0.101:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...
#正常的nginx配置
[root@rs1 ~]# vim /etc/nginx/nginx.conf
。。。内容省略。。。
http {
log_format main '$remote_addr - $remote_user [$time_local] "$request"'
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
。。。内容省略。。。
server {
listen 80;
listen [::]:80;
server_name _;
root /usr/share/nginx/html;
。。。内容省略。。。
}
}
#在访问haproxy后查看nginx日志
[root@rs1 ~]# tail -n 3 /var/log/nginx/access.log
192.168.0.10 - - [10/Jul/2024:15:21:00 +0800] "GET / HTTP/1.1"200 18 "-"
"curl/7.29.0" "-"192.168.0.10 - - [10/Jul/2024:15:26:11 +0800] "GET /
HTTP/1.1"200 18 "-" "curl/7.29.0" "-"
在此日志中是无法看到真实访问源地址的
开启四层透传
#nginx 配置：在访问日志中通过变量$proxy_protocol_addr 记录透传过来的客户端IP
[root@rs1 ~]# vim /etc/nginx/nginx.conf
。。。内容省略。。。
http {
log_format main '$remote_addr - $remote_user [$time_local] "$request"'
' "$proxy_protocol_addr"'
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent" "$http_x_forwarded_for"';
。。。内容省略。。。
server {
listen 80 proxy_protocol; #启用此项，将无法直接访问此网站，只能通过四层代理
访问
listen [::]:80;
server_name _;
root /usr/share/nginx/html;
。。。内容省略。。。
}
}
#修改haproxy
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80
bind 172.25.254.100:80
mode tcp
balance roundrobin
server webserver1 192.168.0.101:80 send-proxy weight 1 check inter 3s fall 3
rise 5
...上面内容省略...
#查看日志内容
[root@rs1 ~]# tail -n 3 /var/log/nginx/access.log
192.168.0.10 - - [10/Jul/2024:15:21:00 +0800] "GET / HTTP/1.1"200 18 "-"
"curl/7.29.0" "-"
192.168.0.10 - - [10/Jul/2024:15:26:11 +0800] "GET / HTTP/1.1"200 18 "-"
"curl/7.29.0" "-"
192.168.0.10 - - [10/Jul/2024:15:41:56 +0800] "GET / HTTP/1.1" "172.25.254.10"200
18 "-" "curl/7.29.0"
七层IP透传
#修改haproxy
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen webserver_80
option forwardfor
bind 172.25.254.100:80
mode http
balance roundrobin
server webserver1 192.168.0.101:80 send-proxy weight 1 check inter 3s fall 3
rise 5
server webserver1 192.168.0.102:80 weight 1 check inter 3s fall 3 rise 5
...上面内容省略...
9、ACL
域名匹配
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
frontend testacl
bind :80
mode http
########### ACL settings #######################
acl web_host hdr_dom(host) www.timinglee.org
########### host ###########################
use_backend timinglee_host if web_host
########### default server ###################
default_backend default_webserver
backend timinglee_host
mode http
server web1 192.168.0.101:80 check weight 1 inter 3s fall 3 rise 5
server web2 192.168.0.102:80 check weight 1 inter 3s fall 3 rise 5
backend default_webserver
mode http
server web1 172.25.254.10:80 check weight 1 inter 3s fall 3 rise 5
...上面内容省略...
#在浏览器所在主机中做地址解析
[root@node10 html]# vim /etc/hosts
172.25.254.100 www.timinglee.org
#测试结果
[root@node10 html]# curl www.timinglee.org
RS1 192.168.0.101
[root@node10 html]# curl www.timinglee.org
RS2 server - 192.168.0.102
[root@node10 html]# curl 172.25.254.100
default web server node10
基于源IP或子网调度访问
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
frontend testacl
bind :80
mode http
########### ACL settings #######################
acl ip_test src 172.25.254.1 192.168.0.0/24
########### host ###########################
use_backend ip_test-host if ip_test
########### default server ###################
default_backend default_webserver
backend ip_test-host
mode http
server web1 192.168.0.101:80 check weight 1 inter 3s fall 3 rise 5
backend default_webserver
mode http
server web1 172.25.254.10:80 check weight 1 inter 3s fall 3 rise 5
测试结果
[172.25.254.10 root@node10 html]# curl 172.25.254.100
default web server node10
[172.25.254.1 Administrator.WIN-20240602BIS] ➤ curl 172.25.254.100
RS1 192.168.0.101
[192.168.0.102 root@rs1 ~]# curl 192.168.0.101
RS1 192.168.0.101
10、自定义错误页面
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
defaults
mode http
...内容省略...
timeout client 1m
timeout server 1m
timeout http-keep-alive 10s
timeout check 10s
maxconn 1000000
errorfile 503 /haproxy/errorpages/503page.http
[root@haproxy ~]# mkdir /haproxy/errorpages/ -p
[root@haproxy ~]# cp /usr/share/haproxy/503.http /haproxy/errorpages/503page.http
[root@haproxy ~]# vim /haproxy/errorpages/503page.http
HTTP/1.0 503 Service Unavailable
Cache-Control: no-cache
Connection: close
Content-Type: text/html;charset=UTF-8^M
<html><body><h1>什么动物生气最安静</h1>
大猩猩！！
</body></html>
测试：
关闭后端的RS主机
然后用浏览器去访问172.25.254.100
11、HAProxy 四层负载
对 MySQL 服务实现四层负载
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
listen mysql_port
bind :3306
mode tcp
balance leastconn
server mysql1 192.168.0.101:3306 check
server mysql2 192.168.0.102:3306 check
#或者使用frontend和backend实现
haproxy ~]# vim /etc/haproxy/haproxy.cfg
...上面内容省略...
frontend mysql_port
bind :3306
mode tcp
use_backend mysql_rs
backend mysql_rs
mode tcp
balance leastconn
server mysql1 192.168.0.101:3306 check
server mysql2 192.168.0.102:3306 check
haproxy ~]# systemctl restart haproxy.service
#在后端服务器安装和配置mariadb服务
rs1 ~]# yum install mariadb-server -y
rs2 ~]# yum install mariadb-server -y
rs1 ~]# vim /etc/my.cnf
[mysqld]
server-id=1 #在另一台主机为
rs2 ~]# vim /etc/my.cnf
[mysqld]
server-id=2 #在另一台主机为
rs1 ~]# systemctl start mariadb
rs2 ~]# systemctl start mariadb
rs1 ~]# mysql -e "grant all on *.* to lee@'%' identified by 'lee';"
rs2 ~]# mysql -e "grant all on *.* to lee@'%' identified by 'lee';"
#测试
[root@node10 ~]# mysql -ulee -plee -h 172.25.254.100 -e "show variables like
'hostname'"
+---------------+-------+
| Variable_name | Value |
+---------------+-------+
| hostname | rs2 |
+---------------+-------+
[root@node10 ~]# mysql -ulee -plee -h 172.25.254.100 -e "show variables like
'hostname'"
+---------------+-------+
| Variable_name | Value |
+---------------+-------+
| hostname | rs1 |
+---------------+-------+
[root@node10 ~]# mysql -ulee -plee -h172.25.254.100 -e "select @@server_id"
+-------------+
| @@server_id |
+-------------+
| 1 |
+-------------+
[root@node10 ~]# mysql -ulee -plee -h172.25.254.100 -e "select @@server_id"
+-------------+
| @@server_id |
+-------------+
| 2 |
+-------------+
12、证书
证书制作
haproxy ~]# mkdir /etc/haproxy/certs/
haproxy ~]# openssl req -newkey rsa:2048 \
-nodes -sha256 –keyout /etc/haproxy/certs/timinglee.org.key \
-x509 -days 365 -out /etc/haproxy/certs/timinglee.org.crt
https配置示例
haproxy ~]# vim /etc/haproxy/haproxy.cfg
frontend webserver
bind *:80
redirect scheme https if !{ ssl_fc }
mode http
use_backend webcluster
frontend webserver-https
bind *:443 ssl crt /etc/haproxy/timinglee.org.pem
mode http
use_backend webcluster
backend webcluster
mode http
balance roundrobin
server web1 172.25.254.200:80 check inter 3s fall 3 rise 5
server web2 172.25.254.201:80 check inter 3s fall 3 rise 5
[root@客户端 ~]#curl -IkL http://172.25.254.100
HTTP/1.1 302 Found
content-length: 0
location: https://www.timinglee.org/
cache-control: no-cache
HTTP/1.1 200 OK
date: Sat, 04 Apr 2020 02:31:31 GMT
server: Apache/2.4.6 (CentOS) PHP/5.4.16
last-modified: Thu, 02 Apr 2020 01:44:13 GMT
etag: "a-5a244f01f8adc"
accept-ranges: bytes
content-length: 10
content-type: text/html; charset=UTF-8
[root@centos6 ~]#curl -Ik https://www.timinglee.org
HTTP/1.1 200 OK
date: Sat, 04 Apr 2020 02:31:50 GMT
server: Apache/2.4.6 (CentOS) PHP/5.4.16
last-modified: Thu, 02 Apr 2020 01:44:28 GMT
etag: "a-5a244f0fd5175"
accept-ranges: bytes
content-length: 10
content-type: text/html; charset=UTF-8