测试拓扑
双臂模式， 使用两个网卡，一个对外，一个对内。
Client host是物理机， RS host都是虚拟机。
LB host是物理机，两个CX5网卡分别在两个子网。
配置文件
用dpvs.conf.sample作为双臂配置文件，其中多一个接口配置，以及worker中多一个接口tx/rx队列
netif_defs {
<init> pktpool_size 1048575
<init> pktpool_cache 256
<init> fdir_mode perfect
# dpdk0 dpdk1 两个接口
<init> device dpdk0 {
rx {
queue_number 8
descriptor_number 1024
rss all
}
tx {
queue_number 8
descriptor_number 1024
}
! mtu 1500
! promisc_mode
! allmulticast
kni_name dpdk0.kni
}
<init> device dpdk1 {
rx {
queue_number 8
descriptor_number 1024
rss all
}
tx {
queue_number 8
descriptor_number 1024
}
! mtu 1500
! promisc_mode
! allmulticast
kni_name dpdk1.kni
}
}
worker_defs {
<init> worker cpu0 {
type master
cpu_id 0
}
# worker中也增加了一个接口tx/rx队列
<init> worker cpu1 {
type slave
cpu_id 1
port dpdk0 {
rx_queue_ids 0
tx_queue_ids 0
! isol_rx_cpu_ids 9
! isol_rxq_ring_sz 1048576
}
port dpdk1 {
rx_queue_ids 0
tx_queue_ids 0
! isol_rx_cpu_ids 9
! isol_rxq_ring_sz 1048576
}
}
启动DPVS
命令行中 -a 98:00.0 -a 98:00.1 设置使用的两张网卡。
./dpvs -c ../conf/dpvs.conf.sample -- -a 98:00.0 -a 98:00.1 -l 0-9
root@r750-132:~/dpvs/bin# ./dpvs -c ../conf/dpvs.conf.sample -- -a 98:00.0 -a 98:00.1 -l 0-8
current thread affinity is set to FFFFFFFF
EAL: Detected 32 lcore(s)
EAL: Detected 2 NUMA nodes
EAL: Detected static linkage of DPDK
EAL: Multi-process socket /var/run/dpdk/rte/mp_socket
EAL: Selected IOVA mode 'PA'
EAL: No available hugepages reported in hugepages-1048576kB
EAL: Probing VFIO support...
EAL: VFIO support initialized
EAL: Probe PCI driver: mlx5_pci (15b3:1017) device: 0000:98:00.0 (socket 1)
EAL: Probe PCI driver: mlx5_pci (15b3:1017) device: 0000:98:00.1 (socket 1)
EAL: No legacy callbacks, legacy socket not created
DPVS: dpvs version: 1.9-8, build on 2025.02.20.15:37:58
DPVS: dpvs-conf-file: ../conf/dpvs.conf.sample
DPVS: dpvs-pid-file: /var/run/dpvs.pid
DPVS: dpvs-ipc-file: /var/run/dpvs.ipc
CFG_FILE: Opening configuration file '../conf/dpvs.conf.sample'.
CFG_FILE: log_level = WARNING
NETIF: dpdk0:rx_queue_number = 8
NETIF: dpdk1:rx_queue_number = 8
NETIF: worker cpu1:dpdk0 rx_queue_id += 0
NETIF: worker cpu1:dpdk0 tx_queue_id += 0
NETIF: worker cpu1:dpdk1 rx_queue_id += 0
NETIF: worker cpu1:dpdk1 tx_queue_id += 0
NETIF: worker cpu2:dpdk0 rx_queue_id += 1
NETIF: worker cpu2:dpdk0 tx_queue_id += 1
NETIF: worker cpu2:dpdk1 rx_queue_id += 1
NETIF: worker cpu2:dpdk1 tx_queue_id += 1
NETIF: worker cpu3:dpdk0 rx_queue_id += 2
NETIF: worker cpu3:dpdk0 tx_queue_id += 2
NETIF: worker cpu3:dpdk1 rx_queue_id += 2
NETIF: worker cpu3:dpdk1 tx_queue_id += 2
NETIF: worker cpu4:dpdk0 rx_queue_id += 3
NETIF: worker cpu4:dpdk0 tx_queue_id += 3
NETIF: worker cpu4:dpdk1 rx_queue_id += 3
NETIF: worker cpu4:dpdk1 tx_queue_id += 3
NETIF: worker cpu5:dpdk0 rx_queue_id += 4
NETIF: worker cpu5:dpdk0 tx_queue_id += 4
NETIF: worker cpu5:dpdk1 rx_queue_id += 4
NETIF: worker cpu5:dpdk1 tx_queue_id += 4
NETIF: worker cpu6:dpdk0 rx_queue_id += 5
NETIF: worker cpu6:dpdk0 tx_queue_id += 5
NETIF: worker cpu6:dpdk1 rx_queue_id += 5
NETIF: worker cpu6:dpdk1 tx_queue_id += 5
NETIF: worker cpu7:dpdk0 rx_queue_id += 6
NETIF: worker cpu7:dpdk0 tx_queue_id += 6
NETIF: worker cpu7:dpdk1 rx_queue_id += 6
NETIF: worker cpu7:dpdk1 tx_queue_id += 6
NETIF: worker cpu8:dpdk0 rx_queue_id += 7
NETIF: worker cpu8:dpdk0 tx_queue_id += 7
NETIF: worker cpu8:dpdk1 rx_queue_id += 7
NETIF: worker cpu8:dpdk1 tx_queue_id += 7
SAPOOL: sapool_filter_enable = on
NETIF: Ethdev port_id=0 invalid tx_offload: 0x1000e, valid value: 0xc96af
NETIF: Ethdev port_id=1 invalid tx_offload: 0x1000e, valid value: 0xc96af
负载均衡配置
# 添加VIP
./dpip addr add 10.0.0.100/32 dev dpdk0
# 添加负载均衡服务 , 轮询模式
./ipvsadm -A -t 10.0.0.100:80 -s rr
# 添加 3个RS， FULLNAT 模式
./ipvsadm -a -t 10.0.0.100:80 -r 192.168.100.3:80 -b
./ipvsadm -a -t 10.0.0.100:80 -r 192.168.100.4:80 -b
./ipvsadm -a -t 10.0.0.100:80 -r 192.168.100.5:80 -b
# 为负载均衡服务 10.0.0.100:80 添加一个LOCAL IP 在dpdk1上
./ipvsadm --add-laddr -z 192.168.100.200 -t 10.0.0.100:80 -F dpdk1
# 添加路由
./dpip route add 10.0.0.0/16 dev dpdk0
./dpip route add 192.168.100.0/24 dev dpdk1
检查配置
服务配置
root@r750-132:~/dpvs/bin# ./ipvsadm -L -n
IP Virtual Server version 1.9.8 (size=0)
Prot LocalAddress:Port Scheduler Flags
-> RemoteAddress:Port Forward Weight ActiveConn InActConn
TCP 10.0.0.100:80 rr
-> 192.168.100.3:0 FullNat 1 0 0
-> 192.168.100.4:0 FullNat 1 0 0
-> 192.168.100.5:0 FullNat 1 0 0
root@r750-132:~/dpvs/bin# ./ipvsadm -G
VIP:VPORT TOTAL SNAT_IP CONFLICTS CONNS
10.0.0.100:80 1
192.168.100.200 0 0
接口路由配置
root@r750-132:~/dpvs/bin# ./dpip addr show
inet 10.0.0.100/32 scope global dpdk0
valid_lft forever preferred_lft forever
inet6 fe80::eaeb:d3ff:fea3:8377/64 scope link dpdk1
valid_lft forever preferred_lft forever
inet 192.168.100.200/32 scope global dpdk1
valid_lft forever preferred_lft forever
inet6 fe80::eaeb:d3ff:fea3:8376/64 scope link dpdk0
valid_lft forever preferred_lft forever
root@r750-132:~/dpvs/bin#
root@r750-132:~/dpvs/bin# ./dpip route show
inet 10.0.0.100/32 via 0.0.0.0 src 0.0.0.0 dev dpdk0 mtu 1500 tos 0 scope host metric 0 proto auto
inet 192.168.100.200/32 via 0.0.0.0 src 0.0.0.0 dev dpdk1 mtu 1500 tos 0 scope host metric 0 proto auto
inet 192.168.100.0/24 via 0.0.0.0 src 0.0.0.0 dev dpdk1 mtu 1500 tos 0 scope link metric 0 proto auto
inet 10.0.0.0/16 via 0.0.0.0 src 0.0.0.0 dev dpdk0 mtu 1500 tos 0 scope link metric 0 proto auto
检查连通性
Client ping dpdk0
# Client ping dpdk0
[root@dkdp192 ~]# ifconfig enp1s0f0np0 10.0.0.192/24
[root@dkdp192 ~]#
[root@dkdp192 ~]# ping 10.0.0.100
PING 10.0.0.100 (10.0.0.100) 56(84) bytes of data.
64 bytes from 10.0.0.100: icmp_seq=1 ttl=64 time=0.153 ms
64 bytes from 10.0.0.100: icmp_seq=2 ttl=64 time=0.068 ms
64 bytes from 10.0.0.100: icmp_seq=3 ttl=64 time=0.080 ms
^C
--- 10.0.0.100 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2063ms
rtt min/avg/max/mdev = 0.068/0.100/0.153/0.038 ms
RS ping dpdk1
root@ubuntu22:~# ping 192.168.100.200
PING 192.168.100.200 (192.168.100.200) 56(84) bytes of data.
64 bytes from 192.168.100.200: icmp_seq=1 ttl=64 time=0.328 ms
64 bytes from 192.168.100.200: icmp_seq=2 ttl=64 time=0.118 ms
64 bytes from 192.168.100.200: icmp_seq=3 ttl=64 time=0.085 ms
^C
--- 192.168.100.200 ping statistics ---
3 packets transmitted, 3 received, 0% packet loss, time 2042ms
rtt min/avg/max/mdev = 0.085/0.177/0.328/0.107 ms
CURL测试
测试成功
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 1 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 0 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 0 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 0 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 1 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 1 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 0 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 1 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 0 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 2 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 1 !
[root@dkdp192 ~]# curl 10.0.0.100:80
This is Server 1 !
[root@dkdp192 ~]#