当前位置：首页 > news >正文

URDMA跑起来

news 来源：原创 2024/5/7 4:01:20

URDMA是利用DPDK的低延时高吞吐的特性实现一个soft RDMA。
项目地址: https://github.com/zrlio/urdma
项目有点旧了，代码4年没更新了，这里记录一下跑起来的过程。

运行环境

Ubunu18.04
Linux kernel 4.15
DPDK 17.05
Intel XXV710 10G网卡

为什么不用更新的ubuntu20.04 和 5.4版本的内核呢？
因为编不过啊，内核模块的接口都改了，参考内核源码文件：uverbs_cmd.c

为什么不用DPDK 20.04 以后的版本呢？
因为编不过啊，rte很多函数和结构命名都变了。

为什么不用virtio, vmxnet3虚拟网卡呢，或者I350千兆物理网卡呢？
因为不支持Flow director啊，尤其是I350，ntuple规则能下发，但是不生效。

依赖库安装

apt install -y autoconf  uthash-dev autotools-dev m4 autoconf2.13 autoconf-archive gnu-standards autoconf-doc libtool rdma-core  librdmacm-dev libnl-route-3-dev libnl-3-dev libibumad-dev libnuma-dev python make pkg-config libjson-c-dev libnuma-dev

DPDK环境

编译DPDK

这里选择DPDK-17.05版本
https://github.com/DPDK/dpdk/releases/tag/v17.05

cd /root/orig_rdma/dpdk-17.05
make config T=x86_64-native-linuxapp-gcc
vim ./build/.config
#修改
CONFIG_RTE_BUILD_SHARED_LIB=y 
CONFIG_RTE_EAL_IGB_UIO=y
CONFIG_RTE_KNI_KMOD=y

export EXTRA_CFLAGS="-w -Wno-address-of-packed-member"

make

绑定网卡设置大页

modprobe uio
insmod /root/orig_rdma/dpdk-17.05/build/kmod/igb_uio.ko

vim /etc/sysctl.conf
vm.nr_hugepages=4096

root@node-2:~# sysctl -p
vm.nr_hugepages = 4096

编译URDMA

修改代码

为liburdma.so增加指定运行库，否则ibverbs应用无法正常启动

root@node-2:~/orig_rdma/urdma/src/liburdma# vim driver.c

391         if (!((*eal_argv)[*eal_argc] = strdup("-d"))) {
392                 goto free_eal_args;
393         }
394         (*eal_argc)++;
395         if (!((*eal_argv)[*eal_argc] = strdup("librte_pmd_i40e.so"))) {
396                 goto free_eal_args;
397         }
398         (*eal_argc)++;
399         if (!((*eal_argv)[*eal_argc] = strdup("-d"))) {
400                 goto free_eal_args;
401         }
402         (*eal_argc)++;
403         if (!((*eal_argv)[*eal_argc] = strdup("librte_mempool_ring.so"))) {
404                 goto free_eal_args;
405         }
406         (*eal_argc)++;

编译

export RTE_SDK=/root/orig_rdma/dpdk-17.05
export RTE_TARGET=build

# 这一条确保不需要install DPDK
export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$RTE_SDK/$RTE_TARGET/lib:/root/orig_rdma/urdma/src/liburdma/.libs

cd /root/orig_rdma/urdma
autoreconf -i
./configure --sysconfdir /etc

make

make install

配置文件

root@node-2:/etc/rdma# cat urdma.json 
{
    "ports": [
        {
            "ipv4_address": "10.2.0.41/24",  ## 接口地址
            "max_qp": "3",					 ## QP数量
            "rx_desc_count": "128"           ## 收包队列长度
        }
    ],
    "socket": "/run/urdma.sock", 			## 域套接字， urdma app与 urdmad交互媒介
    "eal_args": {
        "log-level": 7
    }
}

加载urdma.ko

insmod $RTE_SDK/$RTE_TARGET/kmod/rte_kni.ko
modprobe ib_core
insmod /root/orig_rdma/urdma/src/kmod/urdma.ko

运行urdmad

-d librte_pmd_i40e.so 指定网卡驱动
–file-prefix 指定 hostname

root@node-2:~/orig_rdma/urdma# src/urdmad/urdmad --proc-type=primary --file-prefix=node-2 -l0-1 -d librte_pmd_i40e.so -d librte_mempool_ring.so
EAL: Detected 16 lcore(s)
EAL: No free hugepages reported in hugepages-1048576kB
EAL: Probing VFIO support...
EAL: PCI device 0000:01:00.0 on NUMA socket -1
EAL:   probe driver: 8086:158b net_i40e
EAL: PCI device 0000:01:00.1 on NUMA socket -1
EAL:   probe driver: 8086:158b net_i40e
i40e_dev_filter_ctrl(): Filter type (5) not supported
USER1: port 0 reserves 256 queues for VMDq
port 0 max_qp 3
port 0 tx_desc_count 64 rx_desc_count 128 rx_burst_size 128 tx_burst_size 8
KNI: pci: 01:00:00       8086:158b
USER1: port 0 up mtu=1500 speed 10 Gbps

跑ibverbs应用

检查RDMA设备

root@node-2:~# ibv_devices
EAL: Detected 16 lcore(s)
EAL: Probing VFIO support...
EAL: WARNING: Address Space Layout Randomization (ASLR) is enabled in the kernel.
EAL:    This may cause issues with mapping memory into secondary processes
EAL: PCI device 0000:01:00.0 on NUMA socket -1
EAL:   probe driver: 8086:158b net_i40e
EAL: PCI device 0000:01:00.1 on NUMA socket -1
EAL:   probe driver: 8086:158b net_i40e
    device                 node GUID
    ------              ----------------
    urdma_0             3cfdfee8666c0000

root@node-2:~# ibv_devinfo -d urdma_0
EAL: Detected 16 lcore(s)
EAL: Probing VFIO support...
EAL: WARNING: Address Space Layout Randomization (ASLR) is enabled in the kernel.
EAL:    This may cause issues with mapping memory into secondary processes
EAL: PCI device 0000:01:00.0 on NUMA socket -1
EAL:   probe driver: 8086:158b net_i40e
EAL: PCI device 0000:01:00.1 on NUMA socket -1
EAL:   probe driver: 8086:158b net_i40e
hca_id: urdma_0
        transport:                      iWARP (1)
        node_guid:                      3cfd:fee8:666c:0000
        sys_image_guid:                 3cfd:fee8:666c:0000
        vendor_id:                      0x626d74
        vendor_part_id:                 2070
        hw_ver:                         0x0
        phys_port_cnt:                  1
                port:   1
                        state:                  PORT_ACTIVE (4)
                        max_mtu:                1024 (3)
                        active_mtu:             1024 (3)
                        sm_lid:                 0
                        port_lid:               0
                        port_lmc:               0x00
                        link_layer:             Ethernet

启动一个RDMA Server

root@node-2:~# ib_write_bw -d urdma_0 -R
EAL: Detected 16 lcore(s)
EAL: Probing VFIO support...
EAL: WARNING: Address Space Layout Randomization (ASLR) is enabled in the kernel.
EAL:    This may cause issues with mapping memory into secondary processes
EAL: PCI device 0000:01:00.0 on NUMA socket -1
EAL:   probe driver: 8086:158b net_i40e
EAL: PCI device 0000:01:00.1 on NUMA socket -1
EAL:   probe driver: 8086:158b net_i40e

************************************
* Waiting for client to connect... *
************************************

启动RDMA Client

在另一台机器启动Client 访问Server

root@node-2:~# ib_write_bw -d urdma_0 -R
EAL: Detected 16 lcore(s)
EAL: Probing VFIO support...
EAL: WARNING: Address Space Layout Randomization (ASLR) is enabled in the kernel.
EAL:    This may cause issues with mapping memory into secondary processes
EAL: PCI device 0000:01:00.0 on NUMA socket -1
EAL:   probe driver: 8086:158b net_i40e
EAL: PCI device 0000:01:00.1 on NUMA socket -1
EAL:   probe driver: 8086:158b net_i40e

************************************
* Waiting for client to connect... *
************************************
---------------------------------------------------------------------------------------
                    RDMA_Write BW Test
 Dual-port       : OFF          Device         : urdma_0
 Number of qps   : 1            Transport type : IW
 Connection type : RC           Using SRQ      : OFF
 CQ Moderation   : 100
 Mtu             : 1024[B]
 Link type       : Ethernet
 GID index       : 0
 Max inline data : 0[B]
 rdma_cm QPs     : ON
 Data ex. method : rdma_cm
---------------------------------------------------------------------------------------
 Waiting for client rdma_cm QP to connect
 Please run the same command with the IB/RoCE interface IP
---------------------------------------------------------------------------------------
 local address: LID 0000 QPN 0x6796b3 PSN 0x778bf4
 GID: 60:253:254:232:102:108:00:00:00:00:00:00:00:00:00:00
 remote address: LID 0000 QPN 0x748e03 PSN 0xbf5914
 GID: 60:253:254:232:100:208:00:00:00:00:00:00:00:00:00:00
---------------------------------------------------------------------------------------
 #bytes     #iterations    BW peak[MB/sec]    BW average[MB/sec]   MsgRate[Mpps]
 65536      5000             1090.84            1090.38            0.017446
---------------------------------------------------------------------------------------

root@node-1:~# ib_write_bw -d urdma_0 10.2.0.41 -R
EAL: Detected 16 lcore(s)
EAL: Probing VFIO support...
EAL: WARNING: Address Space Layout Randomization (ASLR) is enabled in the kernel.
EAL:    This may cause issues with mapping memory into secondary processes
EAL: PCI device 0000:01:00.0 on NUMA socket -1
EAL:   probe driver: 8086:158b net_i40e
EAL: PCI device 0000:01:00.1 on NUMA socket -1
EAL:   probe driver: 8086:158b net_i40e
---------------------------------------------------------------------------------------
                    RDMA_Write BW Test
 Dual-port       : OFF          Device         : urdma_0
 Number of qps   : 1            Transport type : IW
 Connection type : RC           Using SRQ      : OFF
 TX depth        : 128
 CQ Moderation   : 100
 Mtu             : 1024[B]
 Link type       : Ethernet
 GID index       : 0
 Max inline data : 0[B]
 rdma_cm QPs     : ON
 Data ex. method : rdma_cm
---------------------------------------------------------------------------------------
 local address: LID 0000 QPN 0x748e03 PSN 0xbf5914
 GID: 60:253:254:232:100:208:00:00:00:00:00:00:00:00:00:00
 remote address: LID 0000 QPN 0x6796b3 PSN 0x778bf4
 GID: 60:253:254:232:102:108:00:00:00:00:00:00:00:00:00:00
---------------------------------------------------------------------------------------
 #bytes     #iterations    BW peak[MB/sec]    BW average[MB/sec]   MsgRate[Mpps]
 65536      5000             1090.84            1090.38            0.017446
---------------------------------------------------------------------------------------