跳到主要内容

shofastup测试

编译打包​

需要使用专门的交叉工具编译，交叉编译工具文件为fsl-qoriq-glibc-x86_64-fsl-toolchain-aarch64-toolchain-2.5.sh，使用如下命令安装：

fsl-qoriq-glibc-x86_64-fsl-toolchain-aarch64-toolchain-2.5.sh -d /opt/cross

若/opt/cross已存在该交叉编译工具无需安装。

注：暂时只能以该交叉工具编译vpp，且openssl、dpdk、vpp都需要以该工具链编译。

交叉编译工具链中包含的openssl库可以用来编译fastup，暂时先不用编译opens数量。

  • openssl
#编译openssl
git checkout OpenSSL_1_1_0g
export CROSS=/opt/cross/sysroots/x86_64-fslsdk-linux/usr/bin/aarch64-fsl-linux/aarch64-fsl-linux-
export CC=aarch64-fsl-linux-gcc
#./Configure linux-aarch64 --prefix=/opt/cross/sysroots/x86_64-fslsdk-linux/usr shared
./Configure linux-aarch64 --prefix=/home/duanyingshou/code/openssl/build_linux_aarch64/ shared
make depend
make CC="/opt/cross/sysroots/x86_64-fslsdk-linux/usr/bin/aarch64-fsl-linux/aarch64-fsl-linux-gcc --sysroot=/opt/cross/sysroots/aarch64-fsl-linux -fPIC" LD="/opt/cross/sysroots/x86_64-fslsdk-linux/usr/bin/aarch64-fsl-linux/aarch64-fsl-linux-ld" AR="/opt/cross/sysroots/x86_64-fslsdk-linux/usr/bin/aarch64-fsl-linux/aarch64-fsl-linux-ar rv " EXTRA_CFLAGS='-g -Ofast -fPIC -ftls-model=local-dynamic' -j 8
make install

  1. 编译时打包/opt/cross/sysroots/x86_64-fslsdk-linux/usr/bin/aarch64-fsl-linux/aarch64-fsl-linux-ar失败

    解决方法添加rv参数

  2. 编译openssl成功，但在编译vpp时openssl报错找不到符号表

    解决方法添加LD="/opt/cross/sysroots/x86_64-fslsdk-linux/usr/bin/aarch64-fsl-linux/aarch64-fsl-linux-ld"

  • dpdk
# 编译dpdk
export CROSS=/opt/cross/sysroots/x86_64-fslsdk-linux/usr/bin/aarch64-fsl-linux/aarch64-fsl-linux-
make config T=arm64-dpaa2-linuxapp-gcc CROSS=aarch64-fsl-linux- CROSS_COMPILE="aarch64-fsl-linux-" CC="/opt/cross/sysroots/x86_64-fslsdk-linux/usr/bin/aarch64-fsl-linux/aarch64-fsl-linux-gcc --sysroot=/opt/cross/sysroots/aarch64-fsl-linux -fPIC -g"  EXTRA_CFLAGS='-g -Ofast -fPIC -ftls-model=local-dynamic'
make T=arm64-dpaa2-linuxapp-gcc -j 4 CC="/opt/cross/sysroots/x86_64-fslsdk-linux/usr/bin/aarch64-fsl-linux/aarch64-fsl-linux-gcc --sysroot=/opt/cross/sysroots/aarch64-fsl-linux -fPIC" EXTRA_CFLAGS='-g -Ofast -fPIC -ftls-model=local-dynamic' CONFIG_RTE_KNI_KMOD=n CONFIG_RTE_EAL_IGB_UIO=n install -j 8
  • vpp

修改vpp下的toolchain.cmake的20和21行如下：

set(CMAKE_C_COMPILER ${toolchain}/usr/bin/$ENV{CROSS_PREFIX}/$ENV{CROSS_PREFIX}-gcc)
set(CMAKE_CXX_COMPILER ${toolchain}/usr/bin/$ENV{CROSS_PREFIX}/$ENV{CROSS_PREFIX}-g++)
#编译vpp
export PLATFORM=dpaa
export DPDK_PATH=/home/duanyingshou/code/dpdk/arm64-dpaa2-linuxapp-gcc
#export OPENSSL_PATH=/home/duanyingshou/code/openssl/build_linux_aarch64/
export CROSS_TOOLCHAIN=/opt/cross/sysroots/x86_64-fslsdk-linux
export CROSS_PREFIX=aarch64-fsl-linux
export CROSS_SYSROOT=/opt/cross/sysroots/aarch64-fsl-linux/
export PATH=$CROSS_TOOLCHAIN/bin:$CROSS_TOOLCHAIN/usr/bin/aarch64-fsl-linux:$PATH
cd build-root
make distclean
#TAG可以使用dpaa_debug，编译时不会优化，方便debug，详见 vpp/build-data/platforms/dpaa.mk
make V=0 PLATFORM=dpaa TAG=dpaa
  • 打包
cp ~/code/fastup_nxp/src/fastuplibs/libup/lib/arm64/*  ~/code/fastup_nxp/build-root/install-dpaa-aarch64/vpp/lib
cp ~/code/fastup_nxp/src/fastuplibs/libutils/lib/arm64/* ~/code/fastup_nxp/build-root/install-dpaa-aarch64/vpp/lib
cd ~/code/fastup_nxp/build-root/install-dpaa-aarch64/
tar zcvf ~/package/vpp_arm64.tar.gz vpp

配置​

  • dpaa2
export ROOT_DPRC=dprc.1
export PARENT_DPRC=dprc.2

export DPCON_COUNT=3
export DPBP_COUNT=4
export DPMCP_COUNT=1
export DPSECI_COUNT=8
export DPIO_COUNT=4
export DPCI_COUNT=2
export DPDMAI_COUNT=2

./vpp/dpaa2/dynamic_dpl.sh dpmac.4
export DPRC=dprc.2
  • hugepage
echo 6 > /proc/sys/vm/nr_hugepages
echo 0 > /proc/sys/vm/swappiness

  • vpp

    • startup.conf

      heapsize 256M
      plugin_path /root/vpp/lib/vpp_plugins

      unix {
        interactive
        #nodaemon
        gid vpp
        log /tmp/vpp.log
        full-coredump
        cli-listen /run/vpp/cli.sock
       # startup-config /etc/vpp/interface.txt
        startup-config /etc/vpp/interface_gtpu.txt
      }

      api-trace {
        on
      }

      api-segment {
        gid vpp
      }

      session {
        evt_qs_memfd_seg
      }

      socksvr {
        socket-name /tmp/vpp-api.sock
      }

      cpu {
              main-core 1
              corelist-workers 3
              #workers 2
      }

      dpdk {
              ## Change default settings for all intefaces
              huge-dir /mnt/hugepages
              no-pci
              num-mem-channels 1
               dev default {
                      ## Number of receive queues, enables RSS
                      ## Default is 1
                      num-rx-queues 1
                      num-rx-desc 40960

                      # rss { ipv4 }

                      ## Number of transmit queues, Default is equal
                      ## to number of worker threads or 1 if no workers treads
                      num-tx-queues 1
              }
              proc-type primary
              #log-level  8
      }

      plugins {
              ## Adjusting the plugin path depending on where the VPP plugins are
              path /root/vpp/lib/vpp_plugins
              vat-path /root/vpp/lib/vpp_api_test_plugins

              plugin default { enable }
              plugin gtpu_plugin.so { disable }
              #plugin fastgtpu_plugin.so { disable }

      }

    • interface.txt

      set int state TenGigabitEthernet0 up
      set int ip address TenGigabitEthernet0 192.168.8.24/24
      set int ip address TenGigabitEthernet0 192.168.9.24/24
      set interface mac address TenGigabitEthernet0 42:4b:54:ae:6e:05
      set int mtu 1500 TenGigabitEthernet0
      set ip arp static TenGigabitEthernet0 192.168.8.25 90:e2:ba:8d:02:f0

    • interface_gtpu.txt

      set int state TenGigabitEthernet0 up
      set int ip address TenGigabitEthernet0 192.168.8.25/16
      set interface mac address TenGigabitEthernet0 42:4b:54:ae:6e:05
      set int mtu 1500 TenGigabitEthernet0

    • 环境变量

      export PATH=/root/vpp/bin:$PATH
      export LD_LIBRARY_PATH=/root/vpp/lib

运行​

  • pktgen

    # -c cpu core mask
    # -m [18:19].1 18core用来做rx，19core用来作tx，使用dpdk的port1
    # -s 1:5gc.pcap 使用port1来回放5gc.pcap
    ./app/x86_64-native-linuxapp-gcc/pktgen -c 0xe0000 --socket-mem 2048 -n 2 -- -P -m [18:19].0 -s 0:5gc.pcap -T --crc-strip

  • vpp

    vpp -c starup.conf

结果​

  • only fastup

    main线程1核、worker线程1核，收发满线速10g，15min无丢包。

  • fastup with fastgtpu, without callback

    main线程1核、worker线程1核，收包满线速10g，15min无丢包。

  • fastup with fastgtpu and callback

    • main线程1核、worker线程1核，收包满线速6g，15min无丢包

    • main线程1核、worker线程1核，超过7g出现rx-miss，运用层收包能力不足，且无法通过调整rx ring来增加收发能力

问题​

dpdk rx-errors丢包​

dpdk rx-miss丢包​

解决方式​

rx-errors​

rx-errors丢包是因为dpdk rx ring过小，只需要在startup.conf中配置dpdk的rx ring即可。

在startup.conf文件中设置num-rx-desc，默认值为1024，按实际测试情况：

dev default {
                ## Number of receive queues, enables RSS
                ## Default is 1
                num-rx-queues 1
                # dpdk rx ring size
                num-rx-desc 8912

                # rss { ipv4 }
                ## to number of worker threads or 1 if no workers treads
                num-tx-queues 1
}

num-rx-desc大小作用效果如下：

  • 无回调

    • 1g线速1024大小不丢包
    • 5g线速4096大小不丢包
    • 10g线速8912大小不丢包

  • 有回调

    • 5g线速8192大小不丢包
    • 6g线速8192大小不丢包
    • 7g线速40960会出现rx-miss

rx-miss​

rx-miss表示网卡收到了包，表示rte_rx_queue已经塞满了数据包，所以该包被丢失。此时该包存在于物理网卡的RX FIFO中，但是不会存在于内存中的rte_rx_queue中。

rx-miss由于应用层取包速度慢，无法及时消耗掉rte_rx_queue中的数据包。

结论​

  • vpp空转、vpp加载fastgtpu插件10g线速无丢包，dpdk的rx ring需设置为8192
  • fastup回调无业务逻辑，6g线速无丢包，超过6g会出现丢包