From 689f7d2c4333daa7cd2a075d420fb1d4d95f7b07 Mon Sep 17 00:00:00 2001 From: "Richard.Yu" Date: Fri, 6 Jan 2023 11:45:03 +0800 Subject: [PATCH] [SAI-PTF][BFN]Enable saiserver test container on bfn container (#13166) Why I did it Enable Test sai api on bfn container with a lightweight container(saiserver). How I did it enable saiserver container on barefoot platform. add docker-saiserver-bfn.mk for building saiserver container in platform/barefoot/docker-saiserver-bfn, add necessary files that needs in saiserver container How to verify it Tested on Intel platform ec9516 Signed-off-by: richardyu-ms Signed-off-by: richardyu-ms --- .azure-pipelines/azure-pipelines-build.yml | 6 ++ .azure-pipelines/build-template.yml | 6 ++ platform/barefoot/docker-saiserver-bfn.mk | 39 ++++++++++ .../barefoot/docker-saiserver-bfn/Dockerfile | 37 --------- .../docker-saiserver-bfn/Dockerfile.j2 | 78 +++++++++++++++++++ .../barefoot/docker-saiserver-bfn/portmap.ini | 33 -------- .../barefoot/docker-saiserver-bfn/profile.ini | 1 - .../docker-saiserver-bfn/ptf_nn_agent.conf | 10 +++ .../docker-saiserver-bfn/sai_tofino.xml | 2 - .../docker-saiserver-bfn/saiserver_start.sh | 32 ++++++++ .../barefoot/docker-saiserver-bfn/start.sh | 8 +- .../docker-saiserver-bfn/supervisord.conf | 3 +- platform/barefoot/libsaithrift-dev.mk | 14 +++- platform/barefoot/rules.mk | 1 + 14 files changed, 189 insertions(+), 81 deletions(-) create mode 100644 platform/barefoot/docker-saiserver-bfn.mk delete mode 100755 platform/barefoot/docker-saiserver-bfn/Dockerfile create mode 100755 platform/barefoot/docker-saiserver-bfn/Dockerfile.j2 delete mode 100644 platform/barefoot/docker-saiserver-bfn/portmap.ini delete mode 100644 platform/barefoot/docker-saiserver-bfn/profile.ini create mode 100644 platform/barefoot/docker-saiserver-bfn/ptf_nn_agent.conf delete mode 100644 platform/barefoot/docker-saiserver-bfn/sai_tofino.xml create mode 100644 platform/barefoot/docker-saiserver-bfn/saiserver_start.sh diff --git a/.azure-pipelines/azure-pipelines-build.yml b/.azure-pipelines/azure-pipelines-build.yml index d21a6066a..09a46573f 100644 --- a/.azure-pipelines/azure-pipelines-build.yml +++ b/.azure-pipelines/azure-pipelines-build.yml @@ -154,6 +154,12 @@ jobs: git stash popd fi + if [ $(GROUP_NAME) == barefoot ]; then + make $BUILD_OPTIONS SAITHRIFT_V2=y ENABLE_SYNCD_RPC=y target/docker-saiserverv2-bfn.gz + pushd ./src/sonic-sairedis/SAI + git stash + popd + fi fi if [ $(syncd_rpc_image) == yes ]; then make $BUILD_OPTIONS ENABLE_SYNCD_RPC=y target/sonic-$(GROUP_NAME).bin diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml index 52bff4aa9..43db5720a 100644 --- a/.azure-pipelines/build-template.yml +++ b/.azure-pipelines/build-template.yml @@ -116,6 +116,12 @@ jobs: git stash popd fi + if [ ${{ parameters.platform }} == barefoot ]; then + make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) SAITHRIFT_V2=y ENABLE_SYNCD_RPC=y target/docker-saiserverv2-bfn.gz + pushd ./src/sonic-sairedis/SAI + git stash + popd + fi fi make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.bin diff --git a/platform/barefoot/docker-saiserver-bfn.mk b/platform/barefoot/docker-saiserver-bfn.mk new file mode 100644 index 000000000..dde7e8888 --- /dev/null +++ b/platform/barefoot/docker-saiserver-bfn.mk @@ -0,0 +1,39 @@ +# docker image for bfn saiserver +# Support two version of saiserver, v2 will use the new sai-ptfv2 +DOCKER_SAISERVER_BFN = docker-saiserver$(SAITHRIFT_VER)-bfn.gz +$(DOCKER_SAISERVER_BFN)_PATH = $(PLATFORM_PATH)/docker-saiserver-bfn + +# Use syncd_init_common.sh to init hardware platform +SYNCD_INIT_COMMON_SCRIPT = syncd_init_common.sh +$(SYNCD_INIT_COMMON_SCRIPT)_PATH = $(SRC_PATH)/sonic-sairedis/syncd/scripts +SONIC_COPY_FILES += $(SYNCD_INIT_COMMON_SCRIPT) + +# Same dependence as syncd +$(DOCKER_SAISERVER_BFN)_DEPENDS += $(SAISERVER) +# Install syncd for reuse the config fun +#$(DOCKER_SAISERVER_BFN)_DEPENDS += $(SYNCD) +$(DOCKER_SAISERVER_BFN)_DEPENDS += $(BFN_SAI) $(BFN_INGRASYS_PLATFORM) $(BFN_PLATFORM) $(LIBTHRIFT_0_14_1) +$(DOCKER_SAISERVER_BFN)_FILES += $(SYNCD_INIT_COMMON_SCRIPT) + +# Same dependence as ENABLE_SYNCD_RPC +$(DOCKER_SAISERVER_BFN)_DEPENDS += $(LIBSAITHRIFT_DEV) $(LIBTHRIFT_0_14_1_DEV) + +# Runtime dependency on sai is set only for syncd +#$(SYNCD)_RDEPENDS += $(BFN_SAI) $(WNC_OSW1800_PLATFORM) $(BFN_INGRASYS_PLATFORM) $(BFN_PLATFORM) +$(DOCKER_SAISERVER_BFN)_RDEPENDS += $(BFN_SAI) $(BFN_INGRASYS_PLATFORM) $(BFN_PLATFORM) + +$(DOCKER_SAISERVER_BFN)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_BULLSEYE) + +SONIC_DOCKER_IMAGES += $(DOCKER_SAISERVER_BFN) +SONIC_BULLSEYE_DOCKERS += $(DOCKER_SAISERVER_BFN) + +# Only Support saiserver v2 +$(DOCKER_SAISERVER_BFN)_CONTAINER_NAME = saiserver$(SAITHRIFT_VER) +$(DOCKER_SAISERVER_BFN)_VERSION = 1.0.0+rpc +$(DOCKER_SAISERVER_BFN)_PACKAGE_NAME = saiserver + +$(DOCKER_SAISERVER_BFN)_RUN_OPT += --privileged -t +$(DOCKER_SAISERVER_BFN)_RUN_OPT += -v /host/machine.conf:/etc/machine.conf +$(DOCKER_SAISERVER_BFN)_RUN_OPT += -v /var/run/docker-saiserver:/var/run/sswsyncd +$(DOCKER_SAISERVER_BFN)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro +$(DOCKER_SAISERVER_BFN)_RUN_OPT += -v /host/warmboot:/var/warmboot diff --git a/platform/barefoot/docker-saiserver-bfn/Dockerfile b/platform/barefoot/docker-saiserver-bfn/Dockerfile deleted file mode 100755 index 6fa5b1818..000000000 --- a/platform/barefoot/docker-saiserver-bfn/Dockerfile +++ /dev/null @@ -1,37 +0,0 @@ -FROM docker-base - -## Make apt-get non-interactive -ENV DEBIAN_FRONTEND=noninteractive - -RUN apt-get update - -COPY ["deps/applibs_*.deb", "/deps/applibs-dev_*.deb", "/deps/sx-complib_*.deb", "/deps/sxd-libs_*.deb", "/deps/sx-scew_*.deb", "/deps/sx-examples_*.deb", "/deps/sx-gen-utils_*.deb", "/deps/python-sdk-api_*.deb", "/deps/iproute2_*.deb", "/deps/mlnx-sai_*.deb", "/deps/libthrift-0.9.3_*.deb", "/deps/libnl-3-200_*.deb", "/deps/libnl-genl-3-200_*.deb", "/deps/libnl-route-3-200_*.deb", "/deps/"] - -RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return 1; }; \ - dpkg_apt /deps/applibs_*.deb \ - && dpkg_apt /deps/applibs-dev_*.deb \ - && dpkg_apt /deps/sx-complib_*.deb \ - && dpkg_apt /deps/sxd-libs_*.deb \ - && dpkg_apt /deps/sx-scew_*.deb \ - && dpkg_apt /deps/sx-examples_*.deb \ - && dpkg_apt /deps/sx-gen-utils_*.deb \ - && dpkg_apt /deps/python-sdk-api_*.deb \ - && dpkg_apt /deps/iproute2_*.deb \ - && dpkg_apt /deps/mlnx-sai_*.deb \ - && dpkg_apt /deps/libthrift-0.9.3_*.deb \ - && dpkg_apt /deps/libnl-3-200_*.deb \ - && dpkg_apt /deps/libnl-genl-3-200_*.deb \ - && dpkg_apt /deps/libnl-route-3-200_*.deb - -COPY ["deps/saiserver", "start.sh", "/usr/bin/"] - -COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] - -COPY ["profile.ini", "portmap.ini", "/etc/sai/"] - - -## Clean up -RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y -RUN rm -rf /deps - -ENTRYPOINT ["/usr/local/bin/supervisord"] diff --git a/platform/barefoot/docker-saiserver-bfn/Dockerfile.j2 b/platform/barefoot/docker-saiserver-bfn/Dockerfile.j2 new file mode 100755 index 000000000..a17cca1a8 --- /dev/null +++ b/platform/barefoot/docker-saiserver-bfn/Dockerfile.j2 @@ -0,0 +1,78 @@ +{% from "dockers/dockerfile-macros.j2" import install_debian_packages %} +FROM docker-config-engine-bullseye-{{DOCKER_USERNAME}}:{{DOCKER_USERTAG}} + +ARG docker_container_name + +## Make apt-get non-interactive +ENV DEBIAN_FRONTEND=noninteractive + +RUN apt-get update + +COPY \ +{% for deb in docker_saiserver_bfn_debs.split(' ') -%} +debs/{{ deb }}{{' '}} +{%- endfor -%} +debs/ + +RUN apt-get install -y \ + libxml2 \ + libpcap-dev \ + libusb-1.0-0 \ + libcurl3-gnutls \ + libunwind8-dev \ + libc-ares2 \ + libedit2 \ + libgoogle-perftools4 + +## Pre-install the fundamental packages +RUN apt-get update \ + && apt-get -y install \ + wget \ + cmake \ + libqt5core5a \ + libqt5network5 \ + python3-setuptools \ + python3-pip \ + python-is-python3 \ + libboost-atomic1.74.0 + +# Install locally-built Debian packages and implicitly install their dependencies +{{ install_debian_packages(docker_saiserver_bfn_debs.split(' ')) }} + +## Support nanomsg, plesae install those package as needed +## If don't need to run cases inside saiserver locally with nn_agent, plesae remove them. +RUN wget https://github.com/nanomsg/nanomsg/archive/1.0.0.tar.gz \ + && tar xvfz 1.0.0.tar.gz \ + && cd nanomsg-1.0.0 \ + && mkdir -p build \ + && cmake . \ + && make install \ + && ldconfig \ + && cd .. \ + && rm -fr nanomsg-1.0.0 \ + && rm -f 1.0.0.tar.gz \ + && pip3 install cffi==1.7.0 \ + && pip3 install --upgrade cffi==1.7.0 \ + && pip3 install wheel \ + && pip3 install nnpy \ + && mkdir -p /opt \ + && cd /opt \ + && wget https://raw.githubusercontent.com/p4lang/ptf/master/ptf_nn/ptf_nn_agent.py \ + && apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y \ + && rm -rf /root/deps + + +COPY ["start.sh", "/usr/bin/"] +COPY ["saiserver_start.sh", "/usr/bin/"] +COPY ["files/syncd_init_common.sh", "/usr/bin/"] +COPY ["supervisord.conf", "/etc/supervisor/conf.d/"] +RUN chmod +x /usr/bin/saiserver_start.sh /usr/bin/syncd_init_common.sh + +## If don't need to run cases inside saiserver locally with nn_agent, plesae remove it. +COPY ["ptf_nn_agent.conf", "/etc/supervisor/conf.d/"] + +## Clean up +RUN apt-get clean -y; apt-get autoclean -y; apt-get autoremove -y +RUN rm -rf /debs + +ENTRYPOINT ["/usr/local/bin/supervisord"] diff --git a/platform/barefoot/docker-saiserver-bfn/portmap.ini b/platform/barefoot/docker-saiserver-bfn/portmap.ini deleted file mode 100644 index 4d3be08ce..000000000 --- a/platform/barefoot/docker-saiserver-bfn/portmap.ini +++ /dev/null @@ -1,33 +0,0 @@ -# alias lanes -Ethernet1 0,1,2,3 -Ethernet2 4,5,6,7 -Ethernet3 8,9,10,11 -Ethernet4 12,13,14,15 -Ethernet5 16,17,18,19 -Ethernet6 20,21,22,23 -Ethernet7 24,25,26,27 -Ethernet8 28,29,30,31 -Ethernet9 32,33,34,35 -Ethernet10 36,37,38,39 -Ethernet11 40,41,42,43 -Ethernet12 44,45,46,47 -Ethernet13 48,49,50,51 -Ethernet14 52,53,54,55 -Ethernet15 56,57,58,59 -Ethernet16 60,61,62,63 -Ethernet17 64,65,66,67 -Ethernet18 68,69,70,71 -Ethernet19 72,73,74,75 -Ethernet20 76,77,78,79 -Ethernet21 80,81,82,83 -Ethernet22 84,85,86,87 -Ethernet23 88,89,90,91 -Ethernet24 92,93,94,95 -Ethernet25 96,97,98,99 -Ethernet26 100,101,102,103 -Ethernet27 104,105,106,107 -Ethernet28 108,109,110,111 -Ethernet29 112,113,114,115 -Ethernet30 116,117,118,119 -Ethernet31 120,121,122,123 -Ethernet32 124,125,126,127 \ No newline at end of file diff --git a/platform/barefoot/docker-saiserver-bfn/profile.ini b/platform/barefoot/docker-saiserver-bfn/profile.ini deleted file mode 100644 index b20b2c6f0..000000000 --- a/platform/barefoot/docker-saiserver-bfn/profile.ini +++ /dev/null @@ -1 +0,0 @@ -SAI_INIT_CONFIG_FILE=/usr/share/sai_tofino.xml diff --git a/platform/barefoot/docker-saiserver-bfn/ptf_nn_agent.conf b/platform/barefoot/docker-saiserver-bfn/ptf_nn_agent.conf new file mode 100644 index 000000000..fa1ed0eb1 --- /dev/null +++ b/platform/barefoot/docker-saiserver-bfn/ptf_nn_agent.conf @@ -0,0 +1,10 @@ +[program:ptf_nn_agent] +command=/usr/bin/python /opt/ptf_nn_agent.py --device-socket 1@tcp://0.0.0.0:10900 -i 1-3@Ethernet12 --set-iface-rcv-buffer=109430400 +process_name=ptf_nn_agent +stdout_logfile=/tmp/ptf_nn_agent.out.log +stderr_logfile=/tmp/ptf_nn_agent.err.log +redirect_stderr=false +autostart=true +autorestart=true +startsecs=1 +numprocs=1 diff --git a/platform/barefoot/docker-saiserver-bfn/sai_tofino.xml b/platform/barefoot/docker-saiserver-bfn/sai_tofino.xml deleted file mode 100644 index 139597f9c..000000000 --- a/platform/barefoot/docker-saiserver-bfn/sai_tofino.xml +++ /dev/null @@ -1,2 +0,0 @@ - - diff --git a/platform/barefoot/docker-saiserver-bfn/saiserver_start.sh b/platform/barefoot/docker-saiserver-bfn/saiserver_start.sh new file mode 100644 index 000000000..06fcf6c3f --- /dev/null +++ b/platform/barefoot/docker-saiserver-bfn/saiserver_start.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Copy from src/sonic-sairedis/syncd/scripts/syncd_start.sh +# Re-use the structure for syncd setup +# Use it to start saiserver +# Script to start syncd using supervisord +# + +# Source the file that holds common code for systemd and supervisord +. /usr/bin/syncd_init_common.sh + +get_saiserver_param() +{ + IFS=' ' read -r -a array <<< "$CMD_ARGS" + for index in "${!array[@]}" + do + #echo "$index ${array[index]}" + if [[ "${array[index]}" == *"-p"* ]]; then + SAI_PROFILE="${array[index+1]}" + fi + if [[ "${array[index]}" == *"-m"* ]]; then + PORT_CONFIG="${array[index+1]}" + fi + done +} + +ENABLE_SAITHRIFT=1 +config_syncd +get_saiserver_param + +echo exec /usr/sbin/saiserver -p ${SAI_PROFILE} -f ${PORT_CONFIG} +exec /usr/sbin/saiserver -p ${SAI_PROFILE} -f ${PORT_CONFIG} +#exec ${CMD} ${CMD_ARGS} diff --git a/platform/barefoot/docker-saiserver-bfn/start.sh b/platform/barefoot/docker-saiserver-bfn/start.sh index 16457d13e..ce82391c9 100755 --- a/platform/barefoot/docker-saiserver-bfn/start.sh +++ b/platform/barefoot/docker-saiserver-bfn/start.sh @@ -1,8 +1,10 @@ #!/usr/bin/env bash +# +# Same method from platform/barefoot/docker-syncd-bfn/start.sh +# +. /opt/bfn/install/bin/dma_setup.sh +# . /opt/bfn/install/bin/bf_kdrv_mod_load /opt/bfn/install -rm -f /var/run/rsyslogd.pid supervisorctl start rsyslogd - supervisorctl start saiserver - diff --git a/platform/barefoot/docker-saiserver-bfn/supervisord.conf b/platform/barefoot/docker-saiserver-bfn/supervisord.conf index e09ac3cbb..4400184e5 100644 --- a/platform/barefoot/docker-saiserver-bfn/supervisord.conf +++ b/platform/barefoot/docker-saiserver-bfn/supervisord.conf @@ -20,7 +20,8 @@ stdout_logfile=syslog stderr_logfile=syslog [program:saiserver] -command=/usr/bin/saiserver -p /etc/sai/profile.ini -f /etc/sai/portmap.ini +environment=LD_LIBRARY_PATH="/opt/bfn/install/lib" +command=/usr/bin/saiserver_start.sh priority=3 autostart=false autorestart=false diff --git a/platform/barefoot/libsaithrift-dev.mk b/platform/barefoot/libsaithrift-dev.mk index 0fb5ca0cb..a35cf13e5 100644 --- a/platform/barefoot/libsaithrift-dev.mk +++ b/platform/barefoot/libsaithrift-dev.mk @@ -1,10 +1,16 @@ # libsaithrift-dev package SAI_VER = 0.9.4 -LIBSAITHRIFT_DEV = libsaithrift-dev_$(SAI_VER)_$(CONFIGURED_ARCH).deb +LIBSAITHRIFT_DEV = libsaithrift$(SAITHRIFT_VER)-dev_$(SAI_VER)_$(CONFIGURED_ARCH).deb $(LIBSAITHRIFT_DEV)_SRC_PATH = $(SRC_PATH)/sonic-sairedis/SAI $(LIBSAITHRIFT_DEV)_DEPENDS += $(LIBTHRIFT_0_14_1) $(LIBTHRIFT_0_14_1_DEV) \ $(PYTHON3_THRIFT_0_14_1) $(THRIFT_0_14_1_COMPILER) \ $(BFN_SAI) +# Support two version of saithift for syncd-rpc +# Support two different versions of thrift +# Only saithriftv2 will build saithriftv2 +ifeq ($(SAITHRIFT_V2),y) +$(LIBSAITHRIFT_DEV)_BUILD_ENV = SAITHRIFTV2=true SAITHRIFT_VER=v2 GEN_SAIRPC_OPTS="--adapter_logger" +endif #$(LIBSAIVS) $(LIBSAIVS_DEV) $(LIBSAIMETADATA) $(LIBSAIMETADATA_DEV) @@ -13,13 +19,13 @@ $(LIBSAITHRIFT_DEV)_DEPENDS += $(LIBTHRIFT_0_14_1) $(LIBTHRIFT_0_14_1_DEV) \ $(LIBSAITHRIFT_DEV)_RDEPENDS += $(LIBTHRIFT_0_14_1) $(BFN_SAI) SONIC_DPKG_DEBS += $(LIBSAITHRIFT_DEV) -PYTHON_SAITHRIFT = python-saithrift_$(SAI_VER)_amd64.deb +PYTHON_SAITHRIFT = python-saithrift$(SAITHRIFT_VER)_$(SAI_VER)_amd64.deb $(eval $(call add_extra_package,$(LIBSAITHRIFT_DEV),$(PYTHON_SAITHRIFT))) -SAISERVER = saiserver_$(SAI_VER)_amd64.deb +SAISERVER = saiserver$(SAITHRIFT_VER)_$(SAI_VER)_amd64.deb $(SAISERVER)_RDEPENDS += $(LIBTHRIFT_0_14_1) $(BFN_SAI) $(eval $(call add_extra_package,$(LIBSAITHRIFT_DEV),$(SAISERVER))) -SAISERVER_DBG = saiserver-dbg_$(SAI_VER)_amd64.deb +SAISERVER_DBG = saiserver$(SAITHRIFT_VER)-dbg_$(SAI_VER)_amd64.deb $(SAISERVER_DBG)_RDEPENDS += $(SAISERVER) $(eval $(call add_extra_package,$(LIBSAITHRIFT_DEV),$(SAISERVER_DBG))) diff --git a/platform/barefoot/rules.mk b/platform/barefoot/rules.mk index 3eaa35fdb..58f52f5ff 100644 --- a/platform/barefoot/rules.mk +++ b/platform/barefoot/rules.mk @@ -16,6 +16,7 @@ include $(PLATFORM_PATH)/bfn-platform.mk #include $(PLATFORM_PATH)/bfn-platform-wnc.mk #include $(PLATFORM_PATH)/bfn-platform-ingrasys.mk include $(PLATFORM_PATH)/bfn-modules.mk +include $(PLATFORM_PATH)/docker-saiserver-bfn.mk SONIC_ALL += $(SONIC_ONE_IMAGE) $(SONIC_ONE_ABOOT) \ $(DOCKER_FPM)