Convert docker-snmp-sv2 to buster (#4529)

* Fix libsnmp-base compilation failure
* Convert docker-snmp-sv2 to buster
* Define install_python3_wheels
* Address review comments
* Address review comments
* Advance snmpagent submodule
* Bump net-snmp to the Buster version
* Revert "Fix libsnmp-base compilation failure"
* use azure storage url

dockers/docker-snmp-sv2/Dockerfile.j2

@@ -1,5 +1,5 @@
-{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python_wheels, copy_files %}
+{% from "dockers/dockerfile-macros.j2" import install_debian_packages, install_python3_wheels, copy_files %}
-FROM docker-config-engine-stretch
+FROM docker-config-engine-buster
 
 ARG docker_container_name
 RUN [ -f /etc/rsyslog.conf ] && sed -ri "s/%syslogtag%/$docker_container_name#%syslogtag%/;" /etc/rsyslog.conf
@@ -18,7 +18,9 @@ ENV DEBIAN_FRONTEND=noninteractive
 # The file referenced (`/usr/share/dpkg/no-pie-compile.specs`) is in the `libdpkg-perl` package on Debian
 RUN apt-get update   && \
     apt-get install -y  \
-        curl            \
+        python3         \
+        python3-pip     \
+        python3-dev     \
         ca-certificates \
         gcc             \
         make            \
@@ -43,11 +45,11 @@ RUN sed -i '/^#.* en_US.* /s/^#//' /etc/locale.gen
 RUN locale-gen
 
 # Install up-to-date version of pip
-RUN curl https://bootstrap.pypa.io/get-pip.py | python3.6
+RUN pip3 install --no-cache-dir setuptools wheel
 
 # Install pyyaml dependency for use by some plugins
 # Install smbus dependency for use by some plugins
-RUN python3.6 -m pip install --no-cache-dir \
+RUN python3 -m pip install --no-cache-dir \
         hiredis                             \
         pyyaml                              \
         smbus
@@ -57,15 +59,14 @@ RUN python3.6 -m pip install --no-cache-dir \
 {{ copy_files("python-wheels/", docker_snmp_sv2_whls.split(' '), "/python-wheels/") }}
 
 # Install locally-built Python wheel dependencies
-{{ install_python_wheels(docker_snmp_sv2_whls.split(' ')) }}
+{{ install_python3_wheels(docker_snmp_sv2_whls.split(' ')) }}
 {% endif %}
 
-RUN python3.6 -m sonic_ax_impl install
+RUN python3 -m sonic_ax_impl install
 
 # Clean up
 RUN apt-get -y purge     \
-        libpython3.6-dev \
+        python3-dev   \
-        libpython3.6     \
         curl             \
         gcc              \
         make             \

dockers/docker-snmp-sv2/supervisord.conf

@@ -34,7 +34,7 @@ stdout_logfile=syslog
 stderr_logfile=syslog
 
 [program:snmp-subagent]
-command=/usr/bin/env python3.6 -m sonic_ax_impl
+command=/usr/bin/env python3 -m sonic_ax_impl
 priority=4
 autostart=false
 autorestart=false

dockers/dockerfile-macros.j2

@@ -9,6 +9,10 @@ RUN dpkg_apt() { [ -f $1 ] && { dpkg -i $1 || apt-get -y install -f; } || return
 RUN cd /python-wheels/ && pip install {{ packages | join(' ') }}
 {%- endmacro %}
 
+{% macro install_python3_wheels(packages) -%}
+RUN cd /python-wheels/ && pip3 install {{ packages | join(' ') }}
+{%- endmacro %}
+
 {% macro copy_files(prefix, files, dest) -%}
 COPY \
     {%- for file in files %}

rules/docker-snmp-sv2.mk

@@ -7,23 +7,21 @@ DOCKER_SNMP_SV2_DBG = $(DOCKER_SNMP_SV2_STEM)-$(DBG_IMAGE_MARK).gz
 $(DOCKER_SNMP_SV2)_PATH = $(DOCKERS_PATH)/docker-snmp-sv2
 
 ## TODO: remove LIBPY3_DEV if we can get pip3 directly
-$(DOCKER_SNMP_SV2)_DEPENDS += $(SNMP) $(SNMPD) $(PY3) $(LIBPY3_DEV)
+$(DOCKER_SNMP_SV2)_DEPENDS += $(SNMP) $(SNMPD)
 
-$(DOCKER_SNMP_SV2)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_DEPENDS)
+$(DOCKER_SNMP_SV2)_DBG_DEPENDS = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_DEPENDS)
 $(DOCKER_SNMP_SV2)_DBG_DEPENDS += $(SNMP_DBG) $(SNMPD_DBG) $(LIBSNMP_DBG)
 
-$(DOCKER_SNMP_SV2)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_STRETCH)_DBG_IMAGE_PACKAGES)
+$(DOCKER_SNMP_SV2)_DBG_IMAGE_PACKAGES = $($(DOCKER_CONFIG_ENGINE_BUSTER)_DBG_IMAGE_PACKAGES)
 
 $(DOCKER_SNMP_SV2)_PYTHON_WHEELS += $(SONIC_PLATFORM_COMMON_PY3) $(SWSSSDK_PY3) $(ASYNCSNMP_PY3)
-$(DOCKER_SNMP_SV2)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_STRETCH)
+$(DOCKER_SNMP_SV2)_LOAD_DOCKERS += $(DOCKER_CONFIG_ENGINE_BUSTER)
 
 SONIC_DOCKER_IMAGES += $(DOCKER_SNMP_SV2)
 SONIC_INSTALL_DOCKER_IMAGES += $(DOCKER_SNMP_SV2)
-SONIC_STRETCH_DOCKERS += $(DOCKER_SNMP_SV2)
 
 SONIC_DOCKER_DBG_IMAGES += $(DOCKER_SNMP_SV2_DBG)
 SONIC_INSTALL_DOCKER_DBG_IMAGES += $(DOCKER_SNMP_SV2_DBG)
-SONIC_STRETCH_DBG_DOCKERS += $(DOCKER_SNMP_SV2_DBG)
 
 $(DOCKER_SNMP_SV2)_CONTAINER_NAME = snmp
 $(DOCKER_SNMP_SV2)_RUN_OPT += --privileged -t

rules/python3.mk

@@ -1,35 +0,0 @@
-PYTHON_VER=3.6.0-1
-PYTHON_PNAME=python3.6
-
-export PYTHON_VER
-export PYTHON_PNAME
-
-LIBPY3_MIN = lib$(PYTHON_PNAME)-minimal_$(PYTHON_VER)_$(CONFIGURED_ARCH).deb
-$(LIBPY3_MIN)_SRC_PATH = $(SRC_PATH)/python3
-$(LIBPY3_MIN)_DEPENDS += 
-$(LIBPY3_MIN)_RDEPENDS += 
-SONIC_MAKE_DEBS += $(LIBPY3_MIN)
-
-LIBPY3_STD = lib$(PYTHON_PNAME)-stdlib_$(PYTHON_VER)_$(CONFIGURED_ARCH).deb
-$(eval $(call add_derived_package,$(LIBPY3_MIN),$(LIBPY3_STD)))
-$(LIBPY3_STD)_DEPENDS += $(LIBMPDECIMAL)
-$(LIBPY3_STD)_RDEPENDS += $(LIBPY3_MIN) $(LIBMPDECIMAL)
-
-LIBPY3 = lib$(PYTHON_PNAME)_$(PYTHON_VER)_$(CONFIGURED_ARCH).deb
-$(eval $(call add_derived_package,$(LIBPY3_MIN),$(LIBPY3)))
-$(LIBPY3)_DEPENDS += $(LIBPY3_STD)
-$(LIBPY3)_RDEPENDS += $(LIBPY3_MIN) $(LIBPY3_STD)
-
-PY3_MIN = $(PYTHON_PNAME)-minimal_$(PYTHON_VER)_$(CONFIGURED_ARCH).deb
-$(eval $(call add_derived_package,$(LIBPY3_MIN),$(PY3_MIN)))
-$(PY3_MIN)_RDEPENDS += $(LIBPY3_MIN)
-
-PY3 = $(PYTHON_PNAME)_$(PYTHON_VER)_$(CONFIGURED_ARCH).deb
-$(eval $(call add_derived_package,$(LIBPY3_MIN),$(PY3)))
-$(PY3)_DEPENDS += $(PY3_MIN) $(LIBPY3_STD)
-$(PY3)_RDEPENDS += $(PY3_MIN) $(LIBPY3_STD)
-
-LIBPY3_DEV = lib$(PYTHON_PNAME)-dev_$(PYTHON_VER)_$(CONFIGURED_ARCH).deb
-$(eval $(call add_derived_package,$(LIBPY3_MIN),$(LIBPY3_DEV)))
-$(LIBPY3_DEV)_DEPENDS += $(LIBPY3) $($(LIBPY3)_DEPENDS)
-$(LIBPY3_DEV)_RDEPENDS += $(LIBPY3) $($(LIBPY3)_RDEPENDS)

rules/snmpd.mk

@@ -1,7 +1,7 @@
 # snmpd package
 
 SNMPD_VERSION = 5.7.3+dfsg
-SNMPD_VERSION_FULL = $(SNMPD_VERSION)-1.5
+SNMPD_VERSION_FULL = $(SNMPD_VERSION)-5
 
 export SNMPD_VERSION SNMPD_VERSION_FULL
 

src/snmpd/Makefile

@@ -19,10 +19,7 @@ $(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
 	rm -rf net-snmp-$(SNMPD_VERSION)
 
 	# download debian net-snmp
-	wget -NO net-snmp_$(SNMPD_VERSION_FULL).dsc "https://sonicstorage.blob.core.windows.net/packages/net-snmp_5.7.3+dfsg-1.5.dsc?sv=2015-04-05&sr=b&sig=vDAYAKlwi7JjF%2FesdJUyf4VIEXPsCfLhqqTqNr75zBs%3D&se=2030-10-12T13%3A59%3A45Z&sp=r"
+	dget -u https://sonicstorage.blob.core.windows.net/debian/pool/main/n/net-snmp/net-snmp_$(SNMPD_VERSION_FULL).dsc
-	wget -NO net-snmp_$(SNMPD_VERSION).orig.tar.xz "https://sonicstorage.blob.core.windows.net/packages/net-snmp_5.7.3+dfsg.orig.tar.xz?sv=2015-04-05&sr=b&sig=UjIh%2FTcHrIEzEV7a%2BV2ZP4ks3xHlAA3wqyxkyV7Ms8I%3D&se=2030-10-12T13%3A58%3A19Z&sp=r"
-	wget -NO net-snmp_$(SNMPD_VERSION_FULL).debian.tar.xz "https://sonicstorage.blob.core.windows.net/packages/net-snmp_5.7.3+dfsg-1.5.debian.tar.xz?sv=2015-04-05&sr=b&sig=xJkmxjtKXYcPe4yR%2FuCA0TXUfT40rj4XUMBaiK9CjsA%3D&se=2030-10-12T14%3A00%3A15Z&sp=r" 
-	dpkg-source -x net-snmp_$(SNMPD_VERSION_FULL).dsc
 
 	pushd net-snmp-$(SNMPD_VERSION)
 	git init

src/snmpd/patch-5.7.3+dfsg/0005-Port-OpenSSL-1.1.0-with-support-for-1.0.2.patch

@@ -1,184 +0,0 @@
-From: Andreas Henriksson <andreas@fatal.se>
-Date: Sat, 23 Dec 2017 22:25:41 +0000
-Subject: [PATCH] Port OpenSSL 1.1.0 with support for 1.0.2
-
-Initial support for OpenSSL 1.1.0
-
-Changes by sebastian@breakpoint.cc:
-- added OpenSSL 1.0.2 glue layer for backwarts compatibility
-- dropped HAVE_EVP_MD_CTX_CREATE + DESTROY and added a check for OpenSSL
-  version instead (and currently 1.0.2 is the only one supported).
-
-BTS: https://bugs.debian.org/828449
-Signed-off-by: Sebastian Andrzej Siewior <sebastian@breakpoint.cc>
----
- apps/snmpusm.c              |   43 ++++++++++++++++++++++++++++++++++++-------
- configure.d/config_os_libs2 |    6 ------
- snmplib/keytools.c          |   13 ++++++-------
- snmplib/scapi.c             |   17 +++++------------
- 4 files changed, 47 insertions(+), 32 deletions(-)
-
---- a/apps/snmpusm.c
-+++ b/apps/snmpusm.c
-@@ -183,6 +183,31 @@ setup_oid(oid * it, size_t * len, u_char
- }
- 
- #if defined(HAVE_OPENSSL_DH_H) && defined(HAVE_LIBCRYPTO)
-+
-+#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
-+
-+static void DH_get0_pqg(const DH *dh,
-+			const BIGNUM **p, const BIGNUM **q, const BIGNUM **g)
-+{
-+	if (p != NULL)
-+		*p = dh->p;
-+	if (q != NULL)
-+		*q = dh->q;
-+	if (g != NULL)
-+		*g = dh->g;
-+}
-+
-+static void DH_get0_key(const DH *dh, const BIGNUM **pub_key,
-+			const BIGNUM **priv_key)
-+{
-+	if (pub_key != NULL)
-+		*pub_key = dh->pub_key;
-+	if (priv_key != NULL)
-+		*priv_key = dh->priv_key;
-+}
-+
-+#endif
-+
- int
- get_USM_DH_key(netsnmp_variable_list *vars, netsnmp_variable_list *dhvar,
-                size_t outkey_len,
-@@ -190,7 +215,7 @@ get_USM_DH_key(netsnmp_variable_list *va
-                oid *keyoid, size_t keyoid_len) {
-     u_char *dhkeychange;
-     DH *dh;
--    BIGNUM *other_pub;
-+    const BIGNUM *p, *g, *pub_key, *other_pub;
-     u_char *key;
-     size_t key_len;
-             
-@@ -205,25 +230,29 @@ get_USM_DH_key(netsnmp_variable_list *va
-         dh = d2i_DHparams(NULL, &cp, dhvar->val_len);
-     }
- 
--    if (!dh || !dh->g || !dh->p) {
-+    if (dh)
-+        DH_get0_pqg(dh, &p, NULL, &g);
-+
-+    if (!dh || !g || !p) {
-         SNMP_FREE(dhkeychange);
-         return SNMPERR_GENERR;
-     }
- 
--    DH_generate_key(dh);
--    if (!dh->pub_key) {
-+    if (!DH_generate_key(dh)) {
-         SNMP_FREE(dhkeychange);
-         return SNMPERR_GENERR;
-     }
-             
--    if (vars->val_len != (unsigned int)BN_num_bytes(dh->pub_key)) {
-+    DH_get0_key(dh, &pub_key, NULL);
-+
-+    if (vars->val_len != (unsigned int)BN_num_bytes(pub_key)) {
-         SNMP_FREE(dhkeychange);
-         fprintf(stderr,"incorrect diffie-helman lengths (%lu != %d)\n",
--                (unsigned long)vars->val_len, BN_num_bytes(dh->pub_key));
-+                (unsigned long)vars->val_len, BN_num_bytes(pub_key));
-         return SNMPERR_GENERR;
-     }
- 
--    BN_bn2bin(dh->pub_key, dhkeychange + vars->val_len);
-+    BN_bn2bin(pub_key, dhkeychange + vars->val_len);
- 
-     key_len = DH_size(dh);
-     if (!key_len) {
---- a/configure.d/config_os_libs2
-+++ b/configure.d/config_os_libs2
-@@ -291,12 +291,6 @@ if test "x$tryopenssl" != "xno" -a "x$tr
-             AC_CHECK_LIB(${CRYPTO}, AES_cfb128_encrypt, 
-                 AC_DEFINE(HAVE_AES_CFB128_ENCRYPT, 1,
-                     [Define to 1 if you have the `AES_cfb128_encrypt' function.]))
--
--            AC_CHECK_LIB(${CRYPTO}, EVP_MD_CTX_create,
--                AC_DEFINE([HAVE_EVP_MD_CTX_CREATE], [],
--                    [Define to 1 if you have the `EVP_MD_CTX_create' function.])
--                AC_DEFINE([HAVE_EVP_MD_CTX_DESTROY], [],
--                    [Define to 1 if you have the `EVP_MD_CTX_destroy' function.]))
-         fi
-         if echo " $transport_result_list " | $GREP "DTLS" > /dev/null; then
- 	    AC_CHECK_LIB(ssl, DTLSv1_method,
---- a/snmplib/keytools.c
-+++ b/snmplib/keytools.c
-@@ -149,13 +149,13 @@ generate_Ku(const oid * hashtype, u_int
-      */
- #ifdef NETSNMP_USE_OPENSSL
- 
--#ifdef HAVE_EVP_MD_CTX_CREATE
-+#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
-     ctx = EVP_MD_CTX_create();
- #else
--    ctx = malloc(sizeof(*ctx));
--    if (!EVP_MD_CTX_init(ctx))
--        return SNMPERR_GENERR;
-+    ctx = EVP_MD_CTX_new();
- #endif
-+    if (!ctx)
-+        return SNMPERR_GENERR;
- #ifndef NETSNMP_DISABLE_MD5
-     if (ISTRANSFORM(hashtype, HMACMD5Auth)) {
-         if (!EVP_DigestInit(ctx, EVP_md5()))
-@@ -259,11 +259,10 @@ generate_Ku(const oid * hashtype, u_int
-     memset(buf, 0, sizeof(buf));
- #ifdef NETSNMP_USE_OPENSSL
-     if (ctx) {
--#ifdef HAVE_EVP_MD_CTX_DESTROY
-+#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
-         EVP_MD_CTX_destroy(ctx);
- #else
--        EVP_MD_CTX_cleanup(ctx);
--        free(ctx);
-+        EVP_MD_CTX_free(ctx);
- #endif
-     }
- #endif
---- a/snmplib/scapi.c
-+++ b/snmplib/scapi.c
-@@ -486,15 +486,10 @@ sc_hash(const oid * hashtype, size_t has
-     }
- 
- /** initialize the pointer */
--#ifdef HAVE_EVP_MD_CTX_CREATE
-+#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
-     cptr = EVP_MD_CTX_create();
- #else
--    cptr = malloc(sizeof(*cptr));
--#if defined(OLD_DES)
--    memset(cptr, 0, sizeof(*cptr));
--#else
--    EVP_MD_CTX_init(cptr);
--#endif
-+    cptr = EVP_MD_CTX_new();
- #endif
-     if (!EVP_DigestInit(cptr, hashfn)) {
-         /* requested hash function is not available */
-@@ -507,13 +502,11 @@ sc_hash(const oid * hashtype, size_t has
- /** do the final pass */
-     EVP_DigestFinal(cptr, MAC, &tmp_len);
-     *MAC_len = tmp_len;
--#ifdef HAVE_EVP_MD_CTX_DESTROY
-+
-+#if (OPENSSL_VERSION_NUMBER < 0x10100000L) || defined(LIBRESSL_VERSION_NUMBER)
-     EVP_MD_CTX_destroy(cptr);
- #else
--#if !defined(OLD_DES)
--    EVP_MD_CTX_cleanup(cptr);
--#endif
--    free(cptr);
-+    EVP_MD_CTX_free(cptr);
- #endif
-     return (rval);

src/snmpd/patch-5.7.3+dfsg/0008-Enable-macro-DEB_BUILD_ARCH_OS-in-order-to-build-ipv.patch

@@ -11,14 +11,12 @@ diff --git a/debian/rules b/debian/rules
 index 4c3b5b6..1fab6a4 100755
 --- a/debian/rules
 +++ b/debian/rules
-@@ -5,6 +5,7 @@
+@@ -4,4 +4,5 @@
- # without -pie build fails during perl module build somehow...
+ export DEB_BUILD_MAINT_OPTIONS := hardening=+all
- export DEB_BUILD_MAINT_OPTIONS := hardening=+all,-pie
  DEB_HOST_MULTIARCH ?= $(shell dpkg-architecture -qDEB_HOST_MULTIARCH)
 +DEB_BUILD_ARCH_OS  ?= $(shell dpkg-architecture -qDEB_BUILD_ARCH_OS)
  
  LIB_VERSION = 30
- UPSTREAM_VERSION = $(shell dpkg-parsechangelog | egrep '^Version:' | cut -f 2 -d ':' | sed 's/ //' | sed 's/~dfsg.*$$//')
 -- 
 2.18.0
 

src/snmpd/patch-5.7.3+dfsg/series

@@ -1,7 +1,5 @@
 0001-SNMP-Stop-spamming-logs-with-statfs-permission-denie.patch
 0002-at.c-properly-check-return-status-from-realloc.-Than.patch
-0003-CHANGES-BUG-2743-snmpd-crashes-when-receiving-a-GetN.patch
-0005-Port-OpenSSL-1.1.0-with-support-for-1.0.2.patch
 0006-From-Jiri-Cervenka-snmpd-Fixed-agentx-crashing-and-or-freezing-on-timeout.patch
 0007-Linux-VRF-5.7.3-Support.patch
 0008-Enable-macro-DEB_BUILD_ARCH_OS-in-order-to-build-ipv.patch