diff --git a/Makefile.cache b/Makefile.cache index f12f866db..c150f175e 100644 --- a/Makefile.cache +++ b/Makefile.cache @@ -357,6 +357,21 @@ define SAVE_CACHE $(if $(call CHECK_WCACHE_ENABLED,$(1)), $(call SAVE_INTO_CACHE,$(1),$(2))) endef +RFS_DEP_FILES := $(wildcard \ + $(addprefix scripts/, build_debian_base_system.sh prepare_debian_image_buildinfo.sh build_mirror_config.sh) \ + $(shell git ls-files files/initramfs-tools) \ + $(shell git ls-files files/image_config) \ + $(shell git ls-files files/apparmor) \ + $(shell git ls-files files/apt) \ + $(shell git ls-files files/sshd) \ + $(shell git ls-files files/dhcp) \ + src/sonic-build-hooks/buildinfo/trusted.gpg.d \ + platform/$(CONFIGURED_PLATFORM)/modules \ + files/docker/docker.service.conf \ + files/build_templates/default_users.json.j2 \ + files/build_scripts/generate_asic_config_checksum.py \ + files/scripts/core_cleanup.py \ + build_debian.sh onie-image.conf) # Set the target path for each target. @@ -384,11 +399,17 @@ $(foreach pkg, $(SONIC_INSTALL_PKGS), \ $(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(FSROOT_PATH))) \ $(eval $(FSROOT_PATH)/$(pkg)_TARGET := $(pkg)) ) +$(foreach pkg, $(SONIC_RFS_TARGETS), \ + $(eval $(pkg)_DST_PATH := $(if $($(pkg)_DST_PATH), $($(pkg)_DST_PATH), $(TARGET_PATH))) \ + $(eval $(pkg)_CACHE_MODE := GIT_CONTENT_SHA) \ + $(eval $(pkg)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)) \ + $(eval $(pkg)_DEP_FILES := $(SONIC_COMMON_BASE_FILES_LIST) $(RFS_DEP_FILES)) \ + $(eval $(TARGET_PATH)/$(pkg)_TARGET := $(pkg)) ) # define the DEP files(.dep and .smdep) and SHA files (.sha and smsha) for each target $(foreach pkg, $(SONIC_MAKE_DEBS) $(SONIC_DPKG_DEBS) $(SONIC_ONLINE_DEBS) $(SONIC_COPY_DEBS) \ $(SONIC_MAKE_FILES) $(SONIC_PYTHON_STDEB_DEBS) $(SONIC_PYTHON_WHEELS) \ - $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES) $(SONIC_INSTALL_PKGS), \ + $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES) $(SONIC_INSTALL_PKGS) $(SONIC_RFS_TARGETS), \ $(eval $(pkg)_MOD_SRC_PATH:=$(if $($(pkg)_SRC_PATH),$($(pkg)_SRC_PATH),$($(pkg)_PATH))) \ $(eval $(pkg)_BASE_PATH:=$(if $($(pkg)_BASE_PATH),$($(pkg)_BASE_PATH),$(CURDIR))) \ $(eval $(pkg)_DEP_FLAGS_FILE:=$($(pkg)_DST_PATH)/$(pkg).flags) \ @@ -489,6 +510,7 @@ $(eval $(call FLAGS_DEP_RULES, $(SONIC_PYTHON_STDEB_DEBS), $(PYTHON_DEBS_PATH),f $(eval $(call FLAGS_DEP_RULES, $(SONIC_PYTHON_WHEELS), $(PYTHON_WHEELS_PATH),flags)) $(eval $(call FLAGS_DEP_RULES, $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES), $(TARGET_PATH),flags)) $(eval $(call FLAGS_DEP_RULES, $(SONIC_INSTALL_PKGS), $(FSROOT_PATH),flags)) +$(eval $(call FLAGS_DEP_RULES, $(SONIC_RFS_TARGETS), $(TARGET_PATH),flags)) @@ -585,6 +607,7 @@ $(eval $(call SHA_DEP_RULES, $(SONIC_PYTHON_STDEB_DEBS), $(PYTHON_DEBS_PATH),dep $(eval $(call SHA_DEP_RULES, $(SONIC_PYTHON_WHEELS), $(PYTHON_WHEELS_PATH),dep)) $(eval $(call SHA_DEP_RULES, $(SONIC_DOCKER_IMAGES) $(SONIC_DOCKER_DBG_IMAGES), $(TARGET_PATH),dep)) $(eval $(call SHA_DEP_RULES, $(SONIC_INSTALL_PKGS), $(FSROOT_PATH),dep)) +$(eval $(call SHA_DEP_RULES, $(SONIC_RFS_TARGETS), $(TARGET_PATH),dep)) @@ -618,6 +641,7 @@ SONIC_CACHE_CLEAN_TARGETS = $(addsuffix -clean,$(addprefix $(TARGET_PATH)/, \ $(SONIC_DOCKER_IMAGES) \ $(SONIC_DOCKER_DBG_IMAGES) \ $(SONIC_SIMPLE_DOCKER_IMAGES) \ + $(SONIC_RFS_TARGETS) \ $(SONIC_INSTALLERS))) $(SONIC_CACHE_CLEAN_TARGETS) :: $(TARGET_PATH)/%-clean : .platform @rm -f $($*_DEP_FLAGS_FILE) $($*_MOD_HASH_FILE) $($*_SMOD_HASH_FILE) \ diff --git a/build_debian.sh b/build_debian.sh index 91abddd63..a6d563ba1 100755 --- a/build_debian.sh +++ b/build_debian.sh @@ -59,6 +59,9 @@ TRUSTED_GPG_DIR=$BUILD_TOOL_PATH/trusted.gpg.d exit 1 } +## Check if not a last stage of RFS build +if [[ $RFS_SPLIT_LAST_STAGE != y ]]; then + ## Prepare the file system directory if [[ -d $FILESYSTEM_ROOT ]]; then sudo rm -rf $FILESYSTEM_ROOT || die "Failed to clean chroot directory" @@ -71,11 +74,6 @@ touch $FILESYSTEM_ROOT/$PLATFORM_DIR/firsttime ## ensure proc is mounted sudo mount proc /proc -t proc || true -## make / as a mountpoint in chroot env, needed by dockerd -pushd $FILESYSTEM_ROOT -sudo mount --bind . . -popd - ## Build the host debian base system echo '[INFO] Build host debian base system...' TARGET_PATH=$TARGET_PATH scripts/build_debian_base_system.sh $CONFIGURED_ARCH $IMAGE_DISTRO $FILESYSTEM_ROOT @@ -576,24 +574,11 @@ if [ -f files/image_config/ntp/ntp-systemd-wrapper ]; then sudo cp ./files/image_config/ntp/ntp-systemd-wrapper $FILESYSTEM_ROOT/usr/lib/ntp/ fi -## Version file +## Version file part 1 sudo mkdir -p $FILESYSTEM_ROOT/etc/sonic if [ -f files/image_config/sonic_release ]; then sudo cp files/image_config/sonic_release $FILESYSTEM_ROOT/etc/sonic/ fi -export build_version="${SONIC_IMAGE_VERSION}" -export debian_version="$(cat $FILESYSTEM_ROOT/etc/debian_version)" -export kernel_version="${kversion}" -export asic_type="${sonic_asic_platform}" -export asic_subtype="${TARGET_MACHINE}" -export commit_id="$(git rev-parse --short HEAD)" -export branch="$(git rev-parse --abbrev-ref HEAD)" -export release="$(if [ -f $FILESYSTEM_ROOT/etc/sonic/sonic_release ]; then cat $FILESYSTEM_ROOT/etc/sonic/sonic_release; fi)" -export build_date="$(date -u)" -export build_number="${BUILD_NUMBER:-0}" -export built_by="$USER@$BUILD_HOSTNAME" -export sonic_os_version="${SONIC_OS_VERSION}" -j2 files/build_templates/sonic_version.yml.j2 | sudo tee $FILESYSTEM_ROOT/etc/sonic/sonic_version.yml # Default users info export password_expire="$( [[ "$CHANGE_DEFAULT_PASSWORD" == "y" ]] && echo true || echo false )" @@ -615,6 +600,60 @@ if [[ ! -f './asic_config_checksum' ]]; then fi sudo cp ./asic_config_checksum $FILESYSTEM_ROOT/etc/sonic/asic_config_checksum +## Check if not a last stage of RFS build +fi + +if [[ $RFS_SPLIT_FIRST_STAGE == y ]]; then + echo '[INFO] Finished with RFS first stage' + echo '[INFO] Umount all' + + ## Display all process details access /proc + sudo LANG=C chroot $FILESYSTEM_ROOT fuser -vm /proc + ## Kill the processes + sudo LANG=C chroot $FILESYSTEM_ROOT fuser -km /proc || true + ## Wait fuser fully kill the processes + sudo timeout 15s bash -c 'until LANG=C chroot $0 umount /proc; do sleep 1; done' $FILESYSTEM_ROOT || true + + sudo rm -f $TARGET_PATH/$RFS_SQUASHFS_NAME + sudo mksquashfs $FILESYSTEM_ROOT $TARGET_PATH/$RFS_SQUASHFS_NAME -Xcompression-level 1 + + exit 0 +fi + +if [[ $RFS_SPLIT_LAST_STAGE == y ]]; then + echo '[INFO] RFS build: second stage' + + ## ensure proc is mounted + sudo mount proc /proc -t proc || true + + sudo fuser -vm $FILESYSTEM_ROOT || true + sudo rm -rf $FILESYSTEM_ROOT + sudo unsquashfs -d $FILESYSTEM_ROOT $TARGET_PATH/$RFS_SQUASHFS_NAME + + ## make / as a mountpoint in chroot env, needed by dockerd + pushd $FILESYSTEM_ROOT + sudo mount --bind . . + popd + + trap_push 'sudo LANG=C chroot $FILESYSTEM_ROOT umount /proc || true' + sudo LANG=C chroot $FILESYSTEM_ROOT mount proc /proc -t proc +fi + +## Version file part 2 +export build_version="${SONIC_IMAGE_VERSION}" +export debian_version="$(cat $FILESYSTEM_ROOT/etc/debian_version)" +export kernel_version="${kversion}" +export asic_type="${sonic_asic_platform}" +export asic_subtype="${TARGET_MACHINE}" +export commit_id="$(git rev-parse --short HEAD)" +export branch="$(git rev-parse --abbrev-ref HEAD)" +export release="$(if [ -f $FILESYSTEM_ROOT/etc/sonic/sonic_release ]; then cat $FILESYSTEM_ROOT/etc/sonic/sonic_release; fi)" +export build_date="$(date -u)" +export build_number="${BUILD_NUMBER:-0}" +export built_by="$USER@$BUILD_HOSTNAME" +export sonic_os_version="${SONIC_OS_VERSION}" +j2 files/build_templates/sonic_version.yml.j2 | sudo tee $FILESYSTEM_ROOT/etc/sonic/sonic_version.yml + if [ -f sonic_debian_extension.sh ]; then ./sonic_debian_extension.sh $FILESYSTEM_ROOT $PLATFORM_DIR $IMAGE_DISTRO fi diff --git a/rules/functions b/rules/functions index 33f4def2a..44a6af3dd 100644 --- a/rules/functions +++ b/rules/functions @@ -143,7 +143,6 @@ SONIC_DBG_DOCKERS += $(2) endef - ############################################################################### ## Utility functions ############################################################################### diff --git a/slave.mk b/slave.mk index f2cbfc8c4..069c83eec 100644 --- a/slave.mk +++ b/slave.mk @@ -448,6 +448,34 @@ else $(info SONiC Build System for $(CONFIGURED_PLATFORM):$(CONFIGURED_ARCH)) endif +# Definition of SONIC_RFS_TARGETS +define rfs_get_installer_dependencies +$(call rfs_build_target_name,$(1),$($(1)_MACHINE)) +endef + +define rfs_build_target_name +$(1)__$(2)__rfs.squashfs +endef + +define rfs_define_target +$(eval rfs_target=$(call rfs_build_target_name,$(1),$($(1)_MACHINE))) +$(eval $(rfs_target)_INSTALLER=$(1)) +$(eval $(rfs_target)_MACHINE=$($(1)_MACHINE)) +$(eval SONIC_RFS_TARGETS+=$(rfs_target)) + +$(if $($(1)_DEPENDENT_MACHINE),\ + $(eval dependent_rfs_target=$(call rfs_build_target_name,$(1),$($(1)_DEPENDENT_MACHINE))) + $(eval $(dependent_rfs_target)_INSTALLER=$(1)) + $(eval $(dependent_rfs_target)_MACHINE=$($(1)_DEPENDENT_MACHINE)) + $(eval SONIC_RFS_TARGETS+=$(dependent_rfs_target)) + $(eval $(rfs_target)_DEPENDENT_RFS=$(dependent_rfs_target))) +endef + +$(foreach installer,$(SONIC_INSTALLERS),$(eval $(call rfs_define_target,$(installer)))) +$(foreach installer, $(SONIC_INSTALLERS), $(eval $(installer)_RFS_DEPENDS=$(call rfs_get_installer_dependencies,$(installer)))) + +SONIC_TARGET_LIST += $(addprefix $(TARGET_PATH)/, $(SONIC_RFS_TARGETS)) + # Overwrite the buildinfo in slave container ifeq ($(filter clean,$(MAKECMDGOALS)),) $(shell DBGOPT='$(DBGOPT)' scripts/prepare_slave_container_buildinfo.sh $(SLAVE_DIR) $(CONFIGURED_ARCH) $(BLDENV)) @@ -1210,6 +1238,62 @@ $(DOCKER_LOAD_TARGETS) : $(TARGET_PATH)/%.gz-load : .platform docker-start $$(TA ## Installers ############################################################################### +$(addprefix $(TARGET_PATH)/, $(SONIC_RFS_TARGETS)) : $(TARGET_PATH)/% : \ + .platform \ + build_debian.sh \ + $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(INITRAMFS_TOOLS) $(LINUX_KERNEL)) \ + $(addsuffix -install,$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(DEBOOTSTRAP))) \ + $$(addprefix $(TARGET_PATH)/,$$($$*_DEPENDENT_RFS)) \ + $(call dpkg_depend,$(TARGET_PATH)/%.dep) + $(HEADER) + + $(call LOAD_CACHE,$*,$@) + + # Skip building the target if it is already loaded from cache + if [ -z '$($*_CACHE_LOADED)' ] ; then + + $(eval installer=$($*_INSTALLER)) + $(eval machine=$($*_MACHINE)) + + export debs_path="$(IMAGE_DISTRO_DEBS_PATH)" + export initramfs_tools="$(IMAGE_DISTRO_DEBS_PATH)/$(INITRAMFS_TOOLS)" + export linux_kernel="$(IMAGE_DISTRO_DEBS_PATH)/$(LINUX_KERNEL)" + export kversion="$(KVERSION)" + export image_type="$($(installer)_IMAGE_TYPE)" + export sonicadmin_user="$(USERNAME)" + export sonic_asic_platform="$(patsubst %-$(CONFIGURED_ARCH),%,$(CONFIGURED_PLATFORM))" + export RFS_SPLIT_FIRST_STAGE=y + export RFS_SPLIT_LAST_STAGE=n + + j2 -f env files/initramfs-tools/union-mount.j2 onie-image.conf > files/initramfs-tools/union-mount + j2 -f env files/initramfs-tools/arista-convertfs.j2 onie-image.conf > files/initramfs-tools/arista-convertfs + + RFS_SQUASHFS_NAME=$* \ + USERNAME="$(USERNAME)" \ + PASSWORD="$(PASSWORD)" \ + CHANGE_DEFAULT_PASSWORD="$(CHANGE_DEFAULT_PASSWORD)" \ + TARGET_MACHINE=$(machine) \ + IMAGE_TYPE=$($(installer)_IMAGE_TYPE) \ + TARGET_PATH=$(TARGET_PATH) \ + TRUSTED_GPG_URLS=$(TRUSTED_GPG_URLS) \ + SONIC_ENABLE_SECUREBOOT_SIGNATURE="$(SONIC_ENABLE_SECUREBOOT_SIGNATURE)" \ + SIGNING_KEY="$(SIGNING_KEY)" \ + SIGNING_CERT="$(SIGNING_CERT)" \ + PACKAGE_URL_PREFIX=$(PACKAGE_URL_PREFIX) \ + DBGOPT='$(DBGOPT)' \ + SONIC_VERSION_CACHE=$(SONIC_VERSION_CACHE) \ + MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) \ + CROSS_BUILD_ENVIRON=$(CROSS_BUILD_ENVIRON) \ + MASTER_KUBERNETES_VERSION=$(MASTER_KUBERNETES_VERSION) \ + MASTER_CRI_DOCKERD=$(MASTER_CRI_DOCKERD) \ + ./build_debian.sh $(LOG) + + $(call SAVE_CACHE,$*,$@) + + fi + + $(FOOTER) + # targets for building installers with base image $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ .platform \ @@ -1264,7 +1348,9 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ $(addprefix $(FILES_PATH)/,$($(SONIC_CTRMGRD)_FILES)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_YANG_MGMT_PY3)) \ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SYSTEM_HEALTH)) \ - $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3)) + $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_HOST_SERVICES_PY3)) \ + $$(addprefix $(TARGET_PATH)/,$$($$*_RFS_DEPENDS)) + $(HEADER) # Pass initramfs and linux kernel explicitly. They are used for all platforms export debs_path="$(IMAGE_DISTRO_DEBS_PATH)" @@ -1420,6 +1506,9 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ chmod +x sonic_debian_extension.sh, ) + export RFS_SPLIT_FIRST_STAGE=n + export RFS_SPLIT_LAST_STAGE=y + # Build images for the MACHINE, DEPENDENT_MACHINE defined. $(foreach dep_machine, $($*_MACHINE) $($*_DEPENDENT_MACHINE), \ DEBUG_IMG="$(INSTALL_DEBUG_TOOLS)" \ @@ -1427,6 +1516,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \ DEBUG_SRC_ARCHIVE_FILE="$(DBG_SRC_ARCHIVE_FILE)" \ scripts/dbg_files.sh + RFS_SQUASHFS_NAME=$*__$(dep_machine)__rfs.squashfs \ DEBUG_IMG="$(INSTALL_DEBUG_TOOLS)" \ DEBUG_SRC_ARCHIVE_FILE="$(DBG_SRC_ARCHIVE_FILE)" \ USERNAME="$(USERNAME)" \ @@ -1521,7 +1611,9 @@ SONIC_CLEAN_TARGETS += $(addsuffix -clean,$(addprefix $(TARGET_PATH)/, \ $(SONIC_DOCKER_IMAGES) \ $(SONIC_DOCKER_DBG_IMAGES) \ $(SONIC_SIMPLE_DOCKER_IMAGES) \ - $(SONIC_INSTALLERS))) + $(SONIC_INSTALLERS) \ + $(SONIC_RFS_TARGETS))) + $(SONIC_CLEAN_TARGETS) :: $(TARGET_PATH)/%-clean : .platform $(Q)rm -rf $(TARGET_PATH)/$* target/versions/dockers/$(subst .gz,,$*)