From 28ea0730b655aa16420cc328c972a905ad6ee2ae Mon Sep 17 00:00:00 2001 From: Thomas Eizinger Date: Thu, 23 Oct 2025 21:09:07 +1100 Subject: [PATCH] feat(apt): import `.deb` files from `import-` directory (#10694) Currently, the `sync-apt.sh` script just generates metadata for all packages found in the `.deb` directory. Unfortunately, this requires the packages to already be uploaded with a certain naming convention, otherwise `apt-ftparchive packages` doesn't actually detect them and creates an empty `Packages` file. The solution here is to extend the `sync-apt.sh` script to normalize the filename to what we need it to be. This requires us to upload the new `.deb` files to the `pool` directory. Instead of messing around with the existing files in there, we slightly change how the `sync-apt.sh` script works. In its new version, it expects packages to be in the `import-stable` and `import-preview` directories. It will then download these, normalize their names and move them to a local `pool-stable` and `pool-preview` directory respectively (potentially overwriting and existing one that is already there, this allows for updating packages). As a final step, it will generate the metadata for all packages in `pool-stable` and `pool-preview`, upload both directories, upload the metadata and then delete the imported `.deb` files. --- .github/workflows/publish-release.yml | 4 +- scripts/sync-apt.sh | 82 ++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 4 deletions(-) diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index 63e078df8..9bbc25512 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -198,7 +198,7 @@ jobs: run: | set -xe - # Download all .deb assets directly to pool + # Download all .deb assets gh release download "${{ inputs.release_name || github.event.release.name }}" --pattern "*.deb" # List downloaded files for verification @@ -209,7 +209,7 @@ jobs: --destination apt \ --source . \ --pattern "*.deb" \ - --destination-path pool \ + --destination-path import-stable \ --overwrite \ --no-progress \ --connection-string "${{ secrets.AZURERM_ARTIFACTS_CONNECTION_STRING }}" diff --git a/scripts/sync-apt.sh b/scripts/sync-apt.sh index 39ea6dcfa..dbe6c0930 100755 --- a/scripts/sync-apt.sh +++ b/scripts/sync-apt.sh @@ -1,5 +1,12 @@ #!/usr/bin/env bash + +# Shell script for syncing the APT repository metadata from a set of `.deb` files. # +# This script maintains two release channels: stable and preview. +# It expects the `.deb` files for these channels to be in the `pool-stable` and `pool-preview` directories. +# To add new packages to the repository, upload them to the `import-stable` and `import-preview` directories NOT to the `pool-` directories. +# The `pool-` directories are referenced by the live repository metadata and the files in there need to atomically change with the metadata. + set -euo pipefail COMPONENT="main" @@ -24,8 +31,12 @@ trap cleanup EXIT for DISTRIBUTION in "stable" "preview"; do POOL_DIR="${WORK_DIR}/pool-${DISTRIBUTION}" + IMPORT_DIR="${WORK_DIR}/import-${DISTRIBUTION}" - echo "Downloading packages for distribution $DISTRIBUTION..." + mkdir --parents "${POOL_DIR}" + mkdir --parents "${IMPORT_DIR}" + + echo "Downloading existing packages for distribution $DISTRIBUTION..." az storage blob download-batch \ --destination "${WORK_DIR}" \ @@ -34,6 +45,55 @@ for DISTRIBUTION in "stable" "preview"; do --connection-string "${AZURERM_ARTIFACTS_CONNECTION_STRING}" \ 2>&1 | grep -v "WARNING" || true + echo "Downloading import packages for distribution $DISTRIBUTION..." + + az storage blob download-batch \ + --destination "${WORK_DIR}" \ + --source apt \ + --pattern "import-${DISTRIBUTION}/*.deb" \ + --connection-string "${AZURERM_ARTIFACTS_CONNECTION_STRING}" \ + 2>&1 | grep -v "WARNING" || true + + if [ "$(ls -A "${IMPORT_DIR}")" ]; then + echo "Normalizing package names..." + + for deb in "${IMPORT_DIR}"/*.deb; do + if [ -f "$deb" ]; then + # Extract metadata from the .deb file + PACKAGE=$(dpkg-deb -f "$deb" Package 2>/dev/null) + VERSION=$(dpkg-deb -f "$deb" Version 2>/dev/null) + ARCH=$(dpkg-deb -f "$deb" Architecture 2>/dev/null) + + # Skip if any field is missing + if [ -z "$PACKAGE" ] || [ -z "$VERSION" ] || [ -z "$ARCH" ]; then + echo "Warning: Could not extract metadata from $(basename "$deb"), skipping" + continue + fi + + # Construct the proper filename + NORMALIZED_NAME="${PACKAGE}_${VERSION}_${ARCH}.deb" + NORMALIZED_PATH="${IMPORT_DIR}/${NORMALIZED_NAME}" + + # Rename if needed + if [ "$(basename "$deb")" == "$NORMALIZED_NAME" ]; then + continue + fi + + echo "Renaming $(basename "$deb") → $NORMALIZED_NAME" + mv "$deb" "$NORMALIZED_PATH" + fi + done + + echo "Importing new packages..." + mv --force --target-directory="${POOL_DIR}/" "${IMPORT_DIR}"/*.deb + fi + + if [ -z "$(ls -A "${POOL_DIR}")" ]; then + echo "No packages for distribution ${DISTRIBUTION}" + + continue + fi + echo "Detecting architectures..." ARCHITECTURES=$(for deb in "${POOL_DIR}"/*.deb; do dpkg-deb -f "$deb" Architecture 2>/dev/null; done | sort -u | tr '\n' ' ') || true @@ -42,11 +102,13 @@ for DISTRIBUTION in "stable" "preview"; do echo "Generating metadata..." mkdir -p "${DISTS_DIR}/${DISTRIBUTION}/${COMPONENT}" + cd "$WORK_DIR" + for ARCH in $ARCHITECTURES; do BINARY_DIR="${DISTS_DIR}/${DISTRIBUTION}/${COMPONENT}/binary-${ARCH}" mkdir -p "${BINARY_DIR}" - apt-ftparchive packages --arch "${ARCH}" "${POOL_DIR}/" >"${BINARY_DIR}/Packages" + apt-ftparchive packages --arch "${ARCH}" "pool-${DISTRIBUTION}" >"${BINARY_DIR}/Packages" gzip -k -f "${BINARY_DIR}/Packages" cat >"${BINARY_DIR}/Release" <