feat(apt): import .deb files from import- directory (#10694)

Currently, the `sync-apt.sh` script just generates metadata for all
packages found in the `.deb` directory. Unfortunately, this requires the
packages to already be uploaded with a certain naming convention,
otherwise `apt-ftparchive packages` doesn't actually detect them and
creates an empty `Packages` file.

The solution here is to extend the `sync-apt.sh` script to normalize the
filename to what we need it to be. This requires us to upload the new
`.deb` files to the `pool` directory. Instead of messing around with the
existing files in there, we slightly change how the `sync-apt.sh` script
works.

In its new version, it expects packages to be in the `import-stable` and
`import-preview` directories. It will then download these, normalize
their names and move them to a local `pool-stable` and `pool-preview`
directory respectively (potentially overwriting and existing one that is
already there, this allows for updating packages).

As a final step, it will generate the metadata for all packages in
`pool-stable` and `pool-preview`, upload both directories, upload the
metadata and then delete the imported `.deb` files.

.github/workflows/publish-release.yml

@@ -198,7 +198,7 @@ jobs:
         run: |
           set -xe
 
-          # Download all .deb assets directly to pool
+          # Download all .deb assets
           gh release download "${{ inputs.release_name || github.event.release.name }}" --pattern "*.deb"
 
           # List downloaded files for verification
@@ -209,7 +209,7 @@ jobs:
           --destination apt \
           --source . \
           --pattern "*.deb" \
-          --destination-path pool \
+          --destination-path import-stable \
           --overwrite \
           --no-progress \
           --connection-string "${{ secrets.AZURERM_ARTIFACTS_CONNECTION_STRING }}"

scripts/sync-apt.sh

@@ -1,5 +1,12 @@
 #!/usr/bin/env bash
+
+# Shell script for syncing the APT repository metadata from a set of `.deb` files.
 #
+# This script maintains two release channels: stable and preview.
+# It expects the `.deb` files for these channels to be in the `pool-stable` and `pool-preview` directories.
+# To add new packages to the repository, upload them to the `import-stable` and `import-preview` directories NOT to the `pool-` directories.
+# The `pool-` directories are referenced by the live repository metadata and the files in there need to atomically change with the metadata.
+
 set -euo pipefail
 
 COMPONENT="main"
@@ -24,8 +31,12 @@ trap cleanup EXIT
 
 for DISTRIBUTION in "stable" "preview"; do
     POOL_DIR="${WORK_DIR}/pool-${DISTRIBUTION}"
+    IMPORT_DIR="${WORK_DIR}/import-${DISTRIBUTION}"
 
-    echo "Downloading packages for distribution $DISTRIBUTION..."
+    mkdir --parents "${POOL_DIR}"
+    mkdir --parents "${IMPORT_DIR}"
+
+    echo "Downloading existing packages for distribution $DISTRIBUTION..."
 
     az storage blob download-batch \
         --destination "${WORK_DIR}" \
@@ -34,6 +45,55 @@ for DISTRIBUTION in "stable" "preview"; do
         --connection-string "${AZURERM_ARTIFACTS_CONNECTION_STRING}" \
         2>&1 | grep -v "WARNING" || true
 
+    echo "Downloading import packages for distribution $DISTRIBUTION..."
+
+    az storage blob download-batch \
+        --destination "${WORK_DIR}" \
+        --source apt \
+        --pattern "import-${DISTRIBUTION}/*.deb" \
+        --connection-string "${AZURERM_ARTIFACTS_CONNECTION_STRING}" \
+        2>&1 | grep -v "WARNING" || true
+
+    if [ "$(ls -A "${IMPORT_DIR}")" ]; then
+        echo "Normalizing package names..."
+
+        for deb in "${IMPORT_DIR}"/*.deb; do
+            if [ -f "$deb" ]; then
+                # Extract metadata from the .deb file
+                PACKAGE=$(dpkg-deb -f "$deb" Package 2>/dev/null)
+                VERSION=$(dpkg-deb -f "$deb" Version 2>/dev/null)
+                ARCH=$(dpkg-deb -f "$deb" Architecture 2>/dev/null)
+
+                # Skip if any field is missing
+                if [ -z "$PACKAGE" ] || [ -z "$VERSION" ] || [ -z "$ARCH" ]; then
+                    echo "Warning: Could not extract metadata from $(basename "$deb"), skipping"
+                    continue
+                fi
+
+                # Construct the proper filename
+                NORMALIZED_NAME="${PACKAGE}_${VERSION}_${ARCH}.deb"
+                NORMALIZED_PATH="${IMPORT_DIR}/${NORMALIZED_NAME}"
+
+                # Rename if needed
+                if [ "$(basename "$deb")" == "$NORMALIZED_NAME" ]; then
+                    continue
+                fi
+
+                echo "Renaming $(basename "$deb") → $NORMALIZED_NAME"
+                mv "$deb" "$NORMALIZED_PATH"
+            fi
+        done
+
+        echo "Importing new packages..."
+        mv --force --target-directory="${POOL_DIR}/" "${IMPORT_DIR}"/*.deb
+    fi
+
+    if [ -z "$(ls -A "${POOL_DIR}")" ]; then
+        echo "No packages for distribution ${DISTRIBUTION}"
+
+        continue
+    fi
+
     echo "Detecting architectures..."
     ARCHITECTURES=$(for deb in "${POOL_DIR}"/*.deb; do dpkg-deb -f "$deb" Architecture 2>/dev/null; done | sort -u | tr '\n' ' ') || true
 
@@ -42,11 +102,13 @@ for DISTRIBUTION in "stable" "preview"; do
     echo "Generating metadata..."
     mkdir -p "${DISTS_DIR}/${DISTRIBUTION}/${COMPONENT}"
 
+    cd "$WORK_DIR"
+
     for ARCH in $ARCHITECTURES; do
         BINARY_DIR="${DISTS_DIR}/${DISTRIBUTION}/${COMPONENT}/binary-${ARCH}"
         mkdir -p "${BINARY_DIR}"
 
-        apt-ftparchive packages --arch "${ARCH}" "${POOL_DIR}/" >"${BINARY_DIR}/Packages"
+        apt-ftparchive packages --arch "${ARCH}" "pool-${DISTRIBUTION}" >"${BINARY_DIR}/Packages"
         gzip -k -f "${BINARY_DIR}/Packages"
 
         cat >"${BINARY_DIR}/Release" <<EOF
@@ -72,6 +134,15 @@ EOF
 
     gpg --default-key "${GPG_KEY_ID}" -abs -o Release.gpg Release
     gpg --default-key "${GPG_KEY_ID}" --clearsign -o InRelease Release
+
+    # Upload new pool directory
+    az storage blob upload-batch \
+        --destination apt \
+        --destination-path "pool-${DISTRIBUTION}" \
+        --source "${POOL_DIR}" \
+        --connection-string "${AZURERM_ARTIFACTS_CONNECTION_STRING}" \
+        --overwrite \
+        --output table
 done
 
 echo "Uploading metadata..."
@@ -82,3 +153,10 @@ az storage blob upload-batch \
     --connection-string "${AZURERM_ARTIFACTS_CONNECTION_STRING}" \
     --overwrite \
     --output table
+
+# Delete import files
+az storage blob delete-batch \
+    --source apt \
+    --pattern "import-*/*.deb" \
+    --connection-string "${AZURERM_ARTIFACTS_CONNECTION_STRING}" \
+    --output table