Modify mini-mirror to build from yaml

This modifies mini-mirror so that instead of parsing through .txt
documents to build the mirror, it instead has one yaml file consisting
of multiple yaml documents with a pre-defined structure that has all
of the same information. This will allow for easier modification and
visibility into the configuration of mini-mirror.

Along with that, this enables the manual setting of CodeName and Label
values of the mirror, as well as the ability to change the name of the
image that the build.sh script creates

Change-Id: I08646258492e8938b8892749172aea23e363af3d

mini-mirror/Dockerfile.ubuntu_bionic

@@ -15,12 +15,13 @@
 FROM ubuntu:18.04 as aptly
 
 ARG APTLY_CONFIG_PATH=etc/aptly.conf
-ARG MIRROR_SOURCE_DIR=sources
+ARG MIRROR_SOURCE_FILE=mini-mirror-sources.yaml
 ARG RELEASE_SIGN_KEY_PATH=etc
 ARG RELEASE_SIGN_KEY_PASSPHRASE
 
 RUN apt-get update
-RUN apt-get install -y gnupg wget
+RUN apt-get install -y gnupg wget jq python-pip
+RUN pip install yq
 
 COPY "${APTLY_CONFIG_PATH}" /etc/aptly.conf
 COPY tools/install_aptly.sh /opt/install_aptly.sh
@@ -31,7 +32,7 @@ ARG APTLY_REFSPEC=allow-custom-codename
 
 RUN /opt/install_aptly.sh
 
-COPY "${MIRROR_SOURCE_DIR}" /opt/sources
+COPY "${MIRROR_SOURCE_FILE}" /opt/mini-mirror-sources.yaml
 COPY "${RELEASE_SIGN_KEY_PATH}" /opt/release.gpg
 COPY tools/publish_snapshots.sh /opt/publish_snapshots.sh
 

mini-mirror/Dockerfile.ubuntu_xenial

@@ -15,12 +15,13 @@
 FROM ubuntu:16.04 as aptly
 
 ARG APTLY_CONFIG_PATH=etc/aptly.conf
-ARG MIRROR_SOURCE_DIR=sources
+ARG MIRROR_SOURCE_FILE=mini-mirror-sources.yaml
 ARG RELEASE_SIGN_KEY_PATH=etc
 ARG RELEASE_SIGN_KEY_PASSPHRASE
 
 RUN apt-get update
-RUN apt-get install -y wget
+RUN apt-get install -y wget jq python-pip
+RUN pip install yq
 
 COPY "${APTLY_CONFIG_PATH}" /etc/aptly.conf
 COPY tools/install_aptly.sh /opt/install_aptly.sh
@@ -31,7 +32,7 @@ ARG APTLY_REFSPEC=allow-custom-codename
 
 RUN /opt/install_aptly.sh
 
-COPY "${MIRROR_SOURCE_DIR}" /opt/sources
+COPY "${MIRROR_SOURCE_FILE}" /opt/mini-mirror-sources.yaml
 COPY "${RELEASE_SIGN_KEY_PATH}" /opt/release.gpg
 COPY tools/publish_snapshots.sh /opt/publish_snapshots.sh
 

mini-mirror/README.rst

@@ -11,50 +11,39 @@ Build Requirements
 Add mirror sources
 ~~~~~~~~~~~~~~~~~~
 
-Mini-mirror requires a directory at build-time that contains the repositories
+Mini-mirror requires a YAML file at build-time that contains the repositories
-and packages that will be mirrored.
+and packages that will be mirrored as different YAML documents.
 
-.. code::
+.. code:: yaml
+    ---
+    name: <Repository name (i.e. the directory a source serves from)>
+    url: <URL link to the source repository>
+    key_url: <URL link to the key for the source repository>
+    codename: *<Override codename for the release file>
+    label: *<Override label for the release file>
+    aptly_config: | # *Inline aptly config JSON file to replace default
+      { }
+    components: # List of Components
+      - <Component>
+    subrepos: # List of repositories within the source repository
+      - distribution: <Distribution name of the repository>
+        packages: # <List of all packages>
+          - name: <Package name>
+            version: *<Version of package to pin to>
+    ...
+    ---
+    # Additional repository document here
+    ...
 
-    sources/
+*Optional
-    | -- source1-prefix/
-         |-- source-name/
-             |-- source.txt
-             |-- packages.txt
-    | -- source2-prefix/
-         |-- source-name/
-             |-- source.txt
-             |-- packages.txt
 
-Sources are defined as directories containing the files:
 
-* source-prefix - a prefix to separate sources that have conflicting
+To specify the location of your sources YAML file, export the following
-  distribution names (i.e. the directory a source serves from).
-* source-name - the name of a source; used for record-keeping.
-* source.txt - contains location and metadata information for a source.
-* packages.txt - contains a list of packages, formatted as `package queries <https://www.aptly.info/doc/feature/query/>`_
-  for a source.
-
-Example ``source.txt`` format:
-
- .. code::
-
-    source_url source_key_url dist components
-
-Example ``packages.txt`` format:
-
-.. code::
-
-    package1
-    package2
-    package3 (>=3.6)
-
-To specify the location of your sources directory, export the following
 environment variable:
 
 .. code:: bash
 
-    export MIRROR_SOURCE_DIR=/path/to/sources
+    export MIRROR_SOURCE_FILE=/path/to/sources.yaml
 
 Generate a signing key
 ~~~~~~~~~~~~~~~~~~~~~~
@@ -92,8 +81,8 @@ environment variable:
 .. NOTE::
 
     Mini-mirror can be configured on a per-repo basis by adding an Aptly config
-    file to the root directory of a source. This overrides the Aptly config
+    file to the .aptly_config key in the YAML document. This overrides
-    file taken from ``APTLY_CONFIG_PATH``.
+    the Aptly config file taken from ``APTLY_CONFIG_PATH``.
 
 Proxy
 ~~~~~

mini-mirror/build.sh

@@ -20,7 +20,8 @@ SCRIPT_DIR=$(dirname "${SCRIPT}")
 ## Only build from main folder
 cd "${SCRIPT_DIR}"/.. || exit
 
-IMAGE="mini-mirror"
+PROJECT_PATH="mini-mirror"
+IMAGE=${IMAGE:-mini-mirror}
 VERSION=${VERSION:-latest}
 DISTRO=${DISTRO:-ubuntu_xenial}
 REGISTRY_URI=${REGISTRY_URI:-"openstackhelm/"}
@@ -31,7 +32,7 @@ HTTPS_PROXY=${HTTPS_PROXY:-""}
 NO_PROXY=${NO_PROXY:-"127.0.0.1,localhost"}
 
 APTLY_CONFIG_PATH=${APTLY_CONFIG_PATH:-"etc/aptly.conf"}
-MIRROR_SOURCE_DIR=${MIRROR_SOURCE_DIR:-"sources"}
+MIRROR_SOURCE_FILE=${MIRROR_SOURCE_FILE:-"mini-mirror-sources.yaml"}
 RELEASE_SIGN_KEY_PATH=${RELEASE_SIGN_KEY_PATH:-"etc"}
 RELEASE_SIGN_KEY_PASSPHRASE=${RELEASE_SIGN_KEY_PASSPHRASE:-""}
 
@@ -42,7 +43,7 @@ APTLY_INSTALL_FROM=${APTLY_INSTALL_FROM:-"source"}
 APTLY_REPO=${APTLY_REPO:-"https://github.com/smstone/aptly.git"}
 APTLY_REFSPEC=${APTLY_REFSPEC:-"allow-custom-codename"}
 
-docker build -f "${IMAGE}"/Dockerfile."${DISTRO}" --network=host \
+docker build -f "${PROJECT_PATH}"/Dockerfile."${DISTRO}" --network=host \
   -t "${REGISTRY_URI}""${IMAGE}":"${VERSION}"-"${DISTRO}""${EXTRA_TAG_INFO}" \
   --build-arg http_proxy="${HTTP_PROXY}" \
   --build-arg https_proxy="${HTTPS_PROXY}" \
@@ -51,12 +52,12 @@ docker build -f "${IMAGE}"/Dockerfile."${DISTRO}" --network=host \
   --build-arg no_proxy="${NO_PROXY}" \
   --build-arg NO_PROXY="${NO_PROXY}" \
   --build-arg APTLY_CONFIG_PATH="${APTLY_CONFIG_PATH}" \
-  --build-arg MIRROR_SOURCE_DIR="${MIRROR_SOURCE_DIR}" \
+  --build-arg MIRROR_SOURCE_FILE="${MIRROR_SOURCE_FILE}" \
   --build-arg RELEASE_SIGN_KEY_PATH="${RELEASE_SIGN_KEY_PATH}" \
   --build-arg RELEASE_SIGN_KEY_PASSPHRASE="${RELEASE_SIGN_KEY_PASSPHRASE}" \
   --build-arg APTLY_INSTALL_FROM="${APTLY_INSTALL_FROM}" \
   --build-arg APTLY_REPO="${APTLY_REPO}" \
   --build-arg APTLY_REFSPEC="${APTLY_REFSPEC}" \
-  ${extra_build_args} "${IMAGE}"
+  ${extra_build_args} "${PROJECT_PATH}"
 
 cd - || exit

mini-mirror/etc/aptly.conf

@@ -2,7 +2,7 @@
   "rootDir": "/opt/.aptly",
   "downloadConcurrency": 4,
   "downloadSpeedLimit": 0,
-  "architectures": [],
+  "architectures": ["amd64"],
   "dependencyFollowSuggests": false,
   "dependencyFollowRecommends": false,
   "dependencyFollowAllVariants": false,

mini-mirror/mini-mirror-sources.yaml

@@ -0,0 +1,43 @@
+---
+name: aptly-example
+url: http://repo.aptly.info/
+key_url: https://www.aptly.info/pubkey.txt
+components:
+  - main
+subrepos:
+  - distribution: squeeze
+    packages:
+      - name: aptly
+...
+# Example additional config for adding docker
+#---
+#name: docker
+#url: https://download.docker.com/linux/ubuntu
+#key_url: https://download.docker.com/linux/ubuntu/gpg
+#aptly_config: |
+#  {
+#    "rootDir": "/opt/.aptly",
+#    "downloadConcurrency": 20,
+#    "downloadSpeedLimit": 0,
+#    "architectures": ["amd64"],
+#    "dependencyFollowSuggests": false,
+#    "dependencyFollowRecommends": false,
+#    "dependencyFollowAllVariants": false,
+#    "dependencyFollowSource": false,
+#    "dependencyVerboseResolve": true,
+#    "gpgDisableSign": true,
+#    "gpgDisableVerify": true,
+#    "gpgProvider": "gpg",
+#    "downloadSourcePackages": false,
+#    "skipLegacyPool": true,
+#    "ppaDistributorID": "ubuntu",
+#    "ppaCodename": ""
+#  }
+#components:
+#  - stable
+#subrepos:
+#  - distribution: xenial
+#    packages:
+#      - name: docker-ce
+#        version: 17.03.3~ce-0~ubuntu-xenial_amd64
+#...

mini-mirror/sources/aptly-example/squeeze/packages.txt

@@ -1 +0,0 @@
-aptly

mini-mirror/sources/aptly-example/squeeze/source.txt

@@ -1 +0,0 @@
-http://repo.aptly.info/ https://www.aptly.info/pubkey.txt squeeze main

mini-mirror/tools/publish_snapshots.sh

@@ -14,23 +14,42 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-set -e
+set -ex
 
-if [ ! -z "$1" ]; then
+if [[ ! -z "$1" ]]; then
   gpg --import /opt/release.gpg
 fi
 
-for source_prefix in /opt/sources/*; do
+sources=$(yq "." /opt/mini-mirror-sources.yaml | jq -s '.')
-  for source in $source_prefix/*; do
+
-    read -r -a info < "${source}"/source.txt
+# Loop to iterate over each document in the YAML file.
-    repo=${info[0]}
+# By base64 encoding and then decoding the output from jq we are able to
-    key=${info[1]}
+# cleanly iterate over the output even if it contains newlines, etc.
-    dist=${info[2]}
+for source in $(echo "${sources}" | jq -r '.[] | @base64' ); do
-    components=${info[*]:3}
+  _source() {
+    echo "${source}" | base64 --decode | jq -r "${*}"
+  }
+
+  source_name=$(_source '.name')
+  repo=$(_source '.url')
+  key=$(_source '.key_url')
+  components=$(_source '.components')
+  label=$(_source '.label')
+  codename=$(_source '.code_name')
+
+  # Loop to iterate over the `subrepo` list in the document
+  for subrepo in $(_source '.subrepos[] | @base64'); do
+    _subrepo() {
+    echo "${subrepo}" | base64 --decode | jq -r "${*}"
+    }
+
+    dist=$(_subrepo '.distribution')
 
     # Use source specific aptly config when provided
-    if [ -f "${source}"/aptly.conf ]; then
+    source_conf=$(_source '.aptly_config')
-      conf="${source}"/aptly.conf
+    if [[ "$source_conf" != "null" ]]; then
+      echo "${source_conf}" > aptly.conf
+      conf=$(pwd)/aptly.conf
     else
       conf=/etc/aptly.conf
     fi
@@ -38,11 +57,21 @@ for source_prefix in /opt/sources/*; do
     # Create package query from well-defined package list.
     #
     #    package1
-    #    package2      ==>      package1 | package2 | package3
+    #    package2      ==>      package1 | package2 (=1.0) | package3
     #    package3
     #
-    packages=$(awk -v ORS=" | " '{ print $1 }' "${source}"/packages.txt)
+
-    packages="${packages::-3}"
+    # Grab packages from .subrepo
+    packages=$(_subrepo '.packages')
+    # Convert any found versions to strings
+    str_versions=$(echo "${packages}" | jq ' .[] | if .version != null then {name: .name, version: .version | tostring} else {name: .name} end')
+    # Format packages <pkg> and versions <ver> to "<pkg> (=@<ver>"
+    formatted_packages=$(echo "${str_versions}" | jq -r '. | join(" (=@")')
+    # Substitute "@<ver>" with "<ver>)" so the new format is "<pkg> (=<ver>)"
+    # and bring the packages on to one line separated by "@"
+    wrap_versions=$(echo "${formatted_packages}" | sed -r "s/@(.*)/\1\)/g" | tr "\n" "@")
+    # Substitute the "@" between packages with " | "
+    package_query=$(echo "${wrap_versions}" | sed -r "s/@/ \| /g" | sed -r "s/ \| $//g")
 
     # Import source key
     wget --no-check-certificate -O - "${key}" | gpg --no-default-keyring \
@@ -51,13 +80,14 @@ for source_prefix in /opt/sources/*; do
     # Create a mirror of each component from a source's repository, update it,
     # and publish a snapshot of it.
     mirrors=()
-    for component in $components; do
+    # Loop to iterate over the `component` list in the document
-      name="${source}-${component}"
+    for component in $(echo "${components}" | jq -r '.[]' ); do
+      name="${source_name}-${dist}-${component}"
       mirrors+=("$name")
 
       aptly mirror create \
           -config="${conf}" \
-          -filter="${packages}" \
+          -filter="${package_query}" \
           -filter-with-deps \
           "${name}" "${repo}" "${dist}" "${component}"
 
@@ -65,28 +95,35 @@ for source_prefix in /opt/sources/*; do
       aptly snapshot create -config="${conf}" "${name}" from mirror "${name}"
     done
 
-    # preserve the codename and label of the source repository
+    # If the codename or label have not been specified then acquire them from
-    codename=$(aptly mirror show ${mirrors[0]} | sed -n 's/^Codename: //p')
+    # the mirror
-    label=$(aptly mirror show ${mirrors[0]} | sed -n 's/^Label: //p')
+    if [[ "${codename}" == "null" ]]; then
+        codename=$(aptly mirror show "${mirrors[0]}" | sed -n 's/^Codename: //p')
+    fi
+    if [[ "${label}" == "null" ]]; then
+        label=$(aptly mirror show "${mirrors[0]}" | sed -n 's/^Label: //p')
+    fi
 
     # Publish snapshot and sign if a key passphrase is provided.
-    com_list=$(echo "${components[@]}" | tr ' ' ',')
+    com_list=$(echo "${components}" | jq -r '. | join(",")')
-    if [ ! -z "$1" ]; then
+    if [[ ! -z "$1" ]]; then
       aptly publish snapshot \
           -batch=true \
+          -config="${conf}" \
           -component="${com_list}" \
           -distribution="${dist}" \
           -passphrase="${1}" \
           -codename="${codename}" \
           -label="${label}" \
-          "${mirrors[@]}" "${source_prefix:13}"
+          "${mirrors[@]}" "${source_name}"
     else
       aptly publish snapshot \
+          -config="${conf}" \
           -component="${com_list}" \
           -distribution="${dist}" \
           -codename="${codename}" \
           -label="${label}" \
-          "${mirrors[@]}" "${source_prefix:13}"
+          "${mirrors[@]}" "${source_name}"
     fi
   done
 done