feat: add ZFS modules (#10)

2026-01-27 10:20:49 +00:00 · 2023-04-01 00:25:21 -05:00
parent b327547cb5
commit fa9949e8ae
4 changed files with 262 additions and 31 deletions
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -18,9 +18,30 @@ env:
    IMAGE_REGISTRY: ghcr.io/${{ github.repository_owner }}

 jobs:
-  push-ghcr:
-    name: Build and push image
+  zfs_versions:
+    name: Get ZFS versions
+    runs-on: ubuntu-latest
+    outputs:
+      latest: ${{ steps.latest.outputs.version }}
+      previous: ${{ steps.previous.outputs.version }}
+    steps:
+      # Use cURL to fetch the given URL, saving the response to `data.json`
+      - name: Fetch zfs versions
+        # Uses project_id from: https://release-monitoring.org/project/11706/
+        run: curl "https://release-monitoring.org/api/v2/versions/?project_id=11706" -o data.json
+      - name: Get latest zfs version
+        id: latest
+        run: echo "version=$(jq -r '.stable_versions[0]' data.json)" >> $GITHUB_OUTPUT
+      - name: Get previous zfs version
+        id: previous
+        run: echo "version=$(jq -r '.stable_versions[1]' data.json)" >> $GITHUB_OUTPUT
+
+  build_zfs:
+    name: Build ZFS
    runs-on: ubuntu-22.04
+    needs: zfs_versions
+    outputs:
+      zfs_pr_prefix: ${{ steps.generate-tags.outputs.zfs_pr_prefix }}
    permissions:
      contents: read
      packages: write
@@ -29,17 +50,12 @@ jobs:
      fail-fast: false
      matrix:
        image_name:
-          - ucore
+          - ucore-zfs-rpm
        coreos_version:
          - stable
          - testing
-        include:
-          - coreos_version: stable
-            is_latest_version: true
-            is_stable_version: true
-          - coreos_version: testing
-            is_latest_version: false
-            is_stable_version: false
+        zfs_version:
+          - ${{ needs.zfs_versions.outputs.latest }}
    steps: 
      # Checkout push-to-registry action GitHub repository
      - name: Checkout Push to Registry action
@@ -52,6 +68,7 @@ jobs:
          # Generate a timestamp for creating an image version history
          TIMESTAMP="$(date +%Y%m%d)"
          COREOS_VERSION="${{ matrix.coreos_version }}"
+
          COMMIT_TAGS=()
          BUILD_TAGS=()

@@ -60,33 +77,27 @@ jobs:
          COMMIT_TAGS+=("pr-${{ github.event.number }}-${COREOS_VERSION}")
          COMMIT_TAGS+=("${SHA_SHORT}-${COREOS_VERSION}")

-          if [[ "${{ matrix.is_latest_version }}" == "true" ]] && \
-             [[ "${{ matrix.is_stable_version }}" == "true" ]]; then
-              COMMIT_TAGS+=("pr-${{ github.event.number }}")
-              COMMIT_TAGS+=("${SHA_SHORT}")
-          fi
-
          BUILD_TAGS=("${COREOS_VERSION}" "${COREOS_VERSION}-${TIMESTAMP}")

-          if [[ "${{ matrix.is_latest_version }}" == "true" ]] && \
-             [[ "${{ matrix.is_stable_version }}" == "true" ]]; then
-              BUILD_TAGS+=("${TIMESTAMP}")
-              BUILD_TAGS+=("latest")
-          fi
-
          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
              echo "Generated the following commit tags: "
              for TAG in "${COMMIT_TAGS[@]}"; do
                  echo "${TAG}"
              done
+
              alias_tags=("${COMMIT_TAGS[@]}")
+              ZFS_PR_PREFIX="pr-${{ github.event.number }}-"
          else
              alias_tags=("${BUILD_TAGS[@]}")
+              ZFS_PR_PREFIX=""
          fi
+          echo "zfs_pr_prefix=${ZFS_PR_PREFIX}" >> $GITHUB_OUTPUT
+
          echo "Generated the following build tags: "
          for TAG in "${BUILD_TAGS[@]}"; do
              echo "${TAG}"
          done
+
          echo "alias_tags=${alias_tags[*]}" >> $GITHUB_OUTPUT

      # Build metadata
@@ -98,9 +109,8 @@ jobs:
            ${{ matrix.image_name }}
          labels: |
            org.opencontainers.image.title=${{ matrix.image_name }}
-            org.opencontainers.image.description=An OCI base image of Fedora CoreOS with batteries included
+            org.opencontainers.image.description=A layer with ZFS RPMs for CoreOS
            io.artifacthub.package.readme-url=https://raw.githubusercontent.com/ublue-os/ucore/main/README.md
-            io.artifacthub.package.logo-url=https://avatars.githubusercontent.com/u/120078124?s=200&v=4

      # Build image using Buildah action
      - name: Build Image
@@ -108,13 +118,14 @@ jobs:
        uses: redhat-actions/buildah-build@v2
        with:
          containerfiles: |
-            ./main/Containerfile
-          context: ./main
+            ./zfs/Containerfile
+          context: ./zfs
          image: ${{ matrix.image_name }}
          tags: |
            ${{ steps.generate-tags.outputs.alias_tags }}
          build-args: |
            COREOS_VERSION=${{ matrix.coreos_version }}
+            ZFS_VERSION=${{ matrix.zfs_version }}
          labels: ${{ steps.meta.outputs.labels }}
          oci: false

@@ -167,7 +178,149 @@ jobs:
          COSIGN_PRIVATE_KEY: ${{ secrets.SIGNING_SECRET }}

      - name: Echo outputs
-        if: github.event_name != 'pull_request'
+        #if: github.event_name != 'pull_request'
+        run: |
+          echo "${{ toJSON(steps.push.outputs) }}"
+
+  build_main:
+    name: Build uCore
+    runs-on: ubuntu-22.04
+    needs: build_zfs
+    permissions:
+      contents: read
+      packages: write
+      id-token: write
+    strategy:
+      fail-fast: false
+      matrix:
+        image_name:
+          - ucore
+        coreos_version:
+          - stable
+          - testing
+        zfs_pr_prefix:
+          - ${{ needs.build_zfs.outputs.zfs_pr_prefix }}
+    steps:
+      # Checkout push-to-registry action GitHub repository
+      - name: Checkout Push to Registry action
+        uses: actions/checkout@v3
+
+      - name: Generate tags
+        id: generate-tags
+        shell: bash
+        run: |
+          # Generate a timestamp for creating an image version history
+          TIMESTAMP="$(date +%Y%m%d)"
+          COREOS_VERSION="${{ matrix.coreos_version }}"
+
+          COMMIT_TAGS=()
+          BUILD_TAGS=()
+
+          # Have tags for tracking builds during pull request
+          SHA_SHORT="$(git rev-parse --short HEAD)"
+          COMMIT_TAGS+=("pr-${{ github.event.number }}-${COREOS_VERSION}")
+          COMMIT_TAGS+=("${SHA_SHORT}-${COREOS_VERSION}")
+
+          BUILD_TAGS=("${COREOS_VERSION}" "${COREOS_VERSION}-${TIMESTAMP}")
+
+          if [[ "${{ github.event_name }}" == "pull_request" ]]; then
+              echo "Generated the following commit tags: "
+              for TAG in "${COMMIT_TAGS[@]}"; do
+                  echo "${TAG}"
+              done
+
+              alias_tags=("${COMMIT_TAGS[@]}")
+          else
+              alias_tags=("${BUILD_TAGS[@]}")
+          fi
+
+          echo "Generated the following build tags: "
+          for TAG in "${BUILD_TAGS[@]}"; do
+              echo "${TAG}"
+          done
+
+          echo "alias_tags=${alias_tags[*]}" >> $GITHUB_OUTPUT
+
+      # Build metadata
+      - name: Image Metadata
+        uses: docker/metadata-action@v4
+        id: meta
+        with:
+          images: |
+            ${{ matrix.image_name }}
+          labels: |
+            org.opencontainers.image.title=${{ matrix.image_name }}
+            org.opencontainers.image.description=An OCI base image of Fedora CoreOS with batteries included
+            io.artifacthub.package.readme-url=https://raw.githubusercontent.com/ublue-os/ucore/main/README.md
+            io.artifacthub.package.logo-url=https://avatars.githubusercontent.com/u/120078124?s=200&v=4
+
+      # Build image using Buildah action
+      - name: Build Image
+        id: build_image
+        uses: redhat-actions/buildah-build@v2
+        with:
+          containerfiles: |
+            ./main/Containerfile
+          context: ./main
+          image: ${{ matrix.image_name }}
+          tags: |
+            ${{ steps.generate-tags.outputs.alias_tags }}
+          build-args: |
+            COREOS_VERSION=${{ matrix.coreos_version }}
+            ZFS_PR_PREFIX=${{ matrix.zfs_pr_prefix }}
+          labels: ${{ steps.meta.outputs.labels }}
+          oci: false
+
+      # Workaround bug where capital letters in your GitHub username make it impossible to push to GHCR.
+      # https://github.com/macbre/push-to-ghcr/issues/12
+      - name: Lowercase Registry
+        id: registry_case
+        uses: ASzc/change-string-case-action@v5
+        with:
+          string: ${{ env.IMAGE_REGISTRY }}
+
+      # Push the image to GHCR (Image Registry)
+      - name: Push To GHCR
+        uses: redhat-actions/push-to-registry@v2
+        id: push
+        #if: github.event_name != 'pull_request'
+        env:
+          REGISTRY_USER: ${{ github.actor }}
+          REGISTRY_PASSWORD: ${{ github.token }}
+        with:
+          image: ${{ steps.build_image.outputs.image }}
+          tags: ${{ steps.build_image.outputs.tags }}
+          registry: ${{ steps.registry_case.outputs.lowercase }}
+          username: ${{ env.REGISTRY_USER }}
+          password: ${{ env.REGISTRY_PASSWORD }}
+          extra-args: |
+            --disable-content-trust
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        if: github.event_name != 'pull_request'
+        with:
+          registry: ghcr.io
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Sign container
+      - uses: sigstore/cosign-installer@v3.0.1
+        if: github.event_name != 'pull_request'
+
+      - name: Sign container image
+        if: github.event_name != 'pull_request'
+        run: |
+          echo "${{ env.COSIGN_PRIVATE_KEY }}" > cosign.key
+          wc -c cosign.key
+          cosign sign -y --key cosign.key ${{ steps.registry_case.outputs.lowercase }}/${{ steps.build_image.outputs.image }}@${TAGS}
+        env:
+          TAGS: ${{ steps.push.outputs.digest }}
+          COSIGN_EXPERIMENTAL: false
+          COSIGN_PRIVATE_KEY: ${{ secrets.SIGNING_SECRET }}
+
+      - name: Echo outputs
+        #if: github.event_name != 'pull_request'
        run: |
          echo "${{ toJSON(steps.push.outputs) }}"

--- a/README.md
+++ b/README.md
@@ -17,8 +17,9 @@ WARNING: This image has **not** been heavily tested, though the underlying compo
  - [cockpit](https://cockpit-project.org)
  - [distrobox](https://github.com/89luca89/distrobox)
  - [duperemove](https://github.com/markfasheh/duperemove)
-  - moby-engine, docker-compose and podman-compose
+  - moby-engine(docker), docker-compose and podman-compose
  - [tailscale](https://tailscale.com) and [wireguard-tools](https://www.wireguard.com)
+  - [ZFS](https://openzfs.github.io/openzfs-docs/Getting%20Started/Fedora/index.html)
 - Enables staging of automatic system updates via rpm-ostreed
 - Disables Zincati auto upgrade/reboot serivce
 - Sets 60 second service stop timeout for reasonably fast shutdowns
@@ -32,10 +33,14 @@ Note: per [cockpit instructions](https://cockpit-project.org/running.html#coreos

 ## Tips and Tricks

+### Immutability and Podman
+
 These images are immutable, you can't, and really shouldn't, install packages like in a mutable "normal" distribution.

 CoreOS expects the user to run services using [podman](https://podman.io). `moby-engine`, the free Docker implementation, is installed for those who desire docker instead of podman.

+### Default Services
+
 To maintain this image's suitability as a minimal container host, most add-on services are not auto-enabled.

 To activate any of the pre-installed `cockpit`, `docker`, or `tailscaled` services:
@@ -44,12 +49,32 @@ To activate any of the pre-installed `cockpit`, `docker`, or `tailscaled` servic
 sudo systemctl enable --now SERVICENAME.service
 ```

+### Docker/Moby and Podman
+
 NOTE: CoreOS [cautions against](https://docs.fedoraproject.org/en-US/fedora-coreos/faq/#_can_i_run_containers_via_docker_and_podman_at_the_same_time) running podman and docker containers at the same time.  Thus, `docker.socket` is disabled by default to prevent accidental activate of docker daemon, given podman is the default.

+### Distrobox
+
 Users may use [distrobox](https://github.com/89luca89/distrobox) to run images of mutable distributions where applications can be installed with traditional package managers. This may be useful for installing interactive utilities such has `htop`, `nmap`, etc. As stated above, however, *services* should run as containers.

+### CoreOS and ostree Docs
+
 It's a good idea to become familar with the [Fedora CoreOS Documentation](https://docs.fedoraproject.org/en-US/fedora-coreos/) as well as the [CoreOS rpm-ostree docs](https://coreos.github.io/rpm-ostree/). Note especially, this image is only possible due to [ostree native containers](https://coreos.github.io/rpm-ostree/container/).

+### ZFS
+
+The ZFS kernel module and tools are pre-installed, but like other services, ZFS is not pre-configured to load on default.
+
+Load it with the command `modprobe zfs` and use `zfs` and `zpool` commands as desired.
+
+Per the [OpenZFS Fedora documentation](https://openzfs.github.io/openzfs-docs/Getting%20Started/Fedora/index.html):
+
+> By default ZFS kernel modules are loaded upon detecting a pool. To always load the modules at boot:
+
+```
+echo zfs > /etc/modules-load.d/zfs.conf
+```
+
 ## How to Install

 ### Prerequsites
--- a/main/Containerfile
+++ b/main/Containerfile
@@ -1,18 +1,22 @@
-ARG IMAGE_NAME="${IMAGE_NAME:-ucore}"
 ARG COREOS_VERSION="${COREOS_VERSION:-stable}"

 FROM quay.io/fedora/fedora-coreos:${COREOS_VERSION}

-ARG IMAGE_NAME
-ARG COREOS_VERSION
+ARG IMAGE_NAME="${IMAGE_NAME:-ucore}"
+ARG COREOS_VERSION="${COREOS_VERSION:-stable}"

 ADD build.sh /tmp/build.sh
 ADD post-install.sh /tmp/post-install.sh
 ADD packages.json /tmp/packages.json

+ARG ZFS_PR_PREFIX="${ZFS_PR_PREFIX}"
+
+COPY --from=ghcr.io/ublue-os/ucore-zfs-rpm:${ZFS_PR_PREFIX}${COREOS_VERSION} / /tmp/rpms
 COPY etc /etc
 COPY usr /usr

+# install locally prepared RPMs (ZFS, etc)
+RUN rpm-ostree install /tmp/rpms/*.rpm
 RUN /tmp/build.sh
 RUN /tmp/post-install.sh
 RUN rm -rf /tmp/* /var/*
--- a/zfs/Containerfile
+++ b/zfs/Containerfile
@@ -0,0 +1,49 @@
+ARG COREOS_VERSION="${COREOS_VERSION}"
+ARG ZFS_VERSION="${ZFS_VERSION}"
+
+FROM quay.io/fedora/fedora-coreos:${COREOS_VERSION} as kernel-query
+
+#We can't use the `uname -r` as it will pick up the host kernel version
+RUN rpm -qa kernel --queryformat '%{VERSION}-%{RELEASE}.%{ARCH}' > /kernel-version.txt
+
+# Using https://openzfs.github.io/openzfs-docs/Developer%20Resources/Custom%20Packages.html
+FROM registry.fedoraproject.org/fedora:latest as builder
+ARG ZFS_VERSION="${ZFS_VERSION}"
+
+COPY --from=kernel-query /kernel-version.txt /kernel-version.txt
+
+WORKDIR /etc/yum.repos.d
+RUN BUILDER_VERSION=$(grep VERSION_ID /etc/os-release | cut -f2 -d=) \
+    && curl -L -O https://src.fedoraproject.org/rpms/fedora-repos/raw/f${BUILDER_VERSION}/f/fedora-updates-archive.repo \
+    && sed -i 's/enabled=AUTO_VALUE/enabled=true/' fedora-updates-archive.repo
+RUN dnf install -y jq dkms gcc make autoconf automake libtool rpm-build libtirpc-devel libblkid-devel \
+    libuuid-devel libudev-devel openssl-devel zlib-devel libaio-devel libattr-devel elfutils-libelf-devel \
+    kernel-$(cat /kernel-version.txt) kernel-modules-$(cat /kernel-version.txt) kernel-devel-$(cat /kernel-version.txt) \
+    python3 python3-devel python3-setuptools python3-cffi libffi-devel git ncompress libcurl-devel
+
+WORKDIR /
+RUN curl -L -O https://github.com/openzfs/zfs/releases/download/zfs-${ZFS_VERSION}/zfs-${ZFS_VERSION}.tar.gz \
+    && tar xzf zfs-${ZFS_VERSION}.tar.gz \
+    && mv zfs-${ZFS_VERSION} /tmp/zfs
+
+WORKDIR /tmp/zfs
+# build
+RUN ./configure \
+        -with-linux=/usr/src/kernels/$(cat /kernel-version.txt)/ \
+        -with-linux-obj=/usr/src/kernels/$(cat /kernel-version.txt)/ \
+    && make -j 1 rpm-utils rpm-kmod
+# sort into directories for easier install later
+RUN mkdir -p /tmp/rpms/{debug,devel,other,src} \
+    && mv *src.rpm /tmp/rpms/src/ \
+    && mv *devel*.rpm /tmp/rpms/devel/ \
+    && mv *debug*.rpm /tmp/rpms/debug/ \
+    && mv zfs-dracut*.rpm /tmp/rpms/other/ \
+    && mv zfs-test*.rpm /tmp/rpms/other/ \
+    && mv *.rpm /tmp/rpms/
+RUN find /tmp/rpms | sort
+
+
+FROM scratch
+
+# Copy build RPMs
+COPY --from=builder /tmp/rpms/ /