refactor(ci): Remove browser-based integration tests (#6435)

Fixes a new issue with puppeteer, chromium 128, and Alpine 3.20 that's
causing failing browser tests.

See more: https://github.com/puppeteer/puppeteer/issues/12189

Failure:

https://github.com/firezone/firezone/actions/runs/10549430305/job/29224528663?pr=6391

Unfortunately, puppeteer's embedded browser doesn't seem to want to run
in Alpine:


https://github.com/firezone/firezone/actions/runs/10563167497/job/29265175731?pr=6435#step:6:56


Fixing this is proving very difficult since we can't seem to use
puppeteer with the latest Alpine images, so I questioned the need to
have these in at all. These tests were added at a time where the DNS
mappings were brittle, so we wanted to verify that relayed and direct
connections held up as we deployed.

This is no longer the case, and we also now have much more unit test
coverage around these things, so given the pain of maintaining these
(and the lack of a current solution to the above), they are removed.

---------

Signed-off-by: Jamil <jamilbk@users.noreply.github.com>

.github/workflows/_build_artifacts.yml

@@ -302,8 +302,6 @@ jobs:
             PACKAGE=${{ matrix.name.package }}
             TARGET=${{ matrix.arch.target }}
           context: rust
-          build-contexts: |
-            browser-tests=${{ github.workspace }}/scripts/tests/browser
           cache-from: |
             type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.name.image_name }}:${{ env.CACHE_TAG }}
             type=registry,ref=${{ steps.login.outputs.registry }}/cache/${{ matrix.name.image_name }}:main

.github/workflows/_integration_tests.yml

@@ -96,16 +96,8 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        # Don't run browser tests on main with the release images because
-        # they don't have chromium installed
-        ref_name:
-          - ${{ github.ref_name }}
-        exclude:
-          - {ref_name: main, test: direct-browser-relay-restart}
-          - {ref_name: main, test: relayed-browser-relay-restart}
-
         test:
-          - name: direct-browser-relay-restart
+          - name: direct-curl-relay-restart
           - name: direct-curl-api-down
           - name: direct-curl-api-relay-down
           - name: direct-curl-api-restart
@@ -118,7 +110,6 @@ jobs:
           - name: dns-failsafe # Uses the default DNS control method
           - name: dns-nm
           - name: relay-graceful-shutdown
-          - name: relayed-browser-relay-restart
           - name: relayed-curl-api-down
           - name: relayed-curl-api-restart
           - name: relayed-curl-relay-restart

rust/Dockerfile

@@ -18,8 +18,7 @@ RUN set -xe \
 ## See https://github.com/LukeMathWalker/cargo-chef/issues/231.
 COPY rust-toolchain.toml rust-toolchain.toml
 
-RUN set -xe \
-  && rustup show
+RUN rustup show
 
 WORKDIR /build
 
@@ -36,6 +35,7 @@ FROM chef AS builder
 
 COPY --from=planner /build/recipe.json .
 
+ARG PACKAGE
 RUN set -xe \
   && cargo chef cook --recipe-path recipe.json --bin ${PACKAGE}
 
@@ -47,12 +47,7 @@ RUN cargo build -p ${PACKAGE} $([ -n "${TARGET}" ] && "--target ${TARGET}")
 # Base image which is used to run the application binary
 FROM alpine:${ALPINE_VERSION} AS runtime_base
 
-# Important!  Update this no-op ENV variable when this Dockerfile
-# is updated with the current date. It will force refresh of all
-# of the base images and things like `apk add` won't be using
-# old cached versions when the Dockerfile is built.
-ENV REFRESHED_AT=2023-10-23 \
-  LANG=C.UTF-8 \
+ENV LANG=C.UTF-8 \
   TERM=xterm \
   RUST_BACKTRACE=1 \
   RUST_LOG=str0m=warn,info
@@ -81,40 +76,36 @@ FROM runtime_base AS runtime_http-test-server
 COPY ./docker-init.sh ./docker-init.sh
 
 # Funnel package specific base image back into `runtime`
+ARG PACKAGE
 FROM runtime_${PACKAGE} AS runtime
 
 ARG PACKAGE
 ENTRYPOINT ["docker-init.sh"]
 ENV PACKAGE=${PACKAGE}
 
-CMD $PACKAGE
+CMD ${PACKAGE}
 
 # used as a base for dev and test
 FROM runtime AS test
 
-RUN set -xe \
-  && apk add --no-cache iperf3 bind-tools iproute2 jq procps
+RUN apk add --no-cache iperf3 bind-tools iproute2 jq procps
 
 # used for local development
 FROM test AS dev
 ARG TARGET
+ARG PACKAGE
 COPY --from=builder /build/target/${TARGET}/debug/${PACKAGE} .
 
 # Build an image for GitHub Actions which includes debug asserts and more test utilities
 FROM test AS debug
 
-ARG TARGET
 ## Build first with `cross build --target ${TARGET} -p ${PACKAGE} && mv /target/${TARGET}/release/${PACKAGE} .`
+ARG PACKAGE
 COPY ${PACKAGE} .
 
-RUN set -xe \
-  && apk add --no-cache nodejs npm chromium
-COPY --from=browser-tests . .
-RUN npm install
-
 # Build a production image from including a binary compiled on the host
 FROM runtime AS release
 
-ARG TARGET
 ## Build first with `cross build --target ${TARGET} -p ${PACKAGE} --release && mv /target/${TARGET}/release/${PACKAGE} .`
+ARG PACKAGE
 COPY ${PACKAGE} .

scripts/tests/browser/.gitignore

@@ -1,2 +0,0 @@
-node_modules/
-dist/

scripts/tests/browser/package.json

@@ -1,17 +0,0 @@
-{
-  "name": "browser-tests",
-  "version": "1.0.0",
-  "scripts": {
-    "load": "npx ts-node src/load.ts",
-    "refresh": "npx ts-node src/refresh.ts"
-  },
-  "dependencies": {
-    "puppeteer": "^22.15.0",
-    "ts-command-line-args": "^2.5.1"
-  },
-  "devDependencies": {
-    "@types/node": "^22.0.2",
-    "ts-node": "^10.9.2",
-    "typescript": "^5.5.4"
-  }
-}

scripts/tests/browser/src/load.ts

@@ -1,12 +0,0 @@
-import { get_args, connectBrowser, retryOrFail } from "./shared.ts";
-
-(async (): Promise<void> => {
-  const args = get_args();
-  const browser = await connectBrowser(args);
-  const page = await browser.newPage();
-
-  await retryOrFail(async () => await page.goto(args.url), args.retries);
-
-  await browser.disconnect();
-  process.exit();
-})();

scripts/tests/browser/src/refresh.ts

@@ -1,32 +0,0 @@
-import { Browser, Page } from "puppeteer";
-import { connectBrowser, get_args, IArgs, retryOrFail } from "./shared.ts";
-
-async function activePage(browser: Browser, args: IArgs): Promise<Page> {
-  const pages = await browser.pages();
-  if (pages.length !== 1) {
-    throw new Error("Either no page found or more pages than expected found");
-  }
-  const page = pages[0];
-
-  const pageUrl = new URL(page.url());
-  const expectedUrl = new URL(args.url);
-  if (pageUrl.origin !== expectedUrl.origin) {
-    throw new Error("Expected page not found");
-  }
-
-  return page;
-}
-
-(async (): Promise<void> => {
-  const args = get_args();
-  const browser = await connectBrowser(args);
-  const page = await activePage(browser, args);
-
-  await retryOrFail(
-    async () => await page.reload({ timeout: 2000 }),
-    args.retries
-  );
-
-  await browser.disconnect();
-  process.exit();
-})();

scripts/tests/browser/src/shared.ts

@@ -1,43 +0,0 @@
-import { parse } from "ts-command-line-args";
-import puppeteer, { Browser, HTTPResponse } from "puppeteer";
-
-export interface IArgs {
-  debugPort: number;
-  url: string;
-  retries: number;
-}
-
-export function get_args(): IArgs {
-  return parse<IArgs>({
-    debugPort: Number,
-    url: String,
-    retries: Number,
-  });
-}
-
-export async function connectBrowser(args: IArgs): Promise<Browser> {
-  return await puppeteer.connect({
-    browserURL: `http://127.0.0.1:${args.debugPort}`,
-  });
-}
-
-export async function retryOrFail(
-  get_page: () => Promise<HTTPResponse | null>,
-  retries: number
-) {
-  while (true) {
-    try {
-      const status: number | undefined = (await get_page())?.status();
-      if (status !== 200) {
-        throw Error(`Failed to load page with status ${status}`);
-      }
-
-      break;
-    } catch (e) {
-      if (retries === 0) {
-        throw e;
-      }
-      retries--;
-    }
-  }
-}

scripts/tests/browser/tsconfig.json

@@ -1,14 +0,0 @@
-{
-  "compilerOptions": {
-    "noEmit": true,
-    "allowImportingTsExtensions": true,
-    "target": "es2016" /* Set the JavaScript language version for emitted JavaScript and include compatible library declarations. */,
-    "module": "commonjs" /* Specify what module code is generated. */,
-    "esModuleInterop": true /* Emit additional JavaScript to ease support for importing CommonJS modules. This enables 'allowSyntheticDefaultImports' for type compatibility. */,
-    "forceConsistentCasingInFileNames": true /* Ensure that casing is correct in imports. */,
-    "strict": true /* Enable all strict type-checking options. */,
-    "outDir": "dist",
-    "skipLibCheck": true /* Skip type checking all .d.ts files. */
-  },
-  "include": ["src/**/*"]
-}

scripts/tests/direct-curl-relay-restart.sh

@@ -5,11 +5,8 @@ HTTPBIN=http://dns.httpbin
 
 docker compose stop relay-2
 
-bootstrap_browser_test_harness
-start_chromium
-
-echo "# Make sure webpage is loaded once"
-load_page $HTTPBIN 1
+echo "# Load page"
+client_curl_resource $HTTPBIN/get
 
 echo "# Simulate rolling deployment of relays"
 docker compose start relay-2
@@ -17,5 +14,5 @@ docker compose kill relay-1 --signal SIGTERM
 
 sleep 1
 
-echo "# Reload page"
-refresh_page $HTTPBIN 10
+echo "# Load page again"
+client_curl_resource $HTTPBIN/get

scripts/tests/lib.sh

@@ -2,34 +2,10 @@
 
 set -euox pipefail
 
-CHROMIUM_PORT=9222
-
 function client() {
     docker compose exec -it client "$@"
 }
 
-# Release images (by design) don't include our browser test harness,
-# so install it here if it's not already present.
-function bootstrap_browser_test_harness() {
-    client which chromium-browser || (
-        client apk add --no-cache nodejs npm chromium &&
-            docker compose cp ./scripts/tests/browser/. client:/bin &&
-            client npm install --prefix /bin
-    )
-}
-
-function start_chromium() {
-    docker compose exec -d -it client chromium-browser --headless --no-sandbox --remote-debugging-port=$CHROMIUM_PORT
-}
-
-function load_page() {
-    client npm run load -- --debugPort $CHROMIUM_PORT --url "$1" --retries "$2"
-}
-
-function refresh_page() {
-    client npm run refresh -- --debugPort $CHROMIUM_PORT --url "$1" --retries "$2"
-}
-
 function gateway() {
     docker compose exec -it gateway "$@"
 }

scripts/tests/relayed-browser-relay-restart.sh

@@ -1,22 +0,0 @@
-#!/usr/bin/env bash
-
-source "./scripts/tests/lib.sh"
-HTTPBIN=http://dns.httpbin
-
-docker compose stop relay-2
-
-install_iptables_drop_rules
-bootstrap_browser_test_harness
-start_chromium
-
-echo "# Make sure webpage is loaded once"
-load_page $HTTPBIN 1
-
-echo "# Simulate rolling deployment of relays"
-docker compose start relay-2
-docker compose kill relay-1 --signal SIGTERM
-
-sleep 1
-
-echo "# Reload page"
-refresh_page $HTTPBIN 10