From ce5885279b6545b548c0ac48d1e6c8542ba4851b Mon Sep 17 00:00:00 2001 From: Ryan Cragun Date: Wed, 23 Oct 2024 15:31:24 -0600 Subject: [PATCH] VAULT-31181: Add `pipeline` tool to Vault (#28536) As the Vault pipeline and release processes evolve over time, so too must the tooling that drives them. Historically we've utilized a combination of CI features and shell scripts that are wrapped into make targets to drive our CI. While this approach has worked, it requires careful consideration of what features to use (bash in CI almost never matches bash in developer machines, etc.) and often requires a deep understanding of several CLI tools (jq, etc). `make` itself also has limitations in user experience, e.g. passing flags. As we're all in on Github Actions as our pipeline coordinator, continuing to utilize and build CLI tools to perform our pipeline tasks makes sense. This PR adds a new CLI tool called `pipeline` which we can use to build new isolated tasks that we can string together in Github Actions. We intend to use this utility as the interface for future release automation work, see VAULT-27514. For the first task in this new `pipeline` tool, I've chosen to build two small sub-commands: * `pipeline releases list-versions` - Allows us to list Vault versions between a range. The range is configurable either by setting `--upper` and/or `--lower` bounds, or by using the `--nminus` to set the N-X to go back from the current branches version. As CE and ENT do not have version parity we also consider the `--edition`, as well as none-to-many `--skip` flags to exclude specific versions. * `pipeline generate enos-dynamic-config` - Which creates dynamic enos configuration based on the branch and the current list of release versions. It takes largely the same flags as the `release list-versions` command, however it also expects a `--dir` for the enos directory and a `--file` where the dynamic configuration will be written. This allows us to dynamically update and feed the latest versions into our sampling algorithm to get coverage over all supported prior versions. We then integrate these new tools into the pipeline itself and cache the dynamic config on a weekly basis. We also cache the pipeline tool itself as it will likely become a repository for pipeline specific tooling. The caching strategy for the `pipeline` tool itself will make most workflows that require it super fast. Signed-off-by: Ryan Cragun --- .../actions/create-dynamic-config/action.yml | 52 ++++ .github/actions/set-up-go/action.yml | 5 +- .github/actions/set-up-pipeline/action.yml | 47 +++ .github/workflows/code-checker.yml | 2 +- .../test-run-enos-scenario-matrix.yml | 13 + .gitignore | 2 +- Makefile | 16 +- enos/Makefile | 9 + enos/README.md | 60 ++-- enos/enos-dynamic-config.hcl | 20 ++ enos/enos-globals.hcl | 20 -- enos/enos-scenario-autopilot.hcl | 12 +- enos/enos-scenario-upgrade.hcl | 16 +- enos/enos-variables.hcl | 9 +- enos/k8s/enos-variables-k8s.hcl | 2 + scripts/go-helper.sh | 6 +- tools/pipeline/go.mod | 65 +++++ tools/pipeline/go.sum | 222 ++++++++++++++ tools/pipeline/internal/cmd/generate.go | 18 ++ .../cmd/generate_enos_dynamic_config.go | 58 ++++ tools/pipeline/internal/cmd/releases.go | 18 ++ .../internal/cmd/releases_list_versions.go | 61 ++++ tools/pipeline/internal/cmd/root.go | 65 +++++ .../pkg/generate/enos_dynamic_config.go | 209 ++++++++++++++ .../pkg/generate/enos_dynamic_config_test.go | 271 ++++++++++++++++++ .../internal/pkg/metadata/metadata.go | 19 ++ .../pipeline/internal/pkg/releases/client.go | 211 ++++++++++++++ .../internal/pkg/releases/client_mock.go | 35 +++ .../internal/pkg/releases/client_test.go | 189 ++++++++++++ .../internal/pkg/releases/list_versions.go | 189 ++++++++++++ .../pkg/releases/list_versions_test.go | 170 +++++++++++ tools/pipeline/main.go | 10 + tools/tools.sh | 31 +- 33 files changed, 2024 insertions(+), 108 deletions(-) create mode 100644 .github/actions/create-dynamic-config/action.yml create mode 100644 .github/actions/set-up-pipeline/action.yml create mode 100644 enos/enos-dynamic-config.hcl create mode 100644 tools/pipeline/go.mod create mode 100644 tools/pipeline/go.sum create mode 100644 tools/pipeline/internal/cmd/generate.go create mode 100644 tools/pipeline/internal/cmd/generate_enos_dynamic_config.go create mode 100644 tools/pipeline/internal/cmd/releases.go create mode 100644 tools/pipeline/internal/cmd/releases_list_versions.go create mode 100644 tools/pipeline/internal/cmd/root.go create mode 100644 tools/pipeline/internal/pkg/generate/enos_dynamic_config.go create mode 100644 tools/pipeline/internal/pkg/generate/enos_dynamic_config_test.go create mode 100644 tools/pipeline/internal/pkg/metadata/metadata.go create mode 100644 tools/pipeline/internal/pkg/releases/client.go create mode 100644 tools/pipeline/internal/pkg/releases/client_mock.go create mode 100644 tools/pipeline/internal/pkg/releases/client_test.go create mode 100644 tools/pipeline/internal/pkg/releases/list_versions.go create mode 100644 tools/pipeline/internal/pkg/releases/list_versions_test.go create mode 100644 tools/pipeline/main.go diff --git a/.github/actions/create-dynamic-config/action.yml b/.github/actions/create-dynamic-config/action.yml new file mode 100644 index 0000000000..3ad4cdea76 --- /dev/null +++ b/.github/actions/create-dynamic-config/action.yml @@ -0,0 +1,52 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +--- +name: Create dynamic pipeline configuration +description: Create dynamic test configuration by restoring existing valid config or creating new config + +inputs: + github-token: + description: An elevated Github token to access private HashiCorp modules. + vault-edition: + description: The vault edition to use when generating the dynamic config + vault-version: + description: The vault version to use when generating the dynamic config + +runs: + using: composite + steps: + - name: dyn-cfg-metadata + id: dyn-cfg-metadata + shell: bash + run: | + # We're using a weekly cache key here so that we only regenerate the configuration on a + # weekly basis. If/when Github decides to purge our tiny config file cache we'll also + # recreate it as necessary. + # + # Uses GITHUB_ENV instead of GITHUB_OUTPUT because composite actions are broken, + # see: https://github.com/actions/cache/issues/803#issuecomment-1793565071 + { + echo "DYNAMIC_CONFIG_KEY=$(date +%Y-%m-%U)" + echo "DYNAMIC_CONFIG_PATH=enos/enos-dynamic-config.hcl" + } | tee -a "$GITHUB_ENV" + - name: Try to restore dynamic config from cache + id: dyn-cfg-cache + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ env.DYNAMIC_CONFIG_PATH }} + key: dyn-cfg-${{ env.DYNAMIC_CONFIG_KEY }} + - if: steps.dyn-cfg-cache.outputs.cache-hit != 'true' + id: dyn-cfg-set-up-pipeline + # If we can't restore it from config then set up pipeline and generate it + name: Set up the pipeline tool + uses: ./.github/actions/set-up-pipeline + with: + github-token: ${{ inputs.github-token }} + - if: steps.dyn-cfg-cache.outputs.cache-hit != 'true' + id: dyn-cfg-generate + name: Create dynamic configuration + shell: bash + run: | + # Make sure that any branch specific dynamic config has been generated + pipeline generate enos-dynamic-config -d ./enos -f enos-dynamic-config.hcl -v ${{ inputs.vault-version }} -e ${{ inputs.vault-edition }} -n 3 --log debug diff --git a/.github/actions/set-up-go/action.yml b/.github/actions/set-up-go/action.yml index 548555d266..f0f848a62d 100644 --- a/.github/actions/set-up-go/action.yml +++ b/.github/actions/set-up-go/action.yml @@ -8,14 +8,11 @@ description: Set up Go with a shared module cache. inputs: github-token: description: An elevated Github token to access private modules if necessary. - type: string no-restore: description: Whether or not to restore the Go module cache on a cache hit - type: boolean - default: false + default: "false" go-version: description: "Override .go-version" - type: string default: "" outputs: diff --git a/.github/actions/set-up-pipeline/action.yml b/.github/actions/set-up-pipeline/action.yml new file mode 100644 index 0000000000..536964ffc6 --- /dev/null +++ b/.github/actions/set-up-pipeline/action.yml @@ -0,0 +1,47 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +--- +name: Install the pipeline tool +description: Install the pipeline tool + +inputs: + github-token: + description: An elevated Github token to access private HashiCorp modules. + +runs: + using: composite + steps: + - uses: ./.github/actions/set-up-go + with: + github-token: ${{ inputs.github-token }} + no-restore: true # Don't download vault's modules for pipeline + - name: pipeline-metadata + id: pipeline-metadata + shell: bash + # Uses GITHUB_ENV instead of GITHUB_OUTPUT because composite actions are broken, + # see: https://github.com/actions/cache/issues/803#issuecomment-1793565071 + run: | + gobin=$(go env GOBIN) + if [[ -z "$gobin" ]]; then + gobin="$(go env GOPATH)/bin" + fi + { + echo "PIPELINE_HASH=$(git ls-tree HEAD tools/pipeline --object-only)" + echo "PIPELINE_PATH=$gobin/pipeline" + } | tee -a "$GITHUB_ENV" + - name: Try to restore pipeline from cache + id: pipeline-cache + uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2 + with: + path: ${{ env.PIPELINE_PATH }} + key: pipeline-${{ env.PIPELINE_HASH }} + - if: steps.pipeline-cache.outputs.cache-hit != 'true' + id: pipeline-build + name: Build pipeline + shell: bash + env: + GOPRIVATE: github.com/hashicorp/* + run: | + git config --global url."https://${{ inputs.github-token }}@github.com".insteadOf https://github.com + make tools-pipeline diff --git a/.github/workflows/code-checker.yml b/.github/workflows/code-checker.yml index 0e4ff173e6..0b3bf049c6 100644 --- a/.github/workflows/code-checker.yml +++ b/.github/workflows/code-checker.yml @@ -80,10 +80,10 @@ jobs: needs: setup steps: - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7 - - uses: ./.github/actions/install-external-tools # for buf and gofumpt - uses: ./.github/actions/set-up-go with: github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + - uses: ./.github/actions/install-external-tools # for buf and gofumpt - name: Go format run: make prep check-go-fmt - name: Protobuf format diff --git a/.github/workflows/test-run-enos-scenario-matrix.yml b/.github/workflows/test-run-enos-scenario-matrix.yml index 826a5a51c7..535e971da9 100644 --- a/.github/workflows/test-run-enos-scenario-matrix.yml +++ b/.github/workflows/test-run-enos-scenario-matrix.yml @@ -55,6 +55,11 @@ jobs: - uses: hashicorp/action-setup-enos@v1 with: github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + - uses: ./.github/actions/create-dynamic-config + with: + github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + vault-version: ${{ inputs.vault-version }} + vault-edition: ${{ inputs.vault-edition }} - id: metadata run: | build_date=$(make ci-get-date) @@ -69,6 +74,8 @@ jobs: # shellcheck disable=2001 vault_version="$(sed 's/+ent/+${{ inputs.vault-edition }}/g' <<< '${{ inputs.vault-version }}')" fi + sample_seed=$(date +%s) + sample=$(enos scenario sample observe "${{ inputs.sample-name }}" --chdir ./enos --min 1 --max "${{ inputs.sample-max }}" --seed "${sample_seed}" --format json | jq -c ".observation.elements") { echo "build-date=${build_date}" echo "vault-version=${vault_version}" @@ -99,6 +106,7 @@ jobs: ENOS_VAR_vault_build_date: ${{ needs.metadata.outputs.build-date }} ENOS_VAR_vault_product_version: ${{ needs.metadata.outputs.vault-version }} ENOS_VAR_vault_revision: ${{ inputs.vault-revision }} + ENOS_VAR_vault_upgrade_initial_version: ${{ matrix.attributes.upgrade_initial_version }} ENOS_VAR_consul_license_path: ./support/consul.hclic ENOS_VAR_vault_license_path: ./support/vault.hclic ENOS_VAR_distro_version_amzz: ${{ matrix.attributes.distro_version_amzn }} @@ -127,6 +135,11 @@ jobs: - uses: hashicorp/action-setup-enos@v1 with: github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + - uses: ./.github/actions/create-dynamic-config + with: + github-token: ${{ secrets.ELEVATED_GITHUB_TOKEN }} + vault-version: ${{ inputs.vault-version }} + vault-edition: ${{ inputs.vault-edition }} - name: Prepare scenario dependencies id: prepare_scenario run: | diff --git a/.gitignore b/.gitignore index 15bcfca6d3..16a10d75ee 100644 --- a/.gitignore +++ b/.gitignore @@ -133,4 +133,4 @@ website/components/node_modules tools/godoctests/.bin tools/gonilnilfunctions/.bin tools/codechecker/.bin -.ci-bootstrap \ No newline at end of file +.ci-bootstrap diff --git a/Makefile b/Makefile index ba499ad740..40022f25a8 100644 --- a/Makefile +++ b/Makefile @@ -297,6 +297,10 @@ check-tools-external: check-tools-internal: @$(CURDIR)/tools/tools.sh check-internal +.PHONY: check-tools-pipeline +check-tools-pipeline: + @$(CURDIR)/tools/tools.sh check-pipeline + check-vault-in-path: @VAULT_BIN=$$(command -v vault) || { echo "vault command not found"; exit 1; }; \ [ -x "$$VAULT_BIN" ] || { echo "$$VAULT_BIN not executable"; exit 1; }; \ @@ -314,6 +318,10 @@ tools-external: tools-internal: @$(CURDIR)/tools/tools.sh install-internal +.PHONY: tools-pipeline +tools-pipeline: + @$(CURDIR)/tools/tools.sh install-pipeline + mysql-database-plugin: @CGO_ENABLED=0 $(GO_CMD) build -o bin/mysql-database-plugin ./plugins/database/mysql/mysql-database-plugin @@ -368,10 +376,6 @@ ci-get-revision: ci-get-version-package: @$(CURDIR)/scripts/ci-helper.sh version-package -.PHONY: ci-install-external-tools -ci-install-external-tools: - @$(CURDIR)/scripts/ci-helper.sh install-external-tools - .PHONY: ci-prepare-ent-legal ci-prepare-ent-legal: @$(CURDIR)/scripts/ci-helper.sh prepare-ent-legal @@ -380,10 +384,6 @@ ci-prepare-ent-legal: ci-prepare-ce-legal: @$(CURDIR)/scripts/ci-helper.sh prepare-ce-legal -.PHONY: ci-update-external-tool-modules -ci-update-external-tool-modules: - @$(CURDIR)/scripts/ci-helper.sh update-external-tool-modules - .PHONY: ci-copywriteheaders ci-copywriteheaders: copywrite headers --plan diff --git a/enos/Makefile b/enos/Makefile index 4a5532b217..24c66eb3b9 100644 --- a/enos/Makefile +++ b/enos/Makefile @@ -1,3 +1,5 @@ +VAULT_VERSION=$$(cat $(CURDIR)/../version/VERSION) + .PHONY: default default: check-fmt shellcheck @@ -10,10 +12,16 @@ fmt: fmt-enos fmt-modules shfmt .PHONY: check-fmt-enos check-fmt-enos: enos fmt --check --diff . + enos fmt --check --diff ./k8s .PHONY: fmt-enos fmt-enos: enos fmt . + enos fmt ./k8s + +.PHONY: gen-enos +gen-enos: + pushd ../tools/pipeline &> /dev/null && go run ./... generate enos-dynamic-config -d ../../enos -f enos-dynamic-config.hcl -e ce -v $(VAULT_VERSION) -n 3 --log info && popd &> /dev/null .PHONY: check-fmt-modules check-fmt-modules: @@ -25,6 +33,7 @@ fmt-modules: .PHONY: validate-enos validate-enos: + enos scenario validate --timeout 30m0s --chdir ./k8s enos scenario validate --timeout 30m0s .PHONY: lint diff --git a/enos/README.md b/enos/README.md index 1ec6b8e13d..81de258fe8 100644 --- a/enos/README.md +++ b/enos/README.md @@ -64,6 +64,27 @@ Variables that are required: See [enos.vars.hcl](./enos.vars.hcl) or [enos-variables.hcl](./enos-variables.hcl) for further descriptions of the variables. +Additional variable information can also be found in the [Scenario Outlines](#scenario_outlines) + +## Scenario Outlines +Enos is capable of producing an outline of each scenario that is defined in a given directory. These +scenarios often include a description of what behavior the scenario performs, which variants are +available, and which variables are required. They also provide a step by step breakdown including +which quality requirments are verifiend by a given step. + +You can generate outlines of all scenarios or specify one via it's name. + +From the `enos` directory: +```bash +enos scenario outline smoke +``` + +There are also HTML versions available for an improved reading experience: +```bash +enos scenario outline --format html > index.html +open index.html +``` + ## Executing Scenarios From the `enos` directory: @@ -96,45 +117,6 @@ enos scenario destroy smoke artifact_source:local Refer to the [Enos documentation](https://github.com/hashicorp/Enos-Docs) for further information regarding installation, execution or composing scenarios. -# Scenarios -There are current two scenarios: `smoke` and `upgrade`. Both begin by building Vault -as specified by the selected `artifact_source` variant (see Variants section below for more -information). - -## Smoke -The [`smoke` scenario](./enos-scenario-smoke.hcl) creates a Vault cluster using -the version from the current branch (either in CI or locally), with the backend -specified by the `backend` variant (`raft` or `consul`). Next, it unseals with the -appropriate method (`awskms` or `shamir`) and performs different verifications -depending on the backend and seal type. - -## Upgrade -The [`upgrade` scenario](./enos-scenario-upgrade.hcl) creates a Vault cluster using -the version specified in `vault_upgrade_initial_release`, with the backend specified -by the `backend` variant (`raft` or `consul`). Next, it upgrades the Vault binary -that is determined by the `artifact_source` variant. After the upgrade, it verifies that -cluster is at the desired version, along with additional verifications. - - -## Autopilot -The [`autopilot` scenario](./enos-scenario-autopilot.hcl) creates a Vault cluster using -the version specified in `vault_upgrade_initial_release`. It writes test data to the Vault cluster. Next, it creates additional nodes with the candidate version of Vault as determined by the `vault_product_version` variable set. -The module uses AWS auto-join to handle discovery and unseals with auto-unseal -or Shamir depending on the `seal` variant. After the new nodes have joined and been -unsealed, it verifies reading stored data on the new nodes. Autopilot upgrade verification checks the upgrade status is "await-server-removal" and the target version is set to the version of upgraded nodes. This test also verifies the undo_logs status for Vault versions 1.13.x - -## Replication -The [`replication` scenario](./enos-scenario-replication.hcl) creates two 3-node Vault clusters and runs following verification steps: - - 1. Writes data on the primary cluster - 1. Enables performance replication - 1. Verifies reading stored data from secondary cluster - 1. Verifies initial replication status between both clusters - 1. Replaces the leader node and one standby node on the primary Vault cluster - 1. Verifies updated replication status between both clusters - - This scenario verifies the performance replication status on both clusters to have their connection_status as "connected" and that the secondary cluster has known_primaries cluster addresses updated to the active nodes IP addresses of the primary Vault cluster. This scenario currently works around issues VAULT-12311 and VAULT-12309. The scenario fails when the primary storage backend is Consul due to issue VAULT-12332 - ## UI Tests The [`ui` scenario](./enos-scenario-ui.hcl) creates a Vault cluster (deployed to AWS) using a version built from the current checkout of the project. Once the cluster is available the UI acceptance tests diff --git a/enos/enos-dynamic-config.hcl b/enos/enos-dynamic-config.hcl new file mode 100644 index 0000000000..4aceebe7ca --- /dev/null +++ b/enos/enos-dynamic-config.hcl @@ -0,0 +1,20 @@ +# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# Code generated by pipeline generate enos-dynamic-config DO NOT EDIT. + +# This file is overwritten in CI as it contains branch specific and sometimes ever-changing values. +# It's checked in here so that enos samples and scenarios can be performed, just be aware that this +# might change out from under you. + +globals { + sample_attributes = { + aws_region = ["us-east-1", "us-west-2"] + distro_version_amzn = ["2023"] + distro_version_leap = ["15.6"] + distro_version_rhel = ["8.10", "9.4"] + distro_version_sles = ["15.6"] + distro_version_ubuntu = ["20.04", "24.04"] + upgrade_initial_version = ["1.16.1", "1.16.2", "1.16.3", "1.17.0-rc1", "1.17.0", "1.17.1", "1.17.2", "1.17.3", "1.17.4", "1.17.5", "1.17.6", "1.18.0-rc1", "1.18.0"] + } +} diff --git a/enos/enos-globals.hcl b/enos/enos-globals.hcl index a17bc357f7..89540ca3ba 100644 --- a/enos/enos-globals.hcl +++ b/enos/enos-globals.hcl @@ -146,32 +146,12 @@ globals { protocol = "udp" }, } - sample_attributes = { - aws_region = ["us-east-1", "us-west-2"] - distro_version_amzn = ["2023"] - distro_version_leap = ["15.6"] - distro_version_rhel = ["8.10", "9.4"] - distro_version_sles = ["15.6"] - distro_version_ubuntu = ["20.04", "24.04"] - } seals = ["awskms", "pkcs11", "shamir"] tags = merge({ "Project Name" : var.project_name "Project" : "Enos", "Environment" : "ci" }, var.tags) - // This reads the VERSION file, strips any pre-release metadata, and selects only initial - // versions that are less than our current version. E.g. A VERSION file containing 1.17.0-beta2 - // would render: semverconstraint(v, "<1.17.0-0") - upgrade_version_stripped = join("-", [split("-", chomp(file("../version/VERSION")))[0], "0"]) - // NOTE: when backporting, make sure that our initial versions are less than that - // release branch's version. Also beware if adding versions below 1.11.x. Some scenarios - // that use this global might not work as expected with earlier versions. Below 1.8.x is - // not supported in any way. - upgrade_all_initial_versions_ce = ["1.8.12", "1.9.10", "1.10.11", "1.11.12", "1.12.11", "1.13.13", "1.14.10", "1.15.6", "1.16.3", "1.17.0"] - upgrade_all_initial_versions_ent = ["1.8.12", "1.9.10", "1.10.11", "1.11.12", "1.12.11", "1.13.13", "1.14.13", "1.15.10", "1.16.4", "1.17.0"] - upgrade_initial_versions_ce = [for v in global.upgrade_all_initial_versions_ce : v if semverconstraint(v, "<${global.upgrade_version_stripped}")] - upgrade_initial_versions_ent = [for v in global.upgrade_all_initial_versions_ent : v if semverconstraint(v, "<${global.upgrade_version_stripped}")] vault_install_dir = { bundle = "/opt/vault/bin" package = "/usr/bin" diff --git a/enos/enos-scenario-autopilot.hcl b/enos/enos-scenario-autopilot.hcl index 036814cecb..bac9d9b7ac 100644 --- a/enos/enos-scenario-autopilot.hcl +++ b/enos/enos-scenario-autopilot.hcl @@ -27,15 +27,9 @@ scenario "autopilot" { config_mode = global.config_modes distro = global.distros edition = global.enterprise_editions - initial_version = global.upgrade_initial_versions_ent ip_version = global.ip_versions seal = global.seals - // Autopilot wasn't available before 1.11.x - exclude { - initial_version = [for e in matrix.initial_version : e if semverconstraint(e, "<1.11.0-0")] - } - // Our local builder always creates bundles exclude { artifact_source = ["local"] @@ -87,7 +81,7 @@ scenario "autopilot" { } manage_service = matrix.artifact_type == "bundle" vault_install_dir = global.vault_install_dir[matrix.artifact_type] - vault_autopilot_default_max_leases = semverconstraint(matrix.initial_version, ">=1.16.0-0") ? "300000" : "" + vault_autopilot_default_max_leases = semverconstraint(var.vault_upgrade_initial_version, ">=1.16.0-0") ? "300000" : "" } step "build_vault" { @@ -251,13 +245,13 @@ scenario "autopilot" { packages = concat(global.packages, global.distro_packages[matrix.distro][global.distro_version[matrix.distro]]) release = { edition = matrix.edition - version = matrix.initial_version + version = var.vault_upgrade_initial_version } seal_attributes = step.create_seal_key.attributes seal_type = matrix.seal storage_backend = "raft" storage_backend_addl_config = { - autopilot_upgrade_version = matrix.initial_version + autopilot_upgrade_version = var.vault_upgrade_initial_version } } } diff --git a/enos/enos-scenario-upgrade.hcl b/enos/enos-scenario-upgrade.hcl index bc746ac127..405cfd47c1 100644 --- a/enos/enos-scenario-upgrade.hcl +++ b/enos/enos-scenario-upgrade.hcl @@ -29,7 +29,6 @@ scenario "upgrade" { consul_version = global.consul_versions distro = global.distros edition = global.editions - initial_version = global.upgrade_initial_versions_ce ip_version = global.ip_versions seal = global.seals @@ -39,19 +38,6 @@ scenario "upgrade" { artifact_type = ["package"] } - // Don't upgrade from super-ancient versions in CI because there are known reliability issues - // in those versions that have already been fixed. - exclude { - initial_version = [for e in matrix.initial_version : e if semverconstraint(e, "<1.11.0-0")] - } - - // FIPS 140-2 editions were not supported until 1.11.x, even though there are 1.10.x binaries - // published. - exclude { - edition = ["ent.fips1402", "ent.hsm.fips1402"] - initial_version = [for e in matrix.initial_version : e if semverconstraint(e, "<1.11.0-0")] - } - // There are no published versions of these artifacts yet. We'll update this to exclude older // versions after our initial publication of these editions for arm64. exclude { @@ -314,7 +300,7 @@ scenario "upgrade" { packages = concat(global.packages, global.distro_packages[matrix.distro][global.distro_version[matrix.distro]]) release = { edition = matrix.edition - version = matrix.initial_version + version = var.vault_upgrade_initial_version } seal_attributes = step.create_seal_key.attributes seal_type = matrix.seal diff --git a/enos/enos-variables.hcl b/enos/enos-variables.hcl index 08909e844d..298f295cd3 100644 --- a/enos/enos-variables.hcl +++ b/enos/enos-variables.hcl @@ -194,11 +194,8 @@ variable "vault_revision" { default = null } -variable "vault_upgrade_initial_release" { +variable "vault_upgrade_initial_version" { description = "The Vault release to deploy before upgrading" - default = { - edition = "ce" - // Vault 1.10.5 has a known issue with retry_join. - version = "1.10.4" - } + type = string + default = "1.13.13" } diff --git a/enos/k8s/enos-variables-k8s.hcl b/enos/k8s/enos-variables-k8s.hcl index 26ea3d0ce8..099b4ec7f6 100644 --- a/enos/k8s/enos-variables-k8s.hcl +++ b/enos/k8s/enos-variables-k8s.hcl @@ -34,9 +34,11 @@ variable "vault_build_date" { variable "vault_revision" { type = string description = "The expected vault revision" + default = "ce" } variable "vault_version" { description = "The expected vault version" type = string + default = "1.18.0" } diff --git a/scripts/go-helper.sh b/scripts/go-helper.sh index 27fc0151cb..9d7c2a9014 100755 --- a/scripts/go-helper.sh +++ b/scripts/go-helper.sh @@ -40,7 +40,7 @@ check_fmt() { echo "Run \`make fmt\` to reformat code." for file in "${malformed[@]}"; do gofumpt -w "$file" - echo "$(git diff --no-color "$file")" + git diff --no-color "$file" done exit 1 fi @@ -79,7 +79,7 @@ mod_download() { while IFS= read -r -d '' mod; do echo "==> Downloading Go modules for $mod to $(go env GOMODCACHE)..." pushd "$(dirname "$mod")" > /dev/null || (echo "failed to push into module dir" && exit 1) - GOOS=linux GOARCH=amd64 go mod download -x + GOOS=linux GOARCH=amd64 GOPRIVATE=github.com/hashicorp go mod download -x popd > /dev/null || (echo "failed to pop out of module dir" && exit 1) done < <(find . -type f -name go.mod -print0) } @@ -89,7 +89,7 @@ mod_tidy() { while IFS= read -r -d '' mod; do echo "==> Tidying $mod..." pushd "$(dirname "$mod")" > /dev/null || (echo "failed to push into module dir" && exit 1) - GOOS=linux GOARCH=amd64 go mod tidy + GOOS=linux GOARCH=amd64 GOPRIVATE=github.com/hashicorp go mod tidy popd > /dev/null || (echo "failed to pop out of module dir" && exit 1) done < <(find . -type f -name go.mod -print0) } diff --git a/tools/pipeline/go.mod b/tools/pipeline/go.mod new file mode 100644 index 0000000000..469ead9384 --- /dev/null +++ b/tools/pipeline/go.mod @@ -0,0 +1,65 @@ +module github.com/hashicorp/vault/tools/pipeline + +go 1.23.2 + +require ( + github.com/Masterminds/semver v1.5.0 + github.com/hashicorp/hcl/v2 v2.22.0 + github.com/hashicorp/releases-api v0.1.23 + github.com/spf13/cobra v1.8.1 + github.com/stretchr/testify v1.9.0 + github.com/veqryn/slog-context v0.7.0 +) + +require ( + github.com/agext/levenshtein v1.2.3 // indirect + github.com/apparentlymart/go-textseg/v15 v15.0.0 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/fatih/color v1.16.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect + github.com/go-openapi/errors v0.22.0 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/loads v0.22.0 // indirect + github.com/go-openapi/runtime v0.26.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/strfmt v0.23.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/validate v0.24.0 // indirect + github.com/google/go-cmp v0.6.0 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-hclog v1.6.3 // indirect + github.com/hashicorp/go-version v1.7.0 // indirect + github.com/imdario/mergo v0.3.15 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/jessevdk/go-flags v1.6.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mitchellh/go-wordwrap v1.0.1 // indirect + github.com/mitchellh/hashstructure v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mitchellh/pointerstructure v1.2.1 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/opentracing/opentracing-go v1.2.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/zclconf/go-cty v1.15.0 // indirect + go.mongodb.org/mongo-driver v1.14.0 // indirect + go.opentelemetry.io/otel v1.20.0 // indirect + go.opentelemetry.io/otel/metric v1.20.0 // indirect + go.opentelemetry.io/otel/trace v1.20.0 // indirect + golang.org/x/mod v0.21.0 // indirect + golang.org/x/net v0.30.0 // indirect + golang.org/x/sync v0.8.0 // indirect + golang.org/x/sys v0.26.0 // indirect + golang.org/x/text v0.19.0 // indirect + golang.org/x/tools v0.26.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect +) diff --git a/tools/pipeline/go.sum b/tools/pipeline/go.sum new file mode 100644 index 0000000000..5fa81af364 --- /dev/null +++ b/tools/pipeline/go.sum @@ -0,0 +1,222 @@ +github.com/DataDog/appsec-internal-go v1.6.0 h1:QHvPOv/O0s2fSI/BraZJNpRDAtdlrRm5APJFZNBxjAw= +github.com/DataDog/appsec-internal-go v1.6.0/go.mod h1:pEp8gjfNLtEOmz+iZqC8bXhu0h4k7NUsW/qiQb34k1U= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 h1:bUMSNsw1iofWiju9yc1f+kBd33E3hMJtq9GuU602Iy8= +github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0/go.mod h1:HzySONXnAgSmIQfL6gOv9hWprKJkx8CicuXuUbmgWfo= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 h1:5nE6N3JSs2IG3xzMthNFhXfOaXlrsdgqmJ73lndFf8c= +github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1/go.mod h1:Vc+snp0Bey4MrrJyiV2tVxxJb6BmLomPvN1RgAvjGaQ= +github.com/DataDog/datadog-go/v5 v5.3.0 h1:2q2qjFOb3RwAZNU+ez27ZVDwErJv5/VpbBPprz7Z+s8= +github.com/DataDog/datadog-go/v5 v5.3.0/go.mod h1:XRDJk1pTc00gm+ZDiBKsjh7oOOtJfYfglVCmFb8C2+Q= +github.com/DataDog/go-libddwaf/v3 v3.2.1 h1:lZPc6UxCOwioHc++nsldKR50FpIrRh1uGnGLuryqnE8= +github.com/DataDog/go-libddwaf/v3 v3.2.1/go.mod h1:AP+7Atb8ftSsrha35wht7+K3R+xuzfVSQhabSO4w6CY= +github.com/DataDog/go-tuf v1.0.2-0.5.2 h1:EeZr937eKAWPxJ26IykAdWA4A0jQXJgkhUjqEI/w7+I= +github.com/DataDog/go-tuf v1.0.2-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0= +github.com/DataDog/sketches-go v1.4.5 h1:ki7VfeNz7IcNafq7yI/j5U/YCkO3LJiMDtXz9OMQbyE= +github.com/DataDog/sketches-go v1.4.5/go.mod h1:7Y8GN8Jf66DLyDhc94zuWA3uHEt/7ttt8jHOBWWrSOg= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= +github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= +github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= +github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= +github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cpuguy83/go-md2man/v2 v2.0.4/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/ebitengine/purego v0.6.0-alpha.5 h1:EYID3JOAdmQ4SNZYJHu9V6IqOeRQDBYxqKAg9PyoHFY= +github.com/ebitengine/purego v0.6.0-alpha.5/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= +github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= +github.com/go-openapi/errors v0.22.0 h1:c4xY/OLxUBSTiepAg3j/MHuAv5mJhnf53LLMWFB+u/w= +github.com/go-openapi/errors v0.22.0/go.mod h1:J3DmZScxCDufmIMsdOuDHxJbdOGC0xtUynjIx092vXE= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/runtime v0.26.0 h1:HYOFtG00FM1UvqrcxbEJg/SwvDRvYLQKGhw2zaQjTcc= +github.com/go-openapi/runtime v0.26.0/go.mod h1:QgRGeZwrUcSHdeh4Ka9Glvo0ug1LC5WyE+EV88plZrQ= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= +github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= +github.com/go-test/deep v1.0.3/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= +github.com/golang-migrate/migrate/v4 v4.14.1 h1:qmRd/rNGjM1r3Ve5gHd5ZplytrD02UcItYNxJ3iUHHE= +github.com/golang-migrate/migrate/v4 v4.14.1/go.mod h1:l7Ks0Au6fYHuUIxUhQ0rcVX1uLlJg54C/VvW7tvxSz0= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.7.0 h1:5tqGy27NaOTB8yJKUZELlFAS/LTKJkrmONwQKeRZfjY= +github.com/hashicorp/go-version v1.7.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/hcl/v2 v2.22.0 h1:hkZ3nCtqeJsDhPRFz5EA9iwcG1hNWGePOTw6oyul12M= +github.com/hashicorp/hcl/v2 v2.22.0/go.mod h1:62ZYHrXgPoX8xBnzl8QzbWq4dyDsDtfCRgIq1rbJEvA= +github.com/hashicorp/releases-api v0.1.23 h1:7cKuNZU5kBO+CsBvTkNDN2XdO0dQdXFtSC8f9IEXb+k= +github.com/hashicorp/releases-api v0.1.23/go.mod h1:RgXaKHrH02RjK2SMAVyHmDPrTrwR0iR6El3W93kh5Vc= +github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= +github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451 h1:WAvSpGf7MsFuzAtK4Vk7R4EVe+liW4x83r4oWu0WHKw= +github.com/jackc/pgerrcode v0.0.0-20201024163028-a0d42d470451/go.mod h1:a/s9Lp5W7n/DD0VrVoyJ00FbP2ytTPDVOivvn2bMlds= +github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= +github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= +github.com/jackc/pgx/v5 v5.5.4 h1:Xp2aQS8uXButQdnCMWNmvx6UysWQQC+u1EoizjguY+8= +github.com/jackc/pgx/v5 v5.5.4/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A= +github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk= +github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4= +github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= +github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= +github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= +github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8= +github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mitchellh/go-testing-interface v1.14.1 h1:jrgshOhYAUVNMAJiKbEu7EqAwgJJ2JqpQmpLJOu07cU= +github.com/mitchellh/go-testing-interface v1.14.1/go.mod h1:gfgS7OtZj6MA4U1UrDRp04twqAjfvlZyCfX3sDjEym8= +github.com/mitchellh/go-wordwrap v1.0.1 h1:TLuKupo69TCn6TQSyGxwI1EblZZEsQ0vMlAFQflz0v0= +github.com/mitchellh/go-wordwrap v1.0.1/go.mod h1:R62XHJLzvMFRBbcrT7m7WgmE1eOyTSsCt+hzestvNj0= +github.com/mitchellh/hashstructure v1.1.0 h1:P6P1hdjqAAknpY/M1CGipelZgp+4y9ja9kmUZPXP+H0= +github.com/mitchellh/hashstructure v1.1.0/go.mod h1:xUDAozZz0Wmdiufv0uyhnHkUTN6/6d8ulp4AwfLKrmA= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.1 h1:ZhBBeX8tSlRpu/FFhXH4RC4OJzFlqsQhoHZAz4x7TIw= +github.com/mitchellh/pointerstructure v1.2.1/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= +github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0= +github.com/outcaste-io/ristretto v0.2.3/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAjPbcMsODrTQUpJ02eNLUoxBg= +github.com/secure-systems-lab/go-securesystemslib v0.7.0/go.mod h1:/2gYnlnHVQ6xeGtfIqFy7Do03K4cdCY0A/GlJLDKLHI= +github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= +github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0= +github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw= +github.com/veqryn/slog-context v0.7.0 h1:Ne7ajlR6Mjs2rQQtpg8k0eO6krR5wzpareh5VpV+V2s= +github.com/veqryn/slog-context v0.7.0/go.mod h1:E+qpdyiQs2YKRxFnX1JjpdFE1z3Ka94Kem2q9ZG6Jjo= +github.com/zclconf/go-cty v1.15.0 h1:tTCRWxsexYUmtt/wVxgDClUe+uQusuI443uL6e+5sXQ= +github.com/zclconf/go-cty v1.15.0/go.mod h1:VvMs5i0vgZdhYawQNq5kePSpLAoz8u1xvZgrPIxfnZE= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940 h1:4r45xpDWB6ZMSMNJFMOjqrGHynW3DIBuR2H9j0ug+Mo= +github.com/zclconf/go-cty-debug v0.0.0-20240509010212-0d6042c53940/go.mod h1:CmBdvvj3nqzfzJ6nTCIwDTPZ56aVGvDrmztiO5g3qrM= +go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80= +go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c= +go.opentelemetry.io/otel v1.20.0 h1:vsb/ggIY+hUjD/zCAQHpzTmndPqv/ml2ArbsbfBYTAc= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel/metric v1.20.0 h1:ZlrO8Hu9+GAhnepmRGhSU7/VkpjrNowxRN9GyKR4wzA= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/sdk v1.14.0 h1:PDCppFRDq8A1jL9v6KMI6dYesaq+DFcDZvjsoGvxGzY= +go.opentelemetry.io/otel/sdk v1.14.0/go.mod h1:bwIC5TjrNG6QDCHNWvW4HLHtUQ4I+VQDsnjhvyZCALM= +go.opentelemetry.io/otel/trace v1.20.0 h1:+yxVAPZPbQhbC3OfAkeIVTky6iTFpcr4SiY9om7mXSQ= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw= +golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U= +golang.org/x/mod v0.21.0 h1:vvrHzRwRfVKSiLrG+d4FMl/Qi4ukBCE6kZlTUkDYRT0= +golang.org/x/mod v0.21.0/go.mod h1:6SkKJ3Xj0I0BrPOZoBy3bdMptDDU9oJrpohJ3eWZ1fY= +golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4= +golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU= +golang.org/x/sync v0.8.0 h1:3NFvSEYkUoMifnESzZl15y791HH1qU2xm6eCJU5ZPXQ= +golang.org/x/sync v0.8.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo= +golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM= +golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.26.0 h1:v/60pFQmzmT9ExmjDv2gGIfi3OqfKoEP6I5+umXlbnQ= +golang.org/x/tools v0.26.0/go.mod h1:TPVVj70c7JJ3WCazhD8OdXcZg/og+b9+tH/KxylGwH0= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 h1:+cNy6SZtPcJQH3LJVLOSmiC7MMxXNOb3PU/VUEz+EhU= +golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/DataDog/dd-trace-go.v1 v1.66.0 h1:025+lLubGtpiDWrRmSOxoFBPIiVRVYRcqP9oLabVOeg= +gopkg.in/DataDog/dd-trace-go.v1 v1.66.0/go.mod h1:Av6AXGmQCQAbDnwNoPiuUz1k3GS8TwQjj+vEdwmEpmM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/square/go-jose.v2 v2.6.0 h1:NGk74WTnPKBNUhNzQX7PYcTLUjoq7mzKk2OKbvwk2iI= +gopkg.in/square/go-jose.v2 v2.6.0/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tools/pipeline/internal/cmd/generate.go b/tools/pipeline/internal/cmd/generate.go new file mode 100644 index 0000000000..3407041ccb --- /dev/null +++ b/tools/pipeline/internal/cmd/generate.go @@ -0,0 +1,18 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import "github.com/spf13/cobra" + +func newGenerateCmd() *cobra.Command { + releases := &cobra.Command{ + Use: "generate", + Short: "Pipeline configuration generation tasks", + Long: "Pipeline configuration generation tasks", + } + + releases.AddCommand(newGenerateEnosDynamicConfigCmd()) + + return releases +} diff --git a/tools/pipeline/internal/cmd/generate_enos_dynamic_config.go b/tools/pipeline/internal/cmd/generate_enos_dynamic_config.go new file mode 100644 index 0000000000..955b71ae37 --- /dev/null +++ b/tools/pipeline/internal/cmd/generate_enos_dynamic_config.go @@ -0,0 +1,58 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import ( + "context" + + "github.com/spf13/cobra" + + "github.com/hashicorp/vault/tools/pipeline/internal/pkg/generate" + "github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases" +) + +// skipVersionsDefault are versions that we skip by default. This list can grow as necessary. +var skipVersionsDefault = []string{ + "1.16.0", // 1.16.0 artifacts were revoked, always skip it if it's in the range +} + +var genEnosDynamicConfigReq = &generate.EnosDynamicConfigReq{ + VersionLister: releases.NewClient(), +} + +func newGenerateEnosDynamicConfigCmd() *cobra.Command { + genCfg := &cobra.Command{ + Use: "enos-dynamic-config", + Short: "Generate dynamic Enos configuration files", + Long: `Create branch specific Enos configuration dynamically. We do this to set the various +sample attribute variables on per-branch basis.`, + RunE: runGenerateEnosDynamicConfig, + } + + genCfg.PersistentFlags().StringVarP(&genEnosDynamicConfigReq.VaultVersion, "version", "v", "", "The version of Vault") + genCfg.PersistentFlags().StringVarP(&genEnosDynamicConfigReq.VaultEdition, "edition", "e", "", "The edition of Vault. Can either be 'ce' or 'enterprise'") + genCfg.PersistentFlags().StringVarP(&genEnosDynamicConfigReq.EnosDir, "dir", "d", ".", "The enos directory to create the configuration in") + genCfg.PersistentFlags().UintVarP(&genEnosDynamicConfigReq.NMinus, "nminus", "n", 3, "Instead of setting a dedicated lower bound, calculate N-X from the upper") + genCfg.PersistentFlags().StringSliceVarP(&genEnosDynamicConfigReq.Skip, "skip", "s", skipVersionsDefault, "Skip these versions. Can be provided none-to-many times") + genCfg.PersistentFlags().StringVarP(&genEnosDynamicConfigReq.FileName, "file", "f", "enos-dynamic-config.hcl", "The name of the file to write the configuration into") + + err := genCfg.MarkPersistentFlagRequired("edition") + if err != nil { + panic(err) + } + err = genCfg.MarkPersistentFlagRequired("version") + if err != nil { + panic(err) + } + + return genCfg +} + +func runGenerateEnosDynamicConfig(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true // Don't spam the usage on failure + + _, err := genEnosDynamicConfigReq.Run(context.TODO()) + + return err +} diff --git a/tools/pipeline/internal/cmd/releases.go b/tools/pipeline/internal/cmd/releases.go new file mode 100644 index 0000000000..4e6cf2c673 --- /dev/null +++ b/tools/pipeline/internal/cmd/releases.go @@ -0,0 +1,18 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import "github.com/spf13/cobra" + +func newReleasesCmd() *cobra.Command { + releases := &cobra.Command{ + Use: "releases", + Short: "Releases API related tasks", + Long: "Releases API related tasks", + } + + releases.AddCommand(newReleasesVersionsBetweenCmd()) + + return releases +} diff --git a/tools/pipeline/internal/cmd/releases_list_versions.go b/tools/pipeline/internal/cmd/releases_list_versions.go new file mode 100644 index 0000000000..693d014538 --- /dev/null +++ b/tools/pipeline/internal/cmd/releases_list_versions.go @@ -0,0 +1,61 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/spf13/cobra" + + "github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases" +) + +var listReleaseVersionsReq = &releases.ListVersionsReq{ + VersionLister: releases.NewClient(), +} + +func newReleasesVersionsBetweenCmd() *cobra.Command { + versions := &cobra.Command{ + Use: "list-versions", + Short: "Create a list of Vault versions between a lower and upper bound", + Long: "Create a list of Vault versions between a lower and upper bound", + RunE: runListVersionsReq, + } + + versions.PersistentFlags().StringVarP(&listReleaseVersionsReq.UpperBound, "upper", "u", "", "The highest version to include") + versions.PersistentFlags().StringVarP(&listReleaseVersionsReq.LowerBound, "lower", "l", "", "The lowest version to include") + versions.PersistentFlags().UintVarP(&listReleaseVersionsReq.NMinus, "nminus", "n", 0, "Instead of setting a dedicated lower bound, calculate N-X from the upper") + versions.PersistentFlags().StringVarP(&listReleaseVersionsReq.LicenseClass, "edition", "e", "", "The edition of Vault. Can either be 'ce' or 'enterprise'") + versions.PersistentFlags().StringSliceVarP(&listReleaseVersionsReq.Skip, "skip", "s", []string{}, "Skip this version. Can be provided none-to-many times") + + err := versions.MarkPersistentFlagRequired("upper") + if err != nil { + panic(err) + } + err = versions.MarkPersistentFlagRequired("edition") + if err != nil { + panic(err) + } + + return versions +} + +func runListVersionsReq(cmd *cobra.Command, args []string) error { + cmd.SilenceUsage = true // Don't spam the usage on failure + + res, err := listReleaseVersionsReq.Run(context.TODO()) + if err != nil { + return err + } + + b, err := json.Marshal(res) + if err != nil { + return err + } + fmt.Println(string(b)) + + return err +} diff --git a/tools/pipeline/internal/cmd/root.go b/tools/pipeline/internal/cmd/root.go new file mode 100644 index 0000000000..2ea852e0ce --- /dev/null +++ b/tools/pipeline/internal/cmd/root.go @@ -0,0 +1,65 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package cmd + +import ( + "fmt" + "log/slog" + "os" + + "github.com/spf13/cobra" + slogctx "github.com/veqryn/slog-context" +) + +type rootCmdCfg struct { + logLevel string +} + +var rootCfg = &rootCmdCfg{} + +func newRootCmd() *cobra.Command { + rootCmd := &cobra.Command{ + Use: "pipeline", + Short: "Execute pipeline tasks", + Long: "Pipeline automation tasks", + } + + rootCmd.PersistentFlags().StringVar(&rootCfg.logLevel, "log", "warn", "Set the log level. One of 'debug', 'info', 'warn', 'error'") + + rootCmd.AddCommand(newGenerateCmd()) + rootCmd.AddCommand(newReleasesCmd()) + + rootCmd.PersistentPreRunE = func(cmd *cobra.Command, args []string) error { + var ll slog.Level + switch rootCfg.logLevel { + case "debug": + ll = slog.LevelDebug + case "info": + ll = slog.LevelInfo + case "warn": + ll = slog.LevelWarn + case "error": + ll = slog.LevelError + default: + return fmt.Errorf("unsupported log level: %s", rootCfg.logLevel) + } + h := slogctx.NewHandler(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: ll}), nil) + slog.SetDefault(slog.New(h)) + + return nil + } + + return rootCmd +} + +// Execute executes the root pipeline command. +func Execute() { + rootCmd := newRootCmd() + rootCmd.SilenceErrors = true // We handle this below + + if err := rootCmd.Execute(); err != nil { + slog.Default().Error(err.Error()) + os.Exit(1) + } +} diff --git a/tools/pipeline/internal/pkg/generate/enos_dynamic_config.go b/tools/pipeline/internal/pkg/generate/enos_dynamic_config.go new file mode 100644 index 0000000000..e0d71ed9a2 --- /dev/null +++ b/tools/pipeline/internal/pkg/generate/enos_dynamic_config.go @@ -0,0 +1,209 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package generate + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "slices" + + "github.com/Masterminds/semver" + slogctx "github.com/veqryn/slog-context" + + "github.com/hashicorp/hcl/v2/gohcl" + "github.com/hashicorp/hcl/v2/hclwrite" + "github.com/hashicorp/vault/tools/pipeline/internal/pkg/metadata" + "github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases" +) + +// EnosDynamicConfigReq is a request to generate dynamic enos configuration +type EnosDynamicConfigReq struct { + VaultEdition string + VaultVersion string + EnosDir string + FileName string + Skip []string + NMinus uint + releases.VersionLister +} + +// EnosDynamicConfigRes is a response from a request to generate dynamic enos configuration +type EnosDynamicConfigRes struct { + Globals *Globals `json:"globals,omitempty" hcl:"globals,block" cty:"globals"` +} + +// Globals are our dynamic globals +type Globals struct { + SampleAttributes *SampleAttrs `json:"sample_attributes,omitempty" hcl:"sample_attributes" cty:"sample_attributes"` +} + +// SampleAttrs are the dynamic sample attributes that we'll write as globals +type SampleAttrs struct { + AWSRegion []string `json:"aws_region,omitempty" hcl:"aws_region" cty:"aws_region"` + DistroVersionAmzn []string `json:"distro_version_amzn,omitempty" hcl:"distro_version_amzn" cty:"distro_version_amzn"` + DistroVersionLeap []string `json:"distro_version_leap,omitempty" hcl:"distro_version_leap" cty:"distro_version_leap"` + DistroVersionRhel []string `json:"distro_version_rhel,omitempty" hcl:"distro_version_rhel" cty:"distro_version_rhel"` + DistroVersionSles []string `json:"distro_version_sles,omitempty" hcl:"distro_version_sles" cty:"distro_version_sles"` + DistroVersionUbuntu []string `json:"distro_version_ubuntu,omitempty" hcl:"distro_version_ubuntu" cty:"distro_version_ubuntu"` + UpgradeInitialVersion []string `json:"upgrade_initial_version,omitempty" hcl:"upgrade_initial_version" cty:"upgrade_initial_version"` +} + +// Validate validates the request parameters +func (e *EnosDynamicConfigReq) Validate(ctx context.Context) error { + if e == nil { + return errors.New("enos dynamic config req: validate: uninitialized") + } + + slog.Default().DebugContext(ctx, "validating enos dynamic config request") + + if e.FileName == "" { + return errors.New("no destination file name set") + } + + if e.VersionLister == nil { + return errors.New("no version lister set") + } + + if !slices.Contains(metadata.Editions, e.VaultEdition) { + return fmt.Errorf("unknown edition: %s", e.VaultEdition) + } + + _, err := semver.NewVersion(e.VaultVersion) + if err != nil { + return fmt.Errorf("invalid version: %s: %w", e.VaultVersion, err) + } + + s, err := os.Stat(e.EnosDir) + if err != nil { + return fmt.Errorf("invalid enos dir: %s: %w", e.EnosDir, err) + } + + if !s.IsDir() { + return fmt.Errorf("invalid enos dir: %s is not a directory", e.EnosDir) + } + + return nil +} + +// Run runs the dynamic configuration request +func (e *EnosDynamicConfigReq) Run(ctx context.Context) (*EnosDynamicConfigRes, error) { + if e == nil { + return nil, fmt.Errorf("enos-dynamic-config-req uninitialized") + } + + ctx = slogctx.Append(ctx, + slog.String("vault-edition", e.VaultEdition), + slog.String("vault-version", e.VaultVersion), + slog.String("file-name", e.FileName), + slog.String("dir", e.EnosDir), + slog.Uint64("n-minnux", uint64(e.NMinus)), + "skip", e.Skip, + ) + slog.Default().DebugContext(ctx, "running enos dynamic config request") + + err := e.Validate(ctx) + if err != nil { + return nil, err + } + + res := &EnosDynamicConfigRes{} + res.Globals, err = e.getGlobals(ctx) + if err != nil { + return nil, err + } + + return res, e.writeFile(ctx, res) +} + +func (e *EnosDynamicConfigReq) getGlobals(ctx context.Context) (*Globals, error) { + var err error + res := &Globals{} + res.SampleAttributes, err = e.getSampleAttrs(ctx) + + return res, err +} + +func (e *EnosDynamicConfigReq) getSampleAttrs(ctx context.Context) (*SampleAttrs, error) { + // Create our HCL body + attrs := &SampleAttrs{ + // Use the cheapest regions + AWSRegion: []string{"us-east-1", "us-west-2"}, + // Current distro defaults + DistroVersionAmzn: []string{"2023"}, + DistroVersionLeap: []string{"15.6"}, + DistroVersionRhel: []string{"8.10", "9.4"}, + DistroVersionSles: []string{"15.6"}, + DistroVersionUbuntu: []string{"20.04", "24.04"}, + } + + // Create our initial upgrade version list. We'll find all released versions between N-3 -> Current + // version, minus any explicitly skipped versions that have been set. Since CE and Ent do not share + // the same version lineage now we'll also have to figure that in as well. + versionReq := &releases.ListVersionsReq{ + VersionLister: e.VersionLister, + LicenseClass: e.VaultEdition, + UpperBound: e.VaultVersion, + NMinus: e.NMinus, + Skip: e.Skip, + } + + versionRes, err := versionReq.Run(ctx) + if err != nil { + return nil, err + } + attrs.UpgradeInitialVersion = versionRes.Versions + + return attrs, nil +} + +// writeFile creates the dynamic config file and writes the dynamic data into it +func (e *EnosDynamicConfigReq) writeFile(ctx context.Context, res *EnosDynamicConfigRes) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + + slog.Default().DebugContext(ctx, "writing enos dynamic config request") + + // Make sure our path is valid + path, err := filepath.Abs(filepath.Join(e.EnosDir, e.FileName)) + if err != nil { + return fmt.Errorf("expanding path dynamic config request path: %w", err) + } + + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0o644) + if err != nil { + return fmt.Errorf("opening dynamic config request destination file: %w", err) + } + defer f.Close() + + if _, err = f.WriteString(`# Copyright (c) HashiCorp, Inc. +# SPDX-License-Identifier: BUSL-1.1 + +# Code generated by pipeline generate enos-dynamic-config DO NOT EDIT. + +# This file is overwritten in CI as it contains branch specific and sometimes ever-changing values. +# It's checked in here so that enos samples and scenarios can be performed, just be aware that this +# might change out from under you. +`); err != nil { + return err + } + + hf := hclwrite.NewEmptyFile() + gohcl.EncodeIntoBody(res, hf.Body()) + bytes := hclwrite.Format(hf.Bytes()) + + slog.Default().InfoContext(ctx, "writing enos dynamic config request", + "path", path, + "hcl", string(bytes), + ) + _, err = f.Write(bytes) + + return err +} diff --git a/tools/pipeline/internal/pkg/generate/enos_dynamic_config_test.go b/tools/pipeline/internal/pkg/generate/enos_dynamic_config_test.go new file mode 100644 index 0000000000..bb395a90a4 --- /dev/null +++ b/tools/pipeline/internal/pkg/generate/enos_dynamic_config_test.go @@ -0,0 +1,271 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package generate + +import ( + "context" + "os" + "path/filepath" + "slices" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/hashicorp/vault/tools/pipeline/internal/pkg/releases" +) + +var testAPIVersions = []string{ + "1.16.10+ent.hsm.fips1402", + "1.16.10+ent.fips1402", + "1.16.10+ent.hsm", + "1.16.10+ent", + "1.17.6+ent.hsm.fips1402", + "1.17.6+ent.fips1402", + "1.17.6+ent.hsm", + "1.17.6+ent", + "1.18.0-rc1+ent.hsm.fips1402", + "1.18.0-rc1+ent.fips1402", + "1.18.0-rc1+ent.hsm", + "1.18.0-rc1+ent", + "1.17.5+ent.hsm.fips1402", + "1.17.5+ent.fips1402", + "1.17.5+ent.hsm", + "1.17.5+ent", + "1.16.9+ent.hsm.fips1402", + "1.16.9+ent.fips1402", + "1.16.9+ent.hsm", + "1.16.9+ent", + "1.17.4+ent.hsm.fips1402", + "1.17.4+ent.fips1402", + "1.17.4+ent.hsm", + "1.17.4+ent", + "1.16.8+ent.hsm.fips1402", + "1.16.8+ent.fips1402", + "1.16.8+ent.hsm", + "1.16.8+ent", + "1.17.3+ent.hsm.fips1402", + "1.17.3+ent.fips1402", + "1.17.3+ent.hsm", + "1.16.7+ent.hsm.fips1402", + "1.17.3+ent", + "1.16.7+ent.fips1402", + "1.16.7+ent.hsm", + "1.16.7+ent", + "1.17.2+ent.hsm.fips1402", + "1.17.2+ent.fips1402", + "1.17.2+ent.hsm", + "1.17.2+ent", + "1.16.6+ent.hsm.fips1402", + "1.16.6+ent.fips1402", + "1.16.6+ent.hsm", + "1.16.6+ent", +} + +var testAllVersions = []string{ + "1.16.6", + "1.16.7", + "1.16.8", + "1.16.9", + "1.16.10", + "1.17.2", + "1.17.3", + "1.17.4", + "1.17.5", + "1.17.6", + "1.18.0-rc1", +} + +func Test_EnosDynamicConfigReq_Validate(t *testing.T) { + t.Parallel() + + for name, test := range map[string]struct { + in *EnosDynamicConfigReq + fail bool + }{ + "ce edition": { + in: &EnosDynamicConfigReq{ + VaultEdition: "ce", + VaultVersion: "1.18.0", + EnosDir: t.TempDir(), + FileName: "test.hcl", + VersionLister: releases.NewMockClient(testAPIVersions), + }, + }, + "oss edition": { + in: &EnosDynamicConfigReq{ + VaultEdition: "oss", + VaultVersion: "1.18.0", + EnosDir: t.TempDir(), + FileName: "test.hcl", + VersionLister: releases.NewMockClient(testAPIVersions), + }, + }, + "ent edition": { + in: &EnosDynamicConfigReq{ + VaultEdition: "ent", + VaultVersion: "1.18.0", + EnosDir: t.TempDir(), + FileName: "test.hcl", + VersionLister: releases.NewMockClient(testAPIVersions), + }, + }, + "enterprise edition": { + in: &EnosDynamicConfigReq{ + VaultEdition: "enterprise", + VaultVersion: "1.18.0", + EnosDir: t.TempDir(), + FileName: "test.hcl", + VersionLister: releases.NewMockClient(testAPIVersions), + }, + }, + "ent.hsm edition": { + in: &EnosDynamicConfigReq{ + VaultEdition: "ent.hsm", + VaultVersion: "1.18.0", + EnosDir: t.TempDir(), + FileName: "test.hcl", + VersionLister: releases.NewMockClient(testAPIVersions), + }, + }, + "ent.fips1402 edition": { + in: &EnosDynamicConfigReq{ + VaultEdition: "ent.fips1402", + VaultVersion: "1.18.0", + EnosDir: t.TempDir(), + FileName: "test.hcl", + VersionLister: releases.NewMockClient(testAPIVersions), + }, + }, + "ent.hsm.fips1402 edition": { + in: &EnosDynamicConfigReq{ + VaultEdition: "ent.hsm.fips1402", + VaultVersion: "1.18.0", + EnosDir: t.TempDir(), + FileName: "test.hcl", + VersionLister: releases.NewMockClient(testAPIVersions), + }, + }, + "unknown edition": { + in: &EnosDynamicConfigReq{ + VaultEdition: "ent.nope", + VaultVersion: "1.18.0", + EnosDir: t.TempDir(), + FileName: "test.hcl", + }, + fail: true, + }, + "invalid version": { + in: &EnosDynamicConfigReq{ + VaultEdition: "ent.hsm.fips1402", + VaultVersion: "vault-1.18.0", + EnosDir: t.TempDir(), + FileName: "test.hcl", + }, + fail: true, + }, + "target dir doesn't exist": { + in: &EnosDynamicConfigReq{ + VaultEdition: "ent.hsm.fips1402", + VaultVersion: "1.18.0", + }, + fail: true, + }, + "no file name": { + in: &EnosDynamicConfigReq{ + VaultEdition: "ent.hsm.fips1402", + VaultVersion: "1.18.0", + EnosDir: t.TempDir(), + }, + fail: true, + }, + "no version lister": { + in: &EnosDynamicConfigReq{ + VaultEdition: "ent.hsm.fips1402", + VaultVersion: "1.18.0", + EnosDir: t.TempDir(), + FileName: "test.hcl", + }, + fail: true, + }, + } { + t.Run(name, func(t *testing.T) { + t.Parallel() + err := test.in.Validate(context.Background()) + if test.fail { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} + +func Test_EnosDynamicConfigReq_Run(t *testing.T) { + t.Parallel() + + for desc, test := range map[string]struct { + req *EnosDynamicConfigReq + res func() *EnosDynamicConfigRes + hcl []byte + fail bool + }{ + "default config": { + req: &EnosDynamicConfigReq{ + FileName: "test.hcl", + VaultEdition: "ent.hsm.fips1402", + VaultVersion: "1.18.0", + Skip: []string{"1.17.2", "1.17.5"}, + NMinus: 2, + EnosDir: t.TempDir(), + VersionLister: releases.NewMockClient(testAPIVersions), + }, + res: func() *EnosDynamicConfigRes { + versions := testAllVersions + versions = slices.DeleteFunc(versions, func(v string) bool { + return v == "1.17.2" || v == "1.17.5" + }) + return &EnosDynamicConfigRes{ + Globals: &Globals{ + SampleAttributes: &SampleAttrs{ + AWSRegion: []string{"us-east-1", "us-west-2"}, + DistroVersionAmzn: []string{"2023"}, + DistroVersionLeap: []string{"15.6"}, + DistroVersionRhel: []string{"8.10, 9.4"}, + DistroVersionSles: []string{"15.6"}, + DistroVersionUbuntu: []string{"20.04", "24.04"}, + UpgradeInitialVersion: versions, + }, + }, + } + }, + hcl: []byte(` +globals { + sample_attributes = { + aws_region = ["us-east-1", "us-west-2"] + distro_version_amzn = ["2023"] + distro_version_leap = ["15.6"] + distro_version_rhel = ["8.10, 9.4"] + distro_version_sles = ["15.6"] + distro_version_ubuntu = ["20.04", "24.04"] + upgrade_initial_version = ["1.16.6", "1.16.7", "1.16.8", "1.16.9", "1.16.10", "1.17.3", "1.17.4", "1.17.6", "1.18.0-rc1"] + } +} +`), + }, + } { + t.Run(desc, func(t *testing.T) { + t.Parallel() + res, err := test.req.Run(context.Background()) + if test.fail { + require.Error(t, err) + return + } + require.NoError(t, err) + require.EqualValues(t, test.res(), res) + b, err := os.ReadFile(filepath.Join(test.req.EnosDir, test.req.FileName)) + require.NoError(t, err) + require.EqualValuesf(t, test.hcl, b, string(b)) + }) + } +} diff --git a/tools/pipeline/internal/pkg/metadata/metadata.go b/tools/pipeline/internal/pkg/metadata/metadata.go new file mode 100644 index 0000000000..b009366762 --- /dev/null +++ b/tools/pipeline/internal/pkg/metadata/metadata.go @@ -0,0 +1,19 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package metadata + +var CeEditions = []string{ + "ce", + "oss", +} + +var EntEditions = []string{ + "ent", + "enterprise", + "ent.fips1402", + "ent.hsm", + "ent.hsm.fips1402", +} + +var Editions = append(CeEditions, EntEditions...) diff --git a/tools/pipeline/internal/pkg/releases/client.go b/tools/pipeline/internal/pkg/releases/client.go new file mode 100644 index 0000000000..86b0474093 --- /dev/null +++ b/tools/pipeline/internal/pkg/releases/client.go @@ -0,0 +1,211 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package releases + +import ( + "context" + "fmt" + "log/slog" + "sort" + "time" + + "github.com/Masterminds/semver" + slogctx "github.com/veqryn/slog-context" + + "github.com/hashicorp/releases-api/pkg/api" + "github.com/hashicorp/releases-api/pkg/client" + "github.com/hashicorp/releases-api/pkg/models" +) + +// Client is an api.releases.hashicorp.com API client. +type Client struct { + Addr string +} + +// VersionLister lists versions from the releases API. +type VersionLister interface { + ListVersions(ctx context.Context, product string, edition LicenseClass, ceil, floor *semver.Version) ([]string, error) +} + +var _ VersionLister = (*Client)(nil) + +type LicenseClass string + +// These map to the licenses classes defined by the releases API. +const ( + LicenseClassNone LicenseClass = "" + LicenseClassCE LicenseClass = "oss" + LicenseClassEnt LicenseClass = "enterprise" + LicenseClassHCP LicenseClass = "hcp" +) + +// NewClient returns a new releases API client. +func NewClient() *Client { + return &Client{ + Addr: "api.releases.hashicorp.com", + } +} + +// GetRelease takes a context, product, edition, and version and returns the relase information. +func (c *Client) GetRelease(ctx context.Context, product string, edition LicenseClass, version string) (*models.Release, error) { + ctx = slogctx.Append(ctx, + slog.String("product", product), + slog.String("edition", string(edition)), + slog.String("version", version), + ) + slog.Default().DebugContext(ctx, "getting release") + + rc := client.New(c.Addr, "", api.V1MimeType) + lc := string(edition) + release, _, err := rc.GetRelease(&client.GetReleaseParams{ + Version: version, + Product: product, + LicenseClass: &lc, + }) + + slog.Default().DebugContext(ctx, "got release", "release", release, "error", err) + + return release, err +} + +// ListVersions takes a context, product, edition, a ceiling version and floor version and returns +// all unique versions between the ceiling and floor range. +func (c *Client) ListVersions(ctx context.Context, product string, edition LicenseClass, ceil, floor *semver.Version) ([]string, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + ctx = slogctx.Append(ctx, + slog.String("product", product), + slog.String("edition", string(edition)), + slog.String("ceil", ceil.String()), + slog.String("floor", floor.String()), + slog.String("addr", c.Addr), + ) + slog.Default().DebugContext(ctx, "listing release versions") + + // The releases API lists releases by their upload order starting by most recent, not sequentially + // by version. It also uses upload timestamps as a method of pagination. Since version upload + // lineages are all mixed up we'll have to start by listing the latest versions and work back + // until we've reached an upload timestamp at our floor version. To do so we'll get the created + // time of our floor and subtract a day which ought to give us a reasonable time floor. + // NOTE: This requires our floor version to actually exist. + floorR, err := c.GetRelease(ctx, product, edition, floor.String()) + if err != nil { + return nil, fmt.Errorf("invalid floor version: %s, %w", floor.String(), err) + } + timeFloor := floorR.TimestampCreated.Add(time.Duration(-24) * time.Hour) + + // We'll do the same for our ceiling but be less restrictive as to allow an infinite ceiling. + // If the ceiling versions exists we'll use it as our initial time ceiling, otherwise we'll + // set it to the current time. + var after time.Time + ceilR, err := c.GetRelease(ctx, product, edition, ceil.String()) + if err == nil { + after = ceilR.TimestampCreated + } else { + after = time.Now() + } + + versions := []string{} + + for { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + slog.Default().DebugContext(ctx, "listing releases", "after", after, "limit", 20) + rc := client.New(c.Addr, "", api.V1MimeType) + releases, _, err := rc.ListReleases(&client.ListReleasesParams{ + Product: product, + AfterTimestamp: after, + Limit: 20, // 20 is the max limit + LicenseClass: string(edition), + }) + if err != nil { + return nil, err + } + + // We've returned all that we can return + if len(releases) < 1 { + break + } + + // Select any release verions that are within our desired range + releaseVersions, err := selectReleaseVersions(releases, ceil, floor) + if err != nil { + return nil, err + } + versions = append(versions, releaseVersions...) + + // Reset our pagination and do another round + after = releases[len(releases)-1].TimestampCreated + + // Short circuit if we've hit our time floor. + if timeFloor.After(after) { + break + } + } + + versions, err = sortVersions(versions) + slog.Default().DebugContext(ctx, "found release versions", "versions", versions, "error", err) + + return versions, err +} + +func sortVersions(in []string) ([]string, error) { + if len(in) < 2 { + return in, nil + } + + c := semver.Collection{} + out := []string{} + for _, ver := range in { + v, err := semver.NewVersion(ver) + if err != nil { + return nil, err + } + c = append(c, v) + } + + sort.Sort(c) + for _, v := range c { + out = append(out, v.String()) + } + + return out, nil +} + +func selectReleaseVersions(releases []*models.Release, ceil *semver.Version, floor *semver.Version) ([]string, error) { + versions := []string{} + if len(releases) < 1 { + return versions, nil + } + + for _, release := range releases { + rv, err := semver.NewVersion(release.Version) + if err != nil { + return nil, err + } + + if rv.GreaterThan(ceil) { + // We've found releases that are too new + continue + } + + if rv.LessThan(floor) { + // We've found a release after our floor so we can skip it. + continue + } + + // We're in-between + versions = append(versions, rv.String()) + } + + return versions, nil +} diff --git a/tools/pipeline/internal/pkg/releases/client_mock.go b/tools/pipeline/internal/pkg/releases/client_mock.go new file mode 100644 index 0000000000..0753733232 --- /dev/null +++ b/tools/pipeline/internal/pkg/releases/client_mock.go @@ -0,0 +1,35 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package releases + +import ( + "context" + + "github.com/Masterminds/semver" + + "github.com/hashicorp/releases-api/pkg/models" +) + +var _ VersionLister = (*MockClient)(nil) + +// MockClient is an in-memory mock of a releases API client for use in testing. +type MockClient struct { + Versions []string +} + +// NewMockClient takes a list of versions and returns a new mock releases API client. +func NewMockClient(versions []string) *MockClient { + return &MockClient{Versions: versions} +} + +// ListVersions takes a context, product, edition, a ceiling version and floor version and returns +// all unique versions between the ceiling and floor range. +func (m *MockClient) ListVersions(ctx context.Context, product string, edition LicenseClass, ceil, floor *semver.Version) ([]string, error) { + releaseVersions := []*models.Release{} + for _, v := range m.Versions { + releaseVersions = append(releaseVersions, &models.Release{Version: v}) + } + + return selectReleaseVersions(releaseVersions, ceil, floor) +} diff --git a/tools/pipeline/internal/pkg/releases/client_test.go b/tools/pipeline/internal/pkg/releases/client_test.go new file mode 100644 index 0000000000..4249330246 --- /dev/null +++ b/tools/pipeline/internal/pkg/releases/client_test.go @@ -0,0 +1,189 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package releases + +import ( + "context" + "testing" + + "github.com/Masterminds/semver" + "github.com/stretchr/testify/require" +) + +var testAPIVersions = []string{ + "1.16.10+ent.hsm.fips1402", + "1.16.10+ent.fips1402", + "1.16.10+ent.hsm", + "1.16.10+ent", + "1.17.6+ent.hsm.fips1402", + "1.17.6+ent.fips1402", + "1.17.6+ent.hsm", + "1.17.6+ent", + "1.18.0-rc1+ent.hsm.fips1402", + "1.18.0-rc1+ent.fips1402", + "1.18.0-rc1+ent.hsm", + "1.18.0-rc1+ent", + "1.17.5+ent.hsm.fips1402", + "1.17.5+ent.fips1402", + "1.17.5+ent.hsm", + "1.17.5+ent", + "1.16.9+ent.hsm.fips1402", + "1.16.9+ent.fips1402", + "1.16.9+ent.hsm", + "1.16.9+ent", + "1.17.4+ent.hsm.fips1402", + "1.17.4+ent.fips1402", + "1.17.4+ent.hsm", + "1.17.4+ent", + "1.16.8+ent.hsm.fips1402", + "1.16.8+ent.fips1402", + "1.16.8+ent.hsm", + "1.16.8+ent", + "1.17.3+ent.hsm.fips1402", + "1.17.3+ent.fips1402", + "1.17.3+ent.hsm", + "1.16.7+ent.hsm.fips1402", + "1.17.3+ent", + "1.16.7+ent.fips1402", + "1.16.7+ent.hsm", + "1.16.7+ent", + "1.17.2+ent.hsm.fips1402", + "1.17.2+ent.fips1402", + "1.17.2+ent.hsm", + "1.17.2+ent", + "1.16.6+ent.hsm.fips1402", + "1.16.6+ent.fips1402", + "1.16.6+ent.hsm", + "1.16.6+ent", +} + +func Test_Client_ListVersions(t *testing.T) { + t.Parallel() + + for desc, test := range map[string]struct { + rf func() (*semver.Version, *semver.Version) + e []string + }{ + "all": { + rf: func() (*semver.Version, *semver.Version) { + ceil, err := semver.NewVersion("1.18.0") + require.NoError(t, err) + floor, err := semver.NewVersion("1.15.0") + require.NoError(t, err) + return ceil, floor + }, + e: testAPIVersions, + }, + "high": { + rf: func() (*semver.Version, *semver.Version) { + ceil, err := semver.NewVersion("1.18.0") + require.NoError(t, err) + floor, err := semver.NewVersion("1.17.0") + require.NoError(t, err) + return ceil, floor + }, + e: []string{ + "1.17.6+ent.hsm.fips1402", + "1.17.6+ent.fips1402", + "1.17.6+ent.hsm", + "1.17.6+ent", + "1.18.0-rc1+ent.hsm.fips1402", + "1.18.0-rc1+ent.fips1402", + "1.18.0-rc1+ent.hsm", + "1.18.0-rc1+ent", + "1.17.5+ent.hsm.fips1402", + "1.17.5+ent.fips1402", + "1.17.5+ent.hsm", + "1.17.5+ent", + "1.17.4+ent.hsm.fips1402", + "1.17.4+ent.fips1402", + "1.17.4+ent.hsm", + "1.17.4+ent", + "1.17.3+ent.hsm.fips1402", + "1.17.3+ent.fips1402", + "1.17.3+ent.hsm", + "1.17.3+ent", + "1.17.2+ent.hsm.fips1402", + "1.17.2+ent.fips1402", + "1.17.2+ent.hsm", + "1.17.2+ent", + }, + }, + "middle": { + rf: func() (*semver.Version, *semver.Version) { + ceil, err := semver.NewVersion("1.17.4") + require.NoError(t, err) + floor, err := semver.NewVersion("1.16.7") + require.NoError(t, err) + return ceil, floor + }, + e: []string{ + "1.16.10+ent.hsm.fips1402", + "1.16.10+ent.fips1402", + "1.16.10+ent.hsm", + "1.16.10+ent", + "1.16.9+ent.hsm.fips1402", + "1.16.9+ent.fips1402", + "1.16.9+ent.hsm", + "1.16.9+ent", + "1.17.4+ent.hsm.fips1402", + "1.17.4+ent.fips1402", + "1.17.4+ent.hsm", + "1.17.4+ent", + "1.16.8+ent.hsm.fips1402", + "1.16.8+ent.fips1402", + "1.16.8+ent.hsm", + "1.16.8+ent", + "1.17.3+ent.hsm.fips1402", + "1.17.3+ent.fips1402", + "1.17.3+ent.hsm", + "1.16.7+ent.hsm.fips1402", + "1.17.3+ent", + "1.16.7+ent.fips1402", + "1.16.7+ent.hsm", + "1.16.7+ent", + "1.17.2+ent.hsm.fips1402", + "1.17.2+ent.fips1402", + "1.17.2+ent.hsm", + "1.17.2+ent", + }, + }, + "low": { + rf: func() (*semver.Version, *semver.Version) { + ceil, err := semver.NewVersion("1.16.9") + require.NoError(t, err) + floor, err := semver.NewVersion("1.15.3") + require.NoError(t, err) + return ceil, floor + }, + e: []string{ + "1.16.9+ent.hsm.fips1402", + "1.16.9+ent.fips1402", + "1.16.9+ent.hsm", + "1.16.9+ent", + "1.16.8+ent.hsm.fips1402", + "1.16.8+ent.fips1402", + "1.16.8+ent.hsm", + "1.16.8+ent", + "1.16.7+ent.hsm.fips1402", + "1.16.7+ent.fips1402", + "1.16.7+ent.hsm", + "1.16.7+ent", + "1.16.6+ent.hsm.fips1402", + "1.16.6+ent.fips1402", + "1.16.6+ent.hsm", + "1.16.6+ent", + }, + }, + } { + t.Run(desc, func(t *testing.T) { + t.Parallel() + client := NewMockClient(testAPIVersions) + ceil, floor := test.rf() + res, err := client.ListVersions(context.Background(), "vault", "enterprise", ceil, floor) + require.NoError(t, err) + require.EqualValues(t, test.e, res) + }) + } +} diff --git a/tools/pipeline/internal/pkg/releases/list_versions.go b/tools/pipeline/internal/pkg/releases/list_versions.go new file mode 100644 index 0000000000..bd2e9e1f00 --- /dev/null +++ b/tools/pipeline/internal/pkg/releases/list_versions.go @@ -0,0 +1,189 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package releases + +import ( + "context" + "errors" + "fmt" + "log/slog" + "slices" + + "github.com/Masterminds/semver" + slogctx "github.com/veqryn/slog-context" + + "github.com/hashicorp/vault/tools/pipeline/internal/pkg/metadata" +) + +// ListVersionsReq is a request to list versions from the releases API. +type ListVersionsReq struct { + UpperBound string + LowerBound string + NMinus uint + Skip []string + LicenseClass string + VersionLister +} + +// ListVersionsRes is a list versions response. +type ListVersionsRes struct { + Versions []string `json:"versions,omitempty"` +} + +// NewListVersionsReq returns a new releases API version lister request. +func NewListVersionsReq() *ListVersionsReq { + return &ListVersionsReq{} +} + +func (req *ListVersionsReq) Validate(ctx context.Context) error { + if req == nil { + return errors.New("releases list versions req: unitialized") + } + + // Allow callers to pass in "oss" or "ce" but always rewrite it to what the releases API expects. + if slices.Contains(metadata.CeEditions, req.LicenseClass) { + req.LicenseClass = string(LicenseClassCE) + } + + // Allow callers to pass in any enterprise edition but always rewrite it to what the releases API + // expects. + if slices.Contains(metadata.EntEditions, req.LicenseClass) { + req.LicenseClass = string(LicenseClassEnt) + } + + if req.LicenseClass != "oss" && req.LicenseClass != "enterprise" { + return fmt.Errorf("releases list versions req: validate: invalid license class: %s: must be 'ce' or 'enterprise'", req.LicenseClass) + } + + if req.VersionLister == nil { + return errors.New("releases list versions req: no version lister has been configured") + } + + if req.LowerBound != "" && req.NMinus != 0 { + return errors.New("releases list versions req: only one of a lower bound floor or nminus option can be configured") + } + + return nil +} + +func (req *ListVersionsReq) VersionRange() (*semver.Version, *semver.Version, error) { + ceil, err := semver.NewVersion(req.UpperBound) + if err != nil { + return nil, nil, fmt.Errorf("releases list versions req: invalid upper bound version: %w", err) + } + + var floor *semver.Version + + if req.LowerBound != "" { + floor, err = semver.NewVersion(req.LowerBound) + if err != nil { + return nil, nil, fmt.Errorf("releases list versions req: invalid lower bound version: %w", err) + } + } else if req.NMinus != 0 { + // This is quite naive. We only consider minor versions and pay no attention to whether or not + // going backwards should bump us back to a different major/minor version. We also do not + // consider preleases here at all so RC's will still go back two minor versions. In the event + // that we bump major versions we'll have to revisit this. + floor, err = semver.NewVersion(req.UpperBound) + if err != nil { + return nil, nil, fmt.Errorf("releases list versions req: invalid upper bound version: %w", err) + } + + minor := floor.Minor() - int64(req.NMinus) + if minor < 0 { + return nil, nil, fmt.Errorf("releases list versions req: impossible nminus version, cannot subtract %d from %d", req.NMinus, floor.Minor()) + } + + // Create a new version with the new minor. Always set the patch to zero to allow for the full + // range. + nv := fmt.Sprintf("%d.%d.0", floor.Major(), minor) + floor, err = semver.NewVersion(nv) + if err != nil { + return nil, nil, fmt.Errorf("releases list versions req: invalid nminus version: %s", nv) + } + } else { + return nil, nil, fmt.Errorf("releases list versions req: no floor version or nminus has been specified") + } + + return floor, ceil, nil +} + +// Run the versions between request by determining our upper and lower version boundaries, using +// them to get a list of versions from the configured VersionLister, and then filtering out any +// skipped versions. +func (req *ListVersionsReq) Run(ctx context.Context) (*ListVersionsRes, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + + ctx = slogctx.Append(ctx, + slog.String("upper-bound", req.UpperBound), + slog.String("lower-bound", req.LowerBound), + slog.Uint64("n-minus", uint64(req.NMinus)), + slog.String("edition", string(req.LicenseClass)), + "skip", req.Skip, + ) + slog.Default().DebugContext(ctx, "running releases list version request") + + err := req.Validate(ctx) + if err != nil { + return nil, err + } + + slog.Default().DebugContext(ctx, "determining version request range") + floor, ceil, err := req.VersionRange() + if err != nil { + return nil, err + } + + versions, err := req.VersionLister.ListVersions( + ctx, "vault", LicenseClass(req.LicenseClass), ceil, floor, + ) + if err != nil { + return nil, err + } + + res := &ListVersionsRes{Versions: []string{}} + seen := map[string]struct{}{} + + for _, v := range versions { + rv, err := semver.NewVersion(v) + if err != nil { + return nil, err + } + + // The releases API will list all editions as seperate release versions, as it should. However, + // we don't make that distinction here. For our purposes we neeed a singular list of all versions + // on a license class basis. As such, we'll drop metadata and only focus on major, minor, patch, + // and prerelease. + nv, err := rv.SetMetadata("") + if err != nil { + return nil, fmt.Errorf("failed to unset metadata: %v", err) + } + + // Since each enterprise version can be listed many times due to the metadata. If we've already + // seen this version we can move on. + if _, ok := seen[nv.String()]; ok { + continue + } + seen[nv.String()] = struct{}{} + + if len(req.Skip) > 0 { + if slices.Contains(req.Skip, nv.String()) || slices.Contains(req.Skip, rv.String()) { + // We're skipping this version + continue + } + } + + // Add it to the versions slice + res.Versions = append(res.Versions, nv.String()) + } + + res.Versions, err = sortVersions(res.Versions) + slog.Default().DebugContext(ctx, "found versions", "versions", res.Versions) + + return res, err +} diff --git a/tools/pipeline/internal/pkg/releases/list_versions_test.go b/tools/pipeline/internal/pkg/releases/list_versions_test.go new file mode 100644 index 0000000000..97b42338db --- /dev/null +++ b/tools/pipeline/internal/pkg/releases/list_versions_test.go @@ -0,0 +1,170 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package releases + +import ( + "context" + "slices" + "testing" + + "github.com/Masterminds/semver" + "github.com/stretchr/testify/require" +) + +var testAllVersions = []string{ + "1.16.6", + "1.16.7", + "1.16.8", + "1.16.9", + "1.16.10", + "1.17.2", + "1.17.3", + "1.17.4", + "1.17.5", + "1.17.6", + "1.18.0-rc1", +} + +func Test_VersionReq_Run(t *testing.T) { + t.Parallel() + + for desc, test := range map[string]struct { + req *ListVersionsReq + res func() *ListVersionsRes + fail bool + }{ + "no lister": { + req: &ListVersionsReq{ + LicenseClass: string(LicenseClassCE), + UpperBound: "1.18.0", + LowerBound: "1.15.0", + }, + fail: true, + }, + "all": { + req: &ListVersionsReq{ + LicenseClass: string(LicenseClassCE), + UpperBound: "1.18.0", + LowerBound: "1.15.0", + VersionLister: NewMockClient(testAPIVersions), + }, + res: func() *ListVersionsRes { return &ListVersionsRes{Versions: testAllVersions} }, + }, + "skips": { + req: &ListVersionsReq{ + LicenseClass: string(LicenseClassCE), + UpperBound: "1.18.0", + LowerBound: "1.15.0", + Skip: []string{"1.16.0", "1.17.1"}, + VersionLister: NewMockClient(testAPIVersions), + }, + res: func() *ListVersionsRes { + versions := slices.Clone(testAllVersions) + return &ListVersionsRes{Versions: slices.DeleteFunc(versions, func(v string) bool { + if v == "1.17.1" || v == "1.16.0" { + return true + } + return false + })} + }, + }, + "correct range upper and lower bound": { + req: &ListVersionsReq{ + LicenseClass: string(LicenseClassCE), + UpperBound: "1.17.5", + LowerBound: "1.16.4", + VersionLister: NewMockClient(testAPIVersions), + }, + res: func() *ListVersionsRes { + versions := slices.Clone(testAllVersions) + return &ListVersionsRes{Versions: slices.DeleteFunc(versions, func(ver string) bool { + u, err := semver.NewVersion("1.17.5") + require.NoError(t, err) + l, err := semver.NewVersion("1.16.4") + require.NoError(t, err) + v, err := semver.NewVersion(ver) + require.NoError(t, err) + if v.GreaterThan(u) || v.LessThan(l) { + return true + } + return false + })} + }, + }, + "correct range nminus 1": { + req: &ListVersionsReq{ + LicenseClass: string(LicenseClassCE), + UpperBound: "1.18.0", + NMinus: 1, + VersionLister: NewMockClient(testAPIVersions), + }, + res: func() *ListVersionsRes { + versions := slices.Clone(testAllVersions) + return &ListVersionsRes{Versions: slices.DeleteFunc(versions, func(ver string) bool { + u, err := semver.NewVersion("1.18.0") + require.NoError(t, err) + l, err := semver.NewVersion("1.17.0") + require.NoError(t, err) + v, err := semver.NewVersion(ver) + require.NoError(t, err) + if v.GreaterThan(u) || v.LessThan(l) { + return true + } + return false + })} + }, + }, + "correct range nminus 2": { + req: &ListVersionsReq{ + LicenseClass: string(LicenseClassCE), + UpperBound: "1.18.0", + NMinus: 2, + VersionLister: NewMockClient(testAPIVersions), + }, + res: func() *ListVersionsRes { return &ListVersionsRes{Versions: testAllVersions} }, + }, + "lower and nminus": { + req: &ListVersionsReq{ + LicenseClass: string(LicenseClassEnt), + UpperBound: "1.18.0", + LowerBound: "1.15.0", + NMinus: 2, + VersionLister: NewMockClient(testAPIVersions), + }, + res: func() *ListVersionsRes { return &ListVersionsRes{Versions: testAllVersions} }, + fail: true, + }, + "invalid upper": { + req: &ListVersionsReq{ + LicenseClass: string(LicenseClassEnt), + UpperBound: "1_18_0", + LowerBound: "1.15.0", + VersionLister: NewMockClient(testAPIVersions), + }, + res: func() *ListVersionsRes { return &ListVersionsRes{Versions: testAllVersions} }, + fail: true, + }, + "invalid lower": { + req: &ListVersionsReq{ + LicenseClass: string(LicenseClassEnt), + UpperBound: "1.18.0", + LowerBound: "1_15_0", + VersionLister: NewMockClient(testAPIVersions), + }, + res: func() *ListVersionsRes { return &ListVersionsRes{Versions: testAllVersions} }, + fail: true, + }, + } { + t.Run(desc, func(t *testing.T) { + t.Parallel() + res, err := test.req.Run(context.Background()) + if test.fail { + require.Error(t, err) + return + } + require.NoError(t, err) + require.EqualValues(t, test.res().Versions, res.Versions) + }) + } +} diff --git a/tools/pipeline/main.go b/tools/pipeline/main.go new file mode 100644 index 0000000000..67675f560d --- /dev/null +++ b/tools/pipeline/main.go @@ -0,0 +1,10 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: BUSL-1.1 + +package main + +import "github.com/hashicorp/vault/tools/pipeline/internal/cmd" + +func main() { + cmd.Execute() +} diff --git a/tools/tools.sh b/tools/tools.sh index 6d01a1d16c..b4f05dc1ec 100755 --- a/tools/tools.sh +++ b/tools/tools.sh @@ -97,9 +97,9 @@ install_internal() { ) echo "==> Installing internal tools..." - pushd "$(repo_root)" &> /dev/null + pushd "$(repo_root)/tools" &> /dev/null for tool in "${tools[@]}"; do - go_install ./tools/"$tool" + go_install ./"$tool" done popd &> /dev/null } @@ -119,6 +119,27 @@ check_internal() { done } +# Install our pipeline tools. In some cases these may require access to internal repositories so +# they are excluded from our baseline toolset. +install_pipeline() { + echo "==> Installing pipeline tools..." + pushd "$(repo_root)/tools/pipeline" &> /dev/null + if env GOPRIVATE=github.com/hashicorp go install ./...; then + echo "--> pipeline ✔" + else + echo "--> pipeline ✖" + popd &> /dev/null + return 1 + fi + popd &> /dev/null +} + +# Check that all required pipeline tools are installed +check_pipeline() { + echo "==> Checking for pipeline tools..." + check_tool pipeline pipeline +} + # Install tools. install() { install_internal @@ -139,12 +160,18 @@ main() { install-internal) install_internal ;; + install-pipeline) + install_pipeline + ;; check-external) check_external ;; check-internal) check_internal ;; + check-pipeline) + check_pipeline + ;; install) install ;;