vault/sdk/testing/stepwise/stepwise.go

// Package stepwise offers types and functions to enable black-box style tests
// that are executed in defined set of steps. Stepwise utilizes "Environments" which
// setup a running instance of Vault and provide a valid API client to execute
// user defined steps against.
package stepwise

import (
	"fmt"
	"os"
	"testing"

	log "github.com/hashicorp/go-hclog"
	"github.com/hashicorp/vault/api"
	"github.com/hashicorp/vault/sdk/helper/consts"
	"github.com/hashicorp/vault/sdk/helper/logging"
)

// TestEnvVar must be set to a non-empty value for acceptance tests to run.
const TestEnvVar = "VAULT_ACC"

// Operation defines operations each step could perform. These are
// intentionally redefined from the logical package in the SDK, so users
// consistently use the stepwise package and not a combination of both stepwise
// and logical.
type Operation string

const (
	WriteOperation  Operation = "create"
	UpdateOperation           = "update"
	ReadOperation             = "read"
	DeleteOperation           = "delete"
	ListOperation             = "list"
	HelpOperation             = "help"
)

// Environment is the interface Environments need to implement to be used in
// Case to execute each Step
type Environment interface {
	// Setup is responsible for creating the Vault cluster for use in the test
	// case.
	Setup() error

	// Client should return a clone of a configured Vault API client to
	// communicate with the Vault cluster created in Setup and managed by this
	// Environment.
	Client() (*api.Client, error)

	// Teardown is responsible for destroying any and all infrastructure created
	// during Setup or otherwise over the course of executing test cases.
	Teardown() error

	// Name returns the name of the environment provider, e.g. Docker, Minikube,
	// et.al.
	Name() string

	// MountPath returns the path the plugin is mounted at
	MountPath() string

	// RootToken returns the root token of the cluster, used for making requests
	// as well as administrative tasks
	RootToken() string
}

// PluginType defines the types of plugins supported
// This type re-create constants as a convienence so users don't need to import/use
// the consts package.
type PluginType consts.PluginType

// These are originally defined in sdk/helper/consts/plugin_types.go
const (
	PluginTypeUnknown PluginType = iota
	PluginTypeCredential
	PluginTypeDatabase
	PluginTypeSecrets
)

func (p PluginType) String() string {
	switch p {
	case PluginTypeUnknown:
		return "unknown"
	case PluginTypeCredential:
		return "auth"
	case PluginTypeDatabase:
		return "database"
	case PluginTypeSecrets:
		return "secret"
	default:
		return "unsupported"
	}
}

// MountOptions are a collection of options each step driver should
// support
type MountOptions struct {
	// MountPathPrefix is an optional prefix to use when mounting the plugin. If
	// omitted the mount path will default to the PluginName with a random suffix.
	MountPathPrefix string

	// Name is used to register the plugin. This can be arbitrary but should be a
	// reasonable value. For an example, if the plugin in test is a secret backend
	// that generates UUIDs with the name "vault-plugin-secrets-uuid", then "uuid"
	// or "test-uuid" would be reasonable. The name is used for lookups in the
	// catalog. See "name" in the "Register Plugin" endpoint docs:
	// - https://www.vaultproject.io/api-docs/system/plugins-catalog#register-plugin
	RegistryName string

	// PluginType is the optional type of plugin. See PluginType const defined
	// above
	PluginType PluginType

	// PluginName represents the name of the plugin that gets compiled. In the
	// standard plugin project file layout, it represents the folder under the
	// cmd/ folder. In the below example UUID project, the PluginName would be
	// "uuid":
	//
	// vault-plugin-secrets-uuid/
	// - backend.go
	// - cmd/
	// ----uuid/
	// ------main.go
	// - path_generate.go
	//
	PluginName string
}

// Step represents a single step of a test Case
type Step struct {
	// Operation defines what action is being taken in this step; write, read,
	// delete, et. al.
	Operation Operation

	// Path is the localized request path. The mount prefix, namespace, and
	// optionally "auth" will be automatically added.
	Path string

	// Arguments to pass in the request. These arguments represent payloads sent
	// to the API.
	Data map[string]interface{}

	// Assert is a function that is called after this step is executed in order to
	// test that the step executed successfully. If this is not set, then the next
	// step will be called
	Assert AssertionFunc

	// Unauthenticated will make the request unauthenticated.
	Unauthenticated bool
}

// AssertionFunc is the callback used for Assert in Steps.
type AssertionFunc func(*api.Secret, error) error

// Case represents a scenario we want to test which involves a series of
// steps to be followed sequentially, evaluating the results after each step.
type Case struct {
	// Environment is used to setup the Vault instance and provide the client that
	// will be used to drive the tests
	Environment Environment

	// Precheck enabls a test case to determine if it should run or not
	Precheck func()

	// Steps are the set of operations that are run for this test case. During
	// execution each step will be logged to output with a 1-based index as it is
	// ran, with the first step logged as step '1' and not step '0'.
	Steps []Step

	// SkipTeardown allows the Environment TeardownFunc to be skipped, leaving any
	// infrastructure created after the test exists. This is useful for debugging
	// during plugin development to examine the state of the Vault cluster after a
	// test runs. Depending on the Environment used this could incur costs the
	// user is responsible for.
	SkipTeardown bool
}

// Run performs an acceptance test on a backend with the given test case.
//
// Tests are not run unless an environmental variable "VAULT_ACC" is
// set to some non-empty value. This is to avoid test cases surprising
// a user by creating real resources.
//
// Tests will fail unless the verbose flag (`go test -v`, or explicitly
// the "-test.v" flag) is set. Because some acceptance tests take quite
// long, we require the verbose flag so users are able to see progress
// output.
func Run(tt TestT, c Case) {
	tt.Helper()
	// We only run acceptance tests if an env var is set because they're
	// slow and generally require some outside configuration.
	checkShouldRun(tt)

	if c.Precheck != nil {
		c.Precheck()
	}

	if c.Environment == nil {
		tt.Fatal("nil driver in acceptance test")
		// return here only used during testing when using mockT type, otherwise
		// Fatal will exit
		return
	}

	logger := logging.NewVaultLogger(log.Trace)

	if err := c.Environment.Setup(); err != nil {
		tt.Fatal(err)
	}

	defer func() {
		if c.SkipTeardown {
			logger.Info("driver Teardown skipped")
			return
		}
		if err := c.Environment.Teardown(); err != nil {
			logger.Error("error in driver teardown:", "error", err)
		}
	}()

	// retrieve the root client from the Environment. If this returns an error,
	// fail immediately
	rootClient, err := c.Environment.Client()
	if err != nil {
		tt.Fatal(err)
	}

	// Trap the rootToken so that we can preform revocation or other tasks in the
	// event any steps remove the token during testing.
	rootToken := c.Environment.RootToken()

	// Defer revocation of any secrets created. We intentionally enclose the
	// responses slice so in the event of a fatal error during test evaluation, we
	// are still able to revoke any leases/secrets created
	var responses []*api.Secret
	defer func() {
		// restore root token for admin tasks
		rootClient.SetToken(rootToken)
		// failedRevokes tracks any errors we get when attempting to revoke a lease
		// to log to users at the end of the test.
		var failedRevokes []*api.Secret
		for _, secret := range responses {
			if secret.LeaseID == "" {
				continue
			}

			if err := rootClient.Sys().Revoke(secret.LeaseID); err != nil {
				tt.Error(fmt.Errorf("error revoking lease: %w", err))
				failedRevokes = append(failedRevokes, secret)
				continue
			}
		}

		// If we have any failed revokes, log it.
		if len(failedRevokes) > 0 {
			for _, s := range failedRevokes {
				tt.Error(fmt.Sprintf(
					"WARNING: Revoking the following secret failed. It may\n"+
						"still exist. Please verify:\n\n%#v",
					s))
			}
		}
	}()

	stepCount := len(c.Steps)
	for i, step := range c.Steps {
		if logger.IsWarn() {
			// range is zero based, so add 1 for a human friendly output of steps.
			progress := fmt.Sprintf("%d/%d", i+1, stepCount)
			logger.Warn("Executing test step", "step_number", progress)
		}

		// reset token in case it was cleared
		client, err := rootClient.Clone()
		if err != nil {
			tt.Fatal(err)
		}

		// TODO: support creating tokens with policies listed in each Step
		client.SetToken(rootToken)

		resp, respErr := makeRequest(tt, c.Environment, step)
		if resp != nil {
			responses = append(responses, resp)
		}

		// Run the associated AssertionFunc, if any. If an error was expected it is
		// sent to the Assert function to validate.
		if step.Assert != nil {
			if err := step.Assert(resp, respErr); err != nil {
				tt.Error(fmt.Errorf("failed step %d: %w", i+1, err))
			}
		}
	}
}

func makeRequest(tt TestT, env Environment, step Step) (*api.Secret, error) {
	tt.Helper()
	client, err := env.Client()
	if err != nil {
		return nil, err
	}

	if step.Unauthenticated {
		token := client.Token()
		client.ClearToken()
		// restore the client token after this request completes
		defer func() {
			client.SetToken(token)
		}()
	}

	path := fmt.Sprintf("%s/%s", env.MountPath(), step.Path)
	switch step.Operation {
	case WriteOperation, UpdateOperation:
		return client.Logical().Write(path, step.Data)
	case ReadOperation:
		// TODO support ReadWithData
		return client.Logical().Read(path)
	case ListOperation:
		return client.Logical().List(path)
	case DeleteOperation:
		return client.Logical().Delete(path)
	default:
		return nil, fmt.Errorf("invalid operation: %s", step.Operation)
	}
}

func checkShouldRun(tt TestT) {
	tt.Helper()
	if os.Getenv(TestEnvVar) == "" {
		tt.Skip(fmt.Sprintf(
			"Acceptance tests skipped unless env '%s' set",
			TestEnvVar))
		return
	}

	// We require verbose mode so that the user knows what is going on.
	if !testing.Verbose() {
		tt.Fatal("Acceptance tests must be run with the -v flag on tests")
	}
}

// TestT is the interface used to handle the test lifecycle of a test.
//
// Users should just use a *testing.T object, which implements this.
type TestT interface {
	Error(args ...interface{})
	Fatal(args ...interface{})
	Skip(args ...interface{})
	Helper()
}