Merge multiple functions for creating consul containers into one. (#6612)

Merge both functions for creating mongodb containers into one.
Add retries to docker container cleanups.
Require $VAULT_ACC be set to enable AWS tests.

builtin/credential/radius/backend_test.go

@@ -9,6 +9,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	logicaltest "github.com/hashicorp/vault/helper/testhelpers/logical"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/ory/dockertest"
@@ -48,10 +49,7 @@ func prepareRadiusTestContainer(t *testing.T) (func(), string, int) {
 	}
 
 	cleanup := func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	port, _ := strconv.Atoi(resource.GetPort("1812/udp"))

builtin/logical/cassandra/backend_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 
 	"github.com/gocql/gocql"
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	logicaltest "github.com/hashicorp/vault/helper/testhelpers/logical"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/mitchellh/mapstructure"
@@ -45,10 +46,7 @@ func prepareCassandraTestContainer(t *testing.T) (func(), string, int) {
 	}
 
 	cleanup := func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	port, _ := strconv.Atoi(resource.GetPort("9042/tcp"))

builtin/logical/consul/backend_test.go

@@ -5,114 +5,17 @@ import (
 	"encoding/base64"
 	"fmt"
 	"log"
-	"os"
 	"reflect"
-	"strings"
 	"testing"
 	"time"
 
 	consulapi "github.com/hashicorp/consul/api"
+	"github.com/hashicorp/vault/helper/testhelpers/consul"
 	logicaltest "github.com/hashicorp/vault/helper/testhelpers/logical"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/mitchellh/mapstructure"
-	"github.com/ory/dockertest"
 )
 
-func prepareTestContainer(t *testing.T, version string) (cleanup func(), retAddress string, consulToken string) {
-	consulToken = os.Getenv("CONSUL_HTTP_TOKEN")
-	retAddress = os.Getenv("CONSUL_HTTP_ADDR")
-	if retAddress != "" {
-		return func() {}, retAddress, consulToken
-	}
-
-	pool, err := dockertest.NewPool("")
-	if err != nil {
-		t.Fatalf("Failed to connect to docker: %s", err)
-	}
-
-	config := `acl { enabled = true default_policy = "deny" }`
-	if strings.HasPrefix(version, "1.3") {
-		config = `datacenter = "test" acl_default_policy = "deny" acl_datacenter = "test" acl_master_token = "test"`
-	}
-
-	dockerOptions := &dockertest.RunOptions{
-		Repository: "consul",
-		Tag:        version,
-		Cmd:        []string{"agent", "-dev", "-client", "0.0.0.0", "-hcl", config},
-	}
-	resource, err := pool.RunWithOptions(dockerOptions)
-	if err != nil {
-		t.Fatalf("Could not start local Consul %s docker container: %s", version, err)
-	}
-
-	cleanup = func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
-	}
-
-	retAddress = fmt.Sprintf("localhost:%s", resource.GetPort("8500/tcp"))
-
-	// exponential backoff-retry
-	if err = pool.Retry(func() error {
-		var err error
-		consulConfig := consulapi.DefaultNonPooledConfig()
-		consulConfig.Address = retAddress
-		consul, err := consulapi.NewClient(consulConfig)
-		if err != nil {
-			return err
-		}
-
-		// For version of Consul < 1.4
-		if strings.HasPrefix(version, "1.3") {
-			consulToken = "test"
-			_, err = consul.KV().Put(&consulapi.KVPair{
-				Key:   "setuptest",
-				Value: []byte("setuptest"),
-			}, &consulapi.WriteOptions{
-				Token: consulToken,
-			})
-			if err != nil {
-				return err
-			}
-			return nil
-		}
-
-		// New default behavior
-		aclbootstrap, _, err := consul.ACL().Bootstrap()
-		if err != nil {
-			return err
-		}
-		consulToken = aclbootstrap.SecretID
-		t.Logf("Generated Master token: %s", consulToken)
-		policy := &consulapi.ACLPolicy{
-			Name:        "test",
-			Description: "test",
-			Rules: `node_prefix "" {
-                policy = "write"
-              }
-
-              service_prefix "" {
-                policy = "read"
-              }
-      `,
-		}
-		q := &consulapi.WriteOptions{
-			Token: consulToken,
-		}
-		_, _, err = consul.ACL().PolicyCreate(policy, q)
-		if err != nil {
-			return err
-		}
-		return nil
-	}); err != nil {
-		cleanup()
-		t.Fatalf("Could not connect to docker: %s", err)
-	}
-	return cleanup, retAddress, consulToken
-}
-
 func TestBackend_Config_Access(t *testing.T) {
 	t.Run("config_access", func(t *testing.T) {
 		t.Parallel()
@@ -135,7 +38,7 @@ func testBackendConfigAccess(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, connURL, connToken := prepareTestContainer(t, version)
+	cleanup, connURL, connToken := consul.PrepareTestContainer(t, version)
 	defer cleanup()
 
 	connData := map[string]interface{}{
@@ -200,7 +103,7 @@ func testBackendRenewRevoke(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, connURL, connToken := prepareTestContainer(t, version)
+	cleanup, connURL, connToken := consul.PrepareTestContainer(t, version)
 	defer cleanup()
 	connData := map[string]interface{}{
 		"address": connURL,
@@ -305,7 +208,7 @@ func testBackendRenewRevoke14(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, connURL, connToken := prepareTestContainer(t, version)
+	cleanup, connURL, connToken := consul.PrepareTestContainer(t, version)
 	defer cleanup()
 	connData := map[string]interface{}{
 		"address": connURL,
@@ -414,7 +317,7 @@ func TestBackend_LocalToken(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cleanup, connURL, connToken := prepareTestContainer(t, "1.4.0-rc1")
+	cleanup, connURL, connToken := consul.PrepareTestContainer(t, "1.4.0-rc1")
 	defer cleanup()
 	connData := map[string]interface{}{
 		"address": connURL,
@@ -556,7 +459,7 @@ func testBackendManagement(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, connURL, connToken := prepareTestContainer(t, version)
+	cleanup, connURL, connToken := consul.PrepareTestContainer(t, version)
 	defer cleanup()
 	connData := map[string]interface{}{
 		"address": connURL,
@@ -600,7 +503,7 @@ func testBackendBasic(t *testing.T, version string) {
 		t.Fatal(err)
 	}
 
-	cleanup, connURL, connToken := prepareTestContainer(t, version)
+	cleanup, connURL, connToken := consul.PrepareTestContainer(t, version)
 	defer cleanup()
 	connData := map[string]interface{}{
 		"address": connURL,

builtin/logical/database/backend_test.go

@@ -15,6 +15,7 @@ import (
 	"github.com/go-test/deep"
 	"github.com/hashicorp/vault/api"
 	"github.com/hashicorp/vault/helper/namespace"
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	vaulthttp "github.com/hashicorp/vault/http"
 	"github.com/hashicorp/vault/plugins/database/postgresql"
 	"github.com/hashicorp/vault/sdk/database/dbplugin"
@@ -50,10 +51,7 @@ func preparePostgresTestContainer(t *testing.T, s logical.Storage, b logical.Bac
 	}
 
 	cleanup = func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	retURL = fmt.Sprintf("postgres://postgres:secret@localhost:%s/database?sslmode=disable", resource.GetPort("5432/tcp"))

builtin/logical/mongodb/backend_test.go

@@ -4,73 +4,20 @@ import (
 	"context"
 	"fmt"
 	"log"
-	"os"
 	"strings"
 	"sync"
 	"testing"
-	"time"
 
 	logicaltest "github.com/hashicorp/vault/helper/testhelpers/logical"
+	"github.com/hashicorp/vault/helper/testhelpers/mongodb"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/mitchellh/mapstructure"
-	dockertest "gopkg.in/ory-am/dockertest.v2"
 )
 
 var (
 	testImagePull sync.Once
 )
 
-func prepareTestContainer(t *testing.T, s logical.Storage, b logical.Backend) (cid dockertest.ContainerID, retURI string) {
-	if os.Getenv("MONGODB_URI") != "" {
-		return "", os.Getenv("MONGODB_URI")
-	}
-
-	// Without this the checks for whether the container has started seem to
-	// never actually pass. There's really no reason to expose the test
-	// containers, so don't.
-	dockertest.BindDockerToLocalhost = "yep"
-
-	testImagePull.Do(func() {
-		dockertest.Pull(dockertest.MongoDBImageName)
-	})
-
-	cid, connErr := dockertest.ConnectToMongoDB(60, 500*time.Millisecond, func(connURI string) bool {
-		connURI = "mongodb://" + connURI
-		// This will cause a validation to run
-		resp, err := b.HandleRequest(context.Background(), &logical.Request{
-			Storage:   s,
-			Operation: logical.UpdateOperation,
-			Path:      "config/connection",
-			Data: map[string]interface{}{
-				"uri": connURI,
-			},
-		})
-		if err != nil || (resp != nil && resp.IsError()) {
-			// It's likely not up and running yet, so return false and try again
-			return false
-		}
-		if resp == nil {
-			t.Fatal("expected warning")
-		}
-
-		retURI = connURI
-		return true
-	})
-
-	if connErr != nil {
-		t.Fatalf("could not connect to database: %v", connErr)
-	}
-
-	return
-}
-
-func cleanupTestContainer(t *testing.T, cid dockertest.ContainerID) {
-	err := cid.KillRemove()
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
 func TestBackend_config_connection(t *testing.T) {
 	var resp *logical.Response
 	var err error
@@ -112,10 +59,8 @@ func TestBackend_basic(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cid, connURI := prepareTestContainer(t, config.StorageView, b)
-	if cid != "" {
-		defer cleanupTestContainer(t, cid)
-	}
+	cleanup, connURI := mongodb.PrepareTestContainer(t, "latest")
+	defer cleanup()
 	connData := map[string]interface{}{
 		"uri": connURI,
 	}
@@ -138,10 +83,8 @@ func TestBackend_roleCrud(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cid, connURI := prepareTestContainer(t, config.StorageView, b)
-	if cid != "" {
-		defer cleanupTestContainer(t, cid)
-	}
+	cleanup, connURI := mongodb.PrepareTestContainer(t, "latest")
+	defer cleanup()
 	connData := map[string]interface{}{
 		"uri": connURI,
 	}
@@ -166,10 +109,8 @@ func TestBackend_leaseWriteRead(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	cid, connURI := prepareTestContainer(t, config.StorageView, b)
-	if cid != "" {
-		defer cleanupTestContainer(t, cid)
-	}
+	cleanup, connURI := mongodb.PrepareTestContainer(t, "latest")
+	defer cleanup()
 	connData := map[string]interface{}{
 		"uri": connURI,
 	}

builtin/logical/mysql/backend_test.go

@@ -9,6 +9,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	logicaltest "github.com/hashicorp/vault/helper/testhelpers/logical"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/mitchellh/mapstructure"
@@ -31,10 +32,7 @@ func prepareTestContainer(t *testing.T) (func(), string) {
 	}
 
 	cleanup := func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	retURL := fmt.Sprintf("root:secret@(localhost:%s)/mysql?parseTime=true", resource.GetPort("3306/tcp"))

builtin/logical/nomad/backend_test.go

@@ -10,6 +10,7 @@ import (
 
 	nomadapi "github.com/hashicorp/nomad/api"
 	"github.com/hashicorp/vault/helper/testhelpers"
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/mitchellh/mapstructure"
 	"github.com/ory/dockertest"
@@ -41,10 +42,7 @@ func prepareTestContainer(t *testing.T) (cleanup func(), retAddress string, noma
 	}
 
 	cleanup = func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	retAddress = fmt.Sprintf("http://localhost:%s/", resource.GetPort("4646/tcp"))

builtin/logical/postgresql/backend_test.go

@@ -11,6 +11,7 @@ import (
 	"reflect"
 	"testing"
 
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	logicaltest "github.com/hashicorp/vault/helper/testhelpers/logical"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/lib/pq"
@@ -34,10 +35,7 @@ func prepareTestContainer(t *testing.T) (cleanup func(), retURL string) {
 	}
 
 	cleanup = func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	retURL = fmt.Sprintf("postgres://postgres:secret@localhost:%s/database?sslmode=disable", resource.GetPort("5432/tcp"))

builtin/logical/rabbitmq/backend_test.go

@@ -8,6 +8,7 @@ import (
 	"strconv"
 	"testing"
 
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	logicaltest "github.com/hashicorp/vault/helper/testhelpers/logical"
 	"github.com/hashicorp/vault/sdk/helper/jsonutil"
 	"github.com/hashicorp/vault/sdk/logical"
@@ -42,10 +43,7 @@ func prepareRabbitMQTestContainer(t *testing.T) (func(), string, int) {
 	}
 
 	cleanup := func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	port, _ := strconv.Atoi(resource.GetPort("15672/tcp"))

helper/awsutil/region_test.go

@@ -86,8 +86,11 @@ func TestGetOrDefaultRegion_ConfigFilesPreferredThird(t *testing.T) {
 }
 
 func TestGetOrDefaultRegion_ConfigFileUnfound(t *testing.T) {
-	configuredRegion := ""
+	if enabled := os.Getenv("VAULT_ACC"); enabled == "" {
+		t.Skip()
+	}
 
+	configuredRegion := ""
 	cleanupEnv := setEnvRegion(t, "")
 	defer cleanupEnv()
 
@@ -125,6 +128,10 @@ func TestGetOrDefaultRegion_EC2InstanceMetadataPreferredFourth(t *testing.T) {
 }
 
 func TestGetOrDefaultRegion_DefaultsToDefaultRegionWhenRegionUnavailable(t *testing.T) {
+	if enabled := os.Getenv("VAULT_ACC"); enabled == "" {
+		t.Skip()
+	}
+
 	configuredRegion := ""
 
 	cleanupEnv := setEnvRegion(t, "")

helper/testhelpers/consul/consulhelper.go

@@ -0,0 +1,105 @@
+package consul
+
+import (
+	"fmt"
+	"os"
+	"strings"
+	"testing"
+
+	consulapi "github.com/hashicorp/consul/api"
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
+	"github.com/ory/dockertest"
+)
+
+func PrepareTestContainer(t *testing.T, version string) (cleanup func(), retAddress string, consulToken string) {
+	t.Logf("preparing test container")
+	consulToken = os.Getenv("CONSUL_HTTP_TOKEN")
+	retAddress = os.Getenv("CONSUL_HTTP_ADDR")
+	if retAddress != "" {
+		return func() {}, retAddress, consulToken
+	}
+
+	pool, err := dockertest.NewPool("")
+	if err != nil {
+		t.Fatalf("Failed to connect to docker: %s", err)
+	}
+
+	config := `acl { enabled = true default_policy = "deny" }`
+	if strings.HasPrefix(version, "1.3") {
+		config = `datacenter = "test" acl_default_policy = "deny" acl_datacenter = "test" acl_master_token = "test"`
+	}
+
+	dockerOptions := &dockertest.RunOptions{
+		Repository: "consul",
+		Tag:        version,
+		Cmd:        []string{"agent", "-dev", "-client", "0.0.0.0", "-hcl", config},
+	}
+	resource, err := pool.RunWithOptions(dockerOptions)
+	if err != nil {
+		t.Fatalf("Could not start local Consul %s docker container: %s", version, err)
+	}
+
+	cleanup = func() {
+		docker.CleanupResource(t, pool, resource)
+	}
+
+	retAddress = fmt.Sprintf("localhost:%s", resource.GetPort("8500/tcp"))
+
+	// exponential backoff-retry
+	if err = pool.Retry(func() error {
+		var err error
+		consulConfig := consulapi.DefaultNonPooledConfig()
+		consulConfig.Address = retAddress
+		consul, err := consulapi.NewClient(consulConfig)
+		if err != nil {
+			return err
+		}
+
+		// For version of Consul < 1.4
+		if strings.HasPrefix(version, "1.3") {
+			consulToken = "test"
+			_, err = consul.KV().Put(&consulapi.KVPair{
+				Key:   "setuptest",
+				Value: []byte("setuptest"),
+			}, &consulapi.WriteOptions{
+				Token: consulToken,
+			})
+			if err != nil {
+				return err
+			}
+			return nil
+		}
+
+		// New default behavior
+		aclbootstrap, _, err := consul.ACL().Bootstrap()
+		if err != nil {
+			return err
+		}
+		consulToken = aclbootstrap.SecretID
+		t.Logf("Generated Master token: %s", consulToken)
+		policy := &consulapi.ACLPolicy{
+			Name:        "test",
+			Description: "test",
+			Rules: `node_prefix "" {
+                policy = "write"
+              }
+
+              service_prefix "" {
+                policy = "read"
+              }
+      `,
+		}
+		q := &consulapi.WriteOptions{
+			Token: consulToken,
+		}
+		_, _, err = consul.ACL().PolicyCreate(policy, q)
+		if err != nil {
+			return err
+		}
+		return nil
+	}); err != nil {
+		cleanup()
+		t.Fatalf("Could not connect to docker: %s", err)
+	}
+	return cleanup, retAddress, consulToken
+}

helper/testhelpers/docker/testhelpers.go

@@ -0,0 +1,25 @@
+package docker
+
+import (
+	"strings"
+	"time"
+
+	"github.com/mitchellh/go-testing-interface"
+	"github.com/ory/dockertest"
+)
+
+func CleanupResource(t testing.T, pool *dockertest.Pool, resource *dockertest.Resource) {
+	var err error
+	for i := 0; i < 10; i++ {
+		err = pool.Purge(resource)
+		if err == nil {
+			return
+		}
+		time.Sleep(1 * time.Second)
+	}
+
+	if strings.Contains(err.Error(), "No such container") {
+		return
+	}
+	t.Fatalf("Failed to cleanup local container: %s", err)
+}

helper/testhelpers/mongodb/mongodbhelper.go

@@ -0,0 +1,56 @@
+package mongodb
+
+import (
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
+	"github.com/ory/dockertest"
+	"gopkg.in/mgo.v2"
+)
+
+func PrepareTestContainer(t *testing.T, version string) (cleanup func(), retURL string) {
+	if os.Getenv("MONGODB_URL") != "" {
+		return func() {}, os.Getenv("MONGODB_URL")
+	}
+
+	pool, err := dockertest.NewPool("")
+	if err != nil {
+		t.Fatalf("Failed to connect to docker: %s", err)
+	}
+
+	resource, err := pool.Run("mongo", "latest", []string{})
+	if err != nil {
+		t.Fatalf("Could not start local mongo docker container: %s", err)
+	}
+
+	cleanup = func() {
+		docker.CleanupResource(t, pool, resource)
+	}
+
+	addr := fmt.Sprintf("localhost:%s", resource.GetPort("27017/tcp"))
+	retURL = "mongodb://" + addr
+
+	// exponential backoff-retry
+	if err = pool.Retry(func() error {
+		session, err := mgo.DialWithInfo(&mgo.DialInfo{
+			Addrs:   []string{addr},
+			Timeout: 10 * time.Second,
+		})
+		if err != nil {
+			return err
+		}
+		defer session.Close()
+
+		session.SetSyncTimeout(1 * time.Minute)
+		session.SetSocketTimeout(1 * time.Minute)
+		return session.Ping()
+	}); err != nil {
+		cleanup()
+		t.Fatalf("Could not connect to mongo docker container: %s", err)
+	}
+
+	return
+}

physical/cassandra/cassandra_test.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/gocql/gocql"
 	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/physical"
 	"github.com/ory/dockertest"
@@ -70,7 +71,7 @@ func prepareCassandraTestContainer(t *testing.T) (func(), string) {
 	}
 
 	cleanup := func() {
-		pool.Purge(resource)
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	setup := func() error {

physical/cockroachdb/cockroachdb_test.go

@@ -6,11 +6,11 @@ import (
 	"os"
 	"testing"
 
-	"github.com/ory/dockertest"
-
 	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/physical"
+	"github.com/ory/dockertest"
 
 	_ "github.com/lib/pq"
 )
@@ -41,10 +41,7 @@ func prepareCockroachDBTestContainer(t *testing.T) (cleanup func(), retURL, tabl
 	}
 
 	cleanup = func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	retURL = fmt.Sprintf("postgresql://root@localhost:%s/?sslmode=disable", resource.GetPort("26257/tcp"))

physical/consul/consul_test.go

@@ -9,13 +9,12 @@ import (
 	"testing"
 	"time"
 
-	log "github.com/hashicorp/go-hclog"
-
 	"github.com/hashicorp/consul/api"
+	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/helper/testhelpers/consul"
 	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/helper/strutil"
 	"github.com/hashicorp/vault/sdk/physical"
-	dockertest "gopkg.in/ory-am/dockertest.v2"
 )
 
 type consulConf map[string]string
@@ -25,12 +24,6 @@ var (
 	testImagePull sync.Once
 )
 
-func testHostIP() string {
-	a := addrCount
-	addrCount++
-	return fmt.Sprintf("127.0.0.%d", a)
-}
-
 func testConsulBackend(t *testing.T) *ConsulBackend {
 	return testConsulBackendConfig(t, &consulConf{})
 }
@@ -492,20 +485,17 @@ func TestConsul_serviceID(t *testing.T) {
 }
 
 func TestConsulBackend(t *testing.T) {
-	var token string
+	consulToken := os.Getenv("CONSUL_HTTP_TOKEN")
 	addr := os.Getenv("CONSUL_HTTP_ADDR")
 	if addr == "" {
-		cid, connURL := prepareTestContainer(t)
-		if cid != "" {
-			defer cleanupTestContainer(t, cid)
-		}
-		addr = connURL
-		token = dockertest.ConsulACLMasterToken
+		cleanup, connURL, token := consul.PrepareTestContainer(t, "1.4.0-rc1")
+		defer cleanup()
+		addr, consulToken = connURL, token
 	}
 
 	conf := api.DefaultConfig()
 	conf.Address = addr
-	conf.Token = token
+	conf.Token = consulToken
 	client, err := api.NewClient(conf)
 	if err != nil {
 		t.Fatalf("err: %v", err)
@@ -533,20 +523,17 @@ func TestConsulBackend(t *testing.T) {
 }
 
 func TestConsulHABackend(t *testing.T) {
-	var token string
+	consulToken := os.Getenv("CONSUL_HTTP_TOKEN")
 	addr := os.Getenv("CONSUL_HTTP_ADDR")
 	if addr == "" {
-		cid, connURL := prepareTestContainer(t)
-		if cid != "" {
-			defer cleanupTestContainer(t, cid)
-		}
-		addr = connURL
-		token = dockertest.ConsulACLMasterToken
+		cleanup, connURL, token := consul.PrepareTestContainer(t, "1.4.0-rc1")
+		defer cleanup()
+		addr, consulToken = connURL, token
 	}
 
 	conf := api.DefaultConfig()
 	conf.Address = addr
-	conf.Token = token
+	conf.Token = consulToken
 	client, err := api.NewClient(conf)
 	if err != nil {
 		t.Fatalf("err: %v", err)
@@ -589,61 +576,3 @@ func TestConsulHABackend(t *testing.T) {
 		t.Fatalf("bad addr: %v", host)
 	}
 }
-
-func prepareTestContainer(t *testing.T) (cid dockertest.ContainerID, retAddress string) {
-	if os.Getenv("CONSUL_HTTP_ADDR") != "" {
-		return "", os.Getenv("CONSUL_HTTP_ADDR")
-	}
-
-	// Without this the checks for whether the container has started seem to
-	// never actually pass. There's really no reason to expose the test
-	// containers, so don't.
-	dockertest.BindDockerToLocalhost = "yep"
-
-	testImagePull.Do(func() {
-		dockertest.Pull(dockertest.ConsulImageName)
-	})
-
-	try := 0
-	cid, connErr := dockertest.ConnectToConsul(60, 500*time.Millisecond, func(connAddress string) bool {
-		try += 1
-		// Build a client and verify that the credentials work
-		config := api.DefaultConfig()
-		config.Address = connAddress
-		config.Token = dockertest.ConsulACLMasterToken
-		client, err := api.NewClient(config)
-		if err != nil {
-			if try > 50 {
-				panic(err)
-			}
-			return false
-		}
-
-		_, err = client.KV().Put(&api.KVPair{
-			Key:   "setuptest",
-			Value: []byte("setuptest"),
-		}, nil)
-		if err != nil {
-			if try > 50 {
-				panic(err)
-			}
-			return false
-		}
-
-		retAddress = connAddress
-		return true
-	})
-
-	if connErr != nil {
-		t.Fatalf("could not connect to consul: %v", connErr)
-	}
-
-	return
-}
-
-func cleanupTestContainer(t *testing.T, cid dockertest.ContainerID) {
-	err := cid.KillRemove()
-	if err != nil {
-		t.Fatal(err)
-	}
-}

physical/couchdb/couchdb_test.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/physical"
 	"github.com/ory/dockertest"
@@ -72,10 +73,7 @@ func prepareCouchdbDBTestContainer(t *testing.T) (cleanup func(), retAddress, us
 
 	retAddress = "http://localhost:" + resource.GetPort("5984/tcp")
 	cleanup = func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local DynamoDB: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	// exponential backoff-retry, because the couchDB may not be able to accept

physical/dynamodb/dynamodb_test.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/go-test/deep"
 	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/physical"
 	"github.com/ory/dockertest"
@@ -380,10 +381,7 @@ func prepareDynamoDBTestContainer(t *testing.T) (cleanup func(), retAddress stri
 
 	retAddress = "http://localhost:" + resource.GetPort("8000/tcp")
 	cleanup = func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local DynamoDB: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	// exponential backoff-retry, because the DynamoDB may not be able to accept

physical/foundationdb/foundationdb_test.go

@@ -151,8 +151,18 @@ func prepareFoundationDBTestDirectory(t *testing.T, topDir string) (func(), stri
 	clusterFile := tmpFile.Name()
 
 	cleanup := func() {
-		pool.Purge(resource)
+		var err error
+		for i := 0; i < 10; i++ {
+			err = pool.Purge(resource)
+			if err == nil {
+				break
+			}
+			time.Sleep(1 * time.Second)
+		}
 		os.Remove(clusterFile)
+		if err != nil {
+			t.Fatalf("Failed to cleanup local container: %s", err)
+		}
 	}
 
 	setup := func() error {

physical/postgresql/postgresql_test.go

@@ -6,11 +6,12 @@ import (
 	"testing"
 
 	log "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/physical"
+	"github.com/ory/dockertest"
 
 	_ "github.com/lib/pq"
-	"github.com/ory/dockertest"
 )
 
 func TestPostgreSQLBackend(t *testing.T) {
@@ -100,10 +101,7 @@ func prepareTestContainer(t *testing.T, logger log.Logger) (cleanup func(), retC
 	retConnString = fmt.Sprintf("postgres://postgres@localhost:%v/postgres?sslmode=disable", resource.GetPort("5432/tcp"))
 
 	cleanup = func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup docker Postgres: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	// Provide a test function to the pool to test if docker instance service is up.

physical/s3/s3_test.go

@@ -26,6 +26,10 @@ func TestS3BackendSseKms(t *testing.T) {
 }
 
 func DoS3BackendTest(t *testing.T, kmsKeyId string) {
+	if enabled := os.Getenv("VAULT_ACC"); enabled == "" {
+		t.Skip()
+	}
+
 	credsConfig := &awsutil.CredentialsConfig{}
 
 	credsChain, err := credsConfig.GenerateCredentialChain()

plugins/database/cassandra/cassandra_test.go

@@ -2,15 +2,15 @@ package cassandra
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"strconv"
 	"testing"
 	"time"
 
-	"fmt"
-
 	"github.com/gocql/gocql"
 	"github.com/hashicorp/errwrap"
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	"github.com/hashicorp/vault/sdk/database/dbplugin"
 	"github.com/ory/dockertest"
 )
@@ -40,10 +40,7 @@ func prepareCassandraTestContainer(t *testing.T) (func(), string, int) {
 	}
 
 	cleanup := func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	port, _ := strconv.Atoi(resource.GetPort("9042/tcp"))

plugins/database/influxdb/influxdb_test.go

@@ -9,6 +9,7 @@ import (
 	"time"
 
 	"github.com/hashicorp/errwrap"
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	"github.com/hashicorp/vault/sdk/database/dbplugin"
 	influx "github.com/influxdata/influxdb/client/v2"
 	"github.com/ory/dockertest"
@@ -37,10 +38,7 @@ func prepareInfluxdbTestContainer(t *testing.T) (func(), string, int) {
 	}
 
 	cleanup := func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	port, _ := strconv.Atoi(resource.GetPort("8086/tcp"))

plugins/database/mongodb/mongodb_test.go

@@ -3,72 +3,21 @@ package mongodb
 import (
 	"context"
 	"fmt"
-	"os"
+	"strings"
 	"testing"
 	"time"
 
-	mgo "gopkg.in/mgo.v2"
-
-	"strings"
-
+	"github.com/hashicorp/vault/helper/testhelpers/mongodb"
 	"github.com/hashicorp/vault/sdk/database/dbplugin"
-	"github.com/ory/dockertest"
+	"gopkg.in/mgo.v2"
 )
 
 const testMongoDBRole = `{ "db": "admin", "roles": [ { "role": "readWrite" } ] }`
 
 const testMongoDBWriteConcern = `{ "wmode": "majority", "wtimeout": 5000 }`
 
-func prepareMongoDBTestContainer(t *testing.T) (cleanup func(), retURL string) {
-	if os.Getenv("MONGODB_URL") != "" {
-		return func() {}, os.Getenv("MONGODB_URL")
-	}
-
-	pool, err := dockertest.NewPool("")
-	if err != nil {
-		t.Fatalf("Failed to connect to docker: %s", err)
-	}
-
-	resource, err := pool.Run("mongo", "latest", []string{})
-	if err != nil {
-		t.Fatalf("Could not start local mongo docker container: %s", err)
-	}
-
-	cleanup = func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
-	}
-
-	retURL = fmt.Sprintf("mongodb://localhost:%s", resource.GetPort("27017/tcp"))
-
-	// exponential backoff-retry
-	if err = pool.Retry(func() error {
-		var err error
-		dialInfo, err := parseMongoURL(retURL)
-		if err != nil {
-			return err
-		}
-
-		session, err := mgo.DialWithInfo(dialInfo)
-		if err != nil {
-			return err
-		}
-		defer session.Close()
-		session.SetSyncTimeout(1 * time.Minute)
-		session.SetSocketTimeout(1 * time.Minute)
-		return session.Ping()
-	}); err != nil {
-		cleanup()
-		t.Fatalf("Could not connect to mongo docker container: %s", err)
-	}
-
-	return
-}
-
 func TestMongoDB_Initialize(t *testing.T) {
-	cleanup, connURL := prepareMongoDBTestContainer(t)
+	cleanup, connURL := mongodb.PrepareTestContainer(t, "latest")
 	defer cleanup()
 
 	connectionDetails := map[string]interface{}{
@@ -92,7 +41,7 @@ func TestMongoDB_Initialize(t *testing.T) {
 }
 
 func TestMongoDB_CreateUser(t *testing.T) {
-	cleanup, connURL := prepareMongoDBTestContainer(t)
+	cleanup, connURL := mongodb.PrepareTestContainer(t, "latest")
 	defer cleanup()
 
 	connectionDetails := map[string]interface{}{
@@ -125,7 +74,7 @@ func TestMongoDB_CreateUser(t *testing.T) {
 }
 
 func TestMongoDB_CreateUser_writeConcern(t *testing.T) {
-	cleanup, connURL := prepareMongoDBTestContainer(t)
+	cleanup, connURL := mongodb.PrepareTestContainer(t, "latest")
 	defer cleanup()
 
 	connectionDetails := map[string]interface{}{
@@ -159,7 +108,7 @@ func TestMongoDB_CreateUser_writeConcern(t *testing.T) {
 }
 
 func TestMongoDB_RevokeUser(t *testing.T) {
-	cleanup, connURL := prepareMongoDBTestContainer(t)
+	cleanup, connURL := mongodb.PrepareTestContainer(t, "latest")
 	defer cleanup()
 
 	connectionDetails := map[string]interface{}{

plugins/database/mysql/mysql_test.go

@@ -9,9 +9,9 @@ import (
 	"testing"
 	"time"
 
-	"github.com/hashicorp/vault/sdk/database/helper/credsutil"
-
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	"github.com/hashicorp/vault/sdk/database/dbplugin"
+	"github.com/hashicorp/vault/sdk/database/helper/credsutil"
 	"github.com/ory/dockertest"
 )
 
@@ -36,10 +36,7 @@ func prepareMySQLTestContainer(t *testing.T, legacy bool) (cleanup func(), retUR
 	}
 
 	cleanup = func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	retURL = fmt.Sprintf("root:secret@(localhost:%s)/mysql?parseTime=true", resource.GetPort("3306/tcp"))

plugins/database/postgresql/postgresql_test.go

@@ -10,6 +10,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	"github.com/hashicorp/vault/sdk/database/dbplugin"
 	"github.com/ory/dockertest"
 )
@@ -34,10 +35,7 @@ func preparePostgresTestContainer(t *testing.T) (cleanup func(), retURL string)
 	}
 
 	cleanup = func() {
-		err := pool.Purge(resource)
-		if err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	retURL = fmt.Sprintf("postgres://postgres:secret@localhost:%s/database?sslmode=disable", resource.GetPort("5432/tcp"))

vault/external_tests/api/api_integration_test.go

@@ -1,29 +1,24 @@
 package api
 
 import (
-	"context"
 	"database/sql"
 	"encoding/base64"
 	"fmt"
-	"net"
-	"net/http"
 	"testing"
-	"time"
 
+	log "github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/vault/api"
 	"github.com/hashicorp/vault/audit"
+	auditFile "github.com/hashicorp/vault/builtin/audit/file"
+	credUserpass "github.com/hashicorp/vault/builtin/credential/userpass"
 	"github.com/hashicorp/vault/builtin/logical/database"
 	"github.com/hashicorp/vault/builtin/logical/pki"
 	"github.com/hashicorp/vault/builtin/logical/transit"
 	"github.com/hashicorp/vault/helper/builtinplugins"
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
+	"github.com/hashicorp/vault/http"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/vault"
-
-	log "github.com/hashicorp/go-hclog"
-
-	auditFile "github.com/hashicorp/vault/builtin/audit/file"
-	credUserpass "github.com/hashicorp/vault/builtin/credential/userpass"
-	vaulthttp "github.com/hashicorp/vault/http"
 	"github.com/ory/dockertest"
 )
 
@@ -67,7 +62,7 @@ func testVaultServerCoreConfig(t testing.TB, coreConfig *vault.CoreConfig) (*api
 	t.Helper()
 
 	cluster := vault.NewTestCluster(t, coreConfig, &vault.TestClusterOptions{
-		HandlerFunc: vaulthttp.Handler,
+		HandlerFunc: http.Handler,
 	})
 	cluster.Start()
 
@@ -89,48 +84,6 @@ func testVaultServerCoreConfig(t testing.TB, coreConfig *vault.CoreConfig) (*api
 	return client, unsealKeys, func() { defer cluster.Cleanup() }
 }
 
-// testVaultServerBad creates an http server that returns a 500 on each request
-// to simulate failures.
-func testVaultServerBad(t testing.TB) (*api.Client, func()) {
-	t.Helper()
-
-	listener, err := net.Listen("tcp", "127.0.0.1:0")
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	server := &http.Server{
-		Addr: "127.0.0.1:0",
-		Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-			http.Error(w, "500 internal server error", http.StatusInternalServerError)
-		}),
-		ReadTimeout:       1 * time.Second,
-		ReadHeaderTimeout: 1 * time.Second,
-		WriteTimeout:      1 * time.Second,
-		IdleTimeout:       1 * time.Second,
-	}
-
-	go func() {
-		if err := server.Serve(listener); err != nil && err != http.ErrServerClosed {
-			t.Fatal(err)
-		}
-	}()
-
-	client, err := api.NewClient(&api.Config{
-		Address: "http://" + listener.Addr().String(),
-	})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	return client, func() {
-		ctx, done := context.WithTimeout(context.Background(), 5*time.Second)
-		defer done()
-
-		server.Shutdown(ctx)
-	}
-}
-
 // testPostgresDB creates a testing postgres database in a Docker container,
 // returning the connection URL and the associated closer function.
 func testPostgresDB(t testing.TB) (string, func()) {
@@ -148,9 +101,7 @@ func testPostgresDB(t testing.TB) (string, func()) {
 	}
 
 	cleanup := func() {
-		if err := pool.Purge(resource); err != nil {
-			t.Fatalf("failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	addr := fmt.Sprintf("postgres://postgres:secret@localhost:%s/database?sslmode=disable", resource.GetPort("5432/tcp"))

vault/seal/transit/transit_acc_test.go

@@ -3,19 +3,24 @@ package transit
 import (
 	"context"
 	"fmt"
+	"os"
 	"path"
 	"reflect"
 	"testing"
 	"time"
 
 	log "github.com/hashicorp/go-hclog"
-	uuid "github.com/hashicorp/go-uuid"
+	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/vault/api"
+	"github.com/hashicorp/vault/helper/testhelpers/docker"
 	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/ory/dockertest"
 )
 
 func TestTransitSeal_Lifecycle(t *testing.T) {
+	if os.Getenv("VAULT_ACC") == "" {
+		t.Skip()
+	}
 	cleanup, retAddress, token, mountPath, keyName, _ := prepareTestContainer(t)
 	defer cleanup()
 
@@ -49,6 +54,9 @@ func TestTransitSeal_Lifecycle(t *testing.T) {
 }
 
 func TestTransitSeal_TokenRenewal(t *testing.T) {
+	if os.Getenv("VAULT_ACC") == "" {
+		t.Skip()
+	}
 	cleanup, retAddress, token, mountPath, keyName, tlsConfig := prepareTestContainer(t)
 	defer cleanup()
 
@@ -135,9 +143,7 @@ func prepareTestContainer(t *testing.T) (cleanup func(), retAddress, token, moun
 	}
 
 	cleanup = func() {
-		if err := pool.Purge(resource); err != nil {
-			t.Fatalf("Failed to cleanup local container: %s", err)
-		}
+		docker.CleanupResource(t, pool, resource)
 	}
 
 	retAddress = fmt.Sprintf("http://127.0.0.1:%s", resource.GetPort("8200/tcp"))