diff --git a/command/operator_diagnose.go b/command/operator_diagnose.go index 47fa7b5c90..c164314aea 100644 --- a/command/operator_diagnose.go +++ b/command/operator_diagnose.go @@ -4,18 +4,26 @@ import ( "context" "encoding/json" "fmt" + "io" "os" "strings" "sync" + "time" "github.com/docker/docker/pkg/ioutils" "github.com/hashicorp/consul/api" log "github.com/hashicorp/go-hclog" + uuid "github.com/hashicorp/go-uuid" + "github.com/hashicorp/vault/helper/metricsutil" + "github.com/hashicorp/vault/internalshared/configutil" "github.com/hashicorp/vault/internalshared/listenerutil" "github.com/hashicorp/vault/internalshared/reloadutil" physconsul "github.com/hashicorp/vault/physical/consul" + "github.com/hashicorp/vault/sdk/physical" "github.com/hashicorp/vault/sdk/version" + sr "github.com/hashicorp/vault/serviceregistration" srconsul "github.com/hashicorp/vault/serviceregistration/consul" + "github.com/hashicorp/vault/vault" "github.com/hashicorp/vault/vault/diagnose" "github.com/mitchellh/cli" "github.com/posener/complete" @@ -23,6 +31,10 @@ import ( const OperatorDiagnoseEnableEnv = "VAULT_DIAGNOSE" +const CoreUninitializedErr = "diagnose cannot attempt this step because core could not be initialized" +const BackendUninitializedErr = "diagnose cannot attempt this step because backend could not be initialized" +const CoreConfigUninitializedErr = "diagnose cannot attempt this step because core config could not be set" + var ( _ cli.Command = (*OperatorDiagnoseCommand)(nil) _ cli.CommandAutocomplete = (*OperatorDiagnoseCommand)(nil) @@ -37,11 +49,12 @@ type OperatorDiagnoseCommand struct { flagConfigs []string cleanupGuard sync.Once - reloadFuncsLock *sync.RWMutex - reloadFuncs *map[string][]reloadutil.ReloadFunc - startedCh chan struct{} // for tests - reloadedCh chan struct{} // for tests - skipEndEnd bool // for tests + reloadFuncsLock *sync.RWMutex + reloadFuncs *map[string][]reloadutil.ReloadFunc + ServiceRegistrations map[string]sr.Factory + startedCh chan struct{} // for tests + reloadedCh chan struct{} // for tests + skipEndEnd bool // for tests } func (c *OperatorDiagnoseCommand) Synopsis() string { @@ -203,18 +216,230 @@ func (c *OperatorDiagnoseCommand) offlineDiagnostics(ctx context.Context) error } else { diagnose.SpotOk(ctx, "parse-config", "") } - // Check Listener Information - // TODO: Run Diagnose checks on the actual net.Listeners - if err := diagnose.Test(ctx, "init-listeners", func(ctx context.Context) error { + var metricSink *metricsutil.ClusterMetricSink + var metricsHelper *metricsutil.MetricsHelper + + var backend *physical.Backend + diagnose.Test(ctx, "storage", func(ctx context.Context) error { + diagnose.Test(ctx, "create-storage-backend", func(ctx context.Context) error { + + b, err := server.setupStorage(config) + if err != nil { + return err + } + backend = &b + return nil + }) + + if config.Storage == nil { + return fmt.Errorf("no storage stanza found in config") + } + + if config.Storage != nil && config.Storage.Type == storageTypeConsul { + diagnose.Test(ctx, "test-storage-tls-consul", func(ctx context.Context) error { + err = physconsul.SetupSecureTLS(api.DefaultConfig(), config.Storage.Config, server.logger, true) + if err != nil { + return err + } + return nil + }) + + diagnose.Test(ctx, "test-consul-direct-access-storage", func(ctx context.Context) error { + dirAccess := diagnose.ConsulDirectAccess(config.Storage.Config) + if dirAccess != "" { + diagnose.Warn(ctx, dirAccess) + } + return nil + }) + } + + // Attempt to use storage backend + if !c.skipEndEnd { + diagnose.Test(ctx, "test-access-storage", diagnose.WithTimeout(30*time.Second, func(ctx context.Context) error { + maxDurationCrudOperation := "write" + maxDuration := time.Duration(0) + uuidSuffix, err := uuid.GenerateUUID() + if err != nil { + return err + } + uuid := "diagnose/latency/" + uuidSuffix + dur, err := diagnose.EndToEndLatencyCheckWrite(ctx, uuid, *backend) + if err != nil { + return err + } + maxDuration = dur + dur, err = diagnose.EndToEndLatencyCheckRead(ctx, uuid, *backend) + if err != nil { + return err + } + if dur > maxDuration { + maxDuration = dur + maxDurationCrudOperation = "read" + } + dur, err = diagnose.EndToEndLatencyCheckDelete(ctx, uuid, *backend) + if err != nil { + return err + } + if dur > maxDuration { + maxDuration = dur + maxDurationCrudOperation = "delete" + } + + if maxDuration > time.Duration(0) { + diagnose.Warn(ctx, diagnose.LatencyWarning+fmt.Sprintf("duration: %s, ", maxDuration)+fmt.Sprintf("operation: %s", maxDurationCrudOperation)) + } + return nil + })) + } + return nil + }) + + var configSR sr.ServiceRegistration + diagnose.Test(ctx, "service-discovery", func(ctx context.Context) error { + if config.ServiceRegistration == nil || config.ServiceRegistration.Config == nil { + return fmt.Errorf("No service registration config") + } + srConfig := config.ServiceRegistration.Config + + diagnose.Test(ctx, "test-serviceregistration-tls-consul", func(ctx context.Context) error { + // SetupSecureTLS for service discovery uses the same cert and key to set up physical + // storage. See the consul package in physical for details. + err = srconsul.SetupSecureTLS(api.DefaultConfig(), srConfig, server.logger, true) + if err != nil { + return err + } + return nil + }) + + if config.ServiceRegistration != nil && config.ServiceRegistration.Type == "consul" { + diagnose.Test(ctx, "test-consul-direct-access-service-discovery", func(ctx context.Context) error { + dirAccess := diagnose.ConsulDirectAccess(config.ServiceRegistration.Config) + if dirAccess != "" { + diagnose.Warn(ctx, dirAccess) + } + return nil + }) + } + return nil + }) + + sealcontext, sealspan := diagnose.StartSpan(ctx, "create-seal") + var seals []vault.Seal + var sealConfigError error + barrierSeal, barrierWrapper, unwrapSeal, seals, sealConfigError, err := setSeal(server, config, make([]string, 0), make(map[string]string)) + // Check error here + if err != nil { + diagnose.Fail(sealcontext, err.Error()) + goto SEALFAIL + } + if sealConfigError != nil { + diagnose.Fail(sealcontext, "seal could not be configured: seals may already be initialized") + goto SEALFAIL + } + + if seals != nil { + for _, seal := range seals { + // Ensure that the seal finalizer is called, even if using verify-only + defer func(seal *vault.Seal) { + sealType := (*seal).BarrierType() + finalizeSealContext, finalizeSealSpan := diagnose.StartSpan(ctx, "finalize-seal-"+sealType) + err = (*seal).Finalize(finalizeSealContext) + if err != nil { + diagnose.Fail(finalizeSealContext, "error finalizing seal") + finalizeSealSpan.End() + } + finalizeSealSpan.End() + }(&seal) + } + } + + if barrierSeal == nil { + diagnose.Fail(sealcontext, "could not create barrier seal! Most likely proper Seal configuration information was not set, but no error was generated") + } + +SEALFAIL: + sealspan.End() + var coreConfig vault.CoreConfig + if err := diagnose.Test(ctx, "setup-core", func(ctx context.Context) error { + var secureRandomReader io.Reader + // prepare a secure random reader for core + secureRandomReader, err = configutil.CreateSecureRandomReaderFunc(config.SharedConfig, barrierWrapper) + if err != nil { + return diagnose.SpotError(ctx, "init-randreader", err) + } + diagnose.SpotOk(ctx, "init-randreader", "") + + if backend == nil { + return fmt.Errorf(BackendUninitializedErr) + } + coreConfig = createCoreConfig(server, config, *backend, configSR, barrierSeal, unwrapSeal, metricsHelper, metricSink, secureRandomReader) + return nil + }); err != nil { + diagnose.Error(ctx, err) + } + + var disableClustering bool + diagnose.Test(ctx, "setup-ha-storage", func(ctx context.Context) error { + if backend == nil { + return fmt.Errorf(BackendUninitializedErr) + } + diagnose.Test(ctx, "create-ha-storage-backend", func(ctx context.Context) error { + // Initialize the separate HA storage backend, if it exists + disableClustering, err = initHaBackend(server, config, &coreConfig, *backend) + if err != nil { + return err + } + return nil + }) + diagnose.Test(ctx, "test-consul-direct-access-storage", func(ctx context.Context) error { + dirAccess := diagnose.ConsulDirectAccess(config.HAStorage.Config) + if dirAccess != "" { + diagnose.Warn(ctx, dirAccess) + } + return nil + }) + if config.HAStorage != nil && config.HAStorage.Type == storageTypeConsul { + diagnose.Test(ctx, "test-storage-tls-consul", func(ctx context.Context) error { + err = physconsul.SetupSecureTLS(api.DefaultConfig(), config.HAStorage.Config, server.logger, true) + if err != nil { + return err + } + return nil + }) + } + return nil + }) + + // Determine the redirect address from environment variables + err = determineRedirectAddr(server, &coreConfig, config) + if err != nil { + return diagnose.SpotError(ctx, "determine-redirect", err) + } + diagnose.SpotOk(ctx, "determine-redirect", "") + + err = findClusterAddress(server, &coreConfig, config, disableClustering) + if err != nil { + return diagnose.SpotError(ctx, "find-cluster-addr", err) + } + diagnose.SpotOk(ctx, "find-cluster-addr", "") + + var lns []listenerutil.Listener + diagnose.Test(ctx, "init-listeners", func(ctx context.Context) error { disableClustering := config.HAStorage.DisableClustering infoKeys := make([]string, 0, 10) info := make(map[string]string) - status, lns, _, errMsg := server.InitListeners(config, disableClustering, &infoKeys, &info) + var listeners []listenerutil.Listener + var status int + diagnose.Test(ctx, "create-listeners", func(ctx context.Context) error { + status, listeners, _, err = server.InitListeners(config, disableClustering, &infoKeys, &info) + if status != 0 { + return err + } + return nil + }) - if status != 0 { - return errMsg - } + lns = listeners // Make sure we close all listeners from this point on listenerCloseFunc := func() { @@ -225,96 +450,39 @@ func (c *OperatorDiagnoseCommand) offlineDiagnostics(ctx context.Context) error defer c.cleanupGuard.Do(listenerCloseFunc) - sanitizedListeners := make([]listenerutil.Listener, 0, len(config.Listeners)) - for _, ln := range lns { - if ln.Config.TLSDisable { - diagnose.Warn(ctx, "TLS is disabled in a Listener config stanza.") - continue - } - if ln.Config.TLSDisableClientCerts { - diagnose.Warn(ctx, "TLS for a listener is turned on without requiring client certs.") - } + diagnose.Test(ctx, "check-listener-tls", func(ctx context.Context) error { + sanitizedListeners := make([]listenerutil.Listener, 0, len(config.Listeners)) + for _, ln := range lns { + if ln.Config.TLSDisable { + diagnose.Warn(ctx, "TLS is disabled in a Listener config stanza.") + continue + } + if ln.Config.TLSDisableClientCerts { + diagnose.Warn(ctx, "TLS for a listener is turned on without requiring client certs.") + } - // Check ciphersuite and load ca/cert/key files - // TODO: TLSConfig returns a reloadFunc and a TLSConfig. We can use this to - // perform an active probe. - _, _, err := listenerutil.TLSConfig(ln.Config, make(map[string]string), c.UI) + // Check ciphersuite and load ca/cert/key files + // TODO: TLSConfig returns a reloadFunc and a TLSConfig. We can use this to + // perform an active probe. + _, _, err := listenerutil.TLSConfig(ln.Config, make(map[string]string), c.UI) + if err != nil { + return err + } + + sanitizedListeners = append(sanitizedListeners, listenerutil.Listener{ + Listener: ln.Listener, + Config: ln.Config, + }) + } + err = diagnose.ListenerChecks(sanitizedListeners) if err != nil { return err } - - sanitizedListeners = append(sanitizedListeners, listenerutil.Listener{ - Listener: ln.Listener, - Config: ln.Config, - }) - } - return diagnose.ListenerChecks(sanitizedListeners) - }); err != nil { - return err - } - - // Errors in these items could stop Vault from starting but are not yet covered: - // TODO: logging configuration - // TODO: SetupTelemetry - if err := diagnose.Test(ctx, "storage", func(ctx context.Context) error { - b, err := server.setupStorage(config) - if err != nil { - return err - } - - dirAccess := diagnose.ConsulDirectAccess(config.HAStorage.Config) - if dirAccess != "" { - diagnose.Warn(ctx, dirAccess) - } - - if config.Storage != nil && config.Storage.Type == storageTypeConsul { - err = physconsul.SetupSecureTLS(api.DefaultConfig(), config.Storage.Config, server.logger, true) - if err != nil { - return err - } - - dirAccess := diagnose.ConsulDirectAccess(config.Storage.Config) - if dirAccess != "" { - diagnose.Warn(ctx, dirAccess) - } - } - - if config.HAStorage != nil && config.HAStorage.Type == storageTypeConsul { - err = physconsul.SetupSecureTLS(api.DefaultConfig(), config.HAStorage.Config, server.logger, true) - if err != nil { - return err - } - } - - // Attempt to use storage backend - if !c.skipEndEnd { - err = diagnose.StorageEndToEndLatencyCheck(ctx, b) - if err != nil { - return err - } - } - - return nil - }); err != nil { - return err - } - - diagnose.Test(ctx, "service-discovery", func(ctx context.Context) error { - srConfig := config.ServiceRegistration.Config - // Initialize the Service Discovery, if there is one - if config.ServiceRegistration != nil && config.ServiceRegistration.Type == "consul" { - // setupStorage populates the srConfig, so no nil checks are necessary. - dirAccess := diagnose.ConsulDirectAccess(config.ServiceRegistration.Config) - if dirAccess != "" { - diagnose.Warn(ctx, dirAccess) - } - - // SetupSecureTLS for service discovery uses the same cert and key to set up physical - // storage. See the consul package in physical for details. - return srconsul.SetupSecureTLS(api.DefaultConfig(), srConfig, server.logger, true) - } + return nil + }) return nil }) + // TODO: Diagnose logging configuration return nil } diff --git a/command/operator_diagnose_test.go b/command/operator_diagnose_test.go index c4a2a7228c..b29eb1f472 100644 --- a/command/operator_diagnose_test.go +++ b/command/operator_diagnose_test.go @@ -44,18 +44,96 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) { Status: diagnose.OkStatus, }, { - Name: "init-listeners", - Status: diagnose.WarningStatus, - Warnings: []string{ - "TLS is disabled in a Listener config stanza.", + Name: "storage", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "create-storage-backend", + Status: diagnose.OkStatus, + }, + { + Name: "test-storage-tls-consul", + Status: diagnose.OkStatus, + }, + { + Name: "test-consul-direct-access-storage", + Status: diagnose.OkStatus, + }, }, }, { - Name: "storage", + Name: "service-discovery", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "test-serviceregistration-tls-consul", + Status: diagnose.OkStatus, + }, + { + Name: "test-consul-direct-access-service-discovery", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "create-seal", Status: diagnose.OkStatus, }, { - Name: "service-discovery", + Name: "setup-core", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "init-randreader", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "setup-ha-storage", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "create-ha-storage-backend", + Status: diagnose.OkStatus, + }, + { + Name: "test-consul-direct-access-storage", + Status: diagnose.OkStatus, + }, + { + Name: "test-storage-tls-consul", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "determine-redirect", + Status: diagnose.OkStatus, + }, + { + Name: "find-cluster-addr", + Status: diagnose.OkStatus, + }, + { + Name: "init-listeners", + Status: diagnose.WarningStatus, + Children: []*diagnose.Result{ + { + Name: "create-listeners", + Status: diagnose.OkStatus, + }, + { + Name: "check-listener-tls", + Status: diagnose.WarningStatus, + Warnings: []string{ + "TLS is disabled in a Listener config stanza.", + }, + }, + }, + }, + { + Name: "finalize-seal-shamir", Status: diagnose.OkStatus, }, }, @@ -71,16 +149,68 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) { Status: diagnose.OkStatus, }, { - Name: "init-listeners", - Status: diagnose.WarningStatus, - Warnings: []string{ - "TLS is disabled in a Listener config stanza.", + Name: "storage", + Status: diagnose.ErrorStatus, + Message: "no storage stanza found in config", + Children: []*diagnose.Result{ + { + Name: "create-storage-backend", + Status: diagnose.ErrorStatus, + }, }, }, { - Name: "storage", + Name: "service-discovery", + Status: diagnose.ErrorStatus, + }, + { + Name: "create-seal", + Status: diagnose.OkStatus, + }, + { + Name: "setup-core", Status: diagnose.ErrorStatus, - Message: "A storage backend must be specified", + Message: BackendUninitializedErr, + Children: []*diagnose.Result{ + { + Name: "init-randreader", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "setup-ha-storage", + Status: diagnose.ErrorStatus, + Message: BackendUninitializedErr, + }, + { + Name: "determine-redirect", + Status: diagnose.OkStatus, + }, + { + Name: "find-cluster-addr", + Status: diagnose.OkStatus, + }, + { + Name: "init-listeners", + Status: diagnose.WarningStatus, + Children: []*diagnose.Result{ + { + Name: "create-listeners", + Status: diagnose.OkStatus, + }, + { + Name: "check-listener-tls", + Status: diagnose.WarningStatus, + Warnings: []string{ + "TLS is disabled in a Listener config stanza.", + }, + }, + }, + }, + { + Name: "finalize-seal-shamir", + Status: diagnose.OkStatus, }, }, }, @@ -94,17 +224,98 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) { Name: "parse-config", Status: diagnose.OkStatus, }, - { - Name: "init-listeners", - Status: diagnose.OkStatus, - }, { Name: "storage", Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "create-storage-backend", + Status: diagnose.OkStatus, + }, + { + Name: "test-storage-tls-consul", + Status: diagnose.OkStatus, + }, + { + Name: "test-consul-direct-access-storage", + Status: diagnose.OkStatus, + }, + }, }, { Name: "service-discovery", Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "test-serviceregistration-tls-consul", + Status: diagnose.OkStatus, + }, + { + Name: "test-consul-direct-access-service-discovery", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "create-seal", + Status: diagnose.OkStatus, + }, + { + Name: "setup-core", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "init-randreader", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "setup-ha-storage", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "create-ha-storage-backend", + Status: diagnose.OkStatus, + }, + { + Name: "test-consul-direct-access-storage", + Status: diagnose.OkStatus, + }, + { + Name: "test-storage-tls-consul", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "determine-redirect", + Status: diagnose.OkStatus, + }, + { + Name: "find-cluster-addr", + Status: diagnose.OkStatus, + }, + { + Name: "init-listeners", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "create-listeners", + Status: diagnose.OkStatus, + }, + { + Name: "check-listener-tls", + Status: diagnose.WarningStatus, + Warnings: []string{ + "TLS is disabled in a Listener config stanza.", + }, + }, + }, + }, + { + Name: "finalize-seal-shamir", + Status: diagnose.OkStatus, }, }, }, @@ -119,15 +330,69 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) { Status: diagnose.OkStatus, }, { - Name: "init-listeners", - Status: diagnose.WarningStatus, - Warnings: []string{ - "TLS is disabled in a Listener config stanza.", + Name: "storage", + Status: diagnose.ErrorStatus, + Children: []*diagnose.Result{ + { + Name: "create-storage-backend", + Status: diagnose.OkStatus, + }, + { + Name: "test-storage-tls-consul", + Status: diagnose.ErrorStatus, + Message: "expired", + }, + { + Name: "test-consul-direct-access-storage", + Status: diagnose.OkStatus, + }, }, }, { - Name: "storage", - Status: diagnose.ErrorStatus, + Name: "service-discovery", + Status: diagnose.WarningStatus, + Children: []*diagnose.Result{ + { + Name: "test-serviceregistration-tls-consul", + Status: diagnose.WarningStatus, + }, + { + Name: "test-consul-direct-access-service-discovery", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "create-seal", + Status: diagnose.OkStatus, + }, + { + Name: "setup-core", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "init-randreader", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "setup-ha-storage", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "create-ha-storage-backend", + Status: diagnose.OkStatus, + }, + { + Name: "test-consul-direct-access-storage", + Status: diagnose.OkStatus, + }, + { + Name: "test-storage-tls-consul", + Status: diagnose.OkStatus, + }, + }, }, }, }, @@ -142,17 +407,68 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) { Status: diagnose.OkStatus, }, { - Name: "init-listeners", + Name: "storage", Status: diagnose.WarningStatus, - Warnings: []string{ - "TLS is disabled in a Listener config stanza.", + Children: []*diagnose.Result{ + { + Name: "create-storage-backend", + Status: diagnose.OkStatus, + }, + { + Name: "test-storage-tls-consul", + Status: diagnose.WarningStatus, + }, + { + Name: "test-consul-direct-access-storage", + Status: diagnose.OkStatus, + }, }, }, { - Name: "storage", + Name: "service-discovery", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "test-serviceregistration-tls-consul", + Status: diagnose.OkStatus, + }, + { + Name: "test-consul-direct-access-service-discovery", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "create-seal", + Status: diagnose.OkStatus, + }, + { + Name: "setup-core", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "init-randreader", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "setup-ha-storage", Status: diagnose.ErrorStatus, - Warnings: []string{ - diagnose.AddrDNExistErr, + Children: []*diagnose.Result{ + { + Name: "create-ha-storage-backend", + Status: diagnose.OkStatus, + }, + { + Name: "test-consul-direct-access-storage", + Status: diagnose.OkStatus, + }, + { + Name: "test-storage-tls-consul", + Status: diagnose.ErrorStatus, + Message: "x509: certificate has expired or is not yet valid", + }, }, }, }, @@ -168,22 +484,39 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) { Status: diagnose.OkStatus, }, { - Name: "init-listeners", - Status: diagnose.WarningStatus, - Warnings: []string{ - "TLS is disabled in a Listener config stanza.", + Name: "storage", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "create-storage-backend", + Status: diagnose.OkStatus, + }, + { + Name: "test-storage-tls-consul", + Status: diagnose.OkStatus, + }, + { + Name: "test-consul-direct-access-storage", + Status: diagnose.OkStatus, + }, }, }, { - Name: "storage", - Status: diagnose.OkStatus, - }, - { - Name: "service-discovery", - Status: diagnose.ErrorStatus, - Message: "failed to verify certificate: x509: certificate has expired or is not yet valid:", - Warnings: []string{ - diagnose.DirAccessErr, + Name: "service-discovery", + Status: diagnose.ErrorStatus, + Children: []*diagnose.Result{ + { + Name: "test-serviceregistration-tls-consul", + Status: diagnose.ErrorStatus, + Message: "failed to verify certificate: x509: certificate has expired or is not yet valid", + }, + { + Name: "test-consul-direct-access-service-discovery", + Status: diagnose.WarningStatus, + Warnings: []string{ + diagnose.DirAccessErr, + }, + }, }, }, }, @@ -198,23 +531,101 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) { Name: "parse-config", Status: diagnose.OkStatus, }, - { - Name: "init-listeners", - Status: diagnose.WarningStatus, - Warnings: []string{ - "TLS is disabled in a Listener config stanza.", - }, - }, { Name: "storage", Status: diagnose.WarningStatus, - Warnings: []string{ - diagnose.DirAccessErr, + Children: []*diagnose.Result{ + { + Name: "create-storage-backend", + Status: diagnose.OkStatus, + }, + { + Name: "test-storage-tls-consul", + Status: diagnose.OkStatus, + }, + { + Name: "test-consul-direct-access-storage", + Status: diagnose.WarningStatus, + Warnings: []string{ + diagnose.DirAccessErr, + }, + }, }, }, { Name: "service-discovery", Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "test-serviceregistration-tls-consul", + Status: diagnose.OkStatus, + }, + { + Name: "test-consul-direct-access-service-discovery", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "create-seal", + Status: diagnose.OkStatus, + }, + { + Name: "setup-core", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "init-randreader", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "setup-ha-storage", + Status: diagnose.OkStatus, + Children: []*diagnose.Result{ + { + Name: "create-ha-storage-backend", + Status: diagnose.OkStatus, + }, + { + Name: "test-consul-direct-access-storage", + Status: diagnose.OkStatus, + }, + { + Name: "test-storage-tls-consul", + Status: diagnose.OkStatus, + }, + }, + }, + { + Name: "determine-redirect", + Status: diagnose.OkStatus, + }, + { + Name: "find-cluster-addr", + Status: diagnose.OkStatus, + }, + { + Name: "init-listeners", + Status: diagnose.WarningStatus, + Children: []*diagnose.Result{ + { + Name: "create-listeners", + Status: diagnose.OkStatus, + }, + { + Name: "check-listener-tls", + Status: diagnose.WarningStatus, + Warnings: []string{ + "TLS is disabled in a Listener config stanza.", + }, + }, + }, + }, + { + Name: "finalize-seal-shamir", + Status: diagnose.OkStatus, }, }, }, @@ -225,7 +636,6 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) { for _, tc := range cases { tc := tc - t.Run(tc.name, func(t *testing.T) { t.Parallel() client, closer := testVaultServer(t) @@ -238,9 +648,12 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) { result := cmd.diagnose.Finalize(context.Background()) for i, exp := range tc.expected { + if i >= len(result.Children) { + t.Fatalf("there are at least %d test cases, but fewer actual results", i) + } act := result.Children[i] if err := compareResult(t, exp, act); err != nil { - t.Fatalf("%v", err) + t.Fatalf("%v, %v, %v", err, act, exp) } } }) @@ -271,7 +684,11 @@ func compareResult(t *testing.T, exp *diagnose.Result, act *diagnose.Result) err } } if len(exp.Children) != len(act.Children) { - return fmt.Errorf("section %s, child count mismatch: %d vs %d", exp.Name, len(exp.Children), len(act.Children)) + errStrings := []string{} + for _, c := range act.Children { + errStrings = append(errStrings, fmt.Sprintf("%+v", c)) + } + return fmt.Errorf(strings.Join(errStrings, ",")) } return nil } diff --git a/command/server.go b/command/server.go index 103294942e..cec8e86929 100644 --- a/command/server.go +++ b/command/server.go @@ -875,6 +875,33 @@ func (c *ServerCommand) setupStorage(config *server.Config) (physical.Backend, e return backend, nil } +func beginServiceRegistration(c *ServerCommand, config *server.Config) (sr.ServiceRegistration, error) { + sdFactory, ok := c.ServiceRegistrations[config.ServiceRegistration.Type] + if !ok { + return nil, fmt.Errorf("Unknown service_registration type %s", config.ServiceRegistration.Type) + } + + namedSDLogger := c.logger.Named("service_registration." + config.ServiceRegistration.Type) + c.allLoggers = append(c.allLoggers, namedSDLogger) + + // Since we haven't even begun starting Vault's core yet, + // we know that Vault is in its pre-running state. + state := sr.State{ + VaultVersion: version.GetVersion().VersionNumber(), + IsInitialized: false, + IsSealed: true, + IsActive: false, + IsPerformanceStandby: false, + } + var err error + configSR, err := sdFactory(config.ServiceRegistration.Config, namedSDLogger, state) + if err != nil { + return nil, fmt.Errorf("Error initializing service_registration of type %s: %s", config.ServiceRegistration.Type, err) + } + + return configSR, nil +} + // InitListeners returns a response code, error message, Listeners, and a TCP Address list. func (c *ServerCommand) InitListeners(config *server.Config, disableClustering bool, infoKeys *[]string, info *map[string]string) (int, []listenerutil.Listener, []*net.TCPAddr, error) { clusterAddrs := []*net.TCPAddr{} @@ -1170,27 +1197,9 @@ func (c *ServerCommand) Run(args []string) int { // Initialize the Service Discovery, if there is one var configSR sr.ServiceRegistration if config.ServiceRegistration != nil { - sdFactory, ok := c.ServiceRegistrations[config.ServiceRegistration.Type] - if !ok { - c.UI.Error(fmt.Sprintf("Unknown service_registration type %s", config.ServiceRegistration.Type)) - return 1 - } - - namedSDLogger := c.logger.Named("service_registration." + config.ServiceRegistration.Type) - c.allLoggers = append(c.allLoggers, namedSDLogger) - - // Since we haven't even begun starting Vault's core yet, - // we know that Vault is in its pre-running state. - state := sr.State{ - VaultVersion: version.GetVersion().VersionNumber(), - IsInitialized: false, - IsSealed: true, - IsActive: false, - IsPerformanceStandby: false, - } - configSR, err = sdFactory(config.ServiceRegistration.Config, namedSDLogger, state) + configSR, err = beginServiceRegistration(c, config) if err != nil { - c.UI.Error(fmt.Sprintf("Error initializing service_registration of type %s: %s", config.ServiceRegistration.Type, err)) + c.UI.Output(err.Error()) return 1 } } @@ -1199,83 +1208,23 @@ func (c *ServerCommand) Run(args []string) int { info := make(map[string]string) info["log level"] = logLevelString infoKeys = append(infoKeys, "log level") + barrierSeal, barrierWrapper, unwrapSeal, seals, sealConfigError, err := setSeal(c, config, infoKeys, info) - var barrierSeal vault.Seal - var unwrapSeal vault.Seal - - var sealConfigError error - var wrapper wrapping.Wrapper - var barrierWrapper wrapping.Wrapper - if c.flagDevAutoSeal { - barrierSeal = vault.NewAutoSeal(vaultseal.NewTestSeal(nil)) - } else { - // Handle the case where no seal is provided - switch len(config.Seals) { - case 0: - config.Seals = append(config.Seals, &configutil.KMS{Type: wrapping.Shamir}) - case 1: - // If there's only one seal and it's disabled assume they want to - // migrate to a shamir seal and simply didn't provide it - if config.Seals[0].Disabled { - config.Seals = append(config.Seals, &configutil.KMS{Type: wrapping.Shamir}) - } - } - for _, configSeal := range config.Seals { - sealType := wrapping.Shamir - if !configSeal.Disabled && os.Getenv("VAULT_SEAL_TYPE") != "" { - sealType = os.Getenv("VAULT_SEAL_TYPE") - configSeal.Type = sealType - } else { - sealType = configSeal.Type - } - - var seal vault.Seal - sealLogger := c.logger.ResetNamed(fmt.Sprintf("seal.%s", sealType)) - c.allLoggers = append(c.allLoggers, sealLogger) - defaultSeal := vault.NewDefaultSeal(&vaultseal.Access{ - Wrapper: aeadwrapper.NewShamirWrapper(&wrapping.WrapperOptions{ - Logger: c.logger.Named("shamir"), - }), - }) - var sealInfoKeys []string - sealInfoMap := map[string]string{} - wrapper, sealConfigError = configutil.ConfigureWrapper(configSeal, &sealInfoKeys, &sealInfoMap, sealLogger) - if sealConfigError != nil { - if !errwrap.ContainsType(sealConfigError, new(logical.KeyNotFoundError)) { - c.UI.Error(fmt.Sprintf( - "Error parsing Seal configuration: %s", sealConfigError)) - return 1 - } - } - if wrapper == nil { - seal = defaultSeal - } else { - seal = vault.NewAutoSeal(&vaultseal.Access{ - Wrapper: wrapper, - }) - } - - infoPrefix := "" - if configSeal.Disabled { - unwrapSeal = seal - infoPrefix = "Old " - } else { - barrierSeal = seal - barrierWrapper = wrapper - } - for _, k := range sealInfoKeys { - infoKeys = append(infoKeys, infoPrefix+k) - info[infoPrefix+k] = sealInfoMap[k] - } + // Check error here + if err != nil { + c.UI.Error(err.Error()) + return 1 + } + if seals != nil { + for _, seal := range seals { // Ensure that the seal finalizer is called, even if using verify-only - defer func() { - err = seal.Finalize(context.Background()) + defer func(seal *vault.Seal) { + err = (*seal).Finalize(context.Background()) if err != nil { c.UI.Error(fmt.Sprintf("Error finalizing seals: %v", err)) } - }() - + }(&seal) } } @@ -1291,235 +1240,36 @@ func (c *ServerCommand) Run(args []string) int { return 1 } - coreConfig := &vault.CoreConfig{ - RawConfig: config, - Physical: backend, - RedirectAddr: config.Storage.RedirectAddr, - StorageType: config.Storage.Type, - HAPhysical: nil, - ServiceRegistration: configSR, - Seal: barrierSeal, - UnwrapSeal: unwrapSeal, - AuditBackends: c.AuditBackends, - CredentialBackends: c.CredentialBackends, - LogicalBackends: c.LogicalBackends, - Logger: c.logger, - DisableSentinelTrace: config.DisableSentinelTrace, - DisableCache: config.DisableCache, - DisableMlock: config.DisableMlock, - MaxLeaseTTL: config.MaxLeaseTTL, - DefaultLeaseTTL: config.DefaultLeaseTTL, - ClusterName: config.ClusterName, - CacheSize: config.CacheSize, - PluginDirectory: config.PluginDirectory, - EnableUI: config.EnableUI, - EnableRaw: config.EnableRawEndpoint, - DisableSealWrap: config.DisableSealWrap, - DisablePerformanceStandby: config.DisablePerformanceStandby, - DisableIndexing: config.DisableIndexing, - AllLoggers: c.allLoggers, - BuiltinRegistry: builtinplugins.Registry, - DisableKeyEncodingChecks: config.DisablePrintableCheck, - MetricsHelper: metricsHelper, - MetricSink: metricSink, - SecureRandomReader: secureRandomReader, - EnableResponseHeaderHostname: config.EnableResponseHeaderHostname, - EnableResponseHeaderRaftNodeID: config.EnableResponseHeaderRaftNodeID, - License: config.License, - LicensePath: config.LicensePath, - } - if c.flagDev { - coreConfig.EnableRaw = true - coreConfig.DevToken = c.flagDevRootTokenID - if c.flagDevLeasedKV { - coreConfig.LogicalBackends["kv"] = vault.LeasedPassthroughBackendFactory - } - if c.flagDevPluginDir != "" { - coreConfig.PluginDirectory = c.flagDevPluginDir - } - if c.flagDevLatency > 0 { - injectLatency := time.Duration(c.flagDevLatency) * time.Millisecond - if _, txnOK := backend.(physical.Transactional); txnOK { - coreConfig.Physical = physical.NewTransactionalLatencyInjector(backend, injectLatency, c.flagDevLatencyJitter, c.logger) - } else { - coreConfig.Physical = physical.NewLatencyInjector(backend, injectLatency, c.flagDevLatencyJitter, c.logger) - } - } - } - + coreConfig := createCoreConfig(c, config, backend, configSR, barrierSeal, unwrapSeal, metricsHelper, metricSink, secureRandomReader) if c.flagDevThreeNode { - return c.enableThreeNodeDevCluster(coreConfig, info, infoKeys, c.flagDevListenAddr, os.Getenv("VAULT_DEV_TEMP_DIR")) + return c.enableThreeNodeDevCluster(&coreConfig, info, infoKeys, c.flagDevListenAddr, os.Getenv("VAULT_DEV_TEMP_DIR")) } if c.flagDevFourCluster { - return enableFourClusterDev(c, coreConfig, info, infoKeys, c.flagDevListenAddr, os.Getenv("VAULT_DEV_TEMP_DIR")) + return enableFourClusterDev(c, &coreConfig, info, infoKeys, c.flagDevListenAddr, os.Getenv("VAULT_DEV_TEMP_DIR")) } - var disableClustering bool - // Initialize the separate HA storage backend, if it exists - var ok bool - if config.HAStorage != nil { - if config.Storage.Type == storageTypeRaft && config.HAStorage.Type == storageTypeRaft { - c.UI.Error("Raft cannot be set both as 'storage' and 'ha_storage'. Setting 'storage' to 'raft' will automatically set it up for HA operations as well") - return 1 - } - - if config.Storage.Type == storageTypeRaft { - c.UI.Error("HA storage cannot be declared when Raft is the storage type") - return 1 - } - - factory, exists := c.PhysicalBackends[config.HAStorage.Type] - if !exists { - c.UI.Error(fmt.Sprintf("Unknown HA storage type %s", config.HAStorage.Type)) - return 1 - - } - - namedHALogger := c.logger.Named("ha." + config.HAStorage.Type) - c.allLoggers = append(c.allLoggers, namedHALogger) - habackend, err := factory(config.HAStorage.Config, namedHALogger) - if err != nil { - c.UI.Error(fmt.Sprintf( - "Error initializing HA storage of type %s: %s", config.HAStorage.Type, err)) - return 1 - - } - - if coreConfig.HAPhysical, ok = habackend.(physical.HABackend); !ok { - c.UI.Error("Specified HA storage does not support HA") - return 1 - } - - if !coreConfig.HAPhysical.HAEnabled() { - c.UI.Error("Specified HA storage has HA support disabled; please consult documentation") - return 1 - } - - coreConfig.RedirectAddr = config.HAStorage.RedirectAddr - disableClustering = config.HAStorage.DisableClustering - - if config.HAStorage.Type == storageTypeRaft && disableClustering { - c.UI.Error("Disable clustering cannot be set to true when Raft is the HA storage type") - return 1 - } - - if !disableClustering { - coreConfig.ClusterAddr = config.HAStorage.ClusterAddr - } - } else { - if coreConfig.HAPhysical, ok = backend.(physical.HABackend); ok { - coreConfig.RedirectAddr = config.Storage.RedirectAddr - disableClustering = config.Storage.DisableClustering - - if (config.Storage.Type == storageTypeRaft) && disableClustering { - c.UI.Error("Disable clustering cannot be set to true when Raft is the storage type") - return 1 - } - - if !disableClustering { - coreConfig.ClusterAddr = config.Storage.ClusterAddr - } - } + disableClustering, err := initHaBackend(c, config, &coreConfig, backend) + if err != nil { + c.UI.Output(err.Error()) + return 1 } - if envRA := os.Getenv("VAULT_API_ADDR"); envRA != "" { - coreConfig.RedirectAddr = envRA - } else if envRA := os.Getenv("VAULT_REDIRECT_ADDR"); envRA != "" { - coreConfig.RedirectAddr = envRA - } else if envAA := os.Getenv("VAULT_ADVERTISE_ADDR"); envAA != "" { - coreConfig.RedirectAddr = envAA - } - - // Attempt to detect the redirect address, if possible - if coreConfig.RedirectAddr == "" { - c.logger.Warn("no `api_addr` value specified in config or in VAULT_API_ADDR; falling back to detection if possible, but this value should be manually set") - } - var detect physical.RedirectDetect - if coreConfig.HAPhysical != nil && coreConfig.HAPhysical.HAEnabled() { - detect, ok = coreConfig.HAPhysical.(physical.RedirectDetect) - } else { - detect, ok = coreConfig.Physical.(physical.RedirectDetect) - } - if ok && coreConfig.RedirectAddr == "" { - redirect, err := c.detectRedirect(detect, config) - if err != nil { - c.UI.Error(fmt.Sprintf("Error detecting api address: %s", err)) - } else if redirect == "" { - c.UI.Error("Failed to detect api address") - } else { - coreConfig.RedirectAddr = redirect - } - } - if coreConfig.RedirectAddr == "" && c.flagDev { - coreConfig.RedirectAddr = fmt.Sprintf("http://%s", config.Listeners[0].Address) + // Determine the redirect address from environment variables + err = determineRedirectAddr(c, &coreConfig, config) + if err != nil { + c.UI.Output(err.Error()) } // After the redirect bits are sorted out, if no cluster address was // explicitly given, derive one from the redirect addr - if disableClustering { - coreConfig.ClusterAddr = "" - } else if envCA := os.Getenv("VAULT_CLUSTER_ADDR"); envCA != "" { - coreConfig.ClusterAddr = envCA - } else { - var addrToUse string - switch { - case coreConfig.ClusterAddr == "" && coreConfig.RedirectAddr != "": - addrToUse = coreConfig.RedirectAddr - case c.flagDev: - addrToUse = fmt.Sprintf("http://%s", config.Listeners[0].Address) - default: - goto CLUSTER_SYNTHESIS_COMPLETE - } - u, err := url.ParseRequestURI(addrToUse) - if err != nil { - c.UI.Error(fmt.Sprintf( - "Error parsing synthesized cluster address %s: %v", addrToUse, err)) - return 1 - } - host, port, err := net.SplitHostPort(u.Host) - if err != nil { - // This sucks, as it's a const in the function but not exported in the package - if strings.Contains(err.Error(), "missing port in address") { - host = u.Host - port = "443" - } else { - c.UI.Error(fmt.Sprintf("Error parsing api address: %v", err)) - return 1 - } - } - nPort, err := strconv.Atoi(port) - if err != nil { - c.UI.Error(fmt.Sprintf( - "Error parsing synthesized address; failed to convert %q to a numeric: %v", port, err)) - return 1 - } - u.Host = net.JoinHostPort(host, strconv.Itoa(nPort+1)) - // Will always be TLS-secured - u.Scheme = "https" - coreConfig.ClusterAddr = u.String() - } - -CLUSTER_SYNTHESIS_COMPLETE: - - if coreConfig.RedirectAddr == coreConfig.ClusterAddr && len(coreConfig.RedirectAddr) != 0 { - c.UI.Error(fmt.Sprintf( - "Address %q used for both API and cluster addresses", coreConfig.RedirectAddr)) + err = findClusterAddress(c, &coreConfig, config, disableClustering) + if err != nil { + c.UI.Output(err.Error()) return 1 } - if coreConfig.ClusterAddr != "" { - // Force https as we'll always be TLS-secured - u, err := url.ParseRequestURI(coreConfig.ClusterAddr) - if err != nil { - c.UI.Error(fmt.Sprintf("Error parsing cluster address %s: %v", coreConfig.ClusterAddr, err)) - return 11 - } - u.Scheme = "https" - coreConfig.ClusterAddr = u.String() - } - // Override the UI enabling config by the environment variable if enableUI := os.Getenv("VAULT_UI"); enableUI != "" { var err error @@ -1538,15 +1288,20 @@ CLUSTER_SYNTHESIS_COMPLETE: } // Apply any enterprise configuration onto the coreConfig. - adjustCoreConfigForEnt(config, coreConfig) + adjustCoreConfigForEnt(config, &coreConfig) // Initialize the core - core, newCoreError := vault.NewCore(coreConfig) + core, newCoreError := vault.NewCore(&coreConfig) if newCoreError != nil { if vault.IsFatalError(newCoreError) { c.UI.Error(fmt.Sprintf("Error initializing core: %s", newCoreError)) return 1 } + c.UI.Warn(wrapAtLength( + "WARNING! A non-fatal error occurred during initialization. Please " + + "check the logs for more information.")) + c.UI.Warn("") + } // Copy the reload funcs pointers back @@ -1650,27 +1405,7 @@ CLUSTER_SYNTHESIS_COMPLETE: // uninitialized. Once one server initializes the storage backend, this // goroutine will pick up the unseal keys and unseal this instance. if !core.IsInSealMigrationMode() { - go func() { - for { - err := core.UnsealWithStoredKeys(context.Background()) - if err == nil { - return - } - - if vault.IsFatalError(err) { - c.logger.Error("error unsealing core", "error", err) - return - } else { - c.logger.Warn("failed to unseal core", "error", err) - } - - select { - case <-c.ShutdownCh: - return - case <-time.After(5 * time.Second): - } - } - }() + go runUnseal(c, core, context.Background()) } // When the underlying storage is raft, kick off retry join if it was specified @@ -1689,176 +1424,24 @@ CLUSTER_SYNTHESIS_COMPLETE: c.WaitGroup = &sync.WaitGroup{} // If service discovery is available, run service discovery - if sd := coreConfig.GetServiceRegistration(); sd != nil { - if err := configSR.Run(c.ShutdownCh, c.WaitGroup, coreConfig.RedirectAddr); err != nil { - c.UI.Error(fmt.Sprintf("Error running service_registration of type %s: %s", config.ServiceRegistration.Type, err)) - return 1 - } + err = runListeners(c, &coreConfig, config, configSR) + if err != nil { + c.UI.Error(err.Error()) + return 1 } // If we're in Dev mode, then initialize the core - if c.flagDev && !c.flagDevSkipInit { - - init, err := c.enableDev(core, coreConfig) - if err != nil { - c.UI.Error(fmt.Sprintf("Error initializing Dev mode: %s", err)) - return 1 - } - - var plugins, pluginsNotLoaded []string - if c.flagDevPluginDir != "" && c.flagDevPluginInit { - - f, err := os.Open(c.flagDevPluginDir) - if err != nil { - c.UI.Error(fmt.Sprintf("Error reading plugin dir: %s", err)) - return 1 - } - - list, err := f.Readdirnames(0) - f.Close() - if err != nil { - c.UI.Error(fmt.Sprintf("Error listing plugins: %s", err)) - return 1 - } - - for _, name := range list { - path := filepath.Join(f.Name(), name) - if err := c.addPlugin(path, init.RootToken, core); err != nil { - if !errwrap.Contains(err, vault.ErrPluginBadType.Error()) { - c.UI.Error(fmt.Sprintf("Error enabling plugin %s: %s", name, err)) - return 1 - } - pluginsNotLoaded = append(pluginsNotLoaded, name) - continue - } - plugins = append(plugins, name) - } - - sort.Strings(plugins) - } - - var qw *quiescenceSink - var qwo sync.Once - qw = &quiescenceSink{ - t: time.AfterFunc(100*time.Millisecond, func() { - qwo.Do(func() { - c.logger.DeregisterSink(qw) - - // Print the big dev mode warning! - c.UI.Warn(wrapAtLength( - "WARNING! dev mode is enabled! In this mode, Vault runs entirely " + - "in-memory and starts unsealed with a single unseal key. The root " + - "token is already authenticated to the CLI, so you can immediately " + - "begin using Vault.")) - c.UI.Warn("") - c.UI.Warn("You may need to set the following environment variable:") - c.UI.Warn("") - - endpointURL := "http://" + config.Listeners[0].Address - if runtime.GOOS == "windows" { - c.UI.Warn("PowerShell:") - c.UI.Warn(fmt.Sprintf(" $env:VAULT_ADDR=\"%s\"", endpointURL)) - c.UI.Warn("cmd.exe:") - c.UI.Warn(fmt.Sprintf(" set VAULT_ADDR=%s", endpointURL)) - } else { - c.UI.Warn(fmt.Sprintf(" $ export VAULT_ADDR='%s'", endpointURL)) - } - - // Unseal key is not returned if stored shares is supported - if len(init.SecretShares) > 0 { - c.UI.Warn("") - c.UI.Warn(wrapAtLength( - "The unseal key and root token are displayed below in case you want " + - "to seal/unseal the Vault or re-authenticate.")) - c.UI.Warn("") - c.UI.Warn(fmt.Sprintf("Unseal Key: %s", base64.StdEncoding.EncodeToString(init.SecretShares[0]))) - } - - if len(init.RecoveryShares) > 0 { - c.UI.Warn("") - c.UI.Warn(wrapAtLength( - "The recovery key and root token are displayed below in case you want " + - "to seal/unseal the Vault or re-authenticate.")) - c.UI.Warn("") - c.UI.Warn(fmt.Sprintf("Recovery Key: %s", base64.StdEncoding.EncodeToString(init.RecoveryShares[0]))) - } - - c.UI.Warn(fmt.Sprintf("Root Token: %s", init.RootToken)) - - if len(plugins) > 0 { - c.UI.Warn("") - c.UI.Warn(wrapAtLength( - "The following dev plugins are registered in the catalog:")) - for _, p := range plugins { - c.UI.Warn(fmt.Sprintf(" - %s", p)) - } - } - - if len(pluginsNotLoaded) > 0 { - c.UI.Warn("") - c.UI.Warn(wrapAtLength( - "The following dev plugins FAILED to be registered in the catalog due to unknown type:")) - for _, p := range pluginsNotLoaded { - c.UI.Warn(fmt.Sprintf(" - %s", p)) - } - } - - c.UI.Warn("") - c.UI.Warn(wrapAtLength( - "Development mode should NOT be used in production installations!")) - c.UI.Warn("") - }) - }), - } - c.logger.RegisterSink(qw) + err = initDevCore(c, &coreConfig, config, core) + if err != nil { + c.UI.Error(err.Error()) + return 1 } // Initialize the HTTP servers - for _, ln := range lns { - if ln.Config == nil { - c.UI.Error("Found nil listener config after parsing") - return 1 - } - handler := vaulthttp.Handler(&vault.HandlerProperties{ - Core: core, - ListenerConfig: ln.Config, - DisablePrintableCheck: config.DisablePrintableCheck, - RecoveryMode: c.flagRecovery, - }) - - if len(ln.Config.XForwardedForAuthorizedAddrs) > 0 { - handler = vaulthttp.WrapForwardedForHandler(handler, ln.Config) - } - - // server defaults - server := &http.Server{ - Handler: handler, - ReadHeaderTimeout: 10 * time.Second, - ReadTimeout: 30 * time.Second, - IdleTimeout: 5 * time.Minute, - ErrorLog: c.logger.StandardLogger(nil), - } - - // override server defaults with config values for read/write/idle timeouts if configured - if ln.Config.HTTPReadHeaderTimeout > 0 { - server.ReadHeaderTimeout = ln.Config.HTTPReadHeaderTimeout - } - if ln.Config.HTTPReadTimeout > 0 { - server.ReadTimeout = ln.Config.HTTPReadTimeout - } - if ln.Config.HTTPWriteTimeout > 0 { - server.WriteTimeout = ln.Config.HTTPWriteTimeout - } - if ln.Config.HTTPIdleTimeout > 0 { - server.IdleTimeout = ln.Config.HTTPIdleTimeout - } - - // server config tests can exit now - if c.flagTestServerConfig { - continue - } - - go server.Serve(ln.Listener) + err = startHttpServers(c, core, config, lns) + if err != nil { + c.UI.Error(err.Error()) + return 1 } if c.flagTestServerConfig { @@ -1877,13 +1460,6 @@ CLUSTER_SYNTHESIS_COMPLETE: } } - if newCoreError != nil { - c.UI.Warn(wrapAtLength( - "WARNING! A non-fatal error occurred during initialization. Please " + - "check the logs for more information.")) - c.UI.Warn("") - } - // Output the header that the server has started if !c.flagCombineLogs { c.UI.Output("==> Vault server started! Log data will stream in below:\n") @@ -2591,6 +2167,493 @@ func CheckStorageMigration(b physical.Backend) (*StorageMigrationStatus, error) return &status, nil } +// setSeal return barrierSeal, barrierWrapper, unwrapSeal, and all the created seals from the configs so we can close them in Run +// The two errors are the sealConfigError and the regular error +func setSeal(c *ServerCommand, config *server.Config, infoKeys []string, info map[string]string) (vault.Seal, wrapping.Wrapper, vault.Seal, []vault.Seal, error, error) { + var barrierSeal vault.Seal + var unwrapSeal vault.Seal + + var sealConfigError error + var wrapper wrapping.Wrapper + var barrierWrapper wrapping.Wrapper + if c.flagDevAutoSeal { + barrierSeal = vault.NewAutoSeal(vaultseal.NewTestSeal(nil)) + return barrierSeal, nil, nil, nil, nil, nil + } + + // Handle the case where no seal is provided + switch len(config.Seals) { + case 0: + config.Seals = append(config.Seals, &configutil.KMS{Type: wrapping.Shamir}) + case 1: + // If there's only one seal and it's disabled assume they want to + // migrate to a shamir seal and simply didn't provide it + if config.Seals[0].Disabled { + config.Seals = append(config.Seals, &configutil.KMS{Type: wrapping.Shamir}) + } + } + var createdSeals []vault.Seal = make([]vault.Seal, len(config.Seals)) + for _, configSeal := range config.Seals { + sealType := wrapping.Shamir + if !configSeal.Disabled && os.Getenv("VAULT_SEAL_TYPE") != "" { + sealType = os.Getenv("VAULT_SEAL_TYPE") + configSeal.Type = sealType + } else { + sealType = configSeal.Type + } + + var seal vault.Seal + sealLogger := c.logger.ResetNamed(fmt.Sprintf("seal.%s", sealType)) + c.allLoggers = append(c.allLoggers, sealLogger) + defaultSeal := vault.NewDefaultSeal(&vaultseal.Access{ + Wrapper: aeadwrapper.NewShamirWrapper(&wrapping.WrapperOptions{ + Logger: c.logger.Named("shamir"), + }), + }) + var sealInfoKeys []string + sealInfoMap := map[string]string{} + wrapper, sealConfigError = configutil.ConfigureWrapper(configSeal, &sealInfoKeys, &sealInfoMap, sealLogger) + if sealConfigError != nil { + if !errwrap.ContainsType(sealConfigError, new(logical.KeyNotFoundError)) { + return barrierSeal, barrierWrapper, unwrapSeal, createdSeals, sealConfigError, fmt.Errorf( + "Error parsing Seal configuration: %s", sealConfigError) + } + } + if wrapper == nil { + seal = defaultSeal + } else { + seal = vault.NewAutoSeal(&vaultseal.Access{ + Wrapper: wrapper, + }) + } + infoPrefix := "" + if configSeal.Disabled { + unwrapSeal = seal + infoPrefix = "Old " + } else { + barrierSeal = seal + barrierWrapper = wrapper + } + for _, k := range sealInfoKeys { + infoKeys = append(infoKeys, infoPrefix+k) + info[infoPrefix+k] = sealInfoMap[k] + } + createdSeals = append(createdSeals, seal) + } + return barrierSeal, barrierWrapper, unwrapSeal, createdSeals, sealConfigError, nil +} + +func initHaBackend(c *ServerCommand, config *server.Config, coreConfig *vault.CoreConfig, backend physical.Backend) (bool, error) { + // Initialize the separate HA storage backend, if it exists + var ok bool + if config.HAStorage != nil { + if config.Storage.Type == storageTypeRaft && config.HAStorage.Type == storageTypeRaft { + return false, fmt.Errorf("Raft cannot be set both as 'storage' and 'ha_storage'. Setting 'storage' to 'raft' will automatically set it up for HA operations as well") + } + + if config.Storage.Type == storageTypeRaft { + return false, fmt.Errorf("HA storage cannot be declared when Raft is the storage type") + } + + factory, exists := c.PhysicalBackends[config.HAStorage.Type] + if !exists { + return false, fmt.Errorf("Unknown HA storage type %s", config.HAStorage.Type) + + } + + namedHALogger := c.logger.Named("ha." + config.HAStorage.Type) + c.allLoggers = append(c.allLoggers, namedHALogger) + habackend, err := factory(config.HAStorage.Config, namedHALogger) + if err != nil { + return false, fmt.Errorf("Error initializing HA storage of type %s: %s", config.HAStorage.Type, err) + + } + + if coreConfig.HAPhysical, ok = habackend.(physical.HABackend); !ok { + return false, fmt.Errorf("Specified HA storage does not support HA") + } + + if !coreConfig.HAPhysical.HAEnabled() { + return false, fmt.Errorf("Specified HA storage has HA support disabled; please consult documentation") + } + + coreConfig.RedirectAddr = config.HAStorage.RedirectAddr + disableClustering := config.HAStorage.DisableClustering + + if config.HAStorage.Type == storageTypeRaft && disableClustering { + return disableClustering, fmt.Errorf("Disable clustering cannot be set to true when Raft is the HA storage type") + } + + if !disableClustering { + coreConfig.ClusterAddr = config.HAStorage.ClusterAddr + } + } else { + if coreConfig.HAPhysical, ok = backend.(physical.HABackend); ok { + coreConfig.RedirectAddr = config.Storage.RedirectAddr + disableClustering := config.Storage.DisableClustering + + if (config.Storage.Type == storageTypeRaft) && disableClustering { + return disableClustering, fmt.Errorf("Disable clustering cannot be set to true when Raft is the storage type") + } + + if !disableClustering { + coreConfig.ClusterAddr = config.Storage.ClusterAddr + } + } + } + return config.DisableClustering, nil +} + +func determineRedirectAddr(c *ServerCommand, coreConfig *vault.CoreConfig, config *server.Config) error { + var retErr error + if envRA := os.Getenv("VAULT_API_ADDR"); envRA != "" { + coreConfig.RedirectAddr = envRA + } else if envRA := os.Getenv("VAULT_REDIRECT_ADDR"); envRA != "" { + coreConfig.RedirectAddr = envRA + } else if envAA := os.Getenv("VAULT_ADVERTISE_ADDR"); envAA != "" { + coreConfig.RedirectAddr = envAA + } + + // Attempt to detect the redirect address, if possible + if coreConfig.RedirectAddr == "" { + c.logger.Warn("no `api_addr` value specified in config or in VAULT_API_ADDR; falling back to detection if possible, but this value should be manually set") + } + + var ok bool + var detect physical.RedirectDetect + if coreConfig.HAPhysical != nil && coreConfig.HAPhysical.HAEnabled() { + detect, ok = coreConfig.HAPhysical.(physical.RedirectDetect) + } else { + detect, ok = coreConfig.Physical.(physical.RedirectDetect) + } + if ok && coreConfig.RedirectAddr == "" { + redirect, err := c.detectRedirect(detect, config) + // the following errors did not cause Run to return, so I'm not returning these + // as errors. + if err != nil { + retErr = fmt.Errorf("Error detecting api address: %s", err) + } else if redirect == "" { + retErr = fmt.Errorf("Failed to detect api address") + } else { + coreConfig.RedirectAddr = redirect + } + } + if coreConfig.RedirectAddr == "" && c.flagDev { + coreConfig.RedirectAddr = fmt.Sprintf("http://%s", config.Listeners[0].Address) + } + return retErr +} + +func findClusterAddress(c *ServerCommand, coreConfig *vault.CoreConfig, config *server.Config, disableClustering bool) error { + if disableClustering { + coreConfig.ClusterAddr = "" + } else if envCA := os.Getenv("VAULT_CLUSTER_ADDR"); envCA != "" { + coreConfig.ClusterAddr = envCA + } else { + var addrToUse string + switch { + case coreConfig.ClusterAddr == "" && coreConfig.RedirectAddr != "": + addrToUse = coreConfig.RedirectAddr + case c.flagDev: + addrToUse = fmt.Sprintf("http://%s", config.Listeners[0].Address) + default: + goto CLUSTER_SYNTHESIS_COMPLETE + } + u, err := url.ParseRequestURI(addrToUse) + if err != nil { + return fmt.Errorf("Error parsing synthesized cluster address %s: %v", addrToUse, err) + } + host, port, err := net.SplitHostPort(u.Host) + if err != nil { + // This sucks, as it's a const in the function but not exported in the package + if strings.Contains(err.Error(), "missing port in address") { + host = u.Host + port = "443" + } else { + return fmt.Errorf("Error parsing api address: %v", err) + } + } + nPort, err := strconv.Atoi(port) + if err != nil { + return fmt.Errorf("Error parsing synthesized address; failed to convert %q to a numeric: %v", port, err) + } + u.Host = net.JoinHostPort(host, strconv.Itoa(nPort+1)) + // Will always be TLS-secured + u.Scheme = "https" + coreConfig.ClusterAddr = u.String() + } + +CLUSTER_SYNTHESIS_COMPLETE: + + if coreConfig.RedirectAddr == coreConfig.ClusterAddr && len(coreConfig.RedirectAddr) != 0 { + return fmt.Errorf("Address %q used for both API and cluster addresses", coreConfig.RedirectAddr) + } + + if coreConfig.ClusterAddr != "" { + // Force https as we'll always be TLS-secured + u, err := url.ParseRequestURI(coreConfig.ClusterAddr) + if err != nil { + return fmt.Errorf("Error parsing cluster address %s: %v", coreConfig.ClusterAddr, err) + } + u.Scheme = "https" + coreConfig.ClusterAddr = u.String() + } + return nil +} + +func runUnseal(c *ServerCommand, core *vault.Core, ctx context.Context) { + for { + err := core.UnsealWithStoredKeys(ctx) + if err == nil { + return + } + + if vault.IsFatalError(err) { + c.logger.Error("error unsealing core", "error", err) + return + } + c.logger.Warn("failed to unseal core", "error", err) + + select { + case <-c.ShutdownCh: + return + case <-time.After(5 * time.Second): + } + } +} + +func createCoreConfig(c *ServerCommand, config *server.Config, backend physical.Backend, configSR sr.ServiceRegistration, barrierSeal, unwrapSeal vault.Seal, + metricsHelper *metricsutil.MetricsHelper, metricSink *metricsutil.ClusterMetricSink, secureRandomReader io.Reader) vault.CoreConfig { + coreConfig := &vault.CoreConfig{ + RawConfig: config, + Physical: backend, + RedirectAddr: config.Storage.RedirectAddr, + StorageType: config.Storage.Type, + HAPhysical: nil, + ServiceRegistration: configSR, + Seal: barrierSeal, + UnwrapSeal: unwrapSeal, + AuditBackends: c.AuditBackends, + CredentialBackends: c.CredentialBackends, + LogicalBackends: c.LogicalBackends, + Logger: c.logger, + DisableSentinelTrace: config.DisableSentinelTrace, + DisableCache: config.DisableCache, + DisableMlock: config.DisableMlock, + MaxLeaseTTL: config.MaxLeaseTTL, + DefaultLeaseTTL: config.DefaultLeaseTTL, + ClusterName: config.ClusterName, + CacheSize: config.CacheSize, + PluginDirectory: config.PluginDirectory, + EnableUI: config.EnableUI, + EnableRaw: config.EnableRawEndpoint, + DisableSealWrap: config.DisableSealWrap, + DisablePerformanceStandby: config.DisablePerformanceStandby, + DisableIndexing: config.DisableIndexing, + AllLoggers: c.allLoggers, + BuiltinRegistry: builtinplugins.Registry, + DisableKeyEncodingChecks: config.DisablePrintableCheck, + MetricsHelper: metricsHelper, + MetricSink: metricSink, + SecureRandomReader: secureRandomReader, + EnableResponseHeaderHostname: config.EnableResponseHeaderHostname, + EnableResponseHeaderRaftNodeID: config.EnableResponseHeaderRaftNodeID, + } + if c.flagDev { + coreConfig.EnableRaw = true + coreConfig.DevToken = c.flagDevRootTokenID + if c.flagDevLeasedKV { + coreConfig.LogicalBackends["kv"] = vault.LeasedPassthroughBackendFactory + } + if c.flagDevPluginDir != "" { + coreConfig.PluginDirectory = c.flagDevPluginDir + } + if c.flagDevLatency > 0 { + injectLatency := time.Duration(c.flagDevLatency) * time.Millisecond + if _, txnOK := backend.(physical.Transactional); txnOK { + coreConfig.Physical = physical.NewTransactionalLatencyInjector(backend, injectLatency, c.flagDevLatencyJitter, c.logger) + } else { + coreConfig.Physical = physical.NewLatencyInjector(backend, injectLatency, c.flagDevLatencyJitter, c.logger) + } + } + } + return *coreConfig +} + +func runListeners(c *ServerCommand, coreConfig *vault.CoreConfig, config *server.Config, configSR sr.ServiceRegistration) error { + if sd := coreConfig.GetServiceRegistration(); sd != nil { + if err := configSR.Run(c.ShutdownCh, c.WaitGroup, coreConfig.RedirectAddr); err != nil { + return fmt.Errorf("Error running service_registration of type %s: %s", config.ServiceRegistration.Type, err) + } + } + return nil +} + +func initDevCore(c *ServerCommand, coreConfig *vault.CoreConfig, config *server.Config, core *vault.Core) error { + if c.flagDev && !c.flagDevSkipInit { + + init, err := c.enableDev(core, coreConfig) + if err != nil { + return fmt.Errorf("Error initializing Dev mode: %s", err) + } + + var plugins, pluginsNotLoaded []string + if c.flagDevPluginDir != "" && c.flagDevPluginInit { + + f, err := os.Open(c.flagDevPluginDir) + if err != nil { + return fmt.Errorf("Error reading plugin dir: %s", err) + } + + list, err := f.Readdirnames(0) + f.Close() + if err != nil { + return fmt.Errorf("Error listing plugins: %s", err) + } + + for _, name := range list { + path := filepath.Join(f.Name(), name) + if err := c.addPlugin(path, init.RootToken, core); err != nil { + if !errwrap.Contains(err, vault.ErrPluginBadType.Error()) { + return fmt.Errorf("Error enabling plugin %s: %s", name, err) + } + pluginsNotLoaded = append(pluginsNotLoaded, name) + continue + } + plugins = append(plugins, name) + } + + sort.Strings(plugins) + } + + var qw *quiescenceSink + var qwo sync.Once + qw = &quiescenceSink{ + t: time.AfterFunc(100*time.Millisecond, func() { + qwo.Do(func() { + c.logger.DeregisterSink(qw) + + // Print the big dev mode warning! + c.UI.Warn(wrapAtLength( + "WARNING! dev mode is enabled! In this mode, Vault runs entirely " + + "in-memory and starts unsealed with a single unseal key. The root " + + "token is already authenticated to the CLI, so you can immediately " + + "begin using Vault.")) + c.UI.Warn("") + c.UI.Warn("You may need to set the following environment variable:") + c.UI.Warn("") + + endpointURL := "http://" + config.Listeners[0].Address + if runtime.GOOS == "windows" { + c.UI.Warn("PowerShell:") + c.UI.Warn(fmt.Sprintf(" $env:VAULT_ADDR=\"%s\"", endpointURL)) + c.UI.Warn("cmd.exe:") + c.UI.Warn(fmt.Sprintf(" set VAULT_ADDR=%s", endpointURL)) + } else { + c.UI.Warn(fmt.Sprintf(" $ export VAULT_ADDR='%s'", endpointURL)) + } + + // Unseal key is not returned if stored shares is supported + if len(init.SecretShares) > 0 { + c.UI.Warn("") + c.UI.Warn(wrapAtLength( + "The unseal key and root token are displayed below in case you want " + + "to seal/unseal the Vault or re-authenticate.")) + c.UI.Warn("") + c.UI.Warn(fmt.Sprintf("Unseal Key: %s", base64.StdEncoding.EncodeToString(init.SecretShares[0]))) + } + + if len(init.RecoveryShares) > 0 { + c.UI.Warn("") + c.UI.Warn(wrapAtLength( + "The recovery key and root token are displayed below in case you want " + + "to seal/unseal the Vault or re-authenticate.")) + c.UI.Warn("") + c.UI.Warn(fmt.Sprintf("Recovery Key: %s", base64.StdEncoding.EncodeToString(init.RecoveryShares[0]))) + } + + c.UI.Warn(fmt.Sprintf("Root Token: %s", init.RootToken)) + + if len(plugins) > 0 { + c.UI.Warn("") + c.UI.Warn(wrapAtLength( + "The following dev plugins are registered in the catalog:")) + for _, p := range plugins { + c.UI.Warn(fmt.Sprintf(" - %s", p)) + } + } + + if len(pluginsNotLoaded) > 0 { + c.UI.Warn("") + c.UI.Warn(wrapAtLength( + "The following dev plugins FAILED to be registered in the catalog due to unknown type:")) + for _, p := range pluginsNotLoaded { + c.UI.Warn(fmt.Sprintf(" - %s", p)) + } + } + + c.UI.Warn("") + c.UI.Warn(wrapAtLength( + "Development mode should NOT be used in production installations!")) + c.UI.Warn("") + }) + }), + } + c.logger.RegisterSink(qw) + } + return nil +} + +// Initialize the HTTP servers +func startHttpServers(c *ServerCommand, core *vault.Core, config *server.Config, lns []listenerutil.Listener) error { + for _, ln := range lns { + if ln.Config == nil { + return fmt.Errorf("Found nil listener config after parsing") + } + handler := vaulthttp.Handler(&vault.HandlerProperties{ + Core: core, + ListenerConfig: ln.Config, + DisablePrintableCheck: config.DisablePrintableCheck, + RecoveryMode: c.flagRecovery, + }) + + if len(ln.Config.XForwardedForAuthorizedAddrs) > 0 { + handler = vaulthttp.WrapForwardedForHandler(handler, ln.Config) + } + + // server defaults + server := &http.Server{ + Handler: handler, + ReadHeaderTimeout: 10 * time.Second, + ReadTimeout: 30 * time.Second, + IdleTimeout: 5 * time.Minute, + ErrorLog: c.logger.StandardLogger(nil), + } + + // override server defaults with config values for read/write/idle timeouts if configured + if ln.Config.HTTPReadHeaderTimeout > 0 { + server.ReadHeaderTimeout = ln.Config.HTTPReadHeaderTimeout + } + if ln.Config.HTTPReadTimeout > 0 { + server.ReadTimeout = ln.Config.HTTPReadTimeout + } + if ln.Config.HTTPWriteTimeout > 0 { + server.WriteTimeout = ln.Config.HTTPWriteTimeout + } + if ln.Config.HTTPIdleTimeout > 0 { + server.IdleTimeout = ln.Config.HTTPIdleTimeout + } + + // server config tests can exit now + if c.flagTestServerConfig { + continue + } + + go server.Serve(ln.Listener) + } + return nil +} + func SetStorageMigration(b physical.Backend, active bool) error { if !active { return b.Delete(context.Background(), storageMigrationLock) diff --git a/command/server/test-fixtures/config_bad_https_storage.hcl b/command/server/test-fixtures/config_bad_https_storage.hcl index 27911d755d..f8b5d7734b 100644 --- a/command/server/test-fixtures/config_bad_https_storage.hcl +++ b/command/server/test-fixtures/config_bad_https_storage.hcl @@ -20,7 +20,7 @@ backend "consul" { ha_backend "consul" { address = "127.0.0.1:8500" bar = "baz" - advertise_addr = "snafu" + advertise_addr = "https://127.0.0.1:8500" disable_clustering = "true" } diff --git a/command/server/test-fixtures/config_diagnose_hastorage_bad_https.hcl b/command/server/test-fixtures/config_diagnose_hastorage_bad_https.hcl index c5187f5615..cc2671942c 100644 --- a/command/server/test-fixtures/config_diagnose_hastorage_bad_https.hcl +++ b/command/server/test-fixtures/config_diagnose_hastorage_bad_https.hcl @@ -11,12 +11,13 @@ listener "tcp" { backend "consul" { foo = "bar" advertise_addr = "foo" - address = "127.0.0.1:1028" + address = "http://remoteconsulserverIP:1028" } ha_backend "consul" { bar = "baz" + address = "https://remoteconsulserverIP:1028" advertise_addr = "snafu" disable_clustering = "true" scheme = "https" diff --git a/command/server/test-fixtures/config_diagnose_ok.hcl b/command/server/test-fixtures/config_diagnose_ok.hcl index 9c1e76d975..9287bc2540 100644 --- a/command/server/test-fixtures/config_diagnose_ok.hcl +++ b/command/server/test-fixtures/config_diagnose_ok.hcl @@ -17,7 +17,7 @@ backend "consul" { ha_backend "consul" { address = "127.0.0.1:8500" bar = "baz" - advertise_addr = "snafu" + advertise_addr = "https://127.0.0.1:8500" disable_clustering = "true" } diff --git a/command/server/test-fixtures/diagnose_ok_storage_direct_access.hcl b/command/server/test-fixtures/diagnose_ok_storage_direct_access.hcl index 4a71fb6065..3b6a9abf29 100644 --- a/command/server/test-fixtures/diagnose_ok_storage_direct_access.hcl +++ b/command/server/test-fixtures/diagnose_ok_storage_direct_access.hcl @@ -17,7 +17,7 @@ backend "consul" { ha_backend "consul" { address = "127.0.0.1:1024" bar = "baz" - advertise_addr = "snafu" + advertise_addr = "https://127.0.0.1:8500" disable_clustering = "true" } diff --git a/command/server/test-fixtures/tls_config_ok.hcl b/command/server/test-fixtures/tls_config_ok.hcl index 44be00930e..0dee4b4836 100644 --- a/command/server/test-fixtures/tls_config_ok.hcl +++ b/command/server/test-fixtures/tls_config_ok.hcl @@ -17,7 +17,7 @@ backend "consul" { ha_backend "consul" { bar = "baz" - advertise_addr = "snafu" + advertise_addr = "http://blah:8500" disable_clustering = "true" address = "127.0.0.1:8500" } diff --git a/vault/diagnose/helpers.go b/vault/diagnose/helpers.go index 4779cfd0dd..fef4cc8f36 100644 --- a/vault/diagnose/helpers.go +++ b/vault/diagnose/helpers.go @@ -202,7 +202,7 @@ func WithTimeout(d time.Duration, f testFunction) testFunction { rch := make(chan error) t := time.NewTimer(d) defer t.Stop() - go f(ctx) + go func() { rch <- f(ctx) }() select { case <-t.C: return fmt.Errorf("timed out after %s", d.String()) diff --git a/vault/diagnose/mock_storage_backend.go b/vault/diagnose/mock_storage_backend.go index dbbc9fb693..dff43ea868 100644 --- a/vault/diagnose/mock_storage_backend.go +++ b/vault/diagnose/mock_storage_backend.go @@ -24,7 +24,7 @@ const ( deleteOp string = "delete" ) -var goodEntry physical.Entry = physical.Entry{Key: secretKey, Value: []byte(secretVal)} +var goodEntry physical.Entry = physical.Entry{Key: "diagnose", Value: []byte(secretVal)} var badEntry physical.Entry = physical.Entry{} type mockStorageBackend struct { @@ -34,7 +34,7 @@ type mockStorageBackend struct { func (m mockStorageBackend) storageLogicGeneralInternal(op string) error { if (m.callType == timeoutCallRead && op == readOp) || (m.callType == timeoutCallWrite && op == writeOp) || (m.callType == timeoutCallDelete && op == deleteOp) { - time.Sleep(25 * time.Second) + time.Sleep(2 * time.Second) } else if m.callType == errCallWrite && op == writeOp { return fmt.Errorf(storageErrStringWrite) } else if m.callType == errCallDelete && op == deleteOp { @@ -53,7 +53,10 @@ func (m mockStorageBackend) Put(ctx context.Context, entry *physical.Entry) erro // Get is used to fetch an entry func (m mockStorageBackend) Get(ctx context.Context, key string) (*physical.Entry, error) { - if m.callType == errCallRead || m.callType == timeoutCallRead { + if m.callType == timeoutCallRead { + return &goodEntry, m.storageLogicGeneralInternal(readOp) + } + if m.callType == errCallRead { return nil, m.storageLogicGeneralInternal(readOp) } if m.callType == badReadCall { @@ -73,3 +76,16 @@ func (m mockStorageBackend) Delete(ctx context.Context, key string) error { func (m mockStorageBackend) List(ctx context.Context, prefix string) ([]string, error) { return nil, fmt.Errorf("method not implemented") } + +func callTypeToOp(ctype string) string { + if ctype == timeoutCallRead || ctype == errCallRead || ctype == badReadCall { + return readOp + } + if ctype == errCallWrite || ctype == storageErrStringWrite || ctype == timeoutCallWrite { + return writeOp + } + if ctype == errCallDelete || ctype == timeoutCallDelete || ctype == storageErrStringDelete { + return deleteOp + } + return "" +} diff --git a/vault/diagnose/storage_checks.go b/vault/diagnose/storage_checks.go index 76f6fa4fe2..34494ede03 100644 --- a/vault/diagnose/storage_checks.go +++ b/vault/diagnose/storage_checks.go @@ -11,71 +11,59 @@ import ( const ( success string = "success" - secretKey string = "diagnose" secretVal string = "diagnoseSecret" - timeOutErr string = "storage call timed out after 20 seconds: " - DirAccessErr string = "consul storage does not connect to local agent, but directly to server" - AddrDNExistErr string = "config address does not exist: 127.0.0.1:8500 will be used" - wrongRWValsPrefix string = "Storage get and put gave wrong values: " + LatencyWarning string = "latency above 100 ms: " + DirAccessErr string = "consul storage does not connect to local agent, but directly to server" + AddrDNExistErr string = "config address does not exist: 127.0.0.1:8500 will be used" + wrongRWValsPrefix string = "Storage get and put gave wrong values: " + latencyThreshold time.Duration = time.Millisecond * 100 ) -// StorageEndToEndLatencyCheck calls Write, Read, and Delete on a secret in the root -// directory of the backend. -// Note: Just checking read, write, and delete for root. It's a very basic check, -// but I don't think we can necessarily do any more than that. We could check list, -// but I don't think List is ever going to break in isolation. -func StorageEndToEndLatencyCheck(ctx context.Context, b physical.Backend) error { - - c2 := make(chan error) - go func() { - err := b.Put(context.Background(), &physical.Entry{Key: secretKey, Value: []byte(secretVal)}) - c2 <- err - }() - select { - case errOut := <-c2: - if errOut != nil { - return errOut - } - case <-time.After(20 * time.Second): - return fmt.Errorf(timeOutErr + "operation: Put") +func EndToEndLatencyCheckWrite(ctx context.Context, uuid string, b physical.Backend) (time.Duration, error) { + start := time.Now() + err := b.Put(context.Background(), &physical.Entry{Key: uuid, Value: []byte(secretVal)}) + duration := time.Since(start) + if err != nil { + return time.Duration(0), err } - - c3 := make(chan *physical.Entry) - c4 := make(chan error) - go func() { - val, err := b.Get(context.Background(), "diagnose") - if err != nil { - c4 <- err - } else { - c3 <- val - } - }() - select { - case err := <-c4: - return err - case val := <-c3: - if val.Key != "diagnose" && string(val.Value) != "diagnose" { - return fmt.Errorf(wrongRWValsPrefix+"expecting diagnose, but got %s, %s", val.Key, val.Value) - } - case <-time.After(20 * time.Second): - return fmt.Errorf(timeOutErr + "operation: Get") + if duration > latencyThreshold { + return duration, nil } + return time.Duration(0), nil +} - c5 := make(chan error) - go func() { - err := b.Delete(context.Background(), "diagnose") - c5 <- err - }() - select { - case errOut := <-c5: - if errOut != nil { - return errOut - } - case <-time.After(20 * time.Second): - return fmt.Errorf(timeOutErr + "operation: Delete") +func EndToEndLatencyCheckRead(ctx context.Context, uuid string, b physical.Backend) (time.Duration, error) { + + start := time.Now() + val, err := b.Get(context.Background(), uuid) + duration := time.Since(start) + if err != nil { + return time.Duration(0), err } - return nil + if val == nil { + return time.Duration(0), fmt.Errorf("no value found when reading generated data") + } + if val.Key != uuid && string(val.Value) != secretVal { + return time.Duration(0), fmt.Errorf(wrongRWValsPrefix+"expecting diagnose, but got %s, %s", val.Key, val.Value) + } + if duration > latencyThreshold { + return duration, nil + } + return time.Duration(0), nil +} +func EndToEndLatencyCheckDelete(ctx context.Context, uuid string, b physical.Backend) (time.Duration, error) { + + start := time.Now() + err := b.Delete(context.Background(), uuid) + duration := time.Since(start) + if err != nil { + return time.Duration(0), err + } + if duration > latencyThreshold { + return duration, nil + } + return time.Duration(0), nil } // ConsulDirectAccess verifies that consul is connecting to local agent, diff --git a/vault/diagnose/storage_checks_test.go b/vault/diagnose/storage_checks_test.go index 91caec376c..0005ec3216 100644 --- a/vault/diagnose/storage_checks_test.go +++ b/vault/diagnose/storage_checks_test.go @@ -4,6 +4,7 @@ import ( "context" "strings" "testing" + "time" "github.com/hashicorp/vault/sdk/physical" ) @@ -15,15 +16,15 @@ func TestStorageTimeout(t *testing.T) { mb physical.Backend }{ { - errSubString: timeOutErr + "operation: Put", + errSubString: LatencyWarning, mb: mockStorageBackend{callType: timeoutCallWrite}, }, { - errSubString: timeOutErr + "operation: Get", + errSubString: LatencyWarning, mb: mockStorageBackend{callType: timeoutCallRead}, }, { - errSubString: timeOutErr + "operation: Delete", + errSubString: LatencyWarning, mb: mockStorageBackend{callType: timeoutCallDelete}, }, { @@ -42,18 +43,31 @@ func TestStorageTimeout(t *testing.T) { errSubString: wrongRWValsPrefix, mb: mockStorageBackend{callType: badReadCall}, }, - { - errSubString: "", - mb: mockStorageBackend{callType: ""}, - }, } for _, tc := range testCases { - outErr := StorageEndToEndLatencyCheck(context.Background(), tc.mb) + var outErr error + var dur time.Duration + uuid := "foo" + backendCallType := tc.mb.(mockStorageBackend).callType + if callTypeToOp(backendCallType) == readOp { + dur, outErr = EndToEndLatencyCheckRead(context.Background(), uuid, tc.mb) + } + if callTypeToOp(backendCallType) == writeOp { + dur, outErr = EndToEndLatencyCheckWrite(context.Background(), uuid, tc.mb) + } + if callTypeToOp(backendCallType) == deleteOp { + dur, outErr = EndToEndLatencyCheckDelete(context.Background(), uuid, tc.mb) + } + if tc.errSubString == "" && outErr == nil { // this is the success case where the Storage Latency check passes continue } + if tc.errSubString == LatencyWarning && dur > time.Duration(0) { + // this is the success case where the Storage Latency check successfully returns nonzero duration + continue + } if !strings.Contains(outErr.Error(), tc.errSubString) { t.Errorf("wrong error: expected %s to be contained in %s", tc.errSubString, outErr) } diff --git a/vault/diagnose/tls_verification.go b/vault/diagnose/tls_verification.go index 6155cf663e..244be03cf6 100644 --- a/vault/diagnose/tls_verification.go +++ b/vault/diagnose/tls_verification.go @@ -10,7 +10,6 @@ import ( "github.com/hashicorp/vault/internalshared/listenerutil" "github.com/hashicorp/vault/sdk/helper/tlsutil" - "github.com/hashicorp/vault/vault" ) const minVersionError = "'tls_min_version' value %q not supported, please specify one of [tls10,tls11,tls12,tls13]" @@ -123,10 +122,3 @@ func TLSFileChecks(certFilePath, keyFilePath string) error { return nil } - -// ServerListenerActiveProbe attempts to use TLS information to set up a TLS server with each listener -// and generate a successful request through to the server. -// TODO -func ServerListenerActiveProbe(core *vault.Core) error { - return fmt.Errorf("Method not implemented") -} diff --git a/vault/init.go b/vault/init.go index 24ea18120d..c7662524e4 100644 --- a/vault/init.go +++ b/vault/init.go @@ -11,6 +11,7 @@ import ( wrapping "github.com/hashicorp/go-kms-wrapping" "github.com/hashicorp/vault/physical/raft" + "github.com/hashicorp/vault/vault/diagnose" "github.com/hashicorp/vault/vault/seal" aeadwrapper "github.com/hashicorp/go-kms-wrapping/wrappers/aead" @@ -466,9 +467,11 @@ func (c *Core) UnsealWithStoredKeys(ctx context.Context) error { // This usually happens when auto-unseal is configured, but the servers have // not been initialized yet. if len(keys) == 0 { + diagnose.Error(ctx, errors.New("stored unseal keys are supported, but none were found")) return NewNonFatalError(errors.New("stored unseal keys are supported, but none were found")) } if len(keys) != 1 { + diagnose.Error(ctx, errors.New("expected exactly one stored key")) return NewNonFatalError(errors.New("expected exactly one stored key")) } @@ -482,6 +485,7 @@ func (c *Core) UnsealWithStoredKeys(ctx context.Context) error { // subset of the required threshold of keys. We still consider this a // "success", since trying again would yield the same result. c.Logger().Warn("vault still sealed after using stored unseal key") + diagnose.Warn(ctx, "vault still sealed after using stored unseal key") } else { c.Logger().Info("unsealed with stored key") }