From 2feeb39b85796709a1b3757cc3abdd776427fa33 Mon Sep 17 00:00:00 2001 From: Scott Miller Date: Tue, 4 May 2021 14:47:56 -0500 Subject: [PATCH] Expose unknown fields and duplicate sections as diagnose warnings (#11455) * Expose unknown fields and duplicate sections as diagnose warnings * section counts not needed, already handled * Address PR feedback * Prune more of the new fields before tests call deep.Equals * Update go.mod --- command/agent/config/config.go | 22 ++++++ command/agent/config/config_test.go | 34 ++++----- command/commands.go | 5 ++ command/server/config.go | 60 ++++++++++++--- command/server/config_test.go | 4 + command/server/config_test_helpers.go | 74 +++++++++++++++---- command/server/test-fixtures/config.hcl | 1 + go.mod | 2 +- go.sum | 7 +- internalshared/configutil/config.go | 8 ++ internalshared/configutil/config_util.go | 57 ++++++++++++++ internalshared/configutil/kms.go | 3 +- internalshared/configutil/listener.go | 3 +- internalshared/configutil/telemetry.go | 12 ++- sdk/helper/strutil/strutil.go | 10 +++ vendor/github.com/hashicorp/hcl/decoder.go | 40 +++++++++- vendor/github.com/hashicorp/hcl/go.mod | 2 + .../vault/sdk/helper/strutil/strutil.go | 10 +++ vendor/modules.txt | 2 +- 19 files changed, 300 insertions(+), 56 deletions(-) diff --git a/command/agent/config/config.go b/command/agent/config/config.go index c96971913c..618e172a51 100644 --- a/command/agent/config/config.go +++ b/command/agent/config/config.go @@ -30,6 +30,20 @@ type Config struct { Templates []*ctconfig.TemplateConfig `hcl:"templates"` } +func (c *Config) Prune() { + for _, l := range c.Listeners { + l.RawConfig = nil + } + c.FoundKeys = nil + c.UnusedKeys = nil + c.SharedConfig.FoundKeys = nil + c.SharedConfig.UnusedKeys = nil + if c.Telemetry != nil { + c.Telemetry.FoundKeys = nil + c.Telemetry.UnusedKeys = nil + } +} + type Retry struct { NumRetries int `hcl:"num_retries"` } @@ -131,6 +145,14 @@ func LoadConfig(path string) (*Config, error) { return nil, err } + // Attribute + ast.Walk(obj, func(n ast.Node) (ast.Node, bool) { + if k, ok := n.(*ast.ObjectKey); ok { + k.Token.Pos.Filename = path + } + return n, true + }) + // Start building the result result := NewConfig() if err := hcl.DecodeObject(result, obj); err != nil { diff --git a/command/agent/config/config_test.go b/command/agent/config/config_test.go index 38a1d19fb7..ac14d2965e 100644 --- a/command/agent/config/config_test.go +++ b/command/agent/config/config_test.go @@ -88,9 +88,7 @@ func TestLoadConfigFile_AgentCache(t *testing.T) { }, } - config.Listeners[0].RawConfig = nil - config.Listeners[1].RawConfig = nil - config.Listeners[2].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -101,9 +99,7 @@ func TestLoadConfigFile_AgentCache(t *testing.T) { } expected.Vault.TLSSkipVerifyRaw = interface{}(true) - config.Listeners[0].RawConfig = nil - config.Listeners[1].RawConfig = nil - config.Listeners[2].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -168,6 +164,7 @@ func TestLoadConfigFile(t *testing.T) { }, } + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -177,6 +174,7 @@ func TestLoadConfigFile(t *testing.T) { t.Fatalf("err: %s", err) } + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -218,6 +216,7 @@ func TestLoadConfigFile_Method_Wrapping(t *testing.T) { }, } + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -248,7 +247,7 @@ func TestLoadConfigFile_AgentCache_NoAutoAuth(t *testing.T) { }, } - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -341,7 +340,7 @@ func TestLoadConfigFile_AgentCache_AutoAuth_NoSink(t *testing.T) { }, } - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -385,7 +384,7 @@ func TestLoadConfigFile_AgentCache_AutoAuth_Force(t *testing.T) { }, } - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -429,7 +428,7 @@ func TestLoadConfigFile_AgentCache_AutoAuth_True(t *testing.T) { }, } - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -484,7 +483,7 @@ func TestLoadConfigFile_AgentCache_AutoAuth_False(t *testing.T) { }, } - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -523,12 +522,7 @@ func TestLoadConfigFile_AgentCache_Persist(t *testing.T) { }, } - config.Listeners[0].RawConfig = nil - if diff := deep.Equal(config, expected); diff != nil { - t.Fatal(diff) - } - - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -644,6 +638,7 @@ func TestLoadConfigFile_Template(t *testing.T) { Templates: tc.expectedTemplates, } + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -744,6 +739,7 @@ func TestLoadConfigFile_Template_NoSinks(t *testing.T) { }, } + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -790,6 +786,7 @@ func TestLoadConfigFile_Vault_Retry(t *testing.T) { }, } + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -834,6 +831,7 @@ func TestLoadConfigFile_Vault_Retry_Empty(t *testing.T) { }, } + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -867,7 +865,7 @@ func TestLoadConfigFile_EnforceConsistency(t *testing.T) { }, } - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } diff --git a/command/commands.go b/command/commands.go index 36834b8a55..81d70674ad 100644 --- a/command/commands.go +++ b/command/commands.go @@ -337,6 +337,11 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) { BaseCommand: getBaseCommand(), }, nil }, + "operator diagnose": func() (cli.Command, error) { + return &OperatorDiagnoseCommand{ + BaseCommand: getBaseCommand(), + }, nil + }, "operator generate-root": func() (cli.Command, error) { return &OperatorGenerateRootCommand{ BaseCommand: getBaseCommand(), diff --git a/command/server/config.go b/command/server/config.go index f53097fec6..63074289a1 100644 --- a/command/server/config.go +++ b/command/server/config.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/hashicorp/hcl/hcl/token" "io" "io/ioutil" "os" @@ -22,6 +23,7 @@ import ( // Config is the configuration for the vault server. type Config struct { + UnusedKeys map[string][]token.Pos `hcl:",unusedKeyPositions"` entConfig *configutil.SharedConfig `hcl:"-"` @@ -41,33 +43,33 @@ type Config struct { EnableUIRaw interface{} `hcl:"ui"` MaxLeaseTTL time.Duration `hcl:"-"` - MaxLeaseTTLRaw interface{} `hcl:"max_lease_ttl"` + MaxLeaseTTLRaw interface{} `hcl:"max_lease_ttl,alias:MaxLeaseTTL"` DefaultLeaseTTL time.Duration `hcl:"-"` - DefaultLeaseTTLRaw interface{} `hcl:"default_lease_ttl"` + DefaultLeaseTTLRaw interface{} `hcl:"default_lease_ttl,alias:DefaultLeaseTTL"` ClusterCipherSuites string `hcl:"cluster_cipher_suites"` PluginDirectory string `hcl:"plugin_directory"` EnableRawEndpoint bool `hcl:"-"` - EnableRawEndpointRaw interface{} `hcl:"raw_storage_endpoint"` + EnableRawEndpointRaw interface{} `hcl:"raw_storage_endpoint,alias:EnableRawEndpoint"` APIAddr string `hcl:"api_addr"` ClusterAddr string `hcl:"cluster_addr"` DisableClustering bool `hcl:"-"` - DisableClusteringRaw interface{} `hcl:"disable_clustering"` + DisableClusteringRaw interface{} `hcl:"disable_clustering,alias:DisableClustering"` DisablePerformanceStandby bool `hcl:"-"` - DisablePerformanceStandbyRaw interface{} `hcl:"disable_performance_standby"` + DisablePerformanceStandbyRaw interface{} `hcl:"disable_performance_standby,alias:DisablePerformanceStandby"` DisableSealWrap bool `hcl:"-"` - DisableSealWrapRaw interface{} `hcl:"disable_sealwrap"` + DisableSealWrapRaw interface{} `hcl:"disable_sealwrap,alias:DisableSealWrap"` DisableIndexing bool `hcl:"-"` - DisableIndexingRaw interface{} `hcl:"disable_indexing"` + DisableIndexingRaw interface{} `hcl:"disable_indexing,alias:DisableIndexing"` DisableSentinelTrace bool `hcl:"-"` - DisableSentinelTraceRaw interface{} `hcl:"disable_sentinel_trace"` + DisableSentinelTraceRaw interface{} `hcl:"disable_sentinel_trace,alias:DisableSentinelTrace"` EnableResponseHeaderHostname bool `hcl:"-"` EnableResponseHeaderHostnameRaw interface{} `hcl:"enable_response_header_hostname"` @@ -76,6 +78,18 @@ type Config struct { EnableResponseHeaderRaftNodeIDRaw interface{} `hcl:"enable_response_header_raft_node_id"` } +const ( + sectionSeal = "Seal" +) + +func (c *Config) Validate(sourceFilePath string) []configutil.ConfigError { + results := configutil.ValidateUnusedFields(c.UnusedKeys, sourceFilePath) + if c.Telemetry != nil { + results = append(results, c.Telemetry.Validate(sourceFilePath)...) + } + return results +} + // DevConfig is a Config that is used for dev mode of Vault. func DevConfig(storageType string) (*Config, error) { hclStr := ` @@ -102,7 +116,7 @@ ui = true ` hclStr = fmt.Sprintf(hclStr, storageType) - parsed, err := ParseConfig(hclStr) + parsed, err := ParseConfig(hclStr, "") if err != nil { return nil, fmt.Errorf("error parsing dev config: %w", err) } @@ -111,6 +125,7 @@ ui = true // Storage is the underlying storage configuration for the server. type Storage struct { + UnusedKeys []string `hcl:",unusedKeys"` Type string RedirectAddr string ClusterAddr string @@ -330,7 +345,7 @@ func LoadConfigFile(path string) (*Config, error) { return nil, err } - conf, err := ParseConfig(string(d)) + conf, err := ParseConfig(string(d), path) if err != nil { return nil, err } @@ -338,7 +353,7 @@ func LoadConfigFile(path string) (*Config, error) { return conf, nil } -func ParseConfig(d string) (*Config, error) { +func ParseConfig(d, source string) (*Config, error) { // Parse! obj, err := hcl.Parse(d) if err != nil { @@ -476,6 +491,15 @@ func ParseConfig(d string) (*Config, error) { return nil, errwrap.Wrapf("error parsing enterprise config: {{err}}", err) } + // Remove all unused keys from Config that were satisfied by SharedConfig. + result.UnusedKeys = configutil.UnusedFieldDifference(result.UnusedKeys, sharedConfig.UnusedKeys, append(result.FoundKeys, sharedConfig.FoundKeys...)) + // Assign file info + for _, v := range result.UnusedKeys { + for _, p := range v { + p.Filename = source + } + } + return result, nil } @@ -816,3 +840,17 @@ func (c *Config) Sanitized() map[string]interface{} { return result } + +func (c *Config) Prune() { + for _, l := range c.Listeners { + l.RawConfig = nil + } + c.FoundKeys = nil + c.UnusedKeys = nil + c.SharedConfig.FoundKeys = nil + c.SharedConfig.UnusedKeys = nil + if c.Telemetry != nil { + c.Telemetry.FoundKeys = nil + c.Telemetry.UnusedKeys = nil + } +} diff --git a/command/server/config_test.go b/command/server/config_test.go index 779518766e..dda4e7821f 100644 --- a/command/server/config_test.go +++ b/command/server/config_test.go @@ -57,3 +57,7 @@ func TestConfigRaftRetryJoin(t *testing.T) { func TestParseSeals(t *testing.T) { testParseSeals(t) } + +func TestUnknownFieldValidation(t *testing.T) { + testUnknownFieldValidation(t) +} diff --git a/command/server/config_test_helpers.go b/command/server/config_test_helpers.go index b026d13eef..5b8f2f03b4 100644 --- a/command/server/config_test_helpers.go +++ b/command/server/config_test_helpers.go @@ -2,6 +2,7 @@ package server import ( "fmt" + "reflect" "strings" "testing" "time" @@ -11,6 +12,7 @@ import ( "github.com/go-test/deep" "github.com/hashicorp/hcl" "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/hcl/hcl/token" "github.com/hashicorp/vault/internalshared/configutil" ) @@ -40,7 +42,7 @@ func testConfigRaftRetryJoin(t *testing.T) { }, }, } - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -150,7 +152,7 @@ func testLoadConfigFile_topLevel(t *testing.T, entropy *configutil.Entropy) { if entropy != nil { expected.Entropy = entropy } - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -239,8 +241,8 @@ func testLoadConfigFile_json2(t *testing.T, entropy *configutil.Entropy) { if entropy != nil { expected.Entropy = entropy } - config.Listeners[0].RawConfig = nil - config.Listeners[1].RawConfig = nil + + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -257,7 +259,7 @@ func testParseEntropy(t *testing.T, oss bool) { mode = "augmentation" }`, outErr: nil, - outEntropy: configutil.Entropy{configutil.EntropyAugmentation}, + outEntropy: configutil.Entropy{Mode: configutil.EntropyAugmentation}, }, { inConfig: `entropy "seal" { @@ -355,7 +357,7 @@ func testLoadConfigFileIntegerAndBooleanValuesCommon(t *testing.T, path string) EnableUIRaw: true, } - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -450,12 +452,57 @@ func testLoadConfigFile(t *testing.T) { addExpectedEntConfig(expected, []string{}) - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } } +func testUnknownFieldValidation(t *testing.T) { + config, err := LoadConfigFile("./test-fixtures/config.hcl") + if err != nil { + t.Fatalf("err: %s", err) + } + + expected := []configutil.ConfigError{ + { + Problem: "unknown field bad_value found in configuration", + Position: token.Pos{ + Filename: "./test-fixtures/config.hcl", + Offset: 603, + Line: 35, + Column: 5, + }, + }, + } + errors := config.Validate("./test-fixtures/config.hcl") + + for _, er1 := range errors { + found := false + for _, ex := range expected { + // Only test the string, pos may change + if ex.Problem == er1.Problem && reflect.DeepEqual(ex.Position, er1.Position) { + found = true + break + } + } + if !found { + t.Fatalf("found unexpected error: %v", er1.String()) + } + } + for _, ex := range expected { + found := false + for _, er1 := range errors { + if ex.Problem == er1.Problem && reflect.DeepEqual(ex.Position, er1.Position) { + found = true + } + } + if !found { + t.Fatalf("could not find expected error: %v", ex.String()) + } + } +} + func testLoadConfigFile_json(t *testing.T) { config, err := LoadConfigFile("./test-fixtures/config.hcl.json") if err != nil { @@ -533,7 +580,7 @@ func testLoadConfigFile_json(t *testing.T) { addExpectedEntConfig(expected, []string{}) - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -597,7 +644,7 @@ func testLoadConfigDir(t *testing.T) { addExpectedEntConfig(expected, []string{"http"}) - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -699,7 +746,7 @@ func testConfig_Sanitized(t *testing.T) { addExpectedEntSanitizedConfig(expected, []string{"http"}) - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(sanitizedConfig, expected); len(diff) > 0 { t.Fatalf("bad, diff: %#v", diff) } @@ -765,7 +812,7 @@ listener "tcp" { }, }, } - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, *expected); diff != nil { t.Fatal(diff) } @@ -825,6 +872,7 @@ func testParseSeals(t *testing.T) { }, }, } + config.Prune() require.Equal(t, config, expected) } @@ -912,7 +960,7 @@ func testLoadConfigFileLeaseMetrics(t *testing.T) { addExpectedEntConfig(expected, []string{}) - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } @@ -945,7 +993,7 @@ func testConfigRaftAutopilot(t *testing.T) { }, }, } - config.Listeners[0].RawConfig = nil + config.Prune() if diff := deep.Equal(config, expected); diff != nil { t.Fatal(diff) } diff --git a/command/server/test-fixtures/config.hcl b/command/server/test-fixtures/config.hcl index da83dd9ab4..d9ec982856 100644 --- a/command/server/test-fixtures/config.hcl +++ b/command/server/test-fixtures/config.hcl @@ -32,6 +32,7 @@ telemetry { dogstatsd_addr = "127.0.0.1:7254" dogstatsd_tags = ["tag_1:val_1", "tag_2:val_2"] metrics_prefix = "myprefix" + bad_value = "shouldn't be here" } sentinel { diff --git a/go.mod b/go.mod index a4917f1325..c7be4a9551 100644 --- a/go.mod +++ b/go.mod @@ -73,7 +73,7 @@ require ( github.com/hashicorp/go-syslog v1.0.0 github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/golang-lru v0.5.4 - github.com/hashicorp/hcl v1.0.1-vault + github.com/hashicorp/hcl v1.0.1-vault-2 github.com/hashicorp/nomad/api v0.0.0-20191220223628-edc62acd919d github.com/hashicorp/raft v1.3.0 github.com/hashicorp/raft-autopilot v0.1.3 diff --git a/go.sum b/go.sum index 47e6b69540..dc15db857e 100644 --- a/go.sum +++ b/go.sum @@ -486,7 +486,6 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= @@ -644,8 +643,8 @@ github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= -github.com/hashicorp/hcl v1.0.1-vault h1:UiJeEzCWAYdVaJr8Xo4lBkTozlW1+1yxVUnpbS1xVEk= -github.com/hashicorp/hcl v1.0.1-vault/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/hcl v1.0.1-vault-2 h1:j0lTHGBdaU13Pc3GaTCdWjmsT22X98bsHnA+ShzIOtg= +github.com/hashicorp/hcl v1.0.1-vault-2/go.mod h1:XYhtn6ijBSAj6n4YqAaf7RBPS4I06AItNorpy+MoQNM= github.com/hashicorp/logutils v1.0.0 h1:dLEQVugN8vlakKOUE3ihGLTZJRB4j+M2cdTm/ORI65Y= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ= @@ -1156,7 +1155,6 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -1228,6 +1226,7 @@ go.opentelemetry.io/otel v0.20.0 h1:eaP0Fqu7SXHwvjiqDq83zImeehOHX8doTvU9AwXON8g= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel/metric v0.20.0 h1:4kzhXFP+btKm4jwxpjIqjs41A7MakRFUS86bqLHTIw8= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= +go.opentelemetry.io/otel/oteltest v0.20.0 h1:HiITxCawalo5vQzdHfKeZurV8x7ljcqAgiWzF6Vaeaw= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0 h1:JsxtGXd06J8jrnya7fdI/U/MR6yXA5DtbZy+qoHQlr8= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= diff --git a/internalshared/configutil/config.go b/internalshared/configutil/config.go index 6ef434cccb..8d2746e709 100644 --- a/internalshared/configutil/config.go +++ b/internalshared/configutil/config.go @@ -2,6 +2,7 @@ package configutil import ( "fmt" + "github.com/hashicorp/hcl/hcl/token" "io/ioutil" "time" @@ -13,6 +14,10 @@ import ( // SharedConfig contains some shared values type SharedConfig struct { + FoundKeys []string `hcl:",decodedFields"` + UnusedKeys UnusedKeyMap `hcl:",unusedKeyPositions"` + Sections map[string][]token.Pos + EntSharedConfig Listeners []*Listener `hcl:"-"` @@ -67,6 +72,7 @@ func ParseConfig(d string) (*SharedConfig, error) { // Start building the result var result SharedConfig + if err := hcl.DecodeObject(&result, obj); err != nil { return nil, err } @@ -75,6 +81,7 @@ func ParseConfig(d string) (*SharedConfig, error) { if result.DefaultMaxRequestDuration, err = parseutil.ParseDurationSecond(result.DefaultMaxRequestDurationRaw); err != nil { return nil, err } + result.FoundKeys = append(result.FoundKeys, "DefaultMaxRequestDuration") result.DefaultMaxRequestDurationRaw = nil } @@ -82,6 +89,7 @@ func ParseConfig(d string) (*SharedConfig, error) { if result.DisableMlock, err = parseutil.ParseBool(result.DisableMlockRaw); err != nil { return nil, err } + result.FoundKeys = append(result.FoundKeys, "DisableMlock") result.DisableMlockRaw = nil } diff --git a/internalshared/configutil/config_util.go b/internalshared/configutil/config_util.go index fc527cc2af..ca108af5ae 100644 --- a/internalshared/configutil/config_util.go +++ b/internalshared/configutil/config_util.go @@ -3,11 +3,30 @@ package configutil import ( + "fmt" + "github.com/asaskevich/govalidator" "github.com/hashicorp/hcl/hcl/ast" + "github.com/hashicorp/hcl/hcl/token" + "github.com/hashicorp/vault/sdk/helper/strutil" ) type EntSharedConfig struct{} +type UnusedKeyMap map[string][]token.Pos + +type ConfigError struct { + Problem string + Position token.Pos +} + +func (c *ConfigError) String() string { + return fmt.Sprintf("%s at %s", c.Problem, c.Position.String()) +} + +type ValidatableConfig interface { + Validate() []ConfigError +} + func (ec *EntSharedConfig) ParseConfig(list *ast.ObjectList) error { return nil } @@ -15,3 +34,41 @@ func (ec *EntSharedConfig) ParseConfig(list *ast.ObjectList) error { func ParseEntropy(result *SharedConfig, list *ast.ObjectList, blockName string) error { return nil } + +// Creates the ConfigErrors for unused fields, which occur in various structs +func ValidateUnusedFields(unusedKeyPositions UnusedKeyMap, sourceFilePath string) []ConfigError { + if unusedKeyPositions == nil { + return nil + } + var errors []ConfigError + for field, positions := range unusedKeyPositions { + problem := fmt.Sprintf("unknown field %s found in configuration", field) + for _, pos := range positions { + if pos.Filename == "" && sourceFilePath != "" { + pos.Filename = sourceFilePath + } + errors = append(errors, ConfigError{ + Problem: problem, + Position: pos, + }) + } + } + return errors +} + +// UnusedFieldDifference returns all the keys in map a that are not present in map b, and also not present in foundKeys. +func UnusedFieldDifference(a, b UnusedKeyMap, foundKeys []string) UnusedKeyMap { + if a == nil { + return nil + } + if b == nil { + return a + } + res := make(UnusedKeyMap) + for k, v := range a { + if _, ok := b[k]; !ok && !strutil.StrListContainsCaseInsensitive(foundKeys, govalidator.UnderscoreToCamelCase(k)) { + res[k] = v + } + } + return res +} diff --git a/internalshared/configutil/kms.go b/internalshared/configutil/kms.go index 16e0ac5054..1beedc36d5 100644 --- a/internalshared/configutil/kms.go +++ b/internalshared/configutil/kms.go @@ -42,7 +42,8 @@ type Entropy struct { // KMS contains KMS configuration for the server type KMS struct { - Type string + UnusedKeys []string `hcl:",unusedKeys"` + Type string // Purpose can be used to allow a string-based specification of what this // KMS is designated for, in situations where we want to allow more than // one KMS to be specified diff --git a/internalshared/configutil/listener.go b/internalshared/configutil/listener.go index ed9c90e325..cbb1194e33 100644 --- a/internalshared/configutil/listener.go +++ b/internalshared/configutil/listener.go @@ -28,7 +28,8 @@ type ListenerProfiling struct { // Listener is the listener configuration for the server. type Listener struct { - RawConfig map[string]interface{} + UnusedKeys []string `hcl:",unusedKeys"` + RawConfig map[string]interface{} Type string Purpose []string `hcl:"-"` diff --git a/internalshared/configutil/telemetry.go b/internalshared/configutil/telemetry.go index 932f11ebcd..0b2b23b24e 100644 --- a/internalshared/configutil/telemetry.go +++ b/internalshared/configutil/telemetry.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "github.com/hashicorp/vault/sdk/helper/parseutil" "time" monitoring "cloud.google.com/go/monitoring/apiv3" @@ -18,7 +19,6 @@ import ( "github.com/hashicorp/hcl" "github.com/hashicorp/hcl/hcl/ast" "github.com/hashicorp/vault/helper/metricsutil" - "github.com/hashicorp/vault/sdk/helper/parseutil" "github.com/mitchellh/cli" "google.golang.org/api/option" ) @@ -33,8 +33,10 @@ const ( // Telemetry is the telemetry configuration for the server type Telemetry struct { - StatsiteAddr string `hcl:"statsite_address"` - StatsdAddr string `hcl:"statsd_address"` + FoundKeys []string `hcl:",decodedFields"` + UnusedKeys UnusedKeyMap `hcl:",unusedKeyPositions"` + StatsiteAddr string `hcl:"statsite_address"` + StatsdAddr string `hcl:"statsd_address"` DisableHostname bool `hcl:"disable_hostname"` EnableHostnameLabel bool `hcl:"enable_hostname_label"` @@ -151,6 +153,10 @@ type Telemetry struct { LeaseMetricsNameSpaceLabels bool `hcl:"add_lease_metrics_namespace_labels"` } +func (t *Telemetry) Validate(source string) []ConfigError { + return ValidateUnusedFields(t.UnusedKeys, source) +} + func (t *Telemetry) GoString() string { return fmt.Sprintf("*%#v", *t) } diff --git a/sdk/helper/strutil/strutil.go b/sdk/helper/strutil/strutil.go index 2083794087..5d0b36b113 100644 --- a/sdk/helper/strutil/strutil.go +++ b/sdk/helper/strutil/strutil.go @@ -32,6 +32,16 @@ func StrListContains(haystack []string, needle string) bool { return false } +// StrListContainsCaseInsensitive looks for a string in a list of strings. +func StrListContainsCaseInsensitive(haystack []string, needle string) bool { + for _, item := range haystack { + if strings.EqualFold(item, needle) { + return true + } + } + return false +} + // StrListSubset checks if a given list is a subset // of another set func StrListSubset(super, sub []string) bool { diff --git a/vendor/github.com/hashicorp/hcl/decoder.go b/vendor/github.com/hashicorp/hcl/decoder.go index bed9ebbe14..ef47a589e3 100644 --- a/vendor/github.com/hashicorp/hcl/decoder.go +++ b/vendor/github.com/hashicorp/hcl/decoder.go @@ -505,7 +505,7 @@ func expandObject(node ast.Node, result reflect.Value) ast.Node { // we need to un-flatten the ast enough to decode newNode := &ast.ObjectItem{ Keys: []*ast.ObjectKey{ - &ast.ObjectKey{ + { Token: keyToken, }, }, @@ -628,6 +628,18 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) decodedFields := make([]string, 0, len(fields)) decodedFieldsVal := make([]reflect.Value, 0) unusedKeysVal := make([]reflect.Value, 0) + + // fill unusedNodeKeys with keys from the AST + // a slice because we have to do equals case fold to match Filter + unusedNodeKeys := make(map[string][]token.Pos, 0) + for _, item := range list.Items { + for _, k := range item.Keys { + fn := k.Token.Value().(string) + sl := unusedNodeKeys[fn] + unusedNodeKeys[fn] = append(sl, k.Token.Pos) + } + } + for _, f := range fields { field, fieldValue := f.field, f.val if !fieldValue.IsValid() { @@ -661,7 +673,7 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) fieldValue.SetString(item.Keys[0].Token.Value().(string)) continue - case "unusedKeys": + case "unusedKeyPositions": unusedKeysVal = append(unusedKeysVal, fieldValue) continue } @@ -682,8 +694,9 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) continue } - // Track the used key + // Track the used keys usedKeys[fieldName] = struct{}{} + unusedNodeKeys = removeCaseFold(unusedNodeKeys, fieldName) // Create the field name and decode. We range over the elements // because we actually want the value. @@ -716,6 +729,13 @@ func (d *decoder) decodeStruct(name string, node ast.Node, result reflect.Value) } } + if len(unusedNodeKeys) > 0 { + // like decodedFields, populated the unusedKeys field(s) + for _, v := range unusedKeysVal { + v.Set(reflect.ValueOf(unusedNodeKeys)) + } + } + return nil } @@ -727,3 +747,17 @@ func findNodeType() reflect.Type { value := reflect.ValueOf(nodeContainer).FieldByName("Node") return value.Type() } + +func removeCaseFold(xs map[string][]token.Pos, y string) map[string][]token.Pos { + var toDel []string + + for i := range xs { + if strings.EqualFold(i, y) { + toDel = append(toDel, i) + } + } + for _, i := range toDel { + delete(xs, i) + } + return xs +} diff --git a/vendor/github.com/hashicorp/hcl/go.mod b/vendor/github.com/hashicorp/hcl/go.mod index 4debbbe358..4c78c89d10 100644 --- a/vendor/github.com/hashicorp/hcl/go.mod +++ b/vendor/github.com/hashicorp/hcl/go.mod @@ -1,3 +1,5 @@ module github.com/hashicorp/hcl +go 1.15 + require github.com/davecgh/go-spew v1.1.1 diff --git a/vendor/github.com/hashicorp/vault/sdk/helper/strutil/strutil.go b/vendor/github.com/hashicorp/vault/sdk/helper/strutil/strutil.go index 2083794087..5d0b36b113 100644 --- a/vendor/github.com/hashicorp/vault/sdk/helper/strutil/strutil.go +++ b/vendor/github.com/hashicorp/vault/sdk/helper/strutil/strutil.go @@ -32,6 +32,16 @@ func StrListContains(haystack []string, needle string) bool { return false } +// StrListContainsCaseInsensitive looks for a string in a list of strings. +func StrListContainsCaseInsensitive(haystack []string, needle string) bool { + for _, item := range haystack { + if strings.EqualFold(item, needle) { + return true + } + } + return false +} + // StrListSubset checks if a given list is a subset // of another set func StrListSubset(super, sub []string) bool { diff --git a/vendor/modules.txt b/vendor/modules.txt index 04313eaa1d..9636792469 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -589,7 +589,7 @@ github.com/hashicorp/go-version ## explicit github.com/hashicorp/golang-lru github.com/hashicorp/golang-lru/simplelru -# github.com/hashicorp/hcl v1.0.1-vault +# github.com/hashicorp/hcl v1.0.1-vault-2 ## explicit github.com/hashicorp/hcl github.com/hashicorp/hcl/hcl/ast