Disable the sys/raw endpoint by default (#3329)

* disable raw endpoint by default

* adding docs

* config option raw -> raw_storage_endpoint

* docs updates

* adding listing on raw endpoint

* reworking tests for enabled raw endpoints

* root protecting base raw endpoint

command/server.go

@@ -260,6 +260,7 @@ func (c *ServerCommand) Run(args []string) int {
 		ClusterName:        config.ClusterName,
 		CacheSize:          config.CacheSize,
 		PluginDirectory:    config.PluginDirectory,
+		EnableRaw:          config.EnableRawEndpoint,
 	}
 	if dev {
 		coreConfig.DevToken = devRootTokenID

command/server/config.go

@@ -46,13 +46,17 @@ type Config struct {
 	ClusterCipherSuites string `hcl:"cluster_cipher_suites"`
 
 	PluginDirectory string `hcl:"plugin_directory"`
+
+	EnableRawEndpoint    bool        `hcl:"-"`
+	EnableRawEndpointRaw interface{} `hcl:"raw_storage_endpoint"`
 }
 
 // DevConfig is a Config that is used for dev mode of Vault.
 func DevConfig(ha, transactional bool) *Config {
 	ret := &Config{
-		DisableCache: false,
-		DisableMlock: true,
+		DisableCache:      false,
+		DisableMlock:      true,
+		EnableRawEndpoint: true,
 
 		Storage: &Storage{
 			Type: "inmem",
@@ -288,6 +292,11 @@ func (c *Config) Merge(c2 *Config) *Config {
 		result.EnableUI = c2.EnableUI
 	}
 
+	result.EnableRawEndpoint = c.EnableRawEndpoint
+	if c2.EnableRawEndpoint {
+		result.EnableRawEndpoint = c2.EnableRawEndpoint
+	}
+
 	result.PluginDirectory = c.PluginDirectory
 	if c2.PluginDirectory != "" {
 		result.PluginDirectory = c2.PluginDirectory
@@ -306,9 +315,8 @@ func LoadConfig(path string, logger log.Logger) (*Config, error) {
 
 	if fi.IsDir() {
 		return LoadConfigDir(path, logger)
-	} else {
-		return LoadConfigFile(path, logger)
 	}
+	return LoadConfigFile(path, logger)
 }
 
 // LoadConfigFile loads the configuration from the given file.
@@ -363,6 +371,12 @@ func ParseConfig(d string, logger log.Logger) (*Config, error) {
 		}
 	}
 
+	if result.EnableRawEndpointRaw != nil {
+		if result.EnableRawEndpoint, err = parseutil.ParseBool(result.EnableRawEndpointRaw); err != nil {
+			return nil, err
+		}
+	}
+
 	list, ok := obj.Node.(*ast.ObjectList)
 	if !ok {
 		return nil, fmt.Errorf("error parsing: file doesn't contain a root object")
@@ -385,6 +399,7 @@ func ParseConfig(d string, logger log.Logger) (*Config, error) {
 		"cluster_name",
 		"cluster_cipher_suites",
 		"plugin_directory",
+		"raw_storage_endpoint",
 	}
 	if err := checkHCLKeys(list, valid); err != nil {
 		return nil, err

command/server/config_test.go

@@ -62,6 +62,9 @@ func TestLoadConfigFile(t *testing.T) {
 		EnableUI:        true,
 		EnableUIRaw:     true,
 
+		EnableRawEndpoint:    true,
+		EnableRawEndpointRaw: true,
+
 		MaxLeaseTTL:        10 * time.Hour,
 		MaxLeaseTTLRaw:     "10h",
 		DefaultLeaseTTL:    10 * time.Hour,
@@ -129,6 +132,9 @@ func TestLoadConfigFile_json(t *testing.T) {
 		DisableMlockRaw:    interface{}(nil),
 		EnableUI:           true,
 		EnableUIRaw:        true,
+
+		EnableRawEndpoint:    true,
+		EnableRawEndpointRaw: true,
 	}
 	if !reflect.DeepEqual(config, expected) {
 		t.Fatalf("expected \n\n%#v\n\n to be \n\n%#v\n\n", config, expected)
@@ -178,6 +184,8 @@ func TestLoadConfigFile_json2(t *testing.T) {
 
 		EnableUI: true,
 
+		EnableRawEndpoint: true,
+
 		Telemetry: &Telemetry{
 			StatsiteAddr:                       "foo",
 			StatsdAddr:                         "bar",
@@ -232,6 +240,8 @@ func TestLoadConfigDir(t *testing.T) {
 
 		EnableUI: true,
 
+		EnableRawEndpoint: true,
+
 		Telemetry: &Telemetry{
 			StatsiteAddr:    "qux",
 			StatsdAddr:      "baz",

command/server/test-fixtures/config-dir/baz.hcl

@@ -4,5 +4,6 @@ telemetry {
     disable_hostname = true
 }
 ui=true
+raw_storage_endpoint=true
 default_lease_ttl = "10h"
 cluster_name = "testcluster"

command/server/test-fixtures/config.hcl

@@ -28,3 +28,4 @@ telemetry {
 max_lease_ttl = "10h"
 default_lease_ttl = "10h"
 cluster_name = "testcluster"
+raw_storage_endpoint = true

command/server/test-fixtures/config.hcl.json

@@ -17,5 +17,6 @@
 	"max_lease_ttl": "10h",
 	"default_lease_ttl": "10h",
 	"cluster_name":"testcluster",
-	"ui":true
+	"ui":true,
+	"raw_storage_endpoint":true
 }

command/server/test-fixtures/config2.hcl.json

@@ -1,5 +1,6 @@
 {
   "ui":true,
+  "raw_storage_endpoint":true,
   "listener":[
     {
       "tcp":{

vault/core.go

@@ -344,6 +344,9 @@ type Core struct {
 	// uiEnabled indicates whether Vault Web UI is enabled or not
 	uiEnabled bool
 
+	// rawEnabled indicates whether the Raw endpoint is enabled
+	rawEnabled bool
+
 	// pluginDirectory is the location vault will look for plugin binaries
 	pluginDirectory string
 
@@ -402,6 +405,9 @@ type CoreConfig struct {
 
 	EnableUI bool `json:"ui" structs:"ui" mapstructure:"ui"`
 
+	// Enable the raw endpoint
+	EnableRaw bool `json:"enable_raw" structs:"enable_raw" mapstructure:"enable_raw"`
+
 	PluginDirectory string `json:"plugin_directory" structs:"plugin_directory" mapstructure:"plugin_directory"`
 
 	ReloadFuncs     *map[string][]reload.ReloadFunc
@@ -462,6 +468,7 @@ func NewCore(conf *CoreConfig) (*Core, error) {
 		clusterListenerShutdownSuccessCh: make(chan struct{}),
 		clusterPeerClusterAddrsCache:     cache.New(3*heartbeatInterval, time.Second),
 		enableMlock:                      !conf.DisableMlock,
+		rawEnabled:                       conf.EnableRaw,
 	}
 
 	if conf.ClusterCipherSuites != "" {

vault/logical_system.go

@@ -22,7 +22,7 @@ var (
 	// protectedPaths cannot be accessed via the raw APIs.
 	// This is both for security and to prevent disrupting Vault.
 	protectedPaths = []string{
-		"core",
+		keyringPath,
 	}
 
 	replicationPaths = func(b *SystemBackend) []*framework.Path {
@@ -59,6 +59,7 @@ func NewSystemBackend(core *Core) *SystemBackend {
 				"remount",
 				"audit",
 				"audit/*",
+				"raw",
 				"raw/*",
 				"replication/primary/secondary-token",
 				"replication/reindex",
@@ -652,25 +653,6 @@ func NewSystemBackend(core *Core) *SystemBackend {
 				HelpDescription: strings.TrimSpace(sysHelp["audit"][1]),
 			},
 
-			&framework.Path{
-				Pattern: "raw/(?P<path>.+)",
-
-				Fields: map[string]*framework.FieldSchema{
-					"path": &framework.FieldSchema{
-						Type: framework.TypeString,
-					},
-					"value": &framework.FieldSchema{
-						Type: framework.TypeString,
-					},
-				},
-
-				Callbacks: map[logical.Operation]framework.OperationFunc{
-					logical.ReadOperation:   b.handleRawRead,
-					logical.UpdateOperation: b.handleRawWrite,
-					logical.DeleteOperation: b.handleRawDelete,
-				},
-			},
-
 			&framework.Path{
 				Pattern: "key-status$",
 
@@ -871,6 +853,28 @@ func NewSystemBackend(core *Core) *SystemBackend {
 
 	b.Backend.Paths = append(b.Backend.Paths, replicationPaths(b)...)
 
+	if core.rawEnabled {
+		b.Backend.Paths = append(b.Backend.Paths, &framework.Path{
+			Pattern: "(raw/?$|raw/(?P<path>.+))",
+
+			Fields: map[string]*framework.FieldSchema{
+				"path": &framework.FieldSchema{
+					Type: framework.TypeString,
+				},
+				"value": &framework.FieldSchema{
+					Type: framework.TypeString,
+				},
+			},
+
+			Callbacks: map[logical.Operation]framework.OperationFunc{
+				logical.ReadOperation:   b.handleRawRead,
+				logical.UpdateOperation: b.handleRawWrite,
+				logical.DeleteOperation: b.handleRawDelete,
+				logical.ListOperation:   b.handleRawList,
+			},
+		})
+	}
+
 	b.Backend.Invalidate = b.invalidate
 
 	return b
@@ -2143,6 +2147,29 @@ func (b *SystemBackend) handleRawDelete(
 	return nil, nil
 }
 
+// handleRawList is used to list directly from the barrier
+func (b *SystemBackend) handleRawList(
+	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {
+	path := data.Get("path").(string)
+	if path != "" && !strings.HasSuffix(path, "/") {
+		path = path + "/"
+	}
+
+	// Prevent access of protected paths
+	for _, p := range protectedPaths {
+		if strings.HasPrefix(path, p) {
+			err := fmt.Sprintf("cannot list '%s'", path)
+			return logical.ErrorResponse(err), logical.ErrInvalidRequest
+		}
+	}
+
+	keys, err := b.Core.barrier.List(path)
+	if err != nil {
+		return handleError(err)
+	}
+	return logical.ListResponse(keys), nil
+}
+
 // handleKeyStatus returns status information about the backend key
 func (b *SystemBackend) handleKeyStatus(
 	req *logical.Request, data *framework.FieldData) (*logical.Response, error) {

vault/logical_system_test.go

@@ -27,6 +27,7 @@ func TestSystemBackend_RootPaths(t *testing.T) {
 		"remount",
 		"audit",
 		"audit/*",
+		"raw",
 		"raw/*",
 		"replication/primary/secondary-token",
 		"replication/reindex",
@@ -1447,7 +1448,7 @@ func TestSystemBackend_disableAudit(t *testing.T) {
 }
 
 func TestSystemBackend_rawRead_Protected(t *testing.T) {
-	b := testSystemBackend(t)
+	b := testSystemBackendRaw(t)
 
 	req := logical.TestRequest(t, logical.ReadOperation, "raw/"+keyringPath)
 	_, err := b.HandleRequest(req)
@@ -1457,7 +1458,7 @@ func TestSystemBackend_rawRead_Protected(t *testing.T) {
 }
 
 func TestSystemBackend_rawWrite_Protected(t *testing.T) {
-	b := testSystemBackend(t)
+	b := testSystemBackendRaw(t)
 
 	req := logical.TestRequest(t, logical.UpdateOperation, "raw/"+keyringPath)
 	_, err := b.HandleRequest(req)
@@ -1467,7 +1468,7 @@ func TestSystemBackend_rawWrite_Protected(t *testing.T) {
 }
 
 func TestSystemBackend_rawReadWrite(t *testing.T) {
-	c, b, _ := testCoreSystemBackend(t)
+	c, b, _ := testCoreSystemBackendRaw(t)
 
 	req := logical.TestRequest(t, logical.UpdateOperation, "raw/sys/policy/test")
 	req.Data["value"] = `path "secret/" { policy = "read" }`
@@ -1503,7 +1504,7 @@ func TestSystemBackend_rawReadWrite(t *testing.T) {
 }
 
 func TestSystemBackend_rawDelete_Protected(t *testing.T) {
-	b := testSystemBackend(t)
+	b := testSystemBackendRaw(t)
 
 	req := logical.TestRequest(t, logical.DeleteOperation, "raw/"+keyringPath)
 	_, err := b.HandleRequest(req)
@@ -1513,7 +1514,7 @@ func TestSystemBackend_rawDelete_Protected(t *testing.T) {
 }
 
 func TestSystemBackend_rawDelete(t *testing.T) {
-	c, b, _ := testCoreSystemBackend(t)
+	c, b, _ := testCoreSystemBackendRaw(t)
 
 	// set the policy!
 	p := &Policy{Name: "test"}
@@ -1589,6 +1590,25 @@ func TestSystemBackend_rotate(t *testing.T) {
 
 func testSystemBackend(t *testing.T) logical.Backend {
 	c, _, _ := TestCoreUnsealed(t)
+	return testSystemBackendInternal(t, c)
+}
+
+func testSystemBackendRaw(t *testing.T) logical.Backend {
+	c, _, _ := TestCoreUnsealedRaw(t)
+	return testSystemBackendInternal(t, c)
+}
+
+func testCoreSystemBackend(t *testing.T) (*Core, logical.Backend, string) {
+	c, _, root := TestCoreUnsealed(t)
+	return c, testSystemBackendInternal(t, c), root
+}
+
+func testCoreSystemBackendRaw(t *testing.T) (*Core, logical.Backend, string) {
+	c, _, root := TestCoreUnsealedRaw(t)
+	return c, testSystemBackendInternal(t, c), root
+}
+
+func testSystemBackendInternal(t *testing.T, c *Core) logical.Backend {
 	bc := &logical.BackendConfig{
 		Logger: c.logger,
 		System: logical.StaticSystemView{
@@ -1606,24 +1626,6 @@ func testSystemBackend(t *testing.T) logical.Backend {
 	return b
 }
 
-func testCoreSystemBackend(t *testing.T) (*Core, logical.Backend, string) {
-	c, _, root := TestCoreUnsealed(t)
-	bc := &logical.BackendConfig{
-		Logger: c.logger,
-		System: logical.StaticSystemView{
-			DefaultLeaseTTLVal: time.Hour * 24,
-			MaxLeaseTTLVal:     time.Hour * 24 * 32,
-		},
-	}
-
-	b := NewSystemBackend(c)
-	err := b.Backend.Setup(bc)
-	if err != nil {
-		t.Fatal(err)
-	}
-	return c, b, root
-}
-
 func TestSystemBackend_PluginCatalog_CRUD(t *testing.T) {
 	c, b, _ := testCoreSystemBackend(t)
 	// Bootstrap the pluginCatalog

vault/seal_testing.go

@@ -107,7 +107,7 @@ func (d *TestSeal) SetRecoveryKey(key []byte) error {
 
 func testCoreUnsealedWithConfigs(t *testing.T, barrierConf, recoveryConf *SealConfig) (*Core, [][]byte, [][]byte, string) {
 	seal := &TestSeal{}
-	core := TestCoreWithSeal(t, seal)
+	core := TestCoreWithSeal(t, seal, false)
 	result, err := core.Initialize(&InitParams{
 		BarrierConfig:  barrierConf,
 		RecoveryConfig: recoveryConf,

vault/testing.go

@@ -85,18 +85,24 @@ oOyBJU/HMVvBfv4g+OVFLVgSwwm6owwsouZ0+D/LasbuHqYyqYqdyPJQYzWA2Y+F
 
 // TestCore returns a pure in-memory, uninitialized core for testing.
 func TestCore(t testing.T) *Core {
-	return TestCoreWithSeal(t, nil)
+	return TestCoreWithSeal(t, nil, false)
+}
+
+// TestCoreRaw returns a pure in-memory, uninitialized core for testing. The raw
+// storage endpoints are enabled with this core.
+func TestCoreRaw(t testing.T) *Core {
+	return TestCoreWithSeal(t, nil, true)
 }
 
 // TestCoreNewSeal returns a pure in-memory, uninitialized core with
 // the new seal configuration.
 func TestCoreNewSeal(t testing.T) *Core {
-	return TestCoreWithSeal(t, &TestSeal{})
+	return TestCoreWithSeal(t, &TestSeal{}, false)
 }
 
 // TestCoreWithSeal returns a pure in-memory, uninitialized core with the
 // specified seal for testing.
-func TestCoreWithSeal(t testing.T, testSeal Seal) *Core {
+func TestCoreWithSeal(t testing.T, testSeal Seal, enableRaw bool) *Core {
 	logger := logformat.NewVaultLogger(log.LevelTrace)
 	physicalBackend, err := physInmem.NewInmem(nil, logger)
 	if err != nil {
@@ -105,6 +111,10 @@ func TestCoreWithSeal(t testing.T, testSeal Seal) *Core {
 
 	conf := testCoreConfig(t, physicalBackend, logger)
 
+	if enableRaw {
+		conf.EnableRaw = true
+	}
+
 	if testSeal != nil {
 		conf.Seal = testSeal
 	}
@@ -198,6 +208,17 @@ func TestCoreUnseal(core *Core, key []byte) (bool, error) {
 // initialized and unsealed.
 func TestCoreUnsealed(t testing.T) (*Core, [][]byte, string) {
 	core := TestCore(t)
+	return testCoreUnsealed(t, core)
+}
+
+// TestCoreUnsealedRaw returns a pure in-memory core that is already
+// initialized, unsealed, and with raw endpoints enabled.
+func TestCoreUnsealedRaw(t testing.T) (*Core, [][]byte, string) {
+	core := TestCoreRaw(t)
+	return testCoreUnsealed(t, core)
+}
+
+func testCoreUnsealed(t testing.T, core *Core) (*Core, [][]byte, string) {
 	keys, token := TestCoreInit(t, core)
 	for _, key := range keys {
 		if _, err := TestCoreUnseal(core, TestKeyCopy(key)); err != nil {

website/source/api/system/raw.html.md

@@ -10,6 +10,10 @@ description: |-
 
 The `/sys/raw` endpoint is access the raw underlying store in Vault.
 
+This endpont is off by default.  See the 
+[Vault configuration documentation](/docs/configuration/index.html) to
+enable.
+
 ## Read Raw
 
 This endpoint reads the value of the key at the given path. This is the raw path
@@ -76,6 +80,41 @@ $ curl \
     https://vault.rocks/v1/sys/raw/secret/foo
 ```
 
+## List Raw
+
+This endpoint returns a list keys for a given path prefix.
+
+**This endpoint requires 'sudo' capability.**
+
+| Method   | Path                         | Produces               |
+| :------- | :--------------------------- | :--------------------- |
+| `LIST`   | `/sys/raw/:prefix` | `200 application/json` |
+| `GET`   | `/sys/raw/:prefix?list=true` | `200 application/json` |
+
+
+### Sample Request
+
+```
+$ curl \
+    --header "X-Vault-Token: ..." \
+    --request LIST \
+    https://vault.rocks/v1/sys/raw/logical
+```
+
+### Sample Response
+
+```json
+{
+  "data":{
+    "keys":[
+      "abcd-1234...",
+      "efgh-1234...",
+      "ijkl-1234..."
+    ]
+  }
+}
+```
+
 ## Delete Raw
 
 This endpoint deletes the key with given path. This is the raw path in the

website/source/docs/configuration/index.html.md

@@ -100,6 +100,10 @@ to specify where the configuration is.
   duration for tokens and secrets. This is specified using a label
   suffix like `"30s"` or `"1h"`.
 
+- `raw_storage_endpoint` `(bool: false)` – Enables the `sys/raw` endpoint which 
+  allows the decryption/encryption of raw data into and out of the security 
+  barrier. This is a highly priveleged endpoint. 
+
 - `ui` `(bool: false, Enterprise-only)` – Enables the built-in web UI, which is
   available on all listeners (address + port) at the `/ui` path. Browsers accessing
   the standard Vault API address will automatically redirect there. This can also