Add path based primary write forwarding (PBPWF) - OSS (#18735)

* Add WriteForwardedStorage to sdk's plugin, logical in OSS

This should allow backends to specify paths to forward write
(storage.Put(...) and storage.Delete(...)) operations for.

Notably, these semantics are subject to change and shouldn't yet be
relied on.

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Collect paths for write forwarding in OSS

This adds a path manager to Core, allowing tracking across all Vault
versions of paths which could use write forwarding if available. In
particular, even on OSS offerings, we'll need to template {{clusterId}}
into the paths, in the event of later upgrading to Enterprise. If we
didn't, we'd end up writing paths which will no longer be accessible
post-migration, due to write forwarding now replacing the sentinel with
the actual cluster identifier.

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Add forwarded writer implementation to OSS

Here, for paths given to us, we determine if we need to do cluster
translation and perform local writing. This is the OSS variant.

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Wire up mount-specific request forwarding in OSS

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Clarify that state lock needs to be held to call HAState in OSS

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Move cluster sentinel constant to sdk/logical

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Expose ClusterID to Plugins via SystemView

This will let plugins learn what the Cluster's ID is, without having to
resort to hacks like writing a random string to its cluster-prefixed
namespace and then reading it once it has replicated.

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

* Add GRPC ClusterID implementation

For any external plugins which wish to use it.

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

Signed-off-by: Alexander Scheel <alex.scheel@hashicorp.com>

sdk/logical/logical.go

@@ -131,6 +131,28 @@ type Paths struct {
 	// should be seal wrapped with extra encryption. It is exact matching
 	// unless it ends with '/' in which case it will be treated as a prefix.
 	SealWrapStorage []string
+
+	// WriteForwardedStorage are storage paths that, when running on a PR
+	// Secondary cluster, cause a GRPC call up to the PR Primary cluster's
+	// active node to handle storage.Put(...) and storage.Delete(...) events.
+	// These paths MUST include a {{clusterId}} literal, which the write layer
+	// will resolve to this cluster's UUID ("replication set" identifier).
+	// storage.List(...) and storage.Get(...) operations occur from the
+	// locally replicated data set, but can use path template expansion to be
+	// identifier agnostic.
+	//
+	// These paths require careful considerations by developers to use. In
+	// particular, writes on secondary clusters will not appear (when a
+	// corresponding read is issued immediately after a write) until the
+	// replication from primary->secondary has occurred. This replication
+	// triggers an InvalidateKey(...) call on the secondary, which can be
+	// used to detect the write has finished syncing. However, this will
+	// likely occur after the request has finished, so it is important to
+	// not block on this occurring.
+	//
+	// On standby nodes, like all storage write operations, this will trigger
+	// an ErrReadOnly return.
+	WriteForwardedStorage []string
 }
 
 type Auditor interface {

sdk/logical/storage.go

@@ -20,6 +20,11 @@ var ErrReadOnly = errors.New("cannot write to readonly storage")
 // storage while the backend is still being setup.
 var ErrSetupReadOnly = errors.New("cannot write to storage during setup")
 
+// Plugins using Paths.WriteForwardedStorage will need to use this sentinel
+// in their path to write cross-cluster. See the description of that parameter
+// for more information.
+const PBPWFClusterSentinel = "{{clusterId}}"
+
 // Storage is the way that logical backends are able read/write data.
 type Storage interface {
 	List(context.Context, string) ([]string, error)

sdk/logical/system_view.go

@@ -89,6 +89,11 @@ type SystemView interface {
 	// GeneratePasswordFromPolicy generates a password from the policy referenced.
 	// If the policy does not exist, this will return an error.
 	GeneratePasswordFromPolicy(ctx context.Context, policyName string) (password string, err error)
+
+	// ClusterID returns the replication ClusterID, for use with path-based
+	// write forwarding (WriteForwardedPaths). This value will be templated
+	// in for the {{cluterId}} sentinel.
+	ClusterID(ctx context.Context) (string, error)
 }
 
 type PasswordPolicy interface {
@@ -119,6 +124,7 @@ type StaticSystemView struct {
 	PluginEnvironment   *PluginEnvironment
 	PasswordPolicies    map[string]PasswordGenerator
 	VersionString       string
+	ClusterUUID         string
 }
 
 type noopAuditor struct{}
@@ -240,3 +246,7 @@ func (d *StaticSystemView) DeletePasswordPolicy(name string) (existed bool) {
 	delete(d.PasswordPolicies, name)
 	return existed
 }
+
+func (d StaticSystemView) ClusterID(ctx context.Context) (string, error) {
+	return d.ClusterUUID, nil
+}

sdk/plugin/grpc_backend_client.go

@@ -124,10 +124,11 @@ func (b *backendGRPCPluginClient) SpecialPaths() *logical.Paths {
 	}
 
 	return &logical.Paths{
-		Root:            reply.Paths.Root,
-		Unauthenticated: reply.Paths.Unauthenticated,
-		LocalStorage:    reply.Paths.LocalStorage,
-		SealWrapStorage: reply.Paths.SealWrapStorage,
+		Root:                  reply.Paths.Root,
+		Unauthenticated:       reply.Paths.Unauthenticated,
+		LocalStorage:          reply.Paths.LocalStorage,
+		SealWrapStorage:       reply.Paths.SealWrapStorage,
+		WriteForwardedStorage: reply.Paths.WriteForwardedStorage,
 	}
 }
 

sdk/plugin/grpc_backend_server.go

@@ -186,10 +186,11 @@ func (b *backendGRPCPluginServer) SpecialPaths(ctx context.Context, args *pb.Emp
 
 	return &pb.SpecialPathsReply{
 		Paths: &pb.Paths{
-			Root:            paths.Root,
-			Unauthenticated: paths.Unauthenticated,
-			LocalStorage:    paths.LocalStorage,
-			SealWrapStorage: paths.SealWrapStorage,
+			Root:                  paths.Root,
+			Unauthenticated:       paths.Unauthenticated,
+			LocalStorage:          paths.LocalStorage,
+			SealWrapStorage:       paths.SealWrapStorage,
+			WriteForwardedStorage: paths.WriteForwardedStorage,
 		},
 	}, nil
 }

sdk/plugin/grpc_system.go

@@ -199,6 +199,15 @@ func (s *gRPCSystemViewClient) GeneratePasswordFromPolicy(ctx context.Context, p
 	return resp.Password, nil
 }
 
+func (s gRPCSystemViewClient) ClusterID(ctx context.Context) (string, error) {
+	reply, err := s.client.ClusterInfo(ctx, &pb.Empty{})
+	if err != nil {
+		return "", err
+	}
+
+	return reply.ClusterID, nil
+}
+
 type gRPCSystemViewServer struct {
 	pb.UnimplementedSystemViewServer
 
@@ -367,3 +376,18 @@ func (s *gRPCSystemViewServer) GeneratePasswordFromPolicy(ctx context.Context, r
 	}
 	return resp, nil
 }
+
+func (s *gRPCSystemViewServer) ClusterInfo(ctx context.Context, _ *pb.Empty) (*pb.ClusterInfoReply, error) {
+	if s.impl == nil {
+		return nil, errMissingSystemView
+	}
+
+	clusterId, err := s.impl.ClusterID(ctx)
+	if err != nil {
+		return &pb.ClusterInfoReply{}, status.Errorf(codes.Internal, "failed to fetch cluster id")
+	}
+
+	return &pb.ClusterInfoReply{
+		ClusterID: clusterId,
+	}, nil
+}

sdk/plugin/pb/backend.proto

@@ -47,6 +47,13 @@ message Paths {
 	// should be seal wrapped with extra encryption. It is exact matching
 	// unless it ends with '/' in which case it will be treated as a prefix.
 	repeated string seal_wrap_storage = 4;
+
+	// WriteForwardedStorage are storage paths that, when running on a PR
+	// Secondary cluster, cause a GRPC call up to the PR Primary cluster's
+	// active node to handle storage.Put(...) and storage.Delete(...) events.
+	//
+	// See extended note in /sdk/logical/logical.go.
+	repeated string write_forwarded_storage = 5;
 }
 
 message Request {
@@ -566,6 +573,12 @@ message GeneratePasswordFromPolicyReply {
 	string password = 1;
 }
 
+message ClusterInfoReply {
+	string cluster_name = 1;
+	string cluster_id = 2;
+	string err = 3;
+}
+
 // SystemView exposes system configuration information in a safe way for plugins
 // to consume. Plugins should implement the client for this service.
 service SystemView {
@@ -618,6 +631,9 @@ service SystemView {
 
 	// GeneratePasswordFromPolicy generates a password from an existing password policy
 	rpc GeneratePasswordFromPolicy(GeneratePasswordFromPolicyRequest) returns (GeneratePasswordFromPolicyReply);
+
+	// ClusterInfo returns the ClusterID information; may be reused if ClusterName is also exposed.
+	rpc ClusterInfo(Empty) returns (ClusterInfoReply);
 }
 
 message Connection {

sdk/plugin/pb/backend_grpc.pb.go

@@ -642,6 +642,8 @@ type SystemViewClient interface {
 	GroupsForEntity(ctx context.Context, in *EntityInfoArgs, opts ...grpc.CallOption) (*GroupsForEntityReply, error)
 	// GeneratePasswordFromPolicy generates a password from an existing password policy
 	GeneratePasswordFromPolicy(ctx context.Context, in *GeneratePasswordFromPolicyRequest, opts ...grpc.CallOption) (*GeneratePasswordFromPolicyReply, error)
+	// ClusterInfo returns the ClusterID information; may be reused if ClusterName is also exposed.
+	ClusterInfo(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ClusterInfoReply, error)
 }
 
 type systemViewClient struct {
@@ -760,6 +762,15 @@ func (c *systemViewClient) GeneratePasswordFromPolicy(ctx context.Context, in *G
 	return out, nil
 }
 
+func (c *systemViewClient) ClusterInfo(ctx context.Context, in *Empty, opts ...grpc.CallOption) (*ClusterInfoReply, error) {
+	out := new(ClusterInfoReply)
+	err := c.cc.Invoke(ctx, "/pb.SystemView/ClusterInfo", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
 // SystemViewServer is the server API for SystemView service.
 // All implementations must embed UnimplementedSystemViewServer
 // for forward compatibility
@@ -802,6 +813,8 @@ type SystemViewServer interface {
 	GroupsForEntity(context.Context, *EntityInfoArgs) (*GroupsForEntityReply, error)
 	// GeneratePasswordFromPolicy generates a password from an existing password policy
 	GeneratePasswordFromPolicy(context.Context, *GeneratePasswordFromPolicyRequest) (*GeneratePasswordFromPolicyReply, error)
+	// ClusterInfo returns the ClusterID information; may be reused if ClusterName is also exposed.
+	ClusterInfo(context.Context, *Empty) (*ClusterInfoReply, error)
 	mustEmbedUnimplementedSystemViewServer()
 }
 
@@ -845,6 +858,9 @@ func (UnimplementedSystemViewServer) GroupsForEntity(context.Context, *EntityInf
 func (UnimplementedSystemViewServer) GeneratePasswordFromPolicy(context.Context, *GeneratePasswordFromPolicyRequest) (*GeneratePasswordFromPolicyReply, error) {
 	return nil, status.Errorf(codes.Unimplemented, "method GeneratePasswordFromPolicy not implemented")
 }
+func (UnimplementedSystemViewServer) ClusterInfo(context.Context, *Empty) (*ClusterInfoReply, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method ClusterInfo not implemented")
+}
 func (UnimplementedSystemViewServer) mustEmbedUnimplementedSystemViewServer() {}
 
 // UnsafeSystemViewServer may be embedded to opt out of forward compatibility for this service.
@@ -1074,6 +1090,24 @@ func _SystemView_GeneratePasswordFromPolicy_Handler(srv interface{}, ctx context
 	return interceptor(ctx, in, info, handler)
 }
 
+func _SystemView_ClusterInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(Empty)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(SystemViewServer).ClusterInfo(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/pb.SystemView/ClusterInfo",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(SystemViewServer).ClusterInfo(ctx, req.(*Empty))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
 // SystemView_ServiceDesc is the grpc.ServiceDesc for SystemView service.
 // It's only intended for direct use with grpc.RegisterService,
 // and not to be introspected or modified (even as a copy)
@@ -1129,6 +1163,10 @@ var SystemView_ServiceDesc = grpc.ServiceDesc{
 			MethodName: "GeneratePasswordFromPolicy",
 			Handler:    _SystemView_GeneratePasswordFromPolicy_Handler,
 		},
+		{
+			MethodName: "ClusterInfo",
+			Handler:    _SystemView_ClusterInfo_Handler,
+		},
 	},
 	Streams:  []grpc.StreamDesc{},
 	Metadata: "sdk/plugin/pb/backend.proto",