Receiving a SIGUSR2 makes Vault log the running goroutines' stacks. (#6240)

* Receiving a SIGUSR2 makes Vault log the running goroutines' stacks.

command/commands.go

@@ -466,6 +466,7 @@ func initCommands(ui, serverCmdUi cli.Ui, runOpts *RunOptions) {
 				PhysicalBackends:   physicalBackends,
 				ShutdownCh:         MakeShutdownCh(),
 				SighupCh:           MakeSighupCh(),
+				SigUSR2Ch:          MakeSigUSR2Ch(),
 			}, nil
 		},
 		"ssh": func() (cli.Command, error) {
@@ -628,3 +629,20 @@ func MakeSighupCh() chan struct{} {
 	}()
 	return resultCh
 }
+
+// MakeSigUSR2Ch returns a channel that can be used for SIGUSR2
+// goroutine logging. This channel will send a message for every
+// SIGUSR2 received.
+func MakeSigUSR2Ch() chan struct{} {
+	resultCh := make(chan struct{})
+
+	signalCh := make(chan os.Signal, 4)
+	signal.Notify(signalCh, syscall.SIGUSR2)
+	go func() {
+		for {
+			<-signalCh
+			resultCh <- struct{}{}
+		}
+	}()
+	return resultCh
+}

command/server.go

@@ -70,6 +70,7 @@ type ServerCommand struct {
 
 	ShutdownCh chan struct{}
 	SighupCh   chan struct{}
+	SigUSR2Ch  chan struct{}
 
 	WaitGroup *sync.WaitGroup
 
@@ -1262,6 +1263,11 @@ CLUSTER_SYNTHESIS_COMPLETE:
 			if err := c.Reload(c.reloadFuncsLock, c.reloadFuncs, c.flagConfigs); err != nil {
 				c.UI.Error(fmt.Sprintf("Error(s) were encountered during reload: %s", err))
 			}
+
+		case <-c.SigUSR2Ch:
+			buf := make([]byte, 32*1024*1024)
+			n := runtime.Stack(buf[:], true)
+			c.logger.Info("goroutine trace", "stack", string(buf[:n]))
 		}
 	}
 

command/server_test.go

@@ -90,6 +90,7 @@ func testServerCommand(tb testing.TB) (*cli.MockUi, *ServerCommand) {
 		},
 		ShutdownCh: MakeShutdownCh(),
 		SighupCh:   MakeSighupCh(),
+		SigUSR2Ch:  MakeSigUSR2Ch(),
 		PhysicalBackends: map[string]physical.Factory{
 			"inmem":    physInmem.NewInmem,
 			"inmem_ha": physInmem.NewInmemHA,