Log when MAP_POPULATE gets disabled (#28526)

* add warning for when MAP_POPULATE mmap flag not set

* Make mmap flags method handle any flags, where MAP_POPULATE is just one of them

* Only have the log print out on restores

* Add test, make logic more consistent

* Add changelog

* Add godoc for test

* Make test less dangerous

changelog/28526.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+physical/raft: Log when the MAP_POPULATE mmap flag gets disabled before opening the database.
+```

physical/raft/bolt_64bit_test.go

@@ -43,3 +43,29 @@ func Test_BoltOptions(t *testing.T) {
 		})
 	}
 }
+
+// TestMmapFlags tests the getMmapFlags function, ensuring it returns the appropriate integer representing the desired mmap flag.
+func TestMmapFlags(t *testing.T) {
+	testCases := []struct {
+		name               string
+		disableMapPopulate bool
+	}{
+		{"MAP_POPULATE is enabled", false},
+		{"MAP_POPULATE disabled by env var", true},
+	}
+
+	for _, tc := range testCases {
+		tc := tc
+
+		t.Run(tc.name, func(t *testing.T) {
+			if tc.disableMapPopulate {
+				t.Setenv("VAULT_RAFT_DISABLE_MAP_POPULATE", "true")
+			}
+
+			isEnabled := usingMapPopulate(getMmapFlags(""))
+			if tc.disableMapPopulate && isEnabled {
+				t.Error("expected MAP_POPULATE to be disabled but it was enabled")
+			}
+		})
+	}
+}

physical/raft/bolt_linux.go

@@ -13,27 +13,49 @@ import (
 
 func init() {
 	getMmapFlags = getMmapFlagsLinux
+	usingMapPopulate = usingMapPopulateLinux
 }
 
 func getMmapFlagsLinux(dbPath string) int {
-	if os.Getenv("VAULT_RAFT_DISABLE_MAP_POPULATE") != "" {
-		return 0
-	}
-	stat, err := os.Stat(dbPath)
-	if err != nil {
-		return 0
-	}
-	size := stat.Size()
-
-	v, err := mem.VirtualMemoryWithContext(context.Background())
-	if err != nil {
-		return 0
-	}
-
-	// We won't worry about swap, since we already tell people not to use it.
-	if v.Total > uint64(size) {
+	if setMapPopulateFlag(dbPath) {
 		return unix.MAP_POPULATE
 	}
 
 	return 0
 }
+
+// setMapPopulateFlag determines whether we should set the MAP_POPULATE flag, which
+// prepopulates page tables to be mapped in the virtual memory space,
+// helping reduce slowness at runtime caused by page faults.
+// We only want to set this flag if we've determined there's enough memory on the system available to do so.
+func setMapPopulateFlag(dbPath string) bool {
+	if os.Getenv("VAULT_RAFT_DISABLE_MAP_POPULATE") != "" {
+		return false
+	}
+	stat, err := os.Stat(dbPath)
+	if err != nil {
+		return false
+	}
+	size := stat.Size()
+
+	v, err := mem.VirtualMemoryWithContext(context.Background())
+	if err != nil {
+		return false
+	}
+
+	// We won't worry about swap, since we already tell people not to use it.
+	if v.Total > uint64(size) {
+		return true
+	}
+
+	return false
+}
+
+// the unix.MAP_POPULATE constant only exists on Linux,
+// so reference to this constant can only live in a *_linux.go file
+func usingMapPopulateLinux(mmapFlag int) bool {
+	if mmapFlag == unix.MAP_POPULATE {
+		return true
+	}
+	return false
+}

physical/raft/fsm.go

@@ -13,6 +13,7 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
 	"sync"
@@ -246,9 +247,11 @@ func (f *FSM) openDBFile(dbPath string) error {
 		return errors.New("can not open empty filename")
 	}
 
+	vaultDbExists := true
 	st, err := os.Stat(dbPath)
 	switch {
 	case err != nil && os.IsNotExist(err):
+		vaultDbExists = false
 	case err != nil:
 		return fmt.Errorf("error checking raft FSM db file %q: %v", dbPath, err)
 	default:
@@ -260,11 +263,16 @@ func (f *FSM) openDBFile(dbPath string) error {
 	}
 
 	opts := boltOptions(dbPath)
+	if runtime.GOOS == "linux" && vaultDbExists && !usingMapPopulate(opts.MmapFlags) {
+		f.logger.Warn("the MAP_POPULATE mmap flag has not been set before opening the FSM database. This may be due to the database file being larger than the available memory on the system, or due to the VAULT_RAFT_DISABLE_MAP_POPULATE environment variable being set. As a result, Vault may be slower to start up.")
+	}
+
 	start := time.Now()
 	boltDB, err := bolt.Open(dbPath, 0o600, opts)
 	if err != nil {
 		return err
 	}
+
 	elapsed := time.Now().Sub(start)
 	f.logger.Debug("time to open database", "elapsed", elapsed, "path", dbPath)
 	metrics.MeasureSince([]string{"raft_storage", "fsm", "open_db_file"}, start)

physical/raft/raft.go

@@ -14,6 +14,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"sync"
 	"sync/atomic"
@@ -74,7 +75,10 @@ const (
 	defaultMaxBatchSize = 128 * 1024
 )
 
-var getMmapFlags = func(string) int { return 0 }
+var (
+	getMmapFlags     = func(string) int { return 0 }
+	usingMapPopulate = func(int) bool { return false }
+)
 
 // Verify RaftBackend satisfies the correct interfaces
 var (
@@ -447,6 +451,7 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend
 		}
 	}
 
+	// Create the log store.
 	// Build an all in-memory setup for dev mode, otherwise prepare a full
 	// disk-based setup.
 	var logStore raft.LogStore
@@ -473,6 +478,7 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend
 		if err != nil {
 			return nil, fmt.Errorf("failed to check if raft.db already exists: %w", err)
 		}
+
 		if backendConfig.RaftWal && raftDbExists {
 			logger.Warn("raft is configured to use raft-wal for storage but existing raft.db detected. raft-wal config will be ignored.")
 			backendConfig.RaftWal = false
@@ -504,6 +510,10 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend
 				MsgpackUseNewTimeFormat: true,
 			}
 
+			if runtime.GOOS == "linux" && raftDbExists && !usingMapPopulate(opts.MmapFlags) {
+				logger.Warn("the MAP_POPULATE mmap flag has not been set before opening the log store database. This may be due to the database file being larger than the available memory on the system, or due to the VAULT_RAFT_DISABLE_MAP_POPULATE environment variable being set. As a result, Vault may be slower to start up.")
+			}
+
 			store, err := raftboltdb.New(raftOptions)
 			if err != nil {
 				return nil, err