Add ulimit check, and tidy unit test cases (#11678)

* Add ulimit check, and tidy unit test cases to avoid needing to have all results and perfect ordering

* Make order independent check recursive

* Fix unit tests

* Try a 5s request timeout

command/operator_diagnose.go

@@ -210,6 +210,9 @@ func (c *OperatorDiagnoseCommand) offlineDiagnostics(ctx context.Context) error
 	ctx, span := diagnose.StartSpan(ctx, "initialization")
 	defer span.End()
 
+	// OS Specific checks
+	// Check open file count
+	diagnose.OSChecks(ctx)
 	diagnose.Test(ctx, "disk-usage", diagnose.DiskUsageCheck)
 
 	server.flagConfigs = c.flagConfigs

command/operator_diagnose_test.go

@@ -39,6 +39,31 @@ func TestOperatorDiagnoseCommand_Run(t *testing.T) {
 				"-config", "./server/test-fixtures/config_diagnose_ok.hcl",
 			},
 			[]*diagnose.Result{
+				{
+					Name:   "open file limits",
+					Status: diagnose.OkStatus,
+				},
+				{
+					Name:   "parse-config",
+					Status: diagnose.OkStatus,
+				},
+				{
+					Name:   "init-listeners",
+					Status: diagnose.WarningStatus,
+					Children: []*diagnose.Result{
+						{
+							Name:   "create-listeners",
+							Status: diagnose.OkStatus,
+						},
+						{
+							Name:   "check-listener-tls",
+							Status: diagnose.WarningStatus,
+							Warnings: []string{
+								"TLS is disabled in a Listener config stanza.",
+							},
+						},
+					},
+				},
 				{
 					Name:   "storage",
 					Status: diagnose.OkStatus,
@@ -314,6 +339,10 @@ func compareResult(exp *diagnose.Result, act *diagnose.Result) error {
 		return fmt.Errorf(strings.Join(errStrings, ","))
 	}
 
+	if len(exp.Children) > 0 {
+		return compareResults(exp.Children, act.Children)
+	}
+
 	if len(exp.Children) > 0 {
 		return compareResults(exp.Children, act.Children)
 	}

vault/diagnose/os_unix.go

@@ -0,0 +1,26 @@
+// +build !windows
+
+package diagnose
+
+import (
+	"context"
+	"fmt"
+	"golang.org/x/sys/unix"
+)
+
+func OSChecks(ctx context.Context) {
+	var limit unix.Rlimit
+	if err := unix.Getrlimit(unix.RLIMIT_NOFILE, &limit); err != nil {
+		SpotError(ctx, "open file limits", fmt.Errorf("could not determine open file limit: %w", err))
+	} else {
+		min := limit.Cur
+		if limit.Max < min {
+			min = limit.Max
+		}
+		if min <= 1024 {
+			SpotWarn(ctx, "open file limits", fmt.Sprintf("set to %d, which may be insufficient.", min))
+		} else {
+			SpotOk(ctx, "open file limits", fmt.Sprintf("set to %d", min))
+		}
+	}
+}

vault/diagnose/os_windows.go

@@ -0,0 +1,11 @@
+// +build windows
+
+package diagnose
+
+import (
+	"context"
+)
+
+func OSChecks(ctx context.Context) {
+	// None so far
+}