Enable undo logs by default (#18692)

* Enable undo logs by default

* add consul test

* update go.mod/sum

* add a better non-existent key

go.mod

@@ -29,7 +29,7 @@ require (
 	github.com/aliyun/alibaba-cloud-sdk-go v1.61.1842
 	github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5
 	github.com/apple/foundationdb/bindings/go v0.0.0-20190411004307-cd5c9d91fad2
-	github.com/armon/go-metrics v0.4.0
+	github.com/armon/go-metrics v0.4.1
 	github.com/armon/go-radix v1.0.0
 	github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef
 	github.com/aws/aws-sdk-go v1.44.128
@@ -62,7 +62,7 @@ require (
 	github.com/google/tink/go v1.6.1
 	github.com/hashicorp/cap v0.2.1-0.20220727210936-60cd1534e220
 	github.com/hashicorp/consul-template v0.29.5
-	github.com/hashicorp/consul/api v1.15.2
+	github.com/hashicorp/consul/api v1.17.0
 	github.com/hashicorp/errwrap v1.1.0
 	github.com/hashicorp/eventlogger v0.1.0
 	github.com/hashicorp/go-cleanhttp v0.5.2
@@ -148,8 +148,8 @@ require (
 	github.com/joyent/triton-go v1.7.1-0.20200416154420-6801d15b779f
 	github.com/kr/pretty v0.3.0
 	github.com/kr/text v0.2.0
-	github.com/mattn/go-colorable v0.1.12
-	github.com/mattn/go-isatty v0.0.14
+	github.com/mattn/go-colorable v0.1.13
+	github.com/mattn/go-isatty v0.0.17
 	github.com/mholt/archiver/v3 v3.5.1
 	github.com/michaelklishin/rabbit-hole/v2 v2.12.0
 	github.com/mikesmitty/edkey v0.0.0-20170222072505-3356ea4e686a
@@ -196,7 +196,7 @@ require (
 	golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8
 	golang.org/x/net v0.4.0
 	golang.org/x/oauth2 v0.1.0
-	golang.org/x/sys v0.3.0
+	golang.org/x/sys v0.4.0
 	golang.org/x/term v0.3.0
 	golang.org/x/tools v0.1.12
 	google.golang.org/api v0.101.0
@@ -336,7 +336,7 @@ require (
 	github.com/hashicorp/logutils v1.0.0 // indirect
 	github.com/hashicorp/mdns v1.0.4 // indirect
 	github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0 // indirect
-	github.com/hashicorp/serf v0.9.7 // indirect
+	github.com/hashicorp/serf v0.10.1 // indirect
 	github.com/hashicorp/vault/api/auth/kubernetes v0.3.0 // indirect
 	github.com/hashicorp/vic v1.5.1-0.20190403131502-bbfe86ec9443 // indirect
 	github.com/hashicorp/yamux v0.0.0-20211028200310-0bc27b27de87 // indirect

go.sum

@@ -232,8 +232,8 @@ github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQh
 github.com/armon/go-metrics v0.3.0/go.mod h1:zXjbSimjXTd7vOpY8B0/2LpvNvDoXBuplAD+gJD3GYs=
 github.com/armon/go-metrics v0.3.9/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
 github.com/armon/go-metrics v0.3.10/go.mod h1:4O98XIr/9W0sxpJ8UaYkvjk10Iff7SnFrb4QAOwNTFc=
-github.com/armon/go-metrics v0.4.0 h1:yCQqn7dwca4ITXb+CbubHmedzaQYHhNhrEXLYUeEe8Q=
-github.com/armon/go-metrics v0.4.0/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
+github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
+github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
 github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI=
 github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
@@ -958,11 +958,11 @@ github.com/hashicorp/cap v0.2.1-0.20220727210936-60cd1534e220 h1:Vgv3jG0kicczshK
 github.com/hashicorp/cap v0.2.1-0.20220727210936-60cd1534e220/go.mod h1:zb3VvIFA0lM2lbmO69NjowV9dJzJnZS89TaM9blXPJA=
 github.com/hashicorp/consul-template v0.29.5 h1:tzEo93RqODAX2cgOe/ke8xcpdPdxg5rxl6d22wE3f6c=
 github.com/hashicorp/consul-template v0.29.5/go.mod h1:SZGBPz/t0JaBwMOqM6q/mG66cBRA8IeDUjOwjO0Pa5M=
-github.com/hashicorp/consul/api v1.15.2 h1:3Q/pDqvJ7udgt/60QOOW/p/PeKioQN+ncYzzCdN2av0=
-github.com/hashicorp/consul/api v1.15.2/go.mod h1:v6nvB10borjOuIwNRZYPZiHKrTM/AyrGtd0WVVodKM8=
+github.com/hashicorp/consul/api v1.17.0 h1:aqytbw31uCPNn37ST+717IyGod+P1eTgSGu3yjRo4bs=
+github.com/hashicorp/consul/api v1.17.0/go.mod h1:ZNwemOPAdgtV4cCx9fqxNmw+PI3vliW6gYin2WD+F2g=
 github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/consul/sdk v0.11.0 h1:HRzj8YSCln2yGgCumN5CL8lYlD3gBurnervJRJAZyC4=
-github.com/hashicorp/consul/sdk v0.11.0/go.mod h1:yPkX5Q6CsxTFMjQQDJwzeNmUUF5NUGGbrDsv9wTb8cw=
+github.com/hashicorp/consul/sdk v0.13.0 h1:lce3nFlpv8humJL8rNrrGHYSKc3q+Kxfeg3Ii1m6ZWU=
+github.com/hashicorp/consul/sdk v0.13.0/go.mod h1:0hs/l5fOVhJy/VdcoaNqUSi2AUs95eF5WKtv+EYIQqE=
 github.com/hashicorp/cronexpr v1.1.1 h1:NJZDd87hGXjoZBdvyCF9mX4DCq5Wy7+A/w+A7q0wn6c=
 github.com/hashicorp/cronexpr v1.1.1/go.mod h1:P4wA0KBl9C5q2hABiMO7cp6jcIg96CDh1Efb3g1PWA4=
 github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
@@ -1077,6 +1077,7 @@ github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/b
 github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
 github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
 github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
@@ -1100,9 +1101,8 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
 github.com/hashicorp/mdns v1.0.1/go.mod h1:4gW7WsVCke5TE7EPeYliwHlRUyBtfCwuFwuMg2DmyNY=
 github.com/hashicorp/mdns v1.0.4 h1:sY0CMhFmjIPDMlTB+HfymFHCaYLhgifZ0QhjaYKD/UQ=
 github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=
-github.com/hashicorp/memberlist v0.3.0/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
-github.com/hashicorp/memberlist v0.3.1 h1:MXgUXLqva1QvpVEDQW1IQLG0wivQAtmFlHRQ+1vWZfM=
-github.com/hashicorp/memberlist v0.3.1/go.mod h1:MS2lj3INKhZjWNqd3N0m3J+Jxf3DAOnAH9VT3Sh9MUE=
+github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM=
+github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0=
 github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0 h1:kBpVVl1sl3MaSrs97e0+pDQhSrqJv9gVbSUrPpVfl1w=
 github.com/hashicorp/net-rpc-msgpackrpc/v2 v2.0.0/go.mod h1:6pdNz0vo0mF0GvhwDG56O3N18qBrAz/XRIcfINfTbwo=
 github.com/hashicorp/nomad/api v0.0.0-20220707195938-75f4c2237b28 h1:fo8EbQ6tc9hYqxik9CAdFMqy48TW8hh2I3znysPqf+0=
@@ -1120,8 +1120,8 @@ github.com/hashicorp/raft-boltdb/v2 v2.0.0-20210421194847-a7e34179d62c h1:oiKun9
 github.com/hashicorp/raft-boltdb/v2 v2.0.0-20210421194847-a7e34179d62c/go.mod h1:kiPs9g148eLShc2TYagUAyKDnD+dH9U+CQKsXzlY9xo=
 github.com/hashicorp/raft-snapshot v1.0.4 h1:EuDuayAJPdiDmVk1ygTDnG2zDzrs0/6/yBuma1IYSow=
 github.com/hashicorp/raft-snapshot v1.0.4/go.mod h1:5sL9eUn72lH5DzsFIJ9jaysITbHksSSszImWSOTC8Ic=
-github.com/hashicorp/serf v0.9.7 h1:hkdgbqizGQHuU5IPqYM1JdSMV8nKfpuOnZYXssk9muY=
-github.com/hashicorp/serf v0.9.7/go.mod h1:TXZNMjZQijwlDvp+r0b63xZ45H7JmCmgg4gpTwn9UV4=
+github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
+github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
 github.com/hashicorp/vault-plugin-auth-alicloud v0.5.4-beta1.0.20221117202053-722c59caa2d0 h1:f4Ay9naDgZwW77q6Jpiy/zMlXC1MDWV2Kwop6uud3f8=
 github.com/hashicorp/vault-plugin-auth-alicloud v0.5.4-beta1.0.20221117202053-722c59caa2d0/go.mod h1:EjGPliIfEWITTGsi8KD/aZgIActKDfDVwStpqpCtrM0=
 github.com/hashicorp/vault-plugin-auth-azure v0.11.2-0.20221108185759-ac6743d5f0f2 h1:cVT7MJAl5uwXFtLMQBA7DDE5GDLEU+1BE03ew1ygY88=
@@ -1378,8 +1378,9 @@ github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcncea
 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
 github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
 github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
-github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40=
 github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
 github.com/mattn/go-ieproxy v0.0.1 h1:qiyop7gCflfhwCzGyeT0gro3sF9AIg9HU98JORTkqfI=
 github.com/mattn/go-ieproxy v0.0.1/go.mod h1:pYabZ6IHcRpFh7vIaLfK7rdcWgFEb3SFJ6/gNWuh88E=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
@@ -1390,8 +1391,10 @@ github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd
 github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84=
 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
 github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
-github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y=
 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
@@ -2255,8 +2258,10 @@ golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.4.0 h1:Zr2JFtRQNX3BCZ8YtxRE9hNJYC8J6I1MVbMg6owUp18=
+golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210615171337-6886f2dfbf5b/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

physical/consul/consul.go

@@ -7,6 +7,7 @@ import (
 	"net/http"
 	"strconv"
 	"strings"
+	"sync/atomic"
 	"time"
 
 	"github.com/armon/go-metrics"
@@ -29,6 +30,9 @@ const (
 	// consistencyModeStrong is the configuration value used to tell
 	// consul to use strong consistency.
 	consistencyModeStrong = "strong"
+
+	// nonExistentKey is used as part of a capabilities check against Consul
+	nonExistentKey = "F35C28E1-7035-40BB-B865-6BED9E3A1B28"
 )
 
 // Verify ConsulBackend satisfies the correct interfaces
@@ -37,11 +41,14 @@ var (
 	_ physical.HABackend     = (*ConsulBackend)(nil)
 	_ physical.Lock          = (*ConsulLock)(nil)
 	_ physical.Transactional = (*ConsulBackend)(nil)
+
+	GetInTxnDisabledError = errors.New("get operations inside transactions are disabled in consul backend")
 )
 
 // ConsulBackend is a physical backend that stores data at specific
 // prefix within Consul. It is used for most production situations as
 // it allows Vault to run on multiple machines in a highly-available manner.
+// failGetInTxn is only used in tests.
 type ConsulBackend struct {
 	client          *api.Client
 	path            string
@@ -51,6 +58,7 @@ type ConsulBackend struct {
 	consistencyMode string
 	sessionTTL      string
 	lockWaitTime    time.Duration
+	failGetInTxn    *uint32
 }
 
 // NewConsulBackend constructs a Consul backend using the given API client
@@ -147,9 +155,9 @@ func NewConsulBackend(conf map[string]string, logger log.Logger) (physical.Backe
 		txn:             client.Txn(),
 		permitPool:      physical.NewPermitPool(maxParInt),
 		consistencyMode: consistencyMode,
-
-		sessionTTL:   sessionTTL,
-		lockWaitTime: lockWaitTime,
+		sessionTTL:      sessionTTL,
+		lockWaitTime:    lockWaitTime,
+		failGetInTxn:    new(uint32),
 	}
 
 	return c, nil
@@ -224,6 +232,33 @@ func SetupSecureTLS(ctx context.Context, consulConf *api.Config, conf map[string
 	return nil
 }
 
+// ExpandedCapabilitiesAvailable tests to see if Consul has KVGetOrEmpty and 128 entries per transaction available
+func (c *ConsulBackend) ExpandedCapabilitiesAvailable(ctx context.Context) bool {
+	available := false
+
+	maxEntries := 128
+	ops := make([]*api.TxnOp, maxEntries)
+	for i := 0; i < maxEntries; i++ {
+		ops[i] = &api.TxnOp{KV: &api.KVTxnOp{
+			Key:  c.path + nonExistentKey,
+			Verb: api.KVGetOrEmpty,
+		}}
+	}
+
+	c.permitPool.Acquire()
+	defer c.permitPool.Release()
+
+	queryOpts := &api.QueryOptions{}
+	queryOpts = queryOpts.WithContext(ctx)
+
+	ok, resp, _, err := c.txn.Txn(ops, queryOpts)
+	if ok && len(resp.Errors) == 0 && err == nil {
+		available = true
+	}
+
+	return available
+}
+
 // Transaction is used to run multiple entries via a transaction.
 func (c *ConsulBackend) Transaction(ctx context.Context, txns []*physical.TxnEntry) error {
 	if len(txns) == 0 {
@@ -231,6 +266,13 @@ func (c *ConsulBackend) Transaction(ctx context.Context, txns []*physical.TxnEnt
 	}
 	defer metrics.MeasureSince([]string{"consul", "transaction"}, time.Now())
 
+	failGetInTxn := atomic.LoadUint32(c.failGetInTxn)
+	for _, t := range txns {
+		if t.Operation == physical.GetOperation && failGetInTxn != 0 {
+			return GetInTxnDisabledError
+		}
+	}
+
 	ops := make([]*api.TxnOp, 0, len(txns))
 	for _, t := range txns {
 		o, err := c.makeApiTxn(t)
@@ -301,8 +343,7 @@ func (c *ConsulBackend) makeApiTxn(txn *physical.TxnEntry) (*api.TxnOp, error) {
 	}
 	switch txn.Operation {
 	case physical.GetOperation:
-		// TODO: This is currently broken. Once Consul releases 1.14, this should be updated to use api.KVGetOrEmpty
-		op.Verb = api.KVGet
+		op.Verb = api.KVGetOrEmpty
 	case physical.DeleteOperation:
 		op.Verb = api.KVDelete
 	case physical.PutOperation:
@@ -409,6 +450,14 @@ func (c *ConsulBackend) List(ctx context.Context, prefix string) ([]string, erro
 	return out, err
 }
 
+func (c *ConsulBackend) FailGetInTxn(fail bool) {
+	var val uint32
+	if fail {
+		val = 1
+	}
+	atomic.StoreUint32(c.failGetInTxn, val)
+}
+
 // LockWith is used for mutual exclusion based on the given key.
 func (c *ConsulBackend) LockWith(key, value string) (physical.Lock, error) {
 	// Create the lock

physical/consul/consul_test.go

@@ -251,11 +251,41 @@ func TestConsul_TooLarge(t *testing.T) {
 	}
 }
 
-func TestConsul_TransactionalBackend_GetTransactionsForNonExistentValues(t *testing.T) {
-	// TODO: unskip this after Consul releases 1.14 and we update our API dep. It currently fails but should pass with Consul 1.14
-	t.SkipNow()
+func TestConsul_ExpandedCapabilitiesAvailable(t *testing.T) {
+	testCases := map[string]bool{
+		"1.13.5": false,
+		"1.14.3": true,
+	}
 
-	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false, true)
+	for version, shouldBeAvailable := range testCases {
+		t.Run(version, func(t *testing.T) {
+			cleanup, config := consul.PrepareTestContainer(t, version, false, true)
+			defer cleanup()
+
+			logger := logging.NewVaultLogger(log.Debug)
+			backendConfig := map[string]string{
+				"address":      config.Address(),
+				"token":        config.Token,
+				"path":         "vault/",
+				"max_parallel": "-1",
+			}
+
+			be, err := NewConsulBackend(backendConfig, logger)
+			if err != nil {
+				t.Fatal(err)
+			}
+			b := be.(*ConsulBackend)
+
+			isAvailable := b.ExpandedCapabilitiesAvailable(context.Background())
+			if isAvailable != shouldBeAvailable {
+				t.Errorf("%t != %t, version %s\n", isAvailable, shouldBeAvailable, version)
+			}
+		})
+	}
+}
+
+func TestConsul_TransactionalBackend_GetTransactionsForNonExistentValues(t *testing.T) {
+	cleanup, config := consul.PrepareTestContainer(t, "1.14.2", false, true)
 	defer cleanup()
 
 	client, err := api.NewClient(config.APIConfig())
@@ -316,10 +346,7 @@ func TestConsul_TransactionalBackend_GetTransactionsForNonExistentValues(t *test
 // TestConsul_TransactionalBackend_GetTransactions tests that passing a slice of transactions to the
 // consul backend will populate values for any transactions that are Get operations.
 func TestConsul_TransactionalBackend_GetTransactions(t *testing.T) {
-	// TODO: unskip this after Consul releases 1.14 and we update our API dep. It currently fails but should pass with Consul 1.14
-	t.SkipNow()
-
-	cleanup, config := consul.PrepareTestContainer(t, "1.4.4", false, true)
+	cleanup, config := consul.PrepareTestContainer(t, "1.14.2", false, true)
 	defer cleanup()
 
 	client, err := api.NewClient(config.APIConfig())

physical/raft/chunking_test.go

@@ -3,14 +3,14 @@ package raft
 import (
 	"bytes"
 	"context"
-	fmt "fmt"
+	"fmt"
 	"os"
 	"testing"
 
-	proto "github.com/golang/protobuf/proto"
+	"github.com/golang/protobuf/proto"
 	"github.com/hashicorp/go-raftchunking"
 	raftchunkingtypes "github.com/hashicorp/go-raftchunking/types"
-	uuid "github.com/hashicorp/go-uuid"
+	"github.com/hashicorp/go-uuid"
 	"github.com/hashicorp/raft"
 	"github.com/hashicorp/raft-boltdb/v2"
 	"github.com/hashicorp/vault/sdk/physical"
@@ -26,7 +26,7 @@ func TestRaft_Chunking_Lifecycle(t *testing.T) {
 	require := require.New(t)
 	assert := assert.New(t)
 
-	b, dir := getRaft(t, true, false)
+	b, dir := GetRaft(t, true, false)
 	defer os.RemoveAll(dir)
 
 	t.Log("applying configuration")
@@ -111,7 +111,7 @@ func TestFSM_Chunking_TermChange(t *testing.T) {
 	require := require.New(t)
 	assert := assert.New(t)
 
-	b, dir := getRaft(t, true, false)
+	b, dir := GetRaft(t, true, false)
 	defer os.RemoveAll(dir)
 
 	t.Log("applying configuration")
@@ -185,7 +185,7 @@ func TestFSM_Chunking_TermChange(t *testing.T) {
 func TestRaft_Chunking_AppliedIndex(t *testing.T) {
 	t.Parallel()
 
-	raft, dir := getRaft(t, true, false)
+	raft, dir := GetRaft(t, true, false)
 	defer os.RemoveAll(dir)
 
 	// Lower the size for tests

physical/raft/raft.go

@@ -12,6 +12,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/armon/go-metrics"
@@ -64,12 +65,12 @@ var (
 	// This is used to reduce disk I/O for the recently committed entries.
 	raftLogCacheSize = 512
 
-	raftState     = "raft/"
-	peersFileName = "peers.json"
-
+	raftState              = "raft/"
+	peersFileName          = "peers.json"
 	restoreOpDelayDuration = 5 * time.Second
+	defaultMaxEntrySize    = uint64(2 * raftchunking.ChunkSize)
 
-	defaultMaxEntrySize = uint64(2 * raftchunking.ChunkSize)
+	GetInTxnDisabledError = errors.New("get operations inside transactions are disabled in raft backend")
 )
 
 // RaftBackend implements the backend interfaces and uses the raft protocol to
@@ -181,6 +182,7 @@ type RaftBackend struct {
 	nonVoter bool
 
 	effectiveSDKVersion string
+	failGetInTxn        *uint32
 }
 
 // LeaderJoinInfo contains information required by a node to join itself as a
@@ -515,6 +517,7 @@ func NewRaftBackend(conf map[string]string, logger log.Logger) (physical.Backend
 		redundancyZone:             conf["autopilot_redundancy_zone"],
 		nonVoter:                   nonVoter,
 		upgradeVersion:             upgradeVersion,
+		failGetInTxn:               new(uint32),
 	}, nil
 }
 
@@ -566,6 +569,14 @@ func (b *RaftBackend) Close() error {
 	return nil
 }
 
+func (b *RaftBackend) FailGetInTxn(fail bool) {
+	var val uint32
+	if fail {
+		val = 1
+	}
+	atomic.StoreUint32(b.failGetInTxn, val)
+}
+
 func (b *RaftBackend) SetEffectiveSDKVersion(sdkVersion string) {
 	b.l.Lock()
 	b.effectiveSDKVersion = sdkVersion
@@ -1563,6 +1574,13 @@ func (b *RaftBackend) Transaction(ctx context.Context, txns []*physical.TxnEntry
 		return err
 	}
 
+	failGetInTxn := atomic.LoadUint32(b.failGetInTxn)
+	for _, t := range txns {
+		if t.Operation == physical.GetOperation && failGetInTxn != 0 {
+			return GetInTxnDisabledError
+		}
+	}
+
 	txnMap := make(map[string]*physical.TxnEntry)
 
 	command := &LogData{

physical/raft/raft_test.go

@@ -27,73 +27,6 @@ import (
 	bolt "go.etcd.io/bbolt"
 )
 
-func getRaft(t testing.TB, bootstrap bool, noStoreState bool) (*RaftBackend, string) {
-	raftDir, err := ioutil.TempDir("", "vault-raft-")
-	if err != nil {
-		t.Fatal(err)
-	}
-	t.Logf("raft dir: %s", raftDir)
-
-	return getRaftWithDir(t, bootstrap, noStoreState, raftDir)
-}
-
-func getRaftWithDir(t testing.TB, bootstrap bool, noStoreState bool, raftDir string) (*RaftBackend, string) {
-	id, err := uuid.GenerateUUID()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	logger := hclog.New(&hclog.LoggerOptions{
-		Name:  fmt.Sprintf("raft-%s", id),
-		Level: hclog.Trace,
-	})
-	logger.Info("raft dir", "dir", raftDir)
-
-	conf := map[string]string{
-		"path":          raftDir,
-		"trailing_logs": "100",
-		"node_id":       id,
-	}
-
-	if noStoreState {
-		conf["doNotStoreLatestState"] = ""
-	}
-
-	backendRaw, err := NewRaftBackend(conf, logger)
-	if err != nil {
-		t.Fatal(err)
-	}
-	backend := backendRaw.(*RaftBackend)
-
-	if bootstrap {
-		err = backend.Bootstrap([]Peer{
-			{
-				ID:      backend.NodeID(),
-				Address: backend.NodeID(),
-			},
-		})
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		err = backend.SetupCluster(context.Background(), SetupOpts{})
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		for {
-			if backend.raft.AppliedIndex() >= 2 {
-				break
-			}
-		}
-
-	}
-
-	backend.DisableAutopilot()
-
-	return backend, raftDir
-}
-
 func connectPeers(nodes ...*RaftBackend) {
 	for _, node := range nodes {
 		for _, peer := range nodes {
@@ -220,7 +153,7 @@ func compareDBs(t *testing.T, boltDB1, boltDB2 *bolt.DB, dataOnly bool) error {
 }
 
 func TestRaft_Backend(t *testing.T) {
-	b, dir := getRaft(t, true, true)
+	b, dir := GetRaft(t, true, true)
 	defer os.RemoveAll(dir)
 
 	physical.ExerciseBackend(t, b)
@@ -316,7 +249,7 @@ func TestRaft_ParseNonVoter(t *testing.T) {
 }
 
 func TestRaft_Backend_LargeKey(t *testing.T) {
-	b, dir := getRaft(t, true, true)
+	b, dir := GetRaft(t, true, true)
 	defer os.RemoveAll(dir)
 
 	key, err := base62.Random(bolt.MaxKeySize + 1)
@@ -344,7 +277,7 @@ func TestRaft_Backend_LargeKey(t *testing.T) {
 }
 
 func TestRaft_Backend_LargeValue(t *testing.T) {
-	b, dir := getRaft(t, true, true)
+	b, dir := GetRaft(t, true, true)
 	defer os.RemoveAll(dir)
 
 	value := make([]byte, defaultMaxEntrySize+1)
@@ -372,7 +305,7 @@ func TestRaft_Backend_LargeValue(t *testing.T) {
 // TestRaft_TransactionalBackend_GetTransactions tests that passing a slice of transactions to the
 // raft backend will populate values for any transactions that are Get operations.
 func TestRaft_TransactionalBackend_GetTransactions(t *testing.T) {
-	b, dir := getRaft(t, true, true)
+	b, dir := GetRaft(t, true, true)
 	defer os.RemoveAll(dir)
 
 	ctx := context.Background()
@@ -429,7 +362,7 @@ func TestRaft_TransactionalBackend_GetTransactions(t *testing.T) {
 }
 
 func TestRaft_TransactionalBackend_LargeKey(t *testing.T) {
-	b, dir := getRaft(t, true, true)
+	b, dir := GetRaft(t, true, true)
 	defer os.RemoveAll(dir)
 
 	value := make([]byte, defaultMaxEntrySize+1)
@@ -468,7 +401,7 @@ func TestRaft_TransactionalBackend_LargeKey(t *testing.T) {
 }
 
 func TestRaft_TransactionalBackend_LargeValue(t *testing.T) {
-	b, dir := getRaft(t, true, true)
+	b, dir := GetRaft(t, true, true)
 	defer os.RemoveAll(dir)
 
 	value := make([]byte, defaultMaxEntrySize+1)
@@ -503,14 +436,14 @@ func TestRaft_TransactionalBackend_LargeValue(t *testing.T) {
 }
 
 func TestRaft_Backend_ListPrefix(t *testing.T) {
-	b, dir := getRaft(t, true, true)
+	b, dir := GetRaft(t, true, true)
 	defer os.RemoveAll(dir)
 
 	physical.ExerciseBackend_ListPrefix(t, b)
 }
 
 func TestRaft_TransactionalBackend(t *testing.T) {
-	b, dir := getRaft(t, true, true)
+	b, dir := GetRaft(t, true, true)
 	defer os.RemoveAll(dir)
 
 	physical.ExerciseTransactionalBackend(t, b)
@@ -518,9 +451,9 @@ func TestRaft_TransactionalBackend(t *testing.T) {
 
 func TestRaft_HABackend(t *testing.T) {
 	t.Skip()
-	raft, dir := getRaft(t, true, true)
+	raft, dir := GetRaft(t, true, true)
 	defer os.RemoveAll(dir)
-	raft2, dir2 := getRaft(t, false, true)
+	raft2, dir2 := GetRaft(t, false, true)
 	defer os.RemoveAll(dir2)
 
 	// Add raft2 to the cluster
@@ -530,9 +463,9 @@ func TestRaft_HABackend(t *testing.T) {
 }
 
 func TestRaft_Backend_ThreeNode(t *testing.T) {
-	raft1, dir := getRaft(t, true, true)
-	raft2, dir2 := getRaft(t, false, true)
-	raft3, dir3 := getRaft(t, false, true)
+	raft1, dir := GetRaft(t, true, true)
+	raft2, dir2 := GetRaft(t, false, true)
+	raft3, dir3 := GetRaft(t, false, true)
 	defer os.RemoveAll(dir)
 	defer os.RemoveAll(dir2)
 	defer os.RemoveAll(dir3)
@@ -553,9 +486,9 @@ func TestRaft_Backend_ThreeNode(t *testing.T) {
 
 func TestRaft_GetOfflineConfig(t *testing.T) {
 	// Create 3 raft nodes
-	raft1, dir1 := getRaft(t, true, true)
-	raft2, dir2 := getRaft(t, false, true)
-	raft3, dir3 := getRaft(t, false, true)
+	raft1, dir1 := GetRaft(t, true, true)
+	raft2, dir2 := GetRaft(t, false, true)
+	raft3, dir3 := GetRaft(t, false, true)
 	defer os.RemoveAll(dir1)
 	defer os.RemoveAll(dir2)
 	defer os.RemoveAll(dir3)
@@ -591,10 +524,10 @@ func TestRaft_GetOfflineConfig(t *testing.T) {
 
 func TestRaft_Recovery(t *testing.T) {
 	// Create 4 raft nodes
-	raft1, dir1 := getRaft(t, true, true)
-	raft2, dir2 := getRaft(t, false, true)
-	raft3, dir3 := getRaft(t, false, true)
-	raft4, dir4 := getRaft(t, false, true)
+	raft1, dir1 := GetRaft(t, true, true)
+	raft2, dir2 := GetRaft(t, false, true)
+	raft3, dir3 := GetRaft(t, false, true)
+	raft4, dir4 := GetRaft(t, false, true)
 	defer os.RemoveAll(dir1)
 	defer os.RemoveAll(dir2)
 	defer os.RemoveAll(dir3)
@@ -678,9 +611,9 @@ func TestRaft_Recovery(t *testing.T) {
 }
 
 func TestRaft_TransactionalBackend_ThreeNode(t *testing.T) {
-	raft1, dir := getRaft(t, true, true)
-	raft2, dir2 := getRaft(t, false, true)
-	raft3, dir3 := getRaft(t, false, true)
+	raft1, dir := GetRaft(t, true, true)
+	raft2, dir2 := GetRaft(t, false, true)
+	raft3, dir3 := GetRaft(t, false, true)
 	defer os.RemoveAll(dir)
 	defer os.RemoveAll(dir2)
 	defer os.RemoveAll(dir3)
@@ -700,7 +633,7 @@ func TestRaft_TransactionalBackend_ThreeNode(t *testing.T) {
 }
 
 func TestRaft_Backend_Performance(t *testing.T) {
-	b, dir := getRaft(t, true, false)
+	b, dir := GetRaft(t, true, false)
 	defer os.RemoveAll(dir)
 
 	defaultConfig := raft.DefaultConfig()
@@ -756,9 +689,9 @@ func TestRaft_Backend_Performance(t *testing.T) {
 }
 
 func BenchmarkDB_Puts(b *testing.B) {
-	raft, dir := getRaft(b, true, false)
+	raft, dir := GetRaft(b, true, false)
 	defer os.RemoveAll(dir)
-	raft2, dir2 := getRaft(b, true, false)
+	raft2, dir2 := GetRaft(b, true, false)
 	defer os.RemoveAll(dir2)
 
 	bench := func(b *testing.B, s physical.Backend, dataSize int) {
@@ -788,7 +721,7 @@ func BenchmarkDB_Puts(b *testing.B) {
 }
 
 func BenchmarkDB_Snapshot(b *testing.B) {
-	raft, dir := getRaft(b, true, false)
+	raft, dir := GetRaft(b, true, false)
 	defer os.RemoveAll(dir)
 
 	data, err := uuid.GenerateRandomBytes(256 * 1024)

physical/raft/snapshot_test.go

@@ -15,7 +15,7 @@ import (
 	"testing"
 	"time"
 
-	hclog "github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-hclog"
 	"github.com/hashicorp/raft"
 	"github.com/hashicorp/vault/sdk/logical"
 	"github.com/hashicorp/vault/sdk/physical"
@@ -55,7 +55,7 @@ func addPeer(t *testing.T, leader, follower *RaftBackend) {
 }
 
 func TestRaft_Snapshot_Loading(t *testing.T) {
-	raft, dir := getRaft(t, true, false)
+	raft, dir := GetRaft(t, true, false)
 	defer os.RemoveAll(dir)
 
 	// Write some data
@@ -139,7 +139,7 @@ func TestRaft_Snapshot_Loading(t *testing.T) {
 }
 
 func TestRaft_Snapshot_Index(t *testing.T) {
-	raft, dir := getRaft(t, true, false)
+	raft, dir := GetRaft(t, true, false)
 	defer os.RemoveAll(dir)
 
 	err := raft.Put(context.Background(), &physical.Entry{
@@ -226,9 +226,9 @@ func TestRaft_Snapshot_Index(t *testing.T) {
 }
 
 func TestRaft_Snapshot_Peers(t *testing.T) {
-	raft1, dir := getRaft(t, true, false)
-	raft2, dir2 := getRaft(t, false, false)
-	raft3, dir3 := getRaft(t, false, false)
+	raft1, dir := GetRaft(t, true, false)
+	raft2, dir2 := GetRaft(t, false, false)
+	raft3, dir3 := GetRaft(t, false, false)
 	defer os.RemoveAll(dir)
 	defer os.RemoveAll(dir2)
 	defer os.RemoveAll(dir3)
@@ -309,9 +309,9 @@ func ensureCommitApplied(t *testing.T, leaderCommitIdx uint64, backend *RaftBack
 }
 
 func TestRaft_Snapshot_Restart(t *testing.T) {
-	raft1, dir := getRaft(t, true, false)
+	raft1, dir := GetRaft(t, true, false)
 	defer os.RemoveAll(dir)
-	raft2, dir2 := getRaft(t, false, false)
+	raft2, dir2 := GetRaft(t, false, false)
 	defer os.RemoveAll(dir2)
 
 	// Write some data
@@ -373,9 +373,9 @@ func TestRaft_Snapshot_Restart(t *testing.T) {
 
 /*
 func TestRaft_Snapshot_ErrorRecovery(t *testing.T) {
-	raft1, dir := getRaft(t, true, false)
-	raft2, dir2 := getRaft(t, false, false)
-	raft3, dir3 := getRaft(t, false, false)
+	raft1, dir := GetRaft(t, true, false)
+	raft2, dir2 := GetRaft(t, false, false)
+	raft3, dir3 := GetRaft(t, false, false)
 	defer os.RemoveAll(dir)
 	defer os.RemoveAll(dir2)
 	defer os.RemoveAll(dir3)
@@ -455,9 +455,9 @@ func TestRaft_Snapshot_ErrorRecovery(t *testing.T) {
 }*/
 
 func TestRaft_Snapshot_Take_Restore(t *testing.T) {
-	raft1, dir := getRaft(t, true, false)
+	raft1, dir := GetRaft(t, true, false)
 	defer os.RemoveAll(dir)
-	raft2, dir2 := getRaft(t, false, false)
+	raft2, dir2 := GetRaft(t, false, false)
 	defer os.RemoveAll(dir2)
 
 	addPeer(t, raft1, raft2)

physical/raft/testing.go

@@ -0,0 +1,78 @@
+package raft
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"testing"
+
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-uuid"
+)
+
+func GetRaft(t testing.TB, bootstrap bool, noStoreState bool) (*RaftBackend, string) {
+	raftDir, err := ioutil.TempDir("", "vault-raft-")
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Logf("raft dir: %s", raftDir)
+
+	return getRaftWithDir(t, bootstrap, noStoreState, raftDir)
+}
+
+func getRaftWithDir(t testing.TB, bootstrap bool, noStoreState bool, raftDir string) (*RaftBackend, string) {
+	id, err := uuid.GenerateUUID()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	logger := hclog.New(&hclog.LoggerOptions{
+		Name:  fmt.Sprintf("raft-%s", id),
+		Level: hclog.Trace,
+	})
+	logger.Info("raft dir", "dir", raftDir)
+
+	conf := map[string]string{
+		"path":          raftDir,
+		"trailing_logs": "100",
+		"node_id":       id,
+	}
+
+	if noStoreState {
+		conf["doNotStoreLatestState"] = ""
+	}
+
+	backendRaw, err := NewRaftBackend(conf, logger)
+	if err != nil {
+		t.Fatal(err)
+	}
+	backend := backendRaw.(*RaftBackend)
+
+	if bootstrap {
+		err = backend.Bootstrap([]Peer{
+			{
+				ID:      backend.NodeID(),
+				Address: backend.NodeID(),
+			},
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		err = backend.SetupCluster(context.Background(), SetupOpts{})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		for {
+			if backend.raft.AppliedIndex() >= 2 {
+				break
+			}
+		}
+
+	}
+
+	backend.DisableAutopilot()
+
+	return backend, raftDir
+}

vault/raft.go

@@ -38,7 +38,7 @@ const (
 
 	// undoLogSafeVersion is the minimum version Vault must be at in order
 	// for undo logs to be turned on.
-	undoLogSafeVersion = "1.12.0-rc1"
+	undoLogSafeVersion = "1.12.0"
 )
 
 var (
@@ -199,6 +199,13 @@ func (c *Core) monitorUndoLogs() error {
 		return nil
 	}
 
+	// If undo logs have been explicitly enabled, likely via VAULT_REPLICATION_USE_UNDO_LOGS, then exit, as presumably
+	// we don't want to be checking for safety if we already know it's safe.
+	if c.UndoLogsEnabled() {
+		logger.Debug("undo logs have been explicitly enabled. exiting monitor.")
+		return nil
+	}
+
 	minimumVersion, err := goversion.NewSemver(undoLogSafeVersion)
 	if err != nil {
 		return fmt.Errorf("minimum undo log version (%q) won't parse: %w", undoLogSafeVersion, err)
@@ -282,7 +289,7 @@ func (c *Core) monitorUndoLogs() error {
 					continue
 				}
 
-				logger.Debug("undo logs have been enabled and this has been persisted to storage. shutting down the checker loop.")
+				logger.Debug("undo logs have been enabled and this has been persisted to storage. shutting down the checker.")
 				return
 			}
 		}
@@ -315,10 +322,10 @@ func (c *Core) setupRaftActiveNode(ctx context.Context) error {
 		return err
 	}
 
-	if c.UndoLogsEnabled() {
-		if err := c.monitorUndoLogs(); err != nil {
-			return err
-		}
+	// We always want to start this watcher - if undo logs are safe to be enabled, it will exit quickly. If not, it
+	// will monitor for safety until they are enabled.
+	if err := c.monitorUndoLogs(); err != nil {
+		return err
 	}
 	return c.startPeriodicRaftTLSRotate(ctx)
 }