logical/framework: only rollback old enough WAL entries

logical/framework/backend.go

@@ -7,6 +7,7 @@ import (
 	"sort"
 	"sync"
 	"text/template"
+	"time"
 
 	"github.com/hashicorp/go-multierror"
 	"github.com/hashicorp/vault/logical"
@@ -34,7 +35,8 @@ type Backend struct {
 	// Rollback is called when a WAL entry (see wal.go) has to be rolled
 	// back. It is called with the data from the entry. Boolean true should
 	// be returned on success. Errors should just be logged.
-	Rollback func(kind string, data interface{}) bool
+	Rollback       func(kind string, data interface{}) bool
+	RollbackMinAge time.Duration
 
 	once    sync.Once
 	pathsRe []*regexp.Regexp
@@ -188,25 +190,43 @@ func (b *Backend) handleRollback(
 	var merr error
 	keys, err := ListWAL(req.Storage)
 	if err != nil {
-		merr = multierror.Append(merr, err)
-		goto RESPOND_ROLLBACK
+		return logical.ErrorResponse(err.Error()), nil
+	}
+	if len(keys) == 0 {
+		return nil, nil
 	}
 
+	// Calculate the minimum time that the WAL entries could be
+	// created in order to be rolled back.
+	age := b.RollbackMinAge
+	if age == 0 {
+		age = 10 * time.Minute
+	}
+	minAge := time.Now().UTC().Add(-1 * age)
+
 	for _, k := range keys {
-		kind, data, err := GetWAL(req.Storage, k)
+		entry, err := GetWAL(req.Storage, k)
 		if err != nil {
 			merr = multierror.Append(merr, err)
 			continue
 		}
+		if entry == nil {
+			continue
+		}
 
-		if b.Rollback(kind, data) {
+		// If the entry isn't old enough, then don't roll it back
+		if !time.Unix(entry.CreatedAt, 0).Before(minAge) {
+			continue
+		}
+
+		// Attempt a rollback
+		if b.Rollback(entry.Kind, entry.Data) {
 			if err := DeleteWAL(req.Storage, k); err != nil {
 				merr = multierror.Append(merr, err)
 			}
 		}
 	}
 
-RESPOND_ROLLBACK:
 	if merr == nil {
 		return nil, nil
 	}

logical/framework/backend_test.go

@@ -4,6 +4,7 @@ import (
 	"reflect"
 	"sync/atomic"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/vault/logical"
 )
@@ -147,7 +148,8 @@ func TestBackendHandleRequest_rollback(t *testing.T) {
 	}
 
 	b := &Backend{
-		Rollback: callback,
+		Rollback:       callback,
+		RollbackMinAge: 1 * time.Millisecond,
 	}
 
 	storage := new(logical.InmemStorage)
@@ -155,6 +157,8 @@ func TestBackendHandleRequest_rollback(t *testing.T) {
 		t.Fatalf("err: %s", err)
 	}
 
+	time.Sleep(10 * time.Millisecond)
+
 	_, err := b.HandleRequest(&logical.Request{
 		Operation: logical.RollbackOperation,
 		Path:      "",

logical/framework/wal.go

@@ -3,6 +3,7 @@ package framework
 import (
 	"encoding/json"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/vault/logical"
 )
@@ -10,6 +11,13 @@ import (
 // WALPrefix is the prefix within Storage where WAL entries will be written.
 const WALPrefix = "wal/"
 
+type WALEntry struct {
+	ID        string      `json:"-"`
+	Kind      string      `json:"type"`
+	Data      interface{} `json:"data"`
+	CreatedAt int64       `json:"created_at"`
+}
+
 // PutWAL writes some data to the WAL.
 //
 // The kind parameter is used by the framework to allow users to store
@@ -26,9 +34,10 @@ const WALPrefix = "wal/"
 // WAL data cannot be modified. You can only add to the WAL and commit existing
 // WAL entries.
 func PutWAL(s logical.Storage, kind string, data interface{}) (string, error) {
-	value, err := json.Marshal(map[string]interface{}{
-		"kind": kind,
-		"data": data,
+	value, err := json.Marshal(&WALEntry{
+		Kind:      kind,
+		Data:      data,
+		CreatedAt: time.Now().UTC().Unix(),
 	})
 	if err != nil {
 		return "", err
@@ -49,21 +58,22 @@ func PutWAL(s logical.Storage, kind string, data interface{}) (string, error) {
 // then nil value is returned.
 //
 // The kind, value, and error are returned.
-func GetWAL(s logical.Storage, id string) (string, interface{}, error) {
+func GetWAL(s logical.Storage, id string) (*WALEntry, error) {
 	entry, err := s.Get(WALPrefix + id)
 	if err != nil {
-		return "", nil, err
+		return nil, err
 	}
 	if entry == nil {
-		return "", nil, nil
+		return nil, nil
 	}
 
-	var result map[string]interface{}
-	if err := json.Unmarshal(entry.Value, &result); err != nil {
-		return "", nil, err
+	var raw WALEntry
+	if err := json.Unmarshal(entry.Value, &raw); err != nil {
+		return nil, err
 	}
+	raw.ID = id
 
-	return result["kind"].(string), result["data"], nil
+	return &raw, nil
 }
 
 // DeleteWAL commits the WAL entry with the given ID. Once comitted,

logical/framework/wal_test.go

@@ -35,26 +35,26 @@ func TestWAL(t *testing.T) {
 	}
 
 	// Should be able to get the value
-	kind, v, err := GetWAL(s, id)
+	entry, err := GetWAL(s, id)
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
-	if kind != "foo" {
-		t.Fatalf("bad: %#v", kind)
+	if entry.Kind != "foo" {
+		t.Fatalf("bad: %#v", entry)
 	}
-	if v != "bar" {
-		t.Fatalf("bad: %#v", v)
+	if entry.Data != "bar" {
+		t.Fatalf("bad: %#v", entry)
 	}
 
 	// Should be able to delete the value
 	if err := DeleteWAL(s, id); err != nil {
 		t.Fatalf("err: %s", err)
 	}
-	_, v, err = GetWAL(s, id)
+	entry, err = GetWAL(s, id)
 	if err != nil {
 		t.Fatalf("err: %s", err)
 	}
-	if v != nil {
-		t.Fatalf("bad: %#v", v)
+	if entry != nil {
+		t.Fatalf("bad: %#v", entry)
 	}
 }