add record expiration support

databunker.yaml

@@ -19,6 +19,7 @@ notification:
   notification_url: "https://httpbin.org/post"
 policy:
   # max time to store records, untill they are deleted
+  max_user_retention_period: "3m"
   max_audit_retention_period: "6m"
   max_session_retention_period: "1h"
   max_shareable_record_retention_period: "1m"

src/bunker.go

@@ -53,9 +53,10 @@ type Config struct {
 		MagicSyncToken  string `yaml:"magic_sync_token"`
 	}
 	Policy struct {
-		MaxAuditRetentionPeriod           string `yaml:"max_audit_retention_period"`
-		MaxSessionRetentionPeriod         string `yaml:"max_session_retention_period"`
-		MaxShareableRecordRetentionPeriod string `yaml:"max_shareable_record_retention_period"`
+		MaxUserRetentionPeriod            string `yaml:"max_user_retention_period" default:"1m"`
+		MaxAuditRetentionPeriod           string `yaml:"max_audit_retention_period" default:"12m"`
+		MaxSessionRetentionPeriod         string `yaml:"max_session_retention_period" default:"1h"`
+		MaxShareableRecordRetentionPeriod string `yaml:"max_shareable_record_retention_period" default:"1m"`
 	}
 	Ssl struct {
 		SslCertificate    string `yaml:"ssl_certificate", envconfig:"SSL_CERTIFICATE"`
@@ -190,6 +191,12 @@ func (e mainEnv) setupRouter() *httprouter.Router {
 	router.GET("/v1/prelogin/:mode/:address/:code/:captcha", e.userPrelogin)
 	router.GET("/v1/login/:mode/:address/:tmp", e.userLogin)
 
+	router.GET("/v1/exp/retain/:exptoken", e.expRetainData)
+	router.GET("/v1/exp/delete/:exptoken", e.expDeleteData)
+	router.GET("/v1/exp/status/:mode/:address", e.expGetStatus)
+	router.POST("/v1/exp/initiate/:mode/:address", e.expInitiate)
+	router.DELETE("/v1/exp/cancel/:mode/:address", e.expCancel)
+
 	router.POST("/v1/sharedrecord/token/:token", e.newSharedRecord)
 	router.GET("/v1/get/:record", e.getRecord)
 

src/expiration_api.go

@@ -0,0 +1,195 @@
+package main
+
+import (
+	"fmt"
+	"net/http"
+
+	uuid "github.com/hashicorp/go-uuid"
+	"github.com/julienschmidt/httprouter"
+	"go.mongodb.org/mongo-driver/bson"
+)
+
+func (e mainEnv) expGetStatus(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	var err error
+	address := ps.ByName("address")
+	mode := ps.ByName("mode")
+	event := audit("get expiration status by "+mode, address, mode, address)
+	defer func() { event.submit(e.db) }()
+	if validateMode(mode) == false {
+		returnError(w, r, "bad mode", 405, nil, event)
+		return
+	}
+	userTOKEN := address
+	var userBson bson.M
+	if mode == "token" {
+		if enforceUUID(w, address, event) == false {
+			return
+		}
+		userBson, err = e.db.lookupUserRecord(address)
+	} else {
+		userBson, err = e.db.lookupUserRecordByIndex(mode, address, e.conf)
+		if userBson != nil {
+			userTOKEN = userBson["token"].(string)
+			event.Record = userTOKEN
+		}
+	}
+	if userBson == nil || err != nil {
+		returnError(w, r, "internal error", 405, nil, event)
+		return
+	}
+	expirationDate := getIntValue(userBson["expdate"])
+	expirationStatus := getStringValue(userBson["expstatus"])
+	expirationToken := getStringValue(userBson["exptoken"])
+	finalJSON := fmt.Sprintf(`{"status":"ok","expdate":%d,"expstatus":"%s","exptoken":"%s"}`,
+		expirationDate, expirationStatus, expirationToken)
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.WriteHeader(200)
+	w.Write([]byte(finalJSON))
+}
+
+func (e mainEnv) expCancel(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	var err error
+	address := ps.ByName("address")
+	mode := ps.ByName("mode")
+	event := audit("clear user expiration by "+mode, address, mode, address)
+	defer func() { event.submit(e.db) }()
+	if validateMode(mode) == false {
+		returnError(w, r, "bad mode", 405, nil, event)
+		return
+	}
+	userTOKEN := address
+	var userBson bson.M
+	if mode == "token" {
+		if enforceUUID(w, address, event) == false {
+			return
+		}
+		userBson, err = e.db.lookupUserRecord(address)
+	} else {
+		userBson, err = e.db.lookupUserRecordByIndex(mode, address, e.conf)
+		if userBson != nil {
+			userTOKEN = userBson["token"].(string)
+			event.Record = userTOKEN
+		}
+	}
+	if userBson == nil || err != nil {
+		returnError(w, r, "internal error", 405, nil, event)
+		return
+	}
+	status := ""
+	err = e.db.updateUserExpStatus(userTOKEN, status)
+	if err != nil {
+		returnError(w, r, "internal error", 405, nil, event)
+		return
+	}
+	finalJSON := `{"status":"ok"}`
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.WriteHeader(200)
+	w.Write([]byte(finalJSON))
+}
+
+func (e mainEnv) expRetainData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	address := ps.ByName("exptoken")
+	mode := "exptoken"
+	event := audit("retain user data by exptoken", address, mode, address)
+	defer func() { event.submit(e.db) }()
+	if enforceUUID(w, address, event) == false {
+		return
+	}
+	userBson, err := e.db.lookupUserRecordByIndex(mode, address, e.conf)
+	if userBson == nil || err != nil {
+		returnError(w, r, "internal error", 405, nil, event)
+		return
+	}
+	userTOKEN := userBson["token"].(string)
+	event.Record = userTOKEN
+	status := "retain"
+	err = e.db.updateUserExpStatus(userTOKEN, status)
+	if err != nil {
+		returnError(w, r, "internal error", 405, nil, event)
+		return
+	}
+	w.WriteHeader(200)
+	w.Write([]byte("OK"))
+}
+
+func (e mainEnv) expDeleteData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	address := ps.ByName("exptoken")
+	mode := "exptoken"
+	event := audit("delete user data by exptoken", address, mode, address)
+	defer func() { event.submit(e.db) }()
+	if enforceUUID(w, address, event) == false {
+		return
+	}
+	resultJSON, userTOKEN, err := e.db.getUserJsonByIndex(address, mode, e.conf)
+	if resultJSON == nil || err != nil {
+		returnError(w, r, "internal error", 405, nil, event)
+		return
+	}
+	event.Record = userTOKEN
+	e.globalUserDelete(userTOKEN)
+	_, err = e.db.deleteUserRecord(resultJSON, userTOKEN)
+	if err != nil {
+		returnError(w, r, "internal error", 405, nil, event)
+		return
+	}
+	w.WriteHeader(200)
+	w.Write([]byte("OK"))
+}
+
+func (e mainEnv) expInitiate(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
+	var err error
+	address := ps.ByName("address")
+	mode := ps.ByName("mode")
+	event := audit("initiate user record expiration by "+mode, address, mode, address)
+	defer func() { event.submit(e.db) }()
+	if validateMode(mode) == false {
+		returnError(w, r, "bad mode", 405, nil, event)
+		return
+	}
+	if e.enforceAdmin(w, r) == "" {
+		return
+	}
+	userTOKEN := address
+	var userBson bson.M
+	if mode == "token" {
+		if enforceUUID(w, address, event) == false {
+			return
+		}
+		userBson, err = e.db.lookupUserRecord(address)
+	} else {
+		userBson, err = e.db.lookupUserRecordByIndex(mode, address, e.conf)
+		if userBson != nil {
+			userTOKEN = userBson["token"].(string)
+			event.Record = userTOKEN
+		}
+	}
+	if userBson == nil || err != nil {
+		returnError(w, r, "internal error", 405, nil, event)
+		return
+	}
+	records, err := getJSONPostData(r)
+	if err != nil {
+		returnError(w, r, "failed to decode request body", 405, err, event)
+		return
+	}
+	expirationStr := getStringValue(records["expiration"])
+	expiration := setExpiration(e.conf.Policy.MaxUserRetentionPeriod, expirationStr)
+	status := getStringValue(records["status"])
+	if len(status) == 0 {
+		status = "wait"
+	}
+	expToken, err := uuid.GenerateUUID()
+	if err != nil {
+		returnError(w, r, "internal error", 405, err, event)
+	}
+	err = e.db.initiateUserExpiration(userTOKEN, expiration, status, expToken)
+	if err != nil {
+		returnError(w, r, "internal error", 405, err, event)
+		return
+	}
+	finalJSON := fmt.Sprintf(`{"status":"ok","exptoken":"%s"}`, expToken)
+	w.Header().Set("Content-Type", "application/json; charset=utf-8")
+	w.WriteHeader(200)
+	w.Write([]byte(finalJSON))
+}
+

src/sessions_api.go

@@ -4,7 +4,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"net/http"
-	"reflect"
 	"strings"
 	uuid "github.com/hashicorp/go-uuid"
 	"github.com/julienschmidt/httprouter"
@@ -117,7 +116,6 @@ func (e mainEnv) newUserSession(w http.ResponseWriter, r *http.Request, ps httpr
 	if e.enforceAuth(w, r, event) == "" {
 		return
 	}
-	expiration := e.conf.Policy.MaxSessionRetentionPeriod
 	records, err := getJSONPostData(r)
 	if err != nil {
 		returnError(w, r, "failed to decode request body", 405, err, event)
@@ -127,13 +125,8 @@ func (e mainEnv) newUserSession(w http.ResponseWriter, r *http.Request, ps httpr
 		returnError(w, r, "empty body", 405, nil, event)
 		return
 	}
-	if value, ok := records["expiration"]; ok {
-		if reflect.TypeOf(value) == reflect.TypeOf("string") {
-			expiration = setExpiration(e.conf.Policy.MaxSessionRetentionPeriod, value.(string))
-		} else {
-			// ignore bad expiration format
-		}
-	}
+	expirationStr := getStringValue(records["expiration"])
+	expiration := setExpiration(e.conf.Policy.MaxSessionRetentionPeriod, expirationStr)
 	jsonData, err := json.Marshal(records)
 	if err != nil {
 		returnError(w, r, "internal error", 405, err, event)

src/storage/mysql-storage.go

@@ -940,6 +940,9 @@ func (dbobj MySQLDB) initUsers() error {
 		`emailidx TINYTEXT,`+
 		`phoneidx TINYTEXT,`+
 		`customidx TINYTEXT,`+
+		`expstatus TINYTEXT,`+
+		`exptoken TINYTEXT,`+
+		`expdate int,`+
 		`tempcodeexp int,`+
 		`tempcode int,`+
 		`data TEXT);`,
@@ -947,7 +950,9 @@ func (dbobj MySQLDB) initUsers() error {
 		`CREATE INDEX users_login ON users (loginidx(36));`,
 		`CREATE INDEX users_email ON users (emailidx(36));`,
 		`CREATE INDEX users_phone ON users (phoneidx(36));`,
-		`CREATE INDEX users_custom ON users (customidx(36));`}
+		`CREATE INDEX users_custom ON users (customidx(36));`,
+		`CREATE INDEX users_expdate ON users (expdate);`,
+		`CREATE INDEX users_exptoken ON users (exptoken(36));`}
 	return dbobj.execQueries(queries)
 }
 

src/storage/sqlite-storage.go

@@ -930,6 +930,9 @@ func (dbobj SQLiteDB) initUsers() error {
 			  emailidx STRING,
 			  phoneidx STRING,
 			  customidx STRING,
+			  expstatus STRING,
+			  exptoken STRING,
+			  expdate int,
 			  tempcodeexp int,
 			  tempcode int,
 			  data TEXT
@@ -938,7 +941,9 @@ func (dbobj SQLiteDB) initUsers() error {
 		`CREATE INDEX users_login ON users (loginidx);`,
 		`CREATE INDEX users_email ON users (emailidx);`,
 		`CREATE INDEX users_phone ON users (phoneidx);`,
-		`CREATE INDEX users_custom ON users (customidx);`}
+		`CREATE INDEX users_custom ON users (customidx);`,
+		`CREATE INDEX users_expdate ON users (expdate);`,
+		`CREATE INDEX users_exptoken ON users (exptoken);`}
 	return dbobj.execQueries(queries)
 }
 

src/users_db.go

@@ -67,6 +67,22 @@ func (dbobj dbcon) createUserRecord(parsedData userJSON, event *auditEvent) (str
 	return userTOKEN, nil
 }
 
+func (dbobj dbcon) initiateUserExpiration(userTOKEN string, expiration string, status string, expToken string) error {
+	bdoc := bson.M{}
+	bdoc["expiration"] = expiration
+	bdoc["expstatus"] = status
+	bdoc["exptoken"] = expToken
+	_, err := dbobj.store.UpdateRecord(storage.TblName.Users, "token", userTOKEN, &bdoc)
+	return err
+}
+
+func (dbobj dbcon) updateUserExpStatus(userTOKEN string, status string) error {
+	bdoc := bson.M{}
+	bdoc["expstatus"] = status
+	_, err := dbobj.store.UpdateRecord(storage.TblName.Users, "token", userTOKEN, &bdoc)
+	return err
+}
+
 func (dbobj dbcon) generateTempLoginCode(userTOKEN string) int32 {
 	rnd := randNum(6)
 	fmt.Printf("random: %d\n", rnd)
@@ -264,6 +280,9 @@ func (dbobj dbcon) lookupUserRecordByIndex(indexName string, indexValue string,
 	if len(indexValue) == 0 {
 		return nil, nil
 	}
+	if indexName == "exptoken" {
+		return dbobj.store.GetRecord(storage.TblName.Users, "exptoken", indexValue)
+	}
 	idxStringHashHex := hashString(dbobj.hash, indexValue)
 	//fmt.Printf("loading by %s, value: %s\n", indexName, indexValue)
 	return dbobj.store.GetRecord(storage.TblName.Users, indexName+"idx", idxStringHashHex)

src/utils.go

@@ -69,6 +69,16 @@ func getStringValue(r interface{}) string {
 	return ""
 }
 
+func getIntValue(r interface{}) int {
+	switch r.(type) {
+		case int:
+			return r.(int)
+		case int32:
+			return int(r.(int32))
+	}
+	return 0
+}
+
 func getInt64Value(records map[string]interface{}, key string) int64 {
 	if value, ok := records[key]; ok {
 		switch value.(type) {