From 45e79280c14914e0b56bfda67f34669be31eb4c1 Mon Sep 17 00:00:00 2001 From: root Date: Tue, 27 Apr 2021 19:15:36 +0000 Subject: [PATCH] add record expiration support --- databunker.yaml | 1 + src/bunker.go | 13 ++- src/expiration_api.go | 195 ++++++++++++++++++++++++++++++++++ src/sessions_api.go | 11 +- src/storage/mysql-storage.go | 7 +- src/storage/sqlite-storage.go | 7 +- src/users_db.go | 19 ++++ src/utils.go | 10 ++ 8 files changed, 249 insertions(+), 14 deletions(-) create mode 100644 src/expiration_api.go diff --git a/databunker.yaml b/databunker.yaml index 23dd863..41343d3 100644 --- a/databunker.yaml +++ b/databunker.yaml @@ -19,6 +19,7 @@ notification: notification_url: "https://httpbin.org/post" policy: # max time to store records, untill they are deleted + max_user_retention_period: "3m" max_audit_retention_period: "6m" max_session_retention_period: "1h" max_shareable_record_retention_period: "1m" diff --git a/src/bunker.go b/src/bunker.go index d6613a6..62c0643 100644 --- a/src/bunker.go +++ b/src/bunker.go @@ -53,9 +53,10 @@ type Config struct { MagicSyncToken string `yaml:"magic_sync_token"` } Policy struct { - MaxAuditRetentionPeriod string `yaml:"max_audit_retention_period"` - MaxSessionRetentionPeriod string `yaml:"max_session_retention_period"` - MaxShareableRecordRetentionPeriod string `yaml:"max_shareable_record_retention_period"` + MaxUserRetentionPeriod string `yaml:"max_user_retention_period" default:"1m"` + MaxAuditRetentionPeriod string `yaml:"max_audit_retention_period" default:"12m"` + MaxSessionRetentionPeriod string `yaml:"max_session_retention_period" default:"1h"` + MaxShareableRecordRetentionPeriod string `yaml:"max_shareable_record_retention_period" default:"1m"` } Ssl struct { SslCertificate string `yaml:"ssl_certificate", envconfig:"SSL_CERTIFICATE"` @@ -190,6 +191,12 @@ func (e mainEnv) setupRouter() *httprouter.Router { router.GET("/v1/prelogin/:mode/:address/:code/:captcha", e.userPrelogin) router.GET("/v1/login/:mode/:address/:tmp", e.userLogin) + router.GET("/v1/exp/retain/:exptoken", e.expRetainData) + router.GET("/v1/exp/delete/:exptoken", e.expDeleteData) + router.GET("/v1/exp/status/:mode/:address", e.expGetStatus) + router.POST("/v1/exp/initiate/:mode/:address", e.expInitiate) + router.DELETE("/v1/exp/cancel/:mode/:address", e.expCancel) + router.POST("/v1/sharedrecord/token/:token", e.newSharedRecord) router.GET("/v1/get/:record", e.getRecord) diff --git a/src/expiration_api.go b/src/expiration_api.go new file mode 100644 index 0000000..ee44c00 --- /dev/null +++ b/src/expiration_api.go @@ -0,0 +1,195 @@ +package main + +import ( + "fmt" + "net/http" + + uuid "github.com/hashicorp/go-uuid" + "github.com/julienschmidt/httprouter" + "go.mongodb.org/mongo-driver/bson" +) + +func (e mainEnv) expGetStatus(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + var err error + address := ps.ByName("address") + mode := ps.ByName("mode") + event := audit("get expiration status by "+mode, address, mode, address) + defer func() { event.submit(e.db) }() + if validateMode(mode) == false { + returnError(w, r, "bad mode", 405, nil, event) + return + } + userTOKEN := address + var userBson bson.M + if mode == "token" { + if enforceUUID(w, address, event) == false { + return + } + userBson, err = e.db.lookupUserRecord(address) + } else { + userBson, err = e.db.lookupUserRecordByIndex(mode, address, e.conf) + if userBson != nil { + userTOKEN = userBson["token"].(string) + event.Record = userTOKEN + } + } + if userBson == nil || err != nil { + returnError(w, r, "internal error", 405, nil, event) + return + } + expirationDate := getIntValue(userBson["expdate"]) + expirationStatus := getStringValue(userBson["expstatus"]) + expirationToken := getStringValue(userBson["exptoken"]) + finalJSON := fmt.Sprintf(`{"status":"ok","expdate":%d,"expstatus":"%s","exptoken":"%s"}`, + expirationDate, expirationStatus, expirationToken) + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(200) + w.Write([]byte(finalJSON)) +} + +func (e mainEnv) expCancel(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + var err error + address := ps.ByName("address") + mode := ps.ByName("mode") + event := audit("clear user expiration by "+mode, address, mode, address) + defer func() { event.submit(e.db) }() + if validateMode(mode) == false { + returnError(w, r, "bad mode", 405, nil, event) + return + } + userTOKEN := address + var userBson bson.M + if mode == "token" { + if enforceUUID(w, address, event) == false { + return + } + userBson, err = e.db.lookupUserRecord(address) + } else { + userBson, err = e.db.lookupUserRecordByIndex(mode, address, e.conf) + if userBson != nil { + userTOKEN = userBson["token"].(string) + event.Record = userTOKEN + } + } + if userBson == nil || err != nil { + returnError(w, r, "internal error", 405, nil, event) + return + } + status := "" + err = e.db.updateUserExpStatus(userTOKEN, status) + if err != nil { + returnError(w, r, "internal error", 405, nil, event) + return + } + finalJSON := `{"status":"ok"}` + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(200) + w.Write([]byte(finalJSON)) +} + +func (e mainEnv) expRetainData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + address := ps.ByName("exptoken") + mode := "exptoken" + event := audit("retain user data by exptoken", address, mode, address) + defer func() { event.submit(e.db) }() + if enforceUUID(w, address, event) == false { + return + } + userBson, err := e.db.lookupUserRecordByIndex(mode, address, e.conf) + if userBson == nil || err != nil { + returnError(w, r, "internal error", 405, nil, event) + return + } + userTOKEN := userBson["token"].(string) + event.Record = userTOKEN + status := "retain" + err = e.db.updateUserExpStatus(userTOKEN, status) + if err != nil { + returnError(w, r, "internal error", 405, nil, event) + return + } + w.WriteHeader(200) + w.Write([]byte("OK")) +} + +func (e mainEnv) expDeleteData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + address := ps.ByName("exptoken") + mode := "exptoken" + event := audit("delete user data by exptoken", address, mode, address) + defer func() { event.submit(e.db) }() + if enforceUUID(w, address, event) == false { + return + } + resultJSON, userTOKEN, err := e.db.getUserJsonByIndex(address, mode, e.conf) + if resultJSON == nil || err != nil { + returnError(w, r, "internal error", 405, nil, event) + return + } + event.Record = userTOKEN + e.globalUserDelete(userTOKEN) + _, err = e.db.deleteUserRecord(resultJSON, userTOKEN) + if err != nil { + returnError(w, r, "internal error", 405, nil, event) + return + } + w.WriteHeader(200) + w.Write([]byte("OK")) +} + +func (e mainEnv) expInitiate(w http.ResponseWriter, r *http.Request, ps httprouter.Params) { + var err error + address := ps.ByName("address") + mode := ps.ByName("mode") + event := audit("initiate user record expiration by "+mode, address, mode, address) + defer func() { event.submit(e.db) }() + if validateMode(mode) == false { + returnError(w, r, "bad mode", 405, nil, event) + return + } + if e.enforceAdmin(w, r) == "" { + return + } + userTOKEN := address + var userBson bson.M + if mode == "token" { + if enforceUUID(w, address, event) == false { + return + } + userBson, err = e.db.lookupUserRecord(address) + } else { + userBson, err = e.db.lookupUserRecordByIndex(mode, address, e.conf) + if userBson != nil { + userTOKEN = userBson["token"].(string) + event.Record = userTOKEN + } + } + if userBson == nil || err != nil { + returnError(w, r, "internal error", 405, nil, event) + return + } + records, err := getJSONPostData(r) + if err != nil { + returnError(w, r, "failed to decode request body", 405, err, event) + return + } + expirationStr := getStringValue(records["expiration"]) + expiration := setExpiration(e.conf.Policy.MaxUserRetentionPeriod, expirationStr) + status := getStringValue(records["status"]) + if len(status) == 0 { + status = "wait" + } + expToken, err := uuid.GenerateUUID() + if err != nil { + returnError(w, r, "internal error", 405, err, event) + } + err = e.db.initiateUserExpiration(userTOKEN, expiration, status, expToken) + if err != nil { + returnError(w, r, "internal error", 405, err, event) + return + } + finalJSON := fmt.Sprintf(`{"status":"ok","exptoken":"%s"}`, expToken) + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(200) + w.Write([]byte(finalJSON)) +} + diff --git a/src/sessions_api.go b/src/sessions_api.go index fbdcd72..a44494b 100644 --- a/src/sessions_api.go +++ b/src/sessions_api.go @@ -4,7 +4,6 @@ import ( "encoding/json" "fmt" "net/http" - "reflect" "strings" uuid "github.com/hashicorp/go-uuid" "github.com/julienschmidt/httprouter" @@ -117,7 +116,6 @@ func (e mainEnv) newUserSession(w http.ResponseWriter, r *http.Request, ps httpr if e.enforceAuth(w, r, event) == "" { return } - expiration := e.conf.Policy.MaxSessionRetentionPeriod records, err := getJSONPostData(r) if err != nil { returnError(w, r, "failed to decode request body", 405, err, event) @@ -127,13 +125,8 @@ func (e mainEnv) newUserSession(w http.ResponseWriter, r *http.Request, ps httpr returnError(w, r, "empty body", 405, nil, event) return } - if value, ok := records["expiration"]; ok { - if reflect.TypeOf(value) == reflect.TypeOf("string") { - expiration = setExpiration(e.conf.Policy.MaxSessionRetentionPeriod, value.(string)) - } else { - // ignore bad expiration format - } - } + expirationStr := getStringValue(records["expiration"]) + expiration := setExpiration(e.conf.Policy.MaxSessionRetentionPeriod, expirationStr) jsonData, err := json.Marshal(records) if err != nil { returnError(w, r, "internal error", 405, err, event) diff --git a/src/storage/mysql-storage.go b/src/storage/mysql-storage.go index cc735b3..f0424d9 100644 --- a/src/storage/mysql-storage.go +++ b/src/storage/mysql-storage.go @@ -940,6 +940,9 @@ func (dbobj MySQLDB) initUsers() error { `emailidx TINYTEXT,`+ `phoneidx TINYTEXT,`+ `customidx TINYTEXT,`+ + `expstatus TINYTEXT,`+ + `exptoken TINYTEXT,`+ + `expdate int,`+ `tempcodeexp int,`+ `tempcode int,`+ `data TEXT);`, @@ -947,7 +950,9 @@ func (dbobj MySQLDB) initUsers() error { `CREATE INDEX users_login ON users (loginidx(36));`, `CREATE INDEX users_email ON users (emailidx(36));`, `CREATE INDEX users_phone ON users (phoneidx(36));`, - `CREATE INDEX users_custom ON users (customidx(36));`} + `CREATE INDEX users_custom ON users (customidx(36));`, + `CREATE INDEX users_expdate ON users (expdate);`, + `CREATE INDEX users_exptoken ON users (exptoken(36));`} return dbobj.execQueries(queries) } diff --git a/src/storage/sqlite-storage.go b/src/storage/sqlite-storage.go index a8f6173..ae80bb3 100644 --- a/src/storage/sqlite-storage.go +++ b/src/storage/sqlite-storage.go @@ -930,6 +930,9 @@ func (dbobj SQLiteDB) initUsers() error { emailidx STRING, phoneidx STRING, customidx STRING, + expstatus STRING, + exptoken STRING, + expdate int, tempcodeexp int, tempcode int, data TEXT @@ -938,7 +941,9 @@ func (dbobj SQLiteDB) initUsers() error { `CREATE INDEX users_login ON users (loginidx);`, `CREATE INDEX users_email ON users (emailidx);`, `CREATE INDEX users_phone ON users (phoneidx);`, - `CREATE INDEX users_custom ON users (customidx);`} + `CREATE INDEX users_custom ON users (customidx);`, + `CREATE INDEX users_expdate ON users (expdate);`, + `CREATE INDEX users_exptoken ON users (exptoken);`} return dbobj.execQueries(queries) } diff --git a/src/users_db.go b/src/users_db.go index 1bb4005..8c351a8 100644 --- a/src/users_db.go +++ b/src/users_db.go @@ -67,6 +67,22 @@ func (dbobj dbcon) createUserRecord(parsedData userJSON, event *auditEvent) (str return userTOKEN, nil } +func (dbobj dbcon) initiateUserExpiration(userTOKEN string, expiration string, status string, expToken string) error { + bdoc := bson.M{} + bdoc["expiration"] = expiration + bdoc["expstatus"] = status + bdoc["exptoken"] = expToken + _, err := dbobj.store.UpdateRecord(storage.TblName.Users, "token", userTOKEN, &bdoc) + return err +} + +func (dbobj dbcon) updateUserExpStatus(userTOKEN string, status string) error { + bdoc := bson.M{} + bdoc["expstatus"] = status + _, err := dbobj.store.UpdateRecord(storage.TblName.Users, "token", userTOKEN, &bdoc) + return err +} + func (dbobj dbcon) generateTempLoginCode(userTOKEN string) int32 { rnd := randNum(6) fmt.Printf("random: %d\n", rnd) @@ -264,6 +280,9 @@ func (dbobj dbcon) lookupUserRecordByIndex(indexName string, indexValue string, if len(indexValue) == 0 { return nil, nil } + if indexName == "exptoken" { + return dbobj.store.GetRecord(storage.TblName.Users, "exptoken", indexValue) + } idxStringHashHex := hashString(dbobj.hash, indexValue) //fmt.Printf("loading by %s, value: %s\n", indexName, indexValue) return dbobj.store.GetRecord(storage.TblName.Users, indexName+"idx", idxStringHashHex) diff --git a/src/utils.go b/src/utils.go index f777b27..1676a6b 100644 --- a/src/utils.go +++ b/src/utils.go @@ -69,6 +69,16 @@ func getStringValue(r interface{}) string { return "" } +func getIntValue(r interface{}) int { + switch r.(type) { + case int: + return r.(int) + case int32: + return int(r.(int32)) + } + return 0 +} + func getInt64Value(records map[string]interface{}, key string) int64 { if value, ok := records[key]; ok { switch value.(type) {