From fd9e9abfceae9c04a2c14eb196d92fd5df4f267a Mon Sep 17 00:00:00 2001 From: yuli Date: Thu, 26 Dec 2024 22:14:23 +0200 Subject: [PATCH] refactore code - add utils directory --- src/utils/go.mod | 3 + src/utils/utils.go | 696 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 699 insertions(+) create mode 100644 src/utils/go.mod create mode 100644 src/utils/utils.go diff --git a/src/utils/go.mod b/src/utils/go.mod new file mode 100644 index 0000000..679f5c7 --- /dev/null +++ b/src/utils/go.mod @@ -0,0 +1,3 @@ +module github.com/securitybunker/databunker/src/utils + +go 1.13 diff --git a/src/utils/utils.go b/src/utils/utils.go new file mode 100644 index 0000000..a97b6a1 --- /dev/null +++ b/src/utils/utils.go @@ -0,0 +1,696 @@ +package main + +import ( + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "log" + "math/rand" + "mime" + "net/http" + "net/url" + "os" + "regexp" + "strconv" + "strings" + "syscall" + "time" + + "github.com/kelseyhightower/envconfig" + "github.com/ttacon/libphonenumber" + "golang.org/x/sys/unix" + "gopkg.in/yaml.v2" +) + +var ( + regexUUID = regexp.MustCompile("^[a-fA-F0-9]{8}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{4}-[a-fA-F0-9]{12}$") + regexBrief = regexp.MustCompile("^[a-z][a-z0-9\\-]{1,64}$") + regexAppName = regexp.MustCompile("^[a-z][a-z0-9\\_]{1,30}$") + regexExpiration = regexp.MustCompile("^([0-9]+)([mhds])?$") + regexHex = regexp.MustCompile("^[a-zA-F0-9]+$") + indexNames = map[string]bool{ + "custom": true, + "email": true, + "login": true, + "phone": true, + "token": true, + } + consentYesStatuses = map[string]bool{ + "1": true, + "accept": true, + "agree": true, + "approve": true, + "given": true, + "good": true, + "ok": true, + "on": true, + "true": true, + "y": true, + "yes": true, + } + basisTypes = map[string]bool{ + "consent": true, + "contract": true, + "legal-requirement": true, + "legitimate-interest": true, + "public-interest": true, + "vital-interest": true, + } +) + +// Consideration why collection of meta data patch was postpone: +// 1. Databunker is not anti-fraud solution +// 2. GDPR stands for data minimalization. +// 3. Do not store what you actually do not NEED. +/* +var interestingHeaders = []string{"x-forwarded", "x-forwarded-for", "x-coming-from", "via", + "forwarded-for", "forwarded", "client-ip", "user-agent", "cookie", "referer"} + +func getMeta(r *http.Request) string { + headers := bson.M{} + for idx, val := range r.Header { + idx0 := strings.ToLower(idx) + log.Printf("Checking header: %s\n", idx0) + if contains(interestingHeaders, idx0) { + headers[idx] = val[0] + } + } + headersStr, _ := json.Marshal(headers) + meta := fmt.Sprintf(`{"clientip":"%s","headers":%s}`, r.RemoteAddr, headersStr) + log.Printf("Meta: %s\n", meta) + return meta +} +*/ + +func getStringValue(r interface{}) string { + if r == nil { + return "" + } + switch r.(type) { + case string: + return strings.TrimSpace(r.(string)) + case []uint8: + return strings.TrimSpace(string(r.([]uint8))) + case float64: + return strconv.Itoa(int(r.(float64))) + } + return "" +} + +func getIntValue(r interface{}) int { + switch r.(type) { + case int: + return r.(int) + case int32: + return int(r.(int32)) + case float64: + return int(r.(float64)) + } + return 0 +} + +func getInt64Value(records map[string]interface{}, key string) int64 { + if value, ok := records[key]; ok { + switch value.(type) { + case int32: + return int64(value.(int32)) + case int64: + return int64(value.(int64)) + } + } + return 0 +} + +// readConfFile() read configuration file. +func readConfFile(cfg *Config, filepath *string) error { + confFile := "databunker.yaml" + if filepath != nil { + if len(*filepath) > 0 { + confFile = *filepath + } + } + log.Printf("Loading configuration file: %s\n", confFile) + f, err := os.Open(confFile) + if err != nil { + return err + } + decoder := yaml.NewDecoder(f) + err = decoder.Decode(cfg) + if err != nil { + return err + } + return nil +} + +// readEnv() process environment variables. +func readEnv(cfg *Config) error { + err := envconfig.Process("", cfg) + return err +} + +func getArgEnvFileVariable(vname string, masterKeyPtr *string) string { + strvalue := "" + if masterKeyPtr != nil && len(*masterKeyPtr) > 0 { + strvalue = *masterKeyPtr + } else if len(os.Getenv(vname)) > 0 { + strvalue = os.Getenv(vname) + } else if len(os.Getenv(vname+"_FILE")) > 0 { + data, _ := os.ReadFile(os.Getenv(vname + "_FILE")) + strvalue = string(data) + } + return strings.TrimSpace(strvalue) +} + +func hashString(md5Salt []byte, src string) string { + stringToHash := append(md5Salt, []byte(src)...) + hashed := sha256.Sum256(stringToHash) + return base64.StdEncoding.EncodeToString(hashed[:]) +} + +func normalizeConsentStatus(status string) string { + status = strings.ToLower(status) + if consentYesStatuses[status] { + return "yes" + } + return "no" +} + +func normalizeBasisType(status string) string { + status = strings.ToLower(status) + if basisTypes[status] { + return status + } + return "consent" +} + +func normalizeBrief(brief string) string { + return strings.ToLower(brief) +} + +func normalizeEmail(email0 string) string { + email, _ := url.QueryUnescape(email0) + email = strings.ToLower(email) + email = strings.TrimSpace(email) + if email0 != email { + log.Printf("Email before normalization: %s, after: %s\n", email0, email) + } + return email +} + +func normalizePhone(phone string, defaultCountry string) string { + // 4444 is a phone number for testing, no need to normilize it + phone = strings.TrimSpace(phone) + if len(phone) == 0 { + return phone + } + if phone == "4444" { + return "4444" + } + if len(defaultCountry) == 0 { + // https://github.com/ttacon/libphonenumber/blob/master/countrycodetoregionmap.go + defaultCountry = "GB" + } + res, err := libphonenumber.Parse(phone, defaultCountry) + if err != nil { + log.Printf("Failed to parse phone number: %s", phone) + return "" + } + phone = "+" + strconv.Itoa(int(*res.CountryCode)) + strconv.FormatUint(*res.NationalNumber, 10) + return phone +} + +func ValidateMode(index string) bool { + return indexNames[strings.ToLower(index)] +} + +func parseFields(fields string) []string { + return strings.Split(fields, ",") +} + +// Binary search implementation for a sorted array of strings +func binarySearch(arr []string, target string) bool { + low := 0 + high := len(arr) - 1 + + for low <= high { + mid := (low + high) / 2 + if arr[mid] == target { + return true + } else if arr[mid] < target { + low = mid + 1 + } else { + high = mid - 1 + } + } + return false +} + +func contains(slice []string, item string) bool { + set := make(map[string]bool, len(slice)) + for _, s := range slice { + set[s] = true + } + return set[item] +} + +func atoi(s string) int32 { + var ( + n uint32 + i int + v byte + ) + for ; i < len(s); i++ { + d := s[i] + if '0' <= d && d <= '9' { + v = d - '0' + } else if 'a' <= d && d <= 'z' { + v = d - 'a' + 10 + } else if 'A' <= d && d <= 'Z' { + v = d - 'A' + 10 + } else { + n = 0 + break + } + n *= uint32(10) + n += uint32(v) + } + return int32(n) +} + +func setExpiration(maxExpiration string, userExpiration string) string { + if len(userExpiration) == 0 { + return maxExpiration + } + userExpirationNum, _ := parseExpiration(userExpiration) + maxExpirationNum, _ := parseExpiration(maxExpiration) + if maxExpirationNum == 0 { + maxExpiration = "6m" + maxExpirationNum, _ = parseExpiration(maxExpiration) + } + if userExpirationNum == 0 { + return maxExpiration + } + if userExpirationNum > maxExpirationNum { + return maxExpiration + } + return userExpiration +} + +func parseExpiration0(expiration string) (int32, error) { + match := regexExpiration.FindStringSubmatch(expiration) + // expiration format: 10d, 10h, 10m, 10s + if len(match) != 3 { + e := fmt.Sprintf("failed to parse expiration value: %s", expiration) + return 0, errors.New(e) + } + num := match[1] + format := match[2] + start := int32(0) + switch format { + case "d": // day + start = start + (atoi(num) * 24 * 3600) + case "h": // hour + start = start + (atoi(num) * 3600) + case "m": // month + start = start + (atoi(num) * 24 * 31 * 3600) + case "s": + start = start + (atoi(num)) + } + return start, nil +} + +func parseExpiration(expiration string) (int32, error) { + match := regexExpiration.FindStringSubmatch(expiration) + // expiration format: 10d, 10h, 10m, 10s + if len(match) == 2 { + return atoi(match[1]), nil + } + if len(match) != 3 { + e := fmt.Sprintf("failed to parse expiration value: %s", expiration) + return 0, errors.New(e) + } + num := match[1] + format := match[2] + if len(format) == 0 { + return atoi(num), nil + } + start := int32(time.Now().Unix()) + switch format { + case "d": // day + start = start + (atoi(num) * 24 * 3600) + case "h": // hour + start = start + (atoi(num) * 3600) + case "m": // month + start = start + (atoi(num) * 24 * 31 * 3600) + case "s": + start = start + (atoi(num)) + } + return start, nil +} + +func lockMemory() error { + return unix.Mlockall(syscall.MCL_CURRENT | syscall.MCL_FUTURE) +} + +func isValidUUID(uuidCode string) bool { + return regexUUID.MatchString(uuidCode) +} + +func isValidApp(app string) bool { + return regexAppName.MatchString(app) +} + +func isValidBrief(brief string) bool { + return regexBrief.MatchString(brief) +} + +func isValidHex(hex1 string) bool { + return regexHex.MatchString(hex1) +} + +func isContainer() bool { + //if _, err := os.Stat("/.dockerenv"); err == nil { + // return true + //} + if len(os.Getenv("KUBERNETES_SERVICE_HOST")) > 0 { + return true + } + if _, err := os.Stat("/var/run/secrets/kubernetes.io"); err == nil { + return true + } + return false +} + +// stringPatternMatch looks for basic human patterns like "*", "*abc*", etc... +func stringPatternMatch(pattern string, value string) bool { + if len(pattern) == 0 { + return false + } + if pattern == "*" { + return true + } + if pattern == value { + return true + } + if strings.HasPrefix(pattern, "*") && strings.HasSuffix(pattern, "*") { + pattern = pattern[1 : len(pattern)-1] + if strings.Contains(value, pattern) { + return true + } + return false + } + if strings.HasPrefix(pattern, "*") { + pattern = pattern[1:] + if strings.HasSuffix(value, pattern) { + return true + } + } else if strings.HasSuffix(pattern, "*") { + pattern = pattern[:len(pattern)-1] + if strings.HasPrefix(value, pattern) { + return true + } + } + return false +} + +func returnError(w http.ResponseWriter, r *http.Request, message string, code int, err error, event *auditEvent) { + log.Printf("[%d] %s %s -> Return error\n", code, r.Method, r.URL.Path) + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(code) + fmt.Fprintf(w, `{"status":"error","message":%q}`, message) + if event != nil { + event.Status = "error" + event.Msg = message + if err != nil { + event.Debug = err.Error() + log.Printf("Generate error response: %s, Error: %s\n", message, err.Error()) + } else { + log.Printf("Generate error response: %s\n", message) + } + } + //http.Error(w, http.StatusText(405), 405) +} + +func returnUUID(w http.ResponseWriter, code string) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(200) + fmt.Fprintf(w, `{"status":"ok","token":%q}`, code) +} + +func (e mainEnv) enforceAuth(w http.ResponseWriter, r *http.Request, event *auditEvent) string { + /* + for key, value := range r.Header { + fmt.Printf("%s => %s\n", key, value) + } + */ + if token, ok := r.Header["X-Bunker-Token"]; ok { + authResult, err := e.db.checkUserAuthXToken(token[0]) + //fmt.Printf("error in auth? error %s - %s\n", err, token[0]) + if err == nil { + if event != nil { + event.Identity = authResult.name + if authResult.ttype == "login" && authResult.token == event.Record { + return authResult.ttype + } + } + if len(authResult.ttype) > 0 && authResult.ttype != "login" { + return authResult.ttype + } + } + /* + if e.db.checkXtoken(token[0]) == true { + if event != nil { + event.Identity = "admin" + } + return true + } + */ + } + log.Printf("403 Access denied\n") + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Access denied")) + if event != nil { + event.Status = "error" + event.Msg = "access denied" + } + return "" +} + +func (e mainEnv) enforceAdmin(w http.ResponseWriter, r *http.Request, event *auditEvent) string { + if token, ok := r.Header["X-Bunker-Token"]; ok { + authResult, err := e.db.checkUserAuthXToken(token[0]) + //fmt.Printf("error in auth? error %s - %s\n", err, token[0]) + if err == nil { + if event != nil { + event.Identity = authResult.name + } + if len(authResult.ttype) > 0 && authResult.ttype != "login" { + return authResult.ttype + } + } + } + log.Printf("403 Access denied\n") + w.WriteHeader(http.StatusForbidden) + w.Write([]byte("Access denied")) + return "" +} + +func EnforceUUID(w http.ResponseWriter, uuidCode string, event *auditEvent) bool { + if isValidUUID(uuidCode) == false { + //fmt.Printf("405 bad uuid in : %s\n", uuidCode) + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(405) + fmt.Fprintf(w, `{"status":"error","message":"bad uuid"}`) + if event != nil { + event.Status = "error" + event.Msg = "bad uuid" + } + return false + } + return true +} + +func getJSONPostMap(r *http.Request) (map[string]interface{}, error) { + cType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil { + log.Printf("ignoring empty content-type: %s\n", err) + return nil, nil + } + cType = strings.ToLower(cType) + records := make(map[string]interface{}) + if r.Method == "DELETE" { + // otherwise data is not parsed! + r.Method = "PATCH" + } + body0, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, err + } + body := strings.TrimSpace(string(body0)) + if len(body) < 3 { + return nil, nil + } + if strings.HasPrefix(cType, "application/x-www-form-urlencoded") { + if body[0] == '{' { + return nil, errors.New("wrong content-type, json instead of url encoded data") + } + form, err := url.ParseQuery(body) + if err != nil { + log.Printf("error to parse HTTP data request: %s\n", err) + return nil, err + } + if len(form) == 0 { + return nil, nil + } + for key, value := range form { + //fmt.Printf("data here %s => %s\n", key, value[0]) + records[key] = value[0] + } + } else if strings.HasPrefix(cType, "application/json") { + err = json.Unmarshal([]byte(body), &records) + if err != nil { + log.Printf("Error in json decode %s", err) + return nil, err + } + } else if strings.HasPrefix(cType, "application/xml") { + err = json.Unmarshal([]byte(body), &records) + if err != nil { + log.Printf("Error in xml/json decode %s", err) + return nil, err + } + } else { + log.Printf("Ignore wrong content type: %s", cType) + maxStrLen := 200 + if len(body) < maxStrLen { + maxStrLen = len(body) + } + log.Printf("Body[max 200 chars]: %s", body[0:maxStrLen]) + return nil, nil + } + return records, nil +} + +func getJSONPostData(r *http.Request) ([]byte, error) { + cType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil { + log.Printf("ignoring empty content-type: %s\n", err) + return nil, nil + } + cType = strings.ToLower(cType) + if r.Method == "DELETE" { + // otherwise data is not parsed! + r.Method = "PATCH" + } + body0, err := ioutil.ReadAll(r.Body) + if err != nil { + return nil, err + } + body := strings.TrimSpace(string(body0)) + if len(body) < 3 { + return nil, nil + } + if strings.HasPrefix(cType, "application/x-www-form-urlencoded") { + if body[0] == '{' || body[0] == '[' { + return nil, errors.New("wrong content-type, json instead of url encoded data") + } + form, err := url.ParseQuery(body) + if err != nil { + log.Printf("error in HTTP data request: %s\n", err) + return nil, err + } + if len(form) == 0 { + return nil, nil + } + records := make(map[string]interface{}) + for key, value := range form { + records[key] = value[0] + } + return json.Marshal(records) + } else if strings.HasPrefix(cType, "application/json") || strings.HasPrefix(cType, "application/xml") { + var data interface{} + err := json.Unmarshal([]byte(body), &data) + if err != nil { + return nil, errors.New("error decoding json data") + } + return json.Marshal(data) + } + log.Printf("Ignore wrong content type: %s", cType) + maxStrLen := 200 + if len(body) < maxStrLen { + maxStrLen = len(body) + } + log.Printf("Body[max 200 chars]: %s", body[0:maxStrLen]) + return nil, errors.New("wrong content-type, not a json string") +} + +func getIndexString(val interface{}) string { + switch val.(type) { + case nil: + return "" + case string: + return strings.TrimSpace(val.(string)) + case []uint8: + return strings.TrimSpace(string(val.([]uint8))) + case int: + return strconv.Itoa(val.(int)) + case int64: + return fmt.Sprintf("%v", val.(int64)) + case float64: + return strconv.Itoa(int(val.(float64))) + } + return "" +} + +func getUserJSON(r *http.Request, defaultCountry string) (userJSON, error) { + var result userJSON + records, err := getJSONPostMap(r) + if err != nil { + return result, err + } + if records == nil { + return result, nil + } + + if value, ok := records["login"]; ok { + result.loginIdx = getIndexString(value) + } + if value, ok := records["email"]; ok { + result.emailIdx = normalizeEmail(getIndexString(value)) + } + if value, ok := records["phone"]; ok { + result.phoneIdx = normalizePhone(getIndexString(value), defaultCountry) + } + if value, ok := records["custom"]; ok { + result.customIdx = getIndexString(value) + } + if value, ok := records["token"]; ok { + result.token = value.(string) + } + result.jsonData, err = json.Marshal(records) + return result, err +} + +var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ") + +func randSeq(n int) string { + b := make([]rune, n) + for i := range b { + b[i] = letters[rand.Intn(len(letters))] + } + return string(b) +} + +var numbers0 = []rune("123456789") +var numbers = []rune("0123456789") + +func randNum(n int) int32 { + b := make([]rune, n) + for i := range b { + b[i] = numbers[rand.Intn(len(numbers))] + } + b[0] = numbers0[rand.Intn(len(numbers0))] + return atoi(string(b)) +}