Initial cloud-services repo - gateway service + pkg modules

pkg/cache/fileids.go

@@ -0,0 +1,147 @@
+package cache
+
+import (
+	"encoding/json"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/db/queries"
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/redis"
+
+	r "github.com/gomodule/redigo/redis"
+	"github.com/pkg/errors"
+)
+
+func RetrieveFileEncryptionParams(client redis.Client, db queries.FileKeysInterface, fileids []string) ([]common.FileKeyResponse, error) {
+	result, err := retrieveFileEncryptionParamsRedis(client, fileids)
+	if err != nil && !errors.Is(err, redis.ErrNilObject) {
+		return nil, err
+	}
+
+	dbFileIDs := missingFileIDs(result, fileids)
+	if len(dbFileIDs) > 0 {
+		rows, err := retrieveFileEncryptionParamsDB(db, dbFileIDs)
+		if err != nil {
+			return nil, err
+		}
+		err = cacheFileEncryptionParamsRedis(client, rows, redisObjectExpireDay)
+		if err != nil {
+			return nil, err
+		}
+
+		result = append(rows, result...)
+	}
+
+	return result, nil
+}
+
+func retrieveFileEncryptionParamsRedis(client redis.Client, fileids []string) ([]common.FileKeyResponse, error) {
+	keys := make([]string, len(fileids))
+
+	for i, fileid := range fileids {
+		keys[i] = redis.FileIDEncryptionParamsKey(fileid)
+	}
+
+	if len(keys) == 0 {
+		return []common.FileKeyResponse{}, nil
+	}
+
+	values, err := client.GetMulti(keys)
+	if err != nil {
+		return nil, err
+	}
+
+	result, err := getFileKeyResponses(values)
+	return result, err
+}
+
+func getFileKeyResponses(values []interface{}) ([]common.FileKeyResponse, error) {
+	result := []common.FileKeyResponse{}
+
+	for _, value := range values {
+		if value == nil {
+			continue
+		}
+
+		file := common.FileKeyResponse{}
+		data, err := r.Bytes(value, nil)
+		if err != nil {
+			file.Error = err.Error()
+			return result, errors.WithStack(err)
+		}
+
+		err = json.Unmarshal(data, &file)
+		if err != nil {
+			return result, errors.WithStack(err)
+		}
+
+		result = append(result, file)
+	}
+
+	return result, nil
+}
+
+func missingFileIDs(items []common.FileKeyResponse, fileids []string) []string {
+	result := []string{}
+	hash := map[string]bool{}
+
+	for _, item := range items {
+		if item.FileID != "" {
+			hash[item.FileID] = true
+		}
+	}
+
+	for _, fileid := range fileids {
+		if !hash[fileid] {
+			result = append(result, fileid)
+		}
+	}
+
+	return result
+}
+
+func retrieveFileEncryptionParamsDB(db queries.FileKeysInterface, fileids []string) ([]common.FileKeyResponse, error) {
+	result := make([]common.FileKeyResponse, len(fileids))
+	hash := map[string]bool{}
+	data, err := db.GetMulti(fileids)
+	if err != nil {
+		return nil, errors.WithStack(err)
+	}
+
+	for i, file := range data {
+		result[i].Apply(&file)
+		hash[file.FileID] = true
+	}
+
+	if len(fileids) != len(data) {
+		starting := len(data)
+		current := 0
+		for _, fileid := range fileids {
+			if !hash[fileid] {
+				logger.Warn().Msgf("file id %s not found", fileid)
+				file := &result[starting+current]
+				file.FileID = fileid
+				file.Error = "not found"
+				current++
+			}
+		}
+	}
+
+	return result, nil
+}
+
+func cacheFileEncryptionParamsRedis(client redis.Client, files []common.FileKeyResponse, expire int) error {
+	batch := redis.NewRedisBatchCommands()
+
+	for _, file := range files {
+		serialized, err := json.Marshal(file)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+		id := redis.FileIDEncryptionParamsKey(file.FileID)
+		batch.Add("SET", id, serialized, "EX", expire)
+	}
+
+	_, err := client.ExecuteBatch(batch)
+	return errors.WithStack(err)
+}