Initial cloud-services repo - gateway service + pkg modules

pkg/remotefileupload/azure.go

@@ -0,0 +1,123 @@
+package remotefileupload
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+
+	"fiskerinc.com/modules/logger"
+	"fiskerinc.com/modules/utils/envtool"
+
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/appendblob"
+	"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
+	"github.com/pkg/errors"
+)
+
+var (
+	azureAccount    = envtool.GetEnv("AZURE_STORAGE_ACCOUNT", "REPLACE_ME")
+	azureAccountKey = envtool.GetEnv("AZURE_STORAGE_ACCESS_KEY", "REPLACE_ME")
+)
+
+// NewAzureUploader creates a new AzureUploader instance using env variables
+func NewAzureUploader(azureStorageContainerName string, azureFileExtension string) (Uploader, error) {
+	a := &AzureUploader{
+		accountName:   azureAccount,
+		containerName: azureStorageContainerName,
+		fileExtension: azureFileExtension,
+	}
+
+	cred, err := azblob.NewSharedKeyCredential(a.accountName, azureAccountKey)
+	if err != nil {
+		return a, errors.WithStack(err)
+	}
+
+	containerPath := fmt.Sprintf("https://%s.blob.core.windows.net/%s/", a.accountName, a.containerName)
+
+	a.containerPath = containerPath
+	a.azureCredentials = cred
+	return a, nil
+}
+
+// AzureUploader stores file location and creds to perform AppendBlock operation to blobs
+type AzureUploader struct {
+	accountName      string
+	containerName    string
+	fileExtension    string
+	containerPath    string
+	azureCredentials *azblob.SharedKeyCredential
+}
+
+// Upload appends new chunk of data to end of blob
+// if blob doesn't exist, creates blob and then appends data
+// logName: A name for if something goes wrong: i.e.: filePath[logIndex]
+// logIndex: What piece of data should be logged if something goes wrong
+func (a *AzureUploader) Upload(block []byte, logValue LogPayload, filePath ...string) (string, error) {
+	ctx := context.Background()
+	blobURL := a.azureBlobURL(a.containerPath, filePath)
+	client, err := appendblob.NewClientWithSharedKeyCredential(blobURL, a.azureCredentials, nil)
+	if err != nil {
+		return "", err
+	}
+
+	logger.Debug().Str(logValue.Title, logValue.Value).Msgf("sending block of length %d to azure container: %s", len(block), blobURL)
+
+	reader := NopCloser(bytes.NewReader(block))
+	func() {
+		// Instead of trying to send data to a blob, and then determining if it exists, lets just check if it exists
+		_, err := client.GetProperties(ctx, nil)
+		if err != nil {
+			if !bloberror.HasCode(err, bloberror.BlobNotFound) {
+				logger.Error().Str(logValue.Title, logValue.Value).Err(err).Send()
+				return
+			}
+			_, err = client.Create(ctx, nil)
+			if err != nil {
+				logger.Error().Str(logValue.Title, logValue.Value).Err(err).Send()
+				return
+			}
+		}
+
+		_, err = client.AppendBlock(ctx, reader, nil)
+		if err != nil {
+			logger.Error().Str(logValue.Title, logValue.Value).Err(err).Send()
+			return
+		}
+
+		logger.Debug().Str(logValue.Title, logValue.Value).Msgf("upload complete")
+	}()
+	return blobURL, nil
+}
+
+// basePath is the url to the blob storage (<account>.azurebloburl.net/<containername>)
+// filepath will be added onto basepath /<your>/<file>/<path>
+func (a *AzureUploader) azureBlobURL(basePath string, filePath []string) string {
+	fileName := fmt.Sprintf("%s%s", "raw", a.fileExtension)
+	finalPath, _ := url.JoinPath(basePath, filePath...)
+	finalPath, _ = url.JoinPath(finalPath, fileName)
+
+	return finalPath
+}
+
+type nopCloser struct {
+	io.ReadSeeker
+}
+
+func (n nopCloser) Close() error {
+	return nil
+}
+
+// NopCloser returns a ReadSeekCloser with a no-op close method wrapping the provided io.ReadSeeker.
+func NopCloser(rs io.ReadSeeker) io.ReadSeekCloser {
+	return nopCloser{rs}
+}
+
+// Azure Account is the whole storage account name such as fiskercloudstg
+// AzureCotnainerName is the name of the specific container such as trexlogs
+// Then the path is the path to the file i.e.: "someVIN", "2023", "05", "03", "raw.log"
+func AzureFilePathLink(AzureAccount, AzureContainerName string, PathPieces ...string) (link string, err error) {
+	link = fmt.Sprintf("https://%s.blob.core.windows.net/%s", AzureAccount, AzureContainerName)
+	return url.JoinPath(link, PathPieces...)
+}