package remotefileupload import ( "bytes" "context" "fmt" "io" "net/url" "fiskerinc.com/modules/logger" "fiskerinc.com/modules/utils/envtool" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/appendblob" "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror" "github.com/pkg/errors" ) var ( azureAccount = envtool.GetEnv("AZURE_STORAGE_ACCOUNT", "REPLACE_ME") azureAccountKey = envtool.GetEnv("AZURE_STORAGE_ACCESS_KEY", "REPLACE_ME") ) // NewAzureUploader creates a new AzureUploader instance using env variables func NewAzureUploader(azureStorageContainerName string, azureFileExtension string) (Uploader, error) { a := &AzureUploader{ accountName: azureAccount, containerName: azureStorageContainerName, fileExtension: azureFileExtension, } cred, err := azblob.NewSharedKeyCredential(a.accountName, azureAccountKey) if err != nil { return a, errors.WithStack(err) } containerPath := fmt.Sprintf("https://%s.blob.core.windows.net/%s/", a.accountName, a.containerName) a.containerPath = containerPath a.azureCredentials = cred return a, nil } // AzureUploader stores file location and creds to perform AppendBlock operation to blobs type AzureUploader struct { accountName string containerName string fileExtension string containerPath string azureCredentials *azblob.SharedKeyCredential } // Upload appends new chunk of data to end of blob // if blob doesn't exist, creates blob and then appends data // logName: A name for if something goes wrong: i.e.: filePath[logIndex] // logIndex: What piece of data should be logged if something goes wrong func (a *AzureUploader) Upload(block []byte, logValue LogPayload, filePath ...string) (string, error) { ctx := context.Background() blobURL := a.azureBlobURL(a.containerPath, filePath) client, err := appendblob.NewClientWithSharedKeyCredential(blobURL, a.azureCredentials, nil) if err != nil { return "", err } logger.Debug().Str(logValue.Title, logValue.Value).Msgf("sending block of length %d to azure container: %s", len(block), blobURL) reader := NopCloser(bytes.NewReader(block)) func() { // Instead of trying to send data to a blob, and then determining if it exists, lets just check if it exists _, err := client.GetProperties(ctx, nil) if err != nil { if !bloberror.HasCode(err, bloberror.BlobNotFound) { logger.Error().Str(logValue.Title, logValue.Value).Err(err).Send() return } _, err = client.Create(ctx, nil) if err != nil { logger.Error().Str(logValue.Title, logValue.Value).Err(err).Send() return } } _, err = client.AppendBlock(ctx, reader, nil) if err != nil { logger.Error().Str(logValue.Title, logValue.Value).Err(err).Send() return } logger.Debug().Str(logValue.Title, logValue.Value).Msgf("upload complete") }() return blobURL, nil } // basePath is the url to the blob storage (.azurebloburl.net/) // filepath will be added onto basepath /// func (a *AzureUploader) azureBlobURL(basePath string, filePath []string) string { fileName := fmt.Sprintf("%s%s", "raw", a.fileExtension) finalPath, _ := url.JoinPath(basePath, filePath...) finalPath, _ = url.JoinPath(finalPath, fileName) return finalPath } type nopCloser struct { io.ReadSeeker } func (n nopCloser) Close() error { return nil } // NopCloser returns a ReadSeekCloser with a no-op close method wrapping the provided io.ReadSeeker. func NopCloser(rs io.ReadSeeker) io.ReadSeekCloser { return nopCloser{rs} } // Azure Account is the whole storage account name such as fiskercloudstg // AzureCotnainerName is the name of the specific container such as trexlogs // Then the path is the path to the file i.e.: "someVIN", "2023", "05", "03", "raw.log" func AzureFilePathLink(AzureAccount, AzureContainerName string, PathPieces ...string) (link string, err error) { link = fmt.Sprintf("https://%s.blob.core.windows.net/%s", AzureAccount, AzureContainerName) return url.JoinPath(link, PathPieces...) }