124 lines
4.0 KiB
Go
124 lines
4.0 KiB
Go
package remotefileupload
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/url"
|
|
|
|
"github.com/fiskerinc/cloud-services/pkg/logger"
|
|
"github.com/fiskerinc/cloud-services/pkg/utils/envtool"
|
|
|
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob"
|
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/appendblob"
|
|
"github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/bloberror"
|
|
"github.com/pkg/errors"
|
|
)
|
|
|
|
var (
|
|
azureAccount = envtool.GetEnv("AZURE_STORAGE_ACCOUNT", "REPLACE_ME")
|
|
azureAccountKey = envtool.GetEnv("AZURE_STORAGE_ACCESS_KEY", "REPLACE_ME")
|
|
)
|
|
|
|
// NewAzureUploader creates a new AzureUploader instance using env variables
|
|
func NewAzureUploader(azureStorageContainerName string, azureFileExtension string) (Uploader, error) {
|
|
a := &AzureUploader{
|
|
accountName: azureAccount,
|
|
containerName: azureStorageContainerName,
|
|
fileExtension: azureFileExtension,
|
|
}
|
|
|
|
cred, err := azblob.NewSharedKeyCredential(a.accountName, azureAccountKey)
|
|
if err != nil {
|
|
return a, errors.WithStack(err)
|
|
}
|
|
|
|
containerPath := fmt.Sprintf("https://%s.blob.core.windows.net/%s/", a.accountName, a.containerName)
|
|
|
|
a.containerPath = containerPath
|
|
a.azureCredentials = cred
|
|
return a, nil
|
|
}
|
|
|
|
// AzureUploader stores file location and creds to perform AppendBlock operation to blobs
|
|
type AzureUploader struct {
|
|
accountName string
|
|
containerName string
|
|
fileExtension string
|
|
containerPath string
|
|
azureCredentials *azblob.SharedKeyCredential
|
|
}
|
|
|
|
// Upload appends new chunk of data to end of blob
|
|
// if blob doesn't exist, creates blob and then appends data
|
|
// logName: A name for if something goes wrong: i.e.: filePath[logIndex]
|
|
// logIndex: What piece of data should be logged if something goes wrong
|
|
func (a *AzureUploader) Upload(block []byte, logValue LogPayload, filePath ...string) (string, error) {
|
|
ctx := context.Background()
|
|
blobURL := a.azureBlobURL(a.containerPath, filePath)
|
|
client, err := appendblob.NewClientWithSharedKeyCredential(blobURL, a.azureCredentials, nil)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
|
|
logger.Debug().Str(logValue.Title, logValue.Value).Msgf("sending block of length %d to azure container: %s", len(block), blobURL)
|
|
|
|
reader := NopCloser(bytes.NewReader(block))
|
|
func() {
|
|
// Instead of trying to send data to a blob, and then determining if it exists, lets just check if it exists
|
|
_, err := client.GetProperties(ctx, nil)
|
|
if err != nil {
|
|
if !bloberror.HasCode(err, bloberror.BlobNotFound) {
|
|
logger.Error().Str(logValue.Title, logValue.Value).Err(err).Send()
|
|
return
|
|
}
|
|
_, err = client.Create(ctx, nil)
|
|
if err != nil {
|
|
logger.Error().Str(logValue.Title, logValue.Value).Err(err).Send()
|
|
return
|
|
}
|
|
}
|
|
|
|
_, err = client.AppendBlock(ctx, reader, nil)
|
|
if err != nil {
|
|
logger.Error().Str(logValue.Title, logValue.Value).Err(err).Send()
|
|
return
|
|
}
|
|
|
|
logger.Debug().Str(logValue.Title, logValue.Value).Msgf("upload complete")
|
|
}()
|
|
return blobURL, nil
|
|
}
|
|
|
|
// basePath is the url to the blob storage (<account>.azurebloburl.net/<containername>)
|
|
// filepath will be added onto basepath /<your>/<file>/<path>
|
|
func (a *AzureUploader) azureBlobURL(basePath string, filePath []string) string {
|
|
fileName := fmt.Sprintf("%s%s", "raw", a.fileExtension)
|
|
finalPath, _ := url.JoinPath(basePath, filePath...)
|
|
finalPath, _ = url.JoinPath(finalPath, fileName)
|
|
|
|
return finalPath
|
|
}
|
|
|
|
type nopCloser struct {
|
|
io.ReadSeeker
|
|
}
|
|
|
|
func (n nopCloser) Close() error {
|
|
return nil
|
|
}
|
|
|
|
// NopCloser returns a ReadSeekCloser with a no-op close method wrapping the provided io.ReadSeeker.
|
|
func NopCloser(rs io.ReadSeeker) io.ReadSeekCloser {
|
|
return nopCloser{rs}
|
|
}
|
|
|
|
// Azure Account is the whole storage account name such as fiskercloudstg
|
|
// AzureCotnainerName is the name of the specific container such as trexlogs
|
|
// Then the path is the path to the file i.e.: "someVIN", "2023", "05", "03", "raw.log"
|
|
func AzureFilePathLink(AzureAccount, AzureContainerName string, PathPieces ...string) (link string, err error) {
|
|
link = fmt.Sprintf("https://%s.blob.core.windows.net/%s", AzureAccount, AzureContainerName)
|
|
return url.JoinPath(link, PathPieces...)
|
|
}
|