Initial cloud-services repo - gateway service + pkg modules
This commit is contained in:
Chris Rai
241
pkg/azurestoragecontainer/asc.go
Normal file
241
pkg/azurestoragecontainer/asc.go
Normal file
@@ -0,0 +1,241 @@
|
||||
package azurestoragecontainer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"fiskerinc.com/modules/logger"
|
||||
"github.com/Azure/azure-storage-blob-go/azblob"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// This set of modules will contain the ability to get the list of files from a container, and create a sas link to download them from
|
||||
|
||||
var azureLogsBlobPath = "https://%s.blob.core.windows.net/%s"
|
||||
|
||||
// For your module to use this they need a CollectionManagement. These store your connection information to
|
||||
// azure. Leave it up to the user to not recreate everytime
|
||||
type CollectionManagement struct {
|
||||
SharedKeyCredential *azblob.SharedKeyCredential
|
||||
BaseLink *url.URL // AzureAccount.blob.core.windows.net/LogsContainerName
|
||||
cachedTokenTime time.Time // The Time we last got the token
|
||||
cachedToken string // Once a day get a new access token
|
||||
cmci CollectionManagementConnectionInformation
|
||||
}
|
||||
|
||||
type CollectionManagementConnectionInformation struct {
|
||||
AzureAccount string
|
||||
AzureContainerName string
|
||||
AzureAccountKey string
|
||||
}
|
||||
|
||||
// Need somekind of one time sync thing. Lets worry about it later. Probably each service needs its own
|
||||
func NewCollectionConnection(cmci CollectionManagementConnectionInformation) (cm *CollectionManagement, err error) {
|
||||
collect := CollectionManagement{}
|
||||
|
||||
link := makeAzureBlobLink(cmci.AzureAccount, cmci.AzureContainerName)
|
||||
parsedURL, err := url.Parse(link)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
return
|
||||
}
|
||||
collect.BaseLink = parsedURL
|
||||
|
||||
cred, err := azblob.NewSharedKeyCredential(cmci.AzureAccount, cmci.AzureAccountKey)
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
collect.SharedKeyCredential = cred
|
||||
|
||||
collect.cmci = cmci
|
||||
|
||||
return &collect, err
|
||||
}
|
||||
|
||||
type FilePath struct {
|
||||
Path string `json:"Path,omitempty"`
|
||||
File bool
|
||||
UnderPaths []*FilePath `json:"UnderPaths,omitempty"`
|
||||
}
|
||||
|
||||
// This is working code to get the full set of all the different files. I am unsure why I wrote this for this ticket,
|
||||
// but could be used to grey out the calender on the ota-portal
|
||||
|
||||
// Generate a file path for all the files
|
||||
// Prefix is the possible begging file path: e.g. vin/year will get all the files under that year folder
|
||||
// rootOnly will only get the root files instead of nested folders
|
||||
func (cm *CollectionManagement) GetFolderStruct(prefix string, rootOnly bool) (fp *FilePath, err error) {
|
||||
fp, err = cm.startRecursiveFilePathSearch(prefix, rootOnly)
|
||||
return
|
||||
}
|
||||
|
||||
// If prefix is empty, start at the root
|
||||
func (cm *CollectionManagement) startRecursiveFilePathSearch(prefix string, rootOnly bool) (startPath *FilePath, err error) {
|
||||
cred := cm.SharedKeyCredential
|
||||
parsedURL := cm.BaseLink
|
||||
containerURL := azblob.NewContainerURL(*parsedURL, azblob.NewPipeline(cred, azblob.PipelineOptions{}))
|
||||
|
||||
//marker := azblob.Marker{}
|
||||
startPath = &FilePath{}
|
||||
startPath.Path = prefix
|
||||
startPath.UnderPaths = make([]*FilePath, 0)
|
||||
if prefix != "" {
|
||||
prefix = prefix + "/"
|
||||
}
|
||||
|
||||
marker := azblob.Marker{}
|
||||
for marker.NotDone() {
|
||||
// This should probably be outside the for loop?
|
||||
hierarchBlob, err := containerURL.ListBlobsHierarchySegment(context.Background(), marker, "/", azblob.ListBlobsSegmentOptions{
|
||||
Prefix: prefix,
|
||||
})
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
return startPath, err
|
||||
}
|
||||
// BlobItems are actual files
|
||||
for _, file := range hierarchBlob.Segment.BlobItems {
|
||||
startPath.UnderPaths = append(startPath.UnderPaths, &FilePath{Path: file.Name[len(prefix):], File: true})
|
||||
}
|
||||
if !rootOnly {
|
||||
for _, path := range hierarchBlob.Segment.BlobPrefixes {
|
||||
startPath.UnderPaths = append(startPath.UnderPaths, recursiveFilePathSearch(path.Name, containerURL))
|
||||
}
|
||||
}
|
||||
|
||||
marker = hierarchBlob.NextMarker
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func makeAzureBlobLink(azureAccountName, azureContainerName string) string {
|
||||
link := fmt.Sprintf(
|
||||
azureLogsBlobPath,
|
||||
azureAccountName,
|
||||
azureContainerName,
|
||||
)
|
||||
return link //fmt.Sprintf("%s / %s", link, sasToken), err
|
||||
}
|
||||
|
||||
// We start at the path and explore all those children
|
||||
// Prefix is the path
|
||||
func recursiveFilePathSearch(prefix string, containerURL azblob.ContainerURL) (fp *FilePath) {
|
||||
fp = &FilePath{
|
||||
UnderPaths: make([]*FilePath, 0),
|
||||
}
|
||||
|
||||
paths := strings.Split(strings.TrimRight(prefix, "/"), "/")
|
||||
fp.Path = paths[len(paths)-1]
|
||||
|
||||
marker := azblob.Marker{}
|
||||
for marker.NotDone() {
|
||||
hierarchBlob, err := containerURL.ListBlobsHierarchySegment(context.Background(), marker, "/", azblob.ListBlobsSegmentOptions{
|
||||
Prefix: prefix,
|
||||
})
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
// BlobItems are actual files
|
||||
for _, file := range hierarchBlob.Segment.BlobItems {
|
||||
_, nextFile := filepath.Split(file.Name)
|
||||
fp.UnderPaths = append(fp.UnderPaths, &FilePath{Path: nextFile, File: true})
|
||||
//file.Name
|
||||
}
|
||||
//path.name includes the whole path
|
||||
for _, path := range hierarchBlob.Segment.BlobPrefixes {
|
||||
fp.UnderPaths = append(fp.UnderPaths, recursiveFilePathSearch(path.Name, containerURL))
|
||||
}
|
||||
marker = hierarchBlob.NextMarker
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Returns a list of full file paths
|
||||
func (fp *FilePath) ReturnFilePaths() []string {
|
||||
filePaths := make([]string, 0)
|
||||
|
||||
for _, uFP := range fp.UnderPaths {
|
||||
if uFP.File {
|
||||
filePaths = append(filePaths, fp.Path+uFP.Path)
|
||||
} else {
|
||||
morePaths := uFP.ReturnFilePaths()
|
||||
for x := range morePaths {
|
||||
filePaths = append(filePaths, fp.Path+morePaths[x])
|
||||
}
|
||||
}
|
||||
}
|
||||
return filePaths
|
||||
}
|
||||
|
||||
// If returns true, remove the file
|
||||
type FileFilter func(fileName string) (remove bool)
|
||||
|
||||
// Assuming FilePath is not a file itself. Only its children are
|
||||
func (fp *FilePath) FilterFiles(ff FileFilter) {
|
||||
fp.filterFilesRecursive(ff)
|
||||
}
|
||||
|
||||
// If the child item returns true to be filtered, we prune it from our child list
|
||||
func (fp *FilePath) filterFilesRecursive(ff FileFilter) bool {
|
||||
if fp.File {
|
||||
return ff(fp.Path)
|
||||
}
|
||||
|
||||
// The index to still keep up to
|
||||
keepDex := 0
|
||||
for pth := range fp.UnderPaths {
|
||||
remove := fp.UnderPaths[pth].filterFilesRecursive(ff)
|
||||
if !remove {
|
||||
fp.UnderPaths[keepDex] = fp.UnderPaths[pth]
|
||||
keepDex++
|
||||
}
|
||||
}
|
||||
fp.UnderPaths = fp.UnderPaths[:keepDex]
|
||||
|
||||
return keepDex == 0
|
||||
}
|
||||
|
||||
func (cm *CollectionManagement) GetAzureBlobLink(filePath string) (link string, err error) {
|
||||
link, err = url.JoinPath(cm.BaseLink.String(), filePath)
|
||||
if err != nil {
|
||||
logger.Err(err).Msg("")
|
||||
return
|
||||
}
|
||||
sasToken, err := cm.getSASAccessTokenOnceADay()
|
||||
if err != nil {
|
||||
logger.Err(err).Msg("")
|
||||
}
|
||||
return link + "?" + sasToken, err
|
||||
}
|
||||
|
||||
func (cm *CollectionManagement) getSASAccessTokenOnceADay() (token string, err error) {
|
||||
if time.Since(cm.cachedTokenTime) > time.Hour*24 {
|
||||
cm.cachedTokenTime = time.Now()
|
||||
cm.cachedToken, err = cm.generateSASToken()
|
||||
}
|
||||
return cm.cachedToken, err
|
||||
}
|
||||
|
||||
func (cm *CollectionManagement) generateSASToken() (token string, err error) {
|
||||
|
||||
sasQueryParams, err := azblob.BlobSASSignatureValues{
|
||||
Protocol: azblob.SASProtocolHTTPS,
|
||||
StartTime: time.Now().UTC(),
|
||||
ExpiryTime: time.Now().UTC().Add(48 * time.Hour),
|
||||
Permissions: azblob.BlobSASPermissions{Read: true, List: true}.String(),
|
||||
IPRange: azblob.IPRange{},
|
||||
ContainerName: cm.cmci.AzureContainerName,
|
||||
}.NewSASQueryParameters(cm.SharedKeyCredential)
|
||||
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Msg("Failed to sas.BlobSignatureValues")
|
||||
return
|
||||
}
|
||||
|
||||
token = sasQueryParams.Encode()
|
||||
return
|
||||
}
|
||||
Reference in New Issue
Block a user