cloud-services/pkg/azurestoragecontainer/asc.go

package azurestoragecontainer

import (
	"context"
	"fmt"
	"net/url"
	"path/filepath"
	"strings"
	"time"

	"fiskerinc.com/modules/logger"
	"github.com/Azure/azure-storage-blob-go/azblob"
	"github.com/pkg/errors"
)

// This set of modules will contain the ability to get the list of files from a container, and create a sas link to download them from

var azureLogsBlobPath = "https://%s.blob.core.windows.net/%s"

// For your module to use this they need a CollectionManagement. These store your connection information to
// azure. Leave it up to the user to not recreate everytime
type CollectionManagement struct {
	SharedKeyCredential *azblob.SharedKeyCredential
	BaseLink            *url.URL  // AzureAccount.blob.core.windows.net/LogsContainerName
	cachedTokenTime     time.Time // The Time we last got the token
	cachedToken         string    // Once a day get a new access token
	cmci                CollectionManagementConnectionInformation
}

type CollectionManagementConnectionInformation struct {
	AzureAccount       string
	AzureContainerName string
	AzureAccountKey    string
}

// Need somekind of one time sync thing. Lets worry about it later. Probably each service needs its own
func NewCollectionConnection(cmci CollectionManagementConnectionInformation) (cm *CollectionManagement, err error) {
	collect := CollectionManagement{}

	link := makeAzureBlobLink(cmci.AzureAccount, cmci.AzureContainerName)
	parsedURL, err := url.Parse(link)
	if err != nil {
		err = errors.WithStack(err)
		return
	}
	collect.BaseLink = parsedURL

	cred, err := azblob.NewSharedKeyCredential(cmci.AzureAccount, cmci.AzureAccountKey)
	if err != nil {
		return nil, errors.WithStack(err)
	}
	collect.SharedKeyCredential = cred

	collect.cmci = cmci

	return &collect, err
}

type FilePath struct {
	Path       string `json:"Path,omitempty"`
	File       bool
	UnderPaths []*FilePath `json:"UnderPaths,omitempty"`
}

// This is working code to get the full set of all the different files. I am unsure why I wrote this for this ticket,
// but could be used to grey out the calender on the ota-portal

// Generate a file path for all the files
// Prefix is the possible begging file path: e.g. vin/year will get all the files under that year folder
// rootOnly will only get the root files instead of nested folders
func (cm *CollectionManagement) GetFolderStruct(prefix string, rootOnly bool) (fp *FilePath, err error) {
	fp, err = cm.startRecursiveFilePathSearch(prefix, rootOnly)
	return
}

// If prefix is empty, start at the root
func (cm *CollectionManagement) startRecursiveFilePathSearch(prefix string, rootOnly bool) (startPath *FilePath, err error) {
	cred := cm.SharedKeyCredential
	parsedURL := cm.BaseLink
	containerURL := azblob.NewContainerURL(*parsedURL, azblob.NewPipeline(cred, azblob.PipelineOptions{}))

	//marker := azblob.Marker{}
	startPath = &FilePath{}
	startPath.Path = prefix
	startPath.UnderPaths = make([]*FilePath, 0)
	if prefix != "" {
		prefix = prefix + "/"
	}

	marker := azblob.Marker{}
	for marker.NotDone() {
		// This should probably be outside the for loop?
		hierarchBlob, err := containerURL.ListBlobsHierarchySegment(context.Background(), marker, "/", azblob.ListBlobsSegmentOptions{
			Prefix: prefix,
		})
		if err != nil {
			err = errors.WithStack(err)
			return startPath, err
		}
		// BlobItems are actual files
		for _, file := range hierarchBlob.Segment.BlobItems {
			startPath.UnderPaths = append(startPath.UnderPaths, &FilePath{Path: file.Name[len(prefix):], File: true})
		}
		if !rootOnly {
			for _, path := range hierarchBlob.Segment.BlobPrefixes {
				startPath.UnderPaths = append(startPath.UnderPaths, recursiveFilePathSearch(path.Name, containerURL))
			}
		}

		marker = hierarchBlob.NextMarker
	}
	return
}

func makeAzureBlobLink(azureAccountName, azureContainerName string) string {
	link := fmt.Sprintf(
		azureLogsBlobPath,
		azureAccountName,
		azureContainerName,
	)
	return link //fmt.Sprintf("%s  /   %s", link, sasToken), err
}

// We start at the path and explore all those children
// Prefix is the path
func recursiveFilePathSearch(prefix string, containerURL azblob.ContainerURL) (fp *FilePath) {
	fp = &FilePath{
		UnderPaths: make([]*FilePath, 0),
	}

	paths := strings.Split(strings.TrimRight(prefix, "/"), "/")
	fp.Path = paths[len(paths)-1]

	marker := azblob.Marker{}
	for marker.NotDone() {
		hierarchBlob, err := containerURL.ListBlobsHierarchySegment(context.Background(), marker, "/", azblob.ListBlobsSegmentOptions{
			Prefix: prefix,
		})
		if err != nil {
			return
		}
		// BlobItems are actual files
		for _, file := range hierarchBlob.Segment.BlobItems {
			_, nextFile := filepath.Split(file.Name)
			fp.UnderPaths = append(fp.UnderPaths, &FilePath{Path: nextFile, File: true})
			//file.Name
		}
		//path.name includes the whole path
		for _, path := range hierarchBlob.Segment.BlobPrefixes {
			fp.UnderPaths = append(fp.UnderPaths, recursiveFilePathSearch(path.Name, containerURL))
		}
		marker = hierarchBlob.NextMarker
	}
	return
}

// Returns a list of full file paths
func (fp *FilePath) ReturnFilePaths() []string {
	filePaths := make([]string, 0)

	for _, uFP := range fp.UnderPaths {
		if uFP.File {
			filePaths = append(filePaths, fp.Path+uFP.Path)
		} else {
			morePaths := uFP.ReturnFilePaths()
			for x := range morePaths {
				filePaths = append(filePaths, fp.Path+morePaths[x])
			}
		}
	}
	return filePaths
}

// If returns true, remove the file
type FileFilter func(fileName string) (remove bool)

// Assuming FilePath is not a file itself. Only its children are
func (fp *FilePath) FilterFiles(ff FileFilter) {
	fp.filterFilesRecursive(ff)
}

// If the child item returns true to be filtered, we prune it from our child list
func (fp *FilePath) filterFilesRecursive(ff FileFilter) bool {
	if fp.File {
		return ff(fp.Path)
	}

	// The index to still keep up to
	keepDex := 0
	for pth := range fp.UnderPaths {
		remove := fp.UnderPaths[pth].filterFilesRecursive(ff)
		if !remove {
			fp.UnderPaths[keepDex] = fp.UnderPaths[pth]
			keepDex++
		}
	}
	fp.UnderPaths = fp.UnderPaths[:keepDex]

	return keepDex == 0
}

func (cm *CollectionManagement) GetAzureBlobLink(filePath string) (link string, err error) {
	link, err = url.JoinPath(cm.BaseLink.String(), filePath)
	if err != nil {
		logger.Err(err).Msg("")
		return
	}
	sasToken, err := cm.getSASAccessTokenOnceADay()
	if err != nil {
		logger.Err(err).Msg("")
	}
	return link + "?" + sasToken, err
}

func (cm *CollectionManagement) getSASAccessTokenOnceADay() (token string, err error) {
	if time.Since(cm.cachedTokenTime) > time.Hour*24 {
		cm.cachedTokenTime = time.Now()
		cm.cachedToken, err = cm.generateSASToken()
	}
	return cm.cachedToken, err
}

func (cm *CollectionManagement) generateSASToken() (token string, err error) {

	sasQueryParams, err := azblob.BlobSASSignatureValues{
		Protocol:      azblob.SASProtocolHTTPS,
		StartTime:     time.Now().UTC(),
		ExpiryTime:    time.Now().UTC().Add(48 * time.Hour),
		Permissions:   azblob.BlobSASPermissions{Read: true, List: true}.String(),
		IPRange:       azblob.IPRange{},
		ContainerName: cm.cmci.AzureContainerName,
	}.NewSASQueryParameters(cm.SharedKeyCredential)

	if err != nil {
		logger.Error().Err(err).Msg("Failed to sas.BlobSignatureValues")
		return
	}

	token = sasQueryParams.Encode()
	return
}