package azurestoragecontainer import ( "context" "fmt" "net/url" "path/filepath" "strings" "time" "fiskerinc.com/modules/logger" "github.com/Azure/azure-storage-blob-go/azblob" "github.com/pkg/errors" ) // This set of modules will contain the ability to get the list of files from a container, and create a sas link to download them from var azureLogsBlobPath = "https://%s.blob.core.windows.net/%s" // For your module to use this they need a CollectionManagement. These store your connection information to // azure. Leave it up to the user to not recreate everytime type CollectionManagement struct { SharedKeyCredential *azblob.SharedKeyCredential BaseLink *url.URL // AzureAccount.blob.core.windows.net/LogsContainerName cachedTokenTime time.Time // The Time we last got the token cachedToken string // Once a day get a new access token cmci CollectionManagementConnectionInformation } type CollectionManagementConnectionInformation struct { AzureAccount string AzureContainerName string AzureAccountKey string } // Need somekind of one time sync thing. Lets worry about it later. Probably each service needs its own func NewCollectionConnection(cmci CollectionManagementConnectionInformation) (cm *CollectionManagement, err error) { collect := CollectionManagement{} link := makeAzureBlobLink(cmci.AzureAccount, cmci.AzureContainerName) parsedURL, err := url.Parse(link) if err != nil { err = errors.WithStack(err) return } collect.BaseLink = parsedURL cred, err := azblob.NewSharedKeyCredential(cmci.AzureAccount, cmci.AzureAccountKey) if err != nil { return nil, errors.WithStack(err) } collect.SharedKeyCredential = cred collect.cmci = cmci return &collect, err } type FilePath struct { Path string `json:"Path,omitempty"` File bool UnderPaths []*FilePath `json:"UnderPaths,omitempty"` } // This is working code to get the full set of all the different files. I am unsure why I wrote this for this ticket, // but could be used to grey out the calender on the ota-portal // Generate a file path for all the files // Prefix is the possible begging file path: e.g. vin/year will get all the files under that year folder // rootOnly will only get the root files instead of nested folders func (cm *CollectionManagement) GetFolderStruct(prefix string, rootOnly bool) (fp *FilePath, err error) { fp, err = cm.startRecursiveFilePathSearch(prefix, rootOnly) return } // If prefix is empty, start at the root func (cm *CollectionManagement) startRecursiveFilePathSearch(prefix string, rootOnly bool) (startPath *FilePath, err error) { cred := cm.SharedKeyCredential parsedURL := cm.BaseLink containerURL := azblob.NewContainerURL(*parsedURL, azblob.NewPipeline(cred, azblob.PipelineOptions{})) //marker := azblob.Marker{} startPath = &FilePath{} startPath.Path = prefix startPath.UnderPaths = make([]*FilePath, 0) if prefix != "" { prefix = prefix + "/" } marker := azblob.Marker{} for marker.NotDone() { // This should probably be outside the for loop? hierarchBlob, err := containerURL.ListBlobsHierarchySegment(context.Background(), marker, "/", azblob.ListBlobsSegmentOptions{ Prefix: prefix, }) if err != nil { err = errors.WithStack(err) return startPath, err } // BlobItems are actual files for _, file := range hierarchBlob.Segment.BlobItems { startPath.UnderPaths = append(startPath.UnderPaths, &FilePath{Path: file.Name[len(prefix):], File: true}) } if !rootOnly { for _, path := range hierarchBlob.Segment.BlobPrefixes { startPath.UnderPaths = append(startPath.UnderPaths, recursiveFilePathSearch(path.Name, containerURL)) } } marker = hierarchBlob.NextMarker } return } func makeAzureBlobLink(azureAccountName, azureContainerName string) string { link := fmt.Sprintf( azureLogsBlobPath, azureAccountName, azureContainerName, ) return link //fmt.Sprintf("%s / %s", link, sasToken), err } // We start at the path and explore all those children // Prefix is the path func recursiveFilePathSearch(prefix string, containerURL azblob.ContainerURL) (fp *FilePath) { fp = &FilePath{ UnderPaths: make([]*FilePath, 0), } paths := strings.Split(strings.TrimRight(prefix, "/"), "/") fp.Path = paths[len(paths)-1] marker := azblob.Marker{} for marker.NotDone() { hierarchBlob, err := containerURL.ListBlobsHierarchySegment(context.Background(), marker, "/", azblob.ListBlobsSegmentOptions{ Prefix: prefix, }) if err != nil { return } // BlobItems are actual files for _, file := range hierarchBlob.Segment.BlobItems { _, nextFile := filepath.Split(file.Name) fp.UnderPaths = append(fp.UnderPaths, &FilePath{Path: nextFile, File: true}) //file.Name } //path.name includes the whole path for _, path := range hierarchBlob.Segment.BlobPrefixes { fp.UnderPaths = append(fp.UnderPaths, recursiveFilePathSearch(path.Name, containerURL)) } marker = hierarchBlob.NextMarker } return } // Returns a list of full file paths func (fp *FilePath) ReturnFilePaths() []string { filePaths := make([]string, 0) for _, uFP := range fp.UnderPaths { if uFP.File { filePaths = append(filePaths, fp.Path+uFP.Path) } else { morePaths := uFP.ReturnFilePaths() for x := range morePaths { filePaths = append(filePaths, fp.Path+morePaths[x]) } } } return filePaths } // If returns true, remove the file type FileFilter func(fileName string) (remove bool) // Assuming FilePath is not a file itself. Only its children are func (fp *FilePath) FilterFiles(ff FileFilter) { fp.filterFilesRecursive(ff) } // If the child item returns true to be filtered, we prune it from our child list func (fp *FilePath) filterFilesRecursive(ff FileFilter) bool { if fp.File { return ff(fp.Path) } // The index to still keep up to keepDex := 0 for pth := range fp.UnderPaths { remove := fp.UnderPaths[pth].filterFilesRecursive(ff) if !remove { fp.UnderPaths[keepDex] = fp.UnderPaths[pth] keepDex++ } } fp.UnderPaths = fp.UnderPaths[:keepDex] return keepDex == 0 } func (cm *CollectionManagement) GetAzureBlobLink(filePath string) (link string, err error) { link, err = url.JoinPath(cm.BaseLink.String(), filePath) if err != nil { logger.Err(err).Msg("") return } sasToken, err := cm.getSASAccessTokenOnceADay() if err != nil { logger.Err(err).Msg("") } return link + "?" + sasToken, err } func (cm *CollectionManagement) getSASAccessTokenOnceADay() (token string, err error) { if time.Since(cm.cachedTokenTime) > time.Hour*24 { cm.cachedTokenTime = time.Now() cm.cachedToken, err = cm.generateSASToken() } return cm.cachedToken, err } func (cm *CollectionManagement) generateSASToken() (token string, err error) { sasQueryParams, err := azblob.BlobSASSignatureValues{ Protocol: azblob.SASProtocolHTTPS, StartTime: time.Now().UTC(), ExpiryTime: time.Now().UTC().Add(48 * time.Hour), Permissions: azblob.BlobSASPermissions{Read: true, List: true}.String(), IPRange: azblob.IPRange{}, ContainerName: cm.cmci.AzureContainerName, }.NewSASQueryParameters(cm.SharedKeyCredential) if err != nil { logger.Error().Err(err).Msg("Failed to sas.BlobSignatureValues") return } token = sasQueryParams.Encode() return }