Initial cloud-services repo - gateway service + pkg modules

2026-01-30 23:14:52 -05:00
commit fbb820d7b3
1037 changed files with 171318 additions and 0 deletions
--- a/pkg/remotefileupload/batchuploader_test.go
+++ b/pkg/remotefileupload/batchuploader_test.go
@@ -0,0 +1,407 @@
+package remotefileupload
+
+import (
+	"bytes"
+	"context"
+	"encoding/gob"
+	"fmt"
+	"math/rand"
+	"sort"
+	"strconv"
+	"strings"
+	"sync"
+	"testing"
+	"time"
+)
+
+func TestBlockUpload(t *testing.T) {
+	t.Skip()
+	azureAccount = "REPLACE_ME"
+	azureAccountKey = "REPLACE_ME"
+
+	RunBatchTimer = false
+	uploader, err := NewAzureBatchUploader("trex-logs", ".txt", 30, ",")
+	if err != nil {
+		t.Error(err)
+	}
+
+	p, ok := uploader.(*AzureBatchUploader)
+	if !ok {
+		t.Error("Could not convert uploader to azure batch uploader")
+	}
+	typedUploader := *p
+
+	filePath := typedUploader.azureBlobURL(typedUploader.containerPath, []string{"4mibUpload"})
+	fakeFile := stringMutex{
+		Body:     []byte{},
+		logValue: LogPayload{Title: "4mibTetFile", Value: "4mibUpload"},
+	}
+
+	// Making it 5 Mibs
+	for x := 0; len(fakeFile.Body) < 1024*1024*5; x++ {
+		fakeFile.Body = append(fakeFile.Body, []byte(fmt.Sprintf("%d,", x))...)
+	}
+	err = typedUploader.uploadBlock(filePath, &fakeFile, context.Background())
+	if err != nil {
+		t.Error(err)
+	}
+}
+
+func TestBlockUploadCheckPath(t *testing.T) {
+	t.Skip()
+	azureAccount = "REPLACE_ME"
+	azureAccountKey = "REPLACE_ME"
+
+	uploader, err := NewAzureBatchUploader("trex-logs", ".txt", 30, ",")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	logP := LogPayload{
+		Title: "Test",
+		Value: "Value",
+	}
+	path, err := uploader.Upload([]byte("Hello This is a file path test"), logP, "/file", "test")
+	if err != nil{
+		t.Error(err)
+		return
+	}
+
+	p, ok := uploader.(*AzureBatchUploader)
+	if !ok {
+		t.Error("Could not convert uploader to azure batch uploader")
+		return
+	}
+	typedUploader := *p
+
+	typedUploader.uploadNow()
+	// Going to this path should give you the file
+	t.Log(path)
+}
+// Adds the same number to all threads in a goroutine
+func TestMutexValues(t *testing.T) {
+	a := AzureBatchUploader{
+		accountName:   "fakeName",
+		containerName: "fakeContainer",
+		fileExtension: ".txt",
+		separator:     []byte{','},
+	}
+
+	ConnectToAzBlob = false
+	a.logsToSend = &logsMapMutex{logs: map[string]*stringMutex{},
+		Mutex: sync.Mutex{}}
+
+	logP := LogPayload{
+		Title: "Test",
+		Value: "Value",
+	}
+	gr := sync.WaitGroup{}
+	gr.Add(100)
+
+	for x := 0; x < 100; x++ {
+		//t.Logf("Number is %d\n", x)
+		go func(y int) {
+			for z := 0; z < 100; z++ {
+				_, _ = a.Upload([]byte(strconv.Itoa(y)), logP, fmt.Sprintf("/file/test%d", z))
+			}
+			gr.Done()
+		}(x)
+	}
+
+	gr.Wait()
+	sendMap := *(a.logsToSend)
+
+	for x := 0; x < 100; x++ {
+		filePath := fmt.Sprintf("file/test%d/raw.txt", x)
+		mstring, ok := sendMap.logs[filePath]
+		if !ok {
+			t.Fail()
+		}
+
+		t.Log(string(mstring.Body))
+		if !checkNumberString(string(mstring.Body), 100, t) {
+			t.Fail()
+		}
+	}
+}
+
+// Adds all numbers to one thread in a goroutine
+func TestMutexValuesOrderSwap(t *testing.T) {
+	a := AzureBatchUploader{
+		accountName:   "fakeName",
+		containerName: "fakeContainer",
+		fileExtension: ".txt",
+		separator:     []byte{','},
+	}
+
+	a.logsToSend = &logsMapMutex{logs: map[string]*stringMutex{},
+		Mutex: sync.Mutex{}}
+
+	logP := LogPayload{
+		Title: "Test",
+		Value: "Value",
+	}
+	gr := sync.WaitGroup{}
+	gr.Add(100)
+
+	for x := 0; x < 100; x++ {
+		//t.Logf("Number is %d\n", x)
+		go func(y int) {
+			for z := 0; z < 100; z++ {
+				_, _ = a.Upload([]byte(strconv.Itoa(z)), logP, fmt.Sprintf("/file/test%d", y))
+			}
+			gr.Done()
+		}(x)
+	}
+
+	gr.Wait()
+	sendMap := *(a.logsToSend)
+
+	for x := 0; x < 100; x++ {
+		filePath := fmt.Sprintf("file/test%d/raw.txt", x)
+		mstring, ok := sendMap.logs[filePath]
+		if !ok {
+			t.Fail()
+		}
+
+		t.Log(string(mstring.Body))
+		if !checkNumberString(string(mstring.Body), 100, t) {
+			t.Fail()
+		}
+	}
+}
+
+func TestDoesMapSwap(t *testing.T) {
+	t.Skip()
+
+	azureAccount = "REPLACE_ME"
+	azureAccountKey = "REPLACE_ME"
+	RunBatchTimer = false
+
+	up, err := NewAzureBatchUploader("trex-logs", ".log", 60, "\n")
+	if err != nil {
+		t.Error(err)
+	}
+	p, ok := up.(*AzureBatchUploader)
+	if !ok {
+		t.Error("Could not convert uploader to azure batch uploader")
+	}
+	a := *p
+	logP := LogPayload{
+		Title: "Test",
+		Value: "Value",
+	}
+
+	wg := sync.WaitGroup{}
+	wg.Add(99)
+	a.Upload([]byte(strconv.Itoa(0)), logP, "/file/test")
+	for x := 1; x < 100; x++ {
+		//t.Logf("Number is %d\n", x)
+		go func(x int) {
+			time.Sleep(time.Millisecond * time.Duration(rand.Int63n(10)))
+			a.Upload([]byte(strconv.Itoa(x)), logP, "/file/test")
+			wg.Done()
+		}(x)
+	}
+	time.Sleep(5 * time.Millisecond)
+	a.uploadNow()
+	wg.Wait()
+	a.uploadNow()
+}
+
+// Writes is the number of numbers to write, threads is how many cars are sending in data
+func benchmarkMutex1(writes, threads int, b *testing.B) {
+	a := AzureBatchUploader{
+		accountName:   "fakeName",
+		containerName: "fakeContainer",
+		fileExtension: ".txt",
+		separator:     []byte{','},
+	}
+
+	a.logsToSend = &logsMapMutex{logs: map[string]*stringMutex{},
+		Mutex: sync.Mutex{}}
+
+	logP := LogPayload{
+		Title: "Test",
+		Value: "Value",
+	}
+	gr := sync.WaitGroup{}
+	gr.Add(threads)
+	for x := 0; x < threads; x++ {
+		//t.Logf("Number is %d\n", x)
+		go func(y int) {
+			for z := 0; z < writes; z++ {
+				p, _ := a.Upload([]byte(strconv.Itoa(z)), logP, fmt.Sprintf("/file/test%d", y))
+				_ = p
+			}
+			gr.Done()
+		}(x)
+
+	}
+	gr.Wait()
+}
+
+func BenchmarkMutex1w100t100(b *testing.B) {
+	benchmarkMutex1(100, 100, b)
+}
+
+func BenchmarkMutex1w100t1000(b *testing.B) {
+	benchmarkMutex1(100, 1000, b)
+}
+
+func BenchmarkMutex1w100t10000(b *testing.B) {
+	benchmarkMutex1(100, 10000, b)
+}
+
+// BenchmarkMutex1w100t100000-16           1000000000               0.5827 ns/op          0 B/op          0 allocs/op
+func BenchmarkMutex1w100t100000(b *testing.B) {
+	benchmarkMutex1(100, 10000, b)
+}
+
+// BenchmarkMutex1w1000t100000-16                 1        4727771437 ns/op        8845213144 B/op 208939787 allocs/op
+// BenchmarkMutex1w1000t100000-16                 1        6521959624 ns/op        9321626472 B/op 228890209 allocs/op
+func BenchmarkMutex1w1000t100000(b *testing.B) {
+	benchmarkMutex1(1000, 10000, b)
+}
+
+func checkNumberString(str string, max int, t *testing.T) (success bool) {
+	var err error
+	stringNumbers := strings.Split(str, ",")
+	numbers := make([]int, len(stringNumbers))
+
+	for x, num := range stringNumbers {
+		numbers[x], err = strconv.Atoi(num)
+		if err != nil {
+			t.Error(err)
+			t.FailNow()
+		}
+	}
+
+	sort.Ints(numbers)
+	for x := 0; x < max; x++ {
+		if numbers[x] != x {
+			return false
+		}
+	}
+
+	return true
+}
+
+// inclusive start, exclusive end
+func checkNumberStringRange(str string, start int, end int, t *testing.T) (success bool) {
+	var err error
+	stringNumbers := strings.Split(str, ",")
+	numbers := make([]int, len(stringNumbers))
+
+	for x, num := range stringNumbers {
+		numbers[x], err = strconv.Atoi(num)
+		if err != nil {
+			t.Error(err)
+			t.FailNow()
+		}
+	}
+
+	sort.Ints(numbers)
+
+	x := start
+	for _, num := range numbers {
+		if num != x {
+			return false
+		}
+		x++
+	}
+
+	return true
+}
+
+// This uses only ~38.439708 megabytes, in actual log sizing
+func BenchmarkTotalSize(b *testing.B) {
+	// This is a longish message
+	testMSG := `{"level":"error","timestamp":"2022-Nov-30 22:17:26.250332","line_number":0,"filename":"dummy","msg":"ws_handshake: The WebSocket handshake was declined by the remote peer"}`
+	RunBatchTimer = false
+	ConnectToAzBlob = false
+	uploader, err := NewAzureBatchUploader("fakeName", ".txt", 30, "\n")
+	if err != nil {
+		b.Error(err)
+	}
+
+	p, ok := uploader.(*AzureBatchUploader)
+	if !ok {
+		b.Error("Could not convert uploader to azure batch uploader")
+	}
+	typedUploader := *p
+
+	// Simulate getting a message over 1 minute every 15 seconds from 50,000 cars
+	for x := 0; x <= 50000; x++ {
+		pl := LogPayload{
+			Title: "VIN",
+			Value: fmt.Sprintf("VINNUMBER%d", x),
+		}
+
+		_, err = typedUploader.Upload([]byte(testMSG), pl, pl.Value)
+		if err != nil {
+			b.Error(err)
+		}
+		_, err = typedUploader.Upload([]byte(testMSG), pl, pl.Value)
+		if err != nil {
+			b.Error(err)
+		}
+		_, err = typedUploader.Upload([]byte(testMSG), pl, pl.Value)
+		if err != nil {
+			b.Error(err)
+		}
+		_, err = typedUploader.Upload([]byte(testMSG), pl, pl.Value)
+		if err != nil {
+			b.Error(err)
+		}
+	}
+
+	b.Log(getRealSizeOf(typedUploader.logsToSend.logs))
+}
+
+func getRealSizeOf(v interface{}) (int, error) {
+	b := new(bytes.Buffer)
+	if err := gob.NewEncoder(b).Encode(v); err != nil {
+		return 0, err
+	}
+	return b.Len(), nil
+}
+
+func TestOrder(t *testing.T){
+
+	RunBatchTimer = false
+	ConnectToAzBlob = false
+	uploader, err := NewAzureBatchUploader("fakeName", ".txt", 30, "\n")
+	p, ok := uploader.(*AzureBatchUploader)
+	if !ok {
+		t.Error("Could not convert uploader to azure batch uploader")
+	}
+	typedUploader := *p
+	if err != nil {
+		t.Error(err)
+	}
+	pl := LogPayload{
+		Title: "VIN",
+		Value: "SomeFakeVin",
+	}
+	for x := 0; x < 1000; x ++{
+		
+		_, err := typedUploader.Upload([]byte(fmt.Sprint(x)), pl, pl.Value)
+		if err != nil {
+			t.Error(err)
+			t.FailNow()
+		}
+	}
+
+	val := typedUploader.logsToSend.logs["https://REPLACE_ME.blob.core.windows.net/fakeName/SomeFakeVin/raw.txt"]
+	numberArray := strings.Split(string(val.Body), "\n")
+	for x := 0; x < len(numberArray) - 1; x ++ {
+		a, _ := strconv.Atoi(numberArray[x])
+		b, _ := strconv.Atoi(numberArray[x+1])
+		if a + 1 != b {
+			t.Logf("Failed got %s before %s\n", numberArray[x], numberArray[x+1])
+			t.Fail()
+		}
+	}
+}