package remotefileupload import ( "bytes" "context" "encoding/gob" "fmt" "math/rand" "sort" "strconv" "strings" "sync" "testing" "time" ) func TestBlockUpload(t *testing.T) { t.Skip() azureAccount = "REPLACE_ME" azureAccountKey = "REPLACE_ME" RunBatchTimer = false uploader, err := NewAzureBatchUploader("trex-logs", ".txt", 30, ",") if err != nil { t.Error(err) } p, ok := uploader.(*AzureBatchUploader) if !ok { t.Error("Could not convert uploader to azure batch uploader") } typedUploader := *p filePath := typedUploader.azureBlobURL(typedUploader.containerPath, []string{"4mibUpload"}) fakeFile := stringMutex{ Body: []byte{}, logValue: LogPayload{Title: "4mibTetFile", Value: "4mibUpload"}, } // Making it 5 Mibs for x := 0; len(fakeFile.Body) < 1024*1024*5; x++ { fakeFile.Body = append(fakeFile.Body, []byte(fmt.Sprintf("%d,", x))...) } err = typedUploader.uploadBlock(filePath, &fakeFile, context.Background()) if err != nil { t.Error(err) } } func TestBlockUploadCheckPath(t *testing.T) { t.Skip() azureAccount = "REPLACE_ME" azureAccountKey = "REPLACE_ME" uploader, err := NewAzureBatchUploader("trex-logs", ".txt", 30, ",") if err != nil { t.Error(err) return } logP := LogPayload{ Title: "Test", Value: "Value", } path, err := uploader.Upload([]byte("Hello This is a file path test"), logP, "/file", "test") if err != nil{ t.Error(err) return } p, ok := uploader.(*AzureBatchUploader) if !ok { t.Error("Could not convert uploader to azure batch uploader") return } typedUploader := *p typedUploader.uploadNow() // Going to this path should give you the file t.Log(path) } // Adds the same number to all threads in a goroutine func TestMutexValues(t *testing.T) { a := AzureBatchUploader{ accountName: "fakeName", containerName: "fakeContainer", fileExtension: ".txt", separator: []byte{','}, } ConnectToAzBlob = false a.logsToSend = &logsMapMutex{logs: map[string]*stringMutex{}, Mutex: sync.Mutex{}} logP := LogPayload{ Title: "Test", Value: "Value", } gr := sync.WaitGroup{} gr.Add(100) for x := 0; x < 100; x++ { //t.Logf("Number is %d\n", x) go func(y int) { for z := 0; z < 100; z++ { _, _ = a.Upload([]byte(strconv.Itoa(y)), logP, fmt.Sprintf("/file/test%d", z)) } gr.Done() }(x) } gr.Wait() sendMap := *(a.logsToSend) for x := 0; x < 100; x++ { filePath := fmt.Sprintf("file/test%d/raw.txt", x) mstring, ok := sendMap.logs[filePath] if !ok { t.Fail() } t.Log(string(mstring.Body)) if !checkNumberString(string(mstring.Body), 100, t) { t.Fail() } } } // Adds all numbers to one thread in a goroutine func TestMutexValuesOrderSwap(t *testing.T) { a := AzureBatchUploader{ accountName: "fakeName", containerName: "fakeContainer", fileExtension: ".txt", separator: []byte{','}, } a.logsToSend = &logsMapMutex{logs: map[string]*stringMutex{}, Mutex: sync.Mutex{}} logP := LogPayload{ Title: "Test", Value: "Value", } gr := sync.WaitGroup{} gr.Add(100) for x := 0; x < 100; x++ { //t.Logf("Number is %d\n", x) go func(y int) { for z := 0; z < 100; z++ { _, _ = a.Upload([]byte(strconv.Itoa(z)), logP, fmt.Sprintf("/file/test%d", y)) } gr.Done() }(x) } gr.Wait() sendMap := *(a.logsToSend) for x := 0; x < 100; x++ { filePath := fmt.Sprintf("file/test%d/raw.txt", x) mstring, ok := sendMap.logs[filePath] if !ok { t.Fail() } t.Log(string(mstring.Body)) if !checkNumberString(string(mstring.Body), 100, t) { t.Fail() } } } func TestDoesMapSwap(t *testing.T) { t.Skip() azureAccount = "REPLACE_ME" azureAccountKey = "REPLACE_ME" RunBatchTimer = false up, err := NewAzureBatchUploader("trex-logs", ".log", 60, "\n") if err != nil { t.Error(err) } p, ok := up.(*AzureBatchUploader) if !ok { t.Error("Could not convert uploader to azure batch uploader") } a := *p logP := LogPayload{ Title: "Test", Value: "Value", } wg := sync.WaitGroup{} wg.Add(99) a.Upload([]byte(strconv.Itoa(0)), logP, "/file/test") for x := 1; x < 100; x++ { //t.Logf("Number is %d\n", x) go func(x int) { time.Sleep(time.Millisecond * time.Duration(rand.Int63n(10))) a.Upload([]byte(strconv.Itoa(x)), logP, "/file/test") wg.Done() }(x) } time.Sleep(5 * time.Millisecond) a.uploadNow() wg.Wait() a.uploadNow() } // Writes is the number of numbers to write, threads is how many cars are sending in data func benchmarkMutex1(writes, threads int, b *testing.B) { a := AzureBatchUploader{ accountName: "fakeName", containerName: "fakeContainer", fileExtension: ".txt", separator: []byte{','}, } a.logsToSend = &logsMapMutex{logs: map[string]*stringMutex{}, Mutex: sync.Mutex{}} logP := LogPayload{ Title: "Test", Value: "Value", } gr := sync.WaitGroup{} gr.Add(threads) for x := 0; x < threads; x++ { //t.Logf("Number is %d\n", x) go func(y int) { for z := 0; z < writes; z++ { p, _ := a.Upload([]byte(strconv.Itoa(z)), logP, fmt.Sprintf("/file/test%d", y)) _ = p } gr.Done() }(x) } gr.Wait() } func BenchmarkMutex1w100t100(b *testing.B) { benchmarkMutex1(100, 100, b) } func BenchmarkMutex1w100t1000(b *testing.B) { benchmarkMutex1(100, 1000, b) } func BenchmarkMutex1w100t10000(b *testing.B) { benchmarkMutex1(100, 10000, b) } // BenchmarkMutex1w100t100000-16 1000000000 0.5827 ns/op 0 B/op 0 allocs/op func BenchmarkMutex1w100t100000(b *testing.B) { benchmarkMutex1(100, 10000, b) } // BenchmarkMutex1w1000t100000-16 1 4727771437 ns/op 8845213144 B/op 208939787 allocs/op // BenchmarkMutex1w1000t100000-16 1 6521959624 ns/op 9321626472 B/op 228890209 allocs/op func BenchmarkMutex1w1000t100000(b *testing.B) { benchmarkMutex1(1000, 10000, b) } func checkNumberString(str string, max int, t *testing.T) (success bool) { var err error stringNumbers := strings.Split(str, ",") numbers := make([]int, len(stringNumbers)) for x, num := range stringNumbers { numbers[x], err = strconv.Atoi(num) if err != nil { t.Error(err) t.FailNow() } } sort.Ints(numbers) for x := 0; x < max; x++ { if numbers[x] != x { return false } } return true } // inclusive start, exclusive end func checkNumberStringRange(str string, start int, end int, t *testing.T) (success bool) { var err error stringNumbers := strings.Split(str, ",") numbers := make([]int, len(stringNumbers)) for x, num := range stringNumbers { numbers[x], err = strconv.Atoi(num) if err != nil { t.Error(err) t.FailNow() } } sort.Ints(numbers) x := start for _, num := range numbers { if num != x { return false } x++ } return true } // This uses only ~38.439708 megabytes, in actual log sizing func BenchmarkTotalSize(b *testing.B) { // This is a longish message testMSG := `{"level":"error","timestamp":"2022-Nov-30 22:17:26.250332","line_number":0,"filename":"dummy","msg":"ws_handshake: The WebSocket handshake was declined by the remote peer"}` RunBatchTimer = false ConnectToAzBlob = false uploader, err := NewAzureBatchUploader("fakeName", ".txt", 30, "\n") if err != nil { b.Error(err) } p, ok := uploader.(*AzureBatchUploader) if !ok { b.Error("Could not convert uploader to azure batch uploader") } typedUploader := *p // Simulate getting a message over 1 minute every 15 seconds from 50,000 cars for x := 0; x <= 50000; x++ { pl := LogPayload{ Title: "VIN", Value: fmt.Sprintf("VINNUMBER%d", x), } _, err = typedUploader.Upload([]byte(testMSG), pl, pl.Value) if err != nil { b.Error(err) } _, err = typedUploader.Upload([]byte(testMSG), pl, pl.Value) if err != nil { b.Error(err) } _, err = typedUploader.Upload([]byte(testMSG), pl, pl.Value) if err != nil { b.Error(err) } _, err = typedUploader.Upload([]byte(testMSG), pl, pl.Value) if err != nil { b.Error(err) } } b.Log(getRealSizeOf(typedUploader.logsToSend.logs)) } func getRealSizeOf(v interface{}) (int, error) { b := new(bytes.Buffer) if err := gob.NewEncoder(b).Encode(v); err != nil { return 0, err } return b.Len(), nil } func TestOrder(t *testing.T){ RunBatchTimer = false ConnectToAzBlob = false uploader, err := NewAzureBatchUploader("fakeName", ".txt", 30, "\n") p, ok := uploader.(*AzureBatchUploader) if !ok { t.Error("Could not convert uploader to azure batch uploader") } typedUploader := *p if err != nil { t.Error(err) } pl := LogPayload{ Title: "VIN", Value: "SomeFakeVin", } for x := 0; x < 1000; x ++{ _, err := typedUploader.Upload([]byte(fmt.Sprint(x)), pl, pl.Value) if err != nil { t.Error(err) t.FailNow() } } val := typedUploader.logsToSend.logs["https://REPLACE_ME.blob.core.windows.net/fakeName/SomeFakeVin/raw.txt"] numberArray := strings.Split(string(val.Body), "\n") for x := 0; x < len(numberArray) - 1; x ++ { a, _ := strconv.Atoi(numberArray[x]) b, _ := strconv.Atoi(numberArray[x+1]) if a + 1 != b { t.Logf("Failed got %s before %s\n", numberArray[x], numberArray[x+1]) t.Fail() } } }