407 lines
9.0 KiB
Go
407 lines
9.0 KiB
Go
package remotefileupload
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/gob"
|
|
"fmt"
|
|
"math/rand"
|
|
"sort"
|
|
"strconv"
|
|
"strings"
|
|
"sync"
|
|
"testing"
|
|
"time"
|
|
)
|
|
|
|
func TestBlockUpload(t *testing.T) {
|
|
t.Skip()
|
|
azureAccount = "REPLACE_ME"
|
|
azureAccountKey = "REPLACE_ME"
|
|
|
|
RunBatchTimer = false
|
|
uploader, err := NewAzureBatchUploader("trex-logs", ".txt", 30, ",")
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
|
|
p, ok := uploader.(*AzureBatchUploader)
|
|
if !ok {
|
|
t.Error("Could not convert uploader to azure batch uploader")
|
|
}
|
|
typedUploader := *p
|
|
|
|
filePath := typedUploader.azureBlobURL(typedUploader.containerPath, []string{"4mibUpload"})
|
|
fakeFile := stringMutex{
|
|
Body: []byte{},
|
|
logValue: LogPayload{Title: "4mibTetFile", Value: "4mibUpload"},
|
|
}
|
|
|
|
// Making it 5 Mibs
|
|
for x := 0; len(fakeFile.Body) < 1024*1024*5; x++ {
|
|
fakeFile.Body = append(fakeFile.Body, []byte(fmt.Sprintf("%d,", x))...)
|
|
}
|
|
err = typedUploader.uploadBlock(filePath, &fakeFile, context.Background())
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
}
|
|
|
|
func TestBlockUploadCheckPath(t *testing.T) {
|
|
t.Skip()
|
|
azureAccount = "REPLACE_ME"
|
|
azureAccountKey = "REPLACE_ME"
|
|
|
|
uploader, err := NewAzureBatchUploader("trex-logs", ".txt", 30, ",")
|
|
if err != nil {
|
|
t.Error(err)
|
|
return
|
|
}
|
|
|
|
logP := LogPayload{
|
|
Title: "Test",
|
|
Value: "Value",
|
|
}
|
|
path, err := uploader.Upload([]byte("Hello This is a file path test"), logP, "/file", "test")
|
|
if err != nil{
|
|
t.Error(err)
|
|
return
|
|
}
|
|
|
|
p, ok := uploader.(*AzureBatchUploader)
|
|
if !ok {
|
|
t.Error("Could not convert uploader to azure batch uploader")
|
|
return
|
|
}
|
|
typedUploader := *p
|
|
|
|
typedUploader.uploadNow()
|
|
// Going to this path should give you the file
|
|
t.Log(path)
|
|
}
|
|
// Adds the same number to all threads in a goroutine
|
|
func TestMutexValues(t *testing.T) {
|
|
a := AzureBatchUploader{
|
|
accountName: "fakeName",
|
|
containerName: "fakeContainer",
|
|
fileExtension: ".txt",
|
|
separator: []byte{','},
|
|
}
|
|
|
|
ConnectToAzBlob = false
|
|
a.logsToSend = &logsMapMutex{logs: map[string]*stringMutex{},
|
|
Mutex: sync.Mutex{}}
|
|
|
|
logP := LogPayload{
|
|
Title: "Test",
|
|
Value: "Value",
|
|
}
|
|
gr := sync.WaitGroup{}
|
|
gr.Add(100)
|
|
|
|
for x := 0; x < 100; x++ {
|
|
//t.Logf("Number is %d\n", x)
|
|
go func(y int) {
|
|
for z := 0; z < 100; z++ {
|
|
_, _ = a.Upload([]byte(strconv.Itoa(y)), logP, fmt.Sprintf("/file/test%d", z))
|
|
}
|
|
gr.Done()
|
|
}(x)
|
|
}
|
|
|
|
gr.Wait()
|
|
sendMap := *(a.logsToSend)
|
|
|
|
for x := 0; x < 100; x++ {
|
|
filePath := fmt.Sprintf("file/test%d/raw.txt", x)
|
|
mstring, ok := sendMap.logs[filePath]
|
|
if !ok {
|
|
t.Fail()
|
|
}
|
|
|
|
t.Log(string(mstring.Body))
|
|
if !checkNumberString(string(mstring.Body), 100, t) {
|
|
t.Fail()
|
|
}
|
|
}
|
|
}
|
|
|
|
// Adds all numbers to one thread in a goroutine
|
|
func TestMutexValuesOrderSwap(t *testing.T) {
|
|
a := AzureBatchUploader{
|
|
accountName: "fakeName",
|
|
containerName: "fakeContainer",
|
|
fileExtension: ".txt",
|
|
separator: []byte{','},
|
|
}
|
|
|
|
a.logsToSend = &logsMapMutex{logs: map[string]*stringMutex{},
|
|
Mutex: sync.Mutex{}}
|
|
|
|
logP := LogPayload{
|
|
Title: "Test",
|
|
Value: "Value",
|
|
}
|
|
gr := sync.WaitGroup{}
|
|
gr.Add(100)
|
|
|
|
for x := 0; x < 100; x++ {
|
|
//t.Logf("Number is %d\n", x)
|
|
go func(y int) {
|
|
for z := 0; z < 100; z++ {
|
|
_, _ = a.Upload([]byte(strconv.Itoa(z)), logP, fmt.Sprintf("/file/test%d", y))
|
|
}
|
|
gr.Done()
|
|
}(x)
|
|
}
|
|
|
|
gr.Wait()
|
|
sendMap := *(a.logsToSend)
|
|
|
|
for x := 0; x < 100; x++ {
|
|
filePath := fmt.Sprintf("file/test%d/raw.txt", x)
|
|
mstring, ok := sendMap.logs[filePath]
|
|
if !ok {
|
|
t.Fail()
|
|
}
|
|
|
|
t.Log(string(mstring.Body))
|
|
if !checkNumberString(string(mstring.Body), 100, t) {
|
|
t.Fail()
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestDoesMapSwap(t *testing.T) {
|
|
t.Skip()
|
|
|
|
azureAccount = "REPLACE_ME"
|
|
azureAccountKey = "REPLACE_ME"
|
|
RunBatchTimer = false
|
|
|
|
up, err := NewAzureBatchUploader("trex-logs", ".log", 60, "\n")
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
p, ok := up.(*AzureBatchUploader)
|
|
if !ok {
|
|
t.Error("Could not convert uploader to azure batch uploader")
|
|
}
|
|
a := *p
|
|
logP := LogPayload{
|
|
Title: "Test",
|
|
Value: "Value",
|
|
}
|
|
|
|
wg := sync.WaitGroup{}
|
|
wg.Add(99)
|
|
a.Upload([]byte(strconv.Itoa(0)), logP, "/file/test")
|
|
for x := 1; x < 100; x++ {
|
|
//t.Logf("Number is %d\n", x)
|
|
go func(x int) {
|
|
time.Sleep(time.Millisecond * time.Duration(rand.Int63n(10)))
|
|
a.Upload([]byte(strconv.Itoa(x)), logP, "/file/test")
|
|
wg.Done()
|
|
}(x)
|
|
}
|
|
time.Sleep(5 * time.Millisecond)
|
|
a.uploadNow()
|
|
wg.Wait()
|
|
a.uploadNow()
|
|
}
|
|
|
|
// Writes is the number of numbers to write, threads is how many cars are sending in data
|
|
func benchmarkMutex1(writes, threads int, b *testing.B) {
|
|
a := AzureBatchUploader{
|
|
accountName: "fakeName",
|
|
containerName: "fakeContainer",
|
|
fileExtension: ".txt",
|
|
separator: []byte{','},
|
|
}
|
|
|
|
a.logsToSend = &logsMapMutex{logs: map[string]*stringMutex{},
|
|
Mutex: sync.Mutex{}}
|
|
|
|
logP := LogPayload{
|
|
Title: "Test",
|
|
Value: "Value",
|
|
}
|
|
gr := sync.WaitGroup{}
|
|
gr.Add(threads)
|
|
for x := 0; x < threads; x++ {
|
|
//t.Logf("Number is %d\n", x)
|
|
go func(y int) {
|
|
for z := 0; z < writes; z++ {
|
|
p, _ := a.Upload([]byte(strconv.Itoa(z)), logP, fmt.Sprintf("/file/test%d", y))
|
|
_ = p
|
|
}
|
|
gr.Done()
|
|
}(x)
|
|
|
|
}
|
|
gr.Wait()
|
|
}
|
|
|
|
func BenchmarkMutex1w100t100(b *testing.B) {
|
|
benchmarkMutex1(100, 100, b)
|
|
}
|
|
|
|
func BenchmarkMutex1w100t1000(b *testing.B) {
|
|
benchmarkMutex1(100, 1000, b)
|
|
}
|
|
|
|
func BenchmarkMutex1w100t10000(b *testing.B) {
|
|
benchmarkMutex1(100, 10000, b)
|
|
}
|
|
|
|
// BenchmarkMutex1w100t100000-16 1000000000 0.5827 ns/op 0 B/op 0 allocs/op
|
|
func BenchmarkMutex1w100t100000(b *testing.B) {
|
|
benchmarkMutex1(100, 10000, b)
|
|
}
|
|
|
|
// BenchmarkMutex1w1000t100000-16 1 4727771437 ns/op 8845213144 B/op 208939787 allocs/op
|
|
// BenchmarkMutex1w1000t100000-16 1 6521959624 ns/op 9321626472 B/op 228890209 allocs/op
|
|
func BenchmarkMutex1w1000t100000(b *testing.B) {
|
|
benchmarkMutex1(1000, 10000, b)
|
|
}
|
|
|
|
func checkNumberString(str string, max int, t *testing.T) (success bool) {
|
|
var err error
|
|
stringNumbers := strings.Split(str, ",")
|
|
numbers := make([]int, len(stringNumbers))
|
|
|
|
for x, num := range stringNumbers {
|
|
numbers[x], err = strconv.Atoi(num)
|
|
if err != nil {
|
|
t.Error(err)
|
|
t.FailNow()
|
|
}
|
|
}
|
|
|
|
sort.Ints(numbers)
|
|
for x := 0; x < max; x++ {
|
|
if numbers[x] != x {
|
|
return false
|
|
}
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
// inclusive start, exclusive end
|
|
func checkNumberStringRange(str string, start int, end int, t *testing.T) (success bool) {
|
|
var err error
|
|
stringNumbers := strings.Split(str, ",")
|
|
numbers := make([]int, len(stringNumbers))
|
|
|
|
for x, num := range stringNumbers {
|
|
numbers[x], err = strconv.Atoi(num)
|
|
if err != nil {
|
|
t.Error(err)
|
|
t.FailNow()
|
|
}
|
|
}
|
|
|
|
sort.Ints(numbers)
|
|
|
|
x := start
|
|
for _, num := range numbers {
|
|
if num != x {
|
|
return false
|
|
}
|
|
x++
|
|
}
|
|
|
|
return true
|
|
}
|
|
|
|
// This uses only ~38.439708 megabytes, in actual log sizing
|
|
func BenchmarkTotalSize(b *testing.B) {
|
|
// This is a longish message
|
|
testMSG := `{"level":"error","timestamp":"2022-Nov-30 22:17:26.250332","line_number":0,"filename":"dummy","msg":"ws_handshake: The WebSocket handshake was declined by the remote peer"}`
|
|
RunBatchTimer = false
|
|
ConnectToAzBlob = false
|
|
uploader, err := NewAzureBatchUploader("fakeName", ".txt", 30, "\n")
|
|
if err != nil {
|
|
b.Error(err)
|
|
}
|
|
|
|
p, ok := uploader.(*AzureBatchUploader)
|
|
if !ok {
|
|
b.Error("Could not convert uploader to azure batch uploader")
|
|
}
|
|
typedUploader := *p
|
|
|
|
// Simulate getting a message over 1 minute every 15 seconds from 50,000 cars
|
|
for x := 0; x <= 50000; x++ {
|
|
pl := LogPayload{
|
|
Title: "VIN",
|
|
Value: fmt.Sprintf("VINNUMBER%d", x),
|
|
}
|
|
|
|
_, err = typedUploader.Upload([]byte(testMSG), pl, pl.Value)
|
|
if err != nil {
|
|
b.Error(err)
|
|
}
|
|
_, err = typedUploader.Upload([]byte(testMSG), pl, pl.Value)
|
|
if err != nil {
|
|
b.Error(err)
|
|
}
|
|
_, err = typedUploader.Upload([]byte(testMSG), pl, pl.Value)
|
|
if err != nil {
|
|
b.Error(err)
|
|
}
|
|
_, err = typedUploader.Upload([]byte(testMSG), pl, pl.Value)
|
|
if err != nil {
|
|
b.Error(err)
|
|
}
|
|
}
|
|
|
|
b.Log(getRealSizeOf(typedUploader.logsToSend.logs))
|
|
}
|
|
|
|
func getRealSizeOf(v interface{}) (int, error) {
|
|
b := new(bytes.Buffer)
|
|
if err := gob.NewEncoder(b).Encode(v); err != nil {
|
|
return 0, err
|
|
}
|
|
return b.Len(), nil
|
|
}
|
|
|
|
func TestOrder(t *testing.T){
|
|
|
|
RunBatchTimer = false
|
|
ConnectToAzBlob = false
|
|
uploader, err := NewAzureBatchUploader("fakeName", ".txt", 30, "\n")
|
|
p, ok := uploader.(*AzureBatchUploader)
|
|
if !ok {
|
|
t.Error("Could not convert uploader to azure batch uploader")
|
|
}
|
|
typedUploader := *p
|
|
if err != nil {
|
|
t.Error(err)
|
|
}
|
|
pl := LogPayload{
|
|
Title: "VIN",
|
|
Value: "SomeFakeVin",
|
|
}
|
|
for x := 0; x < 1000; x ++{
|
|
|
|
_, err := typedUploader.Upload([]byte(fmt.Sprint(x)), pl, pl.Value)
|
|
if err != nil {
|
|
t.Error(err)
|
|
t.FailNow()
|
|
}
|
|
}
|
|
|
|
val := typedUploader.logsToSend.logs["https://REPLACE_ME.blob.core.windows.net/fakeName/SomeFakeVin/raw.txt"]
|
|
numberArray := strings.Split(string(val.Body), "\n")
|
|
for x := 0; x < len(numberArray) - 1; x ++ {
|
|
a, _ := strconv.Atoi(numberArray[x])
|
|
b, _ := strconv.Atoi(numberArray[x+1])
|
|
if a + 1 != b {
|
|
t.Logf("Failed got %s before %s\n", numberArray[x], numberArray[x+1])
|
|
t.Fail()
|
|
}
|
|
}
|
|
} |