Add depot, attendant, jetfire, optimus, ota services with kustomize overlays
This commit is contained in:
Chris Rai
25
services/jetfire/.env.template
Normal file
25
services/jetfire/.env.template
Normal file
@@ -0,0 +1,25 @@
|
||||
CLICKHOUSE_USER="default"
|
||||
CLICKHOUSE_PASS=""
|
||||
|
||||
CLICKHOUSE_FEATURE_TABLE="feature_table"
|
||||
CLICKHOUSE_VEHICLE_SIGNAL_TABLE="vehicle_signal"
|
||||
|
||||
CLICKHOUSE_HOST="localhost"
|
||||
KAFKA_HOSTS="localhost:9093"
|
||||
|
||||
CLICKHOUSE_MAX_CONNS=5
|
||||
|
||||
JETFIRE_VEHICLE_SIGNAL_BATCH_PERIOD_MS=10000
|
||||
|
||||
JETFIRE_FEATURE_BATCH_PERIOD_MS=10000
|
||||
JETFIRE_FEATURE_DOWNSAMPLE_US=1000000
|
||||
|
||||
JETFIRE_TRIP_TIMEOUT_MS=300000
|
||||
JETFIRE_STATE_TIMEOUT_MS=3600000
|
||||
JETFIRE_FUTURE_THRESHOLD_MS=604800000
|
||||
|
||||
JETFIRE_SCHEMA_RESET_PERIOD_MS=3600000
|
||||
|
||||
JETFIRE_MAX_BUFFER_ROWS=1000
|
||||
|
||||
LOG_LEVEL="debug"
|
||||
32
services/jetfire/Dockerfile
Normal file
32
services/jetfire/Dockerfile
Normal file
@@ -0,0 +1,32 @@
|
||||
|
||||
## Build binaries for event detection using cloud_base_go image
|
||||
ARG BASE_IMAGE=cloud_base_go
|
||||
FROM ${BASE_IMAGE} as builder-go
|
||||
|
||||
WORKDIR /build/jetfire
|
||||
|
||||
COPY ./jetfire/go.mod ./jetfire/go.sum ./
|
||||
|
||||
RUN go mod edit -replace fiskerinc.com/modules=../fiskerinc.com/modules \
|
||||
&& go mod download
|
||||
|
||||
COPY ./jetfire ./
|
||||
|
||||
RUN go mod edit -replace fiskerinc.com/modules=../fiskerinc.com/modules \
|
||||
&& go build -tags musl
|
||||
|
||||
|
||||
## Build image for event detection, pulling binaries from builder image
|
||||
FROM alpine:3.17
|
||||
|
||||
RUN apk add --no-cache librdkafka --repository=https://dl-cdn.alpinelinux.org/alpine/edge/community \
|
||||
&& apk add --no-cache ca-certificates
|
||||
|
||||
COPY ./modules_go/logger/log_config .
|
||||
COPY ./jetfire/default-feature-vars.json .
|
||||
COPY --from=builder-go /build/jetfire/jetfire .
|
||||
|
||||
ENV LOG_LEVEL=log_config
|
||||
EXPOSE 8077
|
||||
|
||||
CMD ./jetfire
|
||||
26
services/jetfire/README.md
Normal file
26
services/jetfire/README.md
Normal file
@@ -0,0 +1,26 @@
|
||||
# Jetfire Data Routing Service
|
||||
Jetfire Service listens to CAN Signals on the vehicle_signals topic on Kafka,
|
||||
and performs a pivot operation for populating the feature_table on Clickhouse,
|
||||
and populating the real_time table on Clickhouse.
|
||||
CAN Signals are batched and inserted into vehicle_signal table without any transform performed on the data.
|
||||
|
||||
Feature Table inserts are typically batched at `JETFIRE_FEATURE_BATCH_PERIOD_MS=10000`.
|
||||
|
||||
Vehicle Signal Table inserts are typically batched at `JETFIRE_VEHICLE_SIGNAL_BATCH_PERIOD_MS=10000`
|
||||
|
||||
The schemas for sink tables are fetched periodically, set to `JETFIRE_SCHEMA_RESET_PERIOD_MS=3600000`
|
||||
Additionally, a GET request to the `/reset` endpoint on port `8077` will also trigger an immediate schema reset.
|
||||
|
||||
For more information about Jetfire service, see https://fiskerinc.atlassian.net/wiki/spaces/COM/pages/1401487522/Jetfire+Service
|
||||
|
||||
## Usage
|
||||
|
||||
Copy `./.env.tenmplate` to `./.env` file
|
||||
|
||||
Secrets in the .env file will need to be filled in manually.
|
||||
|
||||
Running jetfire locally
|
||||
```
|
||||
source set_envs.sh
|
||||
go run main.go
|
||||
```
|
||||
54
services/jetfire/controllers/health_check.go
Normal file
54
services/jetfire/controllers/health_check.go
Normal file
@@ -0,0 +1,54 @@
|
||||
package controllers
|
||||
|
||||
import (
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/services"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/health"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
var mismatchTypeError = errors.New("mismatch type error")
|
||||
|
||||
func HealthCheck() {
|
||||
server := health.HealthCheckServer{}
|
||||
err := server.Serve([]health.Config{
|
||||
{
|
||||
Name: "clickhouse",
|
||||
Check: health.NewClickhouseCheck(getClickhouseConsumer),
|
||||
Timeout: time.Second * 1,
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
}
|
||||
}
|
||||
|
||||
func getKafkaConsumer() (health.KafkaConnCheckInterface, error) {
|
||||
client, err := services.GetKafkaConsumer()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conn, ok := client.(health.KafkaConnCheckInterface)
|
||||
if !ok {
|
||||
return nil, errors.WithStack(mismatchTypeError)
|
||||
}
|
||||
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
func getClickhouseConsumer() (health.ClickhouseConnCheckInterface, error) {
|
||||
client, err := services.GetClickhouseConnection()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conn, ok := client.(health.ClickhouseConnCheckInterface)
|
||||
if !ok {
|
||||
return nil, errors.WithStack(mismatchTypeError)
|
||||
}
|
||||
|
||||
return conn, nil
|
||||
}
|
||||
65
services/jetfire/default-feature-vars.json
Normal file
65
services/jetfire/default-feature-vars.json
Normal file
@@ -0,0 +1,65 @@
|
||||
[
|
||||
"BCM_DrFrntDoorSts",
|
||||
"BCM_FrntDrDoorLockSts",
|
||||
"BCM_TotMilg_ODO",
|
||||
"BCM_PwrMod",
|
||||
|
||||
"BMS_AccueChrgTotAh",
|
||||
"BMS_AccueDchaTotAh",
|
||||
"BMS_Bat_Actual_Pack_Capacity",
|
||||
"BMS_Bat_HVmeasure_Current",
|
||||
"BMS_Bat_SoC_usable",
|
||||
"BMS_BattAvrgT",
|
||||
"BMS_Bat_measure_Energy",
|
||||
"BMS_PwrBattChrgDchaCrt1",
|
||||
"BMS_PwrBattChrgDchaCrt2",
|
||||
"BMS_PwrBattRmngCpSOC",
|
||||
"BMS_PwrBattSOH",
|
||||
"BMS_VehChrgDchgMod",
|
||||
"BMS_Cell_Volt_max",
|
||||
"BMS_Cell_Volt_min",
|
||||
|
||||
"BMS_Bat_Coolant_in",
|
||||
"BMS_Bat_Coolant_out",
|
||||
|
||||
"ECC_OutdT",
|
||||
|
||||
"ESP_TotBrkTqReq",
|
||||
"ESP_VehSpd",
|
||||
|
||||
"IBS_StateOfCharge",
|
||||
"IBS_StateOfHealth",
|
||||
"IBS_BatteryVoltage",
|
||||
"IBS_BatteryCurrent",
|
||||
"IBS_BatteryTemperature",
|
||||
"IBS_AvgRi",
|
||||
"IBS_AvailableCapacity",
|
||||
|
||||
"ICC_DispVehSpd",
|
||||
"ICC_FrntWiprCtrl",
|
||||
|
||||
"MCU_F_AlrmLamp_FS",
|
||||
"MCU_F_CrtTq",
|
||||
"MCU_F_HVActvDchaSts",
|
||||
"MCU_R_AlrmLamp_FS",
|
||||
"MCU_R_CrtTq",
|
||||
"MCU_R_HVActvDchaSts",
|
||||
|
||||
"OBC_DCPosRlyCtrlSts",
|
||||
|
||||
"VCU_ACChrgShttrSts",
|
||||
"VCU_APSPerc",
|
||||
"VCU_BrkPedlSts_GB",
|
||||
"VCU_BrkSig",
|
||||
"VCU_ChrgSts",
|
||||
"VCU_ChrgSts_GB",
|
||||
"VCU_ChrgSysOperCmd",
|
||||
"VCU_DCChrgShttrSts",
|
||||
"VCU_GearSig_GB",
|
||||
"VCU_VehChrgDchgMod",
|
||||
"VCU_VehOperMod",
|
||||
|
||||
"TBOX_GPSHei",
|
||||
"TBOX_GPSLati",
|
||||
"TBOX_GPSLongi"
|
||||
]
|
||||
143
services/jetfire/go.mod
Normal file
143
services/jetfire/go.mod
Normal file
@@ -0,0 +1,143 @@
|
||||
module github.com/fiskerinc/cloud-services/services/jetfire
|
||||
|
||||
go 1.25
|
||||
|
||||
toolchain go1.25.0
|
||||
|
||||
require (
|
||||
github.com/ClickHouse/ch-go v0.58.2
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.6.0
|
||||
github.com/fiskerinc/cloud-services/pkg v0.0.0-00010101000000-000000000000
|
||||
github.com/intel-go/fastjson v0.0.0-20170329170629-f846ae58a1ab
|
||||
github.com/julienschmidt/httprouter v1.3.0
|
||||
github.com/pkg/errors v0.9.1
|
||||
github.com/rs/zerolog v1.29.1
|
||||
github.com/sony/gobreaker v0.5.0
|
||||
github.com/stretchr/testify v1.10.0
|
||||
google.golang.org/protobuf v1.36.1
|
||||
gopkg.in/retry.v1 v1.0.3
|
||||
)
|
||||
|
||||
require (
|
||||
github.com/DataDog/appsec-internal-go v1.4.0 // indirect
|
||||
github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 // indirect
|
||||
github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 // indirect
|
||||
github.com/DataDog/datadog-go/v5 v5.3.0 // indirect
|
||||
github.com/DataDog/go-libddwaf/v2 v2.2.3 // indirect
|
||||
github.com/DataDog/go-tuf v1.0.2-0.5.2 // indirect
|
||||
github.com/DataDog/sketches-go v1.4.2 // indirect
|
||||
github.com/KyleBanks/depth v1.2.1 // indirect
|
||||
github.com/Microsoft/go-winio v0.6.1 // indirect
|
||||
github.com/ReneKroon/ttlcache/v2 v2.11.0 // indirect
|
||||
github.com/albenik/bcd v0.0.0-20170831201648-635201416bc7 // indirect
|
||||
github.com/andybalholm/brotli v1.0.6 // indirect
|
||||
github.com/cespare/xxhash/v2 v2.3.0 // indirect
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
|
||||
github.com/dmarkham/enumer v1.5.8 // indirect
|
||||
github.com/dustin/go-humanize v1.0.1 // indirect
|
||||
github.com/ebitengine/purego v0.5.2 // indirect
|
||||
github.com/elliotchance/orderedmap/v2 v2.2.0 // indirect
|
||||
github.com/fiskerinc/cloud-services/pkg/can-go v0.0.0-00010101000000-000000000000 // indirect
|
||||
github.com/frankban/quicktest v1.14.6 // indirect
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 // indirect
|
||||
github.com/go-faster/city v1.0.1 // indirect
|
||||
github.com/go-faster/errors v0.6.1 // indirect
|
||||
github.com/go-logr/logr v1.4.2 // indirect
|
||||
github.com/go-logr/stdr v1.2.2 // indirect
|
||||
github.com/go-openapi/jsonpointer v0.21.0 // indirect
|
||||
github.com/go-openapi/jsonreference v0.21.0 // indirect
|
||||
github.com/go-openapi/spec v0.21.0 // indirect
|
||||
github.com/go-openapi/swag v0.23.0 // indirect
|
||||
github.com/go-pg/pg/v10 v10.11.1 // indirect
|
||||
github.com/go-pg/zerochecker v0.2.0 // indirect
|
||||
github.com/go-playground/locales v0.14.1 // indirect
|
||||
github.com/go-playground/universal-translator v0.18.1 // indirect
|
||||
github.com/go-playground/validator/v10 v10.15.1 // indirect
|
||||
github.com/goccy/go-json v0.10.2 // indirect
|
||||
github.com/golang/mock v1.7.0-rc.1 // indirect
|
||||
github.com/golang/protobuf v1.5.4 // indirect
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/gomodule/redigo v1.8.9 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/gorilla/schema v1.2.0 // indirect
|
||||
github.com/hashicorp/errwrap v1.1.0 // indirect
|
||||
github.com/hashicorp/go-multierror v1.1.1 // indirect
|
||||
github.com/hashicorp/go-version v1.6.0 // indirect
|
||||
github.com/iancoleman/strcase v0.3.0 // indirect
|
||||
github.com/jinzhu/copier v0.3.5 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/josharian/intern v1.0.0 // indirect
|
||||
github.com/klauspost/compress v1.18.2 // indirect
|
||||
github.com/leodido/go-urn v1.2.4 // indirect
|
||||
github.com/lestrrat-go/backoff/v2 v2.0.8 // indirect
|
||||
github.com/lestrrat-go/blackmagic v1.0.1 // indirect
|
||||
github.com/lestrrat-go/httpcc v1.0.1 // indirect
|
||||
github.com/lestrrat-go/iter v1.0.2 // indirect
|
||||
github.com/lestrrat-go/jwx v1.2.25 // indirect
|
||||
github.com/lestrrat-go/option v1.0.1 // indirect
|
||||
github.com/mailru/easyjson v0.7.7 // indirect
|
||||
github.com/mattn/go-colorable v0.1.13 // indirect
|
||||
github.com/mattn/go-isatty v0.0.20 // indirect
|
||||
github.com/montanaflynn/stats v0.7.1 // indirect
|
||||
github.com/onsi/ginkgo v1.16.5 // indirect
|
||||
github.com/onsi/gomega v1.25.0 // indirect
|
||||
github.com/outcaste-io/ristretto v0.2.3 // indirect
|
||||
github.com/pascaldekloe/name v1.0.1 // indirect
|
||||
github.com/paulmach/orb v0.8.0 // indirect
|
||||
github.com/philhofer/fwd v1.1.2 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.22 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
github.com/redis/go-redis/v9 v9.5.1 // indirect
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.7.0 // indirect
|
||||
github.com/segmentio/asm v1.2.0 // indirect
|
||||
github.com/shopspring/decimal v1.3.1 // indirect
|
||||
github.com/sigurn/crc8 v0.0.0-20220107193325-2243fe600f9f // indirect
|
||||
github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a // indirect
|
||||
github.com/swaggo/http-swagger v1.3.3 // indirect
|
||||
github.com/swaggo/swag v1.8.8 // indirect
|
||||
github.com/tinylib/msgp v1.1.8 // indirect
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc // indirect
|
||||
github.com/twmb/franz-go v1.20.6 // indirect
|
||||
github.com/twmb/franz-go/pkg/kadm v1.17.2 // indirect
|
||||
github.com/twmb/franz-go/pkg/kmsg v1.12.0 // indirect
|
||||
github.com/vmihailenco/bufpool v0.1.11 // indirect
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5 // indirect
|
||||
github.com/vmihailenco/tagparser v0.1.2 // indirect
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 // indirect
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 // indirect
|
||||
github.com/xdg-go/scram v1.1.2 // indirect
|
||||
github.com/xdg-go/stringprep v1.0.4 // indirect
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect
|
||||
github.com/xeipuuv/gojsonschema v1.2.0 // indirect
|
||||
github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
|
||||
go.mongodb.org/mongo-driver v1.14.0 // indirect
|
||||
go.opentelemetry.io/otel v1.29.0 // indirect
|
||||
go.opentelemetry.io/otel/metric v1.29.0 // indirect
|
||||
go.opentelemetry.io/otel/trace v1.29.0 // indirect
|
||||
go.uber.org/atomic v1.11.0 // indirect
|
||||
go.uber.org/multierr v1.11.0 // indirect
|
||||
go.uber.org/zap v1.25.0 // indirect
|
||||
go4.org/intern v0.0.0-20230525184215-6c62f75575cb // indirect
|
||||
go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 // indirect
|
||||
golang.org/x/crypto v0.45.0 // indirect
|
||||
golang.org/x/mod v0.29.0 // indirect
|
||||
golang.org/x/net v0.47.0 // indirect
|
||||
golang.org/x/sync v0.18.0 // indirect
|
||||
golang.org/x/sys v0.38.0 // indirect
|
||||
golang.org/x/text v0.31.0 // indirect
|
||||
golang.org/x/time v0.8.0 // indirect
|
||||
golang.org/x/tools v0.38.0 // indirect
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da // indirect
|
||||
gopkg.in/DataDog/dd-trace-go.v1 v1.60.1 // indirect
|
||||
gopkg.in/yaml.v3 v3.0.1 // indirect
|
||||
inet.af/netaddr v0.0.0-20230525184311-b8eac61e914a // indirect
|
||||
mellium.im/sasl v0.3.1 // indirect
|
||||
)
|
||||
|
||||
replace (
|
||||
github.com/fiskerinc/cloud-services/pkg => ../../pkg
|
||||
github.com/fiskerinc/cloud-services/pkg/can-go => ../../pkg/can-go
|
||||
)
|
||||
548
services/jetfire/go.sum
Normal file
548
services/jetfire/go.sum
Normal file
@@ -0,0 +1,548 @@
|
||||
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
|
||||
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
|
||||
github.com/ClickHouse/ch-go v0.58.2 h1:jSm2szHbT9MCAB1rJ3WuCJqmGLi5UTjlNu+f530UTS0=
|
||||
github.com/ClickHouse/ch-go v0.58.2/go.mod h1:Ap/0bEmiLa14gYjCiRkYGbXvbe8vwdrfTYWhsuQ99aw=
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.6.0 h1:NmnPY2Cg4hCqS2ZGBep9EWHfQPAco2Vkpwb02VXtWew=
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.6.0/go.mod h1:SvXuWqDsiHJE3VAn2+3+nz9W9exOSigyskcs4DAcxJQ=
|
||||
github.com/DataDog/appsec-internal-go v1.4.0 h1:KFI8ElxkJOgpw+cUm9TXK/jh5EZvRaWM07sXlxGg9Ck=
|
||||
github.com/DataDog/appsec-internal-go v1.4.0/go.mod h1:ONW8aV6R7Thgb4g0bB9ZQCm+oRgyz5eWiW7XoQ19wIc=
|
||||
github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0 h1:bUMSNsw1iofWiju9yc1f+kBd33E3hMJtq9GuU602Iy8=
|
||||
github.com/DataDog/datadog-agent/pkg/obfuscate v0.48.0/go.mod h1:HzySONXnAgSmIQfL6gOv9hWprKJkx8CicuXuUbmgWfo=
|
||||
github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1 h1:5nE6N3JSs2IG3xzMthNFhXfOaXlrsdgqmJ73lndFf8c=
|
||||
github.com/DataDog/datadog-agent/pkg/remoteconfig/state v0.48.1/go.mod h1:Vc+snp0Bey4MrrJyiV2tVxxJb6BmLomPvN1RgAvjGaQ=
|
||||
github.com/DataDog/datadog-go/v5 v5.3.0 h1:2q2qjFOb3RwAZNU+ez27ZVDwErJv5/VpbBPprz7Z+s8=
|
||||
github.com/DataDog/datadog-go/v5 v5.3.0/go.mod h1:XRDJk1pTc00gm+ZDiBKsjh7oOOtJfYfglVCmFb8C2+Q=
|
||||
github.com/DataDog/go-libddwaf/v2 v2.2.3 h1:LpKE8AYhVrEhlmlw6FGD41udtDf7zW/aMdLNbCXpegQ=
|
||||
github.com/DataDog/go-libddwaf/v2 v2.2.3/go.mod h1:8nX0SYJMB62+fbwYmx5J7zuCGEjiC/RxAo3+AuYJuFE=
|
||||
github.com/DataDog/go-tuf v1.0.2-0.5.2 h1:EeZr937eKAWPxJ26IykAdWA4A0jQXJgkhUjqEI/w7+I=
|
||||
github.com/DataDog/go-tuf v1.0.2-0.5.2/go.mod h1:zBcq6f654iVqmkk8n2Cx81E1JnNTMOAx1UEO/wZR+P0=
|
||||
github.com/DataDog/gostackparse v0.7.0 h1:i7dLkXHvYzHV308hnkvVGDL3BR4FWl7IsXNPz/IGQh4=
|
||||
github.com/DataDog/gostackparse v0.7.0/go.mod h1:lTfqcJKqS9KnXQGnyQMCugq3u1FP6UZMfWR0aitKFMM=
|
||||
github.com/DataDog/sketches-go v1.4.2 h1:gppNudE9d19cQ98RYABOetxIhpTCl4m7CnbRZjvVA/o=
|
||||
github.com/DataDog/sketches-go v1.4.2/go.mod h1:xJIXldczJyyjnbDop7ZZcLxJdV3+7Kra7H1KMgpgkLk=
|
||||
github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc=
|
||||
github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE=
|
||||
github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84=
|
||||
github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
|
||||
github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
|
||||
github.com/ReneKroon/ttlcache/v2 v2.11.0 h1:OvlcYFYi941SBN3v9dsDcC2N8vRxyHcCmJb3Vl4QMoM=
|
||||
github.com/ReneKroon/ttlcache/v2 v2.11.0/go.mod h1:mBxvsNY+BT8qLLd6CuAJubbKo6r0jh3nb5et22bbfGY=
|
||||
github.com/albenik/bcd v0.0.0-20170831201648-635201416bc7 h1:m3Ayfs5OcAlIMEdLIQKubBsVLGee4YMUr14+d1256WE=
|
||||
github.com/albenik/bcd v0.0.0-20170831201648-635201416bc7/go.mod h1:QIAMbrwsnQZ2ES3G26RubSrDB5SPyzsp9Hts5NJdTrI=
|
||||
github.com/andybalholm/brotli v1.0.6 h1:Yf9fFpf49Zrxb9NlQaluyE92/+X7UVHlhMNJN2sxfOI=
|
||||
github.com/andybalholm/brotli v1.0.6/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig=
|
||||
github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A=
|
||||
github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
|
||||
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
|
||||
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
|
||||
github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
github.com/confluentinc/confluent-kafka-go v1.9.2 h1:gV/GxhMBUb03tFWkN+7kdhg+zf+QUM+wVkI9zwh770Q=
|
||||
github.com/confluentinc/confluent-kafka-go/v2 v2.3.0 h1:icCHutJouWlQREayFwCc7lxDAhws08td+W3/gdqgZts=
|
||||
github.com/confluentinc/confluent-kafka-go/v2 v2.3.0/go.mod h1:/VTy8iEpe6mD9pkCH5BhijlUl8ulUXymKv1Qig5Rgb8=
|
||||
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
|
||||
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
|
||||
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc=
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.0-20210816181553-5444fa50b93d/go.mod h1:tmAIfUFEirG/Y8jhZ9M+h36obRZAk/1fcSpXwAVlfqE=
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4=
|
||||
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc=
|
||||
github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 h1:fAjc9m62+UWV/WAFKLNi6ZS0675eEUC9y3AlwSbQu1Y=
|
||||
github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||
github.com/dmarkham/enumer v1.5.8 h1:fIF11F9l5jyD++YYvxcSH5WgHfeaSGPaN/T4kOQ4qEM=
|
||||
github.com/dmarkham/enumer v1.5.8/go.mod h1:d10o8R3t/gROm2p3BXqTkMt2+HMuxEmWCXzorAruYak=
|
||||
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/dvyukov/go-fuzz v0.0.0-20210103155950-6a8e9d1f2415/go.mod h1:11Gm+ccJnvAhCNLlf5+cS9KjtbaD5I5zaZpFMsTHWTw=
|
||||
github.com/ebitengine/purego v0.5.2 h1:r2MQEtkGzZ4LRtFZVAg5bjYKnUbxxloaeuGxH0t7qfs=
|
||||
github.com/ebitengine/purego v0.5.2/go.mod h1:ah1In8AOtksoNK6yk5z1HTJeUkC1Ez4Wk2idgGslMwQ=
|
||||
github.com/elliotchance/orderedmap/v2 v2.2.0 h1:7/2iwO98kYT4XkOjA9mBEIwvi4KpGB4cyHeOFOnj4Vk=
|
||||
github.com/elliotchance/orderedmap/v2 v2.2.0/go.mod h1:85lZyVbpGaGvHvnKa7Qhx7zncAdBIBq6u56Hb1PRU5Q=
|
||||
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
|
||||
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
|
||||
github.com/frankban/quicktest v1.2.2/go.mod h1:Qh/WofXFeiAFII1aEBu529AtJo6Zg2VHscnEsbBnJ20=
|
||||
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
|
||||
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
|
||||
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
|
||||
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
|
||||
github.com/fsnotify/fsnotify v1.8.0 h1:dAwr6QBTBZIkG8roQaJjGof0pp0EeF+tNV7YBP3F/8M=
|
||||
github.com/fsnotify/fsnotify v1.8.0/go.mod h1:8jBTzvmWwFyi3Pb8djgCCO5IBqzKJ/Jwo8TRcHyHii0=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
|
||||
github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
|
||||
github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw=
|
||||
github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw=
|
||||
github.com/go-faster/errors v0.6.1 h1:nNIPOBkprlKzkThvS/0YaX8Zs9KewLCOSFQS5BU06FI=
|
||||
github.com/go-faster/errors v0.6.1/go.mod h1:5MGV2/2T9yvlrbhe9pD9LO5Z/2zCSq2T8j+Jpi2LAyY=
|
||||
github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
|
||||
github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY=
|
||||
github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
|
||||
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
|
||||
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
|
||||
github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ=
|
||||
github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY=
|
||||
github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ=
|
||||
github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4=
|
||||
github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY=
|
||||
github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk=
|
||||
github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE=
|
||||
github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ=
|
||||
github.com/go-pg/pg/v10 v10.11.1 h1:vYwbFpqoMpTDphnzIPshPPepdy3VpzD8qo29OFKp4vo=
|
||||
github.com/go-pg/pg/v10 v10.11.1/go.mod h1:ExJWndhDNNftBdw1Ow83xqpSf4WMSJK8urmXD5VXS1I=
|
||||
github.com/go-pg/zerochecker v0.2.0 h1:pp7f72c3DobMWOb2ErtZsnrPaSvHd2W4o9//8HtF4mU=
|
||||
github.com/go-pg/zerochecker v0.2.0/go.mod h1:NJZ4wKL0NmTtz0GKCoJ8kym6Xn/EQzXRl2OnAe7MmDo=
|
||||
github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
|
||||
github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
|
||||
github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
|
||||
github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
|
||||
github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
|
||||
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
|
||||
github.com/go-playground/validator/v10 v10.15.1 h1:BSe8uhN+xQ4r5guV/ywQI4gO59C2raYcGffYWZEjZzM=
|
||||
github.com/go-playground/validator/v10 v10.15.1/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
|
||||
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
|
||||
github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
|
||||
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
|
||||
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
|
||||
github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
|
||||
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
|
||||
github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
|
||||
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
|
||||
github.com/golang/mock v1.7.0-rc.1 h1:YojYx61/OLFsiv6Rw1Z96LpldJIy31o+UHmwAUMJ6/U=
|
||||
github.com/golang/mock v1.7.0-rc.1/go.mod h1:s42URUywIqd+OcERslBJvOjepvNymP31m3q8d/GkuRs=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
|
||||
github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
|
||||
github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
|
||||
github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
|
||||
github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
|
||||
github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
|
||||
github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
|
||||
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
|
||||
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
|
||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws=
|
||||
github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE=
|
||||
github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
|
||||
github.com/google/go-cmp v0.2.1-0.20190312032427-6f77996f0c42/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
|
||||
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
|
||||
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
|
||||
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 h1:E/LAvt58di64hlYjx7AsNS6C/ysHWYo+2qPCZKTQhRo=
|
||||
github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
|
||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/schema v1.2.0 h1:YufUaxZYCKGFuAq3c96BOhjgd5nmXiOY9NGzF247Tsc=
|
||||
github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
|
||||
github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
|
||||
github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
|
||||
github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
|
||||
github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
|
||||
github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
|
||||
github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
|
||||
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
|
||||
github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=
|
||||
github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
|
||||
github.com/intel-go/fastjson v0.0.0-20170329170629-f846ae58a1ab h1:K7WJJ5AnrQV/6tEh0Qqs19KLzvsq5V15f9CifKii6aU=
|
||||
github.com/intel-go/fastjson v0.0.0-20170329170629-f846ae58a1ab/go.mod h1:xr9Svf97gkxlW+ZDxs47vReKp7m9EUzNhEGOLyBHR+8=
|
||||
github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg=
|
||||
github.com/jinzhu/copier v0.3.5/go.mod h1:DfbEm0FYsaqBcKcFuvmOZb218JkPGtvSHsKg8S8hyyg=
|
||||
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
|
||||
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
|
||||
github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY=
|
||||
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
|
||||
github.com/julienschmidt/httprouter v1.3.0 h1:U0609e9tgbseu3rBINet9P48AI/D3oJs4dN7jwJOQ1U=
|
||||
github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
|
||||
github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/compress v1.18.2 h1:iiPHWW0YrcFgpBYhsA6D1+fqHssJscY/Tm/y2Uqnapk=
|
||||
github.com/klauspost/compress v1.18.2/go.mod h1:R0h/fSBs8DE4ENlcrlib3PsXS61voFxhIs2DeRhCvJ4=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
|
||||
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
|
||||
github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
|
||||
github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
|
||||
github.com/lestrrat-go/backoff/v2 v2.0.8 h1:oNb5E5isby2kiro9AgdHLv5N5tint1AnDVVf2E2un5A=
|
||||
github.com/lestrrat-go/backoff/v2 v2.0.8/go.mod h1:rHP/q/r9aT27n24JQLa7JhSQZCKBBOiM/uP402WwN8Y=
|
||||
github.com/lestrrat-go/blackmagic v1.0.0/go.mod h1:TNgH//0vYSs8VXDCfkZLgIrVTTXQELZffUV0tz3MtdQ=
|
||||
github.com/lestrrat-go/blackmagic v1.0.1 h1:lS5Zts+5HIC/8og6cGHb0uCcNCa3OUt1ygh3Qz2Fe80=
|
||||
github.com/lestrrat-go/blackmagic v1.0.1/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU=
|
||||
github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE=
|
||||
github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E=
|
||||
github.com/lestrrat-go/iter v1.0.1/go.mod h1:zIdgO1mRKhn8l9vrZJZz9TUMMFbQbLeTsbqPDrJ/OJc=
|
||||
github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI=
|
||||
github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4=
|
||||
github.com/lestrrat-go/jwx v1.2.25 h1:tAx93jN2SdPvFn08fHNAhqFJazn5mBBOB8Zli0g0otA=
|
||||
github.com/lestrrat-go/jwx v1.2.25/go.mod h1:zoNuZymNl5lgdcu6P7K6ie2QRll5HVfF4xwxBBK1NxY=
|
||||
github.com/lestrrat-go/option v1.0.0/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
|
||||
github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU=
|
||||
github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I=
|
||||
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
|
||||
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
|
||||
github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
|
||||
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
|
||||
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
|
||||
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
|
||||
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
|
||||
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
|
||||
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
|
||||
github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8eaE=
|
||||
github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
|
||||
github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
|
||||
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
|
||||
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
|
||||
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
|
||||
github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
|
||||
github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
|
||||
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
|
||||
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
|
||||
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
|
||||
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
|
||||
github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
|
||||
github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
|
||||
github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
|
||||
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
|
||||
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
|
||||
github.com/outcaste-io/ristretto v0.2.3 h1:AK4zt/fJ76kjlYObOeNwh4T3asEuaCmp26pOvUOL9w0=
|
||||
github.com/outcaste-io/ristretto v0.2.3/go.mod h1:W8HywhmtlopSB1jeMg3JtdIhf+DYkLAr0VN/s4+MHac=
|
||||
github.com/pascaldekloe/name v1.0.1 h1:9lnXOHeqeHHnWLbKfH6X98+4+ETVqFqxN09UXSjcMb0=
|
||||
github.com/pascaldekloe/name v1.0.1/go.mod h1:Z//MfYJnH4jVpQ9wkclwu2I2MkHmXTlT9wR5UZScttM=
|
||||
github.com/paulmach/orb v0.8.0 h1:W5XAt5yNPNnhaMNEf0xNSkBMJ1LzOzdk2MRlB6EN0Vs=
|
||||
github.com/paulmach/orb v0.8.0/go.mod h1:FWRlTgl88VI1RBx/MkrwWDRhQ96ctqMCh8boXhmqB/A=
|
||||
github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY=
|
||||
github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
|
||||
github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
|
||||
github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU=
|
||||
github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
|
||||
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
|
||||
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
|
||||
github.com/redis/go-redis/v9 v9.5.1 h1:H1X4D3yHPaYrkL5X06Wh6xNVM/pX0Ft4RV0vMGvLBh8=
|
||||
github.com/redis/go-redis/v9 v9.5.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
|
||||
github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052 h1:Qp27Idfgi6ACvFQat5+VJvlYToylpM/hcyLBI3WaKPA=
|
||||
github.com/richardartoul/molecule v1.0.1-0.20221107223329-32cfee06a052/go.mod h1:uvX/8buq8uVeiZiFht+0lqSLBHF+uGV8BrTv8W/SIwk=
|
||||
github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a h1:3QH7VyOaaiUHNrA9Se4YQIRkDTCw1EJls9xTUCaCeRM=
|
||||
github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ=
|
||||
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
|
||||
github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
|
||||
github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
|
||||
github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
|
||||
github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc=
|
||||
github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.7.0 h1:OwvJ5jQf9LnIAS83waAjPbcMsODrTQUpJ02eNLUoxBg=
|
||||
github.com/secure-systems-lab/go-securesystemslib v0.7.0/go.mod h1:/2gYnlnHVQ6xeGtfIqFy7Do03K4cdCY0A/GlJLDKLHI=
|
||||
github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys=
|
||||
github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs=
|
||||
github.com/shopspring/decimal v1.3.1 h1:2Usl1nmF/WZucqkFZhnfFYxxxu8LG21F6nPQBE5gKV8=
|
||||
github.com/shopspring/decimal v1.3.1/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o=
|
||||
github.com/sigurn/crc8 v0.0.0-20220107193325-2243fe600f9f h1:1R9KdKjCNSd7F8iGTxIpoID9prlYH8nuNYKt0XvweHA=
|
||||
github.com/sigurn/crc8 v0.0.0-20220107193325-2243fe600f9f/go.mod h1:vQhwQ4meQEDfahT5kd61wLAF5AAeh5ZPLVI4JJ/tYo8=
|
||||
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
|
||||
github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg=
|
||||
github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
|
||||
github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
|
||||
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
|
||||
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
|
||||
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
|
||||
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
|
||||
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
|
||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||
github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a h1:kAe4YSu0O0UFn1DowNo2MY5p6xzqtJ/wQ7LZynSvGaY=
|
||||
github.com/swaggo/files v0.0.0-20220728132757-551d4a08d97a/go.mod h1:lKJPbtWzJ9JhsTN1k1gZgleJWY/cqq0psdoMmaThG3w=
|
||||
github.com/swaggo/http-swagger v1.3.3 h1:Hu5Z0L9ssyBLofaama21iYaF2VbWyA8jdohaaCGpHsc=
|
||||
github.com/swaggo/http-swagger v1.3.3/go.mod h1:sE+4PjD89IxMPm77FnkDz0sdO+p5lbXzrVWT6OTVVGo=
|
||||
github.com/swaggo/swag v1.8.8 h1:/GgJmrJ8/c0z4R4hoEPZ5UeEhVGdvsII4JbVDLbR7Xc=
|
||||
github.com/swaggo/swag v1.8.8/go.mod h1:ezQVUUhly8dludpVk+/PuwJWvLLanB13ygV5Pr9enSk=
|
||||
github.com/tinylib/msgp v1.1.8 h1:FCXC1xanKO4I8plpHGH2P7koL/RzZs12l/+r7vakfm0=
|
||||
github.com/tinylib/msgp v1.1.8/go.mod h1:qkpG+2ldGg4xRFmx+jfTvZPxfGFhi64BcnL9vkCm/Tw=
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc h1:9lRDQMhESg+zvGYmW5DyG0UqvY96Bu5QYsTLvCHdrgo=
|
||||
github.com/tmthrgd/go-hex v0.0.0-20190904060850-447a3041c3bc/go.mod h1:bciPuU6GHm1iF1pBvUfxfsH0Wmnc2VbpgvbI9ZWuIRs=
|
||||
github.com/twmb/franz-go v1.20.6 h1:TpQTt4QcixJ1cHEmQGPOERvTzo99s8jAutmS7rbSD6w=
|
||||
github.com/twmb/franz-go v1.20.6/go.mod h1:u+FzH2sInp7b9HNVv2cZN8AxdXy6y/AQ1Bkptu4c0FM=
|
||||
github.com/twmb/franz-go/pkg/kadm v1.17.2 h1:g5f1sAxnTkYC6G96pV5u715HWhxd66hWaDZUAQ8xHY8=
|
||||
github.com/twmb/franz-go/pkg/kadm v1.17.2/go.mod h1:ST55zUB+sUS+0y+GcKY/Tf1XxgVilaFpB9I19UubLmU=
|
||||
github.com/twmb/franz-go/pkg/kmsg v1.12.0 h1:CbatD7ers1KzDNgJqPbKOq0Bz/WLBdsTH75wgzeVaPc=
|
||||
github.com/twmb/franz-go/pkg/kmsg v1.12.0/go.mod h1:+DPt4NC8RmI6hqb8G09+3giKObE6uD2Eya6CfqBpeJY=
|
||||
github.com/vmihailenco/bufpool v0.1.11 h1:gOq2WmBrq0i2yW5QJ16ykccQ4wH9UyEsgLm6czKAd94=
|
||||
github.com/vmihailenco/bufpool v0.1.11/go.mod h1:AFf/MOy3l2CFTKbxwt0mp2MwnqjNEs5H/UxrkA5jxTQ=
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.4/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5 h1:5gO0H1iULLWGhs2H5tbAHIZTV8/cYafcFOr9znI5mJU=
|
||||
github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc=
|
||||
github.com/vmihailenco/tagparser v0.1.2 h1:gnjoVuB/kljJ5wICEEOpx98oXMWPLj22G67Vbd1qPqc=
|
||||
github.com/vmihailenco/tagparser v0.1.2/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0 h1:y09buUbR+b5aycVFQs/g70pqKVZNBmxwAhO7/IwNM9g=
|
||||
github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0 h1:Su7DPu48wXMwC3bs7MCNG+z4FhcyEuz5dlvchbq0B0c=
|
||||
github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
|
||||
github.com/xdg-go/scram v1.1.2 h1:FHX5I5B4i4hKRVRBCFRxq1iQRej7WO3hhBuJf+UUySY=
|
||||
github.com/xdg-go/scram v1.1.2/go.mod h1:RT/sEzTbU5y00aCK8UOx6R7YryM0iF1N2MOmC3kKLN4=
|
||||
github.com/xdg-go/stringprep v1.0.4 h1:XLI/Ng3O1Atzq0oBs3TWm+5ZVgkq2aqdlvP9JtoZ6c8=
|
||||
github.com/xdg-go/stringprep v1.0.4/go.mod h1:mPGuuIYwz7CmR2bT9j4GbQqutWS1zV24gijq1dTyGkM=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
|
||||
github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
|
||||
github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
|
||||
github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
|
||||
github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a h1:fZHgsYlfvtyqToslyjUt3VOPF4J7aK/3MPcK7xp3PDk=
|
||||
github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a/go.mod h1:ul22v+Nro/R083muKhosV54bj5niojjWZvU8xrevuH4=
|
||||
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
|
||||
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
|
||||
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
|
||||
go.mongodb.org/mongo-driver v1.14.0 h1:P98w8egYRjYe3XDjxhYJagTokP/H6HzlsnojRgZRd80=
|
||||
go.mongodb.org/mongo-driver v1.14.0/go.mod h1:Vzb0Mk/pa7e6cWw85R4F/endUC3u0U9jGcNU603k65c=
|
||||
go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw=
|
||||
go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8=
|
||||
go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc=
|
||||
go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8=
|
||||
go.opentelemetry.io/otel/sdk v1.29.0 h1:vkqKjk7gwhS8VaWb0POZKmIEDimRCMsopNYnriHyryo=
|
||||
go.opentelemetry.io/otel/sdk v1.29.0/go.mod h1:pM8Dx5WKnvxLCb+8lG1PRNIDxu9g9b9g59Qr7hfAAok=
|
||||
go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4=
|
||||
go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ=
|
||||
go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
|
||||
go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE=
|
||||
go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0=
|
||||
go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A=
|
||||
go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
|
||||
go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
|
||||
go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
|
||||
go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
|
||||
go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c=
|
||||
go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk=
|
||||
go4.org/intern v0.0.0-20211027215823-ae77deb06f29/go.mod h1:cS2ma+47FKrLPdXFpr7CuxiTW3eyJbWew4qx0qtQWDA=
|
||||
go4.org/intern v0.0.0-20230525184215-6c62f75575cb h1:ae7kzL5Cfdmcecbh22ll7lYP3iuUdnfnhiPcSaDgH/8=
|
||||
go4.org/intern v0.0.0-20230525184215-6c62f75575cb/go.mod h1:Ycrt6raEcnF5FTsLiLKkhBTO6DPX3RCUCUVnks3gFJU=
|
||||
go4.org/unsafe/assume-no-moving-gc v0.0.0-20211027215541-db492cf91b37/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
|
||||
go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
|
||||
go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 h1:lGdhQUN/cnWdSH3291CUuxSEqc+AsGTiDxPP3r2J0l4=
|
||||
go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6/go.mod h1:FftLjUGFEDu5k8lt0ddY+HcrH/qU/0qk+H8j9/nTl3E=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
|
||||
golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
|
||||
golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw=
|
||||
golang.org/x/crypto v0.45.0 h1:jMBrvKuj23MTlT0bQEOBcAE0mjg8mK9RXFhRH6nyF3Q=
|
||||
golang.org/x/crypto v0.45.0/go.mod h1:XTGrrkGJve7CYK7J8PEww4aY7gM3qMCElcJQ8n8JdX4=
|
||||
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
|
||||
golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
|
||||
golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY=
|
||||
golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg=
|
||||
golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
|
||||
golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
|
||||
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
|
||||
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
|
||||
golang.org/x/mod v0.29.0 h1:HV8lRxZC4l2cr3Zq1LvtOsi/ThTgWnUk/y64QSs8GwA=
|
||||
golang.org/x/mod v0.29.0/go.mod h1:NyhrlYXJ2H4eJiRy/WDBO6HMqZQ6q9nk4JzS3NuCK+w=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
|
||||
golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
|
||||
golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=
|
||||
golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE=
|
||||
golang.org/x/net v0.47.0 h1:Mx+4dIFzqraBXUugkia1OOvlD6LemFo1ALMHjrXDOhY=
|
||||
golang.org/x/net v0.47.0/go.mod h1:/jNxtkgq5yWUGYkaZGqo27cfGZ1c5Nen03aYrrKpVRU=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.18.0 h1:kr88TuHDroi+UVf+0hZnirlk8o8T+4MrK6mr60WkH/I=
|
||||
golang.org/x/sync v0.18.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
|
||||
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220627191245-f75cf1eec38b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
|
||||
golang.org/x/sys v0.38.0 h1:3yZWxaJjBmCWXqhN1qh02AkOnCQ1poK6oF+a7xWL6Gc=
|
||||
golang.org/x/sys v0.38.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
|
||||
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
|
||||
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
|
||||
golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
|
||||
golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
|
||||
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
|
||||
golang.org/x/text v0.31.0 h1:aC8ghyu4JhP8VojJ2lEHBnochRno1sgL6nEi9WGFGMM=
|
||||
golang.org/x/text v0.31.0/go.mod h1:tKRAlv61yKIjGGHX/4tP1LTbc13YSec1pxVEWXzfoeM=
|
||||
golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
|
||||
golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
|
||||
golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
|
||||
golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
|
||||
golang.org/x/tools v0.0.0-20191108193012-7d206e10da11/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
|
||||
golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
|
||||
golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
|
||||
golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
|
||||
golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=
|
||||
golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
|
||||
golang.org/x/tools v0.1.8/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU=
|
||||
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
|
||||
golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
|
||||
golang.org/x/tools v0.38.0 h1:Hx2Xv8hISq8Lm16jvBZ2VQf+RLmbd7wVUsALibYI/IQ=
|
||||
golang.org/x/tools v0.38.0/go.mod h1:yEsQ/d/YK8cjh0L6rZlY8tgtlKiBNTL14pGDJPJpYQs=
|
||||
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da h1:noIWHXmPHxILtqtCOPIhSt0ABwskkZKjD3bXGnZGpNY=
|
||||
golang.org/x/xerrors v0.0.0-20240903120638-7835f813f4da/go.mod h1:NDW/Ps6MPRej6fsCIbMTohpP40sJ/P/vI1MoTEGwX90=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
|
||||
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
|
||||
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
|
||||
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
|
||||
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
|
||||
google.golang.org/grpc v1.67.3 h1:OgPcDAFKHnH8X3O4WcO4XUc8GRDeKsKReqbQtiCj7N8=
|
||||
google.golang.org/grpc v1.67.3/go.mod h1:YGaHCc6Oap+FzBJTZLBzkGSYt/cvGPFTPxkn7QfSU8s=
|
||||
google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
|
||||
google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
|
||||
google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
|
||||
google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
|
||||
google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
|
||||
google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
|
||||
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
|
||||
google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
|
||||
google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
||||
gopkg.in/DataDog/dd-trace-go.v1 v1.60.1 h1:Sqkq62MxQW/RD+sgZsQuUdHWHyXI4JS5x0lxlxrv2Hk=
|
||||
gopkg.in/DataDog/dd-trace-go.v1 v1.60.1/go.mod h1:6aArYrAHjnuaofJ3lKuSRQbhrBx1LcSpiEYCIScJE5Y=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
|
||||
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
|
||||
gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
|
||||
gopkg.in/retry.v1 v1.0.3 h1:a9CArYczAVv6Qs6VGoLMio99GEs7kY9UzSF9+LD+iGs=
|
||||
gopkg.in/retry.v1 v1.0.3/go.mod h1:FJkXmWiMaAo7xB+xhvDF59zhfjDWyzmyAxiT4dB688g=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
|
||||
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
|
||||
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU=
|
||||
gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU=
|
||||
honnef.co/go/gotraceui v0.2.0 h1:dmNsfQ9Vl3GwbiVD7Z8d/osC6WtGGrasyrC2suc4ZIQ=
|
||||
honnef.co/go/gotraceui v0.2.0/go.mod h1:qHo4/W75cA3bX0QQoSvDjbJa4R8mAyyFjbWAj63XElc=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
inet.af/netaddr v0.0.0-20230525184311-b8eac61e914a h1:1XCVEdxrvL6c0TGOhecLuB7U9zYNdxZEjvOqJreKZiM=
|
||||
inet.af/netaddr v0.0.0-20230525184311-b8eac61e914a/go.mod h1:e83i32mAQOW1LAqEIweALsuK2Uw4mhQadA5r7b0Wobo=
|
||||
mellium.im/sasl v0.3.1 h1:wE0LW6g7U83vhvxjC1IY8DnXM+EU095yeo8XClvCdfo=
|
||||
mellium.im/sasl v0.3.1/go.mod h1:xm59PUYpZHhgQ9ZqoJ5QaCqzWMi8IeS49dhp6plPCzw=
|
||||
97
services/jetfire/handlers/batch.go
Normal file
97
services/jetfire/handlers/batch.go
Normal file
@@ -0,0 +1,97 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/models"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/services"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/utils"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/grpc/kafka_grpc"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/fiskerinc/cloud-services/pkg/utils/envtool"
|
||||
)
|
||||
|
||||
var (
|
||||
//downsampling timers and delays
|
||||
clearCacheTimer = time.Now().Truncate(time.Second)
|
||||
featureDelay = envtool.GetEnvDuration("JETFIRE_FEATURE_DOWNSAMPLE_US", 1000000) * time.Microsecond
|
||||
clearCacheDelay = envtool.GetEnvDuration("JETFIRE_STATE_TIMEOUT_MS", 3600000) * time.Millisecond
|
||||
futureTimeThreshold = envtool.GetEnvDuration("JETFIRE_FUTURE_THRESHOLD_MS", 2*24*60*60*1000) * time.Millisecond
|
||||
)
|
||||
|
||||
// Handles the batch of Signals. returns batchFlag, error
|
||||
// where batchFlag corresponds to FeatureUpdateFlag for which buffers were updated with this batch
|
||||
func HandleSignalBatch(batchData []*kafka_grpc.GRPC_CANSignal, vehicleCache *models.VehicleCache, producerChannel chan models.InsertCommand) (uint, error) {
|
||||
start := time.Now()
|
||||
|
||||
batchFlag := uint(0)
|
||||
|
||||
//Iterate through received can signals and add to cache
|
||||
futureWarning := false //only one future warning per batch of data
|
||||
|
||||
//sets of pointers to insert into insertion buffers
|
||||
featureUpdates := make(map[*models.VehicleState]bool)
|
||||
|
||||
logger.Debug().Msgf("Processing batch %d signals...", len(batchData))
|
||||
|
||||
skipLog := false
|
||||
|
||||
for _, signal := range batchData {
|
||||
signal.Timestamp = utils.FixFloatTimestampScale(signal.Timestamp)
|
||||
if start.Add(futureTimeThreshold).Before(utils.FloatToTime(signal.Timestamp)) && !futureWarning {
|
||||
// Throw out any signals that are from the future.
|
||||
futureWarning = true
|
||||
logger.Warn().Msgf("ignoring signal(s) from %s from future: %f (currently %f)", signal.Vin, signal.Timestamp, utils.TimeToFloat(start))
|
||||
continue
|
||||
}
|
||||
|
||||
err := vehicleCache.UpdateSignal(signal, utils.FeatureUpdateFlag)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
//do not continue yet; attempt to add to vehicle signal buffer
|
||||
}
|
||||
|
||||
//Check vehicle state for update
|
||||
// if signal is not in tracked vars, then skip.
|
||||
// if VIN is not in vehicle cache, then log and skip
|
||||
state, ok := vehicleCache.Cache[signal.Vin]
|
||||
if !ok || state == nil {
|
||||
if len(*vehicleCache.SignalsSet) > 0 {
|
||||
_, ok = (*vehicleCache.SignalsSet)[signal.Name]
|
||||
if ok && !skipLog {
|
||||
// no vin state but signal update should have created a vin state... log warning
|
||||
logger.Error().Msgf("unexpected missing VIN state from cache for VIN {%s} and signal {%s}", signal.Vin, signal.Name)
|
||||
skipLog = true
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
if state.TimeSincePolled(utils.FeatureUpdateFlag) > featureDelay {
|
||||
featureUpdates[state] = true
|
||||
}
|
||||
}
|
||||
|
||||
services.SchemaLock.Lock()
|
||||
defer services.SchemaLock.Unlock()
|
||||
|
||||
//TODO: investigate optimizing selecting VINs that need to be appended to buffer
|
||||
// iterating through maps is slow!
|
||||
|
||||
//batch rows for insertion for feature
|
||||
for vinState := range featureUpdates {
|
||||
err := services.GetFeatureBatch().AppendRow(services.GetFeatureVars(), vinState, producerChannel)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
} else {
|
||||
vinState.SetPollTime(utils.FeatureUpdateFlag)
|
||||
batchFlag |= utils.FeatureUpdateFlag
|
||||
}
|
||||
|
||||
err = services.GetFeatureLastBatch().AppendRow(services.GetFeatureVars(), vinState, producerChannel)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
}
|
||||
}
|
||||
|
||||
return batchFlag, nil
|
||||
}
|
||||
21
services/jetfire/handlers/reset.go
Normal file
21
services/jetfire/handlers/reset.go
Normal file
@@ -0,0 +1,21 @@
|
||||
package handlers
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/services"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
)
|
||||
|
||||
func ResetSchemaDefinitions(w http.ResponseWriter, r *http.Request) {
|
||||
startTime := time.Now()
|
||||
logger.Info().Msg("resetting tracked variables and schemas")
|
||||
services.ResetCacheVars()
|
||||
endTime := time.Now()
|
||||
logger.Debug().Msgf("reset tracked variables and schemas, time taken: %fms; %d variables",
|
||||
float64(endTime.UnixMilli()-startTime.UnixMilli()),
|
||||
len(services.GetFeatureVars()),
|
||||
)
|
||||
}
|
||||
45
services/jetfire/main.go
Normal file
45
services/jetfire/main.go
Normal file
@@ -0,0 +1,45 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/controllers"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/server"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/services"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/kafka"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/fiskerinc/cloud-services/pkg/tracer"
|
||||
"github.com/fiskerinc/cloud-services/pkg/utils/app"
|
||||
)
|
||||
|
||||
var (
|
||||
SERVICE_NAME = "jetfire"
|
||||
)
|
||||
|
||||
func init() {
|
||||
app.Setup(SERVICE_NAME, cleanup)
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
defer cleanup()
|
||||
|
||||
tracer.Start()
|
||||
defer tracer.Stop()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
go controllers.HealthCheck()
|
||||
|
||||
services.ResetCacheVars()
|
||||
go server.StartHTTPServer() //listen for reset requests
|
||||
go server.StartConsumer(ctx, kafka.VehicleSignal)
|
||||
|
||||
select {}
|
||||
}
|
||||
|
||||
func cleanup() {
|
||||
logger.Close()
|
||||
}
|
||||
933
services/jetfire/models/clickhouse.go
Normal file
933
services/jetfire/models/clickhouse.go
Normal file
@@ -0,0 +1,933 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/utils"
|
||||
"math"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/grpc/kafka_grpc"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/fiskerinc/cloud-services/pkg/utils/envtool"
|
||||
"github.com/ClickHouse/ch-go"
|
||||
"github.com/ClickHouse/ch-go/proto"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/rs/zerolog"
|
||||
"github.com/sony/gobreaker"
|
||||
)
|
||||
|
||||
type InsertBlockType int
|
||||
|
||||
const (
|
||||
NilBlockType InsertBlockType = -1 //insert buffers using this block type will have a nil block.
|
||||
|
||||
PivotBlockType InsertBlockType = iota
|
||||
SignalBlockType
|
||||
VinLastBlockType
|
||||
)
|
||||
|
||||
var (
|
||||
initialBlockPoolSize = envtool.GetEnvInt("JETFIRE_MIN_BLOCKS", 2)
|
||||
maxBlockPoolSize = envtool.GetEnvInt("JETFIRE_MAX_BLOCKS", 4)
|
||||
maxBufferBytes = envtool.GetEnvInt("JETFIRE_BUFFER_MAX_BYTES", 128<<20) / maxBlockPoolSize
|
||||
)
|
||||
|
||||
const NO_TIMESTAMP_LOG_SAMPLE_RATE = 10000
|
||||
var logSampler zerolog.Logger
|
||||
// Creating a log sampler
|
||||
func init(){
|
||||
logSampler = logger.Sample(zerolog.RandomSampler(NO_TIMESTAMP_LOG_SAMPLE_RATE))
|
||||
}
|
||||
|
||||
// InsertBuffer abstracts buffer appends and insertions to clickhouse, even for different underlying schemas.
|
||||
// A pool of blocks are allocated for each InsertBuffer.
|
||||
//
|
||||
// These pools do not implement leaky buffer pattern to ensure data order and timeliness.
|
||||
type InsertBuffer struct {
|
||||
InsertTime time.Time
|
||||
insertDelay time.Duration
|
||||
|
||||
blockPool []insertBlock //pool of all blocks, including busy blocks.
|
||||
|
||||
//linked list of available blocks in the pool
|
||||
freeLock sync.Mutex //mutex for linked list
|
||||
freeHead *BlockNode
|
||||
freeTail *BlockNode
|
||||
|
||||
blockType InsertBlockType
|
||||
tripInfo bool
|
||||
|
||||
TableName string
|
||||
}
|
||||
|
||||
// linked list node for insertBlock
|
||||
type BlockNode struct {
|
||||
block insertBlock
|
||||
next *BlockNode
|
||||
}
|
||||
|
||||
// Command struct for inserter goroutine
|
||||
type InsertCommand struct {
|
||||
Buffer *InsertBuffer
|
||||
Block insertBlock
|
||||
}
|
||||
|
||||
func NewInsertBuffer(tripInfo bool, signalNames []string, tableName string, insertDelay time.Duration, blockType InsertBlockType) *InsertBuffer {
|
||||
newBlockPool := []insertBlock{}
|
||||
|
||||
for i := 0; i < initialBlockPoolSize; i++ {
|
||||
var newBlock insertBlock = AllocateNewBlock(signalNames, maxBufferBytes, blockType, tripInfo)
|
||||
|
||||
if newBlock != nil {
|
||||
newBlockPool = append(newBlockPool, newBlock)
|
||||
}
|
||||
}
|
||||
|
||||
newBuffer := new(InsertBuffer)
|
||||
newBuffer.insertDelay = insertDelay
|
||||
newBuffer.blockPool = newBlockPool
|
||||
newBuffer.TableName = tableName
|
||||
newBuffer.blockType = blockType
|
||||
newBuffer.tripInfo = tripInfo
|
||||
|
||||
newBuffer.InitFreeBlocksList()
|
||||
|
||||
return newBuffer
|
||||
}
|
||||
|
||||
// memory block linked list functions. These are for available blocks that are ready to accept data.
|
||||
func (buffer *InsertBuffer) AppendFreeBlock(block insertBlock) {
|
||||
buffer.freeLock.Lock()
|
||||
defer buffer.freeLock.Unlock()
|
||||
|
||||
if buffer.freeHead == nil {
|
||||
buffer.freeHead = &BlockNode{block: block}
|
||||
|
||||
buffer.freeTail = buffer.freeHead
|
||||
return
|
||||
}
|
||||
if buffer.freeTail == nil { //this should never occur!
|
||||
err := errors.Errorf("Unexpected nil tail for InsertBuffer blocks with non nil head!")
|
||||
logger.Error().Err(err).Send()
|
||||
return
|
||||
}
|
||||
|
||||
buffer.freeTail.next = &BlockNode{block: block}
|
||||
buffer.freeTail = buffer.freeTail.next
|
||||
}
|
||||
|
||||
func (buffer *InsertBuffer) PopFreeBlock() insertBlock {
|
||||
buffer.freeLock.Lock()
|
||||
defer buffer.freeLock.Unlock()
|
||||
|
||||
blockNode := buffer.freeHead
|
||||
if blockNode == buffer.freeTail {
|
||||
buffer.freeTail = nil
|
||||
}
|
||||
if blockNode == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
buffer.freeHead = blockNode.next
|
||||
return blockNode.block
|
||||
}
|
||||
|
||||
func (buffer *InsertBuffer) PeekFreeBlock() insertBlock {
|
||||
buffer.freeLock.Lock()
|
||||
defer buffer.freeLock.Unlock()
|
||||
|
||||
blockNode := buffer.freeHead
|
||||
if blockNode == nil {
|
||||
return nil
|
||||
}
|
||||
return blockNode.block
|
||||
}
|
||||
|
||||
// allocates a new block with given params and returns it
|
||||
func AllocateNewBlock(signalNames []string, maxBufferBytes int, blockType InsertBlockType, tripInfo bool) insertBlock {
|
||||
var newBlock insertBlock = nil
|
||||
if blockType == PivotBlockType {
|
||||
newBlock = NewProtoPivotBlock(signalNames, maxBufferBytes, tripInfo)
|
||||
} else if blockType == SignalBlockType {
|
||||
newBlock = NewProtoSignalBlock(signalNames, maxBufferBytes)
|
||||
} else if blockType == VinLastBlockType {
|
||||
newBlock = NewProtoVinLastBlock(signalNames, maxVinCount, tripInfo)
|
||||
}
|
||||
|
||||
return newBlock
|
||||
}
|
||||
|
||||
// appends a row to the buffer.
|
||||
// Row must match the expected type by the underlying proto block.
|
||||
func (buffer *InsertBuffer) AppendRow(signalNames []string, row interface{}, producerChannel chan InsertCommand) error {
|
||||
block := buffer.PeekFreeBlock()
|
||||
loggedWaiting := false
|
||||
|
||||
for block != nil && block.IsFull() {
|
||||
//if the block is full before append, then pop it and put it on the producer channel.
|
||||
buffer.ProduceBlock(producerChannel)
|
||||
block = buffer.PeekFreeBlock()
|
||||
}
|
||||
|
||||
for block == nil {
|
||||
//no more free blocks are available...
|
||||
// if the buffer is able to allocate more blocks, then do so. otherwise, wait for a block...
|
||||
if len(buffer.blockPool) == maxBlockPoolSize {
|
||||
if !loggedWaiting {
|
||||
loggedWaiting = true
|
||||
logger.Warn().Msgf("no available %s blocks for appending, waiting for available block...", buffer.TableName)
|
||||
}
|
||||
//wait
|
||||
time.Sleep(100 * time.Millisecond)
|
||||
block = buffer.PeekFreeBlock()
|
||||
} else {
|
||||
logger.Info().Msgf("%s no available block, allocating new block", buffer.TableName)
|
||||
block = AllocateNewBlock(signalNames, maxBufferBytes, buffer.blockType, buffer.tripInfo)
|
||||
buffer.blockPool = append(buffer.blockPool, block)
|
||||
buffer.AppendFreeBlock(block)
|
||||
}
|
||||
}
|
||||
|
||||
err := block.AppendRow(signalNames, row)
|
||||
|
||||
//if block is full after append, pop it from linked list and put onto producer channel
|
||||
if block.IsFull() || block.TimeSinceThreshold(buffer.insertDelay) {
|
||||
buffer.ProduceBlock(producerChannel)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func (buffer *InsertBuffer) ProduceBlock(producerChannel chan InsertCommand) {
|
||||
if producerChannel != nil {
|
||||
logger.Debug().Msgf("ProduceBlock...")
|
||||
producerChannel <- InsertCommand{
|
||||
Buffer: buffer,
|
||||
Block: buffer.PopFreeBlock(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// searches blockpool for nonEmpty blocks
|
||||
func (buffer *InsertBuffer) getNonEmptyBlock() insertBlock {
|
||||
for _, block := range buffer.blockPool {
|
||||
if block.Len() > 0 {
|
||||
return block
|
||||
}
|
||||
}
|
||||
return buffer.blockPool[0]
|
||||
}
|
||||
|
||||
// returns any nonempty input from buffer
|
||||
func (buffer *InsertBuffer) GetInput() proto.Input {
|
||||
return buffer.getNonEmptyBlock().GetProtoInput(true, nil)
|
||||
}
|
||||
|
||||
// reinitializes the blocks list, assuming any empty block in the pool is free and can be added to list.
|
||||
// returns size of blocks list
|
||||
func (buffer *InsertBuffer) InitFreeBlocksList() int {
|
||||
buffer.freeHead = nil
|
||||
buffer.freeTail = nil
|
||||
count := 0
|
||||
for _, block := range buffer.blockPool {
|
||||
if block.Len() != 0 {
|
||||
continue
|
||||
}
|
||||
buffer.AppendFreeBlock(block)
|
||||
count++
|
||||
}
|
||||
return count
|
||||
}
|
||||
|
||||
// Inserts and flushes only the head block.
|
||||
func InsertAndFlushHead(buffer *InsertBuffer, ctx context.Context, conn *ch.Client, breaker *gobreaker.CircuitBreaker, signalsSet map[string]bool) (int, error) {
|
||||
block := buffer.PopFreeBlock()
|
||||
|
||||
if buffer == nil || block == nil {
|
||||
return 0, errors.Errorf("attempted to insert with nil buffer")
|
||||
}
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
|
||||
rows := block.Len()
|
||||
protoInput := block.GetProtoInput(false, signalsSet)
|
||||
if protoInput == nil {
|
||||
// empty buffer, just return
|
||||
return 0, nil
|
||||
}
|
||||
queryContext, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
|
||||
insertFunc := func() (interface{}, error) {
|
||||
return nil, conn.Do(queryContext, ch.Query{
|
||||
Body: protoInput.Into(buffer.TableName),
|
||||
Input: protoInput,
|
||||
})
|
||||
}
|
||||
_, err := breaker.Execute(insertFunc)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
block.Flush(false)
|
||||
buffer.AppendFreeBlock(block)
|
||||
|
||||
return rows, err
|
||||
}
|
||||
|
||||
// Inserts a block and flushes it, does not readd to free blocks list if error occurred
|
||||
func InsertAndFlush(command InsertCommand, ctx context.Context, conn *ch.Client, breaker *gobreaker.CircuitBreaker, signalsSet map[string]bool) (int, error) {
|
||||
buffer := command.Buffer
|
||||
block := command.Block
|
||||
|
||||
if buffer == nil || block == nil {
|
||||
return 0, errors.Errorf("attempted to insert with nil buffer")
|
||||
}
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
|
||||
rows := block.Len()
|
||||
protoInput := block.GetProtoInput(false, signalsSet)
|
||||
if protoInput == nil {
|
||||
// empty buffer, just return
|
||||
return 0, nil
|
||||
}
|
||||
queryContext, cancel := context.WithTimeout(ctx, 5*time.Minute)
|
||||
defer cancel()
|
||||
|
||||
logger.Debug().Msgf("Inserting %d rows to %s...", rows, conn.ServerInfo().DisplayName)
|
||||
|
||||
insertFunc := func() (interface{}, error) {
|
||||
return nil, conn.Do(queryContext, ch.Query{
|
||||
Body: protoInput.Into(buffer.TableName),
|
||||
Input: protoInput,
|
||||
})
|
||||
}
|
||||
_, err := breaker.Execute(insertFunc)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
block.Flush(false)
|
||||
buffer.AppendFreeBlock(block)
|
||||
|
||||
return rows, err
|
||||
}
|
||||
|
||||
func (buffer *InsertBuffer) Len() int {
|
||||
length := 0
|
||||
for _, block := range buffer.blockPool {
|
||||
length += block.Len()
|
||||
}
|
||||
return length
|
||||
}
|
||||
|
||||
func (buffer *InsertBuffer) Cap() int {
|
||||
length := 0
|
||||
for _, block := range buffer.blockPool {
|
||||
length += block.Cap()
|
||||
}
|
||||
return length
|
||||
}
|
||||
|
||||
func (buffer *InsertBuffer) Resize(signals []string) {
|
||||
for _, block := range buffer.blockPool {
|
||||
block.Resize(signals, maxBufferBytes)
|
||||
}
|
||||
}
|
||||
|
||||
type insertBlock interface {
|
||||
AppendRow(signalNames []string, row interface{}) error
|
||||
GetProtoInput(useLock bool, signalsSet map[string]bool) proto.Input
|
||||
Flush(useLock bool)
|
||||
IsFull() bool
|
||||
GetInsertBlockType() InsertBlockType
|
||||
Resize(signals []string, maxBytes int)
|
||||
Lock()
|
||||
Unlock()
|
||||
Len() int
|
||||
Cap() int
|
||||
IsBusy() bool
|
||||
TimeSinceThreshold(time.Duration) bool
|
||||
}
|
||||
|
||||
type protoPivotBlock struct {
|
||||
lock sync.Mutex
|
||||
busy bool
|
||||
|
||||
length int
|
||||
capacity int
|
||||
|
||||
maxBytes int
|
||||
|
||||
vin proto.ColStr
|
||||
timestamp proto.ColDateTime64
|
||||
tripStart proto.ColDateTime64
|
||||
tripID proto.ColStr
|
||||
data []proto.ColFloat64
|
||||
|
||||
startTime *time.Time
|
||||
|
||||
columnNames []string
|
||||
|
||||
appendTripInfo bool
|
||||
}
|
||||
|
||||
// block type with issue
|
||||
func NewProtoPivotBlock(signalNames []string, maxBufferBytes int, tripInfo bool) *protoPivotBlock {
|
||||
newBlock := protoPivotBlock{
|
||||
columnNames: signalNames,
|
||||
appendTripInfo: tripInfo,
|
||||
}
|
||||
logger.Debug().Msgf("NEW PIVOT BLOCK: %d %d", len(signalNames), maxBufferBytes)
|
||||
newBlock.Resize(signalNames, maxBufferBytes)
|
||||
return &newBlock
|
||||
}
|
||||
|
||||
func (block *protoPivotBlock) Lock() {
|
||||
block.lock.Lock()
|
||||
block.busy = true
|
||||
}
|
||||
|
||||
func (block *protoPivotBlock) Unlock() {
|
||||
block.lock.Unlock()
|
||||
block.busy = false
|
||||
}
|
||||
|
||||
func (block *protoPivotBlock) IsBusy() bool {
|
||||
return block.busy
|
||||
}
|
||||
|
||||
func (block *protoPivotBlock) Len() int {
|
||||
return block.length
|
||||
}
|
||||
|
||||
func (block *protoPivotBlock) Cap() int {
|
||||
return block.capacity
|
||||
}
|
||||
|
||||
func (block *protoPivotBlock) TimeSinceThreshold(threshold time.Duration) bool {
|
||||
return block.startTime == nil || time.Since(*block.startTime) > threshold
|
||||
}
|
||||
|
||||
// Resizes this pivotBlock to the desired number of signal columns, and scales rows to not exceed maxBytes
|
||||
func (block *protoPivotBlock) Resize(signals []string, maxBytes int) {
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
|
||||
if len(signals) == len(block.columnNames) && maxBytes == block.maxBytes {
|
||||
block.Flush(false)
|
||||
return
|
||||
}
|
||||
|
||||
oldCapacity := block.capacity
|
||||
oldWidth := len(block.columnNames)
|
||||
|
||||
block.columnNames = signals
|
||||
block.maxBytes = maxBytes
|
||||
|
||||
block.capacity = block.estimateMaxRows(len(signals), maxBytes)
|
||||
block.length = 0
|
||||
|
||||
logger.Debug().Msgf("resizing block to %d rows, %d columns", block.capacity, len(signals))
|
||||
|
||||
//reallocate memory only if block dimensions have changed
|
||||
if oldCapacity != block.capacity || oldWidth != len(signals) {
|
||||
block.vin.Buf = make([]byte, 0, utils.MaxVinLength*block.capacity)
|
||||
block.vin.Pos = make([]proto.Position, 0, block.capacity)
|
||||
|
||||
block.timestamp.Data = make([]proto.DateTime64, 0, block.capacity)
|
||||
block.tripStart.Data = make([]proto.DateTime64, 0, block.capacity)
|
||||
block.timestamp.WithPrecision(proto.PrecisionNano)
|
||||
block.tripStart.WithPrecision(proto.PrecisionNano)
|
||||
|
||||
block.tripID.Buf = make([]byte, 0, (utils.MaxVinLength+utils.MaxTimestampLength+1)*block.capacity)
|
||||
block.tripID.Pos = make([]proto.Position, 0, block.capacity)
|
||||
|
||||
block.data = make([]proto.ColFloat64, len(signals))
|
||||
for i := range block.data {
|
||||
block.data[i] = make([]float64, 0, block.capacity)
|
||||
}
|
||||
}
|
||||
|
||||
block.Flush(false)
|
||||
}
|
||||
|
||||
func (block *protoPivotBlock) GetInsertBlockType() InsertBlockType {
|
||||
return PivotBlockType
|
||||
}
|
||||
|
||||
func (block *protoPivotBlock) IsFull() bool {
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
return block.length >= block.capacity
|
||||
}
|
||||
|
||||
// Given upper bound for bytes, estimate the maximum number of rows to allocate
|
||||
func (block *protoPivotBlock) estimateMaxRows(numDataColumns int, maxBytes int) int {
|
||||
rowBytes := 20 + 2 + 8 + 8*numDataColumns //vin, timestamp, data columns per row
|
||||
if block.appendTripInfo {
|
||||
rowBytes += 8 + 44 + 2 //tripstart and tripid
|
||||
}
|
||||
return maxBytes / rowBytes
|
||||
}
|
||||
|
||||
// Appends a VehicleState as a row to this Block.
|
||||
func (block *protoPivotBlock) AppendRow(signalNames []string, row interface{}) error {
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
|
||||
if block.length >= block.capacity {
|
||||
return errors.WithStack(utils.ErrInsertFullBlock)
|
||||
}
|
||||
if len(signalNames) != len(block.data) {
|
||||
return errors.WithStack(utils.ErrInsertWrongColumns)
|
||||
}
|
||||
state, valid := row.(*VehicleState)
|
||||
if !valid {
|
||||
return errors.WithStack(utils.ErrInvalidAppendType)
|
||||
}
|
||||
block.length++
|
||||
|
||||
if block.startTime == nil {
|
||||
time := time.Now()
|
||||
block.startTime = &time
|
||||
}
|
||||
|
||||
block.vin.Append(state.VIN)
|
||||
block.timestamp.Append(state.Timestamp)
|
||||
if block.appendTripInfo {
|
||||
block.tripStart.Append(state.TripStart)
|
||||
block.tripID.Append(state.TripID)
|
||||
}
|
||||
|
||||
// 3 hour leeway
|
||||
// Saw a lot of signals with a delay around 3 hours, going to now only check for signals over a day old
|
||||
catchTime := state.Timestamp.Add(-time.Hour * 24)
|
||||
for i, signal := range signalNames {
|
||||
value, valid := state.StateValues[signal]
|
||||
if !valid {
|
||||
value = math.NaN()
|
||||
} else {
|
||||
// If the signal is not included, I'm not going to check its timing.
|
||||
// should no linger see the !ok case
|
||||
// Block that should have my issue
|
||||
signalTime, ok := state.StateTimes[signal]
|
||||
if !ok {
|
||||
// So a lot of signals do not have a timestamp on them, not sure why
|
||||
logSampler.Warn().Str("Location", "protoPivotBlock").Str("VIN", state.VIN).Str("Signal", signal).
|
||||
Str("Location", "protoPivotBlock").Float64("Value", value).Int("Sample Rate", NO_TIMESTAMP_LOG_SAMPLE_RATE).Msg("AppendRow No Timestamp")
|
||||
} else {
|
||||
// If the signal is from 3 hours before the suggested time of the signal
|
||||
if signalTime.Before(catchTime) {
|
||||
logger.Warn().Str("VIN", state.VIN).
|
||||
Str("Signal", signal).
|
||||
Float64("Value", value).
|
||||
Time("timestamp.state", state.Timestamp).
|
||||
Time("timestamp.signal", signalTime).
|
||||
Str("Location", "protoPivotBlock").
|
||||
Msg("AppendRow Timestamp Old")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
block.data[i].Append(value)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Clears and Resets the buffers in the Block.
|
||||
func (block *protoPivotBlock) Flush(useLock bool) {
|
||||
if useLock {
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
}
|
||||
block.length = 0
|
||||
block.startTime = nil
|
||||
|
||||
block.vin.Reset()
|
||||
block.timestamp.Reset()
|
||||
block.tripStart.Reset()
|
||||
block.tripID.Reset()
|
||||
|
||||
for i := range block.data {
|
||||
block.data[i].Reset()
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
// Gets protocol input struct for ch-go batch insertion
|
||||
func (block *protoPivotBlock) GetProtoInput(useLock bool, signalsSet map[string]bool) proto.Input {
|
||||
if useLock {
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
}
|
||||
|
||||
if block.length == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
input := proto.Input{
|
||||
{Name: "VIN", Data: block.vin},
|
||||
{Name: "Timestamp", Data: block.timestamp},
|
||||
}
|
||||
|
||||
if block.appendTripInfo {
|
||||
input = append(input, proto.InputColumn{Name: "TripStart", Data: block.tripStart})
|
||||
input = append(input, proto.InputColumn{Name: "TripID", Data: block.tripID})
|
||||
}
|
||||
|
||||
for i := range block.data {
|
||||
columnName := block.columnNames[i]
|
||||
ok := true
|
||||
if len(signalsSet) > 0 {
|
||||
_, ok = signalsSet[columnName]
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
input = append(input, proto.InputColumn{Name: columnName, Data: block.data[i]})
|
||||
}
|
||||
return input
|
||||
}
|
||||
|
||||
// Block struct for feature_table_last type of schema
|
||||
// This block aggregates only 1 row per VIN. vinIndexMap maps the VIN to a row index.
|
||||
type protoVinLastBlock struct {
|
||||
protoPivotBlock
|
||||
vinIndexMap map[string]int
|
||||
}
|
||||
|
||||
func NewProtoVinLastBlock(signalNames []string, numVINs int, tripInfo bool) *protoVinLastBlock {
|
||||
newBlock := protoVinLastBlock{}
|
||||
newBlock.columnNames = append(newBlock.columnNames, signalNames...)
|
||||
newBlock.vinIndexMap = make(map[string]int)
|
||||
newBlock.appendTripInfo = tripInfo
|
||||
|
||||
logger.Debug().Msgf("NEW VIN LAST BLOCK: %d %d", len(signalNames), maxBufferBytes)
|
||||
|
||||
bytesPerVIN := 20 + 16 + 16 + 20 //VIN (str), Timestamp, TripStart (timestamp), TripID (str)
|
||||
bytesPerVIN += 8 * len(signalNames)
|
||||
|
||||
newBlock.Resize(signalNames, bytesPerVIN*numVINs)
|
||||
return &newBlock
|
||||
}
|
||||
|
||||
func (block *protoVinLastBlock) Flush(useLock bool) {
|
||||
if useLock {
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
}
|
||||
block.protoPivotBlock.Flush(false)
|
||||
|
||||
block.vinIndexMap = make(map[string]int)
|
||||
}
|
||||
|
||||
// Appends a VehicleState as a row to this Block.
|
||||
// Investigate this code vs protoPivotBlock. WHy the differences, why two of the same thing?
|
||||
func (block *protoVinLastBlock) AppendRow(signalNames []string, row interface{}) error {
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
|
||||
if block.length >= block.capacity {
|
||||
return errors.WithStack(utils.ErrInsertFullBlock)
|
||||
}
|
||||
if len(signalNames) != len(block.data) {
|
||||
return errors.WithStack(utils.ErrInsertWrongColumns)
|
||||
}
|
||||
state, valid := row.(*VehicleState)
|
||||
if !valid {
|
||||
return errors.WithStack(utils.ErrInvalidAppendType)
|
||||
}
|
||||
|
||||
vinIndex, ok := block.vinIndexMap[state.VIN]
|
||||
|
||||
if block.startTime == nil {
|
||||
time := time.Now()
|
||||
block.startTime = &time
|
||||
}
|
||||
|
||||
if !ok {
|
||||
// append state to the end of the buffer
|
||||
block.length++
|
||||
|
||||
block.vin.Append(state.VIN)
|
||||
block.timestamp.Append(state.Timestamp)
|
||||
if block.appendTripInfo {
|
||||
block.tripStart.Append(state.TripStart)
|
||||
block.tripID.Append(state.TripID)
|
||||
}
|
||||
|
||||
// catchTime := state.Timestamp.Add(-time.Hour * 3)
|
||||
for i, signal := range signalNames {
|
||||
value, valid := state.StateValues[signal]
|
||||
if !valid {
|
||||
value = math.NaN()
|
||||
} else {
|
||||
// If the signal is not included, I'm not going to check its timing.
|
||||
// should no linger see the !ok case
|
||||
// signalTime, ok := state.StateTimes[signal]
|
||||
// if !ok {
|
||||
// logger.Warn().Str("Location", "protoVinLastBlock, !ok").Msgf("AppendRow no timestamp for %s", signal)
|
||||
// } else {
|
||||
// // If the signal is from 3 hours before the suggested time of the signal
|
||||
// if signalTime.Before(catchTime) {
|
||||
// logger.Warn().Str("VIN", state.VIN).
|
||||
// Str("Signal", signal).
|
||||
// Time("timestamp.state", state.Timestamp).
|
||||
// Time("timestamp.signal", signalTime).
|
||||
// Str("Location", "protoVinLastBlock, !ok").
|
||||
// Msg("AppendRow Timestamp Old")
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
block.data[i].Append(value)
|
||||
}
|
||||
|
||||
block.vinIndexMap[state.VIN] = block.length - 1
|
||||
} else {
|
||||
// override only the row mapped to the vin
|
||||
block.timestamp.Data[vinIndex] = proto.ToDateTime64(state.Timestamp, block.timestamp.Precision)
|
||||
block.tripStart.Data[vinIndex] = proto.ToDateTime64(state.TripStart, block.timestamp.Precision)
|
||||
SetColStr(&block.tripID, state.TripID, vinIndex)
|
||||
|
||||
//catchTime := state.Timestamp.Add(-time.Hour * 3)
|
||||
for i, signal := range signalNames {
|
||||
value, valid := state.StateValues[signal]
|
||||
if !valid {
|
||||
value = math.NaN()
|
||||
} else {
|
||||
// If the signal is not included, I'm not going to check its timing.
|
||||
// should no linger see the !ok case
|
||||
// signalTime, ok := state.StateTimes[signal]
|
||||
// if !ok {
|
||||
// logger.Warn().Str("Location", "protoVinLastBlock, ok").Msgf("AppendRow no timestamp for %s", signal)
|
||||
// } else {
|
||||
// // If the signal is from 3 hours before the suggested time of the signal
|
||||
// if signalTime.Before(catchTime) {
|
||||
// logger.Warn().Str("VIN", state.VIN).
|
||||
// Str("Signal", signal).
|
||||
// Time("timestamp.state", state.Timestamp).
|
||||
// Time("timestamp.signal", signalTime).
|
||||
// Str("Location", "protoVinLastBlock, ok").
|
||||
// Msg("AppendRow Timestamp Old")
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
block.data[i][vinIndex] = value
|
||||
}
|
||||
}
|
||||
return nil
|
||||
|
||||
}
|
||||
|
||||
// Block struct for vehicle_signal type of schema
|
||||
type protoSignalBlock struct {
|
||||
lock sync.Mutex
|
||||
busy bool
|
||||
|
||||
length int
|
||||
capacity int
|
||||
|
||||
maxBytes int
|
||||
|
||||
startTime *time.Time
|
||||
|
||||
vin proto.ColStr
|
||||
timestamp proto.ColDateTime64
|
||||
id proto.ColInt16
|
||||
name proto.ColStr
|
||||
value proto.ColFloat64
|
||||
}
|
||||
|
||||
func NewProtoSignalBlock(signalNames []string, maxBufferBytes int) *protoSignalBlock {
|
||||
newBlock := protoSignalBlock{}
|
||||
newBlock.Resize(signalNames, maxBufferBytes)
|
||||
return &newBlock
|
||||
}
|
||||
|
||||
func (block *protoSignalBlock) Lock() {
|
||||
block.lock.Lock()
|
||||
block.busy = true
|
||||
}
|
||||
|
||||
func (block *protoSignalBlock) Unlock() {
|
||||
block.lock.Unlock()
|
||||
block.busy = false
|
||||
}
|
||||
|
||||
func (block *protoSignalBlock) IsBusy() bool {
|
||||
return block.busy
|
||||
}
|
||||
|
||||
func (block *protoSignalBlock) Len() int {
|
||||
return block.length
|
||||
}
|
||||
|
||||
func (block *protoSignalBlock) Cap() int {
|
||||
return block.capacity
|
||||
}
|
||||
|
||||
func (block *protoSignalBlock) TimeSinceThreshold(threshold time.Duration) bool {
|
||||
return block.startTime == nil || time.Since(*block.startTime) > threshold
|
||||
}
|
||||
|
||||
// Resizes allocated memory for the Block, given maxBytes as the upper bound for memory size
|
||||
func (block *protoSignalBlock) Resize(signals []string, maxBytes int) {
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
|
||||
if maxBytes == block.maxBytes {
|
||||
block.Flush(false)
|
||||
return
|
||||
}
|
||||
|
||||
oldCapacity := block.capacity
|
||||
|
||||
block.maxBytes = maxBytes
|
||||
block.capacity = block.estimateMaxRows(maxBytes)
|
||||
|
||||
logger.Debug().Msgf("resizing block to %d rows", block.capacity)
|
||||
|
||||
//reallocate memory only if block dimensions have changed
|
||||
if oldCapacity != block.capacity {
|
||||
block.vin.Buf = make([]byte, 0, utils.MaxVinLength*block.capacity)
|
||||
block.vin.Pos = make([]proto.Position, 0, block.capacity)
|
||||
|
||||
block.timestamp.Data = make([]proto.DateTime64, 0, block.capacity)
|
||||
block.timestamp.WithPrecision(proto.PrecisionMicro)
|
||||
|
||||
block.id = make([]int16, 0, block.capacity)
|
||||
|
||||
block.name.Buf = make([]byte, 0, 20*block.capacity)
|
||||
block.name.Pos = make([]proto.Position, 0, block.capacity)
|
||||
|
||||
block.value = make([]float64, 0, block.capacity)
|
||||
}
|
||||
|
||||
block.Flush(false)
|
||||
}
|
||||
|
||||
func (block *protoSignalBlock) GetInsertBlockType() InsertBlockType {
|
||||
return SignalBlockType
|
||||
}
|
||||
|
||||
func (block *protoSignalBlock) IsFull() bool {
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
return block.length >= block.capacity
|
||||
}
|
||||
|
||||
// Given upper bound for bytes, estimate the maximum number of rows to allocate
|
||||
func (block *protoSignalBlock) estimateMaxRows(maxBytes int) int {
|
||||
const rowBytes int = 17 + 8 + 2 + 30 + 8 + 2 + 2 //Each row is ~65 bytes.
|
||||
return maxBytes / rowBytes
|
||||
}
|
||||
|
||||
// Clears and Resets the buffers in the Block.
|
||||
func (block *protoSignalBlock) Flush(useLock bool) {
|
||||
if useLock {
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
}
|
||||
block.vin.Reset()
|
||||
block.timestamp.Reset()
|
||||
block.id.Reset()
|
||||
block.name.Reset()
|
||||
block.value.Reset()
|
||||
|
||||
block.startTime = nil
|
||||
block.length = 0
|
||||
}
|
||||
|
||||
// Appends a kafka_grpc.GRPC_CANSignal to this block as a row
|
||||
func (block *protoSignalBlock) AppendRow(signalNames []string, row interface{}) error {
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
|
||||
if block.length >= block.capacity {
|
||||
return errors.WithStack(utils.ErrInsertFullBlock)
|
||||
}
|
||||
signal, valid := row.(*kafka_grpc.GRPC_CANSignal)
|
||||
if !valid {
|
||||
return errors.WithStack(utils.ErrInvalidAppendType)
|
||||
|
||||
}
|
||||
|
||||
if block.startTime == nil {
|
||||
time := time.Now()
|
||||
block.startTime = &time
|
||||
}
|
||||
|
||||
timestamp := utils.FloatToTime(signal.Timestamp)
|
||||
|
||||
block.vin.Append(signal.Vin)
|
||||
block.timestamp.Append(timestamp)
|
||||
block.id.Append(int16(signal.Id))
|
||||
block.name.Append(signal.Name)
|
||||
block.value.Append(signal.Value)
|
||||
|
||||
block.length++
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Gets protocol input struct for ch-go batch insertion
|
||||
func (block *protoSignalBlock) GetProtoInput(useLock bool, signalsSet map[string]bool) proto.Input {
|
||||
if useLock {
|
||||
block.Lock()
|
||||
defer block.Unlock()
|
||||
}
|
||||
|
||||
if block.length == 0 {
|
||||
return nil
|
||||
}
|
||||
input := proto.Input{
|
||||
{Name: "VIN", Data: block.vin},
|
||||
{Name: "Timestamp", Data: block.timestamp},
|
||||
{Name: "Name", Data: block.name},
|
||||
{Name: "Value", Data: block.value},
|
||||
{Name: "ID", Data: block.id},
|
||||
}
|
||||
return input
|
||||
}
|
||||
|
||||
/// helper methods ///
|
||||
|
||||
// Sets the value of string in-place in a proto colstr.
|
||||
// This will reallocate memory as needed.
|
||||
func SetColStr(col *proto.ColStr, value string, index int) {
|
||||
pos := col.Pos[index]
|
||||
oldLen := pos.End - pos.Start
|
||||
newLen := len(value)
|
||||
offset := newLen - oldLen
|
||||
|
||||
newEnd := pos.Start + len(value)
|
||||
|
||||
oldBufLen := len(col.Buf)
|
||||
|
||||
//need to resize buffers
|
||||
if offset > 0 {
|
||||
// grow buffer by appending zero bytes, then truncating the length (not the cap) of slice
|
||||
col.Buf = append(col.Buf, make([]byte, offset)...)[:oldBufLen]
|
||||
}
|
||||
|
||||
// need to shift EVERYTHING after index. This can be slow.
|
||||
if offset != 0 {
|
||||
// copy buffer into itself with offset.
|
||||
copy(col.Buf[newEnd:len(col.Buf)], col.Buf[pos.End:oldBufLen])
|
||||
|
||||
for i := index + 1; i < len(col.Pos); i++ {
|
||||
col.Pos[i].Start += offset
|
||||
col.Pos[i].End += offset
|
||||
}
|
||||
}
|
||||
|
||||
//insert
|
||||
copy(col.Buf[pos.Start:newEnd], value)
|
||||
col.Pos[index].End = newEnd
|
||||
|
||||
}
|
||||
237
services/jetfire/models/vehicle.go
Normal file
237
services/jetfire/models/vehicle.go
Normal file
@@ -0,0 +1,237 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/utils"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/grpc/kafka_grpc"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/fiskerinc/cloud-services/pkg/utils/envtool"
|
||||
)
|
||||
|
||||
var (
|
||||
maxVinCount = envtool.GetEnvInt("JETFIRE_MAX_VINS", 10000)
|
||||
timestampThreshold = 500 * time.Millisecond
|
||||
)
|
||||
|
||||
// Vehicle State stores latest state for a particular VIN.
|
||||
type VehicleState struct {
|
||||
VIN string
|
||||
Timestamp time.Time //data timestamp of last data sample
|
||||
TripStart time.Time //data timestamp of start of trip
|
||||
TripID string //trip id
|
||||
StateValues map[string]float64 //map of values in state. Keys correspond to CAN signal name
|
||||
StateTimes map[string]time.Time //map of timestamps in state. Keys correspond to CAN signal name
|
||||
|
||||
InsertTime time.Time // local timestamp of last received data. Used only for removing old vehicle states from cache
|
||||
|
||||
pollingMap map[uint]time.Time //map of last polling times. Key matches update flag
|
||||
|
||||
// for linked list
|
||||
Next *VehicleState
|
||||
prev *VehicleState
|
||||
}
|
||||
|
||||
func (v *VehicleState) TimeSincePolled(updateIndex uint) time.Duration {
|
||||
pollTime, ok := v.pollingMap[updateIndex]
|
||||
if !ok {
|
||||
pollTime = time.Unix(0, 0)
|
||||
}
|
||||
|
||||
return v.Timestamp.Sub(pollTime)
|
||||
}
|
||||
|
||||
func (v *VehicleState) SetPollTime(updateIndex uint) {
|
||||
v.pollingMap[updateIndex] = v.Timestamp
|
||||
}
|
||||
|
||||
func (v *VehicleState) Clear(newVIN string) {
|
||||
v.VIN = newVIN
|
||||
cacheSize := len(v.StateValues)
|
||||
|
||||
v.StateValues = make(map[string]float64, cacheSize)
|
||||
v.StateTimes = make(map[string]time.Time, cacheSize)
|
||||
v.Timestamp = time.Unix(0, 0)
|
||||
v.TripStart = v.Timestamp
|
||||
v.TripID = ""
|
||||
v.InsertTime = time.Now().UTC()
|
||||
v.pollingMap = make(map[uint]time.Time)
|
||||
}
|
||||
|
||||
// VehicleCache is a table of vehicle states.
|
||||
type VehicleCache struct {
|
||||
Cache map[string]*VehicleState
|
||||
SignalsSet *map[string]bool
|
||||
LastTimestamp time.Time
|
||||
|
||||
// linked list LIFO of States in order of update.
|
||||
StatesListHead *VehicleState //oldest update
|
||||
StatesListTail *VehicleState //newest update
|
||||
}
|
||||
|
||||
func (cache *VehicleCache) Clear() {
|
||||
clear(cache.Cache)
|
||||
|
||||
// disconnect linked list
|
||||
node := cache.StatesListHead
|
||||
for node != nil {
|
||||
next := node.Next
|
||||
node.Next = nil
|
||||
node.prev = nil
|
||||
|
||||
node = next
|
||||
}
|
||||
cache.StatesListHead = nil
|
||||
cache.StatesListTail = nil
|
||||
}
|
||||
|
||||
// removes the node from linked list and appends it to the right end
|
||||
func (cache *VehicleCache) ReinsertRight(node *VehicleState) {
|
||||
if cache.StatesListTail == node {
|
||||
// node is already the tail. don't do anything
|
||||
return
|
||||
}
|
||||
|
||||
// move head ptr if we are moving the first node
|
||||
if cache.StatesListHead == node {
|
||||
cache.StatesListHead = node.Next
|
||||
}
|
||||
|
||||
// remove from list
|
||||
if node.prev != nil && node.Next != nil {
|
||||
p := node.prev
|
||||
n := node.Next
|
||||
n.prev = p
|
||||
p.Next = n
|
||||
} else if node.prev != nil {
|
||||
node.prev.Next = nil
|
||||
} else if node.Next != nil {
|
||||
node.Next.prev = nil
|
||||
}
|
||||
|
||||
node.prev = nil
|
||||
node.Next = nil
|
||||
|
||||
// append to tail
|
||||
if cache.StatesListTail != nil {
|
||||
cache.StatesListTail.Next = node
|
||||
}
|
||||
|
||||
node.prev = cache.StatesListTail
|
||||
cache.StatesListTail = node
|
||||
|
||||
if cache.StatesListHead == nil {
|
||||
cache.StatesListHead = node
|
||||
}
|
||||
}
|
||||
|
||||
// removes left node from the linked list AND from the cache map
|
||||
func (cache *VehicleCache) PopLeft() *VehicleState {
|
||||
if cache.StatesListHead == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
node := cache.StatesListHead
|
||||
cache.StatesListHead = node.Next
|
||||
|
||||
delete(cache.Cache, node.VIN)
|
||||
|
||||
return node
|
||||
}
|
||||
|
||||
// Insert CANSignal into vehicle cache. Allocates new vehicle state as necessary.
|
||||
func (cache *VehicleCache) UpdateSignal(signal *kafka_grpc.GRPC_CANSignal, updateFlag uint) error {
|
||||
if len(*cache.SignalsSet) > 0 {
|
||||
// check if signal is in signals set, skip if not in signals set
|
||||
_, contains := (*cache.SignalsSet)[signal.Name]
|
||||
if !contains {
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
// if VIN is not in cache, allocate new vehicle state for VIN and add to cache
|
||||
_, contains := cache.Cache[signal.Vin]
|
||||
if !contains {
|
||||
if len(cache.Cache) > maxVinCount {
|
||||
oldState := cache.PopLeft()
|
||||
logger.Debug().Msgf("repurposing state %s -> %s", oldState.VIN, signal.Vin)
|
||||
oldState.Clear(signal.Vin)
|
||||
cache.Cache[signal.Vin] = oldState
|
||||
} else {
|
||||
cache.Cache[signal.Vin] = NewVehicleState(signal.Vin, len(*cache.SignalsSet))
|
||||
logger.Debug().Msgf("new vehicle state %s", signal.Vin)
|
||||
}
|
||||
}
|
||||
|
||||
//update value
|
||||
cache.Cache[signal.Vin].UpdateSignal(signal, updateFlag)
|
||||
cache.ReinsertRight(cache.Cache[signal.Vin]) //move state to end of orderly linkedlist
|
||||
cache.LastTimestamp = time.Now().UTC()
|
||||
return nil
|
||||
}
|
||||
|
||||
// constructs a new VehicleState
|
||||
func NewVehicleState(VIN string, cacheSize int) *VehicleState {
|
||||
newState := new(VehicleState)
|
||||
newState.VIN = VIN
|
||||
newState.StateValues = make(map[string]float64, cacheSize)
|
||||
newState.StateTimes = make(map[string]time.Time, cacheSize)
|
||||
newState.Timestamp = time.Unix(0, 0)
|
||||
newState.TripStart = newState.Timestamp
|
||||
newState.TripID = ""
|
||||
newState.InsertTime = time.Now().UTC()
|
||||
newState.pollingMap = make(map[uint]time.Time)
|
||||
return newState
|
||||
}
|
||||
|
||||
// constructs a new VehicleState
|
||||
func NewVehicleStateDefault(VIN string) *VehicleState {
|
||||
return NewVehicleState(VIN, 10)
|
||||
}
|
||||
|
||||
// UpdateSignal() updates the vehicle state cache
|
||||
func (state *VehicleState) UpdateSignal(signal *kafka_grpc.GRPC_CANSignal, updateFlag uint) {
|
||||
// Mark start of new trip if too much time has elapsed between updates
|
||||
signalTime := utils.FloatToTime(signal.Timestamp)
|
||||
|
||||
ignitionTriggered := false
|
||||
// check for vehicle ignition rising edge as a trigger for a new trip.
|
||||
if signal.Name == "BCM_PwrMod" && signal.Value >= 2 && signal.Value <= 4 && !signalTime.Before(state.Timestamp) {
|
||||
pwrMod, ok := state.StateValues["BCM_PwrMod"]
|
||||
ignitionTriggered = !ok || (ok && pwrMod < 2)
|
||||
}
|
||||
|
||||
if signalTime.Sub(state.Timestamp) >= utils.TripTimeout || ignitionTriggered {
|
||||
logger.Debug().Msgf("%s New TripStart: %d, old %d, delta %d", state.VIN, signalTime.Unix(), state.Timestamp.Unix(), signalTime.Sub(state.Timestamp))
|
||||
state.TripStart = signalTime
|
||||
state.TripID = fmt.Sprintf("%s_%d", state.VIN, state.TripStart.Unix())
|
||||
}
|
||||
|
||||
// Update the vehicle timestamp
|
||||
oldTime, ok := state.StateTimes[signal.Name]
|
||||
if !ok {
|
||||
oldTime = state.Timestamp
|
||||
}
|
||||
|
||||
if !signalTime.Add(timestampThreshold).Before(oldTime) {
|
||||
if !signalTime.Before(state.Timestamp) {
|
||||
state.Timestamp = signalTime
|
||||
}
|
||||
// Insert new signal value
|
||||
state.StateValues[signal.Name] = signal.Value
|
||||
state.StateTimes[signal.Name] = signalTime
|
||||
state.InsertTime = time.Now()
|
||||
} else {
|
||||
// if receiving message out of timestamp order, only accept new signal value if
|
||||
// cached value is empty for signal name
|
||||
_, hasSignal := state.StateValues[signal.Name]
|
||||
if !hasSignal {
|
||||
state.StateValues[signal.Name] = signal.Value
|
||||
state.StateTimes[signal.Name] = signalTime
|
||||
state.InsertTime = time.Now()
|
||||
}
|
||||
utils.LogOutOfOrderMsg(signal.Name, signal.Vin)
|
||||
}
|
||||
}
|
||||
164
services/jetfire/server/server_consumer.go
Normal file
164
services/jetfire/server/server_consumer.go
Normal file
@@ -0,0 +1,164 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/handlers"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/models"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/services"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/grpc/kafka_grpc"
|
||||
"github.com/fiskerinc/cloud-services/pkg/kafka"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/fiskerinc/cloud-services/pkg/loggerdataresp"
|
||||
|
||||
"github.com/intel-go/fastjson"
|
||||
"github.com/pkg/errors"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
// StartConsumer runs consumer and puts vehicle signals into a channel for router
|
||||
func StartConsumer(ctx context.Context, topic string) {
|
||||
logger.Debug().Str("StartConsumer", "").Send()
|
||||
defer func() {
|
||||
if err := recover(); err != nil {
|
||||
logger.Error().Msgf("PanicConsumer %v", err)
|
||||
}
|
||||
}()
|
||||
|
||||
producerChannel := make(chan models.InsertCommand, 100)
|
||||
signalsChannel := make(chan *kafka.Message)
|
||||
rebalanceChannel := make(chan struct{})
|
||||
|
||||
consumer, err := services.GetKafkaConsumer()
|
||||
if err != nil {
|
||||
logger.Fatal().Err(err)
|
||||
panic(err)
|
||||
}
|
||||
|
||||
logger.Debug().Msgf("Starting routeEvents and InserterLoop...")
|
||||
|
||||
go routeEvents(signalsChannel, rebalanceChannel, producerChannel)
|
||||
|
||||
go InserterLoop(producerChannel, ctx)
|
||||
|
||||
for {
|
||||
logger.Debug().Msgf("Calling ConsumeOrRebalancedCatch...")
|
||||
consumer.Subscribe([]string{topic})
|
||||
err = consumer.ConsumeOrRebalancedCatch([]string{topic}, signalsChannel, rebalanceChannel)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
}
|
||||
if !loggerdataresp.BadDataError(err, loggerdataresp.EofErrorCheck) {
|
||||
check := consumer.Check(ctx)
|
||||
if check != nil {
|
||||
logger.Error().Err(check).Send()
|
||||
}
|
||||
}
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
// reset the kafka consumer and gc the old one
|
||||
consumer = nil
|
||||
for consumer == nil || err != nil {
|
||||
consumer, err = services.ResetKafkaConsumer()
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// processes signals in batch, handles caching of vehicle state and appending to insertion batch caches
|
||||
func routeEvents(signalsChannel chan *kafka.Message, rebalanceChannel <-chan struct{}, producerChannel chan models.InsertCommand) {
|
||||
var jsonErr error
|
||||
var protoErr error
|
||||
var batchData []*kafka_grpc.GRPC_CANSignal
|
||||
|
||||
initialized := false
|
||||
|
||||
//Consumer loop
|
||||
rebalanceFlag := true
|
||||
for {
|
||||
select {
|
||||
case signal := <-signalsChannel:
|
||||
if signal == nil {
|
||||
continue
|
||||
}
|
||||
rebalanceFlag = true
|
||||
|
||||
if !initialized {
|
||||
// init cache after rebalancing
|
||||
logger.Debug().Msgf("INITIALIZING CACHE...")
|
||||
time.Sleep(time.Second)
|
||||
services.InitCacheFromClickhouse()
|
||||
|
||||
initialized = true
|
||||
logger.Debug().Msgf("INITIALIZED...")
|
||||
}
|
||||
|
||||
batchData, protoErr = unmarshalProtobufSignal(signal)
|
||||
if protoErr != nil {
|
||||
batchData, jsonErr = unmarshalJSONSignal(signal)
|
||||
if jsonErr != nil {
|
||||
logger.Error().Err(protoErr).Msg("Failed to unmarshal signal as either Protobuf or JSON")
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
_, err := handlers.HandleSignalBatch(batchData, services.GetVehicleCache(), producerChannel)
|
||||
if err != nil {
|
||||
logger.Error().Err(errors.WithStack(err)).Send()
|
||||
}
|
||||
|
||||
case <-rebalanceChannel:
|
||||
if rebalanceFlag {
|
||||
rebalanceFlag = false
|
||||
|
||||
logger.Info().Msgf("kafka rebalancing...")
|
||||
initialized = false
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
func unmarshalProtobufSignal(event *kafka.Message) ([]*kafka_grpc.GRPC_CANSignal, error) {
|
||||
if event == nil {
|
||||
err := errors.Errorf("trying to unmarshall null event ptr")
|
||||
return nil, err
|
||||
}
|
||||
batchData := kafka_grpc.GRPC_CANSignalBatchPayload{}
|
||||
err := proto.Unmarshal(event.Value, &batchData)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return batchData.Data.Cansignals, nil
|
||||
}
|
||||
|
||||
func unmarshalJSONSignal(event *kafka.Message) ([]*kafka_grpc.GRPC_CANSignal, error) {
|
||||
if event == nil {
|
||||
err := errors.Errorf("trying to unmarshall null event ptr")
|
||||
return nil, err
|
||||
}
|
||||
//JSON handling is generally slower;
|
||||
//not only is payload much larger but character insertions to fix json format from optimus is very slow.
|
||||
|
||||
batchData := []kafka_grpc.GRPC_CANSignal{}
|
||||
dataBuffer := make([]byte, len(event.Value)+2)
|
||||
copy(dataBuffer[1:], event.Value)
|
||||
dataBuffer[0] = '['
|
||||
dataBuffer[len(event.Value)+1] = ']'
|
||||
|
||||
err := fastjson.Unmarshal(dataBuffer, &batchData)
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ptrs := make([]*kafka_grpc.GRPC_CANSignal, len(batchData))
|
||||
for i := range batchData {
|
||||
ptrs[i] = &batchData[i]
|
||||
}
|
||||
return ptrs, nil
|
||||
}
|
||||
25
services/jetfire/server/server_http.go
Normal file
25
services/jetfire/server/server_http.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/handlers"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/httphandlers"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
|
||||
"github.com/julienschmidt/httprouter"
|
||||
)
|
||||
|
||||
const port string = ":8077"
|
||||
|
||||
func StartHTTPServer() {
|
||||
router := httprouter.New()
|
||||
router.PanicHandler = httphandlers.HttpRouterPanicHandler
|
||||
addHandler(router, http.MethodGet, "/reset", handlers.ResetSchemaDefinitions)
|
||||
logger.Fatal().AnErr("http.ListenAndServe", http.ListenAndServe(port, router)).Send()
|
||||
}
|
||||
|
||||
func addHandler(router *httprouter.Router, method string, path string, handler http.HandlerFunc) {
|
||||
router.HandlerFunc(method, httphandlers.HttpRouterHandleBaseURL(path), handler)
|
||||
}
|
||||
99
services/jetfire/server/server_inserter.go
Normal file
99
services/jetfire/server/server_inserter.go
Normal file
@@ -0,0 +1,99 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"context"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/models"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/services"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/ClickHouse/ch-go"
|
||||
"github.com/ClickHouse/ch-go/proto"
|
||||
"github.com/pkg/errors"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/utils/envtool"
|
||||
)
|
||||
|
||||
var (
|
||||
flushBufferCmd chan interface{}
|
||||
|
||||
schemaResetPeriod = envtool.GetEnvDuration("JETFIRE_SCHEMA_RESET_PERIOD_MS", 3600000*12) * time.Millisecond
|
||||
)
|
||||
|
||||
// loop for Inserter goroutine.
|
||||
// This goroutine is responsible for inserting data into clickhouse
|
||||
func InserterLoop(producerChannel chan models.InsertCommand, ctx context.Context) {
|
||||
resetSchemaTicker := time.NewTicker(schemaResetPeriod)
|
||||
for {
|
||||
select {
|
||||
// reset cache vars every hour
|
||||
case <-resetSchemaTicker.C:
|
||||
logger.Debug().Msgf("<-resetSchemaTicker")
|
||||
services.ResetCacheVars()
|
||||
|
||||
// process flush buffer command
|
||||
case <-flushBufferCmd:
|
||||
logger.Debug().Msgf("<-flushBufferCmd")
|
||||
|
||||
//get clickhouse client
|
||||
client := services.GetShardClient()
|
||||
for client == nil || client.IsClosed() {
|
||||
logger.Error().Err(errors.Errorf("bad chgo client , retrying connection...")).Send()
|
||||
time.Sleep(time.Second)
|
||||
services.InitShardClients()
|
||||
client = services.GetShardClient()
|
||||
}
|
||||
|
||||
logger.Debug().Msgf("InsertFlushAllBuffers??")
|
||||
services.InsertFlushAllBuffers(client)
|
||||
|
||||
// process data to be inserted to clickhouse
|
||||
case command := <-producerChannel:
|
||||
logger.Debug().Msgf("<-producerChannel")
|
||||
// get clickhouse client
|
||||
client := services.GetShardClient()
|
||||
for client == nil || client.IsClosed() {
|
||||
logger.Error().Err(errors.Errorf("bad chgo client , retrying connection...")).Send()
|
||||
time.Sleep(time.Second)
|
||||
services.InitShardClients()
|
||||
client = services.GetShardClient()
|
||||
}
|
||||
|
||||
//insert the queued buffer and block in command
|
||||
var err error
|
||||
retryInsert := true
|
||||
count := 0
|
||||
insertTime := time.Now()
|
||||
for retryInsert {
|
||||
count, err = models.InsertAndFlush(command, ctx, client.GetClient(), client.GetBreaker(), nil)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
|
||||
logger.Error().Err(errors.WithStack(err)).Send()
|
||||
// if the error is a mismatching schema error, then pull new schemas
|
||||
if ch.IsErr(err,
|
||||
proto.ErrIncompatibleColumns,
|
||||
proto.ErrNoSuchColumnInTable,
|
||||
proto.ErrThereIsNoColumn,
|
||||
proto.ErrIncorrectNumberOfColumns,
|
||||
) {
|
||||
services.ResetCacheVars()
|
||||
} else {
|
||||
// client.Connect() //close and restart the clickhouse connection
|
||||
client = services.GetShardClient() //grab a new shard client to retry insert
|
||||
}
|
||||
time.Sleep(time.Second)
|
||||
}
|
||||
if count == 0 {
|
||||
continue
|
||||
}
|
||||
|
||||
//log messages relating to clickhouse insertion
|
||||
logger.Debug().Msgf("done flush Buffer %s, %dms", command.Buffer.TableName, (time.Since(insertTime))/time.Millisecond)
|
||||
if time.Since(insertTime) > 10*time.Second {
|
||||
logger.Warn().Msgf("slow row insertions: took %s to insert %d rows for %s", time.Since(insertTime), count, command.Buffer.TableName)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
303
services/jetfire/services/cache.go
Normal file
303
services/jetfire/services/cache.go
Normal file
@@ -0,0 +1,303 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/models"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/utils"
|
||||
"reflect"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
var (
|
||||
//singleton cache definitions for vehicle states
|
||||
vehicleCache *models.VehicleCache
|
||||
|
||||
// lists of variables needed by destination tables
|
||||
featureVars []string
|
||||
|
||||
// set of all variables needing to be tracked
|
||||
featureSet map[string]bool
|
||||
varsSet map[string]bool
|
||||
|
||||
SchemaLock sync.Mutex
|
||||
)
|
||||
|
||||
// Initializes feature variables by pulling the table schema for destination tables
|
||||
func ResetCacheVars() {
|
||||
SchemaLock.Lock()
|
||||
defer SchemaLock.Unlock()
|
||||
|
||||
conn, err := GetClickhouseConnection()
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
return
|
||||
}
|
||||
featureVars = GetVarList(conn, FEATURE_TABLE, utils.FeatureVarsDefaults)
|
||||
logger.Debug().Msgf("FEATURE VARS %d", len(featureVars))
|
||||
|
||||
featureSet = make(map[string]bool)
|
||||
varsSet = make(map[string]bool)
|
||||
|
||||
for _, key := range featureVars {
|
||||
varsSet[key] = true
|
||||
featureSet[key] = true
|
||||
}
|
||||
logger.Debug().Msgf("VARSSET %d", len(varsSet))
|
||||
|
||||
vCache := GetVehicleCache()
|
||||
vCache.SignalsSet = &varsSet
|
||||
|
||||
//insert and flush the insertion buffers
|
||||
insertClient := GetShardClient()
|
||||
|
||||
//guarantee initialization of singleton buffer
|
||||
//GetVehicleSignalBatch()
|
||||
|
||||
//only insert signals in the intersection of insert buffer columns and updated sink table columns
|
||||
|
||||
fBuffer := GetFeatureBatch()
|
||||
flBuffer := GetFeatureLastBatch()
|
||||
|
||||
if insertClient != nil {
|
||||
ctx := context.Background()
|
||||
models.InsertAndFlushHead(fBuffer, ctx, insertClient.GetClient(), insertClient.GetBreaker(), featureSet)
|
||||
models.InsertAndFlushHead(flBuffer, ctx, insertClient.GetClient(), insertClient.GetBreaker(), featureSet)
|
||||
}
|
||||
//now resize the buffers and reset the column names
|
||||
fBuffer.Resize(featureVars)
|
||||
|
||||
//now resize the buffers and reset the column names
|
||||
flBuffer.Resize(featureVars)
|
||||
}
|
||||
|
||||
// flushes the head node for all buffers.
|
||||
func InsertFlushAllBuffers(client *ChGoClient) {
|
||||
ctx := context.Background()
|
||||
|
||||
//models.InsertAndFlushHead(GetVehicleSignalBatch(), ctx, client.GetClient(), client.GetBreaker(), featureSet)
|
||||
models.InsertAndFlushHead(GetFeatureBatch(), ctx, client.GetClient(), client.GetBreaker(), featureSet)
|
||||
models.InsertAndFlushHead(GetFeatureLastBatch(), ctx, client.GetClient(), client.GetBreaker(), featureSet)
|
||||
}
|
||||
|
||||
func GetFeatureVars() []string {
|
||||
return featureVars
|
||||
}
|
||||
|
||||
// Initializes the VIN cache from clickhouse feature table. This is used during Kafka rebalance events.
|
||||
func InitCacheFromClickhouse() {
|
||||
SchemaLock.Lock()
|
||||
defer SchemaLock.Unlock()
|
||||
|
||||
logger.Debug().Msgf("InitCacheFromClickhouse")
|
||||
//query feature table
|
||||
|
||||
featureColumns := append([]string{"VIN", "Timestamp", "TripStart", "TripID"}, featureVars...)
|
||||
logger.Debug().Msgf("Querying feature table, %d columns", len(featureColumns))
|
||||
|
||||
queryString := fmt.Sprintf("SELECT * FROM %s LIMIT 1 BY VIN", FEATURE_LAST_TABLE)
|
||||
populateCacheFromTable(GetVehicleCache(), queryString, &featureColumns)
|
||||
|
||||
logger.Debug().Msgf("Pulled cache from clickhouse! %d vehicles!", len(GetVehicleCache().Cache))
|
||||
}
|
||||
|
||||
// Performs a query, expecting a dynamic schema from the table. Query results are used to populate cache.
|
||||
// Columns describes the columns expected in the table
|
||||
func populateCacheFromTable(cache *models.VehicleCache, query string, columns *[]string) {
|
||||
ctx := context.Background()
|
||||
logger.Debug().Msgf("query: %s", query)
|
||||
conn, err := GetClickhouseConnection()
|
||||
|
||||
if err != nil {
|
||||
logger.Error().Err(err)
|
||||
return
|
||||
}
|
||||
|
||||
if !HasClickhouseParams() {
|
||||
logger.Error().Msgf("Could not open clickhouse connection. skipping initialization of vehicle caches")
|
||||
return
|
||||
}
|
||||
|
||||
rows, err := QueryWithBreaker(ctx, conn, &query)
|
||||
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
|
||||
if rows == nil {
|
||||
return
|
||||
}
|
||||
|
||||
columnTypes := rows.ColumnTypes()
|
||||
|
||||
row := make([]interface{}, len(columnTypes))
|
||||
|
||||
for i, cType := range columnTypes {
|
||||
kind := cType.ScanType().Kind()
|
||||
scanName := cType.ScanType().Name()
|
||||
|
||||
if kind == reflect.String || strings.Contains(scanName, "string") {
|
||||
row[i] = new(string)
|
||||
} else if kind == reflect.Float64 || kind == reflect.Float32 || strings.Contains(scanName, "float") || strings.Contains(scanName, "decimal") {
|
||||
row[i] = new(float64)
|
||||
} else if kind == reflect.Int64 || kind == reflect.Int || strings.Contains(scanName, "int") {
|
||||
row[i] = new(int64)
|
||||
} else if strings.Contains(scanName, "Time") {
|
||||
row[i] = new(time.Time)
|
||||
} else {
|
||||
row[i] = new(string)
|
||||
}
|
||||
}
|
||||
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
err := rows.Scan(row...)
|
||||
if err != nil {
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
|
||||
vin := *row[0].(*string)
|
||||
timestamp := *row[1].(*time.Time)
|
||||
|
||||
_, contains := cache.Cache[vin]
|
||||
// initialize new state cache as needed
|
||||
if !contains {
|
||||
cache.Cache[vin] = models.NewVehicleState(vin, len(varsSet))
|
||||
}
|
||||
state := cache.Cache[vin]
|
||||
|
||||
if timestamp.After(state.Timestamp) {
|
||||
for i, valPtr := range row {
|
||||
key := (*columns)[i]
|
||||
if key == "VIN" {
|
||||
continue
|
||||
}
|
||||
if key == "Timestamp" {
|
||||
continue
|
||||
}
|
||||
if key == "TripStart" {
|
||||
state.TripStart = *valPtr.(*time.Time)
|
||||
state.TripID = fmt.Sprintf("%s_%d", state.VIN, state.TripStart.Unix())
|
||||
continue
|
||||
}
|
||||
if key == "TripID" {
|
||||
state.TripID = *valPtr.(*string)
|
||||
continue
|
||||
}
|
||||
myValue, hasValue := state.StateValues[key]
|
||||
if !hasValue || myValue == 0 || timestamp.After(state.Timestamp) {
|
||||
state.StateValues[key] = *valPtr.(*float64)
|
||||
}
|
||||
}
|
||||
state.Timestamp = timestamp
|
||||
}
|
||||
}
|
||||
// append states to right
|
||||
for _, state := range cache.Cache {
|
||||
cache.ReinsertRight(state)
|
||||
}
|
||||
}
|
||||
|
||||
// Queries clickhouse and reads into the dest struct
|
||||
func LoadChState(VIN string, dest *models.VehicleState, columns *[]string) error {
|
||||
ctx := context.Background()
|
||||
queryString := fmt.Sprintf("SELECT * FROM %s WHERE VIN='%s' LIMIT 1", FEATURE_LAST_TABLE, VIN)
|
||||
logger.Debug().Msgf("query: %s", queryString)
|
||||
conn, err := GetClickhouseConnection()
|
||||
|
||||
if !HasClickhouseParams() || err != nil {
|
||||
return errors.Errorf("Could not open clickhouse connection. Failed to load VIN state %s", VIN)
|
||||
}
|
||||
|
||||
rows, err := QueryWithBreaker(ctx, conn, &queryString)
|
||||
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
|
||||
if rows == nil {
|
||||
return nil
|
||||
}
|
||||
|
||||
columnTypes := rows.ColumnTypes()
|
||||
|
||||
row := make([]interface{}, len(columnTypes))
|
||||
|
||||
for i, cType := range columnTypes {
|
||||
kind := cType.ScanType().Kind()
|
||||
scanName := cType.ScanType().Name()
|
||||
|
||||
if kind == reflect.String || strings.Contains(scanName, "string") {
|
||||
row[i] = new(string)
|
||||
} else if kind == reflect.Float64 || kind == reflect.Float32 || strings.Contains(scanName, "float") || strings.Contains(scanName, "decimal") {
|
||||
row[i] = new(float64)
|
||||
} else if kind == reflect.Int64 || kind == reflect.Int || strings.Contains(scanName, "int") {
|
||||
row[i] = new(int64)
|
||||
} else if strings.Contains(scanName, "Time") {
|
||||
row[i] = new(time.Time)
|
||||
} else {
|
||||
row[i] = new(string)
|
||||
}
|
||||
}
|
||||
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
err := rows.Scan(row...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
timestamp := *row[1].(*time.Time)
|
||||
|
||||
for i, valPtr := range row {
|
||||
key := (*columns)[i]
|
||||
if key == "VIN" {
|
||||
dest.VIN = *row[0].(*string)
|
||||
continue
|
||||
}
|
||||
if key == "Timestamp" {
|
||||
dest.Timestamp = timestamp
|
||||
continue
|
||||
}
|
||||
if key == "TripStart" {
|
||||
dest.TripStart = *valPtr.(*time.Time)
|
||||
dest.TripID = fmt.Sprintf("%s_%d", dest.VIN, dest.TripStart.Unix())
|
||||
continue
|
||||
}
|
||||
if key == "TripID" {
|
||||
dest.TripID = *valPtr.(*string)
|
||||
continue
|
||||
}
|
||||
_, ok := dest.StateValues[key]
|
||||
if ok {
|
||||
dest.StateValues[key] = *valPtr.(*float64)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
GetVehicleCache().ReinsertRight(dest)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
// Returns Singleton Vehicle Cache; only one is needed for Jetfire service
|
||||
func GetVehicleCache() *models.VehicleCache {
|
||||
if vehicleCache != nil {
|
||||
return vehicleCache
|
||||
}
|
||||
vehicleCache = new(models.VehicleCache)
|
||||
|
||||
vehicleCache.SignalsSet = &varsSet
|
||||
vehicleCache.Cache = make(map[string]*models.VehicleState)
|
||||
vehicleCache.LastTimestamp = utils.FloatToTime(0.0)
|
||||
|
||||
return vehicleCache
|
||||
}
|
||||
201
services/jetfire/services/chgo.go
Normal file
201
services/jetfire/services/chgo.go
Normal file
@@ -0,0 +1,201 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"time"
|
||||
|
||||
fClickhouse "github.com/fiskerinc/cloud-services/pkg/clickhouse"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/ClickHouse/ch-go"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sony/gobreaker"
|
||||
"gopkg.in/retry.v1"
|
||||
)
|
||||
|
||||
/*
|
||||
This file implements an interface for the chgo library.
|
||||
*/
|
||||
|
||||
var (
|
||||
shardClients []*ChGoClient
|
||||
)
|
||||
|
||||
// Gets a shard client at random
|
||||
func GetShardClient() *ChGoClient {
|
||||
if len(shardClients) == 0 {
|
||||
InitShardClients()
|
||||
}
|
||||
|
||||
if len(shardClients) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
index := rand.Intn(len(shardClients))
|
||||
client := shardClients[index]
|
||||
|
||||
isInvalid := client.GetBreaker().State() == gobreaker.StateOpen && !client.GetClient().IsClosed()
|
||||
count := 5
|
||||
|
||||
//resample shards N times to look for a shard client that is active and circruit breaker is not open
|
||||
for isInvalid && count > 0 {
|
||||
count--
|
||||
index = rand.Intn(len(shardClients))
|
||||
client = shardClients[index]
|
||||
isInvalid = client.GetBreaker().State() == gobreaker.StateOpen && !client.GetClient().IsClosed()
|
||||
}
|
||||
if isInvalid {
|
||||
return nil
|
||||
}
|
||||
|
||||
return client
|
||||
}
|
||||
|
||||
func InitShardClients() {
|
||||
|
||||
clear(shardClients)
|
||||
shardClients = shardClients[:0]
|
||||
|
||||
client, err := GetClickhouseConnection()
|
||||
if err != nil {
|
||||
logger.Error().Err(err).Send()
|
||||
}
|
||||
|
||||
shards := []ShardInfo{}
|
||||
err = client.Select(
|
||||
context.Background(),
|
||||
&shards,
|
||||
"SELECT shard_num, replica_num, host_address FROM system.clusters WHERE cluster='default'",
|
||||
)
|
||||
if err != nil {
|
||||
err = errors.WithStack(err)
|
||||
logger.Error().Err(err).Send()
|
||||
}
|
||||
|
||||
for _, s := range shards {
|
||||
shardName := fmt.Sprintf("%d-%d", s.ShardNum, s.ReplicaNum)
|
||||
logger.Debug().Msgf("Creating new shard connection %s, %s", s.HostAddress, shardName)
|
||||
|
||||
client, _ := NewChgoClient(
|
||||
s.HostAddress,
|
||||
fClickhouse.CLICKHOUSE_PORT,
|
||||
shardName,
|
||||
fClickhouse.CLICKHOUSE_DB,
|
||||
fClickhouse.CLICKHOUSE_USER,
|
||||
fClickhouse.CLICKHOUSE_PASS,
|
||||
)
|
||||
shardClients = append(shardClients, client)
|
||||
}
|
||||
|
||||
logger.Info().Msgf("Connected to %d shards", len(shardClients))
|
||||
}
|
||||
|
||||
type ChGoClient struct {
|
||||
client *ch.Client
|
||||
shardName string
|
||||
|
||||
retry retry.Strategy //retry is used for connecting and reconnecting
|
||||
breaker *gobreaker.CircuitBreaker //circuit breaker is only used for insertions
|
||||
|
||||
ch_host string
|
||||
ch_port string
|
||||
ch_db string
|
||||
ch_user string
|
||||
ch_pass string
|
||||
}
|
||||
|
||||
func NewChgoClient(ch_host string, ch_port string, ch_shard string, ch_db string, ch_user string, ch_pass string) (*ChGoClient, error) {
|
||||
newClient := ChGoClient{
|
||||
shardName: ch_shard,
|
||||
ch_host: ch_host,
|
||||
ch_port: ch_port,
|
||||
ch_db: ch_db,
|
||||
ch_user: ch_user,
|
||||
ch_pass: ch_pass,
|
||||
}
|
||||
|
||||
newClient.retry = retry.LimitTime(
|
||||
120*time.Second,
|
||||
retry.Exponential{
|
||||
Initial: 100 * time.Millisecond,
|
||||
},
|
||||
)
|
||||
err := newClient.Connect()
|
||||
|
||||
if err != nil {
|
||||
return nil, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return &newClient, nil
|
||||
}
|
||||
|
||||
func (client *ChGoClient) Connect() error {
|
||||
if client.client != nil && !client.client.IsClosed() {
|
||||
client.client.Close()
|
||||
}
|
||||
|
||||
var err error
|
||||
var newConn *ch.Client
|
||||
|
||||
client.breaker = gobreaker.NewCircuitBreaker(gobreaker.Settings{
|
||||
Name: "clickhouse",
|
||||
MaxRequests: 1,
|
||||
Interval: time.Minute * 1,
|
||||
Timeout: time.Minute * 10,
|
||||
})
|
||||
|
||||
for a := retry.Start(client.retry, nil); a.Next(); {
|
||||
newConn, err = ch.Dial(
|
||||
context.Background(),
|
||||
ch.Options{
|
||||
Address: fmt.Sprintf("%s:%s", client.ch_host, client.ch_port),
|
||||
Database: client.ch_db,
|
||||
User: client.ch_user,
|
||||
Password: client.ch_pass,
|
||||
DialTimeout: 1 * time.Minute,
|
||||
},
|
||||
)
|
||||
|
||||
if err == nil {
|
||||
client.client = newConn
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
err = errors.WithStack(err)
|
||||
return err
|
||||
}
|
||||
|
||||
func (c *ChGoClient) IsClosed() bool {
|
||||
if c.client == nil {
|
||||
return true
|
||||
}
|
||||
return c.client.IsClosed()
|
||||
}
|
||||
|
||||
func (c *ChGoClient) Close() error {
|
||||
if c.client == nil {
|
||||
return nil
|
||||
}
|
||||
return c.client.Close()
|
||||
}
|
||||
|
||||
func (c *ChGoClient) GetClient() *ch.Client {
|
||||
return c.client
|
||||
}
|
||||
|
||||
func (c *ChGoClient) GetShardName() string {
|
||||
return c.shardName
|
||||
}
|
||||
|
||||
func (c *ChGoClient) GetBreaker() *gobreaker.CircuitBreaker {
|
||||
return c.breaker
|
||||
}
|
||||
|
||||
// helper struct for querying clickhouse shard info
|
||||
type ShardInfo struct {
|
||||
ShardNum uint32 `ch:"shard_num"`
|
||||
ReplicaNum uint32 `ch:"replica_num"`
|
||||
HostAddress string `ch:"host_address"`
|
||||
}
|
||||
213
services/jetfire/services/clickhouse.go
Normal file
213
services/jetfire/services/clickhouse.go
Normal file
@@ -0,0 +1,213 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/models"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/utils"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
fClickhouse "github.com/fiskerinc/cloud-services/pkg/clickhouse"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/fiskerinc/cloud-services/pkg/utils/envtool"
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/pkg/errors"
|
||||
"github.com/sony/gobreaker"
|
||||
"gopkg.in/retry.v1"
|
||||
)
|
||||
|
||||
var (
|
||||
conn fClickhouse.ConnInterface
|
||||
clickLock sync.Mutex
|
||||
FEATURE_TABLE = envtool.GetEnv("CLICKHOUSE_FEATURE_TABLE", "feature_table_shard")
|
||||
FEATURE_LAST_TABLE = envtool.GetEnv("CLICKHOUSE_FEATURE_LAST_TABLE", "feature_table_last_shard")
|
||||
VEHICLE_SIGNAL_TABLE = envtool.GetEnv("CLICKHOUSE_VEHICLE_SIGNAL_TABLE", "vehicle_signal_shard")
|
||||
|
||||
clickhouseBreaker = gobreaker.NewCircuitBreaker(gobreaker.Settings{
|
||||
Name: "clickhouse",
|
||||
MaxRequests: 1,
|
||||
Interval: time.Minute * 1,
|
||||
Timeout: time.Second * 60,
|
||||
OnStateChange: utils.BreakerStateChange,
|
||||
ReadyToTrip: func(counts gobreaker.Counts) bool { return counts.Requests > 0 },
|
||||
})
|
||||
|
||||
featureBuffer *models.InsertBuffer
|
||||
featureLastBuffer *models.InsertBuffer
|
||||
vehicleSignalBuffer *models.InsertBuffer
|
||||
|
||||
retryStrategy = retry.LimitTime(
|
||||
120*time.Second,
|
||||
retry.Exponential{
|
||||
Initial: 100 * time.Millisecond,
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
func HasClickhouseParams() bool {
|
||||
return len(envtool.GetEnv("CLICKHOUSE_USER", "")) > 0
|
||||
}
|
||||
|
||||
// Returns singleton instance of clickhouse connection
|
||||
func GetClickhouseConnection() (fClickhouse.ConnInterface, error) {
|
||||
|
||||
clickLock.Lock()
|
||||
defer clickLock.Unlock()
|
||||
if conn != nil {
|
||||
return conn, nil
|
||||
}
|
||||
|
||||
executeWrapper := func() (interface{}, error) {
|
||||
return fClickhouse.NewConn()
|
||||
}
|
||||
|
||||
var err error
|
||||
if conn == nil {
|
||||
//instantiate singleton
|
||||
newConn, err := clickhouseBreaker.Execute(executeWrapper)
|
||||
if err != nil {
|
||||
panic(errors.WithStack(err))
|
||||
}
|
||||
conn = newConn.(clickhouse.Conn)
|
||||
}
|
||||
|
||||
return conn, err
|
||||
}
|
||||
|
||||
func SetClickhouseConn(newConn fClickhouse.ConnInterface) {
|
||||
clickLock.Lock()
|
||||
defer clickLock.Unlock()
|
||||
|
||||
conn = newConn
|
||||
}
|
||||
|
||||
// Returns the buffer struct for batched inserts into Vehicle Signal table
|
||||
func GetVehicleSignalBatch() *models.InsertBuffer {
|
||||
if vehicleSignalBuffer != nil {
|
||||
return vehicleSignalBuffer
|
||||
}
|
||||
|
||||
vehicleSignalBuffer = models.NewInsertBuffer(
|
||||
false,
|
||||
nil,
|
||||
VEHICLE_SIGNAL_TABLE,
|
||||
envtool.GetEnvDuration("JETFIRE_VEHICLE_SIGNAL_BATCH_PERIOD_MS", 10000)*time.Millisecond,
|
||||
models.SignalBlockType,
|
||||
)
|
||||
return vehicleSignalBuffer
|
||||
}
|
||||
|
||||
// Returns the buffer struct for batched inserts into Feature table
|
||||
func GetFeatureBatch() *models.InsertBuffer {
|
||||
if featureBuffer != nil {
|
||||
return featureBuffer
|
||||
}
|
||||
|
||||
featureBuffer = models.NewInsertBuffer(
|
||||
true,
|
||||
featureVars,
|
||||
FEATURE_TABLE,
|
||||
envtool.GetEnvDuration("JETFIRE_FEATURE_BATCH_PERIOD_MS", 10000)*time.Millisecond,
|
||||
models.PivotBlockType,
|
||||
)
|
||||
return featureBuffer
|
||||
}
|
||||
|
||||
func GetFeatureLastBatch() *models.InsertBuffer {
|
||||
if featureLastBuffer != nil {
|
||||
return featureLastBuffer
|
||||
}
|
||||
|
||||
featureLastBuffer = models.NewInsertBuffer(
|
||||
true,
|
||||
featureVars,
|
||||
FEATURE_LAST_TABLE,
|
||||
envtool.GetEnvDuration("JETFIRE_FEATURE_BATCH_PERIOD_MS", 10000)*time.Millisecond,
|
||||
models.VinLastBlockType,
|
||||
)
|
||||
return featureLastBuffer
|
||||
}
|
||||
|
||||
// Queries table schema to get ordered list of can signals
|
||||
func GetVarList(conn fClickhouse.ConnInterface, table string, defaultVarsFile string) []string {
|
||||
var result []string
|
||||
ctx := context.Background()
|
||||
if !HasClickhouseParams() {
|
||||
logger.Warn().Msgf("No clickhouse params found, reading default vars instead.")
|
||||
return utils.ReadVarListFromFile(defaultVarsFile)
|
||||
}
|
||||
|
||||
query := fmt.Sprintf("DESCRIBE %s", table)
|
||||
|
||||
logger.Debug().Msgf("%s", query)
|
||||
|
||||
var rows []descRow
|
||||
err := SelectWithBreaker(ctx, conn, &query, &rows)
|
||||
if err != nil {
|
||||
logger.Warn().Err(err).Msgf("Failed to select data from clickhouse, reading default vars instead...")
|
||||
return utils.ReadVarListFromFile(defaultVarsFile)
|
||||
}
|
||||
|
||||
for _, row := range rows {
|
||||
if row.Name == "VIN" || row.Name == "Timestamp" {
|
||||
continue
|
||||
}
|
||||
if row.Name == "TripID" || row.Name == "TripStart" {
|
||||
continue
|
||||
}
|
||||
result = append(result, row.Name)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func SelectWithBreaker(ctx context.Context, conn fClickhouse.ConnInterface, query *string, buffer interface{}) error {
|
||||
var err error
|
||||
|
||||
selectWrapper := func() (interface{}, error) {
|
||||
return nil, conn.Select(ctx, buffer, *query)
|
||||
}
|
||||
|
||||
for a := retry.Start(retryStrategy, nil); a.Next(); {
|
||||
_, err = clickhouseBreaker.Execute(selectWrapper)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
logger.Error().Err(err).Send()
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
func QueryWithBreaker(ctx context.Context, conn fClickhouse.ConnInterface, query *string) (driver.Rows, error) {
|
||||
var rows interface{}
|
||||
var err error
|
||||
|
||||
queryWrapper := func() (interface{}, error) {
|
||||
return conn.Query(ctx, *query)
|
||||
}
|
||||
|
||||
for a := retry.Start(retryStrategy, nil); a.Next(); {
|
||||
rows, err = clickhouseBreaker.Execute(queryWrapper)
|
||||
if err == nil && rows != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
//exit if rows is nil; do not attempt to convert to driver.Rows
|
||||
if rows == nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return rows.(driver.Rows), err
|
||||
}
|
||||
|
||||
type descRow struct {
|
||||
Name string `ch:"name"`
|
||||
Type string `ch:"type"`
|
||||
Default_type string `ch:"default_type"`
|
||||
Default_expression string `ch:"default_expression"`
|
||||
Comment string `ch:"comment"`
|
||||
Codec_expression string `ch:"codec_expression"`
|
||||
Ttl_expression string `ch:"ttl_expression"`
|
||||
}
|
||||
72
services/jetfire/services/kafka.go
Normal file
72
services/jetfire/services/kafka.go
Normal file
@@ -0,0 +1,72 @@
|
||||
package services
|
||||
|
||||
import (
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/kafka"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/pkg/errors"
|
||||
"gopkg.in/retry.v1"
|
||||
)
|
||||
|
||||
const ServiceName = "jetfire"
|
||||
|
||||
var (
|
||||
consumer kafka.BaseConsumerInterface = nil
|
||||
|
||||
consumerLock sync.Mutex
|
||||
|
||||
kafkaRetry = retry.LimitTime(
|
||||
120*time.Second,
|
||||
retry.Exponential{
|
||||
Initial: 100 * time.Millisecond,
|
||||
})
|
||||
)
|
||||
|
||||
// GetKafkaConsumer returns singleton instance of kafka consumer
|
||||
func GetKafkaConsumer() (kafka.BaseConsumerInterface, error) {
|
||||
consumerLock.Lock()
|
||||
defer consumerLock.Unlock()
|
||||
|
||||
if consumer != nil {
|
||||
return consumer, nil
|
||||
}
|
||||
|
||||
var err error
|
||||
for a := retry.Start(kafkaRetry, nil); a.Next(); {
|
||||
newConsumer, err := kafka.NewBaseConsumer(ServiceName, -1, -1, true)
|
||||
if err == nil {
|
||||
consumer = newConsumer
|
||||
return consumer, nil
|
||||
}
|
||||
logger.Error().Err(err).Send()
|
||||
}
|
||||
err = errors.WithStack(err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// stops the kafka consumer, instantiates new one. gc should clean up old one
|
||||
func ResetKafkaConsumer() (kafka.BaseConsumerInterface, error) {
|
||||
consumerLock.Lock()
|
||||
defer consumerLock.Unlock()
|
||||
|
||||
logger.Info().Msgf("Resetting Kafka Consumer...")
|
||||
|
||||
if consumer != nil {
|
||||
consumer.Stop()
|
||||
consumer = nil
|
||||
}
|
||||
|
||||
var err error
|
||||
for a := retry.Start(kafkaRetry, nil); a.Next(); {
|
||||
newConsumer, err := kafka.NewBaseConsumer(ServiceName, -1, -1, true)
|
||||
if err == nil {
|
||||
consumer = newConsumer
|
||||
return consumer, nil
|
||||
}
|
||||
logger.Error().Err(err).Send()
|
||||
}
|
||||
err = errors.WithStack(err)
|
||||
return nil, err
|
||||
}
|
||||
16
services/jetfire/set_envs.sh
Executable file
16
services/jetfire/set_envs.sh
Executable file
@@ -0,0 +1,16 @@
|
||||
#!/usr/bin/bash
|
||||
|
||||
# This is a tool to just set env vars from .env
|
||||
# This is used instead of a golang dotenv package,
|
||||
# because there are some variables in fiskerinc.com/modules/clickhouse
|
||||
# that are initialized before init() is called.
|
||||
# Usage:
|
||||
# Run this first before running go module locally:
|
||||
#
|
||||
# source ./set_envs.sh
|
||||
# go run main.go
|
||||
#
|
||||
|
||||
set -a
|
||||
source .env
|
||||
set +a
|
||||
103
services/jetfire/tests/bench_test.go
Normal file
103
services/jetfire/tests/bench_test.go
Normal file
@@ -0,0 +1,103 @@
|
||||
package tests
|
||||
|
||||
/*
|
||||
This file implements benchmarks for
|
||||
1. Batch handling of messages.
|
||||
The intention is to measure performance of appending rows to InsertBuffer
|
||||
2. Batch serialization of messages
|
||||
The intention is to measure performance of preparing data for clickhouse insertion
|
||||
|
||||
Because of the size of the input data for this benchmark, it is recommended to set
|
||||
JETFIRE_BUFFER_MAX_BYTES=1073741824
|
||||
|
||||
Or more; otherwise the benchmark may hang while waiting for the nonexistant inserter thread to flush the buffer.
|
||||
*/
|
||||
|
||||
import (
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/handlers"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/services"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/grpc/kafka_grpc"
|
||||
"github.com/ClickHouse/ch-go/proto"
|
||||
"github.com/intel-go/fastjson"
|
||||
)
|
||||
|
||||
var benchmarkJSONPayload = []byte{}
|
||||
var benchmarkBatchPayload = []kafka_grpc.GRPC_CANSignal{}
|
||||
var benchmarkBatchPtrs = []*kafka_grpc.GRPC_CANSignal{}
|
||||
|
||||
func benchInit() {
|
||||
//1176 messages long
|
||||
jsonPath := "test-batch-msg.json"
|
||||
os.Chdir("./tests/")
|
||||
data, err := os.ReadFile(jsonPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
benchmarkJSONPayload = data
|
||||
fastjson.Unmarshal(data, &benchmarkBatchPayload)
|
||||
|
||||
benchmarkBatchPtrs = make([]*kafka_grpc.GRPC_CANSignal, len(benchmarkBatchPayload))
|
||||
for i := range benchmarkBatchPayload {
|
||||
benchmarkBatchPtrs[i] = &benchmarkBatchPayload[i]
|
||||
}
|
||||
services.ResetCacheVars()
|
||||
}
|
||||
|
||||
func benchmarkMessageHandler(batchData []*kafka_grpc.GRPC_CANSignal, b *testing.B) {
|
||||
cache := services.GetVehicleCache()
|
||||
for i := 0; i < b.N; i++ {
|
||||
handlers.HandleSignalBatch(batchData, cache, nil)
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
}
|
||||
|
||||
func BenchmarkMessageHandler(b *testing.B) {
|
||||
benchInit()
|
||||
b.ResetTimer()
|
||||
benchmarkMessageHandler(benchmarkBatchPtrs, b)
|
||||
}
|
||||
|
||||
func BenchmarkSignalSerialization(b *testing.B) {
|
||||
benchInit()
|
||||
|
||||
cache := services.GetVehicleCache()
|
||||
handlers.HandleSignalBatch(benchmarkBatchPtrs, cache, nil)
|
||||
|
||||
b.ResetTimer()
|
||||
dummy := proto.Input{}
|
||||
serializedLength := 0
|
||||
rowsLength := 0
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
dummy = services.GetVehicleSignalBatch().GetInput()
|
||||
serializedLength += len(dummy)
|
||||
rowsLength += services.GetVehicleSignalBatch().Len()
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
}
|
||||
|
||||
func BenchmarkFeatureSerialization(b *testing.B) {
|
||||
benchInit()
|
||||
|
||||
cache := services.GetVehicleCache()
|
||||
handlers.HandleSignalBatch(benchmarkBatchPtrs, cache, nil)
|
||||
|
||||
b.ResetTimer()
|
||||
dummy := proto.Input{}
|
||||
serializedLength := 0
|
||||
rowsLength := 0
|
||||
|
||||
for i := 0; i < b.N; i++ {
|
||||
dummy = services.GetFeatureBatch().GetInput()
|
||||
serializedLength += len(dummy)
|
||||
rowsLength += services.GetFeatureBatch().Len()
|
||||
}
|
||||
|
||||
b.StopTimer()
|
||||
}
|
||||
251
services/jetfire/tests/cache_test.go
Normal file
251
services/jetfire/tests/cache_test.go
Normal file
@@ -0,0 +1,251 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/services"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/utils"
|
||||
"testing"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/grpc/kafka_grpc"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
const testVIN = "TESTVIN1234567890"
|
||||
|
||||
var testCanSignalBatch = []kafka_grpc.GRPC_CANSignal{
|
||||
{Vin: testVIN, Timestamp: 600.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 100},
|
||||
{Vin: testVIN, Timestamp: 601.0, Id: 792, Name: "ESP_VehSpd", Value: 0},
|
||||
{Vin: testVIN, Timestamp: 602.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 90},
|
||||
{Vin: testVIN, Timestamp: 603.0, Id: 792, Name: "ESP_VehSpd", Value: 10},
|
||||
{Vin: testVIN, Timestamp: 604.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 80},
|
||||
{Vin: testVIN, Timestamp: 605.0, Id: 792, Name: "ESP_VehSpd", Value: 20},
|
||||
{Vin: testVIN, Timestamp: 606.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 80},
|
||||
{Vin: testVIN, Timestamp: 607.0, Id: 792, Name: "ESP_VehSpd", Value: 30},
|
||||
{Vin: testVIN, Timestamp: 608.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 70},
|
||||
{Vin: testVIN, Timestamp: 609.0, Id: 792, Name: "ESP_VehSpd", Value: 40},
|
||||
{Vin: testVIN, Timestamp: 608.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 60},
|
||||
{Vin: testVIN, Timestamp: 607.0, Id: 792, Name: "ESP_VehSpd", Value: 50},
|
||||
{Vin: testVIN, Timestamp: 610.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 50},
|
||||
{Vin: testVIN, Timestamp: 611.0, Id: 792, Name: "ESP_VehSpd", Value: 60},
|
||||
{Vin: testVIN, Timestamp: 612.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 40},
|
||||
{Vin: testVIN, Timestamp: 613.0, Id: 792, Name: "ESP_VehSpd", Value: 70},
|
||||
{Vin: testVIN, Timestamp: 614.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 30},
|
||||
{Vin: testVIN, Timestamp: 615.0, Id: 792, Name: "ESP_VehSpd", Value: 80},
|
||||
{Vin: testVIN, Timestamp: 616.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 20},
|
||||
{Vin: testVIN, Timestamp: 617.0, Id: 792, Name: "ESP_VehSpd", Value: 90},
|
||||
{Vin: testVIN, Timestamp: 618.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 60},
|
||||
{Vin: testVIN, Timestamp: 619.0, Id: 792, Name: "ESP_VehSpd", Value: 100},
|
||||
{Vin: testVIN, Timestamp: 650.0, Id: 792, Name: "ESP_VehSpd", Value: 100},
|
||||
|
||||
{Vin: testVIN, Timestamp: 800.0, Id: 792, Name: "ESP_VehSpd", Value: 100},
|
||||
|
||||
{Vin: testVIN, Timestamp: 1500.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 20},
|
||||
{Vin: testVIN, Timestamp: 1501.0, Id: 792, Name: "ESP_VehSpd", Value: 90},
|
||||
{Vin: testVIN, Timestamp: 1502.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 45},
|
||||
{Vin: testVIN, Timestamp: 1503.0, Id: 792, Name: "ESP_VehSpd", Value: 13},
|
||||
|
||||
{Vin: testVIN, Timestamp: 1510.0, Id: 819, Name: "BCM_PwrMod", Value: 0},
|
||||
{Vin: testVIN, Timestamp: 1511.0, Id: 819, Name: "BCM_PwrMod", Value: 2},
|
||||
{Vin: testVIN, Timestamp: 1512.0, Id: 792, Name: "ESP_VehSpd", Value: 11},
|
||||
}
|
||||
|
||||
var testCanSignalOrderBatch = []kafka_grpc.GRPC_CANSignal{
|
||||
{Vin: testVIN, Timestamp: 700.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 100},
|
||||
{Vin: testVIN, Timestamp: 601.0, Id: 792, Name: "ESP_VehSpd", Value: 0},
|
||||
{Vin: testVIN, Timestamp: 603.0, Id: 792, Name: "ESP_VehSpd", Value: 10},
|
||||
{Vin: testVIN, Timestamp: 704.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 80},
|
||||
{Vin: testVIN, Timestamp: 605.0, Id: 792, Name: "ESP_VehSpd", Value: 20},
|
||||
{Vin: testVIN, Timestamp: 607.0, Id: 792, Name: "ESP_VehSpd", Value: 30},
|
||||
{Vin: testVIN, Timestamp: 708.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 70},
|
||||
{Vin: testVIN, Timestamp: 609.0, Id: 792, Name: "ESP_VehSpd", Value: 40},
|
||||
{Vin: testVIN, Timestamp: 607.0, Id: 792, Name: "ESP_VehSpd", Value: 50},
|
||||
{Vin: testVIN, Timestamp: 710.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 50},
|
||||
{Vin: testVIN, Timestamp: 611.0, Id: 792, Name: "ESP_VehSpd", Value: 60},
|
||||
{Vin: testVIN, Timestamp: 613.0, Id: 792, Name: "ESP_VehSpd", Value: 70},
|
||||
{Vin: testVIN, Timestamp: 714.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 30},
|
||||
{Vin: testVIN, Timestamp: 615.0, Id: 792, Name: "ESP_VehSpd", Value: 80},
|
||||
{Vin: testVIN, Timestamp: 617.0, Id: 792, Name: "ESP_VehSpd", Value: 90},
|
||||
{Vin: testVIN, Timestamp: 718.0, Id: 816, Name: "BMS_PwrBattRmngCpSOC", Value: 60},
|
||||
{Vin: testVIN, Timestamp: 619.0, Id: 792, Name: "ESP_VehSpd", Value: 100},
|
||||
}
|
||||
|
||||
func TestVehicleCache(t *testing.T) {
|
||||
cache := services.GetVehicleCache()
|
||||
|
||||
cache.Clear()
|
||||
|
||||
for i, signal := range testCanSignalBatch {
|
||||
cache.UpdateSignal(&signal, 0x0)
|
||||
|
||||
if i == 4 {
|
||||
// testing signal aggregation
|
||||
state, containsState := cache.Cache[testVIN]
|
||||
assert.True(t, containsState)
|
||||
assert.NotNil(t, state)
|
||||
|
||||
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(604.0), 1e8)
|
||||
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(600.0), 1e8)
|
||||
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 80.0)
|
||||
assert.Equal(t, state.StateValues["ESP_VehSpd"], 10.0)
|
||||
}
|
||||
|
||||
if i == 11 {
|
||||
// testing out of order messages
|
||||
state := cache.Cache[testVIN]
|
||||
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(609.0), 1e8)
|
||||
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(600.0), 1e8)
|
||||
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 60.0)
|
||||
assert.Equal(t, state.StateValues["ESP_VehSpd"], 40.0)
|
||||
}
|
||||
|
||||
if i == 17 {
|
||||
// testing signal aggregation after message order is restored
|
||||
state := cache.Cache[testVIN]
|
||||
|
||||
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(615.0), 1e8)
|
||||
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(600.0), 1e8)
|
||||
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 30.0)
|
||||
assert.Equal(t, state.StateValues["ESP_VehSpd"], 80.0)
|
||||
}
|
||||
|
||||
if i == 22 {
|
||||
state := cache.Cache[testVIN]
|
||||
|
||||
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(650.0), 1e8)
|
||||
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(600.0), 1e8)
|
||||
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 60.0)
|
||||
assert.Equal(t, state.StateValues["ESP_VehSpd"], 100.0)
|
||||
}
|
||||
|
||||
if i == 23 {
|
||||
state := cache.Cache[testVIN]
|
||||
|
||||
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(800.0), 1e8)
|
||||
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(600.0), 1e8)
|
||||
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 60.0)
|
||||
assert.Equal(t, state.StateValues["ESP_VehSpd"], 100.0)
|
||||
}
|
||||
|
||||
if i == 25 {
|
||||
state := cache.Cache[testVIN]
|
||||
|
||||
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(1501.0), 1e8)
|
||||
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(1500.0), 1e8)
|
||||
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 20.0)
|
||||
assert.Equal(t, state.StateValues["ESP_VehSpd"], 90.0)
|
||||
}
|
||||
}
|
||||
|
||||
assert.Equal(t, len(cache.Cache), 1)
|
||||
|
||||
// testing large timestamp gap; trigger new trip
|
||||
state := cache.Cache[testVIN]
|
||||
tripStartTime := utils.FloatToTime(1511.0)
|
||||
assert.WithinDuration(t, state.TripStart, tripStartTime, 1e8)
|
||||
assert.Equal(t, state.TripID, fmt.Sprintf("%s_%d", testVIN, 1511))
|
||||
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(1512.0), 1e8)
|
||||
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 45.0)
|
||||
assert.Equal(t, state.StateValues["ESP_VehSpd"], 11.0)
|
||||
}
|
||||
|
||||
func TestVehicleCacheOrderly(t *testing.T) {
|
||||
cache := services.GetVehicleCache()
|
||||
|
||||
cache.Clear()
|
||||
|
||||
for i, signal := range testCanSignalOrderBatch {
|
||||
cache.UpdateSignal(&signal, 0x0)
|
||||
|
||||
if i == 8 {
|
||||
state := cache.Cache[testVIN]
|
||||
|
||||
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(708.0), 1e8)
|
||||
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(700.0), 1e8)
|
||||
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 70.0)
|
||||
assert.Equal(t, state.StateValues["ESP_VehSpd"], 40.0)
|
||||
|
||||
}
|
||||
}
|
||||
state := cache.Cache[testVIN]
|
||||
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(718.0), 1e8)
|
||||
assert.WithinDuration(t, state.TripStart, utils.FloatToTime(700.0), 1e8)
|
||||
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 60.0)
|
||||
assert.Equal(t, state.StateValues["ESP_VehSpd"], 100.0)
|
||||
|
||||
}
|
||||
|
||||
func TestVehicleCacheList(t *testing.T) {
|
||||
cache := services.GetVehicleCache()
|
||||
|
||||
cache.Clear()
|
||||
|
||||
signal := kafka_grpc.GRPC_CANSignal{
|
||||
Vin: "TESTVIN1",
|
||||
Timestamp: 0.0,
|
||||
Id: 816,
|
||||
Name: "ESP_VehSpd",
|
||||
Value: 792,
|
||||
}
|
||||
cache.UpdateSignal(&signal, 0x0)
|
||||
|
||||
signal = kafka_grpc.GRPC_CANSignal{
|
||||
Vin: "TESTVIN2",
|
||||
Timestamp: 0.0,
|
||||
Id: 816,
|
||||
Name: "ESP_VehSpd",
|
||||
Value: 792,
|
||||
}
|
||||
cache.UpdateSignal(&signal, 0x0)
|
||||
|
||||
signal = kafka_grpc.GRPC_CANSignal{
|
||||
Vin: "TESTVIN3",
|
||||
Timestamp: 0.0,
|
||||
Id: 816,
|
||||
Name: "ESP_VehSpd",
|
||||
Value: 792,
|
||||
}
|
||||
cache.UpdateSignal(&signal, 0x0)
|
||||
|
||||
expectedVins := []string{"TESTVIN1", "TESTVIN2", "TESTVIN3"}
|
||||
node := cache.StatesListHead
|
||||
i := 0
|
||||
for node != nil {
|
||||
assert.Equal(t, expectedVins[i], node.VIN)
|
||||
node = node.Next
|
||||
i++
|
||||
}
|
||||
|
||||
node2 := cache.PopLeft()
|
||||
|
||||
expectedVins = []string{"TESTVIN2", "TESTVIN3"}
|
||||
node = cache.StatesListHead
|
||||
i = 0
|
||||
for node != nil {
|
||||
assert.Equal(t, expectedVins[i], node.VIN)
|
||||
node = node.Next
|
||||
i++
|
||||
}
|
||||
|
||||
cache.ReinsertRight(node2)
|
||||
|
||||
expectedVins = []string{"TESTVIN2", "TESTVIN3", "TESTVIN1"}
|
||||
node = cache.StatesListHead
|
||||
i = 0
|
||||
for node != nil {
|
||||
assert.Equal(t, expectedVins[i], node.VIN)
|
||||
node = node.Next
|
||||
i++
|
||||
}
|
||||
|
||||
cache.ReinsertRight(cache.Cache["TESTVIN3"])
|
||||
|
||||
expectedVins = []string{"TESTVIN2", "TESTVIN1", "TESTVIN3"}
|
||||
node = cache.StatesListHead
|
||||
i = 0
|
||||
for node != nil {
|
||||
assert.Equal(t, expectedVins[i], node.VIN)
|
||||
node = node.Next
|
||||
i++
|
||||
}
|
||||
|
||||
}
|
||||
151
services/jetfire/tests/integration_test.go
Normal file
151
services/jetfire/tests/integration_test.go
Normal file
@@ -0,0 +1,151 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/server"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/services"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/utils"
|
||||
"os"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/grpc/kafka_grpc"
|
||||
"github.com/fiskerinc/cloud-services/pkg/kafka"
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/intel-go/fastjson"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"google.golang.org/protobuf/proto"
|
||||
)
|
||||
|
||||
var batchDataSignal = []kafka_grpc.GRPC_CANSignal{}
|
||||
var dataToPublish kafka_grpc.GRPC_CANSignalBatchPayload
|
||||
var startTimestamp = time.Now().UTC()
|
||||
|
||||
func TestIntegration(t *testing.T) {
|
||||
|
||||
t.Skip()
|
||||
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
|
||||
//cleaning up previous run
|
||||
|
||||
conn, err := services.GetClickhouseConnection()
|
||||
if err != nil {
|
||||
logger.Error().Err(err)
|
||||
}
|
||||
|
||||
startTimestamp = time.Now().UTC()
|
||||
|
||||
timestampString := fmt.Sprintf(
|
||||
"%d-%d-%d %d:%d:%d",
|
||||
startTimestamp.Year(),
|
||||
startTimestamp.Month(),
|
||||
startTimestamp.Day(),
|
||||
startTimestamp.Hour(),
|
||||
startTimestamp.Minute(),
|
||||
startTimestamp.Second(),
|
||||
)
|
||||
|
||||
println("cleaning up previous run")
|
||||
conn.Exec(ctx, fmt.Sprintf("ALTER TABLE %s DELETE WHERE VIN=='%s' AND Timestamp<'%s'", services.FEATURE_TABLE, testVIN, timestampString))
|
||||
conn.Exec(ctx, fmt.Sprintf("ALTER TABLE %s DELETE WHERE VIN=='%s' AND Timestamp<'%s'", services.VEHICLE_SIGNAL_TABLE, testVIN, timestampString))
|
||||
|
||||
// conn.Ping(ctx)
|
||||
|
||||
//initialization
|
||||
cache := services.GetVehicleCache()
|
||||
cache.Clear()
|
||||
|
||||
readTestData()
|
||||
producer, err := kafka.NewAsyncProducer(ctx)
|
||||
assert.Nil(t, err)
|
||||
// runJetfire(ctx)
|
||||
|
||||
publishDuration := int64(10)
|
||||
testDuration := time.Duration(publishDuration*2) * time.Second //batch inserts take some time to run. give it some time...
|
||||
|
||||
// begin publishing kafka data. Update message timestamps first.
|
||||
grpcData, err := proto.Marshal(&dataToPublish)
|
||||
assert.Nil(t, err)
|
||||
|
||||
err = producer.ProduceBinary(kafka.VehicleSignal, testVIN, grpcData, nil)
|
||||
assert.Nil(t, err)
|
||||
|
||||
//wait a while since kafka has to rebalance, wait for clickhouse inserts to trigger
|
||||
time.Sleep(testDuration)
|
||||
|
||||
//check clickhouse, count rows
|
||||
println("querying clickhouse feature...")
|
||||
query := fmt.Sprintf("SELECT VIN, Timestamp FROM %s WHERE VIN=='%s' AND Timestamp>='%s'",
|
||||
services.FEATURE_TABLE,
|
||||
testVIN,
|
||||
timestampString,
|
||||
)
|
||||
checkRows(query, 1, t)
|
||||
|
||||
println("querying clickhouse vehicle_signal...")
|
||||
query = fmt.Sprintf("SELECT VIN, Timestamp FROM %s WHERE VIN=='%s' AND Timestamp>='%s'",
|
||||
services.VEHICLE_SIGNAL_TABLE,
|
||||
testVIN,
|
||||
timestampString,
|
||||
)
|
||||
checkRows(query, 1000, t)
|
||||
}
|
||||
|
||||
func checkRows(query string, expected int, t *testing.T) {
|
||||
conn, err := services.GetClickhouseConnection()
|
||||
|
||||
if err != nil {
|
||||
logger.Error().Err(err)
|
||||
return
|
||||
}
|
||||
|
||||
fmt.Println(query)
|
||||
rows, err := conn.Query(context.Background(), query)
|
||||
assert.Nil(t, err)
|
||||
|
||||
count := int(0)
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
count++
|
||||
}
|
||||
assert.GreaterOrEqual(t, count, expected)
|
||||
}
|
||||
|
||||
func readTestData() {
|
||||
//1176 messages long
|
||||
jsonPath := "./test-batch-msg.json"
|
||||
data, err := os.ReadFile(jsonPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
fastjson.Unmarshal(data, &batchDataSignal)
|
||||
|
||||
dataPtr := make([]*kafka_grpc.GRPC_CANSignal, len(batchDataSignal))
|
||||
offset := -1.0
|
||||
for i := range batchDataSignal {
|
||||
dataPtr[i] = &batchDataSignal[i]
|
||||
|
||||
// find min timestamp in batch data
|
||||
if offset < 0 || offset > batchDataSignal[i].Timestamp {
|
||||
offset = dataPtr[i].Timestamp
|
||||
}
|
||||
|
||||
dataPtr[i].Vin = testVIN // in case we need to chagne the test vin to follow proper pattern
|
||||
}
|
||||
for i := range batchDataSignal {
|
||||
dataPtr[i].Timestamp += utils.TimeToFloat(startTimestamp) - offset
|
||||
}
|
||||
dataToPublish = kafka_grpc.GRPC_CANSignalBatchPayload{Data: &kafka_grpc.GRPC_CANSignalData{
|
||||
Cansignals: dataPtr,
|
||||
}}
|
||||
}
|
||||
|
||||
func runJetfire(ctx context.Context) {
|
||||
// first initialize and run jetfire application loops
|
||||
services.ResetCacheVars()
|
||||
go server.StartConsumer(ctx, kafka.VehicleSignal)
|
||||
}
|
||||
39
services/jetfire/tests/message_test.go
Normal file
39
services/jetfire/tests/message_test.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package tests
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/handlers"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/services"
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/utils"
|
||||
"testing"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/grpc/kafka_grpc"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestHandleSignalMessage(t *testing.T) {
|
||||
cache := services.GetVehicleCache()
|
||||
|
||||
ptrArray := make([]*kafka_grpc.GRPC_CANSignal, len(testCanSignalBatch))
|
||||
for i := range testCanSignalBatch {
|
||||
ptrArray[i] = &testCanSignalBatch[i]
|
||||
}
|
||||
|
||||
services.ResetCacheVars()
|
||||
|
||||
cache = services.GetVehicleCache()
|
||||
cache.Clear()
|
||||
|
||||
handlers.HandleSignalBatch(ptrArray, cache, nil)
|
||||
|
||||
assert.Equal(t, len(cache.Cache), 1)
|
||||
|
||||
// testing large timestamp gap; trigger new trip
|
||||
state := cache.Cache[testVIN]
|
||||
tripStartTime := utils.FloatToTime(1511.0)
|
||||
assert.WithinDuration(t, state.TripStart, tripStartTime, 1e8)
|
||||
assert.Equal(t, state.TripID, fmt.Sprintf("%s_%d", testVIN, 1511))
|
||||
assert.WithinDuration(t, state.Timestamp, utils.FloatToTime(1512.0), 1e8)
|
||||
assert.Equal(t, state.StateValues["BMS_PwrBattRmngCpSOC"], 45.0)
|
||||
assert.Equal(t, state.StateValues["ESP_VehSpd"], 11.0)
|
||||
}
|
||||
26
services/jetfire/tests/reset_test.go
Normal file
26
services/jetfire/tests/reset_test.go
Normal file
@@ -0,0 +1,26 @@
|
||||
//go:build reset
|
||||
// +build reset
|
||||
|
||||
package tests
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/services/jetfire/handlers"
|
||||
|
||||
th "github.com/fiskerinc/cloud-services/pkg/testhelper"
|
||||
)
|
||||
|
||||
func TestResetSchema(t *testing.T) {
|
||||
tests := []th.BasicHttpTest{
|
||||
{
|
||||
Name: "Reset",
|
||||
Request: th.MakeTestRequest(http.MethodGet, "http://example.com/reset", nil),
|
||||
ExpectedStatus: http.StatusOK,
|
||||
ExpectedResponse: "",
|
||||
},
|
||||
}
|
||||
|
||||
th.RunBasicHttpTests(t, tests, handlers.ResetSchemaDefinitions)
|
||||
}
|
||||
1
services/jetfire/tests/test-batch-msg.json
Normal file
1
services/jetfire/tests/test-batch-msg.json
Normal file
File diff suppressed because one or more lines are too long
25
services/jetfire/utils/const.go
Normal file
25
services/jetfire/utils/const.go
Normal file
@@ -0,0 +1,25 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"math"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/utils/envtool"
|
||||
)
|
||||
|
||||
var (
|
||||
//other cache constants
|
||||
TripTimeout = time.Duration(envtool.GetEnvInt64("JETFIRE_TRIP_TIMEOUT_MS", 600000)) * time.Millisecond
|
||||
|
||||
EmptySignal = math.NaN()
|
||||
|
||||
//bitwise flags for state updates.
|
||||
//Additional flags can be added for additional sink tables later on
|
||||
FeatureUpdateFlag uint = 0x1
|
||||
LatestUpdateFlag uint = 0x2
|
||||
|
||||
MaxVinLength = 20
|
||||
MaxTimestampLength = 12
|
||||
|
||||
FeatureVarsDefaults = "default-feature-vars.json"
|
||||
)
|
||||
11
services/jetfire/utils/errors.go
Normal file
11
services/jetfire/utils/errors.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package utils
|
||||
|
||||
import "errors"
|
||||
|
||||
var (
|
||||
ErrInsertFullBlock = errors.New("appending row into full block")
|
||||
ErrInsertWrongColumns = errors.New("appending buffer with incorrect number of columns")
|
||||
ErrInvalidAppendType = errors.New("appending invalid type to block")
|
||||
ErrNilClickhouseClient = errors.New("nil clickhouse client")
|
||||
ErrNilKafkaConsumer = errors.New("nil kafka consumer")
|
||||
)
|
||||
63
services/jetfire/utils/functions.go
Normal file
63
services/jetfire/utils/functions.go
Normal file
@@ -0,0 +1,63 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"math"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/intel-go/fastjson"
|
||||
"github.com/sony/gobreaker"
|
||||
)
|
||||
|
||||
// Converts float64 (as decimal seconds in unix epoch) to Time.Time struct
|
||||
func FloatToTime(float float64) time.Time {
|
||||
s := int64(math.Floor(float))
|
||||
ns := int64((float - math.Floor(float)) * 1e9)
|
||||
return time.Unix(
|
||||
s,
|
||||
ns,
|
||||
)
|
||||
}
|
||||
|
||||
func TimeToFloat(timestamp time.Time) float64 {
|
||||
return float64(timestamp.UnixNano()) / 1e9
|
||||
}
|
||||
|
||||
func FixFloatTimestampScale(float float64) float64 {
|
||||
for float > 9999999999 {
|
||||
float /= 1000
|
||||
}
|
||||
return float
|
||||
}
|
||||
|
||||
// marchTimer increments the value at timer until newTime has been reached.
|
||||
// This is used to try to maintain consistent downsample and insertion periods.
|
||||
func MarchTimer(timer *time.Time, newTime *time.Time, delay time.Duration) {
|
||||
for timer.Before(*newTime) {
|
||||
*timer = timer.Add(delay)
|
||||
}
|
||||
}
|
||||
|
||||
func ReadVarListFromFile(varsFile string) []string {
|
||||
data, err := os.ReadFile(varsFile)
|
||||
if err != nil {
|
||||
// when running tests, pwd is in the wrong directory to find the default json files.
|
||||
os.Chdir("..")
|
||||
data, err = os.ReadFile(varsFile)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
|
||||
var result []string
|
||||
err = fastjson.Unmarshal(data, &result)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
return result
|
||||
}
|
||||
|
||||
func BreakerStateChange(name string, from gobreaker.State, to gobreaker.State) {
|
||||
logger.Warn().Stack().Msgf("%s breaker change from %d to %d", name, from, to)
|
||||
}
|
||||
55
services/jetfire/utils/logging.go
Normal file
55
services/jetfire/utils/logging.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package utils
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
"github.com/fiskerinc/cloud-services/pkg/logger"
|
||||
"github.com/rs/zerolog"
|
||||
)
|
||||
|
||||
var (
|
||||
outOfOrderSignalMap = make(map[string]uint)
|
||||
outOfOrderVINMap = make(map[string]uint)
|
||||
outOfOrderCount uint = 0
|
||||
outOfOrderLogDelay = time.Hour
|
||||
outOfOrderTime = time.Now().UTC()
|
||||
)
|
||||
|
||||
// This function is for aggregating and logging out of order incoming messages
|
||||
func LogOutOfOrderMsg(signal string, VIN string) {
|
||||
_, ok := outOfOrderSignalMap[signal]
|
||||
if !ok {
|
||||
outOfOrderSignalMap[signal] = 0
|
||||
}
|
||||
outOfOrderSignalMap[signal] += 1
|
||||
|
||||
_, ok = outOfOrderVINMap[VIN]
|
||||
if !ok {
|
||||
outOfOrderVINMap[VIN] = 0
|
||||
}
|
||||
outOfOrderVINMap[VIN] += 1
|
||||
outOfOrderCount += 1
|
||||
|
||||
if time.Since(outOfOrderTime) > outOfOrderLogDelay {
|
||||
|
||||
signalDict := zerolog.Dict()
|
||||
for k, v := range outOfOrderSignalMap {
|
||||
signalDict.Uint(k, v)
|
||||
}
|
||||
vinDict := zerolog.Dict()
|
||||
for k, v := range outOfOrderVINMap {
|
||||
vinDict.Uint(k, v)
|
||||
}
|
||||
|
||||
logger.Warn().Dict(
|
||||
"Signals", signalDict,
|
||||
).Dict(
|
||||
"VINs", vinDict,
|
||||
).Msgf("Received Out of Order Data! %d out-of-order messages", outOfOrderCount)
|
||||
|
||||
outOfOrderCount = 0
|
||||
outOfOrderTime = time.Now().UTC()
|
||||
clear(outOfOrderSignalMap)
|
||||
clear(outOfOrderVINMap)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user