Initial cloud-services repo - gateway service + pkg modules

2026-01-30 23:14:52 -05:00
commit fbb820d7b3
1037 changed files with 171318 additions and 0 deletions
--- a/pkg/clickhouse/clickhouse.go
+++ b/pkg/clickhouse/clickhouse.go
@@ -0,0 +1,257 @@
+package clickhouse
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"fiskerinc.com/modules/common"
+	"fiskerinc.com/modules/utils/envtool"
+
+	"github.com/ClickHouse/clickhouse-go/v2"
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+	"github.com/pkg/errors"
+)
+
+var (
+	TIMEOUT               = envtool.GetEnvInt("CLICKHOUSE_TIMEOUT", 1)
+	MAX_CONNS             = envtool.GetEnvInt("CLICKHOUSE_MAX_CONNS", 1)
+	CLICKHOUSE_HOST       = envtool.GetEnv("CLICKHOUSE_HOST", "localhost")
+	CLICKHOUSE_PORT       = envtool.GetEnv("CLICKHOUSE_PORT", "9000")
+	CLICKHOUSE_DB         = envtool.GetEnv("CLICKHOUSE_DB", "default")
+	CLICKHOUSE_USER       = envtool.GetEnv("CLICKHOUSE_USER", "")
+	CLICKHOUSE_PASS       = envtool.GetEnv("CLICKHOUSE_PASS", "")
+	VEHICLE_FILTERS_TABLE = envtool.GetEnv("CLICKHOUSE_VEHICLE_FILTERS_TABLE", "can_filter_list_vin")
+	DEFAULT_FILTERS_TABLE = envtool.GetEnv("CLICKHOUSE_DEFAULT_FILTERS_TABLE", "can_filter_list_all")
+)
+
+func NewClient(conn ConnInterface) (ClientInterface, error) {
+	return &Client{conn: conn}, nil
+}
+
+func NewConn() (clickhouse.Conn, error) {
+	return clickhouse.Open(&clickhouse.Options{
+		Addr: []string{fmt.Sprintf("%s:%s", CLICKHOUSE_HOST, CLICKHOUSE_PORT)},
+		Auth: clickhouse.Auth{
+			Database: CLICKHOUSE_DB,
+			Username: CLICKHOUSE_USER,
+			Password: CLICKHOUSE_PASS,
+		},
+		DialTimeout:     time.Second * 60,
+		MaxOpenConns:    MAX_CONNS,
+		MaxIdleConns:    MAX_CONNS,
+		ConnMaxLifetime: 24 * time.Hour,
+		Compression: &clickhouse.Compression{
+			Method: clickhouse.CompressionLZ4,
+		},
+	})
+}
+
+type ClientInterface interface {
+	Select(result interface{}, query string) error
+	RetrieveDefaultFilters() ([]CANFilterSchema, error)
+	RetrieveFiltersForVehicle(vin string) ([]CANFilterSchema, error)
+	SaveDBCInfo(dbc common.DBCDesc) error
+	SaveDBCMessages(ms []common.MessageDesc) error
+	SaveDBCSignals(signals []common.SignalDesc) error
+	SelectDBCsByVersions(versions []string) ([]string, error)
+	SelectDBCSignals(dbc string, options PageQueryOptions) ([]common.SignalDescWithECU, int, error)
+	TruncateDBCDescs() error
+
+	SetConn(conn ConnInterface)
+	Exec(query string) error
+}
+
+type ConnInterface interface {
+	Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error
+	PrepareBatch(ctx context.Context, query string) (driver.Batch, error)
+	AsyncInsert(ctx context.Context, query string, wait bool) error
+	QueryRow(ctx context.Context, query string, args ...interface{}) driver.Row
+	Query(ctx context.Context, query string, args ...interface{}) (driver.Rows, error)
+	Exec(ctx context.Context, query string, args ...interface{}) error
+}
+
+type Client struct {
+	conn ConnInterface
+}
+
+func (c *Client) Select(result interface{}, query string) error {
+	ctx := context.Background()
+
+	err := c.conn.Select(ctx, result, query)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+func (c *Client) RetrieveDefaultFilters() ([]CANFilterSchema, error) {
+	var result []CANFilterSchema
+
+	if err := c.Select(&result, fmt.Sprintf("SELECT ID, Period FROM %s", DEFAULT_FILTERS_TABLE)); err != nil {
+		return result, err
+	}
+
+	return result, nil
+}
+
+func (c *Client) RetrieveFiltersForVehicle(vin string) ([]CANFilterSchema, error) {
+	var result []CANFilterSchema
+
+	if err := c.Select(&result, fmt.Sprintf("SELECT ID, Period FROM %s WHERE VIN='%s'", VEHICLE_FILTERS_TABLE, vin)); err != nil {
+		return result, err
+	}
+
+	return result, nil
+}
+
+func (c *Client) SelectDBCsByVersions(versions []string) ([]string, error) {
+	if len(versions) == 0 {
+		return nil, nil
+	}
+
+	q := fmt.Sprintf("'%s'", strings.Join(versions, "','"))
+
+	var result []string
+	if err := c.Select(&result, fmt.Sprintf("SELECT dbc_name FROM dbcs where dbc_hash IN (%s)", q)); err != nil {
+		return result, err
+	}
+
+	return result, nil
+}
+
+func (c *Client) TruncateDBCDescs() error {
+	err := c.Exec("TRUNCATE TABLE dbc_signals_shard ON CLUSTER default")
+	if err != nil {
+		return err
+	}
+
+	err = c.Exec("TRUNCATE TABLE dbc_messages_shard ON CLUSTER default")
+	if err != nil {
+		return err
+	}
+
+	err = c.Exec("TRUNCATE TABLE dbcs_shard ON CLUSTER default")
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (c *Client) SaveDBCInfo(dbc common.DBCDesc) error {
+	query := fmt.Sprintf(`INSERT INTO dbcs (dbc_hash, dbc_name) VALUES ('%s', '%s')`, dbc.Hash, dbc.Name)
+	return errors.WithStack(c.conn.AsyncInsert(context.Background(), query, true))
+}
+
+func (c *Client) SaveDBCMessages(ms []common.MessageDesc) error {
+	batch, err := c.conn.PrepareBatch(context.Background(),
+		`INSERT INTO dbc_messages (dbc_hash, message_name, message_id, is_extended,
+					send_type, length, description, sender_node, cycle_time_ns, delay_time_ns, ecu_name)`)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	for _, m := range ms {
+		err = batch.Append(m.DBCHash, m.Name, m.ID, m.IsExtended, m.SendType, m.Length, m.Description,
+			m.SenderNode, m.CycleTime, m.DelayTime, m.ECUName)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+	}
+
+	return errors.WithStack(batch.Send())
+}
+
+func (c *Client) SaveDBCSignals(signals []common.SignalDesc) error {
+	batch, err := c.conn.PrepareBatch(context.Background(),
+		`INSERT INTO dbc_signals (
+                         dbc_hash, message_id, signal_name, start,
+                         length, big_endian, signed, multiplexer, multiplexed,
+                         multiplexer_value, offset, scale, min, max, unit,
+                         description, value_descriptions, receiver_nodes, default_value, ecu_name)`)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	for _, s := range signals {
+		err = batch.Append(s.DBCHash, s.MessageID, s.Name, s.Start, s.Length, s.IsBigEndian, s.IsSigned,
+			s.IsMultiplexer, s.IsMultiplexed, s.MultiplexerValue, s.Offset, s.Scale, s.Min, s.Max,
+			s.Unit, s.Description, s.ValueDescriptions, s.ReceiverNodes, s.DefaultValue, s.ECUName)
+		if err != nil {
+			return errors.WithStack(err)
+		}
+	}
+
+	return batch.Send()
+}
+
+func (c *Client) SelectDBCSignals(dbc string, options PageQueryOptions) ([]common.SignalDescWithECU, int, error) {
+	var result []common.SignalDescWithECU
+	chCtx := clickhouse.Context(
+		context.Background(),
+		clickhouse.WithParameters(clickhouse.Parameters{
+			"dbc": dbc,
+
+			// we cannot use keywords like "offset" and "limit" as parameter names
+			"lim":  fmt.Sprint(options.Limit),
+			"offs": fmt.Sprint(options.Offset),
+		}))
+
+	query := CreateDBCSignalQuery(options)
+
+	if err := c.conn.Select(chCtx, &result, query); err != nil {
+		return nil, 0, errors.WithStack(err)
+	}
+
+	var count uint64
+	if err := c.conn.QueryRow(chCtx, "SELECT COUNT() FROM dbc_signals a WHERE a.dbc_hash = {dbc:String}").Scan(&count); err != nil {
+		return nil, 0, errors.WithStack(err)
+	}
+
+	return result, int(count), nil
+}
+
+func (c *Client) SetConn(conn ConnInterface) {
+	c.conn = conn
+}
+
+func (c *Client) Exec(query string) error {
+	ctx := context.Background()
+
+	err := c.conn.Exec(ctx, query)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+type CANFilterSchema struct {
+	ID     int16 `ch:"ID"`
+	Period int32 `ch:"Period"`
+}
+
+type MigrationSchema struct {
+	Version  int64  `ch:"version"`
+	Dirty    uint8  `ch:"dirty"`
+	Sequence uint64 `ch:"sequence"`
+}
+
+func CreateDBCSignalQuery(options PageQueryOptions) string {
+
+	initQuery := `select * from dbc_signals where dbc_hash = {dbc:String}`
+
+	query := initQuery
+	if options.Limit != 0 {
+		query += ` LIMIT {lim:UInt64}`
+	}
+
+	if options.Offset != 0 {
+		query += ` OFFSET {offs:UInt64}`
+	}
+
+	return query
+}
--- a/pkg/clickhouse/clickhouse_test.go
+++ b/pkg/clickhouse/clickhouse_test.go
@@ -0,0 +1,125 @@
+package clickhouse_test
+
+import (
+	"testing"
+
+	"fiskerinc.com/modules/clickhouse"
+	"fiskerinc.com/modules/testhelper"
+)
+
+func TestRetrieveDefaultFilters(t *testing.T) {
+	filters := []clickhouse.CANFilterSchema{
+		{
+			ID:     1,
+			Period: 2,
+		},
+		{
+			ID:     3,
+			Period: 4,
+		},
+	}
+
+	conn := &clickhouse.MockConn{ExpectedResult: filters}
+	client := clickhouse.NewMockClient(conn)
+
+	defaults, err := client.RetrieveDefaultFilters()
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveDefaultFilters", nil, err)
+	}
+
+	for i := range filters {
+		if filters[i].ID != defaults[i].ID {
+			t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveDefaultFilters", filters[i].ID, defaults[i].ID)
+		}
+		if filters[i].Period != defaults[i].Period {
+			t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveDefaultFilters", filters[i].Period, defaults[i].Period)
+		}
+	}
+}
+
+func TestRetrieveFiltersForVehicle(t *testing.T) {
+	filters := []clickhouse.CANFilterSchema{
+		{
+			ID:     1,
+			Period: 2,
+		},
+		{
+			ID:     3,
+			Period: 4,
+		},
+	}
+
+	conn := &clickhouse.MockConn{ExpectedResult: filters}
+	client := clickhouse.NewMockClient(conn)
+
+	defaults, err := client.RetrieveFiltersForVehicle("TESTVIN123")
+	if err != nil {
+		t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveDefaultFilters", nil, err)
+	}
+
+	for i := range filters {
+		if filters[i].ID != defaults[i].ID {
+			t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveDefaultFilters", filters[i].ID, defaults[i].ID)
+		}
+		if filters[i].Period != defaults[i].Period {
+			t.Errorf(testhelper.TestErrorTemplate, "TestRetrieveDefaultFilters", filters[i].Period, defaults[i].Period)
+		}
+	}
+}
+
+func TestCreateDBCSignalQuery(t *testing.T) {
+
+	tests := []struct {
+		name   string
+		input  clickhouse.PageQueryOptions
+		result string
+		err    error
+	}{
+		{
+			name: "with no options",
+			input: clickhouse.PageQueryOptions{
+				Limit:  0,
+				Offset: 0,
+			},
+			result: `select * from dbc_signals where dbc_hash = {dbc:String}`,
+			err: nil,
+		},
+		{
+			name: "with the offset option",
+			input: clickhouse.PageQueryOptions{
+				Limit:  0,
+				Offset: 10,
+			},
+			result: `select * from dbc_signals where dbc_hash = {dbc:String} OFFSET {offs:UInt64}`,
+			err: nil,
+		},
+		{
+			name: "with the limit option",
+			input: clickhouse.PageQueryOptions{
+				Limit:  10,
+				Offset: 0,
+			},
+			result: `select * from dbc_signals where dbc_hash = {dbc:String} LIMIT {lim:UInt64}`,
+			err: nil,
+		},
+		{
+			name: "with the offset and limit options",
+			input: clickhouse.PageQueryOptions{
+				Limit:  100,
+				Offset: 10,
+			},
+			result: `select * from dbc_signals where dbc_hash = {dbc:String} LIMIT {lim:UInt64} OFFSET {offs:UInt64}`,
+			err: nil,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			ans := clickhouse.CreateDBCSignalQuery(test.input)
+			if ans != test.result {
+				t.Errorf("got %s, expected %s", ans, test.result)
+			}
+		})
+	}
+
+}
--- a/pkg/clickhouse/mock.go
+++ b/pkg/clickhouse/mock.go
@@ -0,0 +1,134 @@
+package clickhouse
+
+import (
+	"context"
+	"encoding/json"
+	"reflect"
+
+	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
+
+	"github.com/pkg/errors"
+)
+
+func NewMockClient(conn ConnInterface) ClientInterface {
+	c := &Client{}
+	c.SetConn(conn)
+	return c
+}
+
+type MockConn struct {
+	ExpectedResult   interface{}
+	PrepareBatchMock func(ctx context.Context, query string) (driver.Batch, error)
+	AsyncInsertMock  func(ctx context.Context, query string, wait bool) error
+	QueryRowtMock    func(ctx context.Context, query string, args ...interface{}) driver.Row
+	QueryMock        func(ctx context.Context, query string, args ...interface{}) (driver.Rows, error)
+	Client
+}
+
+func (c *MockConn) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
+	payload, err := json.Marshal(c.ExpectedResult)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	err = json.Unmarshal(payload, dest)
+	if err != nil {
+		return errors.WithStack(err)
+	}
+
+	return nil
+}
+
+func (c *MockConn) PrepareBatch(ctx context.Context, query string) (driver.Batch, error) {
+	return c.PrepareBatchMock(ctx, query)
+}
+
+func (c *MockConn) AsyncInsert(ctx context.Context, query string, wait bool) error {
+	return c.AsyncInsertMock(ctx, query, wait)
+}
+
+func (c *MockConn) QueryRow(ctx context.Context, query string, args ...interface{}) driver.Row {
+	return c.QueryRowtMock(ctx, query, args...)
+}
+
+func (c *MockConn) Query(ctx context.Context, query string, args ...interface{}) (driver.Rows, error) {
+	return RowsMock{}, nil
+}
+
+func (c *MockConn) Exec(ctx context.Context, query string, args ...interface{}) error {
+	return nil
+}
+
+type ColumnTypeMock struct{}
+
+func (c ColumnTypeMock) Name() string {
+	return "name"
+}
+
+func (c ColumnTypeMock) Nullable() bool {
+	return true
+}
+
+func (c ColumnTypeMock) ScanType() reflect.Type {
+	return reflect.TypeOf("")
+}
+
+func (c ColumnTypeMock) DatabaseTypeName() string {
+	return ""
+}
+
+type RowsMock struct {
+	RowsResult interface{}
+}
+
+func (r RowsMock) Next() bool {
+	return false
+}
+
+func (r RowsMock) Scan(dest ...interface{}) error {
+	return nil
+}
+
+func (r RowsMock) ScanStruct(dest interface{}) error {
+	return nil
+}
+
+func (r RowsMock) ColumnTypes() []driver.ColumnType {
+	return []driver.ColumnType{ColumnTypeMock{}}
+}
+
+func (r RowsMock) Totals(dest ...interface{}) error {
+	return nil
+}
+
+func (r RowsMock) Columns() []string {
+	return nil
+}
+
+func (r RowsMock) Close() error {
+	return nil
+}
+
+func (r RowsMock) Err() error {
+	return nil
+}
+
+type RowMock struct {
+	RowResult interface{}
+}
+
+func (r RowMock) Err() error {
+	return nil
+}
+
+func (r RowMock) Scan(dest ...interface{}) error {
+	if len(dest) != 0 {
+		dest[0] = r.RowResult
+	}
+
+	return nil
+}
+
+func (r RowMock) ScanStruct(dest interface{}) error {
+	return nil
+}
--- a/pkg/clickhouse/options.go
+++ b/pkg/clickhouse/options.go
@@ -0,0 +1,31 @@
+package clickhouse
+
+import (
+	"net/http"
+
+	"fiskerinc.com/modules/validator"
+
+	"github.com/gorilla/schema"
+)
+
+const PageQueryOptionsLimitMaximum = 100
+
+type PageQueryOptions struct {
+	Limit  int `json:"limit" validate:"gte=0,lte=100"`
+	Offset int `json:"offset" validate:"gte=0"`
+}
+
+// ParsePageQuery parses PageQueryOptions from http request
+func ParsePageQuery(r *http.Request) (PageQueryOptions, error) {
+	decoder := schema.NewDecoder()
+	options := PageQueryOptions{}
+
+	decoder.SetAliasTag("json")
+	decoder.Decode(&options, r.URL.Query())
+	err := validator.ValidateStruct(options)
+	if err == nil && options.Limit == 0 {
+		options.Limit = PageQueryOptionsLimitMaximum
+	}
+
+	return options, err
+}