Initial cloud-services repo - gateway service + pkg modules
This commit is contained in:
Chris Rai
257
pkg/clickhouse/clickhouse.go
Normal file
257
pkg/clickhouse/clickhouse.go
Normal file
@@ -0,0 +1,257 @@
|
||||
package clickhouse
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"fiskerinc.com/modules/common"
|
||||
"fiskerinc.com/modules/utils/envtool"
|
||||
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
var (
|
||||
TIMEOUT = envtool.GetEnvInt("CLICKHOUSE_TIMEOUT", 1)
|
||||
MAX_CONNS = envtool.GetEnvInt("CLICKHOUSE_MAX_CONNS", 1)
|
||||
CLICKHOUSE_HOST = envtool.GetEnv("CLICKHOUSE_HOST", "localhost")
|
||||
CLICKHOUSE_PORT = envtool.GetEnv("CLICKHOUSE_PORT", "9000")
|
||||
CLICKHOUSE_DB = envtool.GetEnv("CLICKHOUSE_DB", "default")
|
||||
CLICKHOUSE_USER = envtool.GetEnv("CLICKHOUSE_USER", "")
|
||||
CLICKHOUSE_PASS = envtool.GetEnv("CLICKHOUSE_PASS", "")
|
||||
VEHICLE_FILTERS_TABLE = envtool.GetEnv("CLICKHOUSE_VEHICLE_FILTERS_TABLE", "can_filter_list_vin")
|
||||
DEFAULT_FILTERS_TABLE = envtool.GetEnv("CLICKHOUSE_DEFAULT_FILTERS_TABLE", "can_filter_list_all")
|
||||
)
|
||||
|
||||
func NewClient(conn ConnInterface) (ClientInterface, error) {
|
||||
return &Client{conn: conn}, nil
|
||||
}
|
||||
|
||||
func NewConn() (clickhouse.Conn, error) {
|
||||
return clickhouse.Open(&clickhouse.Options{
|
||||
Addr: []string{fmt.Sprintf("%s:%s", CLICKHOUSE_HOST, CLICKHOUSE_PORT)},
|
||||
Auth: clickhouse.Auth{
|
||||
Database: CLICKHOUSE_DB,
|
||||
Username: CLICKHOUSE_USER,
|
||||
Password: CLICKHOUSE_PASS,
|
||||
},
|
||||
DialTimeout: time.Second * 60,
|
||||
MaxOpenConns: MAX_CONNS,
|
||||
MaxIdleConns: MAX_CONNS,
|
||||
ConnMaxLifetime: 24 * time.Hour,
|
||||
Compression: &clickhouse.Compression{
|
||||
Method: clickhouse.CompressionLZ4,
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
type ClientInterface interface {
|
||||
Select(result interface{}, query string) error
|
||||
RetrieveDefaultFilters() ([]CANFilterSchema, error)
|
||||
RetrieveFiltersForVehicle(vin string) ([]CANFilterSchema, error)
|
||||
SaveDBCInfo(dbc common.DBCDesc) error
|
||||
SaveDBCMessages(ms []common.MessageDesc) error
|
||||
SaveDBCSignals(signals []common.SignalDesc) error
|
||||
SelectDBCsByVersions(versions []string) ([]string, error)
|
||||
SelectDBCSignals(dbc string, options PageQueryOptions) ([]common.SignalDescWithECU, int, error)
|
||||
TruncateDBCDescs() error
|
||||
|
||||
SetConn(conn ConnInterface)
|
||||
Exec(query string) error
|
||||
}
|
||||
|
||||
type ConnInterface interface {
|
||||
Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error
|
||||
PrepareBatch(ctx context.Context, query string) (driver.Batch, error)
|
||||
AsyncInsert(ctx context.Context, query string, wait bool) error
|
||||
QueryRow(ctx context.Context, query string, args ...interface{}) driver.Row
|
||||
Query(ctx context.Context, query string, args ...interface{}) (driver.Rows, error)
|
||||
Exec(ctx context.Context, query string, args ...interface{}) error
|
||||
}
|
||||
|
||||
type Client struct {
|
||||
conn ConnInterface
|
||||
}
|
||||
|
||||
func (c *Client) Select(result interface{}, query string) error {
|
||||
ctx := context.Background()
|
||||
|
||||
err := c.conn.Select(ctx, result, query)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) RetrieveDefaultFilters() ([]CANFilterSchema, error) {
|
||||
var result []CANFilterSchema
|
||||
|
||||
if err := c.Select(&result, fmt.Sprintf("SELECT ID, Period FROM %s", DEFAULT_FILTERS_TABLE)); err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Client) RetrieveFiltersForVehicle(vin string) ([]CANFilterSchema, error) {
|
||||
var result []CANFilterSchema
|
||||
|
||||
if err := c.Select(&result, fmt.Sprintf("SELECT ID, Period FROM %s WHERE VIN='%s'", VEHICLE_FILTERS_TABLE, vin)); err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Client) SelectDBCsByVersions(versions []string) ([]string, error) {
|
||||
if len(versions) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
q := fmt.Sprintf("'%s'", strings.Join(versions, "','"))
|
||||
|
||||
var result []string
|
||||
if err := c.Select(&result, fmt.Sprintf("SELECT dbc_name FROM dbcs where dbc_hash IN (%s)", q)); err != nil {
|
||||
return result, err
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (c *Client) TruncateDBCDescs() error {
|
||||
err := c.Exec("TRUNCATE TABLE dbc_signals_shard ON CLUSTER default")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = c.Exec("TRUNCATE TABLE dbc_messages_shard ON CLUSTER default")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = c.Exec("TRUNCATE TABLE dbcs_shard ON CLUSTER default")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) SaveDBCInfo(dbc common.DBCDesc) error {
|
||||
query := fmt.Sprintf(`INSERT INTO dbcs (dbc_hash, dbc_name) VALUES ('%s', '%s')`, dbc.Hash, dbc.Name)
|
||||
return errors.WithStack(c.conn.AsyncInsert(context.Background(), query, true))
|
||||
}
|
||||
|
||||
func (c *Client) SaveDBCMessages(ms []common.MessageDesc) error {
|
||||
batch, err := c.conn.PrepareBatch(context.Background(),
|
||||
`INSERT INTO dbc_messages (dbc_hash, message_name, message_id, is_extended,
|
||||
send_type, length, description, sender_node, cycle_time_ns, delay_time_ns, ecu_name)`)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
for _, m := range ms {
|
||||
err = batch.Append(m.DBCHash, m.Name, m.ID, m.IsExtended, m.SendType, m.Length, m.Description,
|
||||
m.SenderNode, m.CycleTime, m.DelayTime, m.ECUName)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
}
|
||||
|
||||
return errors.WithStack(batch.Send())
|
||||
}
|
||||
|
||||
func (c *Client) SaveDBCSignals(signals []common.SignalDesc) error {
|
||||
batch, err := c.conn.PrepareBatch(context.Background(),
|
||||
`INSERT INTO dbc_signals (
|
||||
dbc_hash, message_id, signal_name, start,
|
||||
length, big_endian, signed, multiplexer, multiplexed,
|
||||
multiplexer_value, offset, scale, min, max, unit,
|
||||
description, value_descriptions, receiver_nodes, default_value, ecu_name)`)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
for _, s := range signals {
|
||||
err = batch.Append(s.DBCHash, s.MessageID, s.Name, s.Start, s.Length, s.IsBigEndian, s.IsSigned,
|
||||
s.IsMultiplexer, s.IsMultiplexed, s.MultiplexerValue, s.Offset, s.Scale, s.Min, s.Max,
|
||||
s.Unit, s.Description, s.ValueDescriptions, s.ReceiverNodes, s.DefaultValue, s.ECUName)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
}
|
||||
|
||||
return batch.Send()
|
||||
}
|
||||
|
||||
func (c *Client) SelectDBCSignals(dbc string, options PageQueryOptions) ([]common.SignalDescWithECU, int, error) {
|
||||
var result []common.SignalDescWithECU
|
||||
chCtx := clickhouse.Context(
|
||||
context.Background(),
|
||||
clickhouse.WithParameters(clickhouse.Parameters{
|
||||
"dbc": dbc,
|
||||
|
||||
// we cannot use keywords like "offset" and "limit" as parameter names
|
||||
"lim": fmt.Sprint(options.Limit),
|
||||
"offs": fmt.Sprint(options.Offset),
|
||||
}))
|
||||
|
||||
query := CreateDBCSignalQuery(options)
|
||||
|
||||
if err := c.conn.Select(chCtx, &result, query); err != nil {
|
||||
return nil, 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
var count uint64
|
||||
if err := c.conn.QueryRow(chCtx, "SELECT COUNT() FROM dbc_signals a WHERE a.dbc_hash = {dbc:String}").Scan(&count); err != nil {
|
||||
return nil, 0, errors.WithStack(err)
|
||||
}
|
||||
|
||||
return result, int(count), nil
|
||||
}
|
||||
|
||||
func (c *Client) SetConn(conn ConnInterface) {
|
||||
c.conn = conn
|
||||
}
|
||||
|
||||
func (c *Client) Exec(query string) error {
|
||||
ctx := context.Background()
|
||||
|
||||
err := c.conn.Exec(ctx, query)
|
||||
if err != nil {
|
||||
return errors.WithStack(err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
type CANFilterSchema struct {
|
||||
ID int16 `ch:"ID"`
|
||||
Period int32 `ch:"Period"`
|
||||
}
|
||||
|
||||
type MigrationSchema struct {
|
||||
Version int64 `ch:"version"`
|
||||
Dirty uint8 `ch:"dirty"`
|
||||
Sequence uint64 `ch:"sequence"`
|
||||
}
|
||||
|
||||
func CreateDBCSignalQuery(options PageQueryOptions) string {
|
||||
|
||||
initQuery := `select * from dbc_signals where dbc_hash = {dbc:String}`
|
||||
|
||||
query := initQuery
|
||||
if options.Limit != 0 {
|
||||
query += ` LIMIT {lim:UInt64}`
|
||||
}
|
||||
|
||||
if options.Offset != 0 {
|
||||
query += ` OFFSET {offs:UInt64}`
|
||||
}
|
||||
|
||||
return query
|
||||
}
|
||||
Reference in New Issue
Block a user