From 7f92d938b49b65bfddf491b0e0e21d0274704bfc Mon Sep 17 00:00:00 2001 From: Raj Singh Date: Sat, 12 Oct 2024 00:23:10 -0500 Subject: [PATCH] Add clickhouse support --- backend/go.mod | 19 +- backend/go.sum | 66 +++++- backend/main.go | 345 +++++++++++++++++++++++++++--- docker-compose.yaml | 38 ++++ examples/config.yaml | 12 +- k8s/fleet-manager/configmap.yaml | 11 +- k8s/fleet-manager/deployment.yaml | 12 +- 7 files changed, 461 insertions(+), 42 deletions(-) create mode 100644 docker-compose.yaml diff --git a/backend/go.mod b/backend/go.mod index 3f845f9..58d41f1 100644 --- a/backend/go.mod +++ b/backend/go.mod @@ -12,6 +12,23 @@ require ( ) require ( + github.com/ClickHouse/ch-go v0.61.5 // indirect + github.com/andybalholm/brotli v1.1.0 // indirect + github.com/go-faster/city v1.0.1 // indirect + github.com/go-faster/errors v0.7.1 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/klauspost/compress v1.17.7 // indirect + github.com/paulmach/orb v0.11.1 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect + github.com/pkg/errors v0.9.1 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/shopspring/decimal v1.4.0 // indirect + go.opentelemetry.io/otel v1.26.0 // indirect + go.opentelemetry.io/otel/trace v1.26.0 // indirect +) + +require ( + github.com/ClickHouse/clickhouse-go/v2 v2.29.0 github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -20,5 +37,5 @@ require ( github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - golang.org/x/sys v0.18.0 // indirect + golang.org/x/sys v0.23.0 // indirect ) diff --git a/backend/go.sum b/backend/go.sum index c942443..8f73763 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -1,9 +1,15 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/ClickHouse/ch-go v0.61.5 h1:zwR8QbYI0tsMiEcze/uIMK+Tz1D3XZXLdNrlaOpeEI4= +github.com/ClickHouse/ch-go v0.61.5/go.mod h1:s1LJW/F/LcFs5HJnuogFMta50kKDO0lf9zzfrbl0RQg= +github.com/ClickHouse/clickhouse-go/v2 v2.29.0 h1:Dj1w59RssRyLgGHXtYaWU0eIM1pJsu9nGPi/btmvAqw= +github.com/ClickHouse/clickhouse-go/v2 v2.29.0/go.mod h1:bLookq6qZJ4Ush/6tOAnJGh1Sf3Sa/nQoMn71p7ZCUE= github.com/actgardner/gogen-avro/v10 v10.1.0/go.mod h1:o+ybmVjEa27AAr35FRqU98DJu1fXES56uXniYFv4yDA= github.com/actgardner/gogen-avro/v10 v10.2.1/go.mod h1:QUhjeHPchheYmMDni/Nx7VB0RsT/ee8YIgGY/xpEQgQ= github.com/actgardner/gogen-avro/v9 v9.1.0/go.mod h1:nyTj6wPqDJoxM3qdnjcLv+EnMDSDFqE0qDpva2QRmKc= +github.com/andybalholm/brotli v1.1.0 h1:eLKJA0d02Lf0mVpIDgYnqXcUn0GqVmEFny3VuID1U3M= +github.com/andybalholm/brotli v1.1.0/go.mod h1:sms7XGricyQI9K10gOSf56VKKWS4oLer58Q+mhRPtnY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU= github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= @@ -42,6 +48,11 @@ github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03D github.com/frankban/quicktest v1.10.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-faster/city v1.0.1 h1:4WAxSZ3V2Ws4QRDrscLEDcibJY8uf41H6AhXDrNDcGw= +github.com/go-faster/city v1.0.1/go.mod h1:jKcUJId49qdW3L1qKHH/3wPeUstCVpVSXTM6vO3VcTw= +github.com/go-faster/errors v0.7.1 h1:MkJTnDoEdi9pDabt1dpWf7AA8/BaSYZqibYyhZ20AYg= +github.com/go-faster/errors v0.7.1/go.mod h1:5ySTjWFiphBs07IKuiL69nxdfd5+fzh1u7FPGZP2quo= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -69,14 +80,17 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20211008130755-947d60d73cc0/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hamba/avro v1.5.6/go.mod h1:3vNT0RLXXpFm2Tb/5KC71ZRJlOroggq1Rcitb6k4Fr8= github.com/heetch/avro v0.3.1/go.mod h1:4xn38Oz/+hiEUTpbVfGVLfvOg0yKLlRP7Q9+gJJILgA= @@ -95,6 +109,11 @@ github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfC github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/juju/qthttptest v0.1.1/go.mod h1:aTlAv8TYaflIiTDIQYzxnl1QdPjAg8Q8qJMErpKy6A4= github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= +github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= @@ -114,8 +133,16 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc= github.com/nrwiersma/avro-benchmarks v0.0.0-20210913175520-21aec48c8f76/go.mod h1:iKyFMidsk/sVYONJRE372sJuX/QTRPacU7imPqqsu7g= +github.com/paulmach/orb v0.11.1 h1:3koVegMC4X/WeiXYz9iswopaTwMem53NzTJuTF20JzU= +github.com/paulmach/orb v0.11.1/go.mod h1:5mULz1xQfs3bmQm63QEJA6lNGujuRafwA5S/EnuLaLU= +github.com/paulmach/protoscan v0.2.1/go.mod h1:SpcSwydNLrxUGSDvXvO0P7g7AuhJ7lcKfDlhJCDw2gY= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_golang v1.15.0 h1:5fCgGYogn0hFdhyhLbw7hEsWxufKtY9klyvdNfFlFhM= @@ -130,27 +157,48 @@ github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB github.com/rogpeppe/clock v0.0.0-20190514195947-2896927a307a/go.mod h1:4r5QyqhjIWCcK8DO4KMclc5Iknq5qVBAlbYYzAbUScQ= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/santhosh-tekuri/jsonschema/v5 v5.0.0/go.mod h1:FKdcjfQW6rpZSnxxUvEA5H/cDPdvJ/SZJQLWWXWGrZ0= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shopspring/decimal v1.4.0 h1:bxl37RwXBklmTi0C79JfXCEBD1cqqHt0bbgBAGFp81k= +github.com/shopspring/decimal v1.4.0/go.mod h1:gawqmDU56v4yIKSwfBSFip1HdCCXN8/+DMd9qYNcwME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/teslamotors/fleet-telemetry v0.3.2 h1:82LUYoT5O8GUO2W/p2rxI4flLBUpdKcffy0dD+rYopQ= github.com/teslamotors/fleet-telemetry v0.3.2/go.mod h1:5VdIlMJw9K6O6WLoOiuiiudB0jeGdSjEm0vZGHCOqY8= +github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI= +github.com/xdg-go/scram v1.1.1/go.mod h1:RaEWvsqvNKKvBPvcKeFjrG2cJqOkHTiyTpzz23ni57g= +github.com/xdg-go/stringprep v1.0.3/go.mod h1:W3f5j4i+9rC0kuIEJL0ky1VpHXQU3ocBgklLGvcBnW8= +github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.mongodb.org/mongo-driver v1.11.4/go.mod h1:PTSz5yu21bkT/wXpkS7WR5f0ddqw5quethTUn9WM+2g= +go.opentelemetry.io/otel v1.26.0 h1:LQwgL5s/1W7YiiRwxf03QGnWLb2HW4pLiAhaA5cZXBs= +go.opentelemetry.io/otel v1.26.0/go.mod h1:UmLkJHUAidDval2EICqBMbnAd0/m2vmpf/dAM+fvFs4= +go.opentelemetry.io/otel/trace v1.26.0 h1:1ieeAUb4y0TE26jUFrCIXKpTuVK7uJGN9/Z/2LP5sQA= +go.opentelemetry.io/otel/trace v1.26.0/go.mod h1:4iDxvGDQuUkHve82hJJ8UqrwswHYsZuWCBllGV2U2y0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -164,6 +212,7 @@ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -171,6 +220,7 @@ golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -180,15 +230,19 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.23.0 h1:YfKFowiIMvtgl1UERQoTPPToxltDeZfbj4H7dVUCwmM= +golang.org/x/sys v0.23.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -196,6 +250,8 @@ golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/backend/main.go b/backend/main.go index 79cf957..7445019 100644 --- a/backend/main.go +++ b/backend/main.go @@ -3,14 +3,17 @@ package main import ( "bytes" "context" + "crypto/tls" "flag" "fmt" + "io" "log" "math" "os" "os/signal" "path/filepath" "strconv" + "strings" "sync" "syscall" "time" @@ -22,8 +25,10 @@ import ( "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/confluentinc/confluent-kafka-go/kafka" + "github.com/ClickHouse/clickhouse-go/v2" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" "github.com/teslamotors/fleet-telemetry/protos" "net/http" @@ -31,9 +36,10 @@ import ( // Config holds the entire configuration structure type Config struct { - Kafka KafkaConfig `yaml:"kafka"` - AWS *S3Config `yaml:"aws,omitempty"` - Local *LocalConfig `yaml:"local,omitempty"` + Kafka KafkaConfig `yaml:"kafka"` + AWS *S3Config `yaml:"aws,omitempty"` + Local *LocalConfig `yaml:"local,omitempty"` + ClickHouse *ClickHouseConfig `yaml:"clickhouse,omitempty"` } // S3Config holds AWS S3 configuration @@ -59,16 +65,31 @@ type KafkaConfig struct { Topic string `yaml:"topic"` } +// ClickHouseConfig holds ClickHouse database configuration +type ClickHouseConfig struct { + Enabled bool `yaml:"enabled"` + Host string `yaml:"host"` + Port int `yaml:"port"` + Database string `yaml:"database"` + Username string `yaml:"username"` + Password string `yaml:"password"` + Secure bool `yaml:"secure"` + TableName string `yaml:"table_name,omitempty"` // Optional: specify a custom table name +} + // Service encapsulates the application's dependencies type Service struct { - Config Config - S3Client *s3.S3 - LocalBackupEnabled bool - LocalBasePath string - KafkaConsumer *kafka.Consumer - PrometheusGauge *prometheus.GaugeVec - PrometheusLatitude *prometheus.GaugeVec - PrometheusLongitude *prometheus.GaugeVec + Config Config + S3Client *s3.S3 + LocalBackupEnabled bool + LocalBasePath string + KafkaConsumer *kafka.Consumer + PrometheusGauge *prometheus.GaugeVec + PrometheusLatitude *prometheus.GaugeVec + PrometheusLongitude *prometheus.GaugeVec + ClickHouseClient clickhouse.Conn + ClickHouseEnabled bool + ClickHouseTableName string } // NewService initializes the service with configurations @@ -95,6 +116,35 @@ func NewService(cfg Config) (*Service, error) { log.Println("AWS S3 configuration not provided. S3 uploads are disabled.") } + // Initialize ClickHouse if configuration is provided and enabled + if service.Config.ClickHouse != nil && service.Config.ClickHouse.Enabled { + chClient, err := configureClickHouse(service.Config.ClickHouse) + if err != nil { + return nil, fmt.Errorf("failed to configure ClickHouse: %w", err) + } + service.ClickHouseClient = chClient + service.ClickHouseEnabled = true + + if err := testClickHouseConnection(chClient); err != nil { + return nil, fmt.Errorf("ClickHouse connection test failed: %w", err) + } + log.Println("ClickHouse connection established successfully.") + + // Set ClickHouse table name, defaulting if not provided + if service.Config.ClickHouse.TableName != "" { + service.ClickHouseTableName = service.Config.ClickHouse.TableName + } else { + service.ClickHouseTableName = "vehicle_data" + } + + // Load existing data into ClickHouse + if err := loadExistingDataIntoClickHouse(service); err != nil { + return nil, fmt.Errorf("failed to load existing data into ClickHouse: %w", err) + } + } else { + log.Println("ClickHouse integration is disabled or not configured.") + } + // Initialize Kafka consumer consumer, err := configureKafka(service.Config.Kafka) if err != nil { @@ -177,6 +227,192 @@ func testS3Connection(s3Svc *s3.S3, bucket string) error { return nil } +// configureClickHouse sets up the ClickHouse client +func configureClickHouse(cfg *ClickHouseConfig) (clickhouse.Conn, error) { + conn, err := clickhouse.Open(&clickhouse.Options{ + Addr: []string{ + fmt.Sprintf("%s:%d", cfg.Host, cfg.Port), + }, + Auth: clickhouse.Auth{ + Database: cfg.Database, + Username: cfg.Username, + Password: cfg.Password, + }, + TLS: &tls.Config{ + InsecureSkipVerify: !cfg.Secure, + }, + // Additional options as needed + }) + if err != nil { + return nil, fmt.Errorf("unable to connect to ClickHouse: %w", err) + } + return conn, nil +} + +// testClickHouseConnection verifies the connection to ClickHouse +func testClickHouseConnection(conn clickhouse.Conn) error { + return conn.Ping(context.Background()) +} + +// loadExistingDataIntoClickHouse loads existing JSON data from S3 or local storage into ClickHouse +func loadExistingDataIntoClickHouse(service *Service) error { + log.Println("Loading existing data from storage into ClickHouse...") + var files []string + var err error + + if service.S3Client != nil { + files, err = listS3JSONFiles(service.S3Client, service.Config.AWS.Bucket) + if err != nil { + return fmt.Errorf("failed to list JSON files in S3: %w", err) + } + } else if service.LocalBackupEnabled { + files, err = listLocalJSONFiles(service.LocalBasePath) + if err != nil { + return fmt.Errorf("failed to list JSON files locally: %w", err) + } + } else { + return fmt.Errorf("no storage (S3 or local) configured to load data from") + } + + for _, file := range files { + var data []byte + if service.S3Client != nil { + data, err = downloadS3JSONFile(service.S3Client, service.Config.AWS.Bucket, file) + if err != nil { + log.Printf("Failed to download S3 file '%s': %v", file, err) + continue + } + } else if service.LocalBackupEnabled { + data, err = os.ReadFile(file) + if err != nil { + log.Printf("Failed to read local file '%s': %v", file, err) + continue + } + } + + vehicleData := &protos.Payload{} + if err := protojson.Unmarshal(data, vehicleData); err != nil { + log.Printf("Failed to unmarshal JSON data from file '%s': %v", file, err) + continue + } + + if err := insertIntoClickHouse(service.ClickHouseClient, service.ClickHouseTableName, vehicleData); err != nil { + log.Printf("Failed to insert data from file '%s' into ClickHouse: %v", file, err) + continue + } + + log.Printf("Successfully loaded data from '%s' into ClickHouse.", file) + } + + log.Println("Completed loading existing data into ClickHouse.") + return nil +} + +// listS3JSONFiles lists all JSON files in the specified S3 bucket +func listS3JSONFiles(s3Svc *s3.S3, bucket string) ([]string, error) { + var files []string + input := &s3.ListObjectsV2Input{ + Bucket: aws.String(bucket), + Prefix: aws.String(""), + } + + err := s3Svc.ListObjectsV2Pages(input, func(page *s3.ListObjectsV2Output, lastPage bool) bool { + for _, obj := range page.Contents { + if strings.HasSuffix(*obj.Key, ".json") { + files = append(files, *obj.Key) + } + } + return !lastPage + }) + + if err != nil { + return nil, err + } + return files, nil +} + +// listLocalJSONFiles lists all JSON files in the specified local directory +func listLocalJSONFiles(basePath string) ([]string, error) { + var files []string + err := filepath.Walk(basePath, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() && strings.HasSuffix(info.Name(), ".json") { + files = append(files, path) + } + return nil + }) + if err != nil { + return nil, err + } + return files, nil +} + +// downloadS3JSONFile downloads a JSON file from S3 +func downloadS3JSONFile(s3Svc *s3.S3, bucket, key string) ([]byte, error) { + output, err := s3Svc.GetObject(&s3.GetObjectInput{ + Bucket: aws.String(bucket), + Key: aws.String(key), + }) + if err != nil { + return nil, err + } + defer output.Body.Close() + + data, err := io.ReadAll(output.Body) + if err != nil { + return nil, err + } + return data, nil +} + +// insertIntoClickHouse inserts vehicle data into ClickHouse +func insertIntoClickHouse(conn clickhouse.Conn, tableName string, data *protos.Payload) error { + ctx := context.Background() + + // Ensure the target table exists. This is optional and can be handled separately. + // Uncomment the following lines if you want the application to create the table automatically. + createTableQuery := fmt.Sprintf(` + CREATE TABLE IF NOT EXISTS %s ( + vin String, + timestamp DateTime, + data String + ) ENGINE = MergeTree() + ORDER BY (vin, timestamp) + `, tableName) + if err := conn.Exec(ctx, createTableQuery); err != nil { + return fmt.Errorf("failed to create ClickHouse table: %w", err) + } + + // Prepare the batch insert + batch, err := conn.PrepareBatch(ctx, fmt.Sprintf("INSERT INTO %s (vin, timestamp, data)", tableName)) + if err != nil { + return fmt.Errorf("failed to prepare ClickHouse batch: %w", err) + } + + // Serialize the data to JSON string + jsonData, err := protojson.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal data to JSON: %w", err) + } + + // Use the current UTC time as the timestamp + timestamp := time.Now().UTC() + + // Append data to the batch + if err := batch.Append(data.Vin, timestamp, string(jsonData)); err != nil { + return fmt.Errorf("failed to append data to ClickHouse batch: %w", err) + } + + // Send the batch to ClickHouse + if err := batch.Send(); err != nil { + return fmt.Errorf("failed to send ClickHouse batch: %w", err) + } + + return nil +} + // configureKafka sets up the Kafka consumer func configureKafka(kafkaCfg KafkaConfig) (*kafka.Consumer, error) { consumerConfig := &kafka.ConfigMap{ @@ -217,29 +453,49 @@ func loadConfig(path string) (Config, error) { return cfg, fmt.Errorf("incomplete Kafka configuration") } + // Optionally, validate ClickHouse configuration if provided and enabled + if cfg.ClickHouse != nil && cfg.ClickHouse.Enabled { + if cfg.ClickHouse.Host == "" || cfg.ClickHouse.Port == 0 || cfg.ClickHouse.Database == "" || + cfg.ClickHouse.Username == "" || cfg.ClickHouse.Password == "" { + return cfg, fmt.Errorf("incomplete ClickHouse configuration") + } + } + + // Optionally, validate AWS configuration if provided + if cfg.AWS != nil { + if err := validateS3Config(cfg.AWS); err != nil { + return cfg, fmt.Errorf("invalid AWS configuration: %w", err) + } + } + return cfg, nil } -// uploadToS3 uploads Protobuf data to the specified S3 bucket with a vin/year/month/day key structure -func uploadToS3(s3Svc *s3.S3, bucket, vin string, data []byte) error { +// uploadToS3 uploads Protobuf data as JSON to the specified S3 bucket with a vin/year/month/day key structure +func uploadToS3(s3Svc *s3.S3, bucket, vin string, data *protos.Payload) error { now := time.Now().UTC() - key := fmt.Sprintf("%s/%04d/%02d/%02d/%04d%02d%02dT%02d%02d%02d.%06dZ.protobuf", + key := fmt.Sprintf("%s/%04d/%02d/%02d/%04d%02d%02dT%02d%02d%02d.%06dZ.json", vin, now.Year(), - now.Month(), + int(now.Month()), now.Day(), - now.Year(), now.Month(), now.Day(), + now.Year(), int(now.Month()), now.Day(), now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/1000, ) + jsonData, err := protojson.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal data to JSON: %w", err) + } + input := &s3.PutObjectInput{ Bucket: aws.String(bucket), Key: aws.String(key), - Body: bytes.NewReader(data), - ContentType: aws.String("application/x-protobuf"), + Body: bytes.NewReader(jsonData), + ContentType: aws.String("application/json"), } - _, err := s3Svc.PutObject(input) + _, err = s3Svc.PutObject(input) if err != nil { return fmt.Errorf("failed to upload to S3 at key '%s': %w", key, err) } @@ -248,25 +504,32 @@ func uploadToS3(s3Svc *s3.S3, bucket, vin string, data []byte) error { return nil } -// backupLocally saves Protobuf data to the local filesystem with a vin/year/month/day folder structure -func backupLocally(basePath, vin string, data []byte) error { +// backupLocally saves Protobuf data as JSON to the local filesystem with a vin/year/month/day folder structure +func backupLocally(basePath, vin string, data *protos.Payload) error { now := time.Now().UTC() dirPath := filepath.Join(basePath, vin, fmt.Sprintf("%04d", now.Year()), - fmt.Sprintf("%02d", now.Month()), + fmt.Sprintf("%02d", int(now.Month())), fmt.Sprintf("%02d", now.Day()), ) if err := os.MkdirAll(dirPath, os.ModePerm); err != nil { return fmt.Errorf("failed to create directories '%s': %w", dirPath, err) } - fileName := fmt.Sprintf("%04d%02d%02dT%02d%02d%02d.%06dZ.protobuf", - now.Year(), now.Month(), now.Day(), + fileName := fmt.Sprintf("%04d%02d%02dT%02d%02d%02d.%06dZ.json", + now.Year(), int(now.Month()), now.Day(), now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/1000) filePath := filepath.Join(dirPath, fileName) - if err := os.WriteFile(filePath, data, 0644); err != nil { + + // Serialize to JSON + jsonData, err := protojson.Marshal(data) + if err != nil { + return fmt.Errorf("failed to marshal data to JSON: %w", err) + } + + if err := os.WriteFile(filePath, jsonData, 0644); err != nil { return fmt.Errorf("failed to write file '%s': %w", filePath, err) } @@ -466,27 +729,34 @@ func startConsumerLoop(service *Service, ctx context.Context, wg *sync.WaitGroup processValue(datum, service, vehicleData.Vin) } - // Serialize data as Protobuf for storage - serializedData, err := proto.Marshal(vehicleData) - if err != nil { - log.Printf("Failed to marshal vehicleData to Protobuf: %v", err) - continue - } + // // Serialize data as JSON for storage + // jsonData, err := protojson.Marshal(vehicleData) + // if err != nil { + // log.Printf("Failed to marshal vehicleData to JSON: %v", err) + // continue + // } // Upload to S3 if enabled if service.S3Client != nil { - if err := uploadToS3(service.S3Client, service.Config.AWS.Bucket, vehicleData.Vin, serializedData); err != nil { + if err := uploadToS3(service.S3Client, service.Config.AWS.Bucket, vehicleData.Vin, vehicleData); err != nil { log.Printf("Failed to upload vehicle data to S3: %v", err) } } // Backup locally if enabled if service.LocalBackupEnabled { - if err := backupLocally(service.LocalBasePath, vehicleData.Vin, serializedData); err != nil { + if err := backupLocally(service.LocalBasePath, vehicleData.Vin, vehicleData); err != nil { log.Printf("Failed to backup vehicle data locally: %v", err) } } + // Insert into ClickHouse if enabled + if service.ClickHouseEnabled { + if err := insertIntoClickHouse(service.ClickHouseClient, service.ClickHouseTableName, vehicleData); err != nil { + log.Printf("Failed to insert vehicle data into ClickHouse: %v", err) + } + } + // Commit the message offset after successful processing if _, err := service.KafkaConsumer.CommitMessage(msg); err != nil { log.Printf("Failed to commit Kafka message: %v", err) @@ -544,6 +814,15 @@ func main() { log.Println("Kafka consumer closed successfully.") } + // Close ClickHouse connection if initialized + if service.ClickHouseEnabled { + if err := service.ClickHouseClient.Close(); err != nil { + log.Printf("Error closing ClickHouse connection: %v", err) + } else { + log.Println("ClickHouse connection closed successfully.") + } + } + // Wait for all goroutines to finish wg.Wait() diff --git a/docker-compose.yaml b/docker-compose.yaml new file mode 100644 index 0000000..92f8207 --- /dev/null +++ b/docker-compose.yaml @@ -0,0 +1,38 @@ +version: '3.8' + +services: + frontend: + build: + context: ./frontend + container_name: fleet-manager-frontend + ports: + - "8501:8501" # Exposes port 8501 + volumes: + - ./examples/config.yaml:/etc/fleet-manager/config.yaml # Mounts config + environment: + CONFIG_PATH: /etc/fleet-manager/config.yaml # Example environment variable + depends_on: + - backend # Ensures backend starts before frontend + + backend: + build: + context: ./backend + container_name: fleet-manager-backend + command: ["/fleet-manager", "-config", "/etc/fleet-manager/config.yaml"] # Overrides the default command + ports: + - "2112:2112" # Exposes port 2112 for metrics + volumes: + - ./examples/config.yaml:/etc/fleet-manager/config.yaml # Mounts config + depends_on: + - clickhouse # Ensures ClickHouse starts before backend + + clickhouse: + image: bitnami/clickhouse:latest + container_name: clickhouse + ports: + - "8123:8123" # HTTP Interface + - "9000:9000" # Native TCP Interface + - "9009:9009" # Inter-server Communication + environment: + CLICKHOUSE_ADMIN_USER: "USERNAME" # Replace with your admin username + CLICKHOUSE_ADMIN_PASSWORD: "PASSWORD" # Replace with your admin password \ No newline at end of file diff --git a/examples/config.yaml b/examples/config.yaml index ae44bbd..941dcab 100644 --- a/examples/config.yaml +++ b/examples/config.yaml @@ -14,4 +14,14 @@ aws: local: enabled: true - basePath: "./data" \ No newline at end of file + basePath: "./data" + +clickhouse: + enabled: true + host: "localhost" + port: 9000 + database: "fleet_telemetry" + username: "USERNAME" + password: "PASSWORD" + secure: false # Set to true if using TLS + table_name: "vehicle_data" # Optional: specify a custom table name \ No newline at end of file diff --git a/k8s/fleet-manager/configmap.yaml b/k8s/fleet-manager/configmap.yaml index e6bb7bc..a949d9a 100644 --- a/k8s/fleet-manager/configmap.yaml +++ b/k8s/fleet-manager/configmap.yaml @@ -18,4 +18,13 @@ data: region: "us-west-1" local: enabled: true - basePath: "./data" \ No newline at end of file + basePath: "./data" + clickhouse: + enabled: true + host: "localhost" + port: 9000 + database: "fleet_telemetry" + username: "USERNAME" + password: "PASSWORD" + secure: false # Set to true if using TLS + table_name: "vehicle_data" # Optional: specify a custom table name \ No newline at end of file diff --git a/k8s/fleet-manager/deployment.yaml b/k8s/fleet-manager/deployment.yaml index 4677c35..e60ac7e 100644 --- a/k8s/fleet-manager/deployment.yaml +++ b/k8s/fleet-manager/deployment.yaml @@ -25,7 +25,6 @@ spec: ports: - containerPort: 8501 name: http - - name: backend image: quay.io/rajsinghcpre/fleet-manager-backend:v0.0.1 command: ["/fleet-manager", "-config", "/etc/fleet-manager/config.yaml"] @@ -37,6 +36,17 @@ spec: ports: - containerPort: 2112 name: metrics + - name: clickhouse + image: bitnami/clickhouse:latest + ports: + - containerPort: 8123 # HTTP Interface + - containerPort: 9000 # Native TCP Interface + - containerPort: 9009 # Inter-server Communication + env: + - name: CLICKHOUSE_ADMIN_USER + value: "USERNAME" + - name: CLICKHOUSE_ADMIN_PASSWORD + value: "PASSWORD" volumes: - name: config-volume configMap: