Merge branch 'main' of github.com:k1LoW/tbls

.github/workflows/ci.yml

@@ -25,7 +25,7 @@ jobs:
       - name: Check out source code
         uses: actions/checkout@v4
 
-      - name: Set up Go on ${{ matrix.platform }}
+      - name: Set up Go
         uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
@@ -72,7 +72,7 @@ jobs:
     name: Test
     strategy:
       matrix:
-        platform: [windows-2019]
+        platform: [windows-2019, windows-latest]
     runs-on: ${{ matrix.platform }}
     steps:
       - name: Use LF
@@ -90,7 +90,7 @@ jobs:
       - name: Check out source code
         uses: actions/checkout@v4
 
-      - name: Set up Go on ${{ matrix.platform }}
+      - name: Set up Go
         uses: actions/setup-go@v4
         with:
           go-version-file: go.mod

.github/workflows/release-test.yml

@@ -6,6 +6,7 @@ on:
       - main
   pull_request:
     paths:
+      - .github/workflows/release-test.yml
       - .goreleaser/*.yml
 
 jobs:
@@ -29,6 +30,11 @@ jobs:
           brew install goreleaser
           brew install sqlite3
 
+      - name: Install
+        run: |
+          go install github.com/k1LoW/tbls@latest
+          tbls version
+
       - name: Release (dry-run)
         run: |
           goreleaser release --config .goreleaser/darwin.yml --clean --snapshot --skip-publish
@@ -49,11 +55,16 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Go on Ubuntu
+      - name: Set up Go
         uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
 
+      - name: Install
+        run: |
+          go install github.com/k1LoW/tbls@latest
+          tbls version
+
       - name: Run GoReleaser (dry-run)
         uses: goreleaser/goreleaser-action@v5
         with:
@@ -64,7 +75,10 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
   windows-release-test:
-    runs-on: windows-2019
+    strategy:
+      matrix:
+        os: [windows-2019, windows-latest]
+    runs-on: ${{ matrix.os }}
     steps:
       - name: Use LF
         run: |
@@ -83,11 +97,16 @@ jobs:
         with:
           fetch-depth: 0
 
-      - name: Set up Go on Windows
+      - name: Set up Go
         uses: actions/setup-go@v4
         with:
           go-version-file: go.mod
 
+      - name: Install
+        run: |
+          go install github.com/k1LoW/tbls@latest
+          tbls version
+
       - name: Run GoReleaser (dry-run)
         uses: goreleaser/goreleaser-action@v5
         with:

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## [v1.72.1](https://github.com/k1LoW/tbls/compare/v1.72.0...v1.72.1) - 2024-01-09
+### Other Changes
+- chore: unnecessary use of fmt.Sprintf by @testwill in https://github.com/k1LoW/tbls/pull/544
+- Add more build environment test by @k1LoW in https://github.com/k1LoW/tbls/pull/546
+- Bump golang.org/x/crypto from 0.14.0 to 0.17.0 by @dependabot in https://github.com/k1LoW/tbls/pull/547
+- Bump github.com/cloudflare/circl from 1.3.3 to 1.3.7 by @dependabot in https://github.com/k1LoW/tbls/pull/549
+- Bump github.com/dvsekhvalnov/jose2go from 1.5.0 to 1.6.0 by @dependabot in https://github.com/k1LoW/tbls/pull/550
+
+## [v1.72.0](https://github.com/k1LoW/tbls/compare/v1.71.1...v1.72.0) - 2023-11-23
+### New Features 🎉
+- feat: [MongoDB] Support multiple type field by @mrtc0 in https://github.com/k1LoW/tbls/pull/540
+
 ## [v1.71.1](https://github.com/k1LoW/tbls/compare/v1.71.0...v1.71.1) - 2023-11-07
 ### Fix bug 🐛
 - fix #535 by @majimaccho in https://github.com/k1LoW/tbls/pull/536

README.md

@@ -591,6 +591,13 @@ dsn: mongodb://mongoadmin:secret@localhost:27017/test
 dsn: mongodb://mongoadmin:secret@localhost:27017/test?sampleSize=20
 ```
 
+If a field has multiple types, the `multipleFieldType` query can be used to list all the types.
+
+``` yaml
+# .tbls.yml
+dsn: mongodb://mongoadmin:secret@localhost:27017/test?sampleSize=20&multipleFieldType=true
+```
+
 **JSON:**
 
 The JSON file output by the `tbls out -t json` command can be read as a datasource.

datasource/mongo.go

@@ -14,7 +14,10 @@ import (
 	"go.mongodb.org/mongo-driver/mongo/readpref"
 )
 
-const defaultSampleSize = 1000
+const (
+	defaultSampleSize        = 1000
+	defaultMultipleFieldType = false
+)
 
 // AnalyzeMongodb analyze `mongodb://`
 func AnalyzeMongodb(urlstr string) (*schema.Schema, error) {
@@ -50,7 +53,11 @@ func AnalyzeMongodb(urlstr string) (*schema.Schema, error) {
 	if err != nil {
 		sampleSize = defaultSampleSize
 	}
-	driver, err := mongodb.New(ctx, client, dbName, sampleSize)
+	multipleFieldType, err := strconv.ParseBool(values.Get("multipleFieldType"))
+	if err != nil {
+		multipleFieldType = defaultMultipleFieldType
+	}
+	driver, err := mongodb.New(ctx, client, dbName, sampleSize, multipleFieldType)
 	if err != nil {
 		return s, err
 	}

drivers/mongodb/mongodb.go

@@ -4,7 +4,9 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"slices"
 	"sort"
+	"strings"
 
 	"github.com/k1LoW/tbls/dict"
 	"github.com/k1LoW/tbls/schema"
@@ -14,19 +16,23 @@ import (
 	"go.mongodb.org/mongo-driver/mongo"
 )
 
+const columnTypeSeparator = ","
+
 type Mongodb struct {
-	ctx        context.Context
-	client     *mongo.Client
-	dbName     string
-	sampleSize int64
+	ctx               context.Context
+	client            *mongo.Client
+	dbName            string
+	sampleSize        int64
+	multipleFieldType bool
 }
 
-func New(ctx context.Context, client *mongo.Client, dbName string, sampleSize int64) (*Mongodb, error) {
+func New(ctx context.Context, client *mongo.Client, dbName string, sampleSize int64, multipleFieldType bool) (*Mongodb, error) {
 	return &Mongodb{
-		ctx:        ctx,
-		client:     client,
-		dbName:     dbName,
-		sampleSize: sampleSize,
+		ctx:               ctx,
+		client:            client,
+		dbName:            dbName,
+		sampleSize:        sampleSize,
+		multipleFieldType: multipleFieldType,
 	}, nil
 }
 
@@ -120,6 +126,10 @@ func (d *Mongodb) listFields(collection *mongo.Collection) ([]*schema.Column, er
 			if !columnInColumns(column, columns) {
 				columns = append(columns, column)
 			}
+
+			if d.multipleFieldType {
+				columns = addColumnType(columns, key, valueType)
+			}
 		}
 	}
 	for _, col := range columns {
@@ -145,6 +155,25 @@ func columnInColumns(a *schema.Column, list []*schema.Column) bool {
 	return false
 }
 
+// addColumnType adds a new type to the specified column
+func addColumnType(list []*schema.Column, columnName, valueType string) []*schema.Column {
+	columns := make([]*schema.Column, len(list))
+
+	for i, col := range list {
+		column := *col
+		if column.Name == columnName {
+			types := append(strings.Split(column.Type, columnTypeSeparator), valueType)
+			slices.Sort(types)
+			uniqTypes := slices.Compact(types)
+			column.Type = strings.Join(uniqTypes, columnTypeSeparator)
+		}
+
+		columns[i] = &column
+	}
+
+	return columns
+}
+
 func (d *Mongodb) listIndexes(collection *mongo.Collection) ([]*schema.Index, error) {
 	indexes := []*schema.Index{}
 	indexSpec, err := collection.Indexes().ListSpecifications(d.ctx)

drivers/mongodb/mongodb_test.go

@@ -5,6 +5,7 @@ package mongodb
 import (
 	"context"
 	"net/url"
+	"reflect"
 	"strings"
 	"testing"
 
@@ -38,7 +39,7 @@ func TestAnalyze(t *testing.T) {
 	s := &schema.Schema{
 		Name: "MongoDB local `docker-mongo-sample-datasets`",
 	}
-	driver, err := New(ctx, client, dbName, 10)
+	driver, err := New(ctx, client, dbName, 10, false)
 	if err != nil {
 		t.Errorf("%v", err)
 	}
@@ -55,3 +56,57 @@ func TestAnalyze(t *testing.T) {
 		t.Errorf("got not empty string.")
 	}
 }
+
+func Test_addColumnType(t *testing.T) {
+	columns := []*schema.Column{
+		{
+			Name: "username",
+			Type: "string",
+		},
+		{
+			Name: "age",
+			Type: "int",
+		},
+	}
+
+	tests := []struct {
+		name       string
+		list       []*schema.Column
+		columnName string
+		valueType  string
+		want       []*schema.Column
+	}{
+		{
+			name:       "Existing types are not added",
+			list:       columns,
+			columnName: "username",
+			valueType:  "string",
+			want:       columns,
+		},
+		{
+			name:       "New types are added with comma separation",
+			list:       columns,
+			columnName: "age",
+			valueType:  "string",
+			want: []*schema.Column{
+				{
+					Name: "username",
+					Type: "string",
+				},
+				{
+					Name: "age",
+					Type: "int,string",
+				},
+			},
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			got := addColumnType(test.list, test.columnName, test.valueType)
+			if !reflect.DeepEqual(got, test.want) {
+				t.Errorf("got %v\nwant %v", got, test.want)
+			}
+		})
+	}
+}

go.mod

@@ -75,11 +75,11 @@ require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/cli/go-gh/v2 v2.3.0 // indirect
 	github.com/cli/safeexec v1.0.1 // indirect
-	github.com/cloudflare/circl v1.3.3 // indirect
+	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe // indirect
 	github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 // indirect
 	github.com/danieljoos/wincred v1.2.0 // indirect
-	github.com/dvsekhvalnov/jose2go v1.5.0 // indirect
+	github.com/dvsekhvalnov/jose2go v1.6.0 // indirect
 	github.com/envoyproxy/go-control-plane v0.11.1 // indirect
 	github.com/envoyproxy/protoc-gen-validate v1.0.2 // indirect
 	github.com/fatih/color v1.15.0 // indirect
@@ -129,14 +129,14 @@ require (
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
 	github.com/zeebo/xxh3 v1.0.2 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/crypto v0.17.0 // indirect
 	golang.org/x/exp v0.0.0-20231006140011-7918f672742d // indirect
 	golang.org/x/mod v0.13.0 // indirect
 	golang.org/x/net v0.17.0 // indirect
 	golang.org/x/sync v0.4.0 // indirect
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/term v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/term v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/tools v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.8 // indirect