Merge branch 'main' into haris/relaxed-plugins

.github/ISSUE_TEMPLATE/4-conduit-release.yml

@@ -0,0 +1,115 @@
+name: 🚢 Conduit release checklist
+description: Use this template to guide you through the Conduit release process.
+title: '[Release] Conduit vX.Y.Z'
+labels:
+  - release
+assignees: []
+
+body:
+  - type: markdown
+    attributes:
+      value: |
+        # Conduit Release Checklist
+
+        This issue serves as a checklist for releasing a new version of Conduit. Follow the steps below to ensure a smooth release process.
+
+  - type: markdown
+    attributes:
+      value: |
+        ## General Information
+
+        A Conduit release includes:
+        - A GitHub release with packages for different OS and architectures, checksums, a changelog, and source code.
+        - A GitHub package for the official Docker image, available on GitHub's Container Registry, tagged with `latest`.
+
+  - type: markdown
+    attributes:
+      value: |
+        ## Before a Release
+
+        ### Update Dependencies
+
+        Update dependencies in the following order, ensuring all repositories are cloned in the same directory:
+
+  - type: checkboxes
+    attributes:
+      label: Update Dependencies
+      options:
+        -  label: '**conduit-commons**: Run `scripts/get-compare-link.sh ../conduit-commons/` to compare the latest tag and `main` branch. If changes are needed, push a new tag.'
+        -  label: '**conduit-connector-protocol**: Update `conduit-commons` if necessary: `go get github.com/conduitio/conduit-commons@vX.Y.Z`. Run `scripts/get-compare-link.sh ../conduit-connector-protocol/` and tag if needed.'
+        -  label: '**conduit-connector-sdk**: Update dependencies (`conduit-commons`, `conduit-connector-protocol`) as needed. Run `scripts/get-compare-link.sh ../conduit-connector-sdk/` and tag if needed.'
+        -  label: '**conduit-processor-sdk**: Update `conduit-commons` if necessary. Run `scripts/get-compare-link.sh ../conduit-processor-sdk/` and tag if needed.'
+        -  label: '**conduit-schema-registry**: Update `conduit-commons` if necessary. Run `scripts/get-compare-link.sh ../conduit-schema-registry/` and tag if needed.'
+        -  label: '**Connector SDK in conduit-connector-template**: Bump the Connector SDK dependency.'
+        -  label: '**conduit-connector-file**: Bump the Connector SDK `scripts/bump-sdk-in-connectors.sh vX.Y.Z`. Run `scripts/get-compare-link.sh ../conduit-connector-file/` and tag if needed.'
+        -  label: '**conduit-connector-kafka**: Bump the Connector SDK `scripts/bump-sdk-in-connectors.sh vX.Y.Z`. Run `scripts/get-compare-link.sh ../conduit-connector-kafka/` and tag if needed.'
+        -  label: '**conduit-connector-generator**: Bump the Connector SDK `scripts/bump-sdk-in-connectors.sh vX.Y.Z`. Run `scripts/get-compare-link.sh ../conduit-connector-generator/` and tag if needed.'
+        -  label: '**conduit-connector-s3**: Bump the Connector SDK `scripts/bump-sdk-in-connectors.sh vX.Y.Z`. Run `scripts/get-compare-link.sh ../conduit-connector-s3/` and tag if needed.'
+        -  label: '**conduit-connector-postgres**: Bump the Connector SDK `scripts/bump-sdk-in-connectors.sh vX.Y.Z`. Run `scripts/get-compare-link.sh ../conduit-connector-postgres/` and tag if needed.'
+        -  label: '**conduit-connector-log**: Bump the Connector SDK `scripts/bump-sdk-in-connectors.sh vX.Y.Z`. Run `scripts/get-compare-link.sh ../conduit-connector-log/` and tag if needed.'
+        -  label: '**Bump built-in connectors on Conduit**: Run `scripts/bump-builtin-connectors.sh`'
+        -  label: '**Release Conduit** (see instructions below).'
+
+
+  - type: markdown
+    attributes:
+      value: |
+        ## Documentation
+
+  - type: checkboxes
+    attributes:
+      label: Documentation
+      options:
+        - label: 'Write a blog post.'
+        - label: 'Regenerate processor documentation on [conduit-site](https://github.com/ConduitIO/conduit-site).'
+        - label: 'Update the banner on the website.'
+        - label: 'Search and replace the latest version in `conduit-site`.'
+        - label: 'Search and replace the latest version in [README.md](../README.md).'
+
+  - type: markdown
+    attributes:
+      value: |
+        ## Releasing Conduit
+
+  - type: checkboxes
+    attributes:
+      label: Releasing Conduit
+      options:
+        - label: 'Trigger a release by pushing a new tag starting with `v` (e.g., `v1.2.3`).'
+        - label: 'Use the script [scripts/tag.sh](https://github.com/ConduitIO/conduit/blob/main/scripts/tag.sh) to ensure version conformity.'
+
+  - type: markdown
+    attributes:
+      value: |
+        ```sh
+        scripts/tag.sh 1.2.3
+        ```
+
+  - type: markdown
+    attributes:
+      value: |
+        ## Nightly Builds
+
+        - Nightly builds (binaries and Docker images) are provided and kept for 7 days.
+        - The latest nightly Docker image is tagged with `latest-nightly`.
+
+  - type: markdown
+    attributes:
+      value: |
+        ## Implementation
+
+        - The GitHub release is created with [GoReleaser](https://github.com/goreleaser/goreleaser/).
+        - Nightly builds are triggered by a GitHub action, defined in [trigger-nightly.yml](/.github/workflows/trigger-nightly.yml).
+
+  - type: markdown
+    attributes:
+      value: |
+        ## Notes
+
+        - The "Trigger nightly build" GitHub action requires a personal access token, not the GitHub token provided by Actions.
+
+        For more information, refer to [Triggering a workflow from a workflow](https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow).
+
+---
+
+Please ensure each step is completed before closing this issue.

.github/ISSUE_TEMPLATE/config.yml

@@ -6,3 +6,6 @@ contact_links:
   - name: 📄 Documentation Issue
     url: https://github.com/ConduitIO/conduit-site
     about: Please report issues regarding documentation here.
+  - name: 💬 Chat with us
+    url: https://discord.com/channels/828680256877363200/933725823813451797
+    about: Join our Discord server to chat with us.

.github/workflows/test.yml

@@ -10,11 +10,20 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Shallow clones should be disabled for a better relevancy of analysis
 
       - name: Set up Go
         uses: actions/setup-go@v5
         with:
           go-version-file: 'go.mod'
 
       - name: Test
-        run: make test-integration GOTEST_FLAGS="-v -count=1"
+        # The test coverate and the test report are needed for the SonarCloud analysis
+        run: make test-integration GOTEST_FLAGS="-v -count=1 -coverprofile coverage.out" 2>&1 | tee test-report.out
+
+      - name: SonarCloud Scan
+        uses: SonarSource/sonarcloud-github-action@v3.1.0
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

.gitignore

@@ -97,3 +97,5 @@ pkg/plugin/processor/standalone/test/wasm_processors/*/processor.wasm
 
 # this one is needed for integration tests
 !pkg/provisioning/test/source-file.txt
+
+golangci-report.xml

cmd/cli/cli.go

@@ -0,0 +1,147 @@
+// Copyright © 2024 Meroxa, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cli
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/conduitio/conduit/pkg/conduit"
+	"github.com/spf13/cobra"
+)
+
+var (
+	initArgs          InitArgs
+	pipelinesInitArgs PipelinesInitArgs
+)
+
+type Instance struct {
+	rootCmd *cobra.Command
+}
+
+// New creates a new CLI Instance.
+func New() *Instance {
+	return &Instance{
+		rootCmd: buildRootCmd(),
+	}
+}
+
+func (i *Instance) Run() {
+	if err := i.rootCmd.Execute(); err != nil {
+		_, _ = fmt.Fprintf(os.Stderr, "%v\n", err)
+		os.Exit(1)
+	}
+}
+
+func buildRootCmd() *cobra.Command {
+	cfg := conduit.DefaultConfig()
+
+	cmd := &cobra.Command{
+		Use:     "conduit",
+		Short:   "Conduit CLI",
+		Long:    "Conduit CLI is a command-line that helps you interact with and manage Conduit.",
+		Version: conduit.Version(true),
+		Run: func(cmd *cobra.Command, args []string) {
+			e := &conduit.Entrypoint{}
+			e.Serve(cfg)
+		},
+	}
+	cmd.CompletionOptions.DisableDefaultCmd = true
+	conduit.Flags(&cfg).VisitAll(cmd.Flags().AddGoFlag)
+
+	// init
+	cmd.AddCommand(buildInitCmd())
+
+	// pipelines
+	cmd.AddGroup(&cobra.Group{
+		ID:    "pipelines",
+		Title: "Pipelines",
+	})
+	cmd.AddCommand(buildPipelinesCmd())
+
+	return cmd
+}
+
+func buildInitCmd() *cobra.Command {
+	initCmd := &cobra.Command{
+		Use:   "init",
+		Short: "Initialize Conduit with a configuration file and directories.",
+		Args:  cobra.NoArgs,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return NewConduitInit(initArgs).Run()
+		},
+	}
+	initCmd.Flags().StringVar(
+		&initArgs.Path,
+		"config.path",
+		"",
+		"path where Conduit will be initialized",
+	)
+
+	return initCmd
+}
+
+func buildPipelinesCmd() *cobra.Command {
+	pipelinesCmd := &cobra.Command{
+		Use:     "pipelines",
+		Short:   "Initialize and manage pipelines",
+		Args:    cobra.NoArgs,
+		GroupID: "pipelines",
+	}
+
+	pipelinesCmd.AddCommand(buildPipelinesInitCmd())
+
+	return pipelinesCmd
+}
+
+func buildPipelinesInitCmd() *cobra.Command {
+	pipelinesInitCmd := &cobra.Command{
+		Use:   "init [pipeline-name]",
+		Short: "Initialize an example pipeline.",
+		Long: `Initialize a pipeline configuration file, with all of parameters for source and destination connectors 
+initialized and described. The source and destination connector can be chosen via flags. If no connectors are chosen, then
+a simple and runnable generator-to-log pipeline is configured.`,
+		Args:    cobra.MaximumNArgs(1),
+		Example: "  conduit pipelines init awesome-pipeline-name --source postgres --destination kafka --path pipelines/pg-to-kafka.yaml",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) > 0 {
+				pipelinesInitArgs.Name = args[0]
+			}
+			return NewPipelinesInit(pipelinesInitArgs).Run()
+		},
+	}
+
+	// Add flags to pipelines init command
+	pipelinesInitCmd.Flags().StringVar(
+		&pipelinesInitArgs.Source,
+		"source",
+		"",
+		"Source connector (any of the built-in connectors).",
+	)
+	pipelinesInitCmd.Flags().StringVar(
+		&pipelinesInitArgs.Destination,
+		"destination",
+		"",
+		"Destination connector (any of the built-in connectors).",
+	)
+	pipelinesInitCmd.Flags().StringVar(
+		&pipelinesInitArgs.Path,
+		"pipelines.path",
+		"./pipelines",
+		"Path where the pipeline will be saved.",
+	)
+
+	return pipelinesInitCmd
+}