chore(ci): Publish to docker.io and use public repos

This commit prepares this repository to become the only place where
vector changes are done. The difference is that it will publish images
to `docker.io` where we can then use for Edge as well as our SaaS
product. This will require us to change our SaaS images to source from
dockerhub instead of gcr, but this is the first step towards that.

There were also changes to use public-only repos for dependencies. This
allows us to remove lots of github auth steps that are no longer needed.
This commit completely re-works `Jenkinsfile` to work from a public-only
repository standpoint.

Ref: LOG-18250

.github/workflows/integration-test-mezmo.yml

@@ -30,8 +30,6 @@ env:
   VERBOSE: true
   CI: true
   PROFILE: debug
-  GITHUB_TOKEN: ${{ secrets.GH_AUTH_TOKEN }}
-  CARGO_NET_GIT_FETCH_WITH_CLI: true
 
 jobs:
   test-integration:
@@ -73,7 +71,6 @@ jobs:
           # - test: 'sumo-logic'
     steps:
       - uses: actions/checkout@v3
-      - run: git config --global url."https://${GITHUB_TOKEN}@github.com".insteadOf ssh://git@github.com
       - run: make test-integration-${{ matrix.test }}
         env:
           SPLUNK_VERSION: ${{ matrix.env.SPLUNK_VERSION }}

Jenkinsfile

@@ -3,25 +3,20 @@ library 'magic-butler-catalogue'
 def WORKSPACE_PATH = "/tmp/workspace/${env.BUILD_TAG.replace('%2F', '/')}"
 def DEFAULT_BRANCH = "master"
 def PROJECT_NAME = "vector"
-def CURRENT_BRANCH = [env.CHANGE_BRANCH, env.BRANCH_NAME]?.find{branch -> branch != null}
-
-def slugify(str) {
-  def s = str.toLowerCase()
-  s = s.replaceAll(/[^a-z0-9\s-\/]/, "").replaceAll(/\s+/, " ").trim()
-  s = s.replaceAll(/[\/\s]/, '-').replaceAll(/-{2,}/, '-')
-  s
-}
+def CURRENT_BRANCH = currentBranch()
+def DOCKER_REPO = "docker.io/mezmohq"
 
 def CREDS = [
   string(
     credentialsId: 'github-api-token',
     variable: 'GITHUB_TOKEN'
   ),
-  aws(credentialsId: 'aws',
+  aws(
+    credentialsId: 'aws',
     accessKeyVariable: 'AWS_ACCESS_KEY_ID',
-    secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'),
+    secretKeyVariable: 'AWS_SECRET_ACCESS_KEY'
+  ),
 ]
-
 def NPMRC = [
     configFile(fileId: 'npmrc', variable: 'NPM_CONFIG_USERCONFIG')
 ]
@@ -71,22 +66,23 @@ pipeline {
     }
   }
   stages {
-    stage('Setup') {
-      steps {
-        sh 'make release-tool'
+    stage('Validate PR Author') {
+      when {
+        expression { env.CHANGE_FORK }
+        not {
+          triggeredBy 'issueCommentCause'
+        }
       }
-    }
-    stage('Check'){
       steps {
-        sh """
-          make check ENVIRONMENT=true
-          make check-fmt ENVIRONMENT=true
-        """
+        error("A maintainer needs to approve this PR for CI by commenting")
       }
     }
+
+
+
     stage('Lint and test release'){
       tools {
-        nodejs 'NodeJS 16'
+        nodejs 'NodeJS 20'
       }
       environment {
         GIT_BRANCH = "${CURRENT_BRANCH}"
@@ -97,15 +93,27 @@ pipeline {
       steps {
         script {
           configFileProvider(NPMRC) {
-            sh 'npm ci'
+            sh 'npm ci --ignore-scripts'
+            sh 'npm run commitlint'
             sh 'npm run release:dry'
           }
         }
-        sh './release-tool lint'
-        sh './release-tool test'
       }
     }
-    stage('Lint and Test'){
+
+    stage('vdev Check'){
+      when {
+        changeRequest() // Only do this during PRs. It's about a 15-min wait.
+      }
+      steps {
+        sh """
+          make check ENVIRONMENT=true
+          make check-fmt ENVIRONMENT=true
+        """
+      }
+    }
+
+    stage('Code'){
       parallel {
         stage('Lint'){
           steps {
@@ -115,7 +123,10 @@ pipeline {
             """
           }
         }
-        stage('Deny'){
+        stage('Check Deny'){
+          when {
+            changeRequest() // PRs only to speed up dev flows. These can be fixed then if they're actionable.
+          }
           steps {
             catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') {
               sh """
@@ -142,12 +153,14 @@ pipeline {
               , push: false
               , tags: [slugify("${CURRENT_BRANCH}-${BUILD_NUMBER}")]
               , dockerfile: "distribution/docker/mezmo/Dockerfile"
+              , docker_repo: DOCKER_REPO
               )
             }
           }
         }
       }
     }
+
     stage('Feature build and publish') {
       when {
         expression {
@@ -156,19 +169,18 @@ pipeline {
       }
       steps {
         script {
-          def tag = slugify("${CURRENT_BRANCH}-${BUILD_NUMBER}")
+          def feature_tag = slugify("${CURRENT_BRANCH}-${BUILD_NUMBER}")
           buildx.build(
             project: PROJECT_NAME
           , push: true
-          , tags: [tag]
+          , tags: [feature_tag]
           , dockerfile: "distribution/docker/mezmo/Dockerfile"
+          , docker_repo: DOCKER_REPO
           )
         }
-        sh './release-tool clean'
-        sh './release-tool build'
-        sh './release-tool publish'
       }
     }
+
     stage('Release and publish') {
       when {
         branch DEFAULT_BRANCH
@@ -177,30 +189,27 @@ pipeline {
         }
       }
       tools {
-        nodejs 'NodeJS 16'
+        nodejs 'NodeJS 20'
       }
       steps {
         script {
+          def version_before = npm.semver().version
           configFileProvider(NPMRC) {
             sh 'npm ci'
             sh 'npm run release'
           }
 
-          def tag = sh (
-            script: "./release-tool debug-RELEASE_VERSION",
-            returnStdout: true
-          ).split(' = ')[1].trim()
-
-          buildx.build(
-            project: PROJECT_NAME
-          , push: true
-          , tags: [tag]
-          , dockerfile: "distribution/docker/mezmo/Dockerfile"
-          )
+          def semver = npm.semver()
+          if (version_before != semver.version) {
+            buildx.build(
+              project: PROJECT_NAME
+            , push: true
+            , tags: [semver.version]
+            , dockerfile: "distribution/docker/mezmo/Dockerfile"
+            , docker_repo: DOCKER_REPO
+            )
+          }
         }
-        sh './release-tool clean'
-        sh './release-tool build'
-        sh './release-tool publish'
       }
     }
   }

Makefile

@@ -158,15 +158,11 @@ define ENVIRONMENT_EXEC
 			--init \
 			--interactive \
 			--env INSIDE_ENVIRONMENT=true \
-			--env GITHUB_TOKEN=$(GITHUB_TOKEN) \
 			--env CI=$(CI) \
 			$(if $(ENVIRONMENT_NETWORK),--network $(ENVIRONMENT_NETWORK),) \
 			--mount type=bind,source=${CURRENT_DIR},target=/git/vectordotdev/vector \
-			--mount type=bind,source=${CURRENT_DIR}/scripts/environment/entrypoint.sh,target=/entrypoint.sh \
 			$(if $(findstring docker,$(CONTAINER_TOOL)),--mount type=bind$(COMMA)source=/var/run/docker.sock$(COMMA)target=/var/run/docker.sock,) \
 			--mount type=volume,source=vector-target,target=/git/vectordotdev/vector/target \
-			--mount type=volume,source=vector-cargo-cache,target=/root/.cargo \
-			--mount type=volume,source=vector-rustup-cache,target=/root/.rustup \
 			$(foreach publish,$(ENVIRONMENT_PUBLISH),--publish $(publish)) \
 			$(ENVIRONMENT_UPSTREAM)
 endef

distribution/docker/mezmo/Dockerfile

@@ -1,6 +1,5 @@
 # use a source that has multi-arch support (amd64+arm64)
 FROM rust:1-slim-bullseye as release-builder
-ARG GITHUB_TOKEN
 
 # deps for compiler
 RUN DEPS='gcc g++ make cmake xz-utils locales python libexpat1-dev gettext libz-dev libssl-dev autoconf pkg-config bzip2 protobuf-compiler libsasl2-dev git libclang-dev' \
@@ -14,7 +13,6 @@ RUN DEPS='gcc g++ make cmake xz-utils locales python libexpat1-dev gettext libz-
   && apt-get clean \
   && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* /usr/share/doc
 
-RUN git config --global url."https://${GITHUB_TOKEN}@github.com".insteadOf ssh://git@github.com
 
 WORKDIR /build
 COPY . .