Refactoring of container creation mechanism (#750)

.github/workflows/testing.yml

@@ -56,9 +56,9 @@ jobs:
           revhash=$(git rev-parse $GITHUB_REF)
           IMAGE_TAG=g${revhash:0:9}
           PUSH_REPO=$PUSH_REGISTRY/${{ github.repository }}
-          PUSH_TAG=$PUSH_REPO:$IMAGE_TAG
+          PUSH_TAG=$PUSH_REPO:udmis-$IMAGE_TAG
           echo PUSH_TAG=$PUSH_TAG >> $GITHUB_ENV
-          udmis/bin/container prep --no-check ${PUSH_REPO%/*}
+          bin/container udmis prep --no-check $PUSH_REPO
           echo Pushing built container as $PUSH_TAG | tee -a $GITHUB_STEP_SUMMARY
       - name: Build and push docker image
         if: ${{ github.event_name == 'push' }}

.gitignore

@@ -23,6 +23,7 @@ credentials.json
 /udmis/out/
 /udmis/var/
 /udmis/.idea/libraries/
+/selfie/build/
 /pubber/build/
 /pubber/out/
 /pubber/pubber/out/

udmis/bin/container → bin/container

@@ -1,22 +1,29 @@
 #!/bin/bash -e
 
 ROOT=$(dirname $0)/..
-cd $ROOT
-DROOT=.
 
 function usage {
     echo Error: $*
-    echo Usage: $0 { prep, build, shell, run, deploy, update, status, logs, stop } [--no-check] [--selfie] [repo]
+    echo Usage: $0 target { prep, build } [--no-check] [repo]
     echo Project: $GCP_PROJECT
     echo Try starting with: $0 build
     exit 1
 }
 
-cmd=$1
-shift || usage missing command
+function fail {
+    echo Error: $*
+    exit 1
+}
+
+target=$1
+cmd=$2
+shift 2 || usage missing arguments
+
+cd ${ROOT}/${target}
+DROOT=$PWD
 
 git_branch=$(git rev-parse --abbrev-ref HEAD)
-git_remote=$(git config --get branch.$git_branch.remote)
+git_remote=$(git config --get branch.$git_branch.remote) || fail missing git remote
 git_url=$(git config --get remote.$git_remote.url)
 git_prefix=${git_url%.*}
 git_prefix=${git_prefix#*.com/}
@@ -25,18 +32,11 @@ git_repo=${git_prefix#*.com:}
 GCP_PROJECT=$(gcloud config get project)
 REPOSITORY=ghcr.io/${git_repo}
 
-IMAGE=udmis
-
 if [[ $1 == "--no-check" ]]; then
     NOCHECK=1
     shift
 fi
 
-if [[ $1 == "--selfie" ]]; then
-    IMAGE=selfie
-    shift
-fi
-
 if [[ -n $1 ]]; then
     REPOSITORY=$1
     shift
@@ -64,25 +64,28 @@ fi
 echo Using GCP project $GCP_PROJECT
 echo Using udmi namespace $UDMI_NAMESPACE
 
-LIBFILE=build/libs/udmis-1.0-SNAPSHOT-all.jar
+LIBFILE=build/libs/${target}-1.0-SNAPSHOT-all.jar
 
 current_user=$USER@$HOSTNAME
 
 revparse=`git rev-parse HEAD`
 
 udmi_ver=g${revparse:0:9}
-udmi_ref=$REPOSITORY:$udmi_ver
+udmi_tag=$target-$udmi_ver
+udmi_ref=$REPOSITORY:$udmi_tag
 
 version=`git describe`
 
 RUNARGS="--rm -ti -v $PWD/var:/udmi -v $HOME/.config:/root/.config --tmpfs /tmp"
-TEMPLATES=$(cd etc; ls k8s_*.yaml)
 
 if [[ $cmd == prep || $cmd == build ]]; then
     rm -rf var tmp && mkdir -p var tmp
+
+    cp -f etc/*.json var/ || true
+
     bin/build
+
     build_time=`date --utc -Imin -r $LIBFILE`
-    cp etc/prod_pod.json var/
     cat <<EOF > var/deployed_version.json
 {
   "udmi_version": "$version",
@@ -92,23 +95,23 @@ if [[ $cmd == prep || $cmd == build ]]; then
 }
 EOF
     more var/deployed_version.json | cat
-    echo Next try: $0 build
 fi
 
 if [[ $cmd == build ]]; then
-    echo Building Dockerfile.$IMAGE
-    docker build -f Dockerfile.$IMAGE -t $IMAGE $DROOT
+    echo Building Dockerfile.$target
+    docker build -f Dockerfile.$target -t $target $DROOT
 fi
 
+TEMPLATES=$(cd etc; ls k8s_*.yaml)
 if [[ $cmd == build && $REPOSITORY != local ]]; then
-    docker tag $IMAGE $udmi_ref
+    docker tag $target $udmi_ref
     docker push $udmi_ref
-    hash=$(docker images --digests ${udmi_ref%:*} | fgrep $udmi_ver | awk '{print $3}')
+    hash=$(docker images --digests ${udmi_ref%:*} | fgrep $udmi_tag | awk '{print $3}')
     ihash=$udmi_ref@$hash
     for file in $TEMPLATES; do
         cp etc/$file tmp/$file
         sed -i tmp/$file \
-            -e "s^@IMAGE-$IMAGE@^$ihash^" \
+            -e "s^@IMAGE-$target@^$ihash^" \
             -e "s^@UDMI_NAMESPACE@^$UDMI_NAMESPACE^" \
             -e "s^@GCP_PROJECT@^$GCP_PROJECT^"
         if diff etc/$file tmp/$file; then
@@ -120,28 +123,12 @@ if [[ $cmd == build && $REPOSITORY != local ]]; then
 fi
 
 if [[ $cmd == prep ]]; then
-    echo Next try: $0 build
+    echo Done with container prep.
 elif [[ $cmd == build ]]; then
-    echo Next try: $0 '{run, deploy, update}'
-elif [[ $cmd == run ]]; then
-    docker run $RUNARGS $IMAGE
-elif [[ $cmd == shell ]]; then
-    docker run $RUNARGS $IMAGE bash
-elif [[ $cmd == deploy ]]; then
-    kubectl apply -f tmp/k8s_pod.yaml
-    echo Next try: $0 status
-elif [[ $cmd == status ]]; then
-    kubectl describe pods udmis-test-pod
-    echo Next try: $0 logs
-elif [[ $cmd == logs ]]; then
-    kubectl logs udmis-test-pod
-elif [[ $cmd == stop ]]; then
-    kubectl delete pod/udmis-test-pod
-elif [[ $cmd == update ]]; then
-    kubectl apply -f tmp/k8s_udmis.yaml
-elif [[ $cmd == remove ]]; then
-    kubectl delete deployment/udmis-pods
+    echo Done with container build.
+    for file in tmp/k8s_*.yaml; do
+        echo kubectl apply -f $target/$file
+    done
 else
-    echo Unknown command $cmd
-    false
+    fail Unknown container command $cmd
 fi

bin/set_project

@@ -0,0 +1,70 @@
+#!/bin/bash -e
+
+if [[ $# != 1 ]]; then
+    echo Usage: $0 gcp_project[/udmi_namespace]
+    false
+fi
+
+full_id=$1
+shift
+
+gcp_project=${full_id%/*}
+udmi_space=${full_id#$gcp_project}
+udmi_space=${udmi_space#/}
+success=$(mktemp)
+rm -f $success
+
+ROOT=$(dirname $0)/..
+cd $ROOT
+
+echo Configuring for GCP project $gcp_project...
+
+current_project=$(gcloud config get project)
+quota_project=$(jq -r .quota_project_id $HOME/.config/gcloud/application_default_credentials.json)
+
+if [[ $quota_project != $gcp_project ]]; then
+    echo Setting gcloud quota project to $gcp_project
+    gcloud auth application-default set-quota-project $gcp_project
+fi
+
+if [[ $gcp_project != $current_project ]]; then
+    echo Setting gcloud defailt project to $gcp_project
+    gcloud --quiet config set project $gcp_project
+fi
+
+saved=$(kubectl config current-context)
+kcontexts=$(kubectl config get-contexts -o name | fgrep _${gcp_project}_)
+
+if [[ -z $kcontexts ]]; then
+    echo No kubectl context defined for project $gcp_project
+    echo "  kubectl config get-contexts -o name | fgrep _${gcp_project}_)"
+    false
+fi
+
+for context in $kcontexts; do
+    echo Checking k8s context $context
+
+    k8s_parts=$(tr '_' ' ' <<<$context)
+    GCP_PROJECT=$(awk '{print $2}' <<<$k8s_parts)
+    GCP_REGION=$(awk '{print $3}' <<<$k8s_parts)
+    K8S_CLUSTER=$(awk '{print $4}' <<<$k8s_parts)
+    UDMI_NAMESPACE=$(awk '{print $5}' <<<$k8s_parts)
+
+    if [[ $GCP_PROJECT == $gcp_project && $UDMI_NAMESPACE == $udmi_space ]]; then
+        echo Matched k8s context $context
+        echo $context > $success
+        break
+    fi
+done
+
+if [[ ! -f $success ]]; then
+    echo No k8s context matching project $gcp_project and udmi namespace ${udmi_space:-\<default\>} found.
+    false
+fi
+
+K8S_CONTEXT=$(< $success)
+
+kubectl config use-context $K8S_CONTEXT
+
+k8s_namespace=$(kubectl config get-contexts --no-headers $K8S_CONTEXT | awk '{ print $5 }')
+echo Using k8s namespace ${k8s_namespace:-default}

bin/update_udmis

@@ -23,7 +23,7 @@ git_prefix=${git_url%.*}
 git_prefix=${git_prefix#*.com/}
 git_repo=${git_prefix#*.com:}
 
-update_to=ghcr.io/${git_repo}:g${full_hash:0:9}
+update_to=ghcr.io/${git_repo}:udmis-g${full_hash:0:9}
 
 echo Updating udmis install to $update_to
 

selfie/bin/build

@@ -0,0 +1,4 @@
+# Empty dummy build script for non-existant selfie source.
+
+mkdir -p build/libs
+touch build/libs/selfie-1.0-SNAPSHOT-all.jar

udmis/Dockerfile.udmis

@@ -14,7 +14,5 @@ ADD bin/ bin/
 ADD var/ var/
 
 ENV CLEARBLADE_CONFIGURATION=/udmi/clearblade.json
-ENV GCP_PROJECT=$GCP_PROJECT
-ENV UDMI_PREFIX=$UDMI_PREFIX
 
 CMD ["/root/bin/run", "var/prod_pod.json"]

udmis/etc/k8s_account.yaml

@@ -4,4 +4,3 @@ metadata:
   annotations:
     iam.gke.io/gcp-service-account: udmi-service@@GCP_PROJECT@.iam.gserviceaccount.com
   name: default
-  namespace: @UDMI_NAMESPACE@

validator/Dockerfile.validator

@@ -0,0 +1,14 @@
+FROM alpine:latest
+
+WORKDIR /root
+
+RUN apk add openjdk17 bash gcompat mosquitto-clients curl jq
+
+# Workaround for https://github.com/grpc/grpc-java/issues/8751
+ENV LD_PRELOAD=/lib/libgcompat.so.0
+
+ADD build/libs/validator-1.0-SNAPSHOT-all.jar build/libs/validator-1.0-SNAPSHOT-all.jar
+
+ADD bin/ bin/
+
+CMD ["/root/bin/automate"]

validator/bin/automate

@@ -0,0 +1,8 @@
+#!/bin/bash -e
+#
+# Automation of tools using PubSub triggers.
+#
+
+env
+
+exec tail -f /dev/null