Create spans for Github workflow jobs and auto-inject into run steps

.github/workflows/test.yaml

@@ -1,4 +1,4 @@
-name: Build
+name: Test
 
 on:
   workflow_call:
@@ -29,7 +29,7 @@ jobs:
       - run: sudo apt-get -y install ./package.deb
       - run: bash -c "cd tests && sudo bash run_tests.sh bash"
 
-  test-os-shell:
+  os-shell:
     needs: smoke
     runs-on: ubuntu-latest
     strategy:
@@ -50,8 +50,28 @@ jobs:
           OS: ${{ matrix.os }}
           SHELL: ${{ matrix.shell }}
 
+  action:
+    needs: _build
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: ./actions/instrument/job
+        env:
+          OTEL_METRICS_EXPORTER: console
+          OTEL_LOGS_EXPORTER: console
+          OTEL_TRACES_EXPORTER: console
+      - run: echo hello world
+      - run: |
+          echo hello world again
+      - run: alias
+      - run: |
+          alias
+      - run: |
+          . otel.sh
+          echo echo world manually injected
+
   all:
-    needs: [package, test-os-shell]
+    needs: [package, os-shell, action]
     runs-on: ubuntu-latest
     steps:
       - run: exit 0

README.md

@@ -143,6 +143,29 @@ If the command represents communication to a third party service (like a HTTP re
 
 Finally, a single root span will be created and activated that represents the script. This span will automatically be deactivated and ended when the script ends.
 
+## Automatic Instrumentation of Github Actions
+To automatically monitor your Github Workflows on job level and to auto-inject into all run steps, add the following step as first in every job you want to observe. You can configure the SDK as described <a href="https://opentelemetry.io/docs/languages/sdk-configuration/">here</a> by adding environment variables to the setup step.
+```yaml
+- uses: plengauer/opentelemetry-bash/actions/instrument/job@main
+  env:
+    OTEL_SERVICE_NAME: 'Test'
+    # ...
+```
+
+A full job may look like this:
+```yaml
+do-something:
+  runs-on: ubuntu-latest
+  steps:
+    - uses: plengauer/opentelemetry-bash/actions/instrument/job@main
+      env:
+        OTEL_SERVICE_NAME: ${{ secrets.SERVICE_NAME }}
+        # ...
+    - run: echo hello world
+    - run: |
+        echo hello world again
+```
+
 ## Manual Instrumentation
 Import the API by referencing the `otelapi.sh` file. This is only necessary if you do not choose a fully automatic approach described above. In case you use automatic instrumentation, the API will be imported automatically for you.
 The SDK needs to be initialized and shut down manually at the start and at the end of your script respectively. All config must be set before the call to `otel_init`. You can configure the underlying SDK with the same variables as any other OpenTelemetry SDK as described <a href="https://opentelemetry.io/docs/languages/sdk-configuration/">here</a>. We recommend not just setting the environment variables, but also exporting them so that automatically injected children inherit the same configuration.

VERSION

@@ -1 +1 @@
-4.5.0
+4.6.0

actions/instrument/job/action.yaml

@@ -0,0 +1,7 @@
+name: 'OpenTelemetry'
+description: 'Observe Github Actions with OpenTelemetry'
+runs:
+  using: 'node20'
+  pre: 'inject_and_init.js'
+  main: 'nop.js'
+  post: 'shutdown.js'

actions/instrument/job/forward.c

@@ -0,0 +1,41 @@
+#include <stdlib.h>
+#include <stdio.h>
+#include <unistd.h>
+
+// forward [ forward arg1 arg2 NULL ] => EXECUTABLE [ EXECUTABLE ARG1 ARG2 arg1 arg2 NULL ]
+
+#ifndef EXECUTABLE
+#error must define executable
+#endif
+
+#define STR(s) XSTR(s)
+#define XSTR(s) #s
+
+int main(int argc, char **argv) {
+    int new_argc = argc;
+#ifdef ARG1
+    new_argc++;
+#endif
+#ifdef ARG2
+    new_argc++;
+#endif
+
+    char **new_argv = (char**) calloc((size_t) new_argc + 1, sizeof(char*));
+    int i = 0;
+    new_argv[i++] = STR(EXECUTABLE);
+#ifdef ARG1
+    new_argv[i++] = STR(ARG1);
+#endif
+#ifdef ARG2
+    new_argv[i++] = STR(ARG2);
+#endif
+    for (int j = 1; j < argc; j++) {
+        new_argv[i++] = argv[j];
+    }
+    new_argv[i++] = NULL;
+
+    execv(STR(EXECUTABLE), new_argv);
+    perror("execv failed");
+    free(new_argv);
+    return 1;
+}

actions/instrument/job/forward.sh

@@ -0,0 +1,11 @@
+#/bin/sh
+set -e
+# sh|dash|bash -e /.../*.sh
+if [ -n "$GITHUB_RUN_ID" ] && [ -f "$3" ] && [ "$(echo "$3" | rev | cut -d . -f 1 | rev)" = "sh" ] && [ "$(echo "$GITHUB_ENV" | rev | cut -d / -f 3- | rev)" = "$(echo "$3" | rev | cut -d / -f 2- | rev)" ]; then
+  file="$3"
+  script="$(cat "$file")"
+  script=". otel.sh
+$script"
+  echo "$script" > "$file"
+fi
+exec "$@"

actions/instrument/job/inject_and_init.bash

@@ -0,0 +1,49 @@
+set -e
+
+root4job_end() {
+  if [ "$(curl "$GITHUB_API_URL/repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/jobs" | jq -r ".jobs[] | select(.name==\"$GITHUB_JOB\") | select(.run_attempt==\"$GITHUB_RUN_ATTEMPT\") | .steps[] | select(.status==\"completed\") | select(.conclusion==\"failure\") | .name" | wc -l)" -gt 0 ]; then
+    otel_span_error "$span_handle"
+  fi
+  otel_span_end "$span_handle"
+  otel_shutdown
+  exit 0
+}
+export -f root4job_end
+
+root4job() {
+  traceparent_file="$1"
+  . otelapi.sh
+  otel_init
+  span_handle="$(otel_span_start CONSUMER "$GITHUB_WORKFLOW / $GITHUB_JOB")"
+  otel_span_activate "$span_handle"
+  echo "$OTEL_TRACEPARENT" > "$traceparent_file"
+  otel_span_deactivate "$span_handle"
+  trap root4job_end SIGUSR1
+  while true; do sleep 1; done
+}
+export -f root4job
+
+root_pid_file="$(mktemp -u | rev | cut -d / -f 2- | rev)/opentelemetry_shell_$GITHUB_RUN_ID.pid"
+traceparent_file="$(mktemp -u)"
+nohup bash -c 'root4job "$@"' bash "$traceparent_file" &> /dev/null &
+echo "$!" > "$root_pid_file"
+
+while ! [ -f "$traceparent_file" ]; do sleep 1; done
+export OTEL_TRACEPARENT="$(cat "$traceparent_file")"
+rm "$traceparent_file"
+
+if [ -z "$OTEL_SERVICE_NAME" ]; then
+  export OTEL_SERVICE_NAME="$(echo "$GITHUB_REPOSITORY" | cut -d / -f 2-) CI"
+fi
+
+printenv | grep '^OTEL_' >> "$GITHUB_ENV"
+
+my_dir="$(echo "$0" | rev | cut -d / -f 2- | rev)"
+new_path_dir="$(mktemp -d)"
+gcc -o "$new_path_dir"/sh_w_otel "$my_dir"/forward.c -DEXECUTABLE="$(which sh)" -DARG1="$my_dir"/forward.sh -DARG2="$(which sh)"
+gcc -o "$new_path_dir"/dash_w_otel "$my_dir"/forward.c -DEXECUTABLE="$(which dash)" -DARG1="$my_dir"/forward.sh -DARG2="$(which dash)"
+gcc -o "$new_path_dir"/bash_w_otel "$my_dir"/forward.c -DEXECUTABLE="$(which bash)" -DARG1="$my_dir"/forward.sh -DARG2="$(which bash)"
+ln --symbolic "$new_path_dir"/sh_w_otel "$new_path_dir"/sh
+ln --symbolic "$new_path_dir"/dash_w_otel "$new_path_dir"/dash
+ln --symbolic "$new_path_dir"/bash_w_otel "$new_path_dir"/bash
+echo "$new_path_dir" >> "$GITHUB_PATH"

actions/instrument/job/inject_and_init.js

@@ -0,0 +1,14 @@
+const { spawn } = require('child_process');
+
+function run(executable, args = []) {
+  return new Promise((resolve, reject) => {
+    const process = spawn(executable, args);
+    process.stdout.on('data', data => console.log('' + data));
+    process.stderr.on('data', data => console.error('' + data));
+    process.on('close', code => code == 0 ? resolve(code) : reject(code));
+    process.on('error', error => reject(new Error(error)));
+  });
+}
+
+run('/bin/sh', [ '-c', 'wget -O - https://raw.githubusercontent.com/plengauer/opentelemetry-bash/main/INSTALL.sh | sh -E' ])
+  .then(() => run('/bin/bash', [ '-e', __dirname + '/inject_and_init.bash' ]))

actions/instrument/job/nop.js

@@ -0,0 +1,5 @@
+const process = require('process');
+if (process.env['GITHUB_REPOSITORY'] == 'plengauer/opentelemetry-bash') {
+  // action is local and pre steps is not supported because the action needs to be checked out first and then its too let to execute a pre step
+  require('./inject_and_init.js');
+}

actions/instrument/job/shutdown.js

@@ -0,0 +1,13 @@
+const { spawn } = require('child_process');
+
+function run(executable, args = []) {
+  return new Promise((resolve, reject) => {
+    const process = spawn(executable, args);
+    process.stdout.on('data', data => console.log('' + data));
+    process.stderr.on('data', data => console.error('' + data));
+    process.on('close', code => code == 0 ? resolve(code) : reject(code));
+    process.on('error', error => reject(new Error(error)));
+  });
+}
+
+run('/bin/sh', [ '-e', __dirname + '/shutdown.sh' ])

actions/instrument/job/shutdown.sh

@@ -0,0 +1,5 @@
+set -e
+root_pid_file="$(mktemp -u | rev | cut -d / -f 2- | rev)/opentelemetry_shell_$GITHUB_RUN_ID.pid"
+root_pid="$(cat "$root_pid_file")"
+kill -USR1 "$root_pid"
+while kill -0 "$root_pid" 2> /dev/null; do sleep 1; done

src/usr/share/opentelemetry_shell/opentelemetry_shell.sh

@@ -306,7 +306,7 @@ _otel_record_exec() {
 
 _otel_start_script() {
   otel_init || return $?
-  if \[ -n "$SSH_CLIENT"  ] && \[ -n "$SSH_CONNECTION" ] && \[ "$(\cat /proc/$PPID/cmdline | \tr -d '\000' | \cut -d' ' -f1)" = "sshd:" ]; then
+  if \[ -n "$SSH_CLIENT"  ] && \[ -n "$SSH_CONNECTION" ] && \[ "$(\cat /proc/$PPID/cmdline | \tr -d '\000' | \cut -d ' ' -f 1)" = "sshd:" ]; then
     _root_span_handle="$(otel_span_start SERVER ssh)"
     otel_span_attribute_typed $_root_span_handle string ssh.ip="$(\echo $SSH_CONNECTION | \cut -d ' ' -f 3)"
     otel_span_attribute_typed $_root_span_handle int ssh.port="$(\echo $SSH_CONNECTION | \cut -d ' ' -f 4)"
@@ -331,6 +331,12 @@ _otel_start_script() {
     _root_span_handle="$(otel_span_start SERVER "$(\echo "$cmdline" | \cut -d . -f 2- | \cut -d ' ' -f 1)")"
     otel_span_attribute_typed $_root_span_handle string debian.package.name="$(\echo "$cmdline" | \rev | \cut -d / -f 1 | \rev | \cut -d . -f 1)"
     otel_span_attribute_typed $_root_span_handle string debian.package.operation="$(\echo "$cmdline" | \cut -d . -f 2-)"
+  elif \[ -n "$GITHUB_RUN_ID" ] && \[ -n "$GITHUB_WORKFLOW" ] && \[ "$(\cat /proc/$PPID/cmdline | \tr '\000-\037' ' ' | \cut -d ' ' -f 1 | \rev | \cut -d / -f 1 | \rev)" = "Runner.Worker" ]; then
+    local name="$GITHUB_WORKFLOW"
+    local kind=CONSUMER
+    if \[ -n "$GITHUB_JOB" ]; then local name="$name / $GITHUB_JOB"; local kind=CONSUMER; fi
+    if \[ -n "$GITHUB_ACTION" ]; then local name="$name / $GITHUB_ACTION"; local kind=SERVER; fi
+    _root_span_handle="$(otel_span_start "$kind" "$name")"
   elif ! \[ "$OTEL_SHELL_AUTO_INJECTED" = TRUE ] && \[ -z "$OTEL_TRACEPARENT" ]; then
     _root_span_handle="$(otel_span_start SERVER "$(_otel_command_self)")"
   elif ! \[ "$OTEL_SHELL_AUTO_INJECTED" = TRUE ] && \[ -n "$OTEL_TRACEPARENT" ]; then

src/usr/share/opentelemetry_shell/opentelemetry_shell_sdk.py

@@ -26,6 +26,34 @@
 from opentelemetry.sdk._logs.export import BatchLogRecordProcessor, ConsoleLogExporter
 from opentelemetry.exporter.otlp.proto.http._log_exporter import OTLPLogExporter
 
+class GithubActionResourceDetector(ResourceDetector):
+    def detect(self) -> Resource:
+        try:
+            if not 'GITHUB_RUN_ID' in os.environ:
+                return Resource.create({});
+            return Resource.create({
+                'github.sha': os.environ.get('GITHUB_SHA', ''),
+                'github.repository.id': os.environ.get('GITHUB_REPOSITORY_ID', ''),
+                'github.repository.name': os.environ.get('GITHUB_REPOSITORY', ''),
+                'github.repository.owner.id': os.environ.get('GITHUB_REPOSITORY_OWNER_ID', ''),
+                'github.repository.owner.name': os.environ.get('GITHUB_REPOSITORY_OWNER', ''),
+                'github.event.ref': os.environ.get('GITHUB_REF', ''),
+                'github.event.ref.sha': os.environ.get('GITHUB_SHA', ''),
+                'github.event.ref.name': os.environ.get('GITHUB_REF_NAME', ''),
+                'github.event.actor.id': os.environ.get('GITHUB_ACTOR_ID', ''),
+                'github.event.actor.name': os.environ.get('GITHUB_ACTOR', ''),
+                'github.event.name': os.environ.get('GITHUB_EVENT_NAME', ''),
+                'github.workflow.run.id': os.environ.get('GITHUB_RUN_ID', ''),
+                'github.workflow.run.attempt': os.environ.get('GITHUB_RUN_ATTEMPT', ''),
+                'github.workflow.ref': os.environ.get('GITHUB_WORKFLOW_REF', ''),
+                'github.workflow.sha': os.environ.get('GITHUB_WORKFLOW_SHA', ''),
+                'github.workflow.name': os.environ.get('GITHUB_WORKFLOW', ''),
+                'github.job.name': os.environ.get('GITHUB_JOB', ''),
+                'github.action.name': os.environ.get('GITHUB_ACTION', ''),
+            })
+        except:
+            return Resource.create({})
+
 class AwsEC2ResourceDetector(ResourceDetector):
     def detect(self) -> Resource:
         try:
@@ -115,6 +143,7 @@ def handle(scope, version, command, arguments):
                 OracleResourceDetector(),
                 KubernetesResourceDetector(),
                 DockerResourceDetector(),
+                GithubActionResourceDetector(),
                 OTELResourceDetector(),
             ]).merge(Resource.create(resource))