Skip to content

Commit

Permalink
Merge branch 'main' into debug-musl-build
Browse files Browse the repository at this point in the history
  • Loading branch information
@iamazeem
iamazeem committed Nov 10, 2024
2 parents 01e75c5 + c47ea2f commit 7d166c6
Show file tree
Hide file tree
Showing 2 changed files with 129 additions and 43 deletions.
9 changes: 8 additions & 1 deletion .github/workflows/benchmarks.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,15 +4,22 @@ on:
workflow_dispatch:
inputs:
workflow-run-id:
description: 'Run ID (optional) [default: latest release]'
description: 'Workflow Run ID'
type: number
required: false
runs:
description: 'No. of runs [min: 5, max: 100]'
type: number
default: 5
required: false

permissions:
actions: read

env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
WORKFLOW_RUN_ID: ${{ inputs.workflow-run-id }}
RUNS: ${{ inputs.runs }}
BENCHMARKS_DIR: .benchmarks

jobs:
Expand Down
163 changes: 121 additions & 42 deletions scripts/ci-run-benchmarks.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,11 @@ set -e

echo "[INF] Running $0"

MIN_RUNS=5
MAX_RUNS=100

OS=${OS:-}
RUNS=${RUNS:-6}
SKIP_FIRST_RUN=${SKIP_FIRST_RUN:-true}
RUNS=${RUNS:-"$MIN_RUNS"}
BENCHMARKS_DIR=${BENCHMARKS_DIR:-".benchmarks"}
ZSV_LINUX_BUILD_COMPILER=${ZSV_LINUX_BUILD_COMPILER:-gcc}
ZSV_TAG=${ZSV_TAG:-}
Expand All @@ -22,14 +24,17 @@ fi

echo "[INF] OS: $OS"
echo "[INF] RUNS: $RUNS"
echo "[INF] SKIP_FIRST_RUN: $SKIP_FIRST_RUN"
echo "[INF] BENCHMARKS_DIR: $BENCHMARKS_DIR"

if [ "$RUNS" -lt 2 ]; then
echo "[ERR] RUNS must be greater than 2!"
exit 1
if [ "$RUNS" -lt "$MIN_RUNS" ] || [ "$RUNS" -gt "$MAX_RUNS" ]; then
echo "[WRN] Invalid RUNS value! [$RUNS]"
echo "[WRN] RUNS must be >= $MIN_RUNS and <= $MAX_RUNS!"
echo "[WRN] Using default minimum value... [RUNS: $MIN_RUNS]"
RUNS="$MAX_RUNS"
fi

RUNS=$((RUNS + 1))

if [ "$ZSV_TAG" = "" ]; then
ZSV_TAG=$(git ls-remote --tags --refs https://github.com/liquidaty/zsv | tail -n1 | cut -d '/' -f3)
fi
Expand All @@ -50,12 +55,12 @@ if [ "$OS" = "linux" ]; then
echo "[INF] Unknown value for ZSV_LINUX_BUILD_COMPILER! [$ZSV_LINUX_BUILD_COMPILER]"
exit 1
fi
OS_COMPILER="$OS [$ZSV_LINUX_BUILD_COMPILER]"
OS_COMPILER="$OS | $ZSV_LINUX_BUILD_COMPILER"
ZSV_TAR_URL="https://github.com/liquidaty/zsv/releases/download/v$ZSV_TAG/zsv-$ZSV_TAG-amd64-linux-$ZSV_LINUX_BUILD_COMPILER.tar.gz"
TSV_TAR_URL="https://github.com/eBay/tsv-utils/releases/download/v2.2.0/tsv-utils-v2.2.0_linux-x86_64_ldc2.tar.gz"
XSV_TAR_URL="https://github.com/BurntSushi/xsv/releases/download/0.13.0/xsv-0.13.0-x86_64-unknown-linux-musl.tar.gz"
elif [ "$OS" = "macos" ] || [ "$OS" = "darwin" ]; then
OS_COMPILER="$OS [gcc]"
OS_COMPILER="$OS | gcc"
ZSV_TAR_URL="https://github.com/liquidaty/zsv/releases/download/v$ZSV_TAG/zsv-$ZSV_TAG-amd64-macosx-gcc.tar.gz"
TSV_TAR_URL="https://github.com/eBay/tsv-utils/releases/download/v2.2.1/tsv-utils-v2.2.1_osx-x86_64_ldc2.tar.gz"
XSV_TAR_URL="https://github.com/BurntSushi/xsv/releases/download/0.13.0/xsv-0.13.0-x86_64-apple-darwin.tar.gz"
Expand Down Expand Up @@ -84,9 +89,9 @@ for URL in "$ZSV_TAR_URL" "$TSV_TAR_URL" "$XSV_TAR_URL"; do
printf "[INF] Downloading... [%s] " "$TAR"
if [ ! -f "$TAR" ]; then
wget -q "$URL"
echo "[DONE]"
else
echo "[SKIPPED]"
echo "[DONE]"
else
echo "[SKIPPED]"
fi
printf "[INF] Extracting... [%s] " "$TAR"
tar xf "$TAR"
Expand All @@ -109,7 +114,7 @@ SELECT_OUTPUT_FILE="select.out"
rm -f "$COUNT_OUTPUT_FILE" "$SELECT_OUTPUT_FILE"

echo "[INF] Running count benchmarks..."
for TOOL in zsv xsv tsv; do
for TOOL in zsv tsv xsv; do
CMD=
if [ "$TOOL" = "zsv" ]; then
CMD="$TOOLS_DIR/zsv count"
Expand All @@ -121,20 +126,18 @@ for TOOL in zsv xsv tsv; do

I=0
while [ "$I" -lt "$RUNS" ]; do
if [ "$SKIP_FIRST_RUN" = true ] && [ "$I" = 0 ]; then
I=$((I + 1))
continue
if [ "$I" != 0 ]; then
{
printf "%d | %s : " "$I" "$TOOL"
(time -p $CMD <"$CSV" >/dev/null) 2>&1 | xargs
} | tee -a "$COUNT_OUTPUT_FILE"
fi
{
printf "%d | %s : " "$I" "$TOOL"
(time -p $CMD <"$CSV" >/dev/null) 2>&1 | xargs
} | tee -a "$COUNT_OUTPUT_FILE"
I=$((I + 1))
done
done

echo "[INF] Running select benchmarks..."
for TOOL in zsv xsv tsv; do
for TOOL in zsv tsv xsv; do
CMD=
if [ "$TOOL" = "zsv" ]; then
CMD="$TOOLS_DIR/zsv select -W -n -- 2 1 3-7"
Expand All @@ -146,48 +149,124 @@ for TOOL in zsv xsv tsv; do

I=0
while [ "$I" -lt "$RUNS" ]; do
if [ "$SKIP_FIRST_RUN" = true ] && [ "$I" = 0 ]; then
I=$((I + 1))
continue
if [ "$I" != 0 ]; then
{
printf "%d | %s : " "$I" "$TOOL"
(time -p $CMD <"$CSV" >/dev/null) 2>&1 | xargs
} | tee -a "$SELECT_OUTPUT_FILE"
fi
{
printf "%d | %s : " "$I" "$TOOL"
(time -p $CMD <"$CSV" >/dev/null) 2>&1 | xargs
} | tee -a "$SELECT_OUTPUT_FILE"
I=$((I + 1))
done
done

echo "[INF] Evaluating results..."

RUNS=$((RUNS - 1))

COUNT_MAX_REAL_TIME=$(cut -d ' ' -f6 <"$COUNT_OUTPUT_FILE" | sort | tail -n1)
COUNT_ZSV_REAL_TIME_VALUES=$(grep zsv "$COUNT_OUTPUT_FILE" | cut -d ' ' -f6 | xargs | tr ' ' '+')
COUNT_ZSV_AVG_REAL_TIME=$(printf %.2f "$(echo "($COUNT_ZSV_REAL_TIME_VALUES) / $RUNS" | bc -l)")
COUNT_TSV_REAL_TIME_VALUES=$(grep tsv "$COUNT_OUTPUT_FILE" | cut -d ' ' -f6 | xargs | tr ' ' '+')
COUNT_TSV_AVG_REAL_TIME=$(printf %.2f "$(echo "($COUNT_TSV_REAL_TIME_VALUES) / $RUNS" | bc -l)")
COUNT_XSV_REAL_TIME_VALUES=$(grep xsv "$COUNT_OUTPUT_FILE" | cut -d ' ' -f6 | xargs | tr ' ' '+')
COUNT_XSV_AVG_REAL_TIME=$(printf %.2f "$(echo "($COUNT_XSV_REAL_TIME_VALUES) / $RUNS" | bc -l)")

SELECT_MAX_REAL_TIME=$(cut -d ' ' -f6 <"$SELECT_OUTPUT_FILE" | sort | tail -n1)
SELECT_ZSV_REAL_TIME_VALUES=$(grep zsv "$SELECT_OUTPUT_FILE" | cut -d ' ' -f6 | xargs | tr ' ' '+')
SELECT_ZSV_AVG_REAL_TIME=$(printf %.2f "$(echo "($SELECT_ZSV_REAL_TIME_VALUES) / $RUNS" | bc -l)")
SELECT_TSV_REAL_TIME_VALUES=$(grep tsv "$SELECT_OUTPUT_FILE" | cut -d ' ' -f6 | xargs | tr ' ' '+')
SELECT_TSV_AVG_REAL_TIME=$(printf %.2f "$(echo "($SELECT_TSV_REAL_TIME_VALUES) / $RUNS" | bc -l)")
SELECT_XSV_REAL_TIME_VALUES=$(grep xsv "$SELECT_OUTPUT_FILE" | cut -d ' ' -f6 | xargs | tr ' ' '+')
SELECT_XSV_AVG_REAL_TIME=$(printf %.2f "$(echo "($SELECT_XSV_REAL_TIME_VALUES) / $RUNS" | bc -l)")

echo "[INF] COUNT_MAX_REAL_TIME: $COUNT_MAX_REAL_TIME"
echo "[INF] COUNT_ZSV_AVG_REAL_TIME: [($COUNT_ZSV_REAL_TIME_VALUES) / $RUNS = $COUNT_ZSV_AVG_REAL_TIME]"
echo "[INF] COUNT_TSV_AVG_REAL_TIME: [($COUNT_TSV_REAL_TIME_VALUES) / $RUNS = $COUNT_TSV_AVG_REAL_TIME]"
echo "[INF] COUNT_XSV_AVG_REAL_TIME: [($COUNT_XSV_REAL_TIME_VALUES) / $RUNS = $COUNT_XSV_AVG_REAL_TIME]"

echo "[INF] SELECT_MAX_REAL_TIME: $SELECT_MAX_REAL_TIME"
echo "[INF] SELECT_ZSV_AVG_REAL_TIME: [($SELECT_ZSV_REAL_TIME_VALUES) / $RUNS = $SELECT_ZSV_AVG_REAL_TIME]"
echo "[INF] SELECT_TSV_AVG_REAL_TIME: [($SELECT_TSV_REAL_TIME_VALUES) / $RUNS = $SELECT_TSV_AVG_REAL_TIME]"
echo "[INF] SELECT_XSV_AVG_REAL_TIME: [($SELECT_XSV_REAL_TIME_VALUES) / $RUNS = $SELECT_XSV_AVG_REAL_TIME]"

COUNT_MAX_REAL_TIME_FOR_CHART=$(printf %.2f "$(echo "$COUNT_MAX_REAL_TIME + ($COUNT_MAX_REAL_TIME * 0.1)" | bc -l)")
SELECT_MAX_REAL_TIME_FOR_CHART=$(printf %.2f "$(echo "$SELECT_MAX_REAL_TIME + ($SELECT_MAX_REAL_TIME * 0.1)" | bc -l)")

MARKDOWN_OUTPUT="benchmarks.md"
echo "[INF] Generating Markdown output... [$MARKDOWN_OUTPUT]"
TIMESTAMP="$(date -u +"%Y-%m-%dT%H:%M:%SZ")"
{
echo '# Benchmarks'
echo
echo "- Timestamp UTC: $TIMESTAMP"
echo "- ZSV build from: $ZSV_BUILD_FROM"
echo "- OS [compiler]: $OS_COMPILER"
echo "## Benchmarks [$OS_COMPILER]"
echo
echo "## Releases Used"
echo "- \`zsv\` build from: $ZSV_BUILD_FROM"
echo "- Builds: [\`zsv\`]($ZSV_TAR_URL), [\`tsv\`]($TSV_TAR_URL), [\`xsv\`]($XSV_TAR_URL)"
echo
echo "- <$ZSV_TAR_URL>"
echo "- <$TSV_TAR_URL>"
echo "- <$XSV_TAR_URL>"
echo "### \`count\`"
echo
echo '## Results'
echo "<details>"
echo "<summary>Runs (click to expand/collapse)</summary>"
echo
echo '### count'
echo
echo '```'
echo '```text'
cat "$COUNT_OUTPUT_FILE"
echo
echo "Averages:"
echo "- zsv: [($COUNT_ZSV_REAL_TIME_VALUES) / $RUNS = $COUNT_ZSV_AVG_REAL_TIME]"
echo "- tsv: [($COUNT_TSV_REAL_TIME_VALUES) / $RUNS = $COUNT_TSV_AVG_REAL_TIME]"
echo "- xsv: [($COUNT_XSV_REAL_TIME_VALUES) / $RUNS = $COUNT_XSV_AVG_REAL_TIME]"
echo '```'
echo
echo '### select'
echo "</details>"
echo
echo '```mermaid'
echo '---'
echo 'config:'
echo ' xyChart:'
echo ' width: 600'
echo ' height: 400'
echo ' themeVariables:'
echo ' xyChart:'
echo ' titleColor: "#1e90ff"'
echo ' plotColorPalette: "#1e90ff"'
echo '---'
echo "xychart-beta"
echo " title \"count speed [lower = faster]\""
echo " x-axis [zsv, tsv, xsv]"
echo " y-axis \"time\" 0 --> $COUNT_MAX_REAL_TIME_FOR_CHART"
echo " bar [$COUNT_ZSV_AVG_REAL_TIME, $COUNT_TSV_AVG_REAL_TIME, $COUNT_XSV_AVG_REAL_TIME]"
echo '```'
echo
echo "### \`select\`"
echo
echo "<details>"
echo "<summary>Runs (click to expand/collapse)</summary>"
echo
echo '```text'
cat "$SELECT_OUTPUT_FILE"
echo
echo "Averages:"
echo "- zsv: [($SELECT_ZSV_REAL_TIME_VALUES) / $RUNS = $SELECT_ZSV_AVG_REAL_TIME]"
echo "- tsv: [($SELECT_TSV_REAL_TIME_VALUES) / $RUNS = $SELECT_TSV_AVG_REAL_TIME]"
echo "- xsv: [($SELECT_XSV_REAL_TIME_VALUES) / $RUNS = $SELECT_XSV_AVG_REAL_TIME]"
echo '```'
echo
echo "</details>"
echo
echo '```mermaid'
echo '---'
echo 'config:'
echo ' xyChart:'
echo ' width: 600'
echo ' height: 400'
echo ' themeVariables:'
echo ' xyChart:'
echo ' titleColor: "#1e90ff"'
echo ' plotColorPalette: "#1e90ff"'
echo '---'
echo "xychart-beta"
echo " title \"select speed [lower = faster]\""
echo " x-axis [zsv, tsv, xsv]"
echo " y-axis \"time\" 0 --> $SELECT_MAX_REAL_TIME_FOR_CHART"
echo " bar [$SELECT_ZSV_AVG_REAL_TIME, $SELECT_TSV_AVG_REAL_TIME, $SELECT_XSV_AVG_REAL_TIME]"
echo '```'
} >"$MARKDOWN_OUTPUT"
echo "[INF] Generated Markdown output successfully!"

Expand Down

0 comments on commit 7d166c6

Please sign in to comment.