Don't do a loadgen release from dev branch, add python3.12,13 to loadgen test, exclude power-checker from auto format

.github/workflows/auto-update-dev.yml

@@ -0,0 +1,34 @@
+name: Auto-Update Dev Branch from Master
+
+on:
+  push:
+    branches:
+      - master  # Trigger workflow on commits to 'dev' branch
+
+jobs:
+  update-main:
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write  # Required to push to protected branches
+
+    steps:
+      - name: Checkout Main Branch
+        uses: actions/checkout@v4
+        with:
+          ref: dev
+          fetch-depth: 0
+          ssh-key: ${{ secrets.DEPLOY_KEY }}
+
+      - name: Configure Git User
+        run: |
+          git config user.name "github-actions"
+          git config user.email "github-actions@github.com"
+
+      - name: Merge auto-update into dev
+        run: |
+          git fetch origin master:master
+          git merge --no-ff master -m "Auto-merge updates from master branch"
+
+      - name: Push Changes to Main
+        run: |
+          git push origin dev

.github/workflows/build_wheels.yml

@@ -7,7 +7,6 @@ on:
     branches:
       - master
       - loadgen-release
-      - dev
     paths:
       - loadgen/**
 

.github/workflows/format.yml

@@ -38,7 +38,7 @@ jobs:
           for FILE in $(git diff --name-only $filter | grep -E '.*\.py$')
           do
             # Check if the file still exists in the working tree
-            if [ -f "$FILE" ]; then
+            if [ -f "$FILE" ] && [ "$FILE" != "tools/submission/power/power_checker.py" ]; then
               autopep8 --in-place -a "$FILE"
               git add "$FILE"
             fi

.github/workflows/reset-branch.yml

@@ -0,0 +1,42 @@
+name: Reset Current Branch to Upstream After Squash Merge
+
+on:
+  workflow_dispatch:
+    inputs:
+      branch:
+        description: 'Branch to reset (leave blank for current branch)'
+        required: false
+        default: 'dev'
+
+jobs:
+  reset-branch:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Detect Current Branch
+        if: ${{ inputs.branch == '' }}
+        run: echo "branch=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_ENV
+
+      - name: Use Input Branch
+        if: ${{ inputs.branch != '' }}
+        run: echo "branch=${{ inputs.branch }}" >> $GITHUB_ENV
+
+      - name: Add Upstream Remote
+        run: |
+          git remote add upstream https://github.com/mlcommons/inference.git
+          git fetch upstream
+      - name: Reset Branch to Upstream
+        run: |
+          git checkout ${{ env.branch }}
+          git reset --hard upstream/${{ env.branch }}
+        if: success()
+
+      - name: Force Push to Origin
+        run: |
+          git push origin ${{ env.branch }} --force-with-lease
+        if: success()

.github/workflows/test-loadgen.yml

@@ -21,7 +21,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11"]
+        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12", "3.13"]
 
     steps:
     - uses: actions/checkout@v3

.github/workflows/test-submission-generation.yml

@@ -0,0 +1,52 @@
+# This workflow will test the submission generation using MLPerf Automation 
+
+name: CM based Submission Generation
+
+on:
+  pull_request:
+    branches: [ "master", "dev" ]
+    paths:
+      - '.github/workflows/test-submission-generation.yml'
+      - '**'  
+      - '!**.md'
+jobs:
+  submission_generation:
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, windows-latest, macos-latest]
+        python-version: [ "3.12" ]
+        division: ["closed", "open", "closed-open"]
+        category: ["datacenter", "edge"]
+        case: ["closed"]
+        action: ["run", "docker"]
+        exclude:
+          - os: macos-latest
+          - os: windows-latest
+          - category: "edge"
+
+    steps:
+    - uses: actions/checkout@v4
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        pip install cm4mlops
+    - name: Pull repo where test cases are uploaded
+      run: |
+        git clone -b submission-generation-examples https://github.com/mlcommons/inference.git submission_generation_examples
+    - name: Run Submission Generation - ${{ matrix.case }} ${{ matrix.action }} ${{ matrix.category }} ${{ matrix.division }} 
+      continue-on-error: true
+      run: |
+        if [ "${{ matrix.case }}" == "closed" ]; then
+          description="Test submission - contains closed edge and datacenter"
+        elif [ "${{ matrix.case }}" == "closed-power" ]; then
+          description="Test submission - contains closed-power edge and datacenter results"
+        fi
+        # Dynamically set the log group to simulate a dynamic step name
+        echo "::group::$description"
+        cm ${{ matrix.action }} script --tags=generate,inference,submission  --adr.compiler.tags=gcc --version=v5.0 --clean --preprocess_submission=yes --submission_base_dir=mysubmissions --results_dir=$PWD/submission_generation_tests/${{ matrix.case }}/ --run-checker --submitter=MLCommons --tar=yes --division=${{ matrix.division }} --env.CM_DETERMINE_MEMORY_CONFIGURATION=yes --quiet
+        cm ${{ matrix.action }} script --tags=run,submission,checker --submitter_id_off=mysubmitter_id --tar=yes --submission_dir=mysubmissions/submissions --submission_tar_file=mysubmission.tar.gz

docs/benchmarks/language/get-llama3_1-405b-data.md

@@ -0,0 +1,41 @@
+---
+hide:
+  - toc
+---
+
+# Text Summarization using LLAMA3.1-405b
+
+## Dataset
+
+The benchmark implementation run command will automatically download the validation and calibration datasets and do the necessary preprocessing. In case you want to download only the datasets, you can use the below commands.
+
+=== "Validation"
+
+    ### Get Validation Dataset
+    ```
+    cm run script --tags=get,dataset,mlperf,inference,llama3,_validation --outdirname=<path to download> -j
+    ```
+
+=== "Calibration"
+
+    ### Get Calibration Dataset
+    ```
+    cm run script --tags=get,dataset,mlperf,inference,llama3,_calibration --outdirname=<path to download> -j
+    ```
+
+## Model
+The benchmark implementation run command will automatically download the required model and do the necessary conversions. In case you want to only download the official model, you can use the below commands.
+
+Get the Official MLPerf LLAMA3.1-405b Model
+
+=== "Pytorch"
+
+    ### Pytorch
+    ```
+    cm run script --tags=get,ml-model,llama3 --outdirname=<path to download> --hf_token=<huggingface access token> -j
+    ```
+
+!!! tip
+
+    Downloading llama3.1-405B model from Hugging Face will require an [**access token**](https://huggingface.co/settings/tokens) which could be generated for your account. Additionally, ensure that your account has access to the [llama3.1-405B](https://huggingface.co/meta-llama/Llama-3.1-405B-Instruct) model. 
+

docs/benchmarks/language/llama3_1-405b.md

@@ -0,0 +1,13 @@
+---
+hide:
+  - toc
+---
+
+# Text Summarization using LLAMA3_1-405b
+
+=== "MLCommons-Python"
+    ## MLPerf Reference Implementation in Python
+
+{{ mlperf_inference_implementation_readme (4, "llama3_1-405b-99", "reference", devices=["CPU","CUDA"]) }}
+
+{{ mlperf_inference_implementation_readme (4, "llama3_1-405b-99.9", "reference", devices=["CPU","CUDA"]) }}

docs/index.md

@@ -163,7 +163,7 @@ The currently valid [MLPerf Inference Benchmarks](index_gh.md) as of MLPerf infe
     - **Dataset Size**: 788,379
     - **QSL Size**: 788,379
 - **Number of Parameters**: 
-- **Reference Model Accuracy**: ACC = ?
+- **Reference Model Accuracy**: ACC = 72.86%
 - **Server Scenario Latency Constraint**: N/A
 - **Equal Issue mode**: True
 - **High accuracy variant**: No