Attempt to integrate rust benchmarking (mix of EZKL benches + existin… #1
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Bench | |
| on: | |
| workflow_dispatch: | |
| push: | |
| branches: | |
| - main | |
| paths: | |
| - .github/workflows/benchmarks.yml | |
| - polylang/Cargo.lock | |
| - polylang/benches/bench.rs | |
| - miden/Cargo.lock | |
| - miden/benches/bench.rs | |
| - risc-zero/host/Cargo.toml | |
| - risc-zero/host/benches/bench.rs | |
| - noir/benches/bench.rs | |
| - noir/Cargo.lock | |
| env: | |
| AWS_REGION: us-east-1 | |
| # comma seperated list of instances to start/stop for this action | |
| INSTANCE_ID: i-0c9244d7e2b3893b5 | |
| jobs: | |
| start-runner: | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: ${{ env.AWS_REGION }} | |
| - name: Start existing EC2 instance | |
| run: | | |
| aws ec2 start-instances --instance-ids ${INSTANCE_ID} | |
| echo "Waiting for instance to be ready..." | |
| aws ec2 wait instance-running --instance-ids ${INSTANCE_ID} | |
| polylang: | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: ['ubuntu-16-shared', 'ubuntu-latest-64-cores'] | |
| needs: start-runner | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Install Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| override: true | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Run Polylang Multi CPU benchmarks | |
| working-directory: ./polylang | |
| if: matrix.runner != 'macos-dedicated' | |
| env: | |
| BENCHY_OUTPUT_DIR: "../.benchmarks/${{ matrix.runner }}" | |
| run: | | |
| cargo bench -F multi-cpu | |
| - name: Run Polylang Metal benchmarks | |
| if: matrix.runner == 'macos-dedicated' | |
| working-directory: ./polylang | |
| env: | |
| BENCHY_OUTPUT_DIR: "../.benchmarks/${{ matrix.runner }}" | |
| run: | | |
| cargo bench | |
| - name: Upload .benchmarks as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| path: .benchmarks/ | |
| miden: | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: ['ubuntu-16-shared', 'ubuntu-latest-64-cores'] | |
| needs: start-runner | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Install Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| override: true | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Run Miden Multi CPU benchmarks | |
| working-directory: ./miden | |
| if: matrix.runner != 'macos-dedicated' | |
| env: | |
| BENCHY_OUTPUT_DIR: "../.benchmarks/${{ matrix.runner }}" | |
| run: | | |
| cargo bench -F multi-cpu | |
| - name: Run Miden Metal benchmarks | |
| if: matrix.runner == 'macos-dedicated' | |
| working-directory: ./miden | |
| env: | |
| BENCHY_OUTPUT_DIR: "../.benchmarks/${{ matrix.runner }}" | |
| run: | | |
| cargo bench -F metal | |
| - name: Upload .benchmarks as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| path: .benchmarks/ | |
| risc_zero: | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: ['ubuntu-16-shared', 'ubuntu-latest-64-cores'] | |
| needs: start-runner | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Install Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| override: true | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Cache cargo bin | |
| uses: actions/cache@v2 | |
| with: | |
| path: ~/.cargo/bin | |
| key: ${{ runner.os }}-cargo-bin-${{ hashFiles('**/Cargo.lock') }} | |
| - name: Install risczero | |
| run: cargo install cargo-risczero && cargo risczero install | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Run Risc-Zero Multi CPU benchmarks | |
| working-directory: ./risc-zero/host | |
| if: matrix.runner != 'macos-dedicated' | |
| env: | |
| BENCHY_OUTPUT_DIR: "../../.benchmarks/${{ matrix.runner }}" | |
| run: | | |
| cargo bench | |
| - name: Run Risc-Zero Metal benchmarks | |
| if: matrix.runner == 'macos-dedicated' | |
| working-directory: ./risc-zero/host | |
| env: | |
| BENCHY_OUTPUT_DIR: "../../.benchmarks/${{ matrix.runner }}" | |
| run: | | |
| cargo bench -F metal | |
| - name: Upload .benchmarks as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| path: .benchmarks/ | |
| noir: | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: ['ubuntu-16-shared', 'ubuntu-latest-64-cores'] | |
| needs: start-runner | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Install Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| override: true | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Install Nix | |
| run: | | |
| sh <(curl -L https://nixos.org/nix/install) | |
| source $HOME/.nix-profile/etc/profile.d/nix.sh | |
| - name: Run cargo bench in Nix shell | |
| working-directory: ./noir | |
| env: | |
| BENCHY_OUTPUT_DIR: "../.benchmarks/${{ matrix.runner }}" | |
| run: | | |
| source $HOME/.nix-profile/etc/profile.d/nix.sh | |
| nix-shell --run "nix-shell -p clang --run 'cargo bench'" | |
| - name: Upload .benchmarks as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| path: .benchmarks/ | |
| leo: | |
| runs-on: ${{ matrix.runner }} | |
| strategy: | |
| matrix: | |
| runner: ['ubuntu-16-shared', 'ubuntu-latest-64-cores'] | |
| needs: start-runner | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Install Rust | |
| uses: actions-rs/toolchain@v1 | |
| with: | |
| toolchain: stable | |
| override: true | |
| - uses: Swatinem/rust-cache@v2 | |
| - name: Run Leo benchmarks | |
| working-directory: ./leo | |
| env: | |
| BENCHY_OUTPUT_DIR: "../.benchmarks/${{ matrix.runner }}" | |
| run: | | |
| cargo bench | |
| - name: Upload .benchmarks as artifact | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| path: .benchmarks/ | |
| commit: | |
| runs-on: ubuntu-latest | |
| needs: [polylang, miden, risc_zero, noir] | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v2 | |
| - name: Download benchmarks artifacts | |
| uses: actions/download-artifact@v3 | |
| with: | |
| path: .benchmarks | |
| - name: Combine results | |
| run: ./combine.py | |
| - name: Copy benchmark results to site | |
| run: | | |
| cp benchmarks.json site/src/fixtures/benchmarks.json | |
| - name: Commit benchmark results | |
| run: | | |
| git config --global user.email "calum.moore@gmail.com" \ | |
| && git config --global user.name "Polybase CI" \ | |
| && git add . \ | |
| && git commit -m "Update benchmarks" \ | |
| && git push | |
| clean-up: | |
| runs-on: ubuntu-latest | |
| needs: [polylang, miden, risc_zero, noir] | |
| if: always() | |
| steps: | |
| - name: Configure AWS credentials | |
| uses: aws-actions/configure-aws-credentials@v1 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: Stop EC2 instance | |
| run: aws ec2 stop-instances --instance-ids ${{ env.INSTANCE_ID }} |