Analyze test output

sergcpp · Oct 18, 2024 · 6ebad93 · 6ebad93
1 parent 3eae39e
commit 6ebad93
Show file tree

Hide file tree

Showing 4 changed files with 181 additions and 9 deletions.
diff --git a/.github/actions/test/action.yml b/.github/actions/test/action.yml
@@ -5,6 +5,8 @@ inputs:
     default: "test_Ray"
   bin-dir:
     required: true
+  out-dir:
+    required: true
   test-args:
     required: true
 runs:
@@ -18,6 +20,7 @@ runs:
         export ASAN_OPTIONS="windows_hook_legacy_allocators=false"
         chmod +x ./${{ inputs.bin-dir }}/${{ inputs.bin-name }}
         WORK_DIR=`pwd`
+        mkdir $WORK_DIR/${{ inputs.out-dir }}
         cd ../../builds/Ray/tests
-        $WORK_DIR/${{ inputs.bin-dir }}/${{ inputs.bin-name }} ${{ inputs.test-args }}
+        $WORK_DIR/${{ inputs.bin-dir }}/${{ inputs.bin-name }} ${{ inputs.test-args }} | tee $WORK_DIR/${{ inputs.out-dir }}/test_Ray_output.txt
       shell: bash
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -99,7 +99,14 @@ jobs:
         uses: ./.github/actions/test
         with:
           bin-dir: "windows-x86_64"
+          out-dir: "windows-x86_64-cpu-output"
           test-args: "--nogpu -j4"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: windows-x86_64-cpu-output
+          path: windows-x86_64-cpu-output/
   test-windows-x86_64-gpu-nv:
     runs-on: [ windows, x86_64, nv ]
     needs:
@@ -115,7 +122,14 @@ jobs:
         uses: ./.github/actions/test
         with:
           bin-dir: "windows-x86_64"
+          out-dir: "windows-x86_64-gpu-nv-output"
           test-args: "--device NV --nocpu -vl 0"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: windows-x86_64-gpu-nv-output
+          path: windows-x86_64-gpu-nv-output/
   test-windows-x86_64-gpu-amd:
     runs-on: [ windows, x86_64, amd ]
     needs:
@@ -131,7 +145,14 @@ jobs:
         uses: ./.github/actions/test
         with:
           bin-dir: "windows-x86_64"
+          out-dir: "windows-x86_64-gpu-amd-output"
           test-args: "--device AMD --nocpu"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: windows-x86_64-gpu-amd-output
+          path: windows-x86_64-gpu-amd-output/
   test-windows-x86_64-gpu-arc:
     runs-on: [ windows, x86_64, arc ]
     needs:
@@ -147,7 +168,14 @@ jobs:
         uses: ./.github/actions/test
         with:
           bin-dir: "windows-x86_64"
+          out-dir: "windows-x86_64-gpu-arc-output"
           test-args: "--device Arc --nocpu"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: windows-x86_64-gpu-arc-output
+          path: windows-x86_64-gpu-arc-output/
   test-windows-x86_64-gpu-xe:
     runs-on: [ windows, x86_64, xe ]
     needs:
@@ -163,7 +191,14 @@ jobs:
         uses: ./.github/actions/test
         with:
           bin-dir: "windows-x86_64"
+          out-dir: "windows-x86_64-gpu-xe-output"
           test-args: "--device Xe --nocpu"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: windows-x86_64-gpu-xe-output
+          path: windows-x86_64-gpu-xe-output/
   test-windows-arm64-cpu:
     runs-on: [ windows, arm64 ]
     needs:
@@ -179,7 +214,14 @@ jobs:
         uses: ./.github/actions/test
         with:
           bin-dir: "windows-arm64"
+          out-dir: "windows-arm64-cpu-output"
           test-args: "--nogpu -j4"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: windows-arm64-cpu-output
+          path: windows-arm64-cpu-output/
   test-windows-arm64-gpu-adreno:
     runs-on: [ windows, arm64, adreno ]
     needs:
@@ -195,7 +237,14 @@ jobs:
         uses: ./.github/actions/test
         with:
           bin-dir: "windows-arm64"
+          out-dir: "windows-arm64-gpu-adreno-output"
           test-args: "--device Adreno --nocpu"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: windows-arm64-gpu-adreno-output
+          path: windows-arm64-gpu-adreno-output/
   test-linux-x86_64-cpu:
     runs-on: [ linux, x86_64 ]
     needs:
@@ -211,7 +260,14 @@ jobs:
         uses: ./.github/actions/test
         with:
           bin-dir: "linux-x86_64"
+          out-dir: "linux-x86_64-cpu-output"
           test-args: "--nogpu -j4"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: linux-x86_64-cpu-output
+          path: linux-x86_64-cpu-output/
   test-linux-x86_64-gpu-amd:
     runs-on: [ linux, x86_64, amd ]
     needs:
@@ -227,7 +283,14 @@ jobs:
         uses: ./.github/actions/test
         with:
           bin-dir: "linux-x86_64"
+          out-dir: "linux-x86_64-gpu-amd-output"
           test-args: "--device AMD --nocpu --nohwrt"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: linux-x86_64-gpu-amd-output
+          path: linux-x86_64-gpu-amd-output/
   test-linux-x86_64-gpu-nv:
     runs-on: [ linux, x86_64, nv ]
     needs:
@@ -243,7 +306,14 @@ jobs:
         uses: ./.github/actions/test
         with:
           bin-dir: "linux-x86_64"
+          out-dir: "linux-x86_64-gpu-nv-output"
           test-args: "--device NV --nocpu"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: linux-x86_64-gpu-nv-output
+          path: linux-x86_64-gpu-nv-output/
   test-macos-arm64-cpu:
     runs-on: [ macos, arm64 ]
     needs:
@@ -259,7 +329,14 @@ jobs:
         uses: ./.github/actions/test
         with:
           bin-dir: "macos-universal"
+          out-dir: "macos-arm64-cpu-output"
           test-args: "--nogpu -j4"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: macos-arm64-cpu-output
+          path: macos-arm64-cpu-output/
   test-macos-arm64-gpu-m1:
     runs-on: [ macos, arm64, m1 ]
     needs:
@@ -275,7 +352,14 @@ jobs:
         uses: ./.github/actions/test
         with:
           bin-dir: "macos-universal"
+          out-dir: "macos-arm64-gpu-m1-output"
           test-args: "--device M1 --nocpu"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: macos-arm64-gpu-m1-output
+          path: macos-arm64-gpu-m1-output/
   test-macos-x86_64-cpu:
     runs-on: [ macos, x86_64 ]
     needs:
@@ -291,4 +375,76 @@ jobs:
         uses: ./.github/actions/test
         with:
           bin-dir: "macos-universal"
+          out-dir: "macos-x86_64-cpu-output"
           test-args: "--nogpu -j4"
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: always()
+        with:
+          name: macos-x86_64-cpu-output
+          path: macos-x86_64-cpu-output/
+  analyze-output:
+    runs-on: [ linux ]
+    needs: [ test-windows-x86_64-cpu, test-windows-x86_64-gpu-nv, test-windows-x86_64-gpu-amd, test-windows-x86_64-gpu-arc, test-windows-x86_64-gpu-xe, test-windows-arm64-cpu, test-windows-arm64-gpu-adreno, test-linux-x86_64-cpu, test-linux-x86_64-gpu-amd, test-linux-x86_64-gpu-nv, test-macos-arm64-cpu, test-macos-arm64-gpu-m1, test-macos-x86_64-cpu ]
+    if: always()
+    steps:
+      - name: Checkout Workflows
+        uses: actions/checkout@v4
+        with:
+          sparse-checkout: scripts/analyze_output.py
+      - uses: actions/download-artifact@v3
+        with:
+          name: windows-x86_64-cpu-output
+          path: windows-x86_64-cpu-output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: windows-x86_64-gpu-nv-output
+          path: windows-x86_64-gpu-nv-output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: windows-x86_64-gpu-amd-output
+          path: windows-x86_64-gpu-amd-output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: windows-x86_64-gpu-arc-output
+          path: windows-x86_64-gpu-arc-output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: windows-x86_64-gpu-xe-output
+          path: windows-x86_64-gpu-xe-output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: windows-arm64-cpu-output
+          path: windows-arm64-cpu-output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: windows-arm64-gpu-adreno-output
+          path: windows-arm64-gpu-adreno-output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: linux-x86_64-cpu-output
+          path: linux-x86_64-cpu-output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: linux-x86_64-gpu-amd-output
+          path: linux-x86_64-gpu-amd-output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: linux-x86_64-gpu-nv-output
+          path: linux-x86_64-gpu-nv-output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: macos-arm64-cpu-output
+          path: macos-arm64-cpu-output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: macos-arm64-gpu-m1-output
+          path: macos-arm64-gpu-m1-output/
+      - uses: actions/download-artifact@v3
+        with:
+          name: macos-x86_64-cpu-output
+          path: macos-x86_64-cpu-output/
+      - name: Analyze Output
+        run: |
+          python scripts/analyze_output.py windows-x86_64-cpu-output/test_Ray_output.txt windows-x86_64-gpu-nv-output/test_Ray_output.txt windows-x86_64-gpu-amd-output/test_Ray_output.txt windows-x86_64-gpu-arc-output/test_Ray_output.txt windows-x86_64-gpu-xe-output/test_Ray_output.txt windows-arm64-cpu-output/test_Ray_output.txt windows-arm64-gpu-adreno-output/test_Ray_output.txt linux-x86_64-cpu-output/test_Ray_output.txt linux-x86_64-gpu-amd-output/test_Ray_output.txt linux-x86_64-gpu-nv-output/test_Ray_output.txt macos-arm64-cpu-output/test_Ray_output.txt macos-arm64-gpu-m1-output/test_Ray_output.txt macos-x86_64-cpu-output/test_Ray_output.txt
+        shell: bash
diff --git a/scripts/analyze_output.py b/scripts/analyze_output.py
@@ -48,7 +48,17 @@ def main():
             except:
                 print("Failed to process ", sys.argv[i])
 
+    # Print failed tests first
+    condition = lambda key, value: value['psnr_tested'] < value['psnr_threshold'] or (value['psnr_tested'] - value['psnr_threshold']) > 0.5 or value['fireflies_tested'] > value['fireflies_threshold'] or (value['fireflies_tested'] - value['fireflies_threshold']) > 100
+    failed_tests = {key: value for key, value in all_tests.items() if condition(key, value)}
+
+    print("-- Failed tests --")
+    print(json.dumps(failed_tests, indent=4))
+
+    print("-- All tests --")
     print(json.dumps(all_tests, indent=4))
 
+    sys.exit(-1 if len(failed_tests) > 0 else 0)
+
 if __name__ == "__main__":
     main()
diff --git a/tests/test_materials.cpp b/tests/test_materials.cpp
@@ -1256,7 +1256,8 @@ void test_alpha_mat4(const char *arch_list[], const char *preferred_device) {
 
 void test_two_sided_mat(const char *arch_list[], const char *preferred_device) {
     const int SampleCount = 12;
-    const int PixThres = 822;
+    const double MinPSNR = 28.50;
+    const int PixThres = 800;
 
     Ray::principled_mat_desc_t front_mat_desc;
     front_mat_desc.base_color[0] = 0.5f;
@@ -1268,7 +1269,7 @@ void test_two_sided_mat(const char *arch_list[], const char *preferred_device) {
 
     const char *textures[] = {"test_data/textures/Fence007A_2K_Opacity.dds"};
 
-    run_material_test(arch_list, preferred_device, "two_sided_mat", front_mat_desc, SampleCount, FastMinPSNR, PixThres,
+    run_material_test(arch_list, preferred_device, "two_sided_mat", front_mat_desc, SampleCount, MinPSNR, PixThres,
                       eDenoiseMethod::None, false, textures, eTestScene::Two_Sided);
 }
 
@@ -1585,7 +1586,8 @@ void test_complex_mat5_sphere_light(const char *arch_list[], const char *preferr
 
 void test_complex_mat5_spot_light(const char *arch_list[], const char *preferred_device) {
     const int SampleCount = 2;
-    const int PixThres = 671;
+    const double MinPSNR = 30.10;
+    const int PixThres = 670;
 
     Ray::principled_mat_desc_t metal_mat_desc;
     metal_mat_desc.base_texture = Ray::TextureHandle{0};
@@ -1599,7 +1601,7 @@ void test_complex_mat5_spot_light(const char *arch_list[], const char *preferred
         "test_data/textures/gold-scuffed_basecolor-boosted.tga", "test_data/textures/gold-scuffed_normal.tga",
         "test_data/textures/gold-scuffed_roughness.tga", "test_data/textures/gold-scuffed_metallic.tga"};
 
-    run_material_test(arch_list, preferred_device, "complex_mat5_spot_light", metal_mat_desc, SampleCount, FastMinPSNR,
+    run_material_test(arch_list, preferred_device, "complex_mat5_spot_light", metal_mat_desc, SampleCount, MinPSNR,
                       PixThres, eDenoiseMethod::None, false, textures, eTestScene::Standard_SpotLight);
 }
 
@@ -1787,7 +1789,8 @@ void test_complex_mat6_sphere_light(const char *arch_list[], const char *preferr
 
 void test_complex_mat6_spot_light(const char *arch_list[], const char *preferred_device) {
     const int SampleCount = 2;
-    const int PixThres = 239;
+    const double MinPSNR = 29.90;
+    const int PixThres = 240;
 
     Ray::principled_mat_desc_t olive_mat_desc;
     olive_mat_desc.base_color[0] = 0.836164f;
@@ -1797,7 +1800,7 @@ void test_complex_mat6_spot_light(const char *arch_list[], const char *preferred
     olive_mat_desc.transmission = 1.0f;
     olive_mat_desc.ior = 2.3f;
 
-    run_material_test(arch_list, preferred_device, "complex_mat6_spot_light", olive_mat_desc, SampleCount, FastMinPSNR,
+    run_material_test(arch_list, preferred_device, "complex_mat6_spot_light", olive_mat_desc, SampleCount, MinPSNR,
                       PixThres, eDenoiseMethod::None, false, nullptr, eTestScene::Standard_SpotLight);
 }
 
@@ -1857,8 +1860,8 @@ void test_complex_mat7_principled(const char *arch_list[], const char *preferred
 
 void test_ray_flags(const char *arch_list[], const char *preferred_device) {
     const int SampleCount = 48;
-    const double MinPSNR = 29.3;
-    const int PixThres = 2129;
+    const double MinPSNR = 29.90;
+    const int PixThres = 1850;
 
     Ray::principled_mat_desc_t unused;
     run_material_test(arch_list, preferred_device, "ray_flags", unused, SampleCount, MinPSNR, PixThres,