diff --git a/toolchain/mfc/test/cases.py b/toolchain/mfc/test/cases.py index 50b211bba..7d3605f7c 100644 --- a/toolchain/mfc/test/cases.py +++ b/toolchain/mfc/test/cases.py @@ -807,7 +807,7 @@ def foreach_example(): continue # # List of currently broken examples -> currently attempting to fix! - brokenCases = ["2D_ibm_cfl_dt", "1D_sodHypo", "2D_viscous", "2D_laplace_pressure_jump", "2D_bubbly_steady_shock", "2D_advection", "2D_hardcodied_ic", "2D_ibm_multiphase", "2D_acoustic_broadband", "1D_inert_shocktube", "1D_reactive_shocktube", "2D_ibm_steady_shock", "3D_performance_test", "3D_ibm_stl_ellipsoid", "3D_sphbubcollapse", "2D_ibm_stl_wedge", "3D_ibm_stl_pyramid"] + brokenCases = ["2D_ibm_cfl_dt", "1D_sodHypo", "2D_viscous", "2D_laplace_pressure_jump", "2D_bubbly_steady_shock", "2D_advection", "2D_hardcodied_ic", "2D_ibm_multiphase", "2D_acoustic_broadband", "1D_inert_shocktube", "1D_reactive_shocktube", "2D_ibm_steady_shock", "3D_performance_test", "3D_ibm_stl_ellipsoid", "3D_sphbubcollapse", "2D_ibm_stl_wedge", "3D_ibm_stl_pyramid", "3D_ibm_bowshock"] if path in brokenCases: continue name = f"{path.split('_')[0]} -> Example -> {'_'.join(path.split('_')[1:])}" diff --git a/toolchain/mfc/test/test.py b/toolchain/mfc/test/test.py index 3e32b5c51..ac0d13d2e 100644 --- a/toolchain/mfc/test/test.py +++ b/toolchain/mfc/test/test.py @@ -19,9 +19,12 @@ nFAIL = 0 nPASS = 0 nSKIP = 0 +errors = [] def __filter(cases_) -> typing.List[TestCase]: cases = cases_[:] + selected_cases = [] + skipped_cases = [] # Check "--from" and "--to" exist and are in the right order bFoundFrom, bFoundTo = (False, False) @@ -51,16 +54,20 @@ def __filter(cases_) -> typing.List[TestCase]: for case in cases[:]: if case.ppn > 1 and not ARG("mpi"): cases.remove(case) + skipped_cases.append(case) if ARG("percent") == 100: - return cases + return cases, skipped_cases - return sample(cases, k=int(len(cases)*ARG("percent")/100.0)) + selected_cases = sample(cases, k=int(len(cases)*ARG("percent")/100.0)) + skipped_cases = [item for item in cases if item not in selected_cases] + return selected_cases, skipped_cases def test(): # pylint: disable=global-statement, global-variable-not-assigned global nFAIL, nPASS, nSKIP + global errors cases = list_cases() @@ -75,7 +82,8 @@ def test(): return - cases = [ _.to_case() for _ in __filter(cases) ] + cases, skipped_cases = __filter(cases) + cases = [ _.to_case() for _ in cases ] if ARG("list"): table = rich.table.Table(title="MFC Test Cases", box=rich.table.box.SIMPLE) @@ -125,9 +133,23 @@ def test(): [ sched.Task(ppn=case.ppn, func=handle_case, args=[case], load=case.get_cell_count()) for case in cases ], ARG("jobs"), ARG("gpus")) + nSKIP = len(skipped_cases) cons.print() cons.unindent() - cons.print(f"\nTest Summary: [bold green]{nPASS}[/bold green] passed, [bold red]{nFAIL}[/bold red] failed, [bold yellow]{nSKIP}[/bold yellow] skipped.") + cons.print(f"\nTest Summary: [bold green]{nPASS}[/bold green] passed, [bold red]{nFAIL}[/bold red] failed, [bold yellow]{nSKIP}[/bold yellow] skipped.\n") + + # Print a summary of all errors at the end if errors exist + if len(errors) != 0: + cons.print(f"[bold red]Failed Cases[/bold red]\n") + for e in errors: + cons.print(e) + + # Print the list of skipped cases + if len(skipped_cases) != 0: + cons.print("[bold yellow]Skipped Cases[/bold yellow]\n") + for c in skipped_cases: + cons.print(f"[bold yellow]{c.trace}[/bold yellow]") + exit(nFAIL) @@ -214,6 +236,7 @@ def _handle_case(case: TestCase, devices: typing.Set[int]): def handle_case(case: TestCase, devices: typing.Set[int]): # pylint: disable=global-statement, global-variable-not-assigned global nFAIL, nPASS, nSKIP + global errors nAttempts = 0 @@ -226,9 +249,11 @@ def handle_case(case: TestCase, devices: typing.Set[int]): except Exception as exc: if nAttempts < ARG("max_attempts"): cons.print(f"[bold yellow] Attempt {nAttempts}: Failed test {case.get_uuid()}. Retrying...[/bold yellow]") + errors.append(f"[bold yellow] Attempt {nAttempts}: Failed test {case.get_uuid()}. Retrying...[/bold yellow]") continue nFAIL += 1 cons.print(f"[bold red]Failed test {case} after {nAttempts} attempt(s).[/bold red]") - cons.print(f"{exc}") + errors.append(f"[bold red]Failed test {case} after {nAttempts} attempt(s).[/bold red]") + errors.append(f"{exc}") return