From 50c561b1420bfd99a9da8f785b19a02d712ebe37 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?=
 <tomas.drvostep@gmail.com>
Date: Mon, 8 Jan 2024 12:43:30 +0100
Subject: [PATCH 1/7] Very WIP JET integration idea

---
 Project.toml        |  1 +
 src/ReTestItems.jl  | 15 +++++++++++++--
 test/internals.jl   |  2 +-
 test/log_capture.jl |  2 +-
 4 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/Project.toml b/Project.toml
index 2271b798..0a65e82a 100644
--- a/Project.toml
+++ b/Project.toml
@@ -4,6 +4,7 @@ version = "1.23.0"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
+JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
diff --git a/src/ReTestItems.jl b/src/ReTestItems.jl
index a882b85c..f231c405 100644
--- a/src/ReTestItems.jl
+++ b/src/ReTestItems.jl
@@ -8,6 +8,7 @@ using Pkg: Pkg
 using TestEnv
 using Logging
 using LoggingExtras
+import JET
 
 export runtests, runtestitem
 export @testsetup, @testitem
@@ -400,7 +401,7 @@ end
 # The provided `worker_num` is only for logging purposes, and not persisted as part of the worker.
 function start_worker(proj_name, nworker_threads, worker_init_expr, ntestitems; worker_num=nothing)
     w = Worker(; threads="$nworker_threads")
-    i = worker_num == nothing ? "" : " $worker_num"
+    i = worker_num === nothing ? "" : " $worker_num"
     # remote_fetch here because we want to make sure the worker is all setup before starting to eval testitems
     remote_fetch(w, quote
         using ReTestItems, Test
@@ -980,7 +981,7 @@ function runtestitem(
         # environment = Module()
         @debugv 1 "Running test item $(repr(name))$(_on_worker())."
         _, stats = @timed_with_compilation _redirect_logs(logs == :eager ? DEFAULT_STDOUT[] : logpath(ti)) do
-            with_source_path(() -> Core.eval(Main, mod_expr), ti.file)
+            with_source_path(() -> (Core.eval(Main, mod_expr); jet_test(ti, mod_expr)), ti.file)
             with_source_path(() -> Core.eval(Main, test_end_mod_expr), ti.file)
             nothing # return nothing as the first return value of @timed_with_compilation
         end
@@ -1035,6 +1036,16 @@ function convert_results_to_be_transferrable(res::Test.Pass)
     return res
 end
 
+function jet_test(ti, mod_expr)
+    if ti.jet !== :none
+        # TODO: Don't round-trip through string, we need to figure out what sort of transformations JET does to a string to produce an Expr
+        # and use that directly.
+        Test.@testset "JET $(repr(ti.jet)) mode" begin
+            JET.test_text(replace(string(mod_expr), "\$(Expr(:softscope, true))" => "eval(Expr(:softscope, true))"), ti.file; mode=ti.jet)
+        end
+    end
+end
+
 convert_results_to_be_transferrable(x) = x
 
 end # module ReTestItems
diff --git a/test/internals.jl b/test/internals.jl
index 67e28118..7d07f268 100644
--- a/test/internals.jl
+++ b/test/internals.jl
@@ -169,7 +169,7 @@ end # `include_testfiles!` testset
 @testset "report_empty_testsets" begin
     using ReTestItems: TestItem, report_empty_testsets, PerfStats, ScheduledForEvaluation
     using Test: DefaultTestSet, Fail, Error
-    ti = TestItem(Ref(42), "Dummy TestItem", "DummyID", [], false, [], 0, nothing, false, "source/path", 42, ".", nothing)
+    ti = TestItem(Ref(42), "Dummy TestItem", "DummyID", [], false, [], 0, nothing, false, "source/path", 42, ".", nothing, :none)
 
     ts = DefaultTestSet("Empty testset")
     report_empty_testsets(ti, ts)
diff --git a/test/log_capture.jl b/test/log_capture.jl
index 78d80fb2..20196e61 100644
--- a/test/log_capture.jl
+++ b/test/log_capture.jl
@@ -33,7 +33,7 @@ end
 @testset "log capture -- reporting" begin
     setup1 = @testsetup module TheTestSetup1 end
     setup2 = @testsetup module TheTestSetup2 end
-    ti = TestItem(Ref(42), "TheTestItem", "ID007", [], false, [], 0, nothing, false, "source/path", 42, ".", nothing)
+    ti = TestItem(Ref(42), "TheTestItem", "ID007", [], false, [], 0, nothing, false, "source/path", 42, ".", nothing, :none)
     push!(ti.testsetups, setup1)
     push!(ti.testsetups, setup2)
     push!(ti.testsets, Test.DefaultTestSet("dummy"))

From ec14054d448a716a451d200d64f6d6e9bb7e7610 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?=
 <tomas.drvostep@gmail.com>
Date: Tue, 9 Jan 2024 13:58:13 +0100
Subject: [PATCH 2/7] Make JET a weak dependency

---
 Project.toml       |  7 ++++++-
 ext/JETExt.jl      | 36 ++++++++++++++++++++++++++++++++++++
 src/ReTestItems.jl | 18 ++++++++----------
 src/macros.jl      | 11 +++++++++--
 4 files changed, 59 insertions(+), 13 deletions(-)
 create mode 100644 ext/JETExt.jl

diff --git a/Project.toml b/Project.toml
index 0a65e82a..81157dff 100644
--- a/Project.toml
+++ b/Project.toml
@@ -4,7 +4,6 @@ version = "1.23.0"
 
 [deps]
 Dates = "ade2ca70-3891-5945-98fb-dc099432e06a"
-JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
 Logging = "56ddb016-857b-54e1-b83d-db4d58db5568"
 LoggingExtras = "e6f89c97-d47a-5376-807f-9c37f3926c36"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
@@ -13,6 +12,12 @@ Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 TestEnv = "1e6cf692-eddd-4d53-88a5-2d735e33781b"
 
+[weakdeps]
+JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
+
+[extensions]
+JETExt = "JET"
+
 [compat]
 Dates = "1"
 Logging = "1"
diff --git a/ext/JETExt.jl b/ext/JETExt.jl
new file mode 100644
index 00000000..2c886a32
--- /dev/null
+++ b/ext/JETExt.jl
@@ -0,0 +1,36 @@
+module JETExt
+
+import JET, ReTestItems, Test
+
+function _analyze_toplevel(ex, file, mode)
+    toplevelex = Expr(:toplevel, ex)
+    analyzer = JET.JETAnalyzer(; mode)
+    config = JET.ToplevelConfig(; mode)
+    res = JET.virtual_process(toplevelex, file, analyzer, config)
+    return JET.JETToplevelResult(analyzer, res, "analyze_toplevel"; mode)
+end
+
+function ReTestItems.jet_test(ti::ReTestItems.TestItem, mod_expr::Expr)
+    ti.jet == :none && return nothing
+
+    onfail(::Function, ::Test.Pass) = nothing
+    onfail(f::Function, ::Test.Fail) = f()
+
+    @assert mod_expr.head === :module "Expected the test item expression to be wrapped in a module, got $(repr(mod_expr.head))"
+    Test.@testset "JET $(repr(ti.jet)) mode" begin
+        result = _analyze_toplevel(mod_expr, ti.file, ti.jet)
+        no_jet_errors = isempty(result)
+        onfail(Test.@test no_jet_errors) do
+            JET.print_reports(
+                stdout,
+                JET.get_reports(result),
+                # Remove the name of the module JET uses for virtualization the code and the name of the module
+                # we wrap the test items in.
+                JET.gen_postprocess(result.res.actual2virtual) ∘ x->replace(x, string("var\"", mod_expr.args[2], "\"") => ""),
+            )
+        end
+    end
+    return nothing
+end
+
+end
diff --git a/src/ReTestItems.jl b/src/ReTestItems.jl
index f231c405..e47a9572 100644
--- a/src/ReTestItems.jl
+++ b/src/ReTestItems.jl
@@ -8,7 +8,6 @@ using Pkg: Pkg
 using TestEnv
 using Logging
 using LoggingExtras
-import JET
 
 export runtests, runtestitem
 export @testsetup, @testitem
@@ -1036,16 +1035,15 @@ function convert_results_to_be_transferrable(res::Test.Pass)
     return res
 end
 
-function jet_test(ti, mod_expr)
-    if ti.jet !== :none
-        # TODO: Don't round-trip through string, we need to figure out what sort of transformations JET does to a string to produce an Expr
-        # and use that directly.
-        Test.@testset "JET $(repr(ti.jet)) mode" begin
-            JET.test_text(replace(string(mod_expr), "\$(Expr(:softscope, true))" => "eval(Expr(:softscope, true))"), ti.file; mode=ti.jet)
-        end
+convert_results_to_be_transferrable(x) = x
+
+# This method signature must be less specific than the overload in ext/JETExt.jl
+function jet_test(ti::Any, mod_expr::Expr)
+    ti.jet == :none && return nothing
+    Test.@testset "JET package extension failure: JET not loaded, ignoring \"jet=$(repr(ti.jet))\"" begin
+        Test.@test_broken false
     end
+    return nothing
 end
 
-convert_results_to_be_transferrable(x) = x
-
 end # module ReTestItems
diff --git a/src/macros.jl b/src/macros.jl
index dbd9426f..fd714a8e 100644
--- a/src/macros.jl
+++ b/src/macros.jl
@@ -125,6 +125,7 @@ struct TestItem
     line::Int
     project_root::String
     code::Any
+    jet::Symbol
     testsetups::Vector{TestSetup} # populated by runtests coordinator
     workerid::Base.RefValue{Int} # populated when the test item is scheduled
     testsets::Vector{DefaultTestSet} # populated when the test item is finished running
@@ -132,10 +133,10 @@ struct TestItem
     stats::Vector{PerfStats} # populated when the test item is finished running
     scheduled_for_evaluation::ScheduledForEvaluation # to keep track of whether the test item has been scheduled for evaluation
 end
-function TestItem(number, name, id, tags, default_imports, setups, retries, timeout, skip, file, line, project_root, code)
+function TestItem(number, name, id, tags, default_imports, setups, retries, timeout, skip, file, line, project_root, code, jet)
     _id = @something(id, repr(hash(name, hash(relpath(file, project_root)))))
     return TestItem(
-        number, name, _id, tags, default_imports, setups, retries, timeout, skip, file, line, project_root, code,
+        number, name, _id, tags, default_imports, setups, retries, timeout, skip, file, line, project_root, code, jet,
         TestSetup[],
         Ref{Int}(0),
         DefaultTestSet[],
@@ -250,6 +251,7 @@ macro testitem(nm, exs...)
     setup = Any[]
     skip = false
     _id = nothing
+    jet = QuoteNode(:none)
     _run = true  # useful for testing `@testitem` itself
     _source = QuoteNode(__source__)
     if length(exs) > 1
@@ -293,6 +295,10 @@ macro testitem(nm, exs...)
             elseif kw == :_source
                 _source = ex.args[2]
                 @assert isa(_source, Union{QuoteNode,Expr})
+            elseif kw == :jet
+                jet = ex.args[2]
+                @assert isa(jet, QuoteNode)
+                @assert jet in QuoteNode.((:none, :typo, :basic, :sound))
             else
                 error("unknown `@testitem` keyword arg `$(ex.args[1])`")
             end
@@ -309,6 +315,7 @@ macro testitem(nm, exs...)
             $String($_source.file), $_source.line,
             $gettls(:__RE_TEST_PROJECT__, "."),
             $q,
+            $jet,
         )
             if !$_run
                 $ti

From 0f9d85e1a5eb40f466d74357956cc9d4df25604b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?=
 <tomas.drvostep@gmail.com>
Date: Tue, 9 Jan 2024 13:58:57 +0100
Subject: [PATCH 3/7] [WIP] Introduce `test/packages.PkgUsingJET.jl`

---
 test/packages/PkgUsingJET.jl/Manifest.toml    | 226 ++++++++++++++++++
 test/packages/PkgUsingJET.jl/Project.toml     |  16 ++
 .../PkgUsingJET.jl/src/PkgUsingJET.jl         |   5 +
 test/packages/PkgUsingJET.jl/test/jet_test.jl |  34 +++
 test/packages/PkgUsingJET.jl/test/runtests.jl |   4 +
 5 files changed, 285 insertions(+)
 create mode 100644 test/packages/PkgUsingJET.jl/Manifest.toml
 create mode 100644 test/packages/PkgUsingJET.jl/Project.toml
 create mode 100644 test/packages/PkgUsingJET.jl/src/PkgUsingJET.jl
 create mode 100644 test/packages/PkgUsingJET.jl/test/jet_test.jl
 create mode 100644 test/packages/PkgUsingJET.jl/test/runtests.jl

diff --git a/test/packages/PkgUsingJET.jl/Manifest.toml b/test/packages/PkgUsingJET.jl/Manifest.toml
new file mode 100644
index 00000000..d9193cef
--- /dev/null
+++ b/test/packages/PkgUsingJET.jl/Manifest.toml
@@ -0,0 +1,226 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.10.0"
+manifest_format = "2.0"
+project_hash = "acba4ed3ade3cccc8954146b28d1516bf39ebd32"
+
+[[deps.ArgTools]]
+uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
+version = "1.1.1"
+
+[[deps.Artifacts]]
+uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+
+[[deps.Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+
+[[deps.CodeTracking]]
+deps = ["InteractiveUtils", "UUIDs"]
+git-tree-sha1 = "c0216e792f518b39b22212127d4a84dc31e4e386"
+uuid = "da1fd8a2-8d9e-5ec2-8556-3022fb5608a2"
+version = "1.3.5"
+
+[[deps.Dates]]
+deps = ["Printf"]
+uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
+
+[[deps.Distributed]]
+deps = ["Random", "Serialization", "Sockets"]
+uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+
+[[deps.Downloads]]
+deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
+uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+version = "1.6.0"
+
+[[deps.FileWatching]]
+uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
+
+[[deps.InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+
+[[deps.JET]]
+deps = ["InteractiveUtils", "JuliaInterpreter", "LoweredCodeUtils", "MacroTools", "Pkg", "PrecompileTools", "Preferences", "Revise", "Test"]
+git-tree-sha1 = "9587e44f478b5fddc70fc3baae60a587deaa3a31"
+uuid = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
+version = "0.8.24"
+
+[[deps.JuliaInterpreter]]
+deps = ["CodeTracking", "InteractiveUtils", "Random", "UUIDs"]
+git-tree-sha1 = "04663b9e1eb0d0eabf76a6d0752e0dac83d53b36"
+uuid = "aa1ae85d-cabe-5617-a682-6adf51b2e16a"
+version = "0.9.28"
+
+[[deps.LibCURL]]
+deps = ["LibCURL_jll", "MozillaCACerts_jll"]
+uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
+version = "0.6.4"
+
+[[deps.LibCURL_jll]]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
+uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
+version = "8.4.0+0"
+
+[[deps.LibGit2]]
+deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"]
+uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
+
+[[deps.LibGit2_jll]]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"]
+uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
+version = "1.6.4+0"
+
+[[deps.LibSSH2_jll]]
+deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
+uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
+version = "1.11.0+1"
+
+[[deps.Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+
+[[deps.Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+
+[[deps.LoggingExtras]]
+deps = ["Dates", "Logging"]
+git-tree-sha1 = "c1dd6d7978c12545b4179fb6153b9250c96b0075"
+uuid = "e6f89c97-d47a-5376-807f-9c37f3926c36"
+version = "1.0.3"
+
+[[deps.LoweredCodeUtils]]
+deps = ["JuliaInterpreter"]
+git-tree-sha1 = "0b8cf121228f7dae022700c1c11ac1f04122f384"
+uuid = "6f1432cf-f94c-5a45-995e-cdbf5db27b0b"
+version = "2.3.2"
+
+[[deps.MacroTools]]
+deps = ["Markdown", "Random"]
+git-tree-sha1 = "b211c553c199c111d998ecdaf7623d1b89b69f93"
+uuid = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+version = "0.5.12"
+
+[[deps.Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+
+[[deps.MbedTLS_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
+version = "2.28.2+1"
+
+[[deps.MozillaCACerts_jll]]
+uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
+version = "2023.1.10"
+
+[[deps.NetworkOptions]]
+uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
+version = "1.2.0"
+
+[[deps.OrderedCollections]]
+git-tree-sha1 = "dfdf5519f235516220579f949664f1bf44e741c5"
+uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
+version = "1.6.3"
+
+[[deps.Pkg]]
+deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "REPL", "Random", "SHA", "Serialization", "TOML", "Tar", "UUIDs", "p7zip_jll"]
+uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+version = "1.10.0"
+
+[[deps.PrecompileTools]]
+deps = ["Preferences"]
+git-tree-sha1 = "03b4c25b43cb84cee5c90aa9b5ea0a78fd848d2f"
+uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+version = "1.2.0"
+
+[[deps.Preferences]]
+deps = ["TOML"]
+git-tree-sha1 = "00805cd429dcb4870060ff49ef443486c262e38e"
+uuid = "21216c6a-2e73-6563-6e65-726566657250"
+version = "1.4.1"
+
+[[deps.Printf]]
+deps = ["Unicode"]
+uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+
+[[deps.REPL]]
+deps = ["InteractiveUtils", "Markdown", "Sockets", "Unicode"]
+uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+
+[[deps.Random]]
+deps = ["SHA"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+
+[[deps.ReTestItems]]
+deps = ["Dates", "Logging", "LoggingExtras", "Pkg", "Serialization", "Sockets", "Test", "TestEnv"]
+path = "../../.."
+uuid = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
+version = "1.23.0"
+weakdeps = ["JET"]
+
+    [deps.ReTestItems.extensions]
+    JETExt = "JET"
+
+[[deps.Requires]]
+deps = ["UUIDs"]
+git-tree-sha1 = "838a3a4188e2ded87a4f9f184b4b0d78a1e91cb7"
+uuid = "ae029012-a4dd-5104-9daa-d747884805df"
+version = "1.3.0"
+
+[[deps.Revise]]
+deps = ["CodeTracking", "Distributed", "FileWatching", "JuliaInterpreter", "LibGit2", "LoweredCodeUtils", "OrderedCollections", "Pkg", "REPL", "Requires", "UUIDs", "Unicode"]
+git-tree-sha1 = "3fe4e5b9cdbb9bbc851c57b149e516acc07f8f72"
+uuid = "295af30f-e4ad-537b-8983-00126c2a3abe"
+version = "3.5.13"
+
+[[deps.SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+version = "0.7.0"
+
+[[deps.Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+
+[[deps.Sockets]]
+uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
+
+[[deps.TOML]]
+deps = ["Dates"]
+uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+version = "1.0.3"
+
+[[deps.Tar]]
+deps = ["ArgTools", "SHA"]
+uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+version = "1.10.0"
+
+[[deps.Test]]
+deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[[deps.TestEnv]]
+deps = ["Pkg"]
+git-tree-sha1 = "c35f69c951ac4f74b8b074f62dfb1e169b351497"
+uuid = "1e6cf692-eddd-4d53-88a5-2d735e33781b"
+version = "1.101.1"
+
+[[deps.UUIDs]]
+deps = ["Random", "SHA"]
+uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+
+[[deps.Unicode]]
+uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
+
+[[deps.Zlib_jll]]
+deps = ["Libdl"]
+uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
+version = "1.2.13+1"
+
+[[deps.nghttp2_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
+version = "1.52.0+1"
+
+[[deps.p7zip_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
+version = "17.4.0+2"
diff --git a/test/packages/PkgUsingJET.jl/Project.toml b/test/packages/PkgUsingJET.jl/Project.toml
new file mode 100644
index 00000000..0a3c85fa
--- /dev/null
+++ b/test/packages/PkgUsingJET.jl/Project.toml
@@ -0,0 +1,16 @@
+name = "PkgUsingJET"
+uuid = "c3f2fd6f-de40-45a4-991e-5ffd14a5ac2f"
+authors = ["Tomáš Drvoštěp <tomas.drvostep@gmail.com>"]
+version = "0.1.0"
+
+[deps]
+JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
+ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
+
+[extras]
+JET = "c3a54625-cd67-489e-a8e7-0a5a0ff4e31b"
+ReTestItems = "817f1d60-ba6b-4fd5-9520-3cf149f6a823"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["JET", "ReTestItems", "Test"]
diff --git a/test/packages/PkgUsingJET.jl/src/PkgUsingJET.jl b/test/packages/PkgUsingJET.jl/src/PkgUsingJET.jl
new file mode 100644
index 00000000..9a44a617
--- /dev/null
+++ b/test/packages/PkgUsingJET.jl/src/PkgUsingJET.jl
@@ -0,0 +1,5 @@
+module PkgUsingJET
+
+greet() = print("Hello World!")
+
+end # module PkgUsingJET
diff --git a/test/packages/PkgUsingJET.jl/test/jet_test.jl b/test/packages/PkgUsingJET.jl/test/jet_test.jl
new file mode 100644
index 00000000..235e21c0
--- /dev/null
+++ b/test/packages/PkgUsingJET.jl/test/jet_test.jl
@@ -0,0 +1,34 @@
+@testsetup module Constants
+    const DEF = 1
+    export DEF
+end
+
+@testitem "Test with JET on" jet=:basic setup=[Constants] begin
+    using .Threads: @spawn
+
+    @test DEF+1 == 2
+
+
+    function foo()
+        r = Ref(2)
+        @spawn ($(r)[] += UNDEF)
+        sleep(0.5)
+        return r[]
+    end
+    @test foo() == 2
+end
+
+@testitem "Test with JET off" setup=[Constants] begin
+    using .Threads: @spawn
+
+    @test DEF+1 == 2
+
+
+    function foo()
+        r = Ref(2)
+        @spawn ($(r)[] += UNDEF)
+        sleep(0.5)
+        return r[]
+    end
+    @test foo() == 2
+end
diff --git a/test/packages/PkgUsingJET.jl/test/runtests.jl b/test/packages/PkgUsingJET.jl/test/runtests.jl
new file mode 100644
index 00000000..bf64aa8e
--- /dev/null
+++ b/test/packages/PkgUsingJET.jl/test/runtests.jl
@@ -0,0 +1,4 @@
+using ReTestItems, PkgUsingJET
+
+runtests(PkgUsingJET; verbose_results=true, nworkers=1)
+runtests(PkgUsingJET; verbose_results=true, nworkers=1, worker_init_expr=quote import JET end)

From db97988acdfe1a81be232119a521f195991e6c77 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?=
 <tomas.drvostep@gmail.com>
Date: Fri, 12 Jan 2024 13:23:55 +0100
Subject: [PATCH 4/7] Progress

---
 ext/JETExt.jl                                 |  9 ++-
 src/ReTestItems.jl                            | 55 ++++++++++++-------
 src/testcontext.jl                            |  1 +
 test/integrationtests.jl                      | 25 ++++++++-
 .../PkgUsingJET.jl/src/PkgUsingJET.jl         |  5 --
 test/packages/PkgUsingJET.jl/test/runtests.jl |  4 --
 test/packages/README.md                       |  1 +
 .../Manifest.toml                             |  0
 .../Project.toml                              |  2 +-
 test/packages/UsingJET.jl/src/UsingJET.jl     |  5 ++
 .../test/jet_test.jl                          |  2 -
 test/packages/UsingJET.jl/test/runtests.jl    |  3 +
 12 files changed, 75 insertions(+), 37 deletions(-)
 delete mode 100644 test/packages/PkgUsingJET.jl/src/PkgUsingJET.jl
 delete mode 100644 test/packages/PkgUsingJET.jl/test/runtests.jl
 rename test/packages/{PkgUsingJET.jl => UsingJET.jl}/Manifest.toml (100%)
 rename test/packages/{PkgUsingJET.jl => UsingJET.jl}/Project.toml (95%)
 create mode 100644 test/packages/UsingJET.jl/src/UsingJET.jl
 rename test/packages/{PkgUsingJET.jl => UsingJET.jl}/test/jet_test.jl (99%)
 create mode 100644 test/packages/UsingJET.jl/test/runtests.jl

diff --git a/ext/JETExt.jl b/ext/JETExt.jl
index 2c886a32..c963ffa9 100644
--- a/ext/JETExt.jl
+++ b/ext/JETExt.jl
@@ -10,15 +10,14 @@ function _analyze_toplevel(ex, file, mode)
     return JET.JETToplevelResult(analyzer, res, "analyze_toplevel"; mode)
 end
 
-function ReTestItems.jet_test(ti::ReTestItems.TestItem, mod_expr::Expr)
-    ti.jet == :none && return nothing
-
+function ReTestItems.jet_test(ti::ReTestItems.TestItem, mod_expr::Expr, jet::Symbol)
+    jet in (:skip, :none) && return nothing
     onfail(::Function, ::Test.Pass) = nothing
     onfail(f::Function, ::Test.Fail) = f()
 
     @assert mod_expr.head === :module "Expected the test item expression to be wrapped in a module, got $(repr(mod_expr.head))"
-    Test.@testset "JET $(repr(ti.jet)) mode" begin
-        result = _analyze_toplevel(mod_expr, ti.file, ti.jet)
+    Test.@testset "JET $(repr(jet)) mode" begin
+        result = _analyze_toplevel(mod_expr, ti.file, jet)
         no_jet_errors = isempty(result)
         onfail(Test.@test no_jet_errors) do
             JET.print_reports(
diff --git a/src/ReTestItems.jl b/src/ReTestItems.jl
index e47a9572..f4acee7e 100644
--- a/src/ReTestItems.jl
+++ b/src/ReTestItems.jl
@@ -17,6 +17,7 @@ const RETESTITEMS_TEMP_FOLDER = mkpath(joinpath(tempdir(), "ReTestItemsTempLogsD
 const DEFAULT_TESTITEM_TIMEOUT = 30*60
 const DEFAULT_RETRIES = 0
 const DEFAULT_MEMORY_THRESHOLD = Ref{Float64}(0.99)
+const DEFAULT_JET = :none # only do a JET pass if a test item explicitly requests it
 
 
 if isdefined(Base, :errormonitor)
@@ -190,6 +191,15 @@ will be run.
   `paths` passed to it cannot contain test files, either because the path doesn't exist or
   the path points to a file which is not a test file. Default is `false`.
   Can also be set using the `RETESTITEMS_VALIDATE_PATHS` environment variable.
+- `jet::Symbol=:none`: The JET mode to use to analyze test items. Can be one of:
+  - `:skip`: disable JET analysis even for test items that have an explicit `jet` mode set
+  - `:none`: don't analyze test items without an explicit `jet` mode set
+  - `:typo`, `:basic`, `:sound`: the JET mode to use for test items without an explicit `jet` mode
+  Default is `:none`. Can also be set using the `RETESTITEMS_JET` environment variable.
+  **Note** that this functionality assumes the JET package is installed and available in the test evironment.
+  When `nworkers > 0`, we'll try to preload the JET for the user on each worker to activate the JET extension,
+  for `nworkers == 0`, the user is expected to import JET themselves if they want to use it.
+  Failure to activate the extension will result in a broken test for test items that explicitly set a `jet` mode.
 """
 function runtests end
 
@@ -235,6 +245,7 @@ function runtests(
     verbose_results::Bool=(logs !== :issues && isinteractive()),
     test_end_expr::Expr=Expr(:block),
     validate_paths::Bool=parse(Bool, get(ENV, "RETESTITEMS_VALIDATE_PATHS", "false")),
+    jet::Symbol=Symbol(get(ENV, "RETESTITEMS_JET", string(DEFAULT_JET))),
 )
     nworker_threads = _validated_nworker_threads(nworker_threads)
     paths′ = _validated_paths(paths, validate_paths)
@@ -243,6 +254,7 @@ function runtests(
     report && logs == :eager && throw(ArgumentError("`report=true` is not compatible with `logs=:eager`"))
     (0 ≤ memory_threshold ≤ 1) || throw(ArgumentError("`memory_threshold` must be between 0 and 1, got $(repr(memory_threshold))"))
     testitem_timeout > 0 || throw(ArgumentError("`testitem_timeout` must be a postive number, got $(repr(testitem_timeout))"))
+    jet in (:skip, :none, :typo, :basic, :sound) || throw(ArgumentError("`jet` must be one of :none, :skip, :typo, :basic, :sound, got $(repr(jet))"))
     # If we were given paths but none were valid, then nothing to run.
     !isempty(paths) && isempty(paths′) && return nothing
     shouldrun_combined(ti) = shouldrun(ti) && _shouldrun(name, ti.name) && _shouldrun(tags, ti.tags)
@@ -254,10 +266,10 @@ function runtests(
     debuglvl = Int(debug)
     if debuglvl > 0
         LoggingExtras.withlevel(LoggingExtras.Debug; verbosity=debuglvl) do
-            _runtests(shouldrun_combined, paths′, nworkers, nworker_threads, worker_init_expr, test_end_expr, timeout, retries, memory_threshold, verbose_results, debuglvl, report, logs)
+            _runtests(shouldrun_combined, paths′, nworkers, nworker_threads, worker_init_expr, test_end_expr, timeout, retries, memory_threshold, verbose_results, debuglvl, report, logs, jet)
         end
     else
-        return _runtests(shouldrun_combined, paths′, nworkers, nworker_threads, worker_init_expr, test_end_expr, timeout, retries, memory_threshold, verbose_results, debuglvl, report, logs)
+        return _runtests(shouldrun_combined, paths′, nworkers, nworker_threads, worker_init_expr, test_end_expr, timeout, retries, memory_threshold, verbose_results, debuglvl, report, logs, jet)
     end
 end
 
@@ -271,7 +283,7 @@ end
 # By tracking and reusing test environments, we can avoid this issue.
 const TEST_ENVS = Dict{String, String}()
 
-function _runtests(shouldrun, paths, nworkers::Int, nworker_threads::String, worker_init_expr::Expr, test_end_expr::Expr, testitem_timeout::Int, retries::Int, memory_threshold::Real, verbose_results::Bool, debug::Int, report::Bool, logs::Symbol)
+function _runtests(shouldrun, paths, nworkers::Int, nworker_threads::String, worker_init_expr::Expr, test_end_expr::Expr, testitem_timeout::Int, retries::Int, memory_threshold::Real, verbose_results::Bool, debug::Int, report::Bool, logs::Symbol, jet::Symbol)
     # Don't recursively call `runtests` e.g. if we `include` a file which calls it.
     # So we ignore the `runtests(...)` call in `test/runtests.jl` when `runtests(...)`
     # was called from the command line.
@@ -291,7 +303,7 @@ function _runtests(shouldrun, paths, nworkers::Int, nworker_threads::String, wor
         if is_running_test_runtests_jl(proj_file)
             # Assume this is `Pkg.test`, so test env already active.
             @debugv 2 "Running in current environment `$(Base.active_project())`"
-            return _runtests_in_current_env(shouldrun, paths, proj_file, nworkers, nworker_threads, worker_init_expr, test_end_expr, testitem_timeout, retries, memory_threshold, verbose_results, debug, report, logs)
+            return _runtests_in_current_env(shouldrun, paths, proj_file, nworkers, nworker_threads, worker_init_expr, test_end_expr, testitem_timeout, retries, memory_threshold, verbose_results, debug, report, logs, jet)
         else
             @debugv 1 "Activating test environment for `$proj_file`"
             orig_proj = Base.active_project()
@@ -304,7 +316,7 @@ function _runtests(shouldrun, paths, nworkers::Int, nworker_threads::String, wor
                     testenv = TestEnv.activate()
                     TEST_ENVS[proj_file] = testenv
                 end
-                _runtests_in_current_env(shouldrun, paths, proj_file, nworkers, nworker_threads, worker_init_expr, test_end_expr, testitem_timeout, retries, memory_threshold, verbose_results, debug, report, logs)
+                _runtests_in_current_env(shouldrun, paths, proj_file, nworkers, nworker_threads, worker_init_expr, test_end_expr, testitem_timeout, retries, memory_threshold, verbose_results, debug, report, logs, jet)
             finally
                 Base.set_active_project(orig_proj)
             end
@@ -314,7 +326,7 @@ end
 
 function _runtests_in_current_env(
     shouldrun, paths, projectfile::String, nworkers::Int, nworker_threads, worker_init_expr::Expr, test_end_expr::Expr,
-    testitem_timeout::Int, retries::Int, memory_threshold::Real, verbose_results::Bool, debug::Int, report::Bool, logs::Symbol,
+    testitem_timeout::Int, retries::Int, memory_threshold::Real, verbose_results::Bool, debug::Int, report::Bool, logs::Symbol, jet::Symbol
 )
     start_time = time()
     proj_name = something(Pkg.Types.read_project(projectfile).name, "")
@@ -340,7 +352,7 @@ function _runtests_in_current_env(
                 run_number = 1
                 max_runs = 1 + max(retries, testitem.retries)
                 while run_number ≤ max_runs
-                    res = runtestitem(testitem, ctx; test_end_expr, verbose_results, logs)
+                    res = runtestitem(testitem, ctx; test_end_expr, verbose_results, logs, jet)
                     ts = res.testset
                     print_errors_and_captured_logs(testitem, run_number; logs)
                     report_empty_testsets(testitem, ts)
@@ -348,7 +360,7 @@ function _runtests_in_current_env(
                     # (the first one is a partial mark, full sweep, the next one is a full mark).
                     GC.gc(true)
                     GC.gc(false)
-                    if any_non_pass(ts) && run_number != max_runs
+                    if res.is_retryable && any_non_pass(ts) && run_number != max_runs
                         run_number += 1
                         @info "Retrying $(repr(testitem.name)). Run=$run_number."
                     else
@@ -379,7 +391,7 @@ function _runtests_in_current_env(
                 ti = starting[i]
                 @spawn begin
                     with_logger(original_logger) do
-                        manage_worker($w, $proj_name, $testitems, $ti, $nworker_threads, $worker_init_expr, $test_end_expr, $testitem_timeout, $retries, $memory_threshold, $verbose_results, $debug, $report, $logs)
+                        manage_worker($w, $proj_name, $testitems, $ti, $nworker_threads, $worker_init_expr, $test_end_expr, $testitem_timeout, $retries, $memory_threshold, $verbose_results, $debug, $report, $logs, $jet)
                     end
                 end
             end
@@ -404,6 +416,7 @@ function start_worker(proj_name, nworker_threads, worker_init_expr, ntestitems;
     # remote_fetch here because we want to make sure the worker is all setup before starting to eval testitems
     remote_fetch(w, quote
         using ReTestItems, Test
+        Core.eval(Module(), :(try import JET catch end))
         Test.TESTSET_PRINT_ENABLE[] = false
         const GLOBAL_TEST_CONTEXT = ReTestItems.TestContext($proj_name, $ntestitems)
         GLOBAL_TEST_CONTEXT.setups_evaled = ReTestItems.TestSetupModules()
@@ -488,7 +501,7 @@ end
 
 function manage_worker(
     worker::Worker, proj_name::AbstractString, testitems::TestItems, testitem::Union{TestItem,Nothing}, nworker_threads, worker_init_expr::Expr, test_end_expr::Expr,
-    default_timeout::Int, retries::Int, memory_threshold::Real, verbose_results::Bool, debug::Int, report::Bool, logs::Symbol
+    default_timeout::Int, retries::Int, memory_threshold::Real, verbose_results::Bool, debug::Int, report::Bool, logs::Symbol, jet::Symbol
 )
     ntestitems = length(testitems.testitems)
     run_number = 1
@@ -503,7 +516,7 @@ function manage_worker(
         end
         testitem.workerid[] = worker.pid
         timeout = something(testitem.timeout, default_timeout)
-        fut = remote_eval(worker, :(ReTestItems.runtestitem($testitem, GLOBAL_TEST_CONTEXT; test_end_expr=$(QuoteNode(test_end_expr)), verbose_results=$verbose_results, logs=$(QuoteNode(logs)))))
+        fut = remote_eval(worker, :(ReTestItems.runtestitem($testitem, GLOBAL_TEST_CONTEXT; test_end_expr=$(QuoteNode(test_end_expr)), verbose_results=$verbose_results, logs=$(QuoteNode(logs)), jet=$(QuoteNode(jet)))))
         max_runs = 1 + max(retries, testitem.retries)
         try
             timer = Timer(timeout) do tm
@@ -535,7 +548,7 @@ function manage_worker(
                 # Run GC to free memory on the worker before next testitem.
                 @debugv 2 "Running GC on $worker"
                 remote_fetch(worker, :(GC.gc(true); GC.gc(false)))
-                if any_non_pass(ts) && run_number != max_runs
+                if testitem_result.is_retryable && any_non_pass(ts) && run_number != max_runs
                     run_number += 1
                     @info "Retrying $(repr(testitem.name)) on $worker. Run=$run_number."
                 else
@@ -891,7 +904,7 @@ function skiptestitem(ti::TestItem, ctx::TestContext; verbose_results::Bool=true
     stats = PerfStats()
     push!(ti.stats, stats)
     log_testitem_skipped(ti, ctx.ntestitems)
-    return TestItemResult(ts, stats)
+    return TestItemResult(ts, stats, false)
 end
 
 
@@ -911,11 +924,13 @@ end
 # when `runtestitem` called directly or `@testitem` called outside of `runtests`.
 function runtestitem(
     ti::TestItem, ctx::TestContext;
-    test_end_expr::Expr=Expr(:block), logs::Symbol=:eager, verbose_results::Bool=true, finish_test::Bool=true,
+    test_end_expr::Expr=Expr(:block), logs::Symbol=:eager, verbose_results::Bool=true, finish_test::Bool=true, jet::Symbol=:none
 )
     if should_skip(ti)::Bool
         return skiptestitem(ti, ctx; verbose_results)
     end
+    jet = jet == :none ? ti.jet : jet
+    retryable_failure = true
     name = ti.name
     log_testitem_start(ti, ctx.ntestitems)
     ts = DefaultTestSet(name; verbose=verbose_results)
@@ -980,8 +995,10 @@ function runtestitem(
         # environment = Module()
         @debugv 1 "Running test item $(repr(name))$(_on_worker())."
         _, stats = @timed_with_compilation _redirect_logs(logs == :eager ? DEFAULT_STDOUT[] : logpath(ti)) do
-            with_source_path(() -> (Core.eval(Main, mod_expr); jet_test(ti, mod_expr)), ti.file)
+            with_source_path(() -> Core.eval(Main, mod_expr), ti.file)
             with_source_path(() -> Core.eval(Main, test_end_mod_expr), ti.file)
+            retryable_failure = any_non_pass(ts) # only failures from actual tests constitute a reson to retry
+            jet_test(ti, mod_expr, jet)          # if JET tests fail, a retry won't help
             nothing # return nothing as the first return value of @timed_with_compilation
         end
         @debugv 1 "Done running test item $(repr(name))$(_on_worker())."
@@ -1010,7 +1027,7 @@ function runtestitem(
     @debugv 2 "Converting results for test item $(repr(name))$(_on_worker())."
     res = convert_results_to_be_transferrable(ts)
     log_testitem_done(ti, ctx.ntestitems)
-    return TestItemResult(res, stats)
+    return TestItemResult(res, stats, retryable_failure)
 end
 
 function convert_results_to_be_transferrable(ts::Test.AbstractTestSet)
@@ -1038,9 +1055,9 @@ end
 convert_results_to_be_transferrable(x) = x
 
 # This method signature must be less specific than the overload in ext/JETExt.jl
-function jet_test(ti::Any, mod_expr::Expr)
-    ti.jet == :none && return nothing
-    Test.@testset "JET package extension failure: JET not loaded, ignoring \"jet=$(repr(ti.jet))\"" begin
+function jet_test(ti::Any, mod_expr::Expr, jet::Symbol)
+    jet in (:skip, :none) && return nothing
+    Test.@testset "JET not loaded, ignoring \"jet=$(repr(jet))\"" begin
         Test.@test_broken false
     end
     return nothing
diff --git a/src/testcontext.jl b/src/testcontext.jl
index f8c9fdc3..3ad61415 100644
--- a/src/testcontext.jl
+++ b/src/testcontext.jl
@@ -193,4 +193,5 @@ end
 struct TestItemResult
     testset::DefaultTestSet
     stats::PerfStats
+    is_retryable::Bool
 end
diff --git a/test/integrationtests.jl b/test/integrationtests.jl
index 42374825..e543b18b 100644
--- a/test/integrationtests.jl
+++ b/test/integrationtests.jl
@@ -636,7 +636,7 @@ end
     tmpdir = joinpath("/tmp", "JL_RETESTITEMS_TEST_TMPDIR")
     # must run with `testitem_timeout < 20` for test to timeout as expected.
     # and must run with `nworkers > 0` for retries to be supported.
-    results = encased_testset(()->runtests(file; nworkers=1, retries=2, testitem_timeout=3))
+    results = encased_testset(()->runtests(file; nworkers=1, retries=2, testitem_timeout=5))
     # Test we _ran_ each test-item the expected number of times
     read_count(x) = parse(Int, read(x, String))
     # Passes on second attempt, so only need to retry once.
@@ -985,6 +985,29 @@ end
         @test n_passed(results_with_end) ≥ 1
         @test length(failures(results_with_end)) ≥ 1
     end
+
+    @testset "UsingJET.jl package" begin
+        # There are two test items, both of which fail basic JET error analysis passes
+        # And 4 passing tests total.
+        results_skip_jet = with_test_package("UsingJET.jl") do
+            runtests(; nworkers=1, jet=:skip)
+        end
+        @test all_passed(results_skip_jet)
+        @test n_passed(results_skip_jet) == 4
+
+        # One of the test items has a `jet` mode set
+        results_default_jet = with_test_package("UsingJET.jl") do
+            runtests(; nworkers=1)
+        end
+        @test length(non_passes(results_default_jet)) == 1
+        @test n_passed(results_default_jet) == 4
+
+        results_force_jet = with_test_package("UsingJET.jl") do
+            runtests(; nworkers=1, jet=:typo)
+        end
+        @test length(non_passes(results_force_jet)) == 2
+        @test n_passed(results_force_jet) == 4
+    end
 end
 
 @testset "Replace workers when we hit memory threshold" begin
diff --git a/test/packages/PkgUsingJET.jl/src/PkgUsingJET.jl b/test/packages/PkgUsingJET.jl/src/PkgUsingJET.jl
deleted file mode 100644
index 9a44a617..00000000
--- a/test/packages/PkgUsingJET.jl/src/PkgUsingJET.jl
+++ /dev/null
@@ -1,5 +0,0 @@
-module PkgUsingJET
-
-greet() = print("Hello World!")
-
-end # module PkgUsingJET
diff --git a/test/packages/PkgUsingJET.jl/test/runtests.jl b/test/packages/PkgUsingJET.jl/test/runtests.jl
deleted file mode 100644
index bf64aa8e..00000000
--- a/test/packages/PkgUsingJET.jl/test/runtests.jl
+++ /dev/null
@@ -1,4 +0,0 @@
-using ReTestItems, PkgUsingJET
-
-runtests(PkgUsingJET; verbose_results=true, nworkers=1)
-runtests(PkgUsingJET; verbose_results=true, nworkers=1, worker_init_expr=quote import JET end)
diff --git a/test/packages/README.md b/test/packages/README.md
index cc35199f..8e3c20f5 100644
--- a/test/packages/README.md
+++ b/test/packages/README.md
@@ -14,3 +14,4 @@ See `test/integrationtests.jl`.
 - *TestProjectFile.jl* - A package which has test-only dependencies declared in a `test/Project.toml`.
 - *MonoRepo.jl* - A package which depends on local, unregistered sub-packages. See `MonoRepo.jl/README.md`.
 - *TestEndExpr.jl* - A package which requires users to uphold an invariant which we would want to test is being upheld by all code run in the tests. This provides a use-case for the `test_end_expr` functionality.
+- *UsingJET.jl* - A package that uses JET in its tests, the tests fail JET error analysis.
\ No newline at end of file
diff --git a/test/packages/PkgUsingJET.jl/Manifest.toml b/test/packages/UsingJET.jl/Manifest.toml
similarity index 100%
rename from test/packages/PkgUsingJET.jl/Manifest.toml
rename to test/packages/UsingJET.jl/Manifest.toml
diff --git a/test/packages/PkgUsingJET.jl/Project.toml b/test/packages/UsingJET.jl/Project.toml
similarity index 95%
rename from test/packages/PkgUsingJET.jl/Project.toml
rename to test/packages/UsingJET.jl/Project.toml
index 0a3c85fa..a19e2273 100644
--- a/test/packages/PkgUsingJET.jl/Project.toml
+++ b/test/packages/UsingJET.jl/Project.toml
@@ -1,4 +1,4 @@
-name = "PkgUsingJET"
+name = "UsingJET"
 uuid = "c3f2fd6f-de40-45a4-991e-5ffd14a5ac2f"
 authors = ["Tomáš Drvoštěp <tomas.drvostep@gmail.com>"]
 version = "0.1.0"
diff --git a/test/packages/UsingJET.jl/src/UsingJET.jl b/test/packages/UsingJET.jl/src/UsingJET.jl
new file mode 100644
index 00000000..55ddba69
--- /dev/null
+++ b/test/packages/UsingJET.jl/src/UsingJET.jl
@@ -0,0 +1,5 @@
+module UsingJET
+
+greet() = print("Hello World!")
+
+end # module UsingJET
diff --git a/test/packages/PkgUsingJET.jl/test/jet_test.jl b/test/packages/UsingJET.jl/test/jet_test.jl
similarity index 99%
rename from test/packages/PkgUsingJET.jl/test/jet_test.jl
rename to test/packages/UsingJET.jl/test/jet_test.jl
index 235e21c0..20ac7358 100644
--- a/test/packages/PkgUsingJET.jl/test/jet_test.jl
+++ b/test/packages/UsingJET.jl/test/jet_test.jl
@@ -8,7 +8,6 @@ end
 
     @test DEF+1 == 2
 
-
     function foo()
         r = Ref(2)
         @spawn ($(r)[] += UNDEF)
@@ -23,7 +22,6 @@ end
 
     @test DEF+1 == 2
 
-
     function foo()
         r = Ref(2)
         @spawn ($(r)[] += UNDEF)
diff --git a/test/packages/UsingJET.jl/test/runtests.jl b/test/packages/UsingJET.jl/test/runtests.jl
new file mode 100644
index 00000000..ed00f376
--- /dev/null
+++ b/test/packages/UsingJET.jl/test/runtests.jl
@@ -0,0 +1,3 @@
+using ReTestItems, UsingJET
+
+runtests(UsingJET; verbose_results=true, nworkers=1, retries=2)

From 06c906f0959c7f4ffaf1b17b34b6d9efb2d8a4f6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?=
 <tomas.drvostep@gmail.com>
Date: Fri, 12 Jan 2024 13:52:24 +0100
Subject: [PATCH 5/7] Add a way for a hard opt-out per test item

---
 src/ReTestItems.jl | 4 +++-
 src/macros.jl      | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/src/ReTestItems.jl b/src/ReTestItems.jl
index f4acee7e..65e6ab17 100644
--- a/src/ReTestItems.jl
+++ b/src/ReTestItems.jl
@@ -929,7 +929,9 @@ function runtestitem(
     if should_skip(ti)::Bool
         return skiptestitem(ti, ctx; verbose_results)
     end
-    jet = jet == :none ? ti.jet : jet
+    jet = (jet == :skip || ti.jet == :skip) ? :skip :
+        ti.jet == :none ? jet : ti.jet
+
     retryable_failure = true
     name = ti.name
     log_testitem_start(ti, ctx.ntestitems)
diff --git a/src/macros.jl b/src/macros.jl
index fd714a8e..65aabfb2 100644
--- a/src/macros.jl
+++ b/src/macros.jl
@@ -298,7 +298,7 @@ macro testitem(nm, exs...)
             elseif kw == :jet
                 jet = ex.args[2]
                 @assert isa(jet, QuoteNode)
-                @assert jet in QuoteNode.((:none, :typo, :basic, :sound))
+                @assert jet in QuoteNode.((:none, :skip, :typo, :basic, :sound))
             else
                 error("unknown `@testitem` keyword arg `$(ex.args[1])`")
             end

From 2b974e2b991aab65c8c213848eb058037e4cfe59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?=
 <tomas.drvostep@gmail.com>
Date: Mon, 15 Jan 2024 11:14:33 +0100
Subject: [PATCH 6/7] Fix JET report filtering

---
 ext/JETExt.jl | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/ext/JETExt.jl b/ext/JETExt.jl
index c963ffa9..30540069 100644
--- a/ext/JETExt.jl
+++ b/ext/JETExt.jl
@@ -18,11 +18,13 @@ function ReTestItems.jet_test(ti::ReTestItems.TestItem, mod_expr::Expr, jet::Sym
     @assert mod_expr.head === :module "Expected the test item expression to be wrapped in a module, got $(repr(mod_expr.head))"
     Test.@testset "JET $(repr(jet)) mode" begin
         result = _analyze_toplevel(mod_expr, ti.file, jet)
-        no_jet_errors = isempty(result)
+        reports = JET.get_reports(result)
+        filtered_reports = JET.configured_reports(reports, ignored_modules=(JET.AnyFrameModule(Test), JET.AnyFrameModule(JET)))
+        no_jet_errors = isempty(filtered_reports)
         onfail(Test.@test no_jet_errors) do
             JET.print_reports(
                 stdout,
-                JET.get_reports(result),
+                filtered_reports,
                 # Remove the name of the module JET uses for virtualization the code and the name of the module
                 # we wrap the test items in.
                 JET.gen_postprocess(result.res.actual2virtual) ∘ x->replace(x, string("var\"", mod_expr.args[2], "\"") => ""),

From 3b52add92c3fbb9ea59fcb2cc9e97d22dd9c3262 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Tom=C3=A1=C5=A1=20Drvo=C5=A1t=C4=9Bp?=
 <tomas.drvostep@gmail.com>
Date: Mon, 15 Jan 2024 13:19:00 +0100
Subject: [PATCH 7/7] More fixes and tweaks

---
 ext/JETExt.jl | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/ext/JETExt.jl b/ext/JETExt.jl
index 30540069..648a6c4c 100644
--- a/ext/JETExt.jl
+++ b/ext/JETExt.jl
@@ -2,12 +2,12 @@ module JETExt
 
 import JET, ReTestItems, Test
 
-function _analyze_toplevel(ex, file, mode)
+function _analyze_toplevel(ex, file, mode, ignore_modules)
     toplevelex = Expr(:toplevel, ex)
-    analyzer = JET.JETAnalyzer(; mode)
-    config = JET.ToplevelConfig(; mode)
+    analyzer = JET.JETAnalyzer(; mode, ignore_missing_comparison=true)
+    config = JET.ToplevelConfig(; concretization_patterns = [])
     res = JET.virtual_process(toplevelex, file, analyzer, config)
-    return JET.JETToplevelResult(analyzer, res, "analyze_toplevel"; mode)
+    return JET.JETToplevelResult(analyzer, res, "analyze_toplevel"; mode, ignore_modules, target_defined_modules=false)
 end
 
 function ReTestItems.jet_test(ti::ReTestItems.TestItem, mod_expr::Expr, jet::Symbol)
@@ -17,14 +17,13 @@ function ReTestItems.jet_test(ti::ReTestItems.TestItem, mod_expr::Expr, jet::Sym
 
     @assert mod_expr.head === :module "Expected the test item expression to be wrapped in a module, got $(repr(mod_expr.head))"
     Test.@testset "JET $(repr(jet)) mode" begin
-        result = _analyze_toplevel(mod_expr, ti.file, jet)
+        result = _analyze_toplevel(mod_expr, ti.file, jet, (JET.AnyFrameModule(Test), JET.AnyFrameModule(JET)))
         reports = JET.get_reports(result)
-        filtered_reports = JET.configured_reports(reports, ignored_modules=(JET.AnyFrameModule(Test), JET.AnyFrameModule(JET)))
-        no_jet_errors = isempty(filtered_reports)
+        no_jet_errors = isempty(reports)
         onfail(Test.@test no_jet_errors) do
             JET.print_reports(
                 stdout,
-                filtered_reports,
+                reports,
                 # Remove the name of the module JET uses for virtualization the code and the name of the module
                 # we wrap the test items in.
                 JET.gen_postprocess(result.res.actual2virtual) ∘ x->replace(x, string("var\"", mod_expr.args[2], "\"") => ""),