diff --git a/src/ReTest.jl b/src/ReTest.jl index 986da2a..0fd0e02 100644 --- a/src/ReTest.jl +++ b/src/ReTest.jl @@ -57,6 +57,7 @@ using .Testset: Testset, Format, print_id Base.@kwdef mutable struct Options verbose::Bool = false # annotated verbosity transient_verbose::Bool = false # verbosity for next run + static_include::Bool = false # whether to execute include at `replace_ts` time end mutable struct TestsetExpr @@ -108,13 +109,25 @@ struct _Invalid global const invalid = _Invalid.instance end +function extract_testsets(dest) + function extractor(x) + if Meta.isexpr(x, :macrocall) && x.args[1] == Symbol("@testset") + push!(dest, x) + nothing # @testset move out of the evaluated file + else + x + end + end +end + # replace unqualified `@testset` by TestsetExpr -function replace_ts(source, mod, x::Expr, parent) +function replace_ts(source, mod, x::Expr, parent; static_include::Bool) if x.head === :macrocall name = x.args[1] if name === Symbol("@testset") @assert x.args[2] isa LineNumberNode - ts, hasbroken = parse_ts(x.args[2], mod, Tuple(x.args[3:end]), parent) + ts, hasbroken = parse_ts(x.args[2], mod, Tuple(x.args[3:end]), parent; + static_include=static_include) ts !== invalid && parent !== nothing && push!(parent.children, ts) ts, false # hasbroken counts only "proper" @test_broken, not recursive ones elseif name === Symbol("@test_broken") @@ -123,7 +136,7 @@ function replace_ts(source, mod, x::Expr, parent) # `@test` is generally called a lot, so it's probably worth it to skip # the containment test in this case x = macroexpand(mod, x, recursive=false) - replace_ts(source, mod, x, parent) + replace_ts(source, mod, x, parent; static_include=static_include) else @goto default end @@ -133,18 +146,34 @@ function replace_ts(source, mod, x::Expr, parent) x.args[end] = path isa AbstractString ? joinpath(sourcepath, path) : :(joinpath($sourcepath, $path)) - x, false + if static_include + length(x.args) == 2 || error("cannot handle include with two arguments: $x") + news = Expr(:block) + insert!(x.args, 2, extract_testsets(news.args)) + try + Core.eval(mod, x) + catch + @warn "could not statically include at $source" + deleteat!(x.args, 2) + return x, false + end + replace_ts(source, mod, news, parent; static_include=static_include) + else + x, false + end else @label default - body_br = map(z -> replace_ts(source, mod, z, parent), x.args) + body_br = map(z -> replace_ts(source, mod, z, parent; static_include=static_include), + x.args) filter!(x -> first(x) !== invalid, body_br) Expr(x.head, first.(body_br)...), any(last.(body_br)) end end -replace_ts(source, mod, x, _) = x, false +replace_ts(source, mod, x, _1; static_include::Bool) = x, false # create a TestsetExpr from @testset's args -function parse_ts(source::LineNumberNode, mod::Module, args::Tuple, parent=nothing) +function parse_ts(source::LineNumberNode, mod::Module, args::Tuple, parent=nothing; + static_include::Bool=false) function tserror(msg) @error msg _file=String(source.file) _line=source.line _module=mod invalid, false @@ -158,6 +187,10 @@ function parse_ts(source::LineNumberNode, mod::Module, args::Tuple, parent=nothi marks = Marks() if parent !== nothing append!(marks.hard, parent.marks.hard) # copy! not available in Julia 1.0 + options.static_include = parent.options.static_include + # if static_include was set in parent, it should have been forwarded also + # through the parse_ts/replace_ts call chains: + @assert static_include == parent.options.static_include end for arg in args[1:end-1] if arg isa String || Meta.isexpr(arg, :string) @@ -207,7 +240,8 @@ function parse_ts(source::LineNumberNode, mod::Module, args::Tuple, parent=nothi end ts = TestsetExpr(source, mod, desc, options, marks, loops, parent) - ts.body, ts.hasbroken = replace_ts(source, mod, tsbody, ts) + ts.body, ts.hasbroken = replace_ts(source, mod, tsbody, ts; + static_include=options.static_include) ts, false # hasbroken counts only "proper" @test_broken, not recursive ones end diff --git a/src/hijack.jl b/src/hijack.jl index 416c93a..cbfc4ee 100644 --- a/src/hijack.jl +++ b/src/hijack.jl @@ -104,7 +104,7 @@ const loaded_testmodules = Dict{Module,Vector{Module}}() """ ReTest.hijack(source, [modname]; - parentmodule::Module=Main, lazy=false, testset::Bool=false, + parentmodule::Module=Main, lazy=false, [include::Symbol], [revise::Bool]) Given test files defined in `source` using the `Test` package, try to load @@ -143,11 +143,10 @@ The `lazy` keyword specifies whether some toplevel expressions should be skipped * `:brutal` means toplevel `@test*` macros are removed, as well as toplevel `begin`, `let`, `for` or `if` blocks. -#### `testset` keyword +#### `include` keyword -The `testset` keyword can help to handle the case where `@testset`s contain -`include` expressions (at the "toplevel" of the testset), like in the -following example: +The `include` keyword can help to handle the case where `@testset`s contain +`include` expressions, like in the following example: ```julia @testset "parent" begin @test true @@ -161,11 +160,28 @@ of the testset which include them. With `ReTest`, the `include` expressions would be evaluated only when the parent testsets are run, so that included testsets are not run themselves, but only "declared". -It the `testset` keyword -is `true`, `hijack` inspects `@testset` expressions and puts `include` -expressions outside of the testset. This is not ideal, but at least allows -`ReTest` to know about all the testsets right after the call to `hijack`, and -to not declare new testsets when parent testsets are run. +If the `include` keyword is set to `:static`, `include(...)` expressions are +evaluated when `@testset` expressions containing them are parsed, before +filtering and before testsets are run. Testsets which are declared (within the +same module) as a side effect of `include(...)` are then inserted in place of +the call to `include(...)`. + +If the `include` keyword is set to `:outline`, `hijack` inspects topelevel +`@testset` expressions and puts toplevel `include(...)` expressions outside of +the containing testset, and should therefore be evaluated immediately. This is +not ideal, but at least allows `ReTest` to know about all the testsets right +after the call to `hijack`, and to not declare new testsets when parent +testsets are run. + +The `:outline` option might be deprecated in the future, and `include=:static` +should generally be preferred. One case where `:outline` might work better is +when the included file defines a submodule: `ReTest` doesn't have the concept +of a nested testset belonging to a different module than the parent testset, +so the best that can be done here is to "outline" such nested testsets; with +`include=:outline`, `hijack` will "process" the content of such submodules +(replace `using Test` by `using ReTest`, etc.), whereas with +`include=:static`, the subdmodules will get defined after `hijack` has +returned (on the first call to `retest` thereafter), so won't be "processed". #### `revise` keyword @@ -180,8 +196,12 @@ and to `false` otherwise. """ function hijack end +# TODO 0.4: remove `testset` kwarg +# TODO: maybe deprecate `include=:outline`? + function hijack(path::AbstractString, modname=nothing; parentmodule::Module=Main, - lazy=false, testset::Bool=false, revise::Maybe{Bool}=nothing) + lazy=false, revise::Maybe{Bool}=nothing, + include::Maybe{Symbol}=nothing, testset::Bool=false) # do first, to error early if necessary Revise = get_revise(revise) @@ -192,22 +212,34 @@ function hijack(path::AbstractString, modname=nothing; parentmodule::Module=Main modname = Symbol(modname) newmod = @eval parentmodule module $modname end - populate_mod!(newmod, path; lazy=lazy, testset=testset, Revise=Revise) + populate_mod!(newmod, path; lazy=lazy, include=setinclude(include, testset), + Revise=Revise) newmod end +function setinclude(include, testset) + if testset + include === nothing || error("cannot specify both `testset` and `include` arguments") + :outline + else + include === nothing || include === :static || include === :outline || + error("`include` keyword only accepts `:static` or `:outline` as value") + include + end +end + # this is just a work-around for v"1.5", where @__MODULE__ can't be used in # expressions; root_module[] is set equal to @__MODULE__ within modules const root_module = Ref{Symbol}() __init__() = root_module[] = gensym("MODULE") -function populate_mod!(mod::Module, path; lazy, Revise, testset) +function populate_mod!(mod::Module, path; lazy, Revise, include) lazy ∈ (true, false, :brutal) || throw(ArgumentError("the `lazy` keyword must be `true`, `false` or `:brutal`")) files = Revise === nothing ? nothing : Dict(path => mod) - substitute!(x) = substitute_retest!(x, lazy, testset, files) + substitute!(x) = substitute_retest!(x, lazy, include, files) @eval mod begin using ReTest # for files which don't have `using Test` @@ -235,7 +267,8 @@ function revise_track(Revise, files) end function hijack(packagemod::Module, modname=nothing; parentmodule::Module=Main, - lazy=false, testset::Bool=false, revise::Maybe{Bool}=nothing) + lazy=false, revise::Maybe{Bool}=nothing, + include::Maybe{Symbol}=nothing, testset::Bool=false) packagepath = pathof(packagemod) packagepath === nothing && packagemod !== Base && throw(ArgumentError("$packagemod is not a package")) @@ -253,13 +286,13 @@ function hijack(packagemod::Module, modname=nothing; parentmodule::Module=Main, else path = joinpath(dirname(dirname(packagepath)), "test", "runtests.jl") hijack(path, modname, parentmodule=parentmodule, - lazy=lazy, testset=testset, revise=revise) + lazy=lazy, testset=testset, include=include, revise=revise) end end -function substitute_retest!(ex, lazy, testset, files=nothing; +function substitute_retest!(ex, lazy, include_, files=nothing; ishijack::Bool=true) - substitute!(x) = substitute_retest!(x, lazy, testset, files, ishijack=ishijack) + substitute!(x) = substitute_retest!(x, lazy, include_, files, ishijack=ishijack) if Meta.isexpr(ex, :using) ishijack || return ex @@ -309,19 +342,24 @@ function substitute_retest!(ex, lazy, testset, files=nothing; ishijack || return ex if lazy != false && ex.args[1] ∈ TEST_MACROS empty_expr!(ex) - elseif testset && ex.args[1] == Symbol("@testset") - # we remove `include` expressions and put them out of the `@testset` - body = ex.args[end] - if body.head == :for - body = body.args[end] + elseif include_ !== nothing && ex.args[1] == Symbol("@testset") + if include_ === :outline + # we remove `include` expressions and put them out of the `@testset` + body = ex.args[end] + if body.head == :for + body = body.args[end] + end + includes = splice!(body.args, findall(body.args) do x + Meta.isexpr(x, :call) && x.args[1] == :include + end) + map!(substitute!, includes, includes) + ex.head = :block + newts = Expr(:macrocall, ex.args...) + push!(empty!(ex.args), newts, includes...) + else # :static + pos = ex.args[2] isa LineNumberNode ? 3 : 2 + insert!(ex.args, pos, :(static_include=true)) end - includes = splice!(body.args, findall(body.args) do x - Meta.isexpr(x, :call) && x.args[1] == :include - end) - map!(substitute!, includes, includes) - ex.head = :block - newts = Expr(:macrocall, ex.args...) - push!(empty!(ex.args), newts, includes...) end elseif ex isa Expr && ex.head ∈ (:block, :let, :for, :while, :if, :try) if lazy == :brutal diff --git a/test/Hijack/test/include_static.jl b/test/Hijack/test/include_static.jl new file mode 100644 index 0000000..96edfb1 --- /dev/null +++ b/test/Hijack/test/include_static.jl @@ -0,0 +1,8 @@ +# include = :static +using Hijack, Test + +@testset "include_static" begin + @test true + push!(Hijack.RUN, 1) + include("include_static_included1.jl") +end diff --git a/test/Hijack/test/include_static_included1.jl b/test/Hijack/test/include_static_included1.jl new file mode 100644 index 0000000..3f0e9f4 --- /dev/null +++ b/test/Hijack/test/include_static_included1.jl @@ -0,0 +1,9 @@ +@testset "include_static_included1 $i" for i=1:2 + @test true + @testset "nested include_static_included1" begin + @test true + push!(Hijack.RUN, 2) + # test that `include` kwarg is forwarded + include("include_static_included2.jl") + end +end diff --git a/test/Hijack/test/include_static_included2.jl b/test/Hijack/test/include_static_included2.jl new file mode 100644 index 0000000..b6cd32a --- /dev/null +++ b/test/Hijack/test/include_static_included2.jl @@ -0,0 +1,4 @@ +@testset "include_static_included2" begin + @test true + push!(Hijack.RUN, 3) +end diff --git a/test/runtests.jl b/test/runtests.jl index 89a6103..ef1647a 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -1790,10 +1790,27 @@ end empty!(Hijack.RUN) @test_throws ArgumentError ReTest.hijack("./Hijack/test/lazy.jl", :HijackWrong, lazy=:wrong) - # test testset=true + # test include=:outline empty!(Hijack.RUN) - ReTest.hijack("./Hijack/test/testset.jl", :HijackTestset, testset=true) + ReTest.hijack("./Hijack/test/testset.jl", :HijackTestset, include=:outline) retest(HijackTestset) @test Hijack.RUN == [1, 2, 3] + + # test include=:static + empty!(Hijack.RUN) + @test_throws ErrorException ReTest.hijack("./Hijack/test/include_static.jl", :HijackInclude, include=:static, testset=true) + @test_throws ErrorException ReTest.hijack("./Hijack/test/include_static.jl", :HijackInclude, include=:notvalid) + ReTest.hijack("./Hijack/test/include_static.jl", :HijackInclude, include=:static) + check(HijackInclude, dry=true, verbose=9, [], output=""" +1| include_static +2| include_static_included1 1 +3| nested include_static_included1 +4| include_static_included2 +2| include_static_included1 2 +3| nested include_static_included1 +4| include_static_included2 +""") + retest(HijackInclude) + @test Hijack.RUN == [1, 2, 3, 2, 3] end end