Skip to content

Commit

Permalink
dryrun: keep track of loop variables from parent testsets
Browse files Browse the repository at this point in the history 
Partially fixes #27.

The full fix might be to make `resolve!` also keep track of these
loop variables; but this would involve storing lists of collected
iterators for each nested testset: once we evaluate an iterator
for filtering in `resolve!`, we have to store it in case
it has some random behavior (e.g. `rand(1:9, i)` where `i` is
a loop variable from a parent testset), in which case re-evaling
it at execution time would lead to inconsistency.
The testset tree would also have to be traversed depth-first by also
unfolding testset-for. This could be a nice improvement if the perfs
don't suffer (instead of storing parent strings subjects and looping
through them, we would store iterators and recurse into them).
  • Loading branch information
@rfourquet
rfourquet committed Aug 1, 2021
1 parent d7a3ff0 commit c3b9527
Show file tree
Hide file tree
Showing 2 changed files with 73 additions and 14 deletions.
60 changes: 46 additions & 14 deletions src/ReTest.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -243,7 +243,13 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
strings = empty!(ts.strings)
desc = ts.desc
ts.loopvalues = nothing # unnecessary ?
ts.loopiters = nothing
loopiters = ts.loopiters =
if ts.loops === nothing
nothing
else
Expr(:tuple, (arg.args[1] for arg in ts.loops)...)
end

if 0 != ts.id != id && !warned[] && has(pat, Integer)
# this can happen when nested testsets are added and Revise is active
@warn "testset IDs have changed since last run"
Expand Down Expand Up @@ -337,7 +343,6 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
else # we have a testset-for with description which needs interpolation, or
# the iterator must be computed to get an iterator counter
xs = ()
loopiters = Expr(:tuple, (arg.args[1] for arg in loops)...)

try
# we need to evaluate roughly the following:
Expand All @@ -359,7 +364,6 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
xs = Core.eval(mod, xsgen)
@assert xs isa Vector
ts.loopvalues = xs
ts.loopiters = loopiters
catch
@assert xs == ()
ts.descwidth = shown ? descwidth(missing) : 0
Expand Down Expand Up @@ -409,15 +413,20 @@ function resolve!(mod::Module, ts::TestsetExpr, pat::Pattern;
run, id
end

eval_desc(mod, ts, x) =
eval_desc(mod, ts, x; stack=false) = # stack => x == iterstack in dryrun
if ts.desc isa String
ts.desc
else
try
Core.eval(mod, quote
let $(ts.loopiters) = $x
$(ts.desc)
end
Core.eval(mod,
if stack
Expr(:let, x, ts.desc)
else
quote
let $(ts.loopiters) = $x
$(ts.desc)
end
end
end)::String
catch
missing
Expand All @@ -442,7 +451,7 @@ function make_ts(ts::TestsetExpr, pat::Pattern, stats, chan)
end
else
c = count(x -> x === nothing, (ts.loopvalues, ts.loopiters))
@assert c == 0 || c == 2
@assert c == 0 || c == 1
if c == 0
loops = [Expr(:(=), ts.loopiters, ts.loopvalues)]
else
Expand Down Expand Up @@ -1448,7 +1457,7 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0,
# external calls:
; maxidw::Int, marks::Bool, tag::Vector, clear::Bool,
# only recursive calls:
evaldesc=true, repeated=nothing, show::Bool=true)
evaldesc=true, repeated=nothing, show::Bool=true, iterstack=Expr(:block))
@assert ts.run
desc = ts.desc

Expand Down Expand Up @@ -1518,7 +1527,8 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0,
for tsc in ts.children
tsc.run || continue
dryrun(mod, tsc, pat, align + 2, subject,
maxidw=maxidw, marks=marks, tag=tag, clear=clear, show=true)
maxidw=maxidw, marks=marks, tag=tag, clear=clear, show=true,
iterstack=iterstack)
end
false, false, false # meaningless unused triple
elseif marks
Expand All @@ -1536,7 +1546,7 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0,
tsc.run || continue
cp, cf, cu = dryrun(mod, tsc, pat, align + 2, subject,
maxidw=maxidw, marks=marks, tag=tag, clear=clear,
show=false)
show=false, iterstack=iterstack)
passes |= cp
fails |= cf
unrun |= cu
Expand Down Expand Up @@ -1585,10 +1595,30 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0,
ts.iter = iter # necessary when reachable is used
dryrun(mod, beginend, pat, align, parentsubj; evaldesc=false,
repeated=repeated, maxidw=maxidw, marks=marks, tag=tag,
clear=clear, show=show)
clear=clear, show=show, iterstack=iterstack)
end

loopvalues = ts.loopvalues
if loopvalues === nothing
# we check whether we can now evaluate loopvalues via iterstack
try
# cf. resolve!
xssym = gensym()
xsgen = quote
let $xssym = []
$(Expr(:for, Expr(:block, ts.loops...),
Expr(:call, Expr(:., :Base, QuoteNode(:push!)),
xssym, ts.loopiters)))
$xssym
end
end
loopvalues = Core.eval(mod, Expr(:let, iterstack, xsgen))
@assert loopvalues isa Vector
catch
@assert loopvalues == nothing
end
end

if loopvalues === nothing
# ts.desc is probably a String (cf. resolve!); if so, don't print repeated
# identitical lines (caveat: if subjects of children would change randomly)
Expand All @@ -1615,7 +1645,8 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0,
else
passes, fails, unrun = false, false, false
for (i, x) in enumerate(loopvalues)
descx = eval_desc(mod, ts, x)
push!(iterstack.args, Expr(:(=), ts.loopiters, x))
descx = eval_desc(mod, ts, iterstack, stack=true)
if descx === missing
# we would usually have `i == 1`, but not in some rare cases;
# once we find an uninterpolated description, we still assume
Expand All @@ -1628,6 +1659,7 @@ function dryrun(mod::Module, ts::TestsetExpr, pat::Pattern, align::Int=0,
else
lp, lf, lu = dryrun_beginend(descx, iter=i)
end
pop!(iterstack.args)
passes |= lp
fails |= lf
unrun |= lu
Expand Down
27 changes: 27 additions & 0 deletions test/runtests.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -635,6 +635,33 @@ end # MultiLoops
check(MultiLoops, "1 1", [(1, 1)])
end

module LoopsVariablesDryrun
# check that even with for-iterators which depend on previous loop variables,
# dryrun mode is able to compute them and corresponding descriptions,
# and filter accordingly

using ReTest

@testset "a$i" for i=1:2
@testset "b$j" for j=1:i
@test true
end
end
end

@chapter Loops begin
# no match, so this it at least filtered for final testsets
check(LoopsVariablesDryrun, "a1/b3", dry=true, verbose=9, [], output="""
1| a1
1| a2
""")
check(LoopsVariablesDryrun, "a2/b1", dry=true, verbose=9, [], output="""
1| a1
1| a2
2| b1
""")
end

# * Anonym ...................................................................

module Anonym
Expand Down

0 comments on commit c3b9527

Please sign in to comment.