From e51738a6794494094cc0fea4e03fd6b6e4fa3c5f Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 18 Mar 2020 12:40:52 +1300 Subject: [PATCH 01/39] add new range methods for random search WIP --- Project.toml | 2 + src/MLJTuning.jl | 5 +- src/range_methods.jl | 146 +++++++++++++++++++++++++ src/ranges.jl | 70 ------------ src/strategies/grid.jl | 4 +- src/strategies/random_search.jl | 132 +++++++++++++++++++++++ test/range_methods.jl | 182 ++++++++++++++++++++++++++++++++ test/ranges.jl | 95 ----------------- test/runtests.jl | 4 +- 9 files changed, 470 insertions(+), 170 deletions(-) create mode 100644 src/range_methods.jl delete mode 100644 src/ranges.jl create mode 100644 src/strategies/random_search.jl create mode 100644 test/range_methods.jl delete mode 100644 test/ranges.jl diff --git a/Project.toml b/Project.toml index 9b7d212..90cfa02 100644 --- a/Project.toml +++ b/Project.toml @@ -6,12 +6,14 @@ version = "0.2.0" [deps] ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" [compat] ComputationalResources = "^0.3" +Distributions = "^0.22" MLJBase = "^0.12" RecipesBase = "^0.8" julia = "^1" diff --git a/src/MLJTuning.jl b/src/MLJTuning.jl index c7aad28..1c75a98 100644 --- a/src/MLJTuning.jl +++ b/src/MLJTuning.jl @@ -17,8 +17,11 @@ export learning_curve!, learning_curve import MLJBase using MLJBase +import MLJBase: Bounded, Unbounded, DoublyUnbounded, + LeftUnbounded, RightUnbounded using RecipesBase using Distributed +import Distributions import ComputationalResources: CPU1, CPUProcesses, CPUThreads, AbstractResource using Random @@ -34,7 +37,7 @@ const DEFAULT_N = 10 include("utilities.jl") include("tuning_strategy_interface.jl") include("tuned_models.jl") -include("ranges.jl") +include("range_methods.jl") include("strategies/explicit.jl") include("strategies/grid.jl") include("plotrecipes.jl") diff --git a/src/range_methods.jl b/src/range_methods.jl new file mode 100644 index 0000000..92772bf --- /dev/null +++ b/src/range_methods.jl @@ -0,0 +1,146 @@ +## BOUNDEDNESS TRAIT + +# For random search and perhaps elsewhere, we need a variation on the +# built-in boundedness notions: +abstract type PositiveUnbounded <: Unbounded end +abstract type Other <: Unbounded end + +boundedness(::NumericRange{<:Any,<:Bounded}) = Bounded +boundedness(::NumericRange{<:Any,<:LeftUnbounded}) = Other +boundedness(::NumericRange{<:Any,<:DoublyUnbounded}) = Other +function boundedness(r::NumericRange{<:Any,<:RightUnbounded}) + if r.lower >= 0 + return PositiveUnbounded + end + return Other +end + + +## PRE-PROCESSING OF USER-SPECIFIED CARTESIAN RANGE OBJECTS + +""" + MLJTuning.grid([rng, ] prototype, ranges, resolutions) + +Given an iterable `ranges` of `ParamRange` objects, and an iterable +`resolutions` of the same length, return a vector of models generated +by cloning and mutating the hyperparameters (fields) of `prototype`, +according to the Cartesian grid defined by the specifed +one-dimensional `ranges` (`ParamRange` objects) and specified +`resolutions`. A resolution of `nothing` for a `NominalRange` +indicates that all values should be used. + +Specification of an `AbstractRNG` object `rng` implies shuffling of +the results. Otherwise models are ordered, with the first +hyperparameter referenced cycling fastest. + +""" +grid(rng::AbstractRNG, prototype::Model, ranges, resolutions) = + shuffle(rng, grid(prototype, ranges, resolutions)) + +function grid(prototype::Model, ranges, resolutions) + + iterators = broadcast(iterator, ranges, resolutions) + + A = MLJBase.unwind(iterators...) + + N = size(A, 1) + map(1:N) do i + clone = deepcopy(prototype) + for k in eachindex(ranges) + field = ranges[k].field + recursive_setproperty!(clone, field, A[i,k]) + end + clone + end +end + +""" + process_grid_range(user_specified_range, resolution, verbosity) + +Utility to convert a user-specified range (see [`Grid`](@ref)) into a +pair of tuples `(ranges, resolutions)`. + +For example, if `r1`, `r2` are `NumericRange`s and `s` is a +NominalRange` with 5 values, then we have: + + julia> MLJTuning.process_grid_range([(r1, 3), r2, s], 42, 1) == + ((r1, r2, s), (3, 42, 5)) + true + +If `verbosity` > 0, then a warning is issued if a `Nominal` range is +paired with a resolution. + +""" +process_grid_range(user_specified_range, args...) = + throw(ArgumentError("Unsupported range. ")) + +process_grid_range(usr::Union{ParamRange,Tuple{ParamRange,Int}}, args...) = + process_grid_range([usr, ], args...) + +function process_grid_range(user_specified_range::AbstractVector, + resolution, verbosity) + # r unpaired: + stand(r) = throw(ArgumentError("Unsupported range. ")) + stand(r::NumericRange) = (r, resolution) + stand(r::NominalRange) = (r, length(r.values)) + + # (r, res): + stand(t::Tuple{NumericRange,Integer}) = t + function stand(t::Tuple{NominalRange,Integer}) + verbosity < 0 || + @warn "Ignoring a resolution specified for a `NominalRange`. " + return (first(t), length(first(t).values)) + end + + ret = zip(stand.(user_specified_range)...) |> collect + return first(ret), last(ret) +end + +""" + process_random_range(user_specified_range, + bounded, + positive_unbounded, + other) + +Utility to convert a user-specified range (see [`RandomSearch`](@ref)) +into an n-tuple of `(field, sampler)` pairs. + +""" +process_random_range(user_specified_range, args...) = + throw(ArgumentError("Unsupported range. ")) + +const DIST = Distributions.Distribution +const DD = Union{DIST, Type{<:DIST}} +const AllowedPairs = Union{Tuple{NumericRange,DD}, + Tuple{NominalRange,AbstractVector{<:AbstractFloat}}} + +process_random_range(user_specified_range::Union{ParamRange, AllowedPairs}, + args...) = + process_random_range([user_specified_range, ], args...) + +function process_random_range(user_specified_range::AbstractVector, + bounded, + positive_unbounded, + other) + @show 1 + + # r not paired: + stand(r) = throw(ArgumentError("Unsupported range. ")) + stand(r::NumericRange) = stand(r, boundedness(r)) + stand(r::NumericRange, ::Type{<:Bounded}) = (r.field, sampler(r, bounded)) + stand(r::NumericRange, ::Type{<:Other}) = (r.field, sampler(r, other)) + stand(r::NumericRange, ::Type{<:PositiveUnbounded}) = + (r.field, sampler(r, positive_unbounded)) + stand(r::NominalRange) = (n = length(r.values); + (r.field, sampler(r, fill(1/n, n)))) + # (r, d): + stand(t::AllowedPairs) = (r = first(t); (r.field, sampler(r, last(t)))) + + # (field, s): + stand(t::Tuple{Union{Symbol,Expr},Any}) = t + + return Tuple(stand.(user_specified_range)) + + # ret = zip(stand.(user_specified_range)...) |> collect + # return first(ret), last(ret) +end diff --git a/src/ranges.jl b/src/ranges.jl deleted file mode 100644 index 37e375d..0000000 --- a/src/ranges.jl +++ /dev/null @@ -1,70 +0,0 @@ -""" - MLJTuning.grid([rng, ] prototype, ranges, resolutions) - -Given an iterable `ranges` of `ParamRange` objects, and an iterable -`resolutions` of the same length, return a vector of models generated -by cloning and mutating the hyperparameters (fields) of `prototype`, -according to the Cartesian grid defined by the specifed -one-dimensional `ranges` (`ParamRange` objects) and specified -`resolutions`. A resolution of `nothing` for a `NominalRange` -indicates that all values should be used. - -Specification of an `AbstractRNG` object `rng` implies shuffling of -the results. Otherwise models are ordered, with the first -hyperparameter referenced cycling fastest. - -""" -grid(rng::AbstractRNG, prototype::Model, ranges, resolutions) = - shuffle(rng, grid(prototype, ranges, resolutions)) - -function grid(prototype::Model, ranges, resolutions) - - iterators = broadcast(iterator, ranges, resolutions) - - A = MLJBase.unwind(iterators...) - - N = size(A, 1) - map(1:N) do i - clone = deepcopy(prototype) - for k in eachindex(ranges) - field = ranges[k].field - recursive_setproperty!(clone, field, A[i,k]) - end - clone - end -end - -""" - process_user_range(user_specified_range, resolution, verbosity) - -Utility to convert user-specified range (see [`Grid`](@ref)) into a -pair of tuples `(ranges, resolutions)`. - -For example, if `r1`, `r2` are `NumericRange`s and `s` is a -NominalRange` with 5 values, then we have: - - julia> MLJTuning.process_user_range([(r1, 3), r2, s], 42, 1) == - ((r1, r2, s), (3, 42, 5)) - true - -If `verbosity` > 0, then a warning is issued if a `Nominal` range is -paired with a resolution. - -""" -process_user_range(user_specified_range, resolution, verbosity) = - process_user_range([user_specified_range, ], resolution, verbosity) -function process_user_range(user_specified_range::AbstractVector, - resolution, verbosity) - stand(r) = throw(ArgumentError("Unsupported range. ")) - stand(r::NumericRange) = (r, resolution) - stand(r::NominalRange) = (r, length(r.values)) - stand(t::Tuple{NumericRange,Integer}) = t - function stand(t::Tuple{NominalRange,Integer}) - verbosity < 0 || - @warn "Ignoring a resolution specified for a `NominalRange`. " - return (first(t), length(first(t).values)) - end - - ret = zip(stand.(user_specified_range)...) |> collect - return first(ret), last(ret) -end diff --git a/src/strategies/grid.jl b/src/strategies/grid.jl index 5c31266..812ddd3 100644 --- a/src/strategies/grid.jl +++ b/src/strategies/grid.jl @@ -83,7 +83,7 @@ end function setup(tuning::Grid, model, user_range, verbosity) ranges, resolutions = - process_user_range(user_range, tuning.resolution, verbosity) + process_grid_range(user_range, tuning.resolution, verbosity) resolutions = adjusted_resolutions(tuning.goal, ranges, resolutions) fields = map(r -> r.field, ranges) @@ -123,7 +123,7 @@ end function default_n(tuning::Grid, user_range) ranges, resolutions = - process_user_range(user_range, tuning.resolution, -1) + process_grid_range(user_range, tuning.resolution, -1) resolutions = adjusted_resolutions(tuning.goal, ranges, resolutions) len(t::Tuple{NumericRange,Integer}) = length(iterator(t[1], t[2])) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl new file mode 100644 index 0000000..c64b095 --- /dev/null +++ b/src/strategies/random_search.jl @@ -0,0 +1,132 @@ +const ParameterName=Union{Symbol,Expr} + +""" + RandomSearch(bounded=Distributions.Uniform, + positive_unbounded=Distributions.Gamma, + others=Normal, + rng=Random.GLOBAL_RNG) + +Instantiate a random search tuning strategy for searching over +Cartesian hyperparemeter domains. + +### Supported ranges: + +- A single one-dimensional range (`ParamRange` object) `r` + +- A pair of the form `(r, d)`, with `r` as above and where `d` is a + probability vector of the same length as `r.values`, if `r` is a + `NominalRange`, and is otherwise: (i) any `Distributions.Univariate` + *instance*; or (ii) one of the *subtypes* of + `Distributions.Univariate` listed in the table below, for automatic + fitting using `Distributions.fit(d, r)` (a distribution whose + support always lies between `r.lower` and `r.upper`. + +- Any pair of the form `(field, s)`, where `field` is the, possibly + nested, name of a field the model to be tuned, and `s` an arbitrary + sampler object for that field. (This only means `rand(rng, s)` is defined and + returns valid values for the field.) + +- Any vector of objects of the above form + +distribution types | for fitting to ranges of this type +--------------------|----------------------------------- +`Arcsine`, `Uniform`, `Biweight`, `Cosine`, `Epanechnikov`, `SymTriangularDist`, `Triweight` | bounded +`Gamma`, `InverseGaussian`, `Poisson` | positive +`Normal`, `Logistic`, `LogNormal`, `Cauchy`, `Gumbel`, `Laplace` | any + +`ParamRange` objects are constructed using the `range` method. + +### Examples: + + range1 = range(model, :hyper1, lower=1, origin=2, unit=1) + + range2 = [(range(model, :hyper1, lower=1, upper=10), Arcsine), + range(model, :hyper2, lower=2, upper=4), + (range(model, :hyper2, lower=2, upper=4), Normal(0, 3)), + range(model, :hyper3, values=[:ball, :tree], [0.3, 0.7])] + + # uniform sampling of :(atom.λ) from [0, 1] without defining a NumericRange: + struct MySampler end + Base.rand(rng::AbstractRNG, ::MySampler) = rand(rng) + range3 = (:(atom.λ), MySampler()) + +### Algorithm + +Models for evaulation are generated by sampling each range `r` using +`rand(rng, s)` where, `s = sampler(r, d)`. See `sampler` for details. If `d` +is not specified, then sampling is uniform (with replacement) in the +case of a `NominalRange`, and is otherwise given by the defaults +specified by the tuning strategy parameters `bounded`, +`positive_unbounded`, and `other`, depending on the `NumericRange` +type. + +See also [`TunedModel`](@ref), [`range`](@ref), [`sampler`](@ref). + +""" +mutable struct RandomSearch <: TuningStrategy + bounded::Distributions.Univariate + positive_unbounded::Distributions.Univariate + others::Distribution.Univariate + rng::Random.AbstractRNG +end + +# Constructor with keywords +function RandomSearch(; bounded=Distributions.Uniform, + positive_unbounded=Distributions.Gamma, + others=Normal, + rng=Random.GLOBAL_RNG) + _rng = rng isa Integer ? Random.MersenneTwister(rng) : rng + return RandomSearch(bounded, positive_unbounded, others, rng) +end + +isnumeric(::Any) = false +isnumeric(::NumericRange) = true + +# function setup(tuning::RandomSearch, model, user_range, verbosity) +# ranges, distributions = # I AM HERE +# process_user_random_range(user_range, tuning.distribution, verbosity) +# distributions = adjusted_distributions(tuning.goal, ranges, distributions) + +# fields = map(r -> r.field, ranges) + +# parameter_scales = scale.(ranges) + +# if tuning.shuffle +# models = grid(tuning.rng, model, ranges, distributions) +# else +# models = grid(model, ranges, distributions) +# end + +# state = (models=models, +# fields=fields, +# parameter_scales=parameter_scales) + +# return state + +# end + +# MLJTuning.models!(tuning::RandomSearch, model, history::Nothing, +# state, verbosity) = state.models +# MLJTuning.models!(tuning::RandomSearch, model, history, +# state, verbosity) = +# state.models[length(history) + 1:end] + +# function tuning_report(tuning::RandomSearch, history, state) + +# plotting = plotting_report(state.fields, state.parameter_scales, history) + +# # todo: remove collects? +# return (history=history, plotting=plotting) + +# end + +# function default_n(tuning::RandomSearch, user_range) +# ranges, distributions = +# process_grid_range(user_range, tuning.distribution, -1) + +# distributions = adjusted_distributions(tuning.goal, ranges, distributions) +# len(t::Tuple{NumericRange,Integer}) = length(iterator(t[1], t[2])) +# len(t::Tuple{NominalRange,Integer}) = t[2] +# return prod(len.(zip(ranges, distributions))) + +# end diff --git a/test/range_methods.jl b/test/range_methods.jl new file mode 100644 index 0000000..c1ee037 --- /dev/null +++ b/test/range_methods.jl @@ -0,0 +1,182 @@ +module TestRanges + +using Test +using MLJBase +using MLJTuning +using Random +import Distributions +const Dist = Distributions + +# `in` for MLJType is overloaded to be `===` based. For purposed of +# testing here, we need `==` based: +function _in(x, itr)::Union{Bool,Missing} + for y in itr + ismissing(y) && return missing + y == x && return true + end + return false +end +_issubset(itr1, itr2) = all(_in(x, itr2) for x in itr1) + +@testset "boundedness traits" begin + r1 = range(Float64, :K, lower=1, upper=10) + r2 = range(Float64, :K, lower=-1, upper=Inf, origin=1, unit=1) + r3 = range(Float64, :K, lower=0, upper=Inf, origin=1, unit=1) + r4 = range(Float64, :K, lower=-Inf, upper=1, origin=0, unit=1) + r5 = range(Float64, :K, lower=-Inf, upper=Inf, origin=1, unit=1) + @test MLJTuning.boundedness(r1) == MLJTuning.Bounded + @test MLJTuning.boundedness(r2) == MLJTuning.Other + @test MLJTuning.boundedness(r3) == MLJTuning.PositiveUnbounded + @test MLJTuning.boundedness(r4) == MLJTuning.Other + @test MLJTuning.boundedness(r5) == MLJTuning.Other +end + +mutable struct DummyModel <: Deterministic + lambda::Float64 + metric::Float64 + kernel::Char +end + +dummy_model = DummyModel(4, 9.5, 'k') + +mutable struct SuperModel <: Deterministic + K::Int64 + model1::DummyModel + model2::DummyModel +end + +dummy_model = DummyModel(1.2, 9.5, 'k') +super_model = SuperModel(4, dummy_model, deepcopy(dummy_model)) + +r1 = range(super_model, :(model1.kernel), values=['c', 'd']) +r2 = range(super_model, :K, lower=1, upper=10, scale=:log10) + +@testset "models from cartesian range and resolutions" begin + + # with method: + m1 = MLJTuning.grid(super_model, [r1, r2], [nothing, 7]) + m1r = MLJTuning.grid(MersenneTwister(123), super_model, [r1, r2], + [nothing, 7]) + + # generate all models by hand: + models1 = [SuperModel(1, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(1, DummyModel(1.2, 9.5, 'd'), dummy_model), + SuperModel(2, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(2, DummyModel(1.2, 9.5, 'd'), dummy_model), + SuperModel(3, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(3, DummyModel(1.2, 9.5, 'd'), dummy_model), + SuperModel(5, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(5, DummyModel(1.2, 9.5, 'd'), dummy_model), + SuperModel(7, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(7, DummyModel(1.2, 9.5, 'd'), dummy_model), + SuperModel(10, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(10, DummyModel(1.2, 9.5, 'd'), dummy_model)] + + @test _issubset(models1, m1) && _issubset(m1, models1) + @test m1r != models1 + @test _issubset(models1, m1r) && _issubset(m1, models1) + + # with method: + m2 = MLJTuning.grid(super_model, [r1, r2], [1, 7]) + + # generate all models by hand: + models2 = [SuperModel(1, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(2, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(3, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(5, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(7, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(10, DummyModel(1.2, 9.5, 'c'), dummy_model)] + + @test _issubset(models2, m2) && _issubset(m2, models2) + +end + +@testset "processing user specification of range in Grid" begin + r1 = range(Int, :h1, lower=1, upper=10) + r2 = range(Int, :h2, lower=20, upper=30) + s = range(Char, :j1, values = ['x', 'y']) + @test_throws ArgumentError MLJTuning.process_grid_range("junk", 42, 1) + @test(@test_logs((:warn, r"Ignoring"), + MLJTuning.process_grid_range((s, 3), 42, 1)) == + ((s, ), (2, ))) + @test MLJTuning.process_grid_range(r1, 42, 1) == ((r1, ), (42, )) + @test MLJTuning.process_grid_range((r1, 3), 42, 1) == ((r1, ), (3, )) + @test MLJTuning.process_grid_range(s, 42, 1) == ((s, ), (2,)) + @test MLJTuning.process_grid_range([(r1, 3), r2, s], 42, 1) == + ((r1, r2, s), (3, 42, 2)) +end + +struct MySampler end +Base.rand(rng::AbstractRNG, ::MySampler) = rand(rng) + +@testset "processing user specification of range in RandomSearch" begin + r1 = range(Int, :h1, lower=1, upper=10, scale=exp) + r2 = range(Int, :h2, lower=5, upper=Inf, origin=10, unit=5) + r3 = range(Char, :j1, values = ['x', 'y']) + s = MySampler() + + @test_throws(ArgumentError, + MLJTuning.process_random_range("junk", + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy)) + @test_throws(ArgumentError, + MLJTuning.process_random_range((r1, "junk"), + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy)) + @test_throws(ArgumentError, + MLJTuning.process_random_range((r3, "junk"), + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy)) + @test_throws(ArgumentError, + MLJTuning.process_random_range(("junk", s), + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy)) + + # unpaired numeric range: + pp = MLJTuning.process_random_range(r1, + Dist.Uniform, # bounded + Dist.Gamma, # positive_unbounded + Dist.Cauchy) # other + @test pp isa Tuple{Tuple{Symbol,MLJBase.NumericSampler}} + p = first(pp) + @test first(p) == :h1 + s = last(p) + @test s.scale == r1.scale + @test s.distribution == Dist.Uniform(1.0, 10.0) + + # unpaired nominal range: + p = MLJTuning.process_random_range(r3, + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy) |> first + @test first(p) == :j1 + s = last(p) + @test s.values == r3.values + @test s.distribution.p == [0.5, 0.5] + @test s.distribution.support == 1:2 + + # (numeric range, distribution instance): + p = MLJTuning.process_random_range((r2, Dist.Poisson(3)), + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy) |> first + @test first(p) == :h2 + s = last(p) + @test s.scale == r2.scale + @test s.distribution == Dist.truncated(Dist.Poisson(3.0), 5.0, Inf) + + # (numeric range, distribution type): + p = MLJTuning.process_random_range((r2, Dist.Poisson), + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy) |> first + s = last(p) + @test s.distribution == Dist.truncated(Poisson(r2.origin), 5.0, Inf) + + +end +true diff --git a/test/ranges.jl b/test/ranges.jl deleted file mode 100644 index 17b3af1..0000000 --- a/test/ranges.jl +++ /dev/null @@ -1,95 +0,0 @@ -module TestRanges - -using Test -using MLJBase -using MLJTuning -using Random - -# `in` for MLJType is overloaded to be `===` based. For purposed of -# testing here, we need `==` based: -function _in(x, itr)::Union{Bool,Missing} - for y in itr - ismissing(y) && return missing - y == x && return true - end - return false -end -_issubset(itr1, itr2) = all(_in(x, itr2) for x in itr1) - -mutable struct DummyModel <: Deterministic - lambda::Float64 - metric::Float64 - kernel::Char -end - -dummy_model = DummyModel(4, 9.5, 'k') - -mutable struct SuperModel <: Deterministic - K::Int64 - model1::DummyModel - model2::DummyModel -end - -dummy_model = DummyModel(1.2, 9.5, 'k') -super_model = SuperModel(4, dummy_model, deepcopy(dummy_model)) - -r1 = range(super_model, :(model1.kernel), values=['c', 'd']) -r2 = range(super_model, :K, lower=1, upper=10, scale=:log10) - -@testset "models from cartesian range and resolutions" begin - - # with method: - m1 = MLJTuning.grid(super_model, [r1, r2], [nothing, 7]) - m1r = MLJTuning.grid(MersenneTwister(123), super_model, [r1, r2], - [nothing, 7]) - - # generate all models by hand: - models1 = [SuperModel(1, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(1, DummyModel(1.2, 9.5, 'd'), dummy_model), - SuperModel(2, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(2, DummyModel(1.2, 9.5, 'd'), dummy_model), - SuperModel(3, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(3, DummyModel(1.2, 9.5, 'd'), dummy_model), - SuperModel(5, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(5, DummyModel(1.2, 9.5, 'd'), dummy_model), - SuperModel(7, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(7, DummyModel(1.2, 9.5, 'd'), dummy_model), - SuperModel(10, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(10, DummyModel(1.2, 9.5, 'd'), dummy_model)] - - @test _issubset(models1, m1) && _issubset(m1, models1) - @test m1r != models1 - @test _issubset(models1, m1r) && _issubset(m1, models1) - - # with method: - m2 = MLJTuning.grid(super_model, [r1, r2], [1, 7]) - - # generate all models by hand: - models2 = [SuperModel(1, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(2, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(3, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(5, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(7, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(10, DummyModel(1.2, 9.5, 'c'), dummy_model)] - - @test _issubset(models2, m2) && _issubset(m2, models2) - -end - -@testset "processing user specification of range" begin - r1 = range(Int, :h1, lower=1, upper=10) - r2 = range(Int, :h2, lower=20, upper=30) - s = range(Char, :j1, values = ['x', 'y']) - @test_throws ArgumentError MLJTuning.process_user_range("junk", 42, 1) - @test(@test_logs((:warn, r"Ignoring"), - MLJTuning.process_user_range((s, 3), 42, 1)) == - ((s, ), (2, ))) - @test MLJTuning.process_user_range(r1, 42, 1) == ((r1, ), (42, )) - @test MLJTuning.process_user_range((r1, 3), 42, 1) == ((r1, ), (3, )) - @test MLJTuning.process_user_range(s, 42, 1) == ((s, ), (2,)) - @test MLJTuning.process_user_range([(r1, 3), r2, s], 42, 1) == - ((r1, r2, s), (3, 42, 2)) -end - -end -true diff --git a/test/runtests.jl b/test/runtests.jl index 4764491..e951237 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,8 +20,8 @@ end @test include("tuned_models.jl") end -@testset "ranges" begin - @test include("ranges.jl") +@testset "range_methods" begin + @test include("range_methods.jl") end @testset "grid" begin From 28cf59dc3eda9b6b0397f28d3c60da18d177e02d Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 18 Mar 2020 18:33:57 +1300 Subject: [PATCH 02/39] update to [compat] MLJBase = "0.12.2" to enable scale(s) extension --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 90cfa02..5296b7a 100644 --- a/Project.toml +++ b/Project.toml @@ -14,7 +14,7 @@ RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" [compat] ComputationalResources = "^0.3" Distributions = "^0.22" -MLJBase = "^0.12" +MLJBase = "^0.12.2" RecipesBase = "^0.8" julia = "^1" From 29119b26c3377b496a24a7934bd08b8fc941b71c Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 26 Mar 2020 13:20:07 +1300 Subject: [PATCH 03/39] resolve some stash pop conflicts --- README.md | 13 ++++ src/MLJTuning.jl | 2 +- src/range_methods.jl | 22 +++--- src/strategies/random_search.jl | 111 +++++++++++++++---------------- src/tuning_strategy_interface.jl | 2 - test/range_methods.jl | 3 +- 6 files changed, 81 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 8f5f7e7..7b21513 100644 --- a/README.md +++ b/README.md @@ -373,6 +373,11 @@ is `fit!` the first time, and not on subsequent calls (unless `force=true`). (Specifically, `MLJBase.fit(::TunedModel, ...)` calls `setup` but `MLJBase.update(::TunedModel, ...)` does not.) +The `setup` function is called once only, when a `TunedModel` machine +is `fit!` the first time, and not on subsequent calls (unless +`force=true`). (Specifically, `MLJBase.fit(::TunedModel, ...)` calls +`setup` but `MLJBase.update(::TunedModel, ...)` does not.) + The `verbosity` is an integer indicating the level of logging: `0` means logging should be restricted to warnings, `-1`, means completely silent. @@ -440,6 +445,14 @@ any number of models. If `models!` returns a number of models exceeding the number needed to complete the history, the list returned is simply truncated. +Some simple tuning strategies, such as `RandomSearch`, will want to +return as many models as possible in one hit. The argument +`n_remaining` is the difference between the current length of the +history and the target number of iterations `tuned_model.n` set by the +user when constructing his `TunedModel` instance, `tuned_model` (or +`default_n(tuning, range)` if left unspecified). + + #### The `best` method: To define what constitutes the "optimal model" ```julia diff --git a/src/MLJTuning.jl b/src/MLJTuning.jl index 1c75a98..68bccf9 100644 --- a/src/MLJTuning.jl +++ b/src/MLJTuning.jl @@ -29,7 +29,7 @@ using Random ## CONSTANTS -const DEFAULT_N = 10 +const DEFAULT_N == 10 # for when `default_n` is not implemented ## INCLUDE FILES diff --git a/src/range_methods.jl b/src/range_methods.jl index 92772bf..be130a8 100644 --- a/src/range_methods.jl +++ b/src/range_methods.jl @@ -1,3 +1,8 @@ +## SCALE FOR SAMPLERS + +TODO: + + ## BOUNDEDNESS TRAIT # For random search and perhaps elsewhere, we need a variation on the @@ -107,14 +112,11 @@ into an n-tuple of `(field, sampler)` pairs. """ process_random_range(user_specified_range, args...) = - throw(ArgumentError("Unsupported range. ")) + throw(ArgumentError("Unsupported range #1. ")) const DIST = Distributions.Distribution -const DD = Union{DIST, Type{<:DIST}} -const AllowedPairs = Union{Tuple{NumericRange,DD}, - Tuple{NominalRange,AbstractVector{<:AbstractFloat}}} -process_random_range(user_specified_range::Union{ParamRange, AllowedPairs}, +process_random_range(user_specified_range::Union{ParamRange, Tuple{Any,Any}}, args...) = process_random_range([user_specified_range, ], args...) @@ -122,10 +124,8 @@ function process_random_range(user_specified_range::AbstractVector, bounded, positive_unbounded, other) - @show 1 - # r not paired: - stand(r) = throw(ArgumentError("Unsupported range. ")) + stand(r) = throw(ArgumentError("Unsupported range #2. ")) stand(r::NumericRange) = stand(r, boundedness(r)) stand(r::NumericRange, ::Type{<:Bounded}) = (r.field, sampler(r, bounded)) stand(r::NumericRange, ::Type{<:Other}) = (r.field, sampler(r, other)) @@ -134,7 +134,11 @@ function process_random_range(user_specified_range::AbstractVector, stand(r::NominalRange) = (n = length(r.values); (r.field, sampler(r, fill(1/n, n)))) # (r, d): - stand(t::AllowedPairs) = (r = first(t); (r.field, sampler(r, last(t)))) + stand(t::Tuple{ParamRange,Any}) = stand(t...) + stand(r, d) = throw(ArgumentError("Unsupported range #3. ")) + stand(r::NominalRange, d::AbstractVector{Float64}) = _stand(r, d) + stand(r::NumericRange, d:: Union{DIST, Type{<:DIST}}) = _stand(r, d) + _stand(r, d) = (r.field, sampler(r, d)) # (field, s): stand(t::Tuple{Union{Symbol,Expr},Any}) = t diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index c64b095..6b295c6 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -3,35 +3,36 @@ const ParameterName=Union{Symbol,Expr} """ RandomSearch(bounded=Distributions.Uniform, positive_unbounded=Distributions.Gamma, - others=Normal, + others=Distributions.Normal, rng=Random.GLOBAL_RNG) -Instantiate a random search tuning strategy for searching over -Cartesian hyperparemeter domains. +Instantiate a random search tuning strategy, for searching over +Cartesian hyperparameter domains, with customizable priors in each +dimenension. ### Supported ranges: -- A single one-dimensional range (`ParamRange` object) `r` +- a single one-dimensional range (`ParamRange` object) `r` -- A pair of the form `(r, d)`, with `r` as above and where `d` is a +- a pair of the form `(r, d)`, with `r` as above and where `d` is a probability vector of the same length as `r.values`, if `r` is a `NominalRange`, and is otherwise: (i) any `Distributions.Univariate` *instance*; or (ii) one of the *subtypes* of `Distributions.Univariate` listed in the table below, for automatic fitting using `Distributions.fit(d, r)` (a distribution whose - support always lies between `r.lower` and `r.upper`. + support always lies between `r.lower` and `r.upper`.) -- Any pair of the form `(field, s)`, where `field` is the, possibly - nested, name of a field the model to be tuned, and `s` an arbitrary - sampler object for that field. (This only means `rand(rng, s)` is defined and - returns valid values for the field.) +- any pair of the form `(field, s)`, where `field` is the (possibly + nested) name of a field of the model to be tuned, and `s` an + arbitrary sampler object for that field. This means only that + `rand(rng, s)` is defined and returns valid values for the field. -- Any vector of objects of the above form +- any vector of objects of the above form distribution types | for fitting to ranges of this type --------------------|----------------------------------- `Arcsine`, `Uniform`, `Biweight`, `Cosine`, `Epanechnikov`, `SymTriangularDist`, `Triweight` | bounded -`Gamma`, `InverseGaussian`, `Poisson` | positive +`Gamma`, `InverseGaussian`, `Poisson` | positive (bounded or unbounded) `Normal`, `Logistic`, `LogNormal`, `Cauchy`, `Gumbel`, `Laplace` | any `ParamRange` objects are constructed using the `range` method. @@ -48,7 +49,7 @@ distribution types | for fitting to ranges of this type # uniform sampling of :(atom.λ) from [0, 1] without defining a NumericRange: struct MySampler end Base.rand(rng::AbstractRNG, ::MySampler) = rand(rng) - range3 = (:(atom.λ), MySampler()) + range3 = (:(atom.λ), MySampler(), range1) ### Algorithm @@ -63,7 +64,7 @@ type. See also [`TunedModel`](@ref), [`range`](@ref), [`sampler`](@ref). """ -mutable struct RandomSearch <: TuningStrategy +mutable struct RandomSearch <: TuningStrategy bounded::Distributions.Univariate positive_unbounded::Distributions.Univariate others::Distribution.Univariate @@ -73,60 +74,54 @@ end # Constructor with keywords function RandomSearch(; bounded=Distributions.Uniform, positive_unbounded=Distributions.Gamma, - others=Normal, + others=Distributions.Normal, rng=Random.GLOBAL_RNG) _rng = rng isa Integer ? Random.MersenneTwister(rng) : rng - return RandomSearch(bounded, positive_unbounded, others, rng) + return RandomSearch(bounded, positive_unbounded, others, _rng) end isnumeric(::Any) = false isnumeric(::NumericRange) = true -# function setup(tuning::RandomSearch, model, user_range, verbosity) -# ranges, distributions = # I AM HERE -# process_user_random_range(user_range, tuning.distribution, verbosity) -# distributions = adjusted_distributions(tuning.goal, ranges, distributions) +# `state`, which is not mutated, consists of a tuple of (field, sampler) +# pairs: +setup(tuning::RandomSearch, model, user_range, verbosity) = + process_user_random_range(user_range, + tuning.bounded, + tuning.positive_unbounded, + tuning.other) + +function MLJTuning.models!(tuning::RandomSearch, + model, + history + state, + verbosity) + + # We generate all remaining models at once. Since the value of + # `tuning.n` can change between calls to `models!` + n_so_far = _length(history) # _length(nothing) = 0 + n = tuning.n === nothing ? DEFAULT_N : tuning.n + n_models = max(0, n - n_so_far) + + return map(1:n_models) do _ + clone = deepcopy(model) + for (fld, s) field_sampler_pairs + recursive_setproperty!(clone, fld, rand(rng, s)) + end + clone + end -# fields = map(r -> r.field, ranges) - -# parameter_scales = scale.(ranges) - -# if tuning.shuffle -# models = grid(tuning.rng, model, ranges, distributions) -# else -# models = grid(model, ranges, distributions) -# end - -# state = (models=models, -# fields=fields, -# parameter_scales=parameter_scales) - -# return state - -# end - -# MLJTuning.models!(tuning::RandomSearch, model, history::Nothing, -# state, verbosity) = state.models -# MLJTuning.models!(tuning::RandomSearch, model, history, -# state, verbosity) = -# state.models[length(history) + 1:end] - -# function tuning_report(tuning::RandomSearch, history, state) - -# plotting = plotting_report(state.fields, state.parameter_scales, history) +end -# # todo: remove collects? -# return (history=history, plotting=plotting) +function tuning_report(tuning::RandomSearch, history, field_sampler_pairs) -# end + fields = first.(field_sampler_pairs) + parameter_scales = map(field_sampler_pairs) do (fld, s) + scale(s) + end -# function default_n(tuning::RandomSearch, user_range) -# ranges, distributions = -# process_grid_range(user_range, tuning.distribution, -1) + plotting = plotting_report(fields, parameter_scales, history) -# distributions = adjusted_distributions(tuning.goal, ranges, distributions) -# len(t::Tuple{NumericRange,Integer}) = length(iterator(t[1], t[2])) -# len(t::Tuple{NominalRange,Integer}) = t[2] -# return prod(len.(zip(ranges, distributions))) + return (history=history, plotting=plotting) -# end +end diff --git a/src/tuning_strategy_interface.jl b/src/tuning_strategy_interface.jl index 4c58b18..9a93e0a 100644 --- a/src/tuning_strategy_interface.jl +++ b/src/tuning_strategy_interface.jl @@ -30,5 +30,3 @@ tuning_report(tuning::TuningStrategy, history, state) = (history=history,) # for declaring the default number of models to evaluate: default_n(tuning::TuningStrategy, range) = DEFAULT_N - - diff --git a/test/range_methods.jl b/test/range_methods.jl index c1ee037..46f8832 100644 --- a/test/range_methods.jl +++ b/test/range_methods.jl @@ -175,8 +175,7 @@ Base.rand(rng::AbstractRNG, ::MySampler) = rand(rng) Dist.Gamma, Dist.Cauchy) |> first s = last(p) - @test s.distribution == Dist.truncated(Poisson(r2.origin), 5.0, Inf) - + @test s.distribution == Dist.truncated(Dist.Poisson(r2.unit), 5.0, Inf) end true From 07b9e21e9062d9730ecf1462811eb0cc5695d30d Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 26 Mar 2020 17:06:02 +1300 Subject: [PATCH 04/39] fix some wrong code due to flawed rebase --- src/MLJTuning.jl | 2 +- src/range_methods.jl | 5 ----- src/strategies/random_search.jl | 9 ++------- test/range_methods.jl | 2 ++ test/tuned_models.jl | 2 +- 5 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/MLJTuning.jl b/src/MLJTuning.jl index 68bccf9..5a8fa81 100644 --- a/src/MLJTuning.jl +++ b/src/MLJTuning.jl @@ -29,7 +29,7 @@ using Random ## CONSTANTS -const DEFAULT_N == 10 # for when `default_n` is not implemented +const DEFAULT_N = 10 # for when `default_n` is not implemented ## INCLUDE FILES diff --git a/src/range_methods.jl b/src/range_methods.jl index be130a8..778ff65 100644 --- a/src/range_methods.jl +++ b/src/range_methods.jl @@ -1,8 +1,3 @@ -## SCALE FOR SAMPLERS - -TODO: - - ## BOUNDEDNESS TRAIT # For random search and perhaps elsewhere, we need a variation on the diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index 6b295c6..a556ed1 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -95,15 +95,10 @@ function MLJTuning.models!(tuning::RandomSearch, model, history state, + n_remaining, verbosity) - # We generate all remaining models at once. Since the value of - # `tuning.n` can change between calls to `models!` - n_so_far = _length(history) # _length(nothing) = 0 - n = tuning.n === nothing ? DEFAULT_N : tuning.n - n_models = max(0, n - n_so_far) - - return map(1:n_models) do _ + return map(1:n_remaining) do _ clone = deepcopy(model) for (fld, s) field_sampler_pairs recursive_setproperty!(clone, fld, rand(rng, s)) diff --git a/test/range_methods.jl b/test/range_methods.jl index 46f8832..cd1c1be 100644 --- a/test/range_methods.jl +++ b/test/range_methods.jl @@ -177,5 +177,7 @@ Base.rand(rng::AbstractRNG, ::MySampler) = rand(rng) s = last(p) @test s.distribution == Dist.truncated(Dist.Poisson(r2.unit), 5.0, Inf) +end + end true diff --git a/test/tuned_models.jl b/test/tuned_models.jl index 18e4340..ee5a68a 100644 --- a/test/tuned_models.jl +++ b/test/tuned_models.jl @@ -23,7 +23,7 @@ y = 2*x1 .+ 5*x2 .- 3*x3 .+ 0.4*rand(N); m(K) = KNNRegressor(K=K) r = [m(K) for K in 2:13] -# TODO: replace the above with the line below and fix post an issue on +# TODO: replace the above with the line below and post an issue on # the failure (a bug in Distributed, I reckon): # r = (m(K) for K in 2:13) From ab1d51563f2e666d967a6b163b2312751072eaa3 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 27 Mar 2020 11:35:00 +1300 Subject: [PATCH 05/39] add MLJModelInterface as [deps] --- Project.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Project.toml b/Project.toml index 5296b7a..551ebaf 100644 --- a/Project.toml +++ b/Project.toml @@ -8,6 +8,7 @@ ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d" +MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" @@ -15,6 +16,7 @@ RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" ComputationalResources = "^0.3" Distributions = "^0.22" MLJBase = "^0.12.2" +MLJModelInterface = "^0.2" RecipesBase = "^0.8" julia = "^1" From 9706d20f2e093111b823db8305b3f12a05f9a648 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 27 Mar 2020 12:11:27 +1300 Subject: [PATCH 06/39] add isrecorded(model, history) convenience method --- src/MLJTuning.jl | 2 ++ src/strategies/explicit.jl | 6 ++++-- src/tuning_strategy_interface.jl | 29 +++++++++++++++++++++++++++++ test/tuned_models.jl | 15 +++++++++++++-- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/MLJTuning.jl b/src/MLJTuning.jl index 5a8fa81..bf6f676 100644 --- a/src/MLJTuning.jl +++ b/src/MLJTuning.jl @@ -17,6 +17,8 @@ export learning_curve!, learning_curve import MLJBase using MLJBase +# TODO: rm next import after is_same_except is imported into MLJBase +import MLJModelInterface import MLJBase: Bounded, Unbounded, DoublyUnbounded, LeftUnbounded, RightUnbounded using RecipesBase diff --git a/src/strategies/explicit.jl b/src/strategies/explicit.jl index 2cfe913..0a17319 100644 --- a/src/strategies/explicit.jl +++ b/src/strategies/explicit.jl @@ -34,8 +34,10 @@ function MLJTuning.models!(tuning::Explicit, while i < n_remaining next === nothing && break m, s = next - push!(models, m) - i += 1 + if !isrecorded(m, history) + push!(models, m) + i += 1 + end next = iterate(range, s) end diff --git a/src/tuning_strategy_interface.jl b/src/tuning_strategy_interface.jl index 9a93e0a..6a43fe6 100644 --- a/src/tuning_strategy_interface.jl +++ b/src/tuning_strategy_interface.jl @@ -1,6 +1,11 @@ +## TYPES TO BE SUBTYPED + abstract type TuningStrategy <: MLJBase.MLJType end MLJBase.show_as_constructed(::Type{<:TuningStrategy}) = true + +## METHODS TO BE IMPLEMENTED + # for initialization of state (compulsory) setup(tuning::TuningStrategy, model, range, verbosity) = range @@ -30,3 +35,27 @@ tuning_report(tuning::TuningStrategy, history, state) = (history=history,) # for declaring the default number of models to evaluate: default_n(tuning::TuningStrategy, range) = DEFAULT_N + + +## CONVENIENCE METHODS + +""" + MLJTuning.isrecorded(model, history) + MLJTuning.isrecorded(model, history, exceptions::Symbol...) + +Test if `history` has an entry for some model `m` sharing the same +hyperparameter values as `model`, with the possible exception of fields +specified in `exceptions`. + +More precisely, the requirement is that +`MLJModelInterface.is_same_except(m, model, exceptions...)` be true. + +""" +isrecorded(model::MLJBase.Model, ::Nothing) = false +function isrecorded(model::MLJBase.Model, history)::Bool + for (metamodel, _) in history + MLJModelInterface.is_same_except(_first(metamodel), model) && + return true + end + return false +end diff --git a/test/tuned_models.jl b/test/tuned_models.jl index ee5a68a..9b62e9c 100644 --- a/test/tuned_models.jl +++ b/test/tuned_models.jl @@ -38,6 +38,17 @@ r = [m(K) for K in 2:13] TunedModel(model=first(r), tuning=Explicit(), range=r)) end +@testset "duplicate models ignored" begin + s = [m(K) for K in 2:13] + push!(s, m(2)) + tm = TunedModel(model=first(r), tuning=Explicit(), + range=r, resampling=CV(nfolds=2), + measures=[rms, l1]) + fitresult, meta_state, report = fit(tm, 0, X, y); + history, _, state = meta_state; + @test length(history) == length(2:13) +end + results = [(evaluate(model, X, y, resampling=CV(nfolds=2), measure=rms, @@ -120,9 +131,9 @@ end) catch MethodError DEFAULT_N end - + end - + end @testset_accelerated("passing of model metadata", accel, From 0d46f89dd55e46f613999be762708828f6711994 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 27 Mar 2020 13:48:57 +1300 Subject: [PATCH 07/39] revert to having no isrecorded method --- src/strategies/explicit.jl | 6 ++---- src/tuning_strategy_interface.jl | 24 ------------------------ test/tuned_models.jl | 22 ++++++++++++---------- 3 files changed, 14 insertions(+), 38 deletions(-) diff --git a/src/strategies/explicit.jl b/src/strategies/explicit.jl index 0a17319..2cfe913 100644 --- a/src/strategies/explicit.jl +++ b/src/strategies/explicit.jl @@ -34,10 +34,8 @@ function MLJTuning.models!(tuning::Explicit, while i < n_remaining next === nothing && break m, s = next - if !isrecorded(m, history) - push!(models, m) - i += 1 - end + push!(models, m) + i += 1 next = iterate(range, s) end diff --git a/src/tuning_strategy_interface.jl b/src/tuning_strategy_interface.jl index 6a43fe6..d1827eb 100644 --- a/src/tuning_strategy_interface.jl +++ b/src/tuning_strategy_interface.jl @@ -35,27 +35,3 @@ tuning_report(tuning::TuningStrategy, history, state) = (history=history,) # for declaring the default number of models to evaluate: default_n(tuning::TuningStrategy, range) = DEFAULT_N - - -## CONVENIENCE METHODS - -""" - MLJTuning.isrecorded(model, history) - MLJTuning.isrecorded(model, history, exceptions::Symbol...) - -Test if `history` has an entry for some model `m` sharing the same -hyperparameter values as `model`, with the possible exception of fields -specified in `exceptions`. - -More precisely, the requirement is that -`MLJModelInterface.is_same_except(m, model, exceptions...)` be true. - -""" -isrecorded(model::MLJBase.Model, ::Nothing) = false -function isrecorded(model::MLJBase.Model, history)::Bool - for (metamodel, _) in history - MLJModelInterface.is_same_except(_first(metamodel), model) && - return true - end - return false -end diff --git a/test/tuned_models.jl b/test/tuned_models.jl index 9b62e9c..ea586ff 100644 --- a/test/tuned_models.jl +++ b/test/tuned_models.jl @@ -38,16 +38,18 @@ r = [m(K) for K in 2:13] TunedModel(model=first(r), tuning=Explicit(), range=r)) end -@testset "duplicate models ignored" begin - s = [m(K) for K in 2:13] - push!(s, m(2)) - tm = TunedModel(model=first(r), tuning=Explicit(), - range=r, resampling=CV(nfolds=2), - measures=[rms, l1]) - fitresult, meta_state, report = fit(tm, 0, X, y); - history, _, state = meta_state; - @test length(history) == length(2:13) -end +# @testset "duplicate models warning" begin +# s = [m(K) for K in 2:13] +# push!(s, m(13)) +# tm = TunedModel(model=first(s), tuning=Explicit(), +# range=s, resampling=CV(nfolds=2), +# measures=[rms, l1]) +# @test_logs((:info, r"Attempting"), +# (:warn, r"A model already"), +# fitresult, meta_state, report = fit(tm, 1, X, y)) +# history, _, state = meta_state; +# @test length(history) == length(2:13) + 1 +# end results = [(evaluate(model, X, y, resampling=CV(nfolds=2), From d787b2b70dd3820139b780d6e12eef5ab86e3ead Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Tue, 31 Mar 2020 18:24:05 +1300 Subject: [PATCH 08/39] add RandomSearch tests for constructor, setup and models --- src/MLJTuning.jl | 5 +- src/range_methods.jl | 6 +- src/strategies/random_search.jl | 45 ++++++++------- test/runtests.jl | 4 ++ test/strategies/random_search.jl | 96 ++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 27 deletions(-) create mode 100644 test/strategies/random_search.jl diff --git a/src/MLJTuning.jl b/src/MLJTuning.jl index bf6f676..f4c4f3e 100644 --- a/src/MLJTuning.jl +++ b/src/MLJTuning.jl @@ -7,7 +7,7 @@ module MLJTuning export TunedModel # defined in strategies/: -export Explicit, Grid +export Explicit, Grid, RandomSearch # defined in learning_curves.jl: export learning_curve!, learning_curve @@ -17,8 +17,6 @@ export learning_curve!, learning_curve import MLJBase using MLJBase -# TODO: rm next import after is_same_except is imported into MLJBase -import MLJModelInterface import MLJBase: Bounded, Unbounded, DoublyUnbounded, LeftUnbounded, RightUnbounded using RecipesBase @@ -42,6 +40,7 @@ include("tuned_models.jl") include("range_methods.jl") include("strategies/explicit.jl") include("strategies/grid.jl") +include("strategies/random_search.jl") include("plotrecipes.jl") include("learning_curves.jl") diff --git a/src/range_methods.jl b/src/range_methods.jl index 778ff65..d38586e 100644 --- a/src/range_methods.jl +++ b/src/range_methods.jl @@ -15,9 +15,6 @@ function boundedness(r::NumericRange{<:Any,<:RightUnbounded}) return Other end - -## PRE-PROCESSING OF USER-SPECIFIED CARTESIAN RANGE OBJECTS - """ MLJTuning.grid([rng, ] prototype, ranges, resolutions) @@ -54,6 +51,9 @@ function grid(prototype::Model, ranges, resolutions) end end + +## PRE-PROCESSING OF USER-SPECIFIED CARTESIAN RANGE OBJECTS + """ process_grid_range(user_specified_range, resolution, verbosity) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index a556ed1..e83a44e 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -3,7 +3,7 @@ const ParameterName=Union{Symbol,Expr} """ RandomSearch(bounded=Distributions.Uniform, positive_unbounded=Distributions.Gamma, - others=Distributions.Normal, + other=Distributions.Normal, rng=Random.GLOBAL_RNG) Instantiate a random search tuning strategy, for searching over @@ -16,11 +16,12 @@ dimenension. - a pair of the form `(r, d)`, with `r` as above and where `d` is a probability vector of the same length as `r.values`, if `r` is a - `NominalRange`, and is otherwise: (i) any `Distributions.Univariate` - *instance*; or (ii) one of the *subtypes* of - `Distributions.Univariate` listed in the table below, for automatic - fitting using `Distributions.fit(d, r)` (a distribution whose - support always lies between `r.lower` and `r.upper`.) + `NominalRange`, and is otherwise: (i) any + `Distributions.UnivariateDistribution` *instance*; or (ii) one of + the *subtypes* of `Distributions.UnivariateDistribution` listed in + the table below, for automatic fitting using `Distributions.fit(d, + r)` (a distribution whose support always lies between `r.lower` and + `r.upper`.) - any pair of the form `(field, s)`, where `field` is the (possibly nested) name of a field of the model to be tuned, and `s` an @@ -65,47 +66,49 @@ See also [`TunedModel`](@ref), [`range`](@ref), [`sampler`](@ref). """ mutable struct RandomSearch <: TuningStrategy - bounded::Distributions.Univariate - positive_unbounded::Distributions.Univariate - others::Distribution.Univariate + bounded + positive_unbounded + other rng::Random.AbstractRNG end # Constructor with keywords function RandomSearch(; bounded=Distributions.Uniform, positive_unbounded=Distributions.Gamma, - others=Distributions.Normal, + other=Distributions.Normal, rng=Random.GLOBAL_RNG) + (bounded isa Type{<:Distributions.UnivariateDistribution} && + positive_unbounded isa Type{<:Distributions.UnivariateDistribution} && + other isa Type{<:Distributions.UnivariateDistribution}) || + error("`bounded`, `positive_unbounded` and `other` "* + "must all be subtypes of "* + "`Distributions.UnivariateDistribution`. ") + _rng = rng isa Integer ? Random.MersenneTwister(rng) : rng - return RandomSearch(bounded, positive_unbounded, others, _rng) + return RandomSearch(bounded, positive_unbounded, other, _rng) end -isnumeric(::Any) = false -isnumeric(::NumericRange) = true - # `state`, which is not mutated, consists of a tuple of (field, sampler) # pairs: setup(tuning::RandomSearch, model, user_range, verbosity) = - process_user_random_range(user_range, + process_random_range(user_range, tuning.bounded, tuning.positive_unbounded, tuning.other) function MLJTuning.models!(tuning::RandomSearch, model, - history - state, + history, + state, # tuple of (field, sampler) pairs n_remaining, verbosity) - return map(1:n_remaining) do _ clone = deepcopy(model) - for (fld, s) field_sampler_pairs - recursive_setproperty!(clone, fld, rand(rng, s)) + for (fld, s) in state + recursive_setproperty!(clone, fld, rand(tuning.rng, s)) end clone end - end function tuning_report(tuning::RandomSearch, history, field_sampler_pairs) diff --git a/test/runtests.jl b/test/runtests.jl index e951237..f93150c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -28,6 +28,10 @@ end @test include("strategies/grid.jl") end +@testset "random search" begin + @test include("strategies/random_search.jl") +end + @testset "learning curves" begin @test include("learning_curves.jl") end diff --git a/test/strategies/random_search.jl b/test/strategies/random_search.jl new file mode 100644 index 0000000..b924da0 --- /dev/null +++ b/test/strategies/random_search.jl @@ -0,0 +1,96 @@ +module TestRandomSearch + +using Test +using MLJBase +using MLJTuning +import Distributions +import Random +import Random.seed! +seed!(1234) + +const Dist = Distributions + +x1 = rand(100); +x2 = rand(100); +x3 = rand(100) +X = (x1=x1, x2=x2, x3=x3); +y = 2*x1 .+ 5*x2 .- 3*x3 .+ 0.2*rand(100); + +mutable struct DummyModel <: Deterministic + lambda::Int + metric::Int + kernel::Char +end + +mutable struct SuperModel <: Deterministic + K::Int64 + model1::DummyModel + model2::DummyModel +end + +MLJBase.fit(::DummyModel, verbosity::Int, X, y) = std(y), nothing, nothing +MLJBase.predict(::DummyModel, fitresult, Xnew) = fitresult + +dummy_model = DummyModel(1, 9, 'k') +super_model = SuperModel(4, dummy_model, deepcopy(dummy_model)) + +r0 = range(super_model, :(model1.kernel), values=['c', 'd']) +r1 = range(super_model, :(model1.lambda), lower=1, upper=3) +r2 = range(super_model, :K, lower=0, upper=Inf, origin=2, unit=3) + +@testset "Constructor" begin + @test_throws Exception RandomSearch(bounded=Dist.Uniform(1,2)) + @test_throws Exception RandomSearch(positive_unbounded=Dist.Poisson(1)) + @test_throws Exception RandomSearch(bounded=Dist.Uniform(1,2)) +end + +@testset "setup" begin + user_range = [r0, (r1, Dist.SymTriangularDist), r2] + tuning = RandomSearch(positive_unbounded=Dist.Gamma, rng=123) + + @test MLJTuning.default_n(tuning, user_range) == MLJTuning.DEFAULT_N + + p0, p1, p2 = MLJTuning.setup(tuning, super_model, user_range, 3) + @test first.([p0, p1, p2]) == [:(model1.kernel), :(model1.lambda), :K] + + s0, s1, s2 = last.([p0, p1, p2]) + @test s0.distribution == Dist.Categorical(0.5, 0.5) + @test s1.distribution == Dist.SymTriangularDist(2,1) + γ = s2.distribution + @test mean(γ) == 2 + @test std(γ) == 3 +end + +@testset "models!" begin + N = 10000 + model = DummyModel(1, 1, 'k') + r1 = range(model, :lambda, lower=0, upper=1) + r2 = range(model, :metric, lower=-1, upper=1) + user_range = [r1, r2] + tuning = RandomSearch(rng=1) + tuned_model = TunedModel(model=model, + tuning=tuning, + n=N, + range=user_range, + measures=[rms,mae]) + state = MLJTuning.setup(tuning, model, user_range, 3) + my_models = MLJTuning.models!(tuning, + model, + nothing, # history + state, + N, # n_remaining + 0) + + # check the samples of each hyperparam have expected distritution: + lambdas = map(m -> m.lambda, my_models) + metrics = map(m -> m.metric, my_models) + a, b = values(Dist.countmap(lambdas)) + @test abs(a/b - 1) < 0.06 + dict = Dist.countmap(metrics) + a, b, c = dict[-1], dict[0], dict[1] + @test abs(b/a - 2) < 0.06 + @test abs(b/c - 2) < 0.06 +end + +end # module +true From 646147c808da72248cd66d11b455f80cc78b76f0 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 1 Apr 2020 17:24:04 +1300 Subject: [PATCH 09/39] add remaining random search tests --- src/tuned_models.jl | 8 ++++--- test/strategies/random_search.jl | 36 +++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/tuned_models.jl b/src/tuned_models.jl index 8372031..3fa7b51 100644 --- a/src/tuned_models.jl +++ b/src/tuned_models.jl @@ -217,16 +217,17 @@ function event(metamodel, model = _first(metamodel) metadata = _last(metamodel) resampling_machine.model.model = model - verb = (verbosity == 2 ? 0 : verbosity - 1) + verb = (verbosity >= 2 ? verbosity - 3 : verbosity - 1) fit!(resampling_machine, verbosity=verb) e = evaluate(resampling_machine) r = result(tuning, history, state, e, metadata) if verbosity > 2 - println(params(model)) + println("hyperparameters: $(params(model))") end + if verbosity > 1 - println("$r") + println("result: $r") end return model, r @@ -300,6 +301,7 @@ function build(history, history, state, acceleration) + history = _vcat(history, Δhistory) end return history diff --git a/test/strategies/random_search.jl b/test/strategies/random_search.jl index b924da0..aa10afd 100644 --- a/test/strategies/random_search.jl +++ b/test/strategies/random_search.jl @@ -28,8 +28,9 @@ mutable struct SuperModel <: Deterministic model2::DummyModel end -MLJBase.fit(::DummyModel, verbosity::Int, X, y) = std(y), nothing, nothing -MLJBase.predict(::DummyModel, fitresult, Xnew) = fitresult +MLJBase.fit(::DummyModel, verbosity::Int, X, y) = mean(y), nothing, nothing +MLJBase.predict(::DummyModel, fitresult, Xnew) = + fill(fitresult, schema(Xnew).nrows) dummy_model = DummyModel(1, 9, 'k') super_model = SuperModel(4, dummy_model, deepcopy(dummy_model)) @@ -57,7 +58,7 @@ end @test s0.distribution == Dist.Categorical(0.5, 0.5) @test s1.distribution == Dist.SymTriangularDist(2,1) γ = s2.distribution - @test mean(γ) == 2 + @test mean(γ) == 2 @test std(γ) == 3 end @@ -92,5 +93,34 @@ end @test abs(b/c - 2) < 0.06 end +@testset "tuned model using random search and its report" begin + N = 4 + model = DummyModel(1, 1, 'k') + r1 = range(model, :lambda, lower=0, upper=1) + r2 = range(model, :metric, lower=-1, upper=1) + user_range = [r1, r2] + tuning = RandomSearch(rng=1) + tuned_model = TunedModel(model=model, + tuning=tuning, + n=N, + resampling=Holdout(fraction_train=0.5), + range=user_range, + measures=[rms,mae]) + mach = machine(tuned_model, X, y) + fit!(mach, verbosity=0) + + # model predicts mean of training target, so: + train, test = partition(eachindex(y), 0.5) + μ = mean(y[train]) + error = mean((y[test] .- μ).^2) |> sqrt + + r = report(mach) + @test r.plotting.parameter_names == + ["lambda", "metric"] + @test r.plotting.parameter_scales == [:linear, :linear] + @test r.plotting.measurements ≈ fill(error, N) + @test size(r.plotting.parameter_values) == (N, 2) +end + end # module true From 33fa87e58bb631e1bde065457df705dee417a38d Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 1 Apr 2020 17:24:49 +1300 Subject: [PATCH 10/39] add RandomSearch to readme update readme update readme --- README.md | 10 ++++++---- test/strategies/random_search.jl | 12 ++++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7b21513..797fd3d 100644 --- a/README.md +++ b/README.md @@ -61,14 +61,14 @@ This repository contains: developers to conveniently implement common hyperparameter optimization strategies, such as: - - [x] search a list of explicitly specified models `list = [model1, - model2, ...]` + - [x] search models generated by an arbitrary iterator, eg `models = [model1, + model2, ...]` (built-in `Explicit` strategy) - - [x] grid search + - [x] grid search (built-in `Grid` strategy) - [ ] Latin hypercubes - - [ ] random search + - [x] random search (built-in `RandomSearch` strategy) - [ ] bandit @@ -232,6 +232,8 @@ In setting up a tuning task, the user constructs an instance of the ### Implementation requirements for new tuning strategies +As sample implemenations, see [/src/strategies/](/src/strategies) + #### Summary of functions Several functions are part of the tuning strategy API: diff --git a/test/strategies/random_search.jl b/test/strategies/random_search.jl index aa10afd..192e235 100644 --- a/test/strategies/random_search.jl +++ b/test/strategies/random_search.jl @@ -18,7 +18,7 @@ y = 2*x1 .+ 5*x2 .- 3*x3 .+ 0.2*rand(100); mutable struct DummyModel <: Deterministic lambda::Int - metric::Int + alpha::Int kernel::Char end @@ -66,7 +66,7 @@ end N = 10000 model = DummyModel(1, 1, 'k') r1 = range(model, :lambda, lower=0, upper=1) - r2 = range(model, :metric, lower=-1, upper=1) + r2 = range(model, :alpha, lower=-1, upper=1) user_range = [r1, r2] tuning = RandomSearch(rng=1) tuned_model = TunedModel(model=model, @@ -84,10 +84,10 @@ end # check the samples of each hyperparam have expected distritution: lambdas = map(m -> m.lambda, my_models) - metrics = map(m -> m.metric, my_models) + alphas = map(m -> m.alpha, my_models) a, b = values(Dist.countmap(lambdas)) @test abs(a/b - 1) < 0.06 - dict = Dist.countmap(metrics) + dict = Dist.countmap(alphas) a, b, c = dict[-1], dict[0], dict[1] @test abs(b/a - 2) < 0.06 @test abs(b/c - 2) < 0.06 @@ -97,7 +97,7 @@ end N = 4 model = DummyModel(1, 1, 'k') r1 = range(model, :lambda, lower=0, upper=1) - r2 = range(model, :metric, lower=-1, upper=1) + r2 = range(model, :alpha, lower=-1, upper=1) user_range = [r1, r2] tuning = RandomSearch(rng=1) tuned_model = TunedModel(model=model, @@ -116,7 +116,7 @@ end r = report(mach) @test r.plotting.parameter_names == - ["lambda", "metric"] + ["lambda", "alpha"] @test r.plotting.parameter_scales == [:linear, :linear] @test r.plotting.measurements ≈ fill(error, N) @test size(r.plotting.parameter_values) == (N, 2) From c797cea50bb084005c5590fe0ff82bbe645b45ab Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 1 Apr 2020 17:44:59 +1300 Subject: [PATCH 11/39] fix spelling in docstring --- src/strategies/random_search.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index e83a44e..a3f14a2 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -8,7 +8,7 @@ const ParameterName=Union{Symbol,Expr} Instantiate a random search tuning strategy, for searching over Cartesian hyperparameter domains, with customizable priors in each -dimenension. +dimension. ### Supported ranges: @@ -54,7 +54,7 @@ distribution types | for fitting to ranges of this type ### Algorithm -Models for evaulation are generated by sampling each range `r` using +Models for evaluation are generated by sampling each range `r` using `rand(rng, s)` where, `s = sampler(r, d)`. See `sampler` for details. If `d` is not specified, then sampling is uniform (with replacement) in the case of a `NominalRange`, and is otherwise given by the defaults From 8811817b59b84904a8b2573773e098b63c3603ed Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 1 Apr 2020 17:46:28 +1300 Subject: [PATCH 12/39] minor --- src/strategies/random_search.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index a3f14a2..be11435 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -10,7 +10,7 @@ Instantiate a random search tuning strategy, for searching over Cartesian hyperparameter domains, with customizable priors in each dimension. -### Supported ranges: +### Supported ranges - a single one-dimensional range (`ParamRange` object) `r` @@ -38,7 +38,7 @@ distribution types | for fitting to ranges of this type `ParamRange` objects are constructed using the `range` method. -### Examples: +### Examples range1 = range(model, :hyper1, lower=1, origin=2, unit=1) From ac6b5f5d7d29814881bc69adca547d429dd72e2f Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 1 Apr 2020 19:42:28 +1300 Subject: [PATCH 13/39] tweak doc-string range examples --- src/strategies/random_search.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index be11435..5783fb7 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -40,10 +40,10 @@ distribution types | for fitting to ranges of this type ### Examples - range1 = range(model, :hyper1, lower=1, origin=2, unit=1) + range1 = range(model, :hyper1, lower=0, upper=1) range2 = [(range(model, :hyper1, lower=1, upper=10), Arcsine), - range(model, :hyper2, lower=2, upper=4), + range(model, :hyper2, lower=2, upper=Inf, unit=1, origin=3), (range(model, :hyper2, lower=2, upper=4), Normal(0, 3)), range(model, :hyper3, values=[:ball, :tree], [0.3, 0.7])] From 5c55f19178652d393bdc222e0627630737e020e2 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 3 Apr 2020 08:20:40 +1300 Subject: [PATCH 14/39] update doc strings --- src/strategies/grid.jl | 5 ++++- src/strategies/random_search.jl | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/strategies/grid.jl b/src/strategies/grid.jl index 812ddd3..2afc6f8 100644 --- a/src/strategies/grid.jl +++ b/src/strategies/grid.jl @@ -9,7 +9,10 @@ default `resolution` in each numeric dimension. ### Supported ranges: -- A single one-dimensional range (`ParamRange` object) `r`, or pair of +A single one-dimensional range or vector of one-dimensioinal ranges +can be specified. Specically, a range can consist of: + +- A single one-dimensional range (ie, `ParamRange` object) `r`, or pair of the form `(r, res)` where `res` specifies a resolution to override the default `resolution`. diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index 5783fb7..59c05a2 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -12,6 +12,11 @@ dimension. ### Supported ranges +A single one-dimensional range or vector of one-dimensioinal ranges +can be specified. If not paired with a prior, then one is fitted, +according to fallback distribution types specified by the tuning +strategy hyperparameters. Specifically, a range can consist of: + - a single one-dimensional range (`ParamRange` object) `r` - a pair of the form `(r, d)`, with `r` as above and where `d` is a From b755b068ef23c4922e5670946ebe40c62cbca829 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 3 Apr 2020 09:53:43 +1300 Subject: [PATCH 15/39] enable multiple ranges for same field; WIP: tests needed --- src/strategies/random_search.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index 59c05a2..fc200bf 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -109,6 +109,7 @@ function MLJTuning.models!(tuning::RandomSearch, verbosity) return map(1:n_remaining) do _ clone = deepcopy(model) + Random.shuffle!(tuning.rng, state |> collect) for (fld, s) in state recursive_setproperty!(clone, fld, rand(tuning.rng, s)) end From 36d646f004238f117b0f6fe88c65dcb8c04d5dfc Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 3 Apr 2020 11:23:55 +1300 Subject: [PATCH 16/39] add tests for multiple samplers per field --- src/strategies/random_search.jl | 16 +++++++++------- test/strategies/random_search.jl | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index fc200bf..cb1a3ad 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -45,16 +45,18 @@ distribution types | for fitting to ranges of this type ### Examples + using Distributions + range1 = range(model, :hyper1, lower=0, upper=1) range2 = [(range(model, :hyper1, lower=1, upper=10), Arcsine), - range(model, :hyper2, lower=2, upper=Inf, unit=1, origin=3), + range(model, :hyper2, lower=2, upper=Inf, unit=1, origin=3), (range(model, :hyper2, lower=2, upper=4), Normal(0, 3)), - range(model, :hyper3, values=[:ball, :tree], [0.3, 0.7])] + (range(model, :hyper3, values=[:ball, :tree]), [0.3, 0.7])] # uniform sampling of :(atom.λ) from [0, 1] without defining a NumericRange: struct MySampler end - Base.rand(rng::AbstractRNG, ::MySampler) = rand(rng) + Base.rand(rng::Random.AbstractRNG, ::MySampler) = rand(rng) range3 = (:(atom.λ), MySampler(), range1) ### Algorithm @@ -93,13 +95,13 @@ function RandomSearch(; bounded=Distributions.Uniform, return RandomSearch(bounded, positive_unbounded, other, _rng) end -# `state`, which is not mutated, consists of a tuple of (field, sampler) -# pairs: +# `state` consists of a tuple of (field, sampler) pairs (that gets +# shuffled each iteration): setup(tuning::RandomSearch, model, user_range, verbosity) = process_random_range(user_range, tuning.bounded, tuning.positive_unbounded, - tuning.other) + tuning.other) |> collect function MLJTuning.models!(tuning::RandomSearch, model, @@ -109,7 +111,7 @@ function MLJTuning.models!(tuning::RandomSearch, verbosity) return map(1:n_remaining) do _ clone = deepcopy(model) - Random.shuffle!(tuning.rng, state |> collect) + Random.shuffle!(tuning.rng, state) for (fld, s) in state recursive_setproperty!(clone, fld, rand(tuning.rng, s)) end diff --git a/test/strategies/random_search.jl b/test/strategies/random_search.jl index 192e235..55501ad 100644 --- a/test/strategies/random_search.jl +++ b/test/strategies/random_search.jl @@ -122,5 +122,31 @@ end @test size(r.plotting.parameter_values) == (N, 2) end +struct ConstantSampler + c +end +Base.rand(rng::Random.AbstractRNG, s::ConstantSampler) = s.c + +@testset "multiple samplers for single field" begin + N = 1000 + model = DummyModel(1, 1, 'k') + r = range(model, :alpha, lower=-1, upper=1) + user_range = [(:lambda, ConstantSampler(0)), + r, + (:lambda, ConstantSampler(1))] + tuning = RandomSearch(rng=123) + tuned_model = TunedModel(model=model, + tuning=tuning, + n=N, + range=user_range, + measures=[rms,mae]) + mach = fit!(machine(tuned_model, X, y)) + my_models = first.(report(mach).history); + lambdas = map(m -> m.lambda, my_models); + a, b = values(Dist.countmap(lambdas)) + @test abs(a/b -1) < 0.04 + @test a + b == N +end + end # module true From 1a959bacdef0092741da04c7f84b7d2d8e3cd7a4 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 3 Apr 2020 11:42:44 +1300 Subject: [PATCH 17/39] update doc-string --- src/strategies/random_search.jl | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index cb1a3ad..35dc921 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -35,6 +35,11 @@ strategy hyperparameters. Specifically, a range can consist of: - any vector of objects of the above form +A range vector may contain multiple entries for the same model field, +as in `range = [(:lambda, s1), (:alpha, s), (:lambda, s2)]`. In that +case the entry used in each iteration is random. See more under +[Algorithm](@ref). + distribution types | for fitting to ranges of this type --------------------|----------------------------------- `Arcsine`, `Uniform`, `Biweight`, `Cosine`, `Epanechnikov`, `SymTriangularDist`, `Triweight` | bounded @@ -61,13 +66,17 @@ distribution types | for fitting to ranges of this type ### Algorithm -Models for evaluation are generated by sampling each range `r` using -`rand(rng, s)` where, `s = sampler(r, d)`. See `sampler` for details. If `d` -is not specified, then sampling is uniform (with replacement) in the -case of a `NominalRange`, and is otherwise given by the defaults -specified by the tuning strategy parameters `bounded`, -`positive_unbounded`, and `other`, depending on the `NumericRange` -type. +In each iteration, a model is generated for evaluation by mutating the +fields of a deep copy of `model`. The range vector is shuffled and the +fields sampled according to the new order (repeated fields being +mutated more than once). For a `range` entry of the form `(field, s)` +the algorithm calls `rand(rng, s)` and mutates the field `field` of +the model clone to have this value. For an entry of the form `(r, d)`, +`s` is substituted with `sampler(r, d)`. If no `d` is specified, then +sampling is uniform (with replacement) if `r` is a `NominalRange`, and +is otherwise given by the defaults specified by the tuning strategy +parameters `bounded`, `positive_unbounded`, and `other`, depending on +the field values of the `NumericRange` object `r`. See also [`TunedModel`](@ref), [`range`](@ref), [`sampler`](@ref). From 85291d3ba818e49841ac3d4d6b083ff8bf5273de Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 18 Mar 2020 12:40:52 +1300 Subject: [PATCH 18/39] add new range methods for random search WIP --- Project.toml | 2 + src/MLJTuning.jl | 5 +- src/range_methods.jl | 146 +++++++++++++++++++++++++ src/ranges.jl | 70 ------------ src/strategies/grid.jl | 4 +- src/strategies/random_search.jl | 132 +++++++++++++++++++++++ test/range_methods.jl | 182 ++++++++++++++++++++++++++++++++ test/ranges.jl | 95 ----------------- test/runtests.jl | 4 +- 9 files changed, 470 insertions(+), 170 deletions(-) create mode 100644 src/range_methods.jl delete mode 100644 src/ranges.jl create mode 100644 src/strategies/random_search.jl create mode 100644 test/range_methods.jl delete mode 100644 test/ranges.jl diff --git a/Project.toml b/Project.toml index 5e90565..b75f22f 100644 --- a/Project.toml +++ b/Project.toml @@ -6,12 +6,14 @@ version = "0.3.0" [deps] ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" +Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" [compat] ComputationalResources = "^0.3" +Distributions = "^0.22" MLJBase = "^0.12" RecipesBase = "^0.8" julia = "^1" diff --git a/src/MLJTuning.jl b/src/MLJTuning.jl index c7aad28..1c75a98 100644 --- a/src/MLJTuning.jl +++ b/src/MLJTuning.jl @@ -17,8 +17,11 @@ export learning_curve!, learning_curve import MLJBase using MLJBase +import MLJBase: Bounded, Unbounded, DoublyUnbounded, + LeftUnbounded, RightUnbounded using RecipesBase using Distributed +import Distributions import ComputationalResources: CPU1, CPUProcesses, CPUThreads, AbstractResource using Random @@ -34,7 +37,7 @@ const DEFAULT_N = 10 include("utilities.jl") include("tuning_strategy_interface.jl") include("tuned_models.jl") -include("ranges.jl") +include("range_methods.jl") include("strategies/explicit.jl") include("strategies/grid.jl") include("plotrecipes.jl") diff --git a/src/range_methods.jl b/src/range_methods.jl new file mode 100644 index 0000000..92772bf --- /dev/null +++ b/src/range_methods.jl @@ -0,0 +1,146 @@ +## BOUNDEDNESS TRAIT + +# For random search and perhaps elsewhere, we need a variation on the +# built-in boundedness notions: +abstract type PositiveUnbounded <: Unbounded end +abstract type Other <: Unbounded end + +boundedness(::NumericRange{<:Any,<:Bounded}) = Bounded +boundedness(::NumericRange{<:Any,<:LeftUnbounded}) = Other +boundedness(::NumericRange{<:Any,<:DoublyUnbounded}) = Other +function boundedness(r::NumericRange{<:Any,<:RightUnbounded}) + if r.lower >= 0 + return PositiveUnbounded + end + return Other +end + + +## PRE-PROCESSING OF USER-SPECIFIED CARTESIAN RANGE OBJECTS + +""" + MLJTuning.grid([rng, ] prototype, ranges, resolutions) + +Given an iterable `ranges` of `ParamRange` objects, and an iterable +`resolutions` of the same length, return a vector of models generated +by cloning and mutating the hyperparameters (fields) of `prototype`, +according to the Cartesian grid defined by the specifed +one-dimensional `ranges` (`ParamRange` objects) and specified +`resolutions`. A resolution of `nothing` for a `NominalRange` +indicates that all values should be used. + +Specification of an `AbstractRNG` object `rng` implies shuffling of +the results. Otherwise models are ordered, with the first +hyperparameter referenced cycling fastest. + +""" +grid(rng::AbstractRNG, prototype::Model, ranges, resolutions) = + shuffle(rng, grid(prototype, ranges, resolutions)) + +function grid(prototype::Model, ranges, resolutions) + + iterators = broadcast(iterator, ranges, resolutions) + + A = MLJBase.unwind(iterators...) + + N = size(A, 1) + map(1:N) do i + clone = deepcopy(prototype) + for k in eachindex(ranges) + field = ranges[k].field + recursive_setproperty!(clone, field, A[i,k]) + end + clone + end +end + +""" + process_grid_range(user_specified_range, resolution, verbosity) + +Utility to convert a user-specified range (see [`Grid`](@ref)) into a +pair of tuples `(ranges, resolutions)`. + +For example, if `r1`, `r2` are `NumericRange`s and `s` is a +NominalRange` with 5 values, then we have: + + julia> MLJTuning.process_grid_range([(r1, 3), r2, s], 42, 1) == + ((r1, r2, s), (3, 42, 5)) + true + +If `verbosity` > 0, then a warning is issued if a `Nominal` range is +paired with a resolution. + +""" +process_grid_range(user_specified_range, args...) = + throw(ArgumentError("Unsupported range. ")) + +process_grid_range(usr::Union{ParamRange,Tuple{ParamRange,Int}}, args...) = + process_grid_range([usr, ], args...) + +function process_grid_range(user_specified_range::AbstractVector, + resolution, verbosity) + # r unpaired: + stand(r) = throw(ArgumentError("Unsupported range. ")) + stand(r::NumericRange) = (r, resolution) + stand(r::NominalRange) = (r, length(r.values)) + + # (r, res): + stand(t::Tuple{NumericRange,Integer}) = t + function stand(t::Tuple{NominalRange,Integer}) + verbosity < 0 || + @warn "Ignoring a resolution specified for a `NominalRange`. " + return (first(t), length(first(t).values)) + end + + ret = zip(stand.(user_specified_range)...) |> collect + return first(ret), last(ret) +end + +""" + process_random_range(user_specified_range, + bounded, + positive_unbounded, + other) + +Utility to convert a user-specified range (see [`RandomSearch`](@ref)) +into an n-tuple of `(field, sampler)` pairs. + +""" +process_random_range(user_specified_range, args...) = + throw(ArgumentError("Unsupported range. ")) + +const DIST = Distributions.Distribution +const DD = Union{DIST, Type{<:DIST}} +const AllowedPairs = Union{Tuple{NumericRange,DD}, + Tuple{NominalRange,AbstractVector{<:AbstractFloat}}} + +process_random_range(user_specified_range::Union{ParamRange, AllowedPairs}, + args...) = + process_random_range([user_specified_range, ], args...) + +function process_random_range(user_specified_range::AbstractVector, + bounded, + positive_unbounded, + other) + @show 1 + + # r not paired: + stand(r) = throw(ArgumentError("Unsupported range. ")) + stand(r::NumericRange) = stand(r, boundedness(r)) + stand(r::NumericRange, ::Type{<:Bounded}) = (r.field, sampler(r, bounded)) + stand(r::NumericRange, ::Type{<:Other}) = (r.field, sampler(r, other)) + stand(r::NumericRange, ::Type{<:PositiveUnbounded}) = + (r.field, sampler(r, positive_unbounded)) + stand(r::NominalRange) = (n = length(r.values); + (r.field, sampler(r, fill(1/n, n)))) + # (r, d): + stand(t::AllowedPairs) = (r = first(t); (r.field, sampler(r, last(t)))) + + # (field, s): + stand(t::Tuple{Union{Symbol,Expr},Any}) = t + + return Tuple(stand.(user_specified_range)) + + # ret = zip(stand.(user_specified_range)...) |> collect + # return first(ret), last(ret) +end diff --git a/src/ranges.jl b/src/ranges.jl deleted file mode 100644 index 37e375d..0000000 --- a/src/ranges.jl +++ /dev/null @@ -1,70 +0,0 @@ -""" - MLJTuning.grid([rng, ] prototype, ranges, resolutions) - -Given an iterable `ranges` of `ParamRange` objects, and an iterable -`resolutions` of the same length, return a vector of models generated -by cloning and mutating the hyperparameters (fields) of `prototype`, -according to the Cartesian grid defined by the specifed -one-dimensional `ranges` (`ParamRange` objects) and specified -`resolutions`. A resolution of `nothing` for a `NominalRange` -indicates that all values should be used. - -Specification of an `AbstractRNG` object `rng` implies shuffling of -the results. Otherwise models are ordered, with the first -hyperparameter referenced cycling fastest. - -""" -grid(rng::AbstractRNG, prototype::Model, ranges, resolutions) = - shuffle(rng, grid(prototype, ranges, resolutions)) - -function grid(prototype::Model, ranges, resolutions) - - iterators = broadcast(iterator, ranges, resolutions) - - A = MLJBase.unwind(iterators...) - - N = size(A, 1) - map(1:N) do i - clone = deepcopy(prototype) - for k in eachindex(ranges) - field = ranges[k].field - recursive_setproperty!(clone, field, A[i,k]) - end - clone - end -end - -""" - process_user_range(user_specified_range, resolution, verbosity) - -Utility to convert user-specified range (see [`Grid`](@ref)) into a -pair of tuples `(ranges, resolutions)`. - -For example, if `r1`, `r2` are `NumericRange`s and `s` is a -NominalRange` with 5 values, then we have: - - julia> MLJTuning.process_user_range([(r1, 3), r2, s], 42, 1) == - ((r1, r2, s), (3, 42, 5)) - true - -If `verbosity` > 0, then a warning is issued if a `Nominal` range is -paired with a resolution. - -""" -process_user_range(user_specified_range, resolution, verbosity) = - process_user_range([user_specified_range, ], resolution, verbosity) -function process_user_range(user_specified_range::AbstractVector, - resolution, verbosity) - stand(r) = throw(ArgumentError("Unsupported range. ")) - stand(r::NumericRange) = (r, resolution) - stand(r::NominalRange) = (r, length(r.values)) - stand(t::Tuple{NumericRange,Integer}) = t - function stand(t::Tuple{NominalRange,Integer}) - verbosity < 0 || - @warn "Ignoring a resolution specified for a `NominalRange`. " - return (first(t), length(first(t).values)) - end - - ret = zip(stand.(user_specified_range)...) |> collect - return first(ret), last(ret) -end diff --git a/src/strategies/grid.jl b/src/strategies/grid.jl index 5c31266..812ddd3 100644 --- a/src/strategies/grid.jl +++ b/src/strategies/grid.jl @@ -83,7 +83,7 @@ end function setup(tuning::Grid, model, user_range, verbosity) ranges, resolutions = - process_user_range(user_range, tuning.resolution, verbosity) + process_grid_range(user_range, tuning.resolution, verbosity) resolutions = adjusted_resolutions(tuning.goal, ranges, resolutions) fields = map(r -> r.field, ranges) @@ -123,7 +123,7 @@ end function default_n(tuning::Grid, user_range) ranges, resolutions = - process_user_range(user_range, tuning.resolution, -1) + process_grid_range(user_range, tuning.resolution, -1) resolutions = adjusted_resolutions(tuning.goal, ranges, resolutions) len(t::Tuple{NumericRange,Integer}) = length(iterator(t[1], t[2])) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl new file mode 100644 index 0000000..c64b095 --- /dev/null +++ b/src/strategies/random_search.jl @@ -0,0 +1,132 @@ +const ParameterName=Union{Symbol,Expr} + +""" + RandomSearch(bounded=Distributions.Uniform, + positive_unbounded=Distributions.Gamma, + others=Normal, + rng=Random.GLOBAL_RNG) + +Instantiate a random search tuning strategy for searching over +Cartesian hyperparemeter domains. + +### Supported ranges: + +- A single one-dimensional range (`ParamRange` object) `r` + +- A pair of the form `(r, d)`, with `r` as above and where `d` is a + probability vector of the same length as `r.values`, if `r` is a + `NominalRange`, and is otherwise: (i) any `Distributions.Univariate` + *instance*; or (ii) one of the *subtypes* of + `Distributions.Univariate` listed in the table below, for automatic + fitting using `Distributions.fit(d, r)` (a distribution whose + support always lies between `r.lower` and `r.upper`. + +- Any pair of the form `(field, s)`, where `field` is the, possibly + nested, name of a field the model to be tuned, and `s` an arbitrary + sampler object for that field. (This only means `rand(rng, s)` is defined and + returns valid values for the field.) + +- Any vector of objects of the above form + +distribution types | for fitting to ranges of this type +--------------------|----------------------------------- +`Arcsine`, `Uniform`, `Biweight`, `Cosine`, `Epanechnikov`, `SymTriangularDist`, `Triweight` | bounded +`Gamma`, `InverseGaussian`, `Poisson` | positive +`Normal`, `Logistic`, `LogNormal`, `Cauchy`, `Gumbel`, `Laplace` | any + +`ParamRange` objects are constructed using the `range` method. + +### Examples: + + range1 = range(model, :hyper1, lower=1, origin=2, unit=1) + + range2 = [(range(model, :hyper1, lower=1, upper=10), Arcsine), + range(model, :hyper2, lower=2, upper=4), + (range(model, :hyper2, lower=2, upper=4), Normal(0, 3)), + range(model, :hyper3, values=[:ball, :tree], [0.3, 0.7])] + + # uniform sampling of :(atom.λ) from [0, 1] without defining a NumericRange: + struct MySampler end + Base.rand(rng::AbstractRNG, ::MySampler) = rand(rng) + range3 = (:(atom.λ), MySampler()) + +### Algorithm + +Models for evaulation are generated by sampling each range `r` using +`rand(rng, s)` where, `s = sampler(r, d)`. See `sampler` for details. If `d` +is not specified, then sampling is uniform (with replacement) in the +case of a `NominalRange`, and is otherwise given by the defaults +specified by the tuning strategy parameters `bounded`, +`positive_unbounded`, and `other`, depending on the `NumericRange` +type. + +See also [`TunedModel`](@ref), [`range`](@ref), [`sampler`](@ref). + +""" +mutable struct RandomSearch <: TuningStrategy + bounded::Distributions.Univariate + positive_unbounded::Distributions.Univariate + others::Distribution.Univariate + rng::Random.AbstractRNG +end + +# Constructor with keywords +function RandomSearch(; bounded=Distributions.Uniform, + positive_unbounded=Distributions.Gamma, + others=Normal, + rng=Random.GLOBAL_RNG) + _rng = rng isa Integer ? Random.MersenneTwister(rng) : rng + return RandomSearch(bounded, positive_unbounded, others, rng) +end + +isnumeric(::Any) = false +isnumeric(::NumericRange) = true + +# function setup(tuning::RandomSearch, model, user_range, verbosity) +# ranges, distributions = # I AM HERE +# process_user_random_range(user_range, tuning.distribution, verbosity) +# distributions = adjusted_distributions(tuning.goal, ranges, distributions) + +# fields = map(r -> r.field, ranges) + +# parameter_scales = scale.(ranges) + +# if tuning.shuffle +# models = grid(tuning.rng, model, ranges, distributions) +# else +# models = grid(model, ranges, distributions) +# end + +# state = (models=models, +# fields=fields, +# parameter_scales=parameter_scales) + +# return state + +# end + +# MLJTuning.models!(tuning::RandomSearch, model, history::Nothing, +# state, verbosity) = state.models +# MLJTuning.models!(tuning::RandomSearch, model, history, +# state, verbosity) = +# state.models[length(history) + 1:end] + +# function tuning_report(tuning::RandomSearch, history, state) + +# plotting = plotting_report(state.fields, state.parameter_scales, history) + +# # todo: remove collects? +# return (history=history, plotting=plotting) + +# end + +# function default_n(tuning::RandomSearch, user_range) +# ranges, distributions = +# process_grid_range(user_range, tuning.distribution, -1) + +# distributions = adjusted_distributions(tuning.goal, ranges, distributions) +# len(t::Tuple{NumericRange,Integer}) = length(iterator(t[1], t[2])) +# len(t::Tuple{NominalRange,Integer}) = t[2] +# return prod(len.(zip(ranges, distributions))) + +# end diff --git a/test/range_methods.jl b/test/range_methods.jl new file mode 100644 index 0000000..c1ee037 --- /dev/null +++ b/test/range_methods.jl @@ -0,0 +1,182 @@ +module TestRanges + +using Test +using MLJBase +using MLJTuning +using Random +import Distributions +const Dist = Distributions + +# `in` for MLJType is overloaded to be `===` based. For purposed of +# testing here, we need `==` based: +function _in(x, itr)::Union{Bool,Missing} + for y in itr + ismissing(y) && return missing + y == x && return true + end + return false +end +_issubset(itr1, itr2) = all(_in(x, itr2) for x in itr1) + +@testset "boundedness traits" begin + r1 = range(Float64, :K, lower=1, upper=10) + r2 = range(Float64, :K, lower=-1, upper=Inf, origin=1, unit=1) + r3 = range(Float64, :K, lower=0, upper=Inf, origin=1, unit=1) + r4 = range(Float64, :K, lower=-Inf, upper=1, origin=0, unit=1) + r5 = range(Float64, :K, lower=-Inf, upper=Inf, origin=1, unit=1) + @test MLJTuning.boundedness(r1) == MLJTuning.Bounded + @test MLJTuning.boundedness(r2) == MLJTuning.Other + @test MLJTuning.boundedness(r3) == MLJTuning.PositiveUnbounded + @test MLJTuning.boundedness(r4) == MLJTuning.Other + @test MLJTuning.boundedness(r5) == MLJTuning.Other +end + +mutable struct DummyModel <: Deterministic + lambda::Float64 + metric::Float64 + kernel::Char +end + +dummy_model = DummyModel(4, 9.5, 'k') + +mutable struct SuperModel <: Deterministic + K::Int64 + model1::DummyModel + model2::DummyModel +end + +dummy_model = DummyModel(1.2, 9.5, 'k') +super_model = SuperModel(4, dummy_model, deepcopy(dummy_model)) + +r1 = range(super_model, :(model1.kernel), values=['c', 'd']) +r2 = range(super_model, :K, lower=1, upper=10, scale=:log10) + +@testset "models from cartesian range and resolutions" begin + + # with method: + m1 = MLJTuning.grid(super_model, [r1, r2], [nothing, 7]) + m1r = MLJTuning.grid(MersenneTwister(123), super_model, [r1, r2], + [nothing, 7]) + + # generate all models by hand: + models1 = [SuperModel(1, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(1, DummyModel(1.2, 9.5, 'd'), dummy_model), + SuperModel(2, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(2, DummyModel(1.2, 9.5, 'd'), dummy_model), + SuperModel(3, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(3, DummyModel(1.2, 9.5, 'd'), dummy_model), + SuperModel(5, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(5, DummyModel(1.2, 9.5, 'd'), dummy_model), + SuperModel(7, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(7, DummyModel(1.2, 9.5, 'd'), dummy_model), + SuperModel(10, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(10, DummyModel(1.2, 9.5, 'd'), dummy_model)] + + @test _issubset(models1, m1) && _issubset(m1, models1) + @test m1r != models1 + @test _issubset(models1, m1r) && _issubset(m1, models1) + + # with method: + m2 = MLJTuning.grid(super_model, [r1, r2], [1, 7]) + + # generate all models by hand: + models2 = [SuperModel(1, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(2, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(3, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(5, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(7, DummyModel(1.2, 9.5, 'c'), dummy_model), + SuperModel(10, DummyModel(1.2, 9.5, 'c'), dummy_model)] + + @test _issubset(models2, m2) && _issubset(m2, models2) + +end + +@testset "processing user specification of range in Grid" begin + r1 = range(Int, :h1, lower=1, upper=10) + r2 = range(Int, :h2, lower=20, upper=30) + s = range(Char, :j1, values = ['x', 'y']) + @test_throws ArgumentError MLJTuning.process_grid_range("junk", 42, 1) + @test(@test_logs((:warn, r"Ignoring"), + MLJTuning.process_grid_range((s, 3), 42, 1)) == + ((s, ), (2, ))) + @test MLJTuning.process_grid_range(r1, 42, 1) == ((r1, ), (42, )) + @test MLJTuning.process_grid_range((r1, 3), 42, 1) == ((r1, ), (3, )) + @test MLJTuning.process_grid_range(s, 42, 1) == ((s, ), (2,)) + @test MLJTuning.process_grid_range([(r1, 3), r2, s], 42, 1) == + ((r1, r2, s), (3, 42, 2)) +end + +struct MySampler end +Base.rand(rng::AbstractRNG, ::MySampler) = rand(rng) + +@testset "processing user specification of range in RandomSearch" begin + r1 = range(Int, :h1, lower=1, upper=10, scale=exp) + r2 = range(Int, :h2, lower=5, upper=Inf, origin=10, unit=5) + r3 = range(Char, :j1, values = ['x', 'y']) + s = MySampler() + + @test_throws(ArgumentError, + MLJTuning.process_random_range("junk", + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy)) + @test_throws(ArgumentError, + MLJTuning.process_random_range((r1, "junk"), + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy)) + @test_throws(ArgumentError, + MLJTuning.process_random_range((r3, "junk"), + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy)) + @test_throws(ArgumentError, + MLJTuning.process_random_range(("junk", s), + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy)) + + # unpaired numeric range: + pp = MLJTuning.process_random_range(r1, + Dist.Uniform, # bounded + Dist.Gamma, # positive_unbounded + Dist.Cauchy) # other + @test pp isa Tuple{Tuple{Symbol,MLJBase.NumericSampler}} + p = first(pp) + @test first(p) == :h1 + s = last(p) + @test s.scale == r1.scale + @test s.distribution == Dist.Uniform(1.0, 10.0) + + # unpaired nominal range: + p = MLJTuning.process_random_range(r3, + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy) |> first + @test first(p) == :j1 + s = last(p) + @test s.values == r3.values + @test s.distribution.p == [0.5, 0.5] + @test s.distribution.support == 1:2 + + # (numeric range, distribution instance): + p = MLJTuning.process_random_range((r2, Dist.Poisson(3)), + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy) |> first + @test first(p) == :h2 + s = last(p) + @test s.scale == r2.scale + @test s.distribution == Dist.truncated(Dist.Poisson(3.0), 5.0, Inf) + + # (numeric range, distribution type): + p = MLJTuning.process_random_range((r2, Dist.Poisson), + Dist.Uniform, + Dist.Gamma, + Dist.Cauchy) |> first + s = last(p) + @test s.distribution == Dist.truncated(Poisson(r2.origin), 5.0, Inf) + + +end +true diff --git a/test/ranges.jl b/test/ranges.jl deleted file mode 100644 index 17b3af1..0000000 --- a/test/ranges.jl +++ /dev/null @@ -1,95 +0,0 @@ -module TestRanges - -using Test -using MLJBase -using MLJTuning -using Random - -# `in` for MLJType is overloaded to be `===` based. For purposed of -# testing here, we need `==` based: -function _in(x, itr)::Union{Bool,Missing} - for y in itr - ismissing(y) && return missing - y == x && return true - end - return false -end -_issubset(itr1, itr2) = all(_in(x, itr2) for x in itr1) - -mutable struct DummyModel <: Deterministic - lambda::Float64 - metric::Float64 - kernel::Char -end - -dummy_model = DummyModel(4, 9.5, 'k') - -mutable struct SuperModel <: Deterministic - K::Int64 - model1::DummyModel - model2::DummyModel -end - -dummy_model = DummyModel(1.2, 9.5, 'k') -super_model = SuperModel(4, dummy_model, deepcopy(dummy_model)) - -r1 = range(super_model, :(model1.kernel), values=['c', 'd']) -r2 = range(super_model, :K, lower=1, upper=10, scale=:log10) - -@testset "models from cartesian range and resolutions" begin - - # with method: - m1 = MLJTuning.grid(super_model, [r1, r2], [nothing, 7]) - m1r = MLJTuning.grid(MersenneTwister(123), super_model, [r1, r2], - [nothing, 7]) - - # generate all models by hand: - models1 = [SuperModel(1, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(1, DummyModel(1.2, 9.5, 'd'), dummy_model), - SuperModel(2, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(2, DummyModel(1.2, 9.5, 'd'), dummy_model), - SuperModel(3, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(3, DummyModel(1.2, 9.5, 'd'), dummy_model), - SuperModel(5, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(5, DummyModel(1.2, 9.5, 'd'), dummy_model), - SuperModel(7, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(7, DummyModel(1.2, 9.5, 'd'), dummy_model), - SuperModel(10, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(10, DummyModel(1.2, 9.5, 'd'), dummy_model)] - - @test _issubset(models1, m1) && _issubset(m1, models1) - @test m1r != models1 - @test _issubset(models1, m1r) && _issubset(m1, models1) - - # with method: - m2 = MLJTuning.grid(super_model, [r1, r2], [1, 7]) - - # generate all models by hand: - models2 = [SuperModel(1, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(2, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(3, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(5, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(7, DummyModel(1.2, 9.5, 'c'), dummy_model), - SuperModel(10, DummyModel(1.2, 9.5, 'c'), dummy_model)] - - @test _issubset(models2, m2) && _issubset(m2, models2) - -end - -@testset "processing user specification of range" begin - r1 = range(Int, :h1, lower=1, upper=10) - r2 = range(Int, :h2, lower=20, upper=30) - s = range(Char, :j1, values = ['x', 'y']) - @test_throws ArgumentError MLJTuning.process_user_range("junk", 42, 1) - @test(@test_logs((:warn, r"Ignoring"), - MLJTuning.process_user_range((s, 3), 42, 1)) == - ((s, ), (2, ))) - @test MLJTuning.process_user_range(r1, 42, 1) == ((r1, ), (42, )) - @test MLJTuning.process_user_range((r1, 3), 42, 1) == ((r1, ), (3, )) - @test MLJTuning.process_user_range(s, 42, 1) == ((s, ), (2,)) - @test MLJTuning.process_user_range([(r1, 3), r2, s], 42, 1) == - ((r1, r2, s), (3, 42, 2)) -end - -end -true diff --git a/test/runtests.jl b/test/runtests.jl index 4764491..e951237 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -20,8 +20,8 @@ end @test include("tuned_models.jl") end -@testset "ranges" begin - @test include("ranges.jl") +@testset "range_methods" begin + @test include("range_methods.jl") end @testset "grid" begin From c39b2f8f7c92f1d6358bfc53c9ddf19dcc0561c2 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 18 Mar 2020 18:33:57 +1300 Subject: [PATCH 19/39] update to [compat] MLJBase = "0.12.2" to enable scale(s) extension --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index b75f22f..bfcb6eb 100644 --- a/Project.toml +++ b/Project.toml @@ -14,7 +14,7 @@ RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" [compat] ComputationalResources = "^0.3" Distributions = "^0.22" -MLJBase = "^0.12" +MLJBase = "^0.12.2" RecipesBase = "^0.8" julia = "^1" From e8e79792541f297ed5e7c544cfd49d310e6ff474 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 26 Mar 2020 13:20:07 +1300 Subject: [PATCH 20/39] resolve some stash pop conflicts --- README.md | 13 ++++ src/MLJTuning.jl | 2 +- src/range_methods.jl | 22 +++--- src/strategies/random_search.jl | 111 +++++++++++++++---------------- src/tuning_strategy_interface.jl | 2 - test/range_methods.jl | 3 +- 6 files changed, 81 insertions(+), 72 deletions(-) diff --git a/README.md b/README.md index 8f5f7e7..7b21513 100644 --- a/README.md +++ b/README.md @@ -373,6 +373,11 @@ is `fit!` the first time, and not on subsequent calls (unless `force=true`). (Specifically, `MLJBase.fit(::TunedModel, ...)` calls `setup` but `MLJBase.update(::TunedModel, ...)` does not.) +The `setup` function is called once only, when a `TunedModel` machine +is `fit!` the first time, and not on subsequent calls (unless +`force=true`). (Specifically, `MLJBase.fit(::TunedModel, ...)` calls +`setup` but `MLJBase.update(::TunedModel, ...)` does not.) + The `verbosity` is an integer indicating the level of logging: `0` means logging should be restricted to warnings, `-1`, means completely silent. @@ -440,6 +445,14 @@ any number of models. If `models!` returns a number of models exceeding the number needed to complete the history, the list returned is simply truncated. +Some simple tuning strategies, such as `RandomSearch`, will want to +return as many models as possible in one hit. The argument +`n_remaining` is the difference between the current length of the +history and the target number of iterations `tuned_model.n` set by the +user when constructing his `TunedModel` instance, `tuned_model` (or +`default_n(tuning, range)` if left unspecified). + + #### The `best` method: To define what constitutes the "optimal model" ```julia diff --git a/src/MLJTuning.jl b/src/MLJTuning.jl index 1c75a98..68bccf9 100644 --- a/src/MLJTuning.jl +++ b/src/MLJTuning.jl @@ -29,7 +29,7 @@ using Random ## CONSTANTS -const DEFAULT_N = 10 +const DEFAULT_N == 10 # for when `default_n` is not implemented ## INCLUDE FILES diff --git a/src/range_methods.jl b/src/range_methods.jl index 92772bf..be130a8 100644 --- a/src/range_methods.jl +++ b/src/range_methods.jl @@ -1,3 +1,8 @@ +## SCALE FOR SAMPLERS + +TODO: + + ## BOUNDEDNESS TRAIT # For random search and perhaps elsewhere, we need a variation on the @@ -107,14 +112,11 @@ into an n-tuple of `(field, sampler)` pairs. """ process_random_range(user_specified_range, args...) = - throw(ArgumentError("Unsupported range. ")) + throw(ArgumentError("Unsupported range #1. ")) const DIST = Distributions.Distribution -const DD = Union{DIST, Type{<:DIST}} -const AllowedPairs = Union{Tuple{NumericRange,DD}, - Tuple{NominalRange,AbstractVector{<:AbstractFloat}}} -process_random_range(user_specified_range::Union{ParamRange, AllowedPairs}, +process_random_range(user_specified_range::Union{ParamRange, Tuple{Any,Any}}, args...) = process_random_range([user_specified_range, ], args...) @@ -122,10 +124,8 @@ function process_random_range(user_specified_range::AbstractVector, bounded, positive_unbounded, other) - @show 1 - # r not paired: - stand(r) = throw(ArgumentError("Unsupported range. ")) + stand(r) = throw(ArgumentError("Unsupported range #2. ")) stand(r::NumericRange) = stand(r, boundedness(r)) stand(r::NumericRange, ::Type{<:Bounded}) = (r.field, sampler(r, bounded)) stand(r::NumericRange, ::Type{<:Other}) = (r.field, sampler(r, other)) @@ -134,7 +134,11 @@ function process_random_range(user_specified_range::AbstractVector, stand(r::NominalRange) = (n = length(r.values); (r.field, sampler(r, fill(1/n, n)))) # (r, d): - stand(t::AllowedPairs) = (r = first(t); (r.field, sampler(r, last(t)))) + stand(t::Tuple{ParamRange,Any}) = stand(t...) + stand(r, d) = throw(ArgumentError("Unsupported range #3. ")) + stand(r::NominalRange, d::AbstractVector{Float64}) = _stand(r, d) + stand(r::NumericRange, d:: Union{DIST, Type{<:DIST}}) = _stand(r, d) + _stand(r, d) = (r.field, sampler(r, d)) # (field, s): stand(t::Tuple{Union{Symbol,Expr},Any}) = t diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index c64b095..6b295c6 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -3,35 +3,36 @@ const ParameterName=Union{Symbol,Expr} """ RandomSearch(bounded=Distributions.Uniform, positive_unbounded=Distributions.Gamma, - others=Normal, + others=Distributions.Normal, rng=Random.GLOBAL_RNG) -Instantiate a random search tuning strategy for searching over -Cartesian hyperparemeter domains. +Instantiate a random search tuning strategy, for searching over +Cartesian hyperparameter domains, with customizable priors in each +dimenension. ### Supported ranges: -- A single one-dimensional range (`ParamRange` object) `r` +- a single one-dimensional range (`ParamRange` object) `r` -- A pair of the form `(r, d)`, with `r` as above and where `d` is a +- a pair of the form `(r, d)`, with `r` as above and where `d` is a probability vector of the same length as `r.values`, if `r` is a `NominalRange`, and is otherwise: (i) any `Distributions.Univariate` *instance*; or (ii) one of the *subtypes* of `Distributions.Univariate` listed in the table below, for automatic fitting using `Distributions.fit(d, r)` (a distribution whose - support always lies between `r.lower` and `r.upper`. + support always lies between `r.lower` and `r.upper`.) -- Any pair of the form `(field, s)`, where `field` is the, possibly - nested, name of a field the model to be tuned, and `s` an arbitrary - sampler object for that field. (This only means `rand(rng, s)` is defined and - returns valid values for the field.) +- any pair of the form `(field, s)`, where `field` is the (possibly + nested) name of a field of the model to be tuned, and `s` an + arbitrary sampler object for that field. This means only that + `rand(rng, s)` is defined and returns valid values for the field. -- Any vector of objects of the above form +- any vector of objects of the above form distribution types | for fitting to ranges of this type --------------------|----------------------------------- `Arcsine`, `Uniform`, `Biweight`, `Cosine`, `Epanechnikov`, `SymTriangularDist`, `Triweight` | bounded -`Gamma`, `InverseGaussian`, `Poisson` | positive +`Gamma`, `InverseGaussian`, `Poisson` | positive (bounded or unbounded) `Normal`, `Logistic`, `LogNormal`, `Cauchy`, `Gumbel`, `Laplace` | any `ParamRange` objects are constructed using the `range` method. @@ -48,7 +49,7 @@ distribution types | for fitting to ranges of this type # uniform sampling of :(atom.λ) from [0, 1] without defining a NumericRange: struct MySampler end Base.rand(rng::AbstractRNG, ::MySampler) = rand(rng) - range3 = (:(atom.λ), MySampler()) + range3 = (:(atom.λ), MySampler(), range1) ### Algorithm @@ -63,7 +64,7 @@ type. See also [`TunedModel`](@ref), [`range`](@ref), [`sampler`](@ref). """ -mutable struct RandomSearch <: TuningStrategy +mutable struct RandomSearch <: TuningStrategy bounded::Distributions.Univariate positive_unbounded::Distributions.Univariate others::Distribution.Univariate @@ -73,60 +74,54 @@ end # Constructor with keywords function RandomSearch(; bounded=Distributions.Uniform, positive_unbounded=Distributions.Gamma, - others=Normal, + others=Distributions.Normal, rng=Random.GLOBAL_RNG) _rng = rng isa Integer ? Random.MersenneTwister(rng) : rng - return RandomSearch(bounded, positive_unbounded, others, rng) + return RandomSearch(bounded, positive_unbounded, others, _rng) end isnumeric(::Any) = false isnumeric(::NumericRange) = true -# function setup(tuning::RandomSearch, model, user_range, verbosity) -# ranges, distributions = # I AM HERE -# process_user_random_range(user_range, tuning.distribution, verbosity) -# distributions = adjusted_distributions(tuning.goal, ranges, distributions) +# `state`, which is not mutated, consists of a tuple of (field, sampler) +# pairs: +setup(tuning::RandomSearch, model, user_range, verbosity) = + process_user_random_range(user_range, + tuning.bounded, + tuning.positive_unbounded, + tuning.other) + +function MLJTuning.models!(tuning::RandomSearch, + model, + history + state, + verbosity) + + # We generate all remaining models at once. Since the value of + # `tuning.n` can change between calls to `models!` + n_so_far = _length(history) # _length(nothing) = 0 + n = tuning.n === nothing ? DEFAULT_N : tuning.n + n_models = max(0, n - n_so_far) + + return map(1:n_models) do _ + clone = deepcopy(model) + for (fld, s) field_sampler_pairs + recursive_setproperty!(clone, fld, rand(rng, s)) + end + clone + end -# fields = map(r -> r.field, ranges) - -# parameter_scales = scale.(ranges) - -# if tuning.shuffle -# models = grid(tuning.rng, model, ranges, distributions) -# else -# models = grid(model, ranges, distributions) -# end - -# state = (models=models, -# fields=fields, -# parameter_scales=parameter_scales) - -# return state - -# end - -# MLJTuning.models!(tuning::RandomSearch, model, history::Nothing, -# state, verbosity) = state.models -# MLJTuning.models!(tuning::RandomSearch, model, history, -# state, verbosity) = -# state.models[length(history) + 1:end] - -# function tuning_report(tuning::RandomSearch, history, state) - -# plotting = plotting_report(state.fields, state.parameter_scales, history) +end -# # todo: remove collects? -# return (history=history, plotting=plotting) +function tuning_report(tuning::RandomSearch, history, field_sampler_pairs) -# end + fields = first.(field_sampler_pairs) + parameter_scales = map(field_sampler_pairs) do (fld, s) + scale(s) + end -# function default_n(tuning::RandomSearch, user_range) -# ranges, distributions = -# process_grid_range(user_range, tuning.distribution, -1) + plotting = plotting_report(fields, parameter_scales, history) -# distributions = adjusted_distributions(tuning.goal, ranges, distributions) -# len(t::Tuple{NumericRange,Integer}) = length(iterator(t[1], t[2])) -# len(t::Tuple{NominalRange,Integer}) = t[2] -# return prod(len.(zip(ranges, distributions))) + return (history=history, plotting=plotting) -# end +end diff --git a/src/tuning_strategy_interface.jl b/src/tuning_strategy_interface.jl index 4c58b18..9a93e0a 100644 --- a/src/tuning_strategy_interface.jl +++ b/src/tuning_strategy_interface.jl @@ -30,5 +30,3 @@ tuning_report(tuning::TuningStrategy, history, state) = (history=history,) # for declaring the default number of models to evaluate: default_n(tuning::TuningStrategy, range) = DEFAULT_N - - diff --git a/test/range_methods.jl b/test/range_methods.jl index c1ee037..46f8832 100644 --- a/test/range_methods.jl +++ b/test/range_methods.jl @@ -175,8 +175,7 @@ Base.rand(rng::AbstractRNG, ::MySampler) = rand(rng) Dist.Gamma, Dist.Cauchy) |> first s = last(p) - @test s.distribution == Dist.truncated(Poisson(r2.origin), 5.0, Inf) - + @test s.distribution == Dist.truncated(Dist.Poisson(r2.unit), 5.0, Inf) end true From e68ab7c5899e6208a5d27e8111d927cacfa63056 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Thu, 26 Mar 2020 17:06:02 +1300 Subject: [PATCH 21/39] fix some wrong code due to flawed rebase --- src/MLJTuning.jl | 2 +- src/range_methods.jl | 5 ----- src/strategies/random_search.jl | 9 ++------- test/range_methods.jl | 2 ++ test/tuned_models.jl | 2 +- 5 files changed, 6 insertions(+), 14 deletions(-) diff --git a/src/MLJTuning.jl b/src/MLJTuning.jl index 68bccf9..5a8fa81 100644 --- a/src/MLJTuning.jl +++ b/src/MLJTuning.jl @@ -29,7 +29,7 @@ using Random ## CONSTANTS -const DEFAULT_N == 10 # for when `default_n` is not implemented +const DEFAULT_N = 10 # for when `default_n` is not implemented ## INCLUDE FILES diff --git a/src/range_methods.jl b/src/range_methods.jl index be130a8..778ff65 100644 --- a/src/range_methods.jl +++ b/src/range_methods.jl @@ -1,8 +1,3 @@ -## SCALE FOR SAMPLERS - -TODO: - - ## BOUNDEDNESS TRAIT # For random search and perhaps elsewhere, we need a variation on the diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index 6b295c6..a556ed1 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -95,15 +95,10 @@ function MLJTuning.models!(tuning::RandomSearch, model, history state, + n_remaining, verbosity) - # We generate all remaining models at once. Since the value of - # `tuning.n` can change between calls to `models!` - n_so_far = _length(history) # _length(nothing) = 0 - n = tuning.n === nothing ? DEFAULT_N : tuning.n - n_models = max(0, n - n_so_far) - - return map(1:n_models) do _ + return map(1:n_remaining) do _ clone = deepcopy(model) for (fld, s) field_sampler_pairs recursive_setproperty!(clone, fld, rand(rng, s)) diff --git a/test/range_methods.jl b/test/range_methods.jl index 46f8832..cd1c1be 100644 --- a/test/range_methods.jl +++ b/test/range_methods.jl @@ -177,5 +177,7 @@ Base.rand(rng::AbstractRNG, ::MySampler) = rand(rng) s = last(p) @test s.distribution == Dist.truncated(Dist.Poisson(r2.unit), 5.0, Inf) +end + end true diff --git a/test/tuned_models.jl b/test/tuned_models.jl index 18e4340..ee5a68a 100644 --- a/test/tuned_models.jl +++ b/test/tuned_models.jl @@ -23,7 +23,7 @@ y = 2*x1 .+ 5*x2 .- 3*x3 .+ 0.4*rand(N); m(K) = KNNRegressor(K=K) r = [m(K) for K in 2:13] -# TODO: replace the above with the line below and fix post an issue on +# TODO: replace the above with the line below and post an issue on # the failure (a bug in Distributed, I reckon): # r = (m(K) for K in 2:13) From e03518253adf7d5331af22ebd299d12fd66b944b Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 27 Mar 2020 11:35:00 +1300 Subject: [PATCH 22/39] add MLJModelInterface as [deps] --- Project.toml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/Project.toml b/Project.toml index bfcb6eb..7dd9390 100644 --- a/Project.toml +++ b/Project.toml @@ -8,6 +8,7 @@ ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3" Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" Distributions = "31c24e10-a181-5473-b8eb-7969acd0382f" MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d" +MLJModelInterface = "e80e1ace-859a-464e-9ed9-23947d8ae3ea" Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" @@ -15,6 +16,7 @@ RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" ComputationalResources = "^0.3" Distributions = "^0.22" MLJBase = "^0.12.2" +MLJModelInterface = "^0.2" RecipesBase = "^0.8" julia = "^1" From f1936bc68a3d144b019099bd94790b85d10f5d5a Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 27 Mar 2020 12:11:27 +1300 Subject: [PATCH 23/39] add isrecorded(model, history) convenience method --- src/MLJTuning.jl | 2 ++ src/strategies/explicit.jl | 6 ++++-- src/tuning_strategy_interface.jl | 29 +++++++++++++++++++++++++++++ test/tuned_models.jl | 15 +++++++++++++-- 4 files changed, 48 insertions(+), 4 deletions(-) diff --git a/src/MLJTuning.jl b/src/MLJTuning.jl index 5a8fa81..bf6f676 100644 --- a/src/MLJTuning.jl +++ b/src/MLJTuning.jl @@ -17,6 +17,8 @@ export learning_curve!, learning_curve import MLJBase using MLJBase +# TODO: rm next import after is_same_except is imported into MLJBase +import MLJModelInterface import MLJBase: Bounded, Unbounded, DoublyUnbounded, LeftUnbounded, RightUnbounded using RecipesBase diff --git a/src/strategies/explicit.jl b/src/strategies/explicit.jl index 2cfe913..0a17319 100644 --- a/src/strategies/explicit.jl +++ b/src/strategies/explicit.jl @@ -34,8 +34,10 @@ function MLJTuning.models!(tuning::Explicit, while i < n_remaining next === nothing && break m, s = next - push!(models, m) - i += 1 + if !isrecorded(m, history) + push!(models, m) + i += 1 + end next = iterate(range, s) end diff --git a/src/tuning_strategy_interface.jl b/src/tuning_strategy_interface.jl index 9a93e0a..6a43fe6 100644 --- a/src/tuning_strategy_interface.jl +++ b/src/tuning_strategy_interface.jl @@ -1,6 +1,11 @@ +## TYPES TO BE SUBTYPED + abstract type TuningStrategy <: MLJBase.MLJType end MLJBase.show_as_constructed(::Type{<:TuningStrategy}) = true + +## METHODS TO BE IMPLEMENTED + # for initialization of state (compulsory) setup(tuning::TuningStrategy, model, range, verbosity) = range @@ -30,3 +35,27 @@ tuning_report(tuning::TuningStrategy, history, state) = (history=history,) # for declaring the default number of models to evaluate: default_n(tuning::TuningStrategy, range) = DEFAULT_N + + +## CONVENIENCE METHODS + +""" + MLJTuning.isrecorded(model, history) + MLJTuning.isrecorded(model, history, exceptions::Symbol...) + +Test if `history` has an entry for some model `m` sharing the same +hyperparameter values as `model`, with the possible exception of fields +specified in `exceptions`. + +More precisely, the requirement is that +`MLJModelInterface.is_same_except(m, model, exceptions...)` be true. + +""" +isrecorded(model::MLJBase.Model, ::Nothing) = false +function isrecorded(model::MLJBase.Model, history)::Bool + for (metamodel, _) in history + MLJModelInterface.is_same_except(_first(metamodel), model) && + return true + end + return false +end diff --git a/test/tuned_models.jl b/test/tuned_models.jl index ee5a68a..9b62e9c 100644 --- a/test/tuned_models.jl +++ b/test/tuned_models.jl @@ -38,6 +38,17 @@ r = [m(K) for K in 2:13] TunedModel(model=first(r), tuning=Explicit(), range=r)) end +@testset "duplicate models ignored" begin + s = [m(K) for K in 2:13] + push!(s, m(2)) + tm = TunedModel(model=first(r), tuning=Explicit(), + range=r, resampling=CV(nfolds=2), + measures=[rms, l1]) + fitresult, meta_state, report = fit(tm, 0, X, y); + history, _, state = meta_state; + @test length(history) == length(2:13) +end + results = [(evaluate(model, X, y, resampling=CV(nfolds=2), measure=rms, @@ -120,9 +131,9 @@ end) catch MethodError DEFAULT_N end - + end - + end @testset_accelerated("passing of model metadata", accel, From 093d43e4483824cad1ebc77a9b54b91e9cfdf4fd Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 27 Mar 2020 13:48:57 +1300 Subject: [PATCH 24/39] revert to having no isrecorded method --- src/strategies/explicit.jl | 6 ++---- src/tuning_strategy_interface.jl | 24 ------------------------ test/tuned_models.jl | 22 ++++++++++++---------- 3 files changed, 14 insertions(+), 38 deletions(-) diff --git a/src/strategies/explicit.jl b/src/strategies/explicit.jl index 0a17319..2cfe913 100644 --- a/src/strategies/explicit.jl +++ b/src/strategies/explicit.jl @@ -34,10 +34,8 @@ function MLJTuning.models!(tuning::Explicit, while i < n_remaining next === nothing && break m, s = next - if !isrecorded(m, history) - push!(models, m) - i += 1 - end + push!(models, m) + i += 1 next = iterate(range, s) end diff --git a/src/tuning_strategy_interface.jl b/src/tuning_strategy_interface.jl index 6a43fe6..d1827eb 100644 --- a/src/tuning_strategy_interface.jl +++ b/src/tuning_strategy_interface.jl @@ -35,27 +35,3 @@ tuning_report(tuning::TuningStrategy, history, state) = (history=history,) # for declaring the default number of models to evaluate: default_n(tuning::TuningStrategy, range) = DEFAULT_N - - -## CONVENIENCE METHODS - -""" - MLJTuning.isrecorded(model, history) - MLJTuning.isrecorded(model, history, exceptions::Symbol...) - -Test if `history` has an entry for some model `m` sharing the same -hyperparameter values as `model`, with the possible exception of fields -specified in `exceptions`. - -More precisely, the requirement is that -`MLJModelInterface.is_same_except(m, model, exceptions...)` be true. - -""" -isrecorded(model::MLJBase.Model, ::Nothing) = false -function isrecorded(model::MLJBase.Model, history)::Bool - for (metamodel, _) in history - MLJModelInterface.is_same_except(_first(metamodel), model) && - return true - end - return false -end diff --git a/test/tuned_models.jl b/test/tuned_models.jl index 9b62e9c..ea586ff 100644 --- a/test/tuned_models.jl +++ b/test/tuned_models.jl @@ -38,16 +38,18 @@ r = [m(K) for K in 2:13] TunedModel(model=first(r), tuning=Explicit(), range=r)) end -@testset "duplicate models ignored" begin - s = [m(K) for K in 2:13] - push!(s, m(2)) - tm = TunedModel(model=first(r), tuning=Explicit(), - range=r, resampling=CV(nfolds=2), - measures=[rms, l1]) - fitresult, meta_state, report = fit(tm, 0, X, y); - history, _, state = meta_state; - @test length(history) == length(2:13) -end +# @testset "duplicate models warning" begin +# s = [m(K) for K in 2:13] +# push!(s, m(13)) +# tm = TunedModel(model=first(s), tuning=Explicit(), +# range=s, resampling=CV(nfolds=2), +# measures=[rms, l1]) +# @test_logs((:info, r"Attempting"), +# (:warn, r"A model already"), +# fitresult, meta_state, report = fit(tm, 1, X, y)) +# history, _, state = meta_state; +# @test length(history) == length(2:13) + 1 +# end results = [(evaluate(model, X, y, resampling=CV(nfolds=2), From e059a688faaf3caf45de76e036c6530241583e97 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Tue, 31 Mar 2020 18:24:05 +1300 Subject: [PATCH 25/39] add RandomSearch tests for constructor, setup and models --- src/MLJTuning.jl | 5 +- src/range_methods.jl | 6 +- src/strategies/random_search.jl | 45 ++++++++------- test/runtests.jl | 4 ++ test/strategies/random_search.jl | 96 ++++++++++++++++++++++++++++++++ 5 files changed, 129 insertions(+), 27 deletions(-) create mode 100644 test/strategies/random_search.jl diff --git a/src/MLJTuning.jl b/src/MLJTuning.jl index bf6f676..f4c4f3e 100644 --- a/src/MLJTuning.jl +++ b/src/MLJTuning.jl @@ -7,7 +7,7 @@ module MLJTuning export TunedModel # defined in strategies/: -export Explicit, Grid +export Explicit, Grid, RandomSearch # defined in learning_curves.jl: export learning_curve!, learning_curve @@ -17,8 +17,6 @@ export learning_curve!, learning_curve import MLJBase using MLJBase -# TODO: rm next import after is_same_except is imported into MLJBase -import MLJModelInterface import MLJBase: Bounded, Unbounded, DoublyUnbounded, LeftUnbounded, RightUnbounded using RecipesBase @@ -42,6 +40,7 @@ include("tuned_models.jl") include("range_methods.jl") include("strategies/explicit.jl") include("strategies/grid.jl") +include("strategies/random_search.jl") include("plotrecipes.jl") include("learning_curves.jl") diff --git a/src/range_methods.jl b/src/range_methods.jl index 778ff65..d38586e 100644 --- a/src/range_methods.jl +++ b/src/range_methods.jl @@ -15,9 +15,6 @@ function boundedness(r::NumericRange{<:Any,<:RightUnbounded}) return Other end - -## PRE-PROCESSING OF USER-SPECIFIED CARTESIAN RANGE OBJECTS - """ MLJTuning.grid([rng, ] prototype, ranges, resolutions) @@ -54,6 +51,9 @@ function grid(prototype::Model, ranges, resolutions) end end + +## PRE-PROCESSING OF USER-SPECIFIED CARTESIAN RANGE OBJECTS + """ process_grid_range(user_specified_range, resolution, verbosity) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index a556ed1..e83a44e 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -3,7 +3,7 @@ const ParameterName=Union{Symbol,Expr} """ RandomSearch(bounded=Distributions.Uniform, positive_unbounded=Distributions.Gamma, - others=Distributions.Normal, + other=Distributions.Normal, rng=Random.GLOBAL_RNG) Instantiate a random search tuning strategy, for searching over @@ -16,11 +16,12 @@ dimenension. - a pair of the form `(r, d)`, with `r` as above and where `d` is a probability vector of the same length as `r.values`, if `r` is a - `NominalRange`, and is otherwise: (i) any `Distributions.Univariate` - *instance*; or (ii) one of the *subtypes* of - `Distributions.Univariate` listed in the table below, for automatic - fitting using `Distributions.fit(d, r)` (a distribution whose - support always lies between `r.lower` and `r.upper`.) + `NominalRange`, and is otherwise: (i) any + `Distributions.UnivariateDistribution` *instance*; or (ii) one of + the *subtypes* of `Distributions.UnivariateDistribution` listed in + the table below, for automatic fitting using `Distributions.fit(d, + r)` (a distribution whose support always lies between `r.lower` and + `r.upper`.) - any pair of the form `(field, s)`, where `field` is the (possibly nested) name of a field of the model to be tuned, and `s` an @@ -65,47 +66,49 @@ See also [`TunedModel`](@ref), [`range`](@ref), [`sampler`](@ref). """ mutable struct RandomSearch <: TuningStrategy - bounded::Distributions.Univariate - positive_unbounded::Distributions.Univariate - others::Distribution.Univariate + bounded + positive_unbounded + other rng::Random.AbstractRNG end # Constructor with keywords function RandomSearch(; bounded=Distributions.Uniform, positive_unbounded=Distributions.Gamma, - others=Distributions.Normal, + other=Distributions.Normal, rng=Random.GLOBAL_RNG) + (bounded isa Type{<:Distributions.UnivariateDistribution} && + positive_unbounded isa Type{<:Distributions.UnivariateDistribution} && + other isa Type{<:Distributions.UnivariateDistribution}) || + error("`bounded`, `positive_unbounded` and `other` "* + "must all be subtypes of "* + "`Distributions.UnivariateDistribution`. ") + _rng = rng isa Integer ? Random.MersenneTwister(rng) : rng - return RandomSearch(bounded, positive_unbounded, others, _rng) + return RandomSearch(bounded, positive_unbounded, other, _rng) end -isnumeric(::Any) = false -isnumeric(::NumericRange) = true - # `state`, which is not mutated, consists of a tuple of (field, sampler) # pairs: setup(tuning::RandomSearch, model, user_range, verbosity) = - process_user_random_range(user_range, + process_random_range(user_range, tuning.bounded, tuning.positive_unbounded, tuning.other) function MLJTuning.models!(tuning::RandomSearch, model, - history - state, + history, + state, # tuple of (field, sampler) pairs n_remaining, verbosity) - return map(1:n_remaining) do _ clone = deepcopy(model) - for (fld, s) field_sampler_pairs - recursive_setproperty!(clone, fld, rand(rng, s)) + for (fld, s) in state + recursive_setproperty!(clone, fld, rand(tuning.rng, s)) end clone end - end function tuning_report(tuning::RandomSearch, history, field_sampler_pairs) diff --git a/test/runtests.jl b/test/runtests.jl index e951237..f93150c 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -28,6 +28,10 @@ end @test include("strategies/grid.jl") end +@testset "random search" begin + @test include("strategies/random_search.jl") +end + @testset "learning curves" begin @test include("learning_curves.jl") end diff --git a/test/strategies/random_search.jl b/test/strategies/random_search.jl new file mode 100644 index 0000000..b924da0 --- /dev/null +++ b/test/strategies/random_search.jl @@ -0,0 +1,96 @@ +module TestRandomSearch + +using Test +using MLJBase +using MLJTuning +import Distributions +import Random +import Random.seed! +seed!(1234) + +const Dist = Distributions + +x1 = rand(100); +x2 = rand(100); +x3 = rand(100) +X = (x1=x1, x2=x2, x3=x3); +y = 2*x1 .+ 5*x2 .- 3*x3 .+ 0.2*rand(100); + +mutable struct DummyModel <: Deterministic + lambda::Int + metric::Int + kernel::Char +end + +mutable struct SuperModel <: Deterministic + K::Int64 + model1::DummyModel + model2::DummyModel +end + +MLJBase.fit(::DummyModel, verbosity::Int, X, y) = std(y), nothing, nothing +MLJBase.predict(::DummyModel, fitresult, Xnew) = fitresult + +dummy_model = DummyModel(1, 9, 'k') +super_model = SuperModel(4, dummy_model, deepcopy(dummy_model)) + +r0 = range(super_model, :(model1.kernel), values=['c', 'd']) +r1 = range(super_model, :(model1.lambda), lower=1, upper=3) +r2 = range(super_model, :K, lower=0, upper=Inf, origin=2, unit=3) + +@testset "Constructor" begin + @test_throws Exception RandomSearch(bounded=Dist.Uniform(1,2)) + @test_throws Exception RandomSearch(positive_unbounded=Dist.Poisson(1)) + @test_throws Exception RandomSearch(bounded=Dist.Uniform(1,2)) +end + +@testset "setup" begin + user_range = [r0, (r1, Dist.SymTriangularDist), r2] + tuning = RandomSearch(positive_unbounded=Dist.Gamma, rng=123) + + @test MLJTuning.default_n(tuning, user_range) == MLJTuning.DEFAULT_N + + p0, p1, p2 = MLJTuning.setup(tuning, super_model, user_range, 3) + @test first.([p0, p1, p2]) == [:(model1.kernel), :(model1.lambda), :K] + + s0, s1, s2 = last.([p0, p1, p2]) + @test s0.distribution == Dist.Categorical(0.5, 0.5) + @test s1.distribution == Dist.SymTriangularDist(2,1) + γ = s2.distribution + @test mean(γ) == 2 + @test std(γ) == 3 +end + +@testset "models!" begin + N = 10000 + model = DummyModel(1, 1, 'k') + r1 = range(model, :lambda, lower=0, upper=1) + r2 = range(model, :metric, lower=-1, upper=1) + user_range = [r1, r2] + tuning = RandomSearch(rng=1) + tuned_model = TunedModel(model=model, + tuning=tuning, + n=N, + range=user_range, + measures=[rms,mae]) + state = MLJTuning.setup(tuning, model, user_range, 3) + my_models = MLJTuning.models!(tuning, + model, + nothing, # history + state, + N, # n_remaining + 0) + + # check the samples of each hyperparam have expected distritution: + lambdas = map(m -> m.lambda, my_models) + metrics = map(m -> m.metric, my_models) + a, b = values(Dist.countmap(lambdas)) + @test abs(a/b - 1) < 0.06 + dict = Dist.countmap(metrics) + a, b, c = dict[-1], dict[0], dict[1] + @test abs(b/a - 2) < 0.06 + @test abs(b/c - 2) < 0.06 +end + +end # module +true From 42473bb0c89bf47f3bbabf598d5ccdffd21b5421 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 1 Apr 2020 17:24:04 +1300 Subject: [PATCH 26/39] add remaining random search tests --- src/tuned_models.jl | 8 ++++--- test/strategies/random_search.jl | 36 +++++++++++++++++++++++++++++--- 2 files changed, 38 insertions(+), 6 deletions(-) diff --git a/src/tuned_models.jl b/src/tuned_models.jl index 8372031..3fa7b51 100644 --- a/src/tuned_models.jl +++ b/src/tuned_models.jl @@ -217,16 +217,17 @@ function event(metamodel, model = _first(metamodel) metadata = _last(metamodel) resampling_machine.model.model = model - verb = (verbosity == 2 ? 0 : verbosity - 1) + verb = (verbosity >= 2 ? verbosity - 3 : verbosity - 1) fit!(resampling_machine, verbosity=verb) e = evaluate(resampling_machine) r = result(tuning, history, state, e, metadata) if verbosity > 2 - println(params(model)) + println("hyperparameters: $(params(model))") end + if verbosity > 1 - println("$r") + println("result: $r") end return model, r @@ -300,6 +301,7 @@ function build(history, history, state, acceleration) + history = _vcat(history, Δhistory) end return history diff --git a/test/strategies/random_search.jl b/test/strategies/random_search.jl index b924da0..aa10afd 100644 --- a/test/strategies/random_search.jl +++ b/test/strategies/random_search.jl @@ -28,8 +28,9 @@ mutable struct SuperModel <: Deterministic model2::DummyModel end -MLJBase.fit(::DummyModel, verbosity::Int, X, y) = std(y), nothing, nothing -MLJBase.predict(::DummyModel, fitresult, Xnew) = fitresult +MLJBase.fit(::DummyModel, verbosity::Int, X, y) = mean(y), nothing, nothing +MLJBase.predict(::DummyModel, fitresult, Xnew) = + fill(fitresult, schema(Xnew).nrows) dummy_model = DummyModel(1, 9, 'k') super_model = SuperModel(4, dummy_model, deepcopy(dummy_model)) @@ -57,7 +58,7 @@ end @test s0.distribution == Dist.Categorical(0.5, 0.5) @test s1.distribution == Dist.SymTriangularDist(2,1) γ = s2.distribution - @test mean(γ) == 2 + @test mean(γ) == 2 @test std(γ) == 3 end @@ -92,5 +93,34 @@ end @test abs(b/c - 2) < 0.06 end +@testset "tuned model using random search and its report" begin + N = 4 + model = DummyModel(1, 1, 'k') + r1 = range(model, :lambda, lower=0, upper=1) + r2 = range(model, :metric, lower=-1, upper=1) + user_range = [r1, r2] + tuning = RandomSearch(rng=1) + tuned_model = TunedModel(model=model, + tuning=tuning, + n=N, + resampling=Holdout(fraction_train=0.5), + range=user_range, + measures=[rms,mae]) + mach = machine(tuned_model, X, y) + fit!(mach, verbosity=0) + + # model predicts mean of training target, so: + train, test = partition(eachindex(y), 0.5) + μ = mean(y[train]) + error = mean((y[test] .- μ).^2) |> sqrt + + r = report(mach) + @test r.plotting.parameter_names == + ["lambda", "metric"] + @test r.plotting.parameter_scales == [:linear, :linear] + @test r.plotting.measurements ≈ fill(error, N) + @test size(r.plotting.parameter_values) == (N, 2) +end + end # module true From 96c3a862772014ac4e8cddfef6f13af71e254274 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 1 Apr 2020 17:24:49 +1300 Subject: [PATCH 27/39] add RandomSearch to readme update readme update readme --- README.md | 10 ++++++---- test/strategies/random_search.jl | 12 ++++++------ 2 files changed, 12 insertions(+), 10 deletions(-) diff --git a/README.md b/README.md index 7b21513..797fd3d 100644 --- a/README.md +++ b/README.md @@ -61,14 +61,14 @@ This repository contains: developers to conveniently implement common hyperparameter optimization strategies, such as: - - [x] search a list of explicitly specified models `list = [model1, - model2, ...]` + - [x] search models generated by an arbitrary iterator, eg `models = [model1, + model2, ...]` (built-in `Explicit` strategy) - - [x] grid search + - [x] grid search (built-in `Grid` strategy) - [ ] Latin hypercubes - - [ ] random search + - [x] random search (built-in `RandomSearch` strategy) - [ ] bandit @@ -232,6 +232,8 @@ In setting up a tuning task, the user constructs an instance of the ### Implementation requirements for new tuning strategies +As sample implemenations, see [/src/strategies/](/src/strategies) + #### Summary of functions Several functions are part of the tuning strategy API: diff --git a/test/strategies/random_search.jl b/test/strategies/random_search.jl index aa10afd..192e235 100644 --- a/test/strategies/random_search.jl +++ b/test/strategies/random_search.jl @@ -18,7 +18,7 @@ y = 2*x1 .+ 5*x2 .- 3*x3 .+ 0.2*rand(100); mutable struct DummyModel <: Deterministic lambda::Int - metric::Int + alpha::Int kernel::Char end @@ -66,7 +66,7 @@ end N = 10000 model = DummyModel(1, 1, 'k') r1 = range(model, :lambda, lower=0, upper=1) - r2 = range(model, :metric, lower=-1, upper=1) + r2 = range(model, :alpha, lower=-1, upper=1) user_range = [r1, r2] tuning = RandomSearch(rng=1) tuned_model = TunedModel(model=model, @@ -84,10 +84,10 @@ end # check the samples of each hyperparam have expected distritution: lambdas = map(m -> m.lambda, my_models) - metrics = map(m -> m.metric, my_models) + alphas = map(m -> m.alpha, my_models) a, b = values(Dist.countmap(lambdas)) @test abs(a/b - 1) < 0.06 - dict = Dist.countmap(metrics) + dict = Dist.countmap(alphas) a, b, c = dict[-1], dict[0], dict[1] @test abs(b/a - 2) < 0.06 @test abs(b/c - 2) < 0.06 @@ -97,7 +97,7 @@ end N = 4 model = DummyModel(1, 1, 'k') r1 = range(model, :lambda, lower=0, upper=1) - r2 = range(model, :metric, lower=-1, upper=1) + r2 = range(model, :alpha, lower=-1, upper=1) user_range = [r1, r2] tuning = RandomSearch(rng=1) tuned_model = TunedModel(model=model, @@ -116,7 +116,7 @@ end r = report(mach) @test r.plotting.parameter_names == - ["lambda", "metric"] + ["lambda", "alpha"] @test r.plotting.parameter_scales == [:linear, :linear] @test r.plotting.measurements ≈ fill(error, N) @test size(r.plotting.parameter_values) == (N, 2) From 80957fc26aae57d8a0a94d884acddf632f016b39 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 1 Apr 2020 17:44:59 +1300 Subject: [PATCH 28/39] fix spelling in docstring --- src/strategies/random_search.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index e83a44e..a3f14a2 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -8,7 +8,7 @@ const ParameterName=Union{Symbol,Expr} Instantiate a random search tuning strategy, for searching over Cartesian hyperparameter domains, with customizable priors in each -dimenension. +dimension. ### Supported ranges: @@ -54,7 +54,7 @@ distribution types | for fitting to ranges of this type ### Algorithm -Models for evaulation are generated by sampling each range `r` using +Models for evaluation are generated by sampling each range `r` using `rand(rng, s)` where, `s = sampler(r, d)`. See `sampler` for details. If `d` is not specified, then sampling is uniform (with replacement) in the case of a `NominalRange`, and is otherwise given by the defaults From 27c0515560bc5a54e461b8e3094c82aa91591a75 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 1 Apr 2020 17:46:28 +1300 Subject: [PATCH 29/39] minor --- src/strategies/random_search.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index a3f14a2..be11435 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -10,7 +10,7 @@ Instantiate a random search tuning strategy, for searching over Cartesian hyperparameter domains, with customizable priors in each dimension. -### Supported ranges: +### Supported ranges - a single one-dimensional range (`ParamRange` object) `r` @@ -38,7 +38,7 @@ distribution types | for fitting to ranges of this type `ParamRange` objects are constructed using the `range` method. -### Examples: +### Examples range1 = range(model, :hyper1, lower=1, origin=2, unit=1) From f9f91f7ca032009adb41ffbce237bb8531e98117 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Wed, 1 Apr 2020 19:42:28 +1300 Subject: [PATCH 30/39] tweak doc-string range examples --- src/strategies/random_search.jl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index be11435..5783fb7 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -40,10 +40,10 @@ distribution types | for fitting to ranges of this type ### Examples - range1 = range(model, :hyper1, lower=1, origin=2, unit=1) + range1 = range(model, :hyper1, lower=0, upper=1) range2 = [(range(model, :hyper1, lower=1, upper=10), Arcsine), - range(model, :hyper2, lower=2, upper=4), + range(model, :hyper2, lower=2, upper=Inf, unit=1, origin=3), (range(model, :hyper2, lower=2, upper=4), Normal(0, 3)), range(model, :hyper3, values=[:ball, :tree], [0.3, 0.7])] From d1ecb1f8f137a88dcda8fa70125635e775d3510b Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 3 Apr 2020 08:20:40 +1300 Subject: [PATCH 31/39] update doc strings --- src/strategies/grid.jl | 5 ++++- src/strategies/random_search.jl | 5 +++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/src/strategies/grid.jl b/src/strategies/grid.jl index 812ddd3..2afc6f8 100644 --- a/src/strategies/grid.jl +++ b/src/strategies/grid.jl @@ -9,7 +9,10 @@ default `resolution` in each numeric dimension. ### Supported ranges: -- A single one-dimensional range (`ParamRange` object) `r`, or pair of +A single one-dimensional range or vector of one-dimensioinal ranges +can be specified. Specically, a range can consist of: + +- A single one-dimensional range (ie, `ParamRange` object) `r`, or pair of the form `(r, res)` where `res` specifies a resolution to override the default `resolution`. diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index 5783fb7..59c05a2 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -12,6 +12,11 @@ dimension. ### Supported ranges +A single one-dimensional range or vector of one-dimensioinal ranges +can be specified. If not paired with a prior, then one is fitted, +according to fallback distribution types specified by the tuning +strategy hyperparameters. Specifically, a range can consist of: + - a single one-dimensional range (`ParamRange` object) `r` - a pair of the form `(r, d)`, with `r` as above and where `d` is a From de693a1f5895361cfb4be88d521bd7692c43ae93 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 3 Apr 2020 09:53:43 +1300 Subject: [PATCH 32/39] enable multiple ranges for same field; WIP: tests needed --- src/strategies/random_search.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index 59c05a2..fc200bf 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -109,6 +109,7 @@ function MLJTuning.models!(tuning::RandomSearch, verbosity) return map(1:n_remaining) do _ clone = deepcopy(model) + Random.shuffle!(tuning.rng, state |> collect) for (fld, s) in state recursive_setproperty!(clone, fld, rand(tuning.rng, s)) end From 301dd4684a17282ede8ba05c9830712e769cc59a Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 3 Apr 2020 11:23:55 +1300 Subject: [PATCH 33/39] add tests for multiple samplers per field --- src/strategies/random_search.jl | 16 +++++++++------- test/strategies/random_search.jl | 26 ++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index fc200bf..cb1a3ad 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -45,16 +45,18 @@ distribution types | for fitting to ranges of this type ### Examples + using Distributions + range1 = range(model, :hyper1, lower=0, upper=1) range2 = [(range(model, :hyper1, lower=1, upper=10), Arcsine), - range(model, :hyper2, lower=2, upper=Inf, unit=1, origin=3), + range(model, :hyper2, lower=2, upper=Inf, unit=1, origin=3), (range(model, :hyper2, lower=2, upper=4), Normal(0, 3)), - range(model, :hyper3, values=[:ball, :tree], [0.3, 0.7])] + (range(model, :hyper3, values=[:ball, :tree]), [0.3, 0.7])] # uniform sampling of :(atom.λ) from [0, 1] without defining a NumericRange: struct MySampler end - Base.rand(rng::AbstractRNG, ::MySampler) = rand(rng) + Base.rand(rng::Random.AbstractRNG, ::MySampler) = rand(rng) range3 = (:(atom.λ), MySampler(), range1) ### Algorithm @@ -93,13 +95,13 @@ function RandomSearch(; bounded=Distributions.Uniform, return RandomSearch(bounded, positive_unbounded, other, _rng) end -# `state`, which is not mutated, consists of a tuple of (field, sampler) -# pairs: +# `state` consists of a tuple of (field, sampler) pairs (that gets +# shuffled each iteration): setup(tuning::RandomSearch, model, user_range, verbosity) = process_random_range(user_range, tuning.bounded, tuning.positive_unbounded, - tuning.other) + tuning.other) |> collect function MLJTuning.models!(tuning::RandomSearch, model, @@ -109,7 +111,7 @@ function MLJTuning.models!(tuning::RandomSearch, verbosity) return map(1:n_remaining) do _ clone = deepcopy(model) - Random.shuffle!(tuning.rng, state |> collect) + Random.shuffle!(tuning.rng, state) for (fld, s) in state recursive_setproperty!(clone, fld, rand(tuning.rng, s)) end diff --git a/test/strategies/random_search.jl b/test/strategies/random_search.jl index 192e235..55501ad 100644 --- a/test/strategies/random_search.jl +++ b/test/strategies/random_search.jl @@ -122,5 +122,31 @@ end @test size(r.plotting.parameter_values) == (N, 2) end +struct ConstantSampler + c +end +Base.rand(rng::Random.AbstractRNG, s::ConstantSampler) = s.c + +@testset "multiple samplers for single field" begin + N = 1000 + model = DummyModel(1, 1, 'k') + r = range(model, :alpha, lower=-1, upper=1) + user_range = [(:lambda, ConstantSampler(0)), + r, + (:lambda, ConstantSampler(1))] + tuning = RandomSearch(rng=123) + tuned_model = TunedModel(model=model, + tuning=tuning, + n=N, + range=user_range, + measures=[rms,mae]) + mach = fit!(machine(tuned_model, X, y)) + my_models = first.(report(mach).history); + lambdas = map(m -> m.lambda, my_models); + a, b = values(Dist.countmap(lambdas)) + @test abs(a/b -1) < 0.04 + @test a + b == N +end + end # module true From e8148eafc1d5f80b9a105cda3f45868cd42701d9 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 3 Apr 2020 11:42:44 +1300 Subject: [PATCH 34/39] update doc-string --- src/strategies/random_search.jl | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index cb1a3ad..35dc921 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -35,6 +35,11 @@ strategy hyperparameters. Specifically, a range can consist of: - any vector of objects of the above form +A range vector may contain multiple entries for the same model field, +as in `range = [(:lambda, s1), (:alpha, s), (:lambda, s2)]`. In that +case the entry used in each iteration is random. See more under +[Algorithm](@ref). + distribution types | for fitting to ranges of this type --------------------|----------------------------------- `Arcsine`, `Uniform`, `Biweight`, `Cosine`, `Epanechnikov`, `SymTriangularDist`, `Triweight` | bounded @@ -61,13 +66,17 @@ distribution types | for fitting to ranges of this type ### Algorithm -Models for evaluation are generated by sampling each range `r` using -`rand(rng, s)` where, `s = sampler(r, d)`. See `sampler` for details. If `d` -is not specified, then sampling is uniform (with replacement) in the -case of a `NominalRange`, and is otherwise given by the defaults -specified by the tuning strategy parameters `bounded`, -`positive_unbounded`, and `other`, depending on the `NumericRange` -type. +In each iteration, a model is generated for evaluation by mutating the +fields of a deep copy of `model`. The range vector is shuffled and the +fields sampled according to the new order (repeated fields being +mutated more than once). For a `range` entry of the form `(field, s)` +the algorithm calls `rand(rng, s)` and mutates the field `field` of +the model clone to have this value. For an entry of the form `(r, d)`, +`s` is substituted with `sampler(r, d)`. If no `d` is specified, then +sampling is uniform (with replacement) if `r` is a `NominalRange`, and +is otherwise given by the defaults specified by the tuning strategy +parameters `bounded`, `positive_unbounded`, and `other`, depending on +the field values of the `NumericRange` object `r`. See also [`TunedModel`](@ref), [`range`](@ref), [`sampler`](@ref). From bebbdea02a4a9bbd1444b875b6678b2de75e2a60 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 3 Apr 2020 13:16:35 +1300 Subject: [PATCH 35/39] extend [compat] Distributions = "^0.22,^0.23" --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 7dd9390..0a37218 100644 --- a/Project.toml +++ b/Project.toml @@ -14,7 +14,7 @@ RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01" [compat] ComputationalResources = "^0.3" -Distributions = "^0.22" +Distributions = "^0.22,^0.23" MLJBase = "^0.12.2" MLJModelInterface = "^0.2" RecipesBase = "^0.8" From b379a0b5c8ffe6dc59f0e504d1b28f8abacf0d59 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 3 Apr 2020 14:42:25 +1300 Subject: [PATCH 36/39] further doc-string additions --- README.md | 5 ++-- src/strategies/grid.jl | 3 +- src/strategies/random_search.jl | 6 ++-- src/tuned_models.jl | 49 +++++++++++++++++++++++++++++++-- 4 files changed, 55 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 797fd3d..06c382d 100644 --- a/README.md +++ b/README.md @@ -502,8 +502,9 @@ where: model - `tuning_report(::MyTuningStrategy, ...)` is a method the implementer - may overload. It should return a named tuple. The fallback is to - return the raw history: + may overload. It should return a named tuple with `history` as one + of the keys (the format up to the implementation.) The fallback is + to return the raw history: ```julia MLJTuning.tuning_report(tuning, history, state) = (history=history,) diff --git a/src/strategies/grid.jl b/src/strategies/grid.jl index 2afc6f8..31f8e6f 100644 --- a/src/strategies/grid.jl +++ b/src/strategies/grid.jl @@ -10,7 +10,8 @@ default `resolution` in each numeric dimension. ### Supported ranges: A single one-dimensional range or vector of one-dimensioinal ranges -can be specified. Specically, a range can consist of: +can be specified. Specifically, in `Grid` search, the `range` field +of a `TunedModel` instance can be: - A single one-dimensional range (ie, `ParamRange` object) `r`, or pair of the form `(r, res)` where `res` specifies a resolution to override diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index 35dc921..4277660 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -15,7 +15,8 @@ dimension. A single one-dimensional range or vector of one-dimensioinal ranges can be specified. If not paired with a prior, then one is fitted, according to fallback distribution types specified by the tuning -strategy hyperparameters. Specifically, a range can consist of: +strategy hyperparameters. Specifically, in `RandomSearch`, the `range` +field of a `TunedModel` instance can be: - a single one-dimensional range (`ParamRange` object) `r` @@ -37,8 +38,7 @@ strategy hyperparameters. Specifically, a range can consist of: A range vector may contain multiple entries for the same model field, as in `range = [(:lambda, s1), (:alpha, s), (:lambda, s2)]`. In that -case the entry used in each iteration is random. See more under -[Algorithm](@ref). +case the entry used in each iteration is random. distribution types | for fitting to ranges of this type --------------------|----------------------------------- diff --git a/src/tuned_models.jl b/src/tuned_models.jl index 3fa7b51..998ad3b 100644 --- a/src/tuned_models.jl +++ b/src/tuned_models.jl @@ -115,19 +115,64 @@ key | value `best_model` | optimal model instance `best_fitted_params`| learned parameters of the optimal model -The named tuple `report(mach)` has these keys/values: +The named tuple `report(mach)` includes these keys/values: key | value --------------------|-------------------------------------------------- `best_model` | optimal model instance `best_result` | corresponding "result" entry in the history `best_report` | report generated by fitting the optimal model +`history` | tuning strategy-specific history of all evaluations plus others specific to the `tuning` strategy, such as `history=...`. +### Hyperparameter summary + +- `model`: `Supervised` model prototype that is cloned and mutated to + generate models for evaluation + +- `tuning`: tuning strategy to be applied (eg, `Grid()`, `RandomSearch()`) + +- `resampling`: resampling strategy (eg, `Holdout()`, `CV()`), + `StratifiedCV()` to be applied in performance evaluations + +- `measure`: measure or measures to be applied in performance + evaluations; only the first used in optimization (unless the + strategy is multi-objective) but all reported to the history + +- `weights`: sample weights to be passed the measure(s) in performance + evaluations, if supported (see important note above for behaviour in + unspecified case) + +- `repeats`: for generating train/test sets multiple times in + resampling; see [`evaluate!`](@ref) for + details + +- `operation=predict`: operation to be applied to each fitted model; + usually `predict` but `predict_mean`, `predict_median` and + `predict_mode` can be used for `Probabilistic` models, in + conjunction with `Deterministic` measures + +- `range`: range object; see tuning strategy specific documentation + for supported types + +- `n`: number of iterations (ie, models to be evaluated); if left + unspecified then `default_n(tuning, + range)` is used + +- `train_best=true`: whether to train the optimal model + +- `acceleration`: mode of parallelization for tuning strategies that + support this + +- `acceleration_resampling`: mode of parallelization for resampling + +- `check_measure`: whether to check `measure` is compatible with the + specified `model` and `operation`) + """ -function TunedModel(;model=nothing, +function TunedModel(; model=nothing, tuning=Grid(), resampling=MLJBase.Holdout(), measures=nothing, From da4692aea32e465bf499c05a0f0adeb030875ec7 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 3 Apr 2020 15:04:30 +1300 Subject: [PATCH 37/39] more doc tweaks --- src/tuned_models.jl | 35 +++++++++++++++++------------------ 1 file changed, 17 insertions(+), 18 deletions(-) diff --git a/src/tuned_models.jl b/src/tuned_models.jl index 998ad3b..06e4e84 100644 --- a/src/tuned_models.jl +++ b/src/tuned_models.jl @@ -127,15 +127,15 @@ key | value plus others specific to the `tuning` strategy, such as `history=...`. -### Hyperparameter summary +### Summary of key-word arguments - `model`: `Supervised` model prototype that is cloned and mutated to generate models for evaluation -- `tuning`: tuning strategy to be applied (eg, `Grid()`, `RandomSearch()`) +- `tuning=Grid()`: tuning strategy to be applied (eg, `RandomSearch()`) -- `resampling`: resampling strategy (eg, `Holdout()`, `CV()`), - `StratifiedCV()` to be applied in performance evaluations +- `resampling=Holdout()`: resampling strategy (eg, `Holdout()`, `CV()`), + `StratifiedCV()`) to be applied in performance evaluations - `measure`: measure or measures to be applied in performance evaluations; only the first used in optimization (unless the @@ -145,28 +145,27 @@ plus others specific to the `tuning` strategy, such as `history=...`. evaluations, if supported (see important note above for behaviour in unspecified case) -- `repeats`: for generating train/test sets multiple times in - resampling; see [`evaluate!`](@ref) for - details +- `repeats=1`: for generating train/test sets multiple times in + resampling; see [`evaluate!`](@ref) for details - `operation=predict`: operation to be applied to each fitted model; - usually `predict` but `predict_mean`, `predict_median` and - `predict_mode` can be used for `Probabilistic` models, in - conjunction with `Deterministic` measures + usually `predict` but `predict_mean`, `predict_median` or + `predict_mode` can be used for `Probabilistic` models, if + the specified measures are `Deterministic` -- `range`: range object; see tuning strategy specific documentation - for supported types +- `range`: range object; tuning strategy documentation describes + supported types -- `n`: number of iterations (ie, models to be evaluated); if left - unspecified then `default_n(tuning, - range)` is used +- `n`: number of iterations (ie, models to be evaluated); set by + tuning strategy if left unspecified - `train_best=true`: whether to train the optimal model -- `acceleration`: mode of parallelization for tuning strategies that - support this +- `acceleration=default_resource()`: mode of parallelization for + tuning strategies that support this -- `acceleration_resampling`: mode of parallelization for resampling +- `acceleration_resampling=CPU1()`: mode of parallelization for + resampling - `check_measure`: whether to check `measure` is compatible with the specified `model` and `operation`) From b7c6d24c71c44c01a556b303325909c9955637c7 Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 3 Apr 2020 15:23:25 +1300 Subject: [PATCH 38/39] more doc tweaks --- README.md | 2 +- src/strategies/random_search.jl | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 06c382d..e4e13c5 100644 --- a/README.md +++ b/README.md @@ -232,7 +232,7 @@ In setting up a tuning task, the user constructs an instance of the ### Implementation requirements for new tuning strategies -As sample implemenations, see [/src/strategies/](/src/strategies) +As sample implementations, see [/src/strategies/](/src/strategies) #### Summary of functions diff --git a/src/strategies/random_search.jl b/src/strategies/random_search.jl index 4277660..8f98fe4 100644 --- a/src/strategies/random_search.jl +++ b/src/strategies/random_search.jl @@ -20,14 +20,18 @@ field of a `TunedModel` instance can be: - a single one-dimensional range (`ParamRange` object) `r` -- a pair of the form `(r, d)`, with `r` as above and where `d` is a - probability vector of the same length as `r.values`, if `r` is a - `NominalRange`, and is otherwise: (i) any - `Distributions.UnivariateDistribution` *instance*; or (ii) one of - the *subtypes* of `Distributions.UnivariateDistribution` listed in - the table below, for automatic fitting using `Distributions.fit(d, - r)` (a distribution whose support always lies between `r.lower` and - `r.upper`.) +- a pair of the form `(r, d)`, with `r` as above and where `d` is: + + - a probability vector of the same length as `r.values` (`r` a + `NominalRange`) + + - any `Distributions.UnivariateDistribution` *instance* (`r` a + `NumericRange`) + + - one of the *subtypes* of `Distributions.UnivariateDistribution` + listed in the table below, for automatic fitting using + `Distributions.fit(d, r)`, a distribution whose support always + lies between `r.lower` and `r.upper` (`r` a `NumericRange`) - any pair of the form `(field, s)`, where `field` is the (possibly nested) name of a field of the model to be tuned, and `s` an From 9dcfa02af2fa235e38873c92f4fd78509b60a3eb Mon Sep 17 00:00:00 2001 From: "Anthony D. Blaom" Date: Fri, 3 Apr 2020 15:28:52 +1300 Subject: [PATCH 39/39] bump version = "0.3.1" --- Project.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Project.toml b/Project.toml index 0a37218..67a2451 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MLJTuning" uuid = "03970b2e-30c4-11ea-3135-d1576263f10f" authors = ["Anthony D. Blaom "] -version = "0.3.0" +version = "0.3.1" [deps] ComputationalResources = "ed09eef8-17a6-5b46-8889-db040fac31e3"