MLJ.jl
MLJ.jl copied to clipboard
Integration test failures: Classifiers
edited A new package MLJTestIntegration.jl is under development to provide integration tests for the MLJ ecosystem at large. Some issues have been revealed there for the regressors in the following table:
julia> fails |> DataFrame
Row │ name package test exception
─────┼───────────────────────────────────────────────────────────────────────────────────────────────
1 │ DecisionTreeClassifier BetaML ensemble_prediction MethodError(predict, (WrappedEns…
2 │ NuSVC LIBSVM tuned_pipe_evaluation CompositeException(Any[TaskFaile…
3 │ PegasosClassifier BetaML ensemble_prediction ArgumentError("Adding two `Univa…
4 │ RandomForestClassifier BetaML ensemble_prediction MethodError(predict, (WrappedEns…
5 │ SVMNuClassifier ScikitLearn tuned_pipe_evaluation CompositeException(Any[TaskFaile…
An example on how to reproduce is given at the end.
TODO: Open issues for each of the above:
- [x] 1, 4: https://github.com/JuliaAI/MLJEnsembles.jl/pull/19
- [x] 3 related to https://github.com/sylvaticus/BetaML.jl/issues/31
- [ ] 2.
- [ ] 5.
Intermittently I am also getting a problem with KernelPerceptronClassifier
, tracked:
- [x] here.
using Pkg
Pkg.add("BetaML")
Pkg.add(url="https://github.com/JuliaAI/MLJTestIntegration.jl", rev="dev")
using BetaML, MLJTestIntegration
julia> MLJTestIntegration.test_single_target_classifiers(
[(name="DecisionTreeClassifier", package_name="BetaML"),],
level=3,
throw=true
)
ERROR: MethodError: no method matching predict(::MLJEnsembles.WrappedEnsemble{Tuple{DecisionNode{Float64}, CategoricalArrays.CategoricalValue{String, UInt32}}, DecisionTreeClassifier}, ::Vector{Float64}, ::NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}}, ::Type{MLJModelInterface.Probabilistic}, ::Type{AbstractVector{var"#s149"} where var"#s149"<:Union{Missing, ScientificTypesBase.Finite}})
Closest candidates are:
predict(::MLJEnsembles.WrappedEnsemble, ::Any, ::Any, ::Type{MLJModelInterface.Probabilistic}, ::Type{var"#s23"} where var"#s23"<:(AbstractVector{var"#s1"} where var"#s1"<:ScientificTypesBase.Continuous)) at /Users/anthony/.julia/packages/MLJEnsembles/i1fek/src/ensembles.jl:93
predict(::MLJEnsembles.WrappedEnsemble, ::Any, ::Any, ::Type{MLJModelInterface.Probabilistic}, ::Type{var"#s23"} where var"#s23"<:(AbstractVector{var"#s1"} where var"#s1"<:ScientificTypesBase.Finite)) at /Users/anthony/.julia/packages/MLJEnsembles/i1fek/src/ensembles.jl:76
predict(::MLJBase.Machine{var"#s326", C} where {var"#s326"<:MLJModelInterface.Static, C}, ::Any, ::Any...) at /Users/anthony/.julia/packages/MLJBase/QZctr/src/operations.jl:93
...
Stacktrace:
[1] attempt(f::MLJTestIntegration.var"#24#25"{DecisionTreeClassifier, Tuple{NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}}, CategoricalArrays.CategoricalVector{String, UInt32, String, CategoricalArrays.CategoricalValue{String, UInt32}, Union{}}}}, message::String; throw::Bool)
@ MLJTestIntegration ~/MLJ/MLJTestIntegration/src/attemptors.jl:17
[2] ensemble_prediction(::DecisionTreeClassifier, ::NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}}, ::Vararg{Any, N} where N; throw::Bool, verbosity::Int64)
@ MLJTestIntegration ~/MLJ/MLJTestIntegration/src/attemptors.jl:149
[3] test(::Vector{NamedTuple{(:name, :package_name), Tuple{String, String}}}, ::NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}}, ::Vararg{Any, N} where N; mod::Module, level::Int64, throw::Bool, verbosity::Int64)
@ MLJTestIntegration ~/MLJ/MLJTestIntegration/src/test.jl:297
[4] _test(proxies::Vector{NamedTuple{(:name, :package_name), Tuple{String, String}}}, data::Tuple{NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}}, CategoricalArrays.CategoricalVector{String, UInt32, String, CategoricalArrays.CategoricalValue{String, UInt32}, Union{}}}; ignore::Bool, verbosity::Int64, kwargs::Base.Iterators.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:level, :throw), Tuple{Int64, Bool}}})
@ MLJTestIntegration ~/MLJ/MLJTestIntegration/src/special_cases.jl:26
[5] test_single_target_classifiers(args::Vector{NamedTuple{(:name, :package_name), Tuple{String, String}}}; kwargs::Base.Iterators.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:level, :throw), Tuple{Int64, Bool}}})
@ MLJTestIntegration ~/MLJ/MLJTestIntegration/src/special_cases.jl:40
[6] top-level scope
@ REPL[32]:1
caused by: MethodError: no method matching predict(::MLJEnsembles.WrappedEnsemble{Tuple{DecisionNode{Float64}, CategoricalArrays.CategoricalValue{String, UInt32}}, DecisionTreeClassifier}, ::Vector{Float64}, ::NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}}, ::Type{MLJModelInterface.Probabilistic}, ::Type{AbstractVector{var"#s149"} where var"#s149"<:Union{Missing, ScientificTypesBase.Finite}})
Closest candidates are:
predict(::MLJEnsembles.WrappedEnsemble, ::Any, ::Any, ::Type{MLJModelInterface.Probabilistic}, ::Type{var"#s23"} where var"#s23"<:(AbstractVector{var"#s1"} where var"#s1"<:ScientificTypesBase.Continuous)) at /Users/anthony/.julia/packages/MLJEnsembles/i1fek/src/ensembles.jl:93
predict(::MLJEnsembles.WrappedEnsemble, ::Any, ::Any, ::Type{MLJModelInterface.Probabilistic}, ::Type{var"#s23"} where var"#s23"<:(AbstractVector{var"#s1"} where var"#s1"<:ScientificTypesBase.Finite)) at /Users/anthony/.julia/packages/MLJEnsembles/i1fek/src/ensembles.jl:76
predict(::MLJBase.Machine{var"#s326", C} where {var"#s326"<:MLJModelInterface.Static, C}, ::Any, ::Any...) at /Users/anthony/.julia/packages/MLJBase/QZctr/src/operations.jl:93
...
Stacktrace:
[1] predict(wens::MLJEnsembles.WrappedEnsemble{Tuple{DecisionNode{Float64}, CategoricalArrays.CategoricalValue{String, UInt32}}, DecisionTreeClassifier}, atomic_weights::Vector{Float64}, Xnew::NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}})
@ MLJEnsembles ~/.julia/packages/MLJEnsembles/i1fek/src/ensembles.jl:34
[2] predict(model::MLJEnsembles.ProbabilisticEnsembleModel{DecisionTreeClassifier}, fitresult::MLJEnsembles.WrappedEnsemble{Tuple{DecisionNode{Float64}, CategoricalArrays.CategoricalValue{String, UInt32}}, DecisionTreeClassifier}, Xnew::NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}})
@ MLJEnsembles ~/.julia/packages/MLJEnsembles/i1fek/src/ensembles.jl:559
[3] predict(mach::MLJBase.Machine{MLJEnsembles.ProbabilisticEnsembleModel{DecisionTreeClassifier}, true}, Xraw::NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}})
@ MLJBase ~/.julia/packages/MLJBase/QZctr/src/operations.jl:85
[4] (::MLJTestIntegration.var"#24#25"{DecisionTreeClassifier, Tuple{NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}}, CategoricalArrays.CategoricalVector{String, UInt32, String, CategoricalArrays.CategoricalValue{String, UInt32}, Union{}}}})()
@ MLJTestIntegration ~/MLJ/MLJTestIntegration/src/attemptors.jl:154
[5] attempt(f::MLJTestIntegration.var"#24#25"{DecisionTreeClassifier, Tuple{NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}}, CategoricalArrays.CategoricalVector{String, UInt32, String, CategoricalArrays.CategoricalValue{String, UInt32}, Union{}}}}, message::String; throw::Bool)
@ MLJTestIntegration ~/MLJ/MLJTestIntegration/src/attemptors.jl:15
[6] ensemble_prediction(::DecisionTreeClassifier, ::NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}}, ::Vararg{Any, N} where N; throw::Bool, verbosity::Int64)
@ MLJTestIntegration ~/MLJ/MLJTestIntegration/src/attemptors.jl:149
[7] test(::Vector{NamedTuple{(:name, :package_name), Tuple{String, String}}}, ::NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}}, ::Vararg{Any, N} where N; mod::Module, level::Int64, throw::Bool, verbosity::Int64)
@ MLJTestIntegration ~/MLJ/MLJTestIntegration/src/test.jl:297
[8] _test(proxies::Vector{NamedTuple{(:name, :package_name), Tuple{String, String}}}, data::Tuple{NamedTuple{(:FL, :RW), Tuple{Vector{Float64}, Vector{Float64}}}, CategoricalArrays.CategoricalVector{String, UInt32, String, CategoricalArrays.CategoricalValue{String, UInt32}, Union{}}}; ignore::Bool, verbosity::Int64, kwargs::Base.Iterators.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:level, :throw), Tuple{Int64, Bool}}})
@ MLJTestIntegration ~/MLJ/MLJTestIntegration/src/special_cases.jl:26
[9] test_single_target_classifiers(args::Vector{NamedTuple{(:name, :package_name), Tuple{String, String}}}; kwargs::Base.Iterators.Pairs{Symbol, Integer, Tuple{Symbol, Symbol}, NamedTuple{(:level, :throw), Tuple{Int64, Bool}}})
@ MLJTestIntegration ~/MLJ/MLJTestIntegration/src/special_cases.jl:40
[10] top-level scope
@ REPL[32]:1
Further failures when adding multithreading tests:
3×4 DataFrame
Row │ name package_name test exception
│ Any Any Any Any
─────┼───────────────────────────────────────────────────────────────────────────────────────────────────
1 │ LinearSVC LIBSVM accelerated_evaluation AssertionError("Different comput…
2 │ MultinomialClassifier MLJLinearModels accelerated_evaluation AssertionError("Different comput…
3 │ SVMLinearClassifier ScikitLearn accelerated_evaluation AssertionError("Different comput…
The error indicates that evaluate
gives different results for acceleration=CPU1()
and acceleration=CPUThreads()
, despite the fact the model appears to give the same predictions for repeated retraining on a CPU1(). And this despite the fact that resmapling was simple holdout. Note that 1 and 3 wrap the same C library LIBSVM.
- [x] 1, 3
- [x] 2
(30 other classifiers have no problems with this new test)
To reproduce use MLJTestIntegration#multi-threading, and call MLJTestIntegration.test_single_target_classifiers
as indicated in previous post, but with level=4
.
Strange, I don't experience the same error for MultinomialClassifier
on my PC when running julia with 4 threads.
julia> using MLJ
julia> function _make_binary()
data = MLJ.load_crabs()
y_, X = unpack(data, ==(:sp), col->col in [:FL, :RW])
y = coerce(y_, MLJ.OrderedFactor)
return X, y
end
_make_binary (generic function with 1 method)
julia> X, y = _make_binary()
((FL = [8.1, 8.8, 9.2, 9.6, 9.8, 10.8, 11.1, 11.6, 11.8, 11.8 … 20.3, 20.5, 20.6, 20.9, 21.3, 21.4, 21.7, 21.9, 22.5, 23.1], RW = [6.7, 7.7, 7.8, 7.9, 8.0, 9.0, 9.9, 9.1, 9.6, 10.5 … 16.0, 17.5, 17.5, 16.5, 18.4, 18.0, 17.1, 17.2, 17.2, 20.2]), CategoricalArrays.CategoricalValue{String, UInt32}["B", "B", "B", "B", "B", "B", "B", "B", "B", "B" … "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"])
julia> resources = [CPU1(), CPUThreads()]
2-element Vector{ComputationalResources.AbstractCPU{Nothing}}:
CPU1{Nothing}(nothing)
CPUThreads{Nothing}(nothing)
julia> mc = @load MultinomialClassifier
[ Info: For silent loading, specify `verbosity=0`.
import MLJLinearModels ✔
MLJLinearModels.MultinomialClassifier
julia> model = mc()
MultinomialClassifier(
lambda = 1.0,
gamma = 0.0,
penalty = :l2,
fit_intercept = true,
penalize_intercept = false,
scale_penalty_with_samples = true,
solver = nothing)
julia> data = (X, y)
((FL = [8.1, 8.8, 9.2, 9.6, 9.8, 10.8, 11.1, 11.6, 11.8, 11.8 … 20.3, 20.5, 20.6, 20.9, 21.3, 21.4, 21.7, 21.9, 22.5, 23.1], RW = [6.7, 7.7, 7.8, 7.9, 8.0, 9.0, 9.9, 9.1, 9.6, 10.5 … 16.0, 17.5, 17.5, 16.5, 18.4, 18.0, 17.1, 17.2, 17.2, 20.2]), CategoricalArrays.CategoricalValue{String, UInt32}["B", "B", "B", "B", "B", "B", "B", "B", "B", "B" … "O", "O", "O", "O", "O", "O", "O", "O", "O", "O"])
julia> measure = MLJ.MLJBase.default_measure(model)
LogLoss(
tol = 2.220446049250313e-16)
julia> es = map(resources) do resource
evaluate(model, data...;
measure=measure,
resampling=CV(nfolds=4),
acceleration=resource,
verbosity=0)
end
2-element Vector{PerformanceEvaluation{Vector{LogLoss{Float64}}, Vector{Float64}, Vector{typeof(predict)}, Vector{Vector{Float64}}, Vector{Vector{Vector{Float64}}}, Vector{NamedTuple{(:classes, :coefs, :intercept), Tuple{CategoricalArrays.CategoricalVector{String, UInt32, String, CategoricalArrays.CategoricalValue{String, UInt32}, Union{}}, Vector{Pair{Symbol, Float64}}, Float64}}}, Vector{NamedTuple{(), Tuple{}}}}}:
PerformanceEvaluation(18.0,)
PerformanceEvaluation(18.0,)
julia> ms = map(e->sort(e.per_fold[1]), es)
2-element Vector{Vector{Float64}}:
[2.2204460492503136e-16, 2.2204460492503136e-16, 36.04365338911719, 36.04365338911719]
[2.2204460492503136e-16, 2.2204460492503136e-16, 36.04365338911719, 36.04365338911719]
julia> m = first(ms)
4-element Vector{Float64}:
2.2204460492503136e-16
2.2204460492503136e-16
36.04365338911719
36.04365338911719
julia> all(≈(m), collect(ms)[2:end])
true
@OkonSamuel Thanks for checking. Those issues at this comment seemed to be resolved, at least for now.
These are either sorted or tracked here.