SemOptimizerResult: streamline optim results

Author: Alexey Stukalov

docs/src/developer/optimizer.md

@@ -30,7 +30,6 @@ update_observed(optimizer::SemOptimizerName, observed::SemObserved; kwargs...) =
 ### additional methods
 ############################################################################################
 
-algorithm(optimizer::SemOptimizerName) = optimizer.algorithm
 options(optimizer::SemOptimizerName) = optimizer.options
 ```
 
@@ -68,7 +67,7 @@ The method has to return a `SemFit` object that consists of the minimum of the o
 In addition, you might want to provide methods to access properties of your optimization result:
 
 ```julia
-optimizer(res::MyOptimizationResult) = ...
+algorithm_name(res::MyOptimizationResult) = ...
 n_iterations(res::MyOptimizationResult) = ...
 convergence(res::MyOptimizationResult) = ...
 ```

ext/SEMNLOptExt/NLopt.jl

@@ -118,31 +118,32 @@ SEM.update_observed(optimizer::SemOptimizerNLopt, observed::SemObserved; kwargs.
 ### additional methods
 ############################################################################################
 
-SEM.algorithm(optimizer::SemOptimizerNLopt) = optimizer.algorithm
 local_algorithm(optimizer::SemOptimizerNLopt) = optimizer.local_algorithm
 SEM.options(optimizer::SemOptimizerNLopt) = optimizer.options
 local_options(optimizer::SemOptimizerNLopt) = optimizer.local_options
 equality_constraints(optimizer::SemOptimizerNLopt) = optimizer.equality_constraints
 inequality_constraints(optimizer::SemOptimizerNLopt) = optimizer.inequality_constraints
 
-struct NLoptResult
+# wrapper for the NLopt optimization result
+struct NLoptResult <: SEM.SemOptimizerResult{SemOptimizerNLopt}
+    optimizer::SemOptimizerNLopt
     result::Any
     problem::Any
 end
 
-SEM.optimizer(res::NLoptResult) = res.problem.algorithm
+SEM.algorithm_name(res::NLoptResult) = res.problem.algorithm
 SEM.n_iterations(res::NLoptResult) = res.problem.numevals
 SEM.convergence(res::NLoptResult) = res.result[3]
 
-# construct SemFit from fitted NLopt object
-function SemFit_NLopt(optimization_result, model::AbstractSem, start_val, opt)
-    return SemFit(
-        optimization_result[1],
-        optimization_result[2],
-        start_val,
-        model,
-        NLoptResult(optimization_result, opt),
-    )
+# construct NLopt.jl problem
+function NLopt_problem(algorithm, options, npar)
+    problem = Opt(algorithm, npar)
+
+    for (key, val) in pairs(options)
+        setproperty!(problem, key, val)
+    end
+
+    return problem
 end
 
 # fit method
@@ -152,8 +153,8 @@ function SEM.fit(
     start_params::AbstractVector;
     kwargs...,
 )
-    opt = construct_NLopt(optim.algorithm, optim.options, nparams(model))
-    opt.min_objective =
+    problem = NLopt_problem(optim.algorithm, optim.options, nparams(model))
+    problem.min_objective =
         (par, G) -> SEM.evaluate!(
             zero(eltype(par)),
             !isnothing(G) && !isempty(G) ? G : nothing,
@@ -162,36 +163,27 @@ function SEM.fit(
             par,
         )
     for (f, tol) in optim.inequality_constraints
-        inequality_constraint!(opt, f, tol)
+        inequality_constraint!(problem, f, tol)
     end
     for (f, tol) in optim.equality_constraints
-        equality_constraint!(opt, f, tol)
+        equality_constraint!(problem, f, tol)
     end
 
     if !isnothing(optim.local_algorithm)
-        opt_local =
-            construct_NLopt(optim.local_algorithm, optim.local_options, nparams(model))
-        opt.local_optimizer = opt_local
+        problem.local_optimizer =
+            NLopt_problem(optim.local_algorithm, optim.local_options, nparams(model))
     end
 
     # fit
-    result = NLopt.optimize(opt, start_params)
+    result = NLopt.optimize(problem, start_params)
 
-    return SemFit_NLopt(result, model, start_params, opt)
-end
-
-############################################################################################
-### additional functions
-############################################################################################
-
-function construct_NLopt(algorithm, options, npar)
-    opt = Opt(algorithm, npar)
-
-    for (key, val) in pairs(options)
-        setproperty!(opt, key, val)
-    end
-
-    return opt
+    return SemFit(
+        result[1], # minimum
+        result[2], # optimal params
+        start_val,
+        model,
+        NLoptResult(optim, result, problem),
+    )
 end
 
 ############################################################################################

ext/SEMProximalOptExt/ProximalAlgorithms.jl

@@ -41,12 +41,6 @@ SEM.sem_optimizer_subtype(::Val{:Proximal}) = SemOptimizerProximal
 SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwargs...) =
     optimizer
 
-############################################################################################
-### additional methods
-############################################################################################
-
-SEM.algorithm(optimizer::SemOptimizerProximal) = optimizer.algorithm
-
 ############################################################################
 ### Model fitting
 ############################################################################
@@ -58,8 +52,11 @@ function ProximalAlgorithms.value_and_gradient(model::AbstractSem, params)
     return obj, grad
 end
 
-mutable struct ProximalResult
-    result::Any
+# wrapper for the Proximal optimization result
+struct ProximalResult{O <: SemOptimizer{:Proximal}} <: SEM.SemOptimizerResult{O}
+    optimizer::O
+    minimum::Float64
+    n_iterations::Int
 end
 
 function SEM.fit(
@@ -69,36 +66,20 @@ function SEM.fit(
     kwargs...,
 )
     if isnothing(optim.operator_h)
-        solution, iterations =
+        solution, niterations =
             optim.algorithm(x0 = start_params, f = model, g = optim.operator_g)
     else
-        solution, iterations = optim.algorithm(
+        solution, niterations = optim.algorithm(
             x0 = start_params,
             f = model,
             g = optim.operator_g,
             h = optim.operator_h,
         )
     end
 
-    minimum = objective!(model, solution)
+    optim_res = ProximalResult(optim, objective!(model, solution), niterations)
 
-    optimization_result = Dict(
-        :minimum => minimum,
-        :iterations => iterations,
-        :algorithm => optim.algorithm,
-        :operator_g => optim.operator_g,
-    )
-
-    isnothing(optim.operator_h) ||
-        push!(optimization_result, :operator_h => optim.operator_h)
-
-    return SemFit(
-        minimum,
-        solution,
-        start_params,
-        model,
-        ProximalResult(optimization_result),
-    )
+    return SemFit(optim_res.minimum, solution, start_params, model, optim_res)
 end
 
 ############################################################################################
@@ -125,10 +106,9 @@ function Base.show(io::IO, struct_inst::SemOptimizerProximal)
 end
 
 function Base.show(io::IO, result::ProximalResult)
-    print(io, "Minimum:          $(round(result.result[:minimum]; digits = 2)) \n")
-    print(io, "No. evaluations:  $(result.result[:iterations]) \n")
-    print(io, "Operator:         $(nameof(typeof(result.result[:operator_g]))) \n")
-    if haskey(result.result, :operator_h)
-        print(io, "Second Operator:  $(nameof(typeof(result.result[:operator_h]))) \n")
-    end
+    print(io, "Minimum:          $(round(result.minimum; digits = 2)) \n")
+    print(io, "No. evaluations:  $(result.n_iterations) \n")
+    print(io, "Operator:         $(nameof(typeof(result.optimizer.operator_g))) \n")
+    op_h = result.optimizer.operator_h
+    isnothing(op_h) || print(io, "Second Operator:  $(nameof(typeof(op_h))) \n")
 end

src/frontend/fit/SemFit.jl

@@ -13,7 +13,7 @@ Fitted structural equation model.
 - `model(::SemFit)`
 - `optimization_result(::SemFit)`
 
-- `optimizer(::SemFit)` -> optimization algorithm
+- `algorithm_name(::SemFit)` -> optimization algorithm
 - `n_iterations(::SemFit)` -> number of iterations
 - `convergence(::SemFit)` -> convergence properties
 """
@@ -63,6 +63,6 @@ optimization_result(sem_fit::SemFit) = sem_fit.optimization_result
 
 # optimizer properties
 optimizer_engine(sem_fit::SemFit) = optimizer_engine(optimization_result(sem_fit))
-optimizer(sem_fit::SemFit) = optimizer(optimization_result(sem_fit))
+algorithm_name(sem_fit::SemFit) = algorithm_name(optimization_result(sem_fit))
 n_iterations(sem_fit::SemFit) = n_iterations(optimization_result(sem_fit))
 convergence(sem_fit::SemFit) = convergence(optimization_result(sem_fit))

src/frontend/fit/summary.jl

@@ -7,7 +7,8 @@ function details(sem_fit::SemFit; show_fitmeasures = false, color = :light_cyan,
         color = color,
     )
     print("\n")
-    println("Optimization algorithm:      $(optimizer(sem_fit))")
+    println("Optimization engine:         $(optimizer_engine(sem_fit))")
+    println("Optimization algorithm:      $(algorithm_name(sem_fit))")
     println("Convergence:                 $(convergence(sem_fit))")
     println("No. iterations/evaluations:  $(n_iterations(sem_fit))")
     print("\n")

src/optimizer/abstract.jl

@@ -84,6 +84,10 @@ For a list of available engines, call [`optimizer_engines`](@ref).
 """
 optimizer_engine_doc(engine) = Base.Docs.doc(sem_optimizer_subtype(engine))
 
+optimizer(result::SemOptimizerResult) = result.optimizer
+
+optimizer_engine(result::SemOptimizerResult) = optimizer_engine(result.optimizer)
+
 """
     fit([optim::SemOptimizer], model::AbstractSem;
         [engine::Symbol], start_val = start_val, kwargs...)

src/optimizer/optim.jl

@@ -67,26 +67,17 @@ update_observed(optimizer::SemOptimizerOptim, observed::SemObserved; kwargs...)
 ### additional methods
 ############################################################################################
 
-algorithm(optimizer::SemOptimizerOptim) = optimizer.algorithm
 options(optimizer::SemOptimizerOptim) = optimizer.options
 
-function SemFit(
-    optimization_result::Optim.MultivariateOptimizationResults,
-    model::AbstractSem,
-    start_val,
-)
-    return SemFit(
-        optimization_result.minimum,
-        optimization_result.minimizer,
-        start_val,
-        model,
-        optimization_result,
-    )
+# wrapper for the Optim.jl result
+struct SemOptimResult{O <: SemOptimizerOptim} <: SemOptimizerResult{O}
+    optimizer::O
+    result::Optim.MultivariateOptimizationResults
 end
 
-optimizer(res::Optim.MultivariateOptimizationResults) = Optim.summary(res)
-n_iterations(res::Optim.MultivariateOptimizationResults) = Optim.iterations(res)
-convergence(res::Optim.MultivariateOptimizationResults) = Optim.converged(res)
+algorithm_name(res::SemOptimResult) = Optim.summary(res.result)
+n_iterations(res::SemOptimResult) = Optim.iterations(res.result)
+convergence(res::SemOptimResult) = Optim.converged(res.result)
 
 function fit(
     optim::SemOptimizerOptim,
@@ -133,5 +124,11 @@ function fit(
             optim.options,
         )
     end
-    return SemFit(result, model, start_params)
+    return SemFit(
+        result.minimum,
+        result.minimizer,
+        start_params,
+        model,
+        SemOptimResult(optim, result),
+    )
 end

src/types.jl

@@ -81,6 +81,9 @@ Base.:*(x::SemWeight, y) = x.w * y
 
 abstract type SemOptimizer{E} end
 
+# wrapper around optimization result
+abstract type SemOptimizerResult{O <: SemOptimizer} end
+
 """
 Supertype of all objects that can serve as the observed field of a SEM.
 Pre-processes data and computes sufficient statistics for example.

test/examples/proximal/l0.jl

@@ -45,7 +45,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML)
 fit_prox = fit(model_prox, engine = :Proximal, operator_g = prox_operator)
 
 @testset "l0 | solution_unregularized" begin
-    @test fit_prox.optimization_result.result[:iterations] < 1000
+    @test n_iterations(fit_prox.optimization_result) < 1000
     @test maximum(abs.(solution(sem_fit) - solution(fit_prox))) < 0.002
 end
 
@@ -57,7 +57,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML)
 fit_prox = fit(model_prox, engine = :Proximal, operator_g = prox_operator)
 
 @testset "l0 | solution_regularized" begin
-    @test fit_prox.optimization_result.result[:iterations] < 1000
+    @test n_iterations(fit_prox.optimization_result) < 1000
     @test solution(fit_prox)[31] == 0.0
     @test abs(
         StructuralEquationModels.minimum(fit_prox) -

test/examples/proximal/lasso.jl

@@ -43,7 +43,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML)
 fit_prox = fit(model_prox, engine = :Proximal, operator_g = NormL1(λ))
 
 @testset "lasso | solution_unregularized" begin
-    @test fit_prox.optimization_result.result[:iterations] < 1000
+    @test n_iterations(fit_prox.optimization_result) < 1000
     @test maximum(abs.(solution(sem_fit) - solution(fit_prox))) < 0.002
 end
 
@@ -55,7 +55,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML)
 fit_prox = fit(model_prox, engine = :Proximal, operator_g = NormL1(λ))
 
 @testset "lasso | solution_regularized" begin
-    @test fit_prox.optimization_result.result[:iterations] < 1000
+    @test n_iterations(fit_prox.optimization_result) < 1000
     @test all(solution(fit_prox)[16:20] .< solution(sem_fit)[16:20])
     @test StructuralEquationModels.minimum(fit_prox) -
           StructuralEquationModels.minimum(sem_fit) < 0.03