SemOptimizerResult: optim wrapper

Author: Alexey Stukalov

ext/SEMNLOptExt/NLopt.jl

@@ -128,7 +128,9 @@ local_options(optimizer::SemOptimizerNLopt) = optimizer.local_options
 equality_constraints(optimizer::SemOptimizerNLopt) = optimizer.equality_constraints
 inequality_constraints(optimizer::SemOptimizerNLopt) = optimizer.inequality_constraints
 
-struct NLoptResult
+# wrapper for the NLopt optimization result
+struct NLoptResult <: SEM.SemOptimizerResult{SemOptimizerNLopt}
+    optimizer::SemOptimizerNLopt
     result::Any
     problem::Any
 end
@@ -137,27 +139,15 @@ SEM.algorithm_name(res::NLoptResult) = res.problem.algorithm
 SEM.n_iterations(res::NLoptResult) = res.problem.numevals
 SEM.convergence(res::NLoptResult) = res.result[3]
 
-# construct SemFit from fitted NLopt object
-function SemFit_NLopt(optimization_result, model::AbstractSem, start_val, optim, opt)
-    return SemFit(
-        optimization_result[1],
-        optimization_result[2],
-        start_val,
-        model,
-        optim,
-        NLoptResult(optimization_result, opt),
-    )
-end
-
 # fit method
 function SEM.fit(
     optim::SemOptimizerNLopt,
     model::AbstractSem,
     start_params::AbstractVector;
     kwargs...,
 )
-    opt = construct_NLopt(optim.algorithm, optim.options, nparams(model))
-    opt.min_objective =
+    problem = NLopt_problem(optim.algorithm, optim.options, nparams(model))
+    problem.min_objective =
         (par, G) -> SEM.evaluate!(
             zero(eltype(par)),
             !isnothing(G) && !isempty(G) ? G : nothing,
@@ -166,36 +156,42 @@ function SEM.fit(
             par,
         )
     for (f, tol) in optim.inequality_constraints
-        inequality_constraint!(opt, f, tol)
+        inequality_constraint!(problem, f, tol)
     end
     for (f, tol) in optim.equality_constraints
-        equality_constraint!(opt, f, tol)
+        equality_constraint!(problem, f, tol)
     end
 
     if !isnothing(optim.local_algorithm)
-        opt_local =
-            construct_NLopt(optim.local_algorithm, optim.local_options, nparams(model))
-        opt.local_optimizer = opt_local
+        problem.local_optimizer =
+            NLopt_problem(optim.local_algorithm, optim.local_options, nparams(model))
     end
 
     # fit
-    result = NLopt.optimize(opt, start_params)
+    result = NLopt.optimize(problem, start_params)
 
-    return SemFit_NLopt(result, model, start_params, optim, opt)
+    return SemFit(
+        result[1], # minimum
+        result[2], # optimal params
+        start_val,
+        model,
+        NLoptResult(optim, result, problem),
+    )
 end
 
 ############################################################################################
 ### additional functions
 ############################################################################################
 
-function construct_NLopt(algorithm, options, npar)
-    opt = Opt(algorithm, npar)
+# construct NLopt.jl problem
+function NLopt_problem(algorithm, options, npar)
+    problem = Opt(algorithm, npar)
 
     for (key, val) in pairs(options)
-        setproperty!(opt, key, val)
+        setproperty!(problem, key, val)
     end
 
-    return opt
+    return problem
 end
 
 ############################################################################################

ext/SEMProximalOptExt/ProximalAlgorithms.jl

@@ -47,8 +47,10 @@ SEM.update_observed(optimizer::SemOptimizerProximal, observed::SemObserved; kwar
 ### Model fitting
 ############################################################################
 
-mutable struct ProximalResult
-    result::Any
+# wrapper for the Proximal optimization result
+struct ProximalResult{O <: SemOptimizer{:Proximal}} <: SEM.SemOptimizerResult{O}
+    optimizer::O
+    n_iterations::Int
 end
 
 ## connect to ProximalAlgorithms.jl
@@ -65,49 +67,39 @@ function SEM.fit(
     kwargs...,
 )
     if isnothing(optim.operator_h)
-        solution, iterations =
+        solution, niterations =
             optim.algorithm(x0 = start_params, f = model, g = optim.operator_g)
     else
-        solution, iterations = optim.algorithm(
+        solution, niterations = optim.algorithm(
             x0 = start_params,
             f = model,
             g = optim.operator_g,
             h = optim.operator_h,
         )
     end
 
-    minimum = objective!(model, solution)
-
-    optimization_result = Dict(
-        :minimum => minimum,
-        :iterations => iterations,
-        :algorithm => optim.algorithm,
-        :operator_g => optim.operator_g,
-    )
-
-    isnothing(optim.operator_h) ||
-        push!(optimization_result, :operator_h => optim.operator_h)
-
     return SemFit(
-        minimum,
+        objective!(model, solution), # minimum
         solution,
         start_params,
         model,
-        optim,
-        ProximalResult(optimization_result),
+        ProximalResult(optim, niterations),
     )
 end
 
 ############################################################################################
 ### additional methods
 ############################################################################################
 
-SEM.algorithm_name(res::ProximalResult) = SEM.algorithm_name(res.result[:algorithm])
-SEM.algorithm_name(::ProximalAlgorithms.IterativeAlgorithm{I,H,S,D,K}) where 
-    {I, H, S, D, K} = nameof(I)
+SEM.algorithm_name(res::ProximalResult) = SEM.algorithm_name(res.optimizer.algorithm)
+SEM.algorithm_name(
+    ::ProximalAlgorithms.IterativeAlgorithm{I, H, S, D, K},
+) where {I, H, S, D, K} = nameof(I)
 
-SEM.convergence(::ProximalResult) = "No standard convergence criteria for proximal \n algorithms available."
-SEM.n_iterations(res::ProximalResult) = res.result[:iterations]
+SEM.convergence(
+    ::ProximalResult,
+) = "No standard convergence criteria for proximal \n algorithms available."
+SEM.n_iterations(res::ProximalResult) = res.n_iterations
 
 ############################################################################################
 # pretty printing
@@ -119,10 +111,8 @@ function Base.show(io::IO, struct_inst::SemOptimizerProximal)
 end
 
 function Base.show(io::IO, result::ProximalResult)
-    print(io, "Minimum:          $(round(result.result[:minimum]; digits = 2)) \n")
-    print(io, "No. evaluations:  $(result.result[:iterations]) \n")
-    print(io, "Operator:         $(nameof(typeof(result.result[:operator_g]))) \n")
-    if haskey(result.result, :operator_h)
-        print(io, "Second Operator:  $(nameof(typeof(result.result[:operator_h]))) \n")
-    end
+    print(io, "No. evaluations:  $(result.n_iterations) \n")
+    print(io, "Operator:         $(nameof(typeof(result.optimizer.operator_g))) \n")
+    op_h = result.optimizer.operator_h
+    isnothing(op_h) || print(io, "Second Operator:  $(nameof(typeof(op_h))) \n")
 end

src/frontend/fit/SemFit.jl

@@ -17,12 +17,11 @@ Fitted structural equation model.
 - `n_iterations(::SemFit)` -> number of iterations
 - `convergence(::SemFit)` -> convergence properties
 """
-mutable struct SemFit{Mi, So, St, Mo, Op, O}
+mutable struct SemFit{Mi, So, St, Mo, O}
     minimum::Mi
     solution::So
     start_val::St
     model::Mo
-    optimizer::Op
     optimization_result::O
 end
 
@@ -64,6 +63,6 @@ optimization_result(sem_fit::SemFit) = sem_fit.optimization_result
 
 # optimizer properties
 algorithm_name(sem_fit::SemFit) = algorithm_name(sem_fit.optimization_result)
-optimizer_engine(sem_fit::SemFit) = optimizer_engine(sem_fit.optimizer)
+optimizer_engine(sem_fit::SemFit) = optimizer_engine(sem_fit.optimization_result)
 n_iterations(sem_fit::SemFit) = n_iterations(optimization_result(sem_fit))
 convergence(sem_fit::SemFit) = convergence(optimization_result(sem_fit))

src/optimizer/abstract.jl

@@ -79,6 +79,10 @@ For a list of available engines, call [`optimizer_engines`](@ref).
 """
 optimizer_engine_doc(engine) = doc(SemOptimizer_impltype(engine))
 
+optimizer(result::SemOptimizerResult) = result.optimizer
+
+optimizer_engine(result::SemOptimizerResult) = optimizer_engine(result.optimizer)
+
 """
     fit([optim::SemOptimizer], model::AbstractSem;
             [engine::Symbol], start_val = start_val, kwargs...)

src/optimizer/optim.jl

@@ -75,9 +75,15 @@ update_observed(optimizer::SemOptimizerOptim, observed::SemObserved; kwargs...)
 
 options(optimizer::SemOptimizerOptim) = optimizer.options
 
-algorithm_name(res::Optim.MultivariateOptimizationResults) = Optim.summary(res)
-n_iterations(res::Optim.MultivariateOptimizationResults) = Optim.iterations(res)
-convergence(res::Optim.MultivariateOptimizationResults) = Optim.converged(res)
+# wrapper for the Optim.jl result
+struct SemOptimResult{O <: SemOptimizerOptim} <: SemOptimizerResult{O}
+    optimizer::O
+    result::Optim.MultivariateOptimizationResults
+end
+
+algorithm_name(res::SemOptimResult) = Optim.summary(res.result)
+n_iterations(res::SemOptimResult) = Optim.iterations(res.result)
+convergence(res::SemOptimResult) = Optim.converged(res.result)
 
 function fit(
     optim::SemOptimizerOptim,
@@ -129,6 +135,6 @@ function fit(
         result.minimizer,
         start_params,
         model,
-        optim,
-        result)
+        SemOptimResult(optim, result),
+    )
 end

src/types.jl

@@ -81,6 +81,9 @@ Base.:*(x::SemWeight, y) = x.w * y
 
 abstract type SemOptimizer{E} end
 
+# wrapper around optimization result
+abstract type SemOptimizerResult{O <: SemOptimizer} end
+
 """
 Supertype of all objects that can serve as the observed field of a SEM.
 Pre-processes data and computes sufficient statistics for example.

test/examples/proximal/l0.jl

@@ -45,7 +45,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML)
 fit_prox = fit(model_prox, engine = :Proximal, operator_g = prox_operator)
 
 @testset "l0 | solution_unregularized" begin
-    @test fit_prox.optimization_result.result[:iterations] < 1000
+    @test n_iterations(fit_prox.optimization_result) < 1000
     @test maximum(abs.(solution(sem_fit) - solution(fit_prox))) < 0.002
 end
 
@@ -57,7 +57,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML)
 fit_prox = fit(model_prox, engine = :Proximal, operator_g = prox_operator)
 
 @testset "l0 | solution_regularized" begin
-    @test fit_prox.optimization_result.result[:iterations] < 1000
+    @test n_iterations(fit_prox.optimization_result) < 1000
     @test solution(fit_prox)[31] == 0.0
     @test abs(
         StructuralEquationModels.minimum(fit_prox) -

test/examples/proximal/lasso.jl

@@ -43,7 +43,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML)
 fit_prox = fit(model_prox, engine = :Proximal, operator_g = NormL1(λ))
 
 @testset "lasso | solution_unregularized" begin
-    @test fit_prox.optimization_result.result[:iterations] < 1000
+    @test n_iterations(fit_prox.optimization_result) < 1000
     @test maximum(abs.(solution(sem_fit) - solution(fit_prox))) < 0.002
 end
 
@@ -55,7 +55,7 @@ model_prox = Sem(specification = partable, data = dat, loss = SemML)
 fit_prox = fit(model_prox, engine = :Proximal, operator_g = NormL1(λ))
 
 @testset "lasso | solution_regularized" begin
-    @test fit_prox.optimization_result.result[:iterations] < 1000
+    @test n_iterations(fit_prox.optimization_result) < 1000
     @test all(solution(fit_prox)[16:20] .< solution(sem_fit)[16:20])
     @test StructuralEquationModels.minimum(fit_prox) -
           StructuralEquationModels.minimum(sem_fit) < 0.03