From 459fd51a1d41695a2940dd5cc6dd4e7dbac72b89 Mon Sep 17 00:00:00 2001 From: AshtonSBradley Date: Sat, 23 May 2026 14:46:33 +1200 Subject: [PATCH 1/8] Update reference image tests for v3 --- reference/compare.jl | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/reference/compare.jl b/reference/compare.jl index 9a6ce92..aa75c8b 100644 --- a/reference/compare.jl +++ b/reference/compare.jl @@ -7,7 +7,7 @@ using TOML include("references.jl") -function download_refimages(tag = "refimages-v2") +function download_refimages(tag = "refimages-v3") url = "https://github.com/Kolaru/MathTeXEngine.jl/releases/download/$tag/reference_images.tar" images_tar = joinpath(@__DIR__, "reference_images.tar") images = joinpath(@__DIR__, "reference_images") @@ -42,7 +42,7 @@ end @info "Downloading reference images" reference_images = download_refimages() - @inof "Reference images downloaded in $reference_images" + @info "Reference images downloaded in $reference_images" @info "Generating comparison images" comparison_images = joinpath(@__DIR__, "comparison_images") @@ -73,7 +73,9 @@ end axcurrent = fig[2, 2] = Axis(fig, aspect=DataAspect()) image!(axcurrent, rotr90(img)) - save(joinpath(path, image_path), fig) + comparison_path = joinpath(path, image_path) + mkpath(dirname(comparison_path)) + save(comparison_path, fig) end @test img == refimg end From 51686de9fcf575d4a8b6ffef5f7be351ee9fbe46 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Richard?= Date: Sat, 23 May 2026 13:29:58 +0200 Subject: [PATCH 2/8] use refimages-v2 --- reference/compare.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference/compare.jl b/reference/compare.jl index aa75c8b..cfc01ab 100644 --- a/reference/compare.jl +++ b/reference/compare.jl @@ -7,7 +7,7 @@ using TOML include("references.jl") -function download_refimages(tag = "refimages-v3") +function download_refimages(tag = "refimages-v2") url = "https://github.com/Kolaru/MathTeXEngine.jl/releases/download/$tag/reference_images.tar" images_tar = joinpath(@__DIR__, "reference_images.tar") images = joinpath(@__DIR__, "reference_images") From d48115f96b75ee00d346be3e12ff917ff1e16101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Richard?= Date: Sat, 23 May 2026 14:08:50 +0200 Subject: [PATCH 3/8] Upload the correct reference images --- reference/compare.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reference/compare.jl b/reference/compare.jl index cfc01ab..aa75c8b 100644 --- a/reference/compare.jl +++ b/reference/compare.jl @@ -7,7 +7,7 @@ using TOML include("references.jl") -function download_refimages(tag = "refimages-v2") +function download_refimages(tag = "refimages-v3") url = "https://github.com/Kolaru/MathTeXEngine.jl/releases/download/$tag/reference_images.tar" images_tar = joinpath(@__DIR__, "reference_images.tar") images = joinpath(@__DIR__, "reference_images") From f81521a199873a5f52d0ce1c238d8858c271aae5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Richard?= Date: Sat, 23 May 2026 18:28:05 +0200 Subject: [PATCH 4/8] Better image saved when reference test fails --- reference/Project.toml | 1 + reference/compare.jl | 38 +++++++++++++++++++++++++++++--------- reference/references.jl | 2 +- 3 files changed, 31 insertions(+), 10 deletions(-) diff --git a/reference/Project.toml b/reference/Project.toml index af7ebde..ddf33a1 100644 --- a/reference/Project.toml +++ b/reference/Project.toml @@ -1,5 +1,6 @@ [deps] CairoMakie = "13f3f980-e62b-5c42-98c6-ff1f3baf88f0" +Colors = "5ae59095-9a9b-59fe-a467-6f913c188581" FileIO = "5789e2e9-d7fb-5bc7-8068-2c6fae9b9549" LaTeXStrings = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f" MathTeXEngine = "0a4f8689-d25c-4efe-a92b-7142dfc1aa53" diff --git a/reference/compare.jl b/reference/compare.jl index aa75c8b..c8f5176 100644 --- a/reference/compare.jl +++ b/reference/compare.jl @@ -59,19 +59,39 @@ end for image_path in image_paths(REFERENCES) @info "Comparing $image_path" - refimg = load(joinpath(reference_images, image_path)) - img = load(joinpath(comparison_images, image_path)) + refimg = rotr90(load(joinpath(reference_images, image_path))) + img = rotr90(load(joinpath(comparison_images, image_path))) if img != refimg @info "Saving the reference comparison for '$image_path'." - fig = Figure() - fig[1, 1] = Label(fig, "Reference", tellwidth=false) - axref = fig[2, 1] = Axis(fig, aspect=DataAspect()) - image!(axref, rotr90(refimg)) + fig = Figure(size = (3*size(img, 1), size(img, 2))) + Label(fig[1, 1], "Reference $(size(refimg))", tellwidth=false) + axref = Axis(fig[2, 1], aspect=DataAspect()) + hidedecorations!(axref) + image!(axref, refimg) - fig[1, 2] = Label(fig, "Current", tellwidth=false) - axcurrent = fig[2, 2] = Axis(fig, aspect=DataAspect()) - image!(axcurrent, rotr90(img)) + Label(fig[1, 2], "Current $(size(img))", tellwidth=false) + axcurrent = Axis(fig[2, 2], aspect=DataAspect()) + hidedecorations!(axcurrent) + image!(axcurrent, img) + + if size(img) == size(refimg) + overlayed = fill(colorant"white", size(img)) + for i in axes(img, 1) + for j in axes(img, 2) + overlayed[i, j] = RGB( + convert(Gray, img[i, j]), + convert(Gray, refimg[i, j]), + 1.0 + ) + end + end + + Label(fig[1, 3], "Overlayed (reference in teal, current in magenta)", tellwidth=false) + axoverlayed = Axis(fig[2, 3], aspect=DataAspect()) + hidedecorations!(axoverlayed) + image!(axoverlayed, overlayed) + end comparison_path = joinpath(path, image_path) mkpath(dirname(comparison_path)) diff --git a/reference/references.jl b/reference/references.jl index 0a0a018..585be57 100644 --- a/reference/references.jl +++ b/reference/references.jl @@ -20,7 +20,7 @@ const SUPPORTED_FONTS = [ ] function generate(destination_folder, references = REFERENCES, fonts = SUPPORTED_FONTS) - @info "Generating reference images in folder $destination_folder" + @info "Generating images in folder $destination_folder" path = mkpath(destination_folder) failures = Dict() From 5f540d5d1dee07bd9625e8072d15a3d7c9afce62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Richard?= Date: Sat, 23 May 2026 18:37:52 +0200 Subject: [PATCH 5/8] Add missing import --- reference/compare.jl | 1 + 1 file changed, 1 insertion(+) diff --git a/reference/compare.jl b/reference/compare.jl index c8f5176..920cee6 100644 --- a/reference/compare.jl +++ b/reference/compare.jl @@ -1,4 +1,5 @@ using CairoMakie +using Colors using FileIO using MathTeXEngine using Tar From 3f794312436bd0e19ba932009abb75fa3046219e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Richard?= Date: Sat, 23 May 2026 20:56:37 +0200 Subject: [PATCH 6/8] Allow up to a 10% error on the worst pixel in reference test --- reference/compare.jl | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/reference/compare.jl b/reference/compare.jl index 920cee6..6c2d203 100644 --- a/reference/compare.jl +++ b/reference/compare.jl @@ -38,6 +38,15 @@ function image_paths(references) return paths end +function image_distance(img1, img2) + diffs = ( + abs.(red.(img1) - red.(img2)) + + abs.(green.(img1) - green.(img2)) + + abs.(blue.(img1) - blue.(img2)) + ) / 3 + return maximum(diffs) +end + @testset "Reference images" begin @info "Reference test started" @@ -63,7 +72,7 @@ end refimg = rotr90(load(joinpath(reference_images, image_path))) img = rotr90(load(joinpath(comparison_images, image_path))) - if img != refimg + if image_distance(img, refimg) > 0.1 @info "Saving the reference comparison for '$image_path'." fig = Figure(size = (3*size(img, 1), size(img, 2))) Label(fig[1, 1], "Reference $(size(refimg))", tellwidth=false) From 691cb4cd61a038393da5e5682c6bf02ed2575e02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Richard?= Date: Sat, 23 May 2026 23:55:47 +0200 Subject: [PATCH 7/8] Better difference image to try to see something in CI --- reference/compare.jl | 35 +++++++++++------------------------ 1 file changed, 11 insertions(+), 24 deletions(-) diff --git a/reference/compare.jl b/reference/compare.jl index 6c2d203..ce29310 100644 --- a/reference/compare.jl +++ b/reference/compare.jl @@ -39,14 +39,11 @@ function image_paths(references) end function image_distance(img1, img2) - diffs = ( - abs.(red.(img1) - red.(img2)) + - abs.(green.(img1) - green.(img2)) + - abs.(blue.(img1) - blue.(img2)) - ) / 3 - return maximum(diffs) + return maximum(image_difference(img1, img2)) ./ 100 end +image_difference(img1, img2) = colordiff.(img1, img2) + @testset "Reference images" begin @info "Reference test started" @@ -73,38 +70,28 @@ end img = rotr90(load(joinpath(comparison_images, image_path))) if image_distance(img, refimg) > 0.1 - @info "Saving the reference comparison for '$image_path'." + @info "Saving the reference comparison for '$image_path' (image difference $(image_distance(img, refimg)))" fig = Figure(size = (3*size(img, 1), size(img, 2))) Label(fig[1, 1], "Reference $(size(refimg))", tellwidth=false) - axref = Axis(fig[2, 1], aspect=DataAspect()) + axref = Axis(fig[2, 1], aspect = DataAspect()) hidedecorations!(axref) image!(axref, refimg) Label(fig[1, 2], "Current $(size(img))", tellwidth=false) - axcurrent = Axis(fig[2, 2], aspect=DataAspect()) + axcurrent = Axis(fig[2, 2], aspect = DataAspect()) hidedecorations!(axcurrent) image!(axcurrent, img) if size(img) == size(refimg) - overlayed = fill(colorant"white", size(img)) - for i in axes(img, 1) - for j in axes(img, 2) - overlayed[i, j] = RGB( - convert(Gray, img[i, j]), - convert(Gray, refimg[i, j]), - 1.0 - ) - end - end - - Label(fig[1, 3], "Overlayed (reference in teal, current in magenta)", tellwidth=false) - axoverlayed = Axis(fig[2, 3], aspect=DataAspect()) - hidedecorations!(axoverlayed) - image!(axoverlayed, overlayed) + Label(fig[1, 3], "Image difference (maximum difference $(float(image_distance(img, refimg))))", tellwidth=false) + axdiff = Axis(fig[2, 3], aspect = DataAspect()) + hidedecorations!(axdiff) + image!(axdiff, 1 .- image_difference(img, refimg)) end comparison_path = joinpath(path, image_path) mkpath(dirname(comparison_path)) + @info "Saving the reference plot at $comparison_path" save(comparison_path, fig) end @test img == refimg From 61e843453af99f9b281e8bc1cd9783363385018b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Beno=C3=AEt=20Richard?= Date: Sun, 24 May 2026 00:55:01 +0200 Subject: [PATCH 8/8] Count the number of bad pixels to judge reference tests --- reference/compare.jl | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/reference/compare.jl b/reference/compare.jl index ce29310..cebdecb 100644 --- a/reference/compare.jl +++ b/reference/compare.jl @@ -38,11 +38,8 @@ function image_paths(references) return paths end -function image_distance(img1, img2) - return maximum(image_difference(img1, img2)) ./ 100 -end - -image_difference(img1, img2) = colordiff.(img1, img2) +n_bad_pixels(img1, img2) = count(>(0.5), image_difference(img1, img2)) +image_difference(img1, img2) = colordiff.(img1, img2) ./ 100 @testset "Reference images" begin @info "Reference test started" @@ -69,8 +66,8 @@ image_difference(img1, img2) = colordiff.(img1, img2) refimg = rotr90(load(joinpath(reference_images, image_path))) img = rotr90(load(joinpath(comparison_images, image_path))) - if image_distance(img, refimg) > 0.1 - @info "Saving the reference comparison for '$image_path' (image difference $(image_distance(img, refimg)))" + if (failed = n_bad_pixels(img, refimg) >= 10) + @info "Saving the reference comparison for '$image_path' (image difference $(n_bad_pixels(img, refimg)))" fig = Figure(size = (3*size(img, 1), size(img, 2))) Label(fig[1, 1], "Reference $(size(refimg))", tellwidth=false) axref = Axis(fig[2, 1], aspect = DataAspect()) @@ -83,10 +80,13 @@ image_difference(img1, img2) = colordiff.(img1, img2) image!(axcurrent, img) if size(img) == size(refimg) - Label(fig[1, 3], "Image difference (maximum difference $(float(image_distance(img, refimg))))", tellwidth=false) + Label(fig[1, 3], "Difference ($(n_bad_pixels(img, refimg)) bad pixels). Blue: reference. Orange: current.", tellwidth=false) axdiff = Axis(fig[2, 3], aspect = DataAspect()) hidedecorations!(axdiff) - image!(axdiff, 1 .- image_difference(img, refimg)) + diff = Gray.(img) - Gray.(refimg) + overlay = RGB.(Gray.(refimg), Gray.(img)./2 .+ Gray.(refimg)./2, Gray.(img)) + + image!(axdiff, overlay) end comparison_path = joinpath(path, image_path) @@ -94,6 +94,6 @@ image_difference(img1, img2) = colordiff.(img1, img2) @info "Saving the reference plot at $comparison_path" save(comparison_path, fig) end - @test img == refimg + @test !failed end end