Merge branch 'test-fixes' into 'develop'

fix image comparison and add reference and test images as CI artifacts

See merge request psyplot/psy-simple!39

Author: Chilipp

.gitlab-ci.yml

@@ -69,6 +69,8 @@ test:
     name: pipfile
     paths:
       - "ci/matrix/${SCENARIO}/*"
+      - "tests/reference_figures/*.png"
+      - "tests/test_figures/*.png"
     expire_in: 30 days
   coverage: '/(?i)total.*? (100(?:\.0+)?\%|[1-9]?\d(?:\.\d+)?\%)$/'
 

tests/_base_testing.py

@@ -1,6 +1,5 @@
 """Base test setup for the psy-simple test suite."""
 
-
 # SPDX-FileCopyrightText: 2021-2024 Helmholtz-Zentrum Hereon
 # SPDX-FileCopyrightText: 2020-2021 Helmholtz-Zentrum Geesthacht
 # SPDX-FileCopyrightText: 2016-2024 University of Lausanne
@@ -11,6 +10,7 @@
 import os
 import subprocess as spr
 import sys
+import time
 from unittest import TestCase
 
 import numpy as np
@@ -97,11 +97,21 @@ def compare_figures(self, fname, tol=5, **kwargs):
         from matplotlib.testing.compare import compare_images
 
         plt.savefig(os.path.join(self.odir, fname), **kwargs)
-        results = compare_images(
-            os.path.join(ref_dir, fname),
-            os.path.join(self.odir, fname),
-            tol=tol,
-        )
+        try:
+            results = compare_images(
+                os.path.join(ref_dir, fname),
+                os.path.join(self.odir, fname),
+                tol=tol,
+            )
+        except Exception:
+            # output file might be empty because not yet written, so just try
+            # again
+            time.sleep(3)
+            results = compare_images(
+                os.path.join(ref_dir, fname),
+                os.path.join(self.odir, fname),
+                tol=tol,
+            )
         self.assertIsNone(results, msg=results)
 
     def assertAlmostArrayEqual(