diff --git a/.coveragerc b/.coveragerc index 5eda44064..912faae6c 100644 --- a/.coveragerc +++ b/.coveragerc @@ -8,3 +8,4 @@ omit = */test_* */__init__.py */run* */out3Plot/out* + */com9MoTVoellmy/_buildMoTVoellmy.py diff --git a/.github/workflows/buildAndUploadPyPi.yml b/.github/workflows/buildAndUploadPyPi.yml index 291bbe739..7b9bc757c 100644 --- a/.github/workflows/buildAndUploadPyPi.yml +++ b/.github/workflows/buildAndUploadPyPi.yml @@ -1,5 +1,6 @@ # Deploy to pypi for all plattforms + name: Build and upload to PyPI on: @@ -29,6 +30,14 @@ jobs: env: CIBW_PLATFORM: ${{ matrix.platform || 'auto' }} # CIBW_BUILD_FRONTEND: "build[uv]" + CIBW_BEFORE_BUILD_LINUX: > + python {package}/avaframe/com9MoTVoellmy/_buildMoTVoellmy.py + CIBW_BEFORE_BUILD_WINDOWS: > + choco install mingw -y && + python {package}/avaframe/com9MoTVoellmy/_buildMoTVoellmy.py + CIBW_BEFORE_BUILD_MACOS: > + python {package}/avaframe/com9MoTVoellmy/_buildMoTVoellmy.py + CIBW_ENVIRONMENT: GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }} - uses: actions/upload-artifact@v7 with: @@ -51,12 +60,14 @@ jobs: run: | python -m pip install --upgrade pip pip install numpy cython build -# - name: Compile cython code -# run: | -# python setup.py build_ext --inplace -# - name: Install avaframe -# run: | -# pip install . + - name: Compile MoT-Voellmy binaries + run: | + python avaframe/com9MoTVoellmy/_buildMoTVoellmy.py + sudo apt-get install -y gcc-mingw-w64-x86-64 + C_FILE=$(ls avaframe/com9MoTVoellmy/MoT-Voellmy.*.c 2>/dev/null | head -1) + if [ -n "$C_FILE" ]; then + x86_64-w64-mingw32-gcc -Wall -pedantic -o avaframe/com9MoTVoellmy/MoT-Voellmy_win.exe "$C_FILE" -lm + fi - name: create source distribution run: python -m build # - name: Build sdist diff --git a/.github/workflows/runTestSinglePython.yml b/.github/workflows/runTestSinglePython.yml index 5d60600ac..38d676baa 100644 --- a/.github/workflows/runTestSinglePython.yml +++ b/.github/workflows/runTestSinglePython.yml @@ -28,6 +28,7 @@ jobs: cache: false - name: Pixi run pytest run: | + pixi run compilemot pixi run pytest -ra --cov --cov-report=xml --cov-report lcov:cov.info --cov-config=.coveragerc - uses: qltysh/qlty-action/coverage@main with: diff --git a/.gitignore b/.gitignore index c4189b393..af359298f 100644 --- a/.gitignore +++ b/.gitignore @@ -44,6 +44,11 @@ avaframe/com1DFA/*.so /.pixi/ +# MoT-Voellmy compiled binaries and downloaded sources +avaframe/com9MoTVoellmy/MoT-Voellmy_linux.exe +avaframe/com9MoTVoellmy/MoT-Voellmy_win.exe +avaframe/com9MoTVoellmy/MoT-Voellmy.*.c + # auxilliary geodata .xml files (created by GIS software packages) *.aux.xml diff --git a/MANIFEST.in b/MANIFEST.in index 4f33aed1c..6016d9108 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -3,6 +3,7 @@ include avaframe/version.py include avaframe/in3Utils/logging.conf include avaframe/com1DFA/*.pyx include avaframe/com9MoTVoellmy/MoT-Voellmy*.exe +include avaframe/com9MoTVoellmy/_buildMoTVoellmy.py recursive-include avaframe *Cfg.ini global-exclude local_*Cfg.ini prune benchmarks diff --git a/avaframe/com9MoTVoellmy/MoT-Voellmy_linux.exe b/avaframe/com9MoTVoellmy/MoT-Voellmy_linux.exe deleted file mode 100755 index 859db791d..000000000 Binary files a/avaframe/com9MoTVoellmy/MoT-Voellmy_linux.exe and /dev/null differ diff --git a/avaframe/com9MoTVoellmy/MoT-Voellmy_win.exe b/avaframe/com9MoTVoellmy/MoT-Voellmy_win.exe deleted file mode 100644 index 8b2a6e11f..000000000 Binary files a/avaframe/com9MoTVoellmy/MoT-Voellmy_win.exe and /dev/null differ diff --git a/avaframe/com9MoTVoellmy/_buildMoTVoellmy.py b/avaframe/com9MoTVoellmy/_buildMoTVoellmy.py new file mode 100644 index 000000000..d2a273d6f --- /dev/null +++ b/avaframe/com9MoTVoellmy/_buildMoTVoellmy.py @@ -0,0 +1,191 @@ +"""Build MoT-Voellmy binary for the current platform. + +Usage: + python _buildMoTVoellmy.py [path/to/source.c] + +If a path is provided, that C source file is used. Otherwise, the script +discovers and downloads the latest source from the upstream GitHub repo. + +On success, the compiled binary is written to this directory alongside the script. +On failure, the script exits with a non-zero code (for pixi task / CI). +When imported from setup.py, it returns a status instead of exiting. +""" + +import os +import platform +import re +import shutil +import subprocess +import sys +import urllib.request + +upstreamRepo = "norwegian-geotechnical-institute/MoT-Voellmy" +upstreamApi = f"https://api.github.com/repos/{upstreamRepo}" +upstreamContents = f"{upstreamApi}/contents/" +sourcePattern = re.compile(r"^MoT-Voellmy\..*\.c$") + +outputDir = os.path.dirname(os.path.abspath(__file__)) + + +def _getReleaseTag(): + """Return the tag name of the latest GitHub release, or None.""" + headers = {} + token = os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + + try: + url = f"{upstreamApi}/releases/latest" + req = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(req, timeout=30) as resp: + import json + + data = json.loads(resp.read().decode()) + tag = data.get("tag_name") + if tag: + print(f"Latest upstream release: {tag}") + return tag + except Exception as e: + print(f"Failed to query GitHub releases: {e}", file=sys.stderr) + return None + + +def _findGithubSource(): + """Query the latest GitHub release for the .c source file. + + Returns (download_url, filename) or None if not found. + """ + headers = {} + token = os.environ.get("GITHUB_TOKEN") + if token: + headers["Authorization"] = f"Bearer {token}" + + tag = _getReleaseTag() + ref = tag if tag else "main" + + try: + url = f"{upstreamContents}?ref={ref}" + req = urllib.request.Request(url, headers=headers) + with urllib.request.urlopen(req, timeout=30) as resp: + import json + + contents = json.loads(resp.read().decode()) + except Exception as e: + print(f"Failed to query GitHub API: {e}", file=sys.stderr) + return None + + for item in contents: + if item.get("type") != "file": + continue + name = item.get("name", "") + if sourcePattern.match(name): + url = item.get("download_url") + if url: + print(f"Found upstream source: {name}") + return url, name + + print("No MoT-Voellmy source file found in upstream repo", file=sys.stderr) + return None + + +def _downloadSource(url, destPath): + """Download a file from url to destPath.""" + print(f"Downloading {url} ...") + try: + with urllib.request.urlopen(url, timeout=60) as resp: + with open(destPath, "wb") as f: + shutil.copyfileobj(resp, f) + print(f"Downloaded to {destPath}") + return True + except Exception as e: + print(f"Download failed: {e}", file=sys.stderr) + return False + + +def _compile(sourcePath): + """Compile sourcePath for the current platform. + + Returns True on success, False on failure. + """ + system = platform.system() + + if system == "Linux": + outName = "MoT-Voellmy_linux.exe" + cmd = ["gcc", "-Wall", "-pedantic", "-o", outName, sourcePath, "-lm"] + elif system == "Windows": + outName = "MoT-Voellmy_win.exe" + cmd = ["gcc", "-Wall", "-pedantic", "-o", outName, sourcePath, "-lm"] + elif system == "Darwin": + outName = "MoT-Voellmy_mac.exe" + cmd = ["gcc", "-Wall", "-pedantic", "-arch", "arm64", "-arch", "x86_64", + "-o", outName, sourcePath, "-lm"] + else: + print(f"Unknown platform: {system}", file=sys.stderr) + return False + + # Run from outputDir so the binary lands alongside the Python module + print(f"Compiling: {' '.join(cmd)}") + try: + result = subprocess.run(cmd, cwd=outputDir, capture_output=True, text=True) + if result.returncode != 0: + print(f"Compilation failed:\n{result.stderr}", file=sys.stderr) + return False + except FileNotFoundError: + print( + f"gcc not found. Install gcc (e.g. MinGW on Windows) or download " + f"the precompiled binary from https://github.com/norwegian-geotechnical-institute/" + f"MoT-Voellmy and copy it to the com9MoTVoellmy directory as {outName}.", + file=sys.stderr, + ) + return False + + # Make executable on Unix + outPath = os.path.join(outputDir, outName) + if system != "Windows": + os.chmod(outPath, 0o755) + + print(f"Compiled {outPath}") + return True + + +def buildMoTVoellmy(sourcePath=None): + """Compile MoT-Voellmy binary. + + Args: + sourcePath: Optional path to a local .c file. If None, downloads + the latest source from the upstream GitHub repo. + + Returns: + True if compilation succeeded, False otherwise. + """ + # 1. CLI argument + if sourcePath is not None: + if not os.path.isfile(sourcePath): + print(f"Source file not found: {sourcePath}", file=sys.stderr) + return False + return _compile(sourcePath) + + # 2. GitHub download + result = _findGithubSource() + if result is None: + return False + + url, filename = result + dest = os.path.join(outputDir, filename) + + # Use cached copy if available and download is a fallback + if not os.path.isfile(dest): + if not _downloadSource(url, dest): + return False + + return _compile(dest) + + +def main(): + source = sys.argv[1] if len(sys.argv) > 1 else None + success = buildMoTVoellmy(source) + sys.exit(0 if success else 1) + + +if __name__ == "__main__": + main() diff --git a/avaframe/com9MoTVoellmy/com9MoTVoellmy.py b/avaframe/com9MoTVoellmy/com9MoTVoellmy.py index b6f621174..032fe5cb2 100644 --- a/avaframe/com9MoTVoellmy/com9MoTVoellmy.py +++ b/avaframe/com9MoTVoellmy/com9MoTVoellmy.py @@ -173,9 +173,7 @@ def com9MoTVoellmyTask(rcfFile): if os.name == "nt": exeName = "MoT-Voellmy_win.exe" elif platform.system() == "Darwin": - message = "MoT-Voellmy does not support MacOS at the moment" - log.error(message) - raise OSError(message) + exeName = "./MoT-Voellmy_mac.exe" else: exeName = "./MoT-Voellmy_linux.exe" diff --git a/avaframe/com9MoTVoellmy/com9MoTVoellmyCfg.ini b/avaframe/com9MoTVoellmy/com9MoTVoellmyCfg.ini index 4b4aa50e8..0e233bcb7 100644 --- a/avaframe/com9MoTVoellmy/com9MoTVoellmyCfg.ini +++ b/avaframe/com9MoTVoellmy/com9MoTVoellmyCfg.ini @@ -6,7 +6,7 @@ modelType = dfa # list of simulations that shall be performed (null, ent, res, entres, available (use all available input data)) -simTypeList = res +simTypeList = available #+++++Release thickness++++ diff --git a/avaframe/tests/test_com9MoTVoellmy.py b/avaframe/tests/test_com9MoTVoellmy.py index 8d931876f..22d71a258 100644 --- a/avaframe/tests/test_com9MoTVoellmy.py +++ b/avaframe/tests/test_com9MoTVoellmy.py @@ -1,9 +1,15 @@ """Tests for com9MoTVoellmy module""" -import pytest +import configparser +import logging import pathlib +import shutil from unittest.mock import patch + +import pytest + from avaframe.com9MoTVoellmy import com9MoTVoellmy +from avaframe.in3Utils import cfgUtils def test_com9MoTVoellmyTask_windows(tmp_path): @@ -69,8 +75,8 @@ def test_com9MoTVoellmyTask_linux(tmp_path): assert result == command -def test_com9MoTVoellmyTask_macOS_raises_error(tmp_path): - """Test that com9MoTVoellmyTask raises OSError on macOS""" +def test_com9MoTVoellmyTask_macOS_uses_mac_binary(tmp_path): + """Test that com9MoTVoellmyTask uses the macOS binary on Darwin""" rcfFile = tmp_path / "test_config.rcf" rcfFile.write_text("test config") @@ -80,16 +86,14 @@ def test_com9MoTVoellmyTask_macOS_raises_error(tmp_path): patch("os.chdir"), patch("os.path.dirname"), patch("os.path.abspath"), + patch("avaframe.com9MoTVoellmy.com9MoTVoellmy.mT.runAndCheckMoT") as mock_run, ): - # Verify OSError is raised for macOS - with pytest.raises(OSError, match="MoT-Voellmy does not support MacOS"): - com9MoTVoellmy.com9MoTVoellmyTask(rcfFile) + command = com9MoTVoellmy.com9MoTVoellmyTask(rcfFile) + assert command[0] == "./MoT-Voellmy_mac.exe" def test_com9MoTVoellmyTask_verifyLogging(tmp_path, caplog): """Test that com9MoTVoellmyTask logs the simulation run""" - import logging - rcfFile = tmp_path / "test_config.rcf" rcfFile.write_text("test config") @@ -139,8 +143,6 @@ def test_com9MoTVoellmyTask_rcfFilePathHandling(tmp_path): def test_com9MoTVoellmyPostprocess_directoryCreation(tmp_path): """Test that postprocess creates necessary output directories""" - import configparser - avalancheDir = tmp_path / "avaTest" avalancheDir.mkdir() @@ -177,8 +179,6 @@ def test_com9MoTVoellmyPostprocess_directoryCreation(tmp_path): def test_com9MoTVoellmyPostprocess_filesCopied(tmp_path): """Test that postprocess copies all expected files""" - import configparser - avalancheDir = tmp_path / "avaTest" avalancheDir.mkdir() @@ -227,8 +227,6 @@ def test_com9MoTVoellmyPostprocess_filesCopied(tmp_path): def test_com9MoTVoellmyPostprocess_directoriesCopied(tmp_path): """Test that postprocess copies timestep directories""" - import configparser - avalancheDir = tmp_path / "avaTest" avalancheDir.mkdir() @@ -262,8 +260,6 @@ def test_com9MoTVoellmyPostprocess_directoriesCopied(tmp_path): def test_com9MoTVoellmyPostprocess_plotsGenerated(tmp_path): """Test that postprocess generates plots""" - import configparser - avalancheDir = tmp_path / "avaTest" avalancheDir.mkdir() @@ -294,8 +290,6 @@ def test_com9MoTVoellmyPostprocess_plotsGenerated(tmp_path): def test_com9MoTVoellmyPostprocess_multipleSimulations(tmp_path): """Test postprocess with multiple simulations""" - import configparser - avalancheDir = tmp_path / "avaTest" avalancheDir.mkdir() @@ -329,3 +323,51 @@ def test_com9MoTVoellmyPostprocess_multipleSimulations(tmp_path): # Verify copyMoTDirs called correct number of times: 2 dirs * 3 sims = 6 assert mockCopyDirs.call_count == 6 + + +def test_com9MoTVoellmyIntegrationRun(tmp_path, caplog): + """Integration test: run com9MoTVoellmy with avaKot (null) and verify output files""" + testDir = pathlib.Path(__file__).parents[0] + avaSrc = testDir / ".." / "data" / "avaKot" + avaDir = tmp_path / "avaKot" + shutil.copytree(avaSrc / "Inputs", avaDir / "Inputs") + + binPath = pathlib.Path(com9MoTVoellmy.__file__).parent / "MoT-Voellmy_linux.exe" + if not binPath.exists(): + pytest.skip("MoT-Voellmy binary not found -- run `pixi run compilemot`") + + cfgMain = configparser.ConfigParser() + cfgMain["MAIN"] = {"avalancheDir": str(avaDir), "nCPU": "auto", "CPUPercent": "90"} + cfgMain["FLAGS"] = {"showPlot": "False", "savePlot": "True"} + + cfgCom9 = cfgUtils.getModuleConfig(com9MoTVoellmy) + cfgCom9["GENERAL"]["simTypeList"] = "null" + + with caplog.at_level(logging.INFO): + com9MoTVoellmy.com9MoTVoellmyMain(cfgMain, cfgInfo=cfgCom9) + + # Verify no errors during run (only main process log is captured) + errors = [r for r in caplog.records if r.levelno >= logging.ERROR] + assert len(errors) == 0, f"Errors in log: {[e.message for e in errors]}" + + # Verify parallel computation ran (logged in main process) + assert any( + "Overall" in r.message and "computation took" in r.message + for r in caplog.records + ), "No parallel computation summary found" + + # Verify log says simulations were performed + assert any( + "The following simulations will be performed" in r.message + for r in caplog.records + ) + + # Verify output files exist + peakDir = avaDir / "Outputs" / "com9MoTVoellmy" / "peakFiles" + assert peakDir.exists() + ascFiles = sorted(peakDir.glob("*.asc")) + assert len(ascFiles) >= 3, f"Expected >= 3 asc files, got {len(ascFiles)}: {ascFiles}" + + cfgDir = avaDir / "Outputs" / "com9MoTVoellmy" / "configurationFiles" + rcfFiles = sorted(cfgDir.glob("*.rcf")) + assert len(rcfFiles) >= 1, f"No RCF files found in {cfgDir}" diff --git a/docs/moduleCom9MoTVoellmy.rst b/docs/moduleCom9MoTVoellmy.rst index 5533af9f5..5e08610ef 100644 --- a/docs/moduleCom9MoTVoellmy.rst +++ b/docs/moduleCom9MoTVoellmy.rst @@ -73,6 +73,32 @@ plots, and configuration files. MoT Voellmy via the script variant ---------------------------------- +Binary +^^^^^^ + +The MoT-Voellmy binary is compiled automatically during ``pip install`` and in released +wheelsm, no manual steps needed. The build process fetches the latest C source from the +`upstream repository `_ +and compiles it with ``gcc``. + +To compile manually (e.g. after updating the upstream source): + + .. code-block:: bash + + pixi run compilemot + +To compile from a local C source file instead of downloading from GitHub: + + .. code-block:: bash + + pixi run compilemot /path/to/MoT-Voellmy.c + +If ``gcc`` is not available, download the precompiled binary from the upstream repository +and copy it to ``avaframe/com9MoTVoellmy/``. On Windows, install MinGW to use ``gcc``. + +The expected binary names are ``MoT-Voellmy_linux.exe``, ``MoT-Voellmy_win.exe``, or +``MoT-Voellmy_mac.exe`` depending on your platform. + To run ^^^^^ diff --git a/pyproject.toml b/pyproject.toml index c32effd63..458d80a3f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -125,6 +125,7 @@ avaframe = "*" build = "python setup.py build_ext --inplace" clean = "python -c \"import pathlib; [f.unlink() for f in pathlib.Path('avaframe').rglob('*.so')]; [f.unlink() for f in pathlib.Path('avaframe').rglob('*.c')]\"" rebuild = { depends-on = ["clean", "build"] } +compilemot = "python avaframe/com9MoTVoellmy/_buildMoTVoellmy.py" #Feature qgis [tool.pixi.feature.qgis.dependencies] diff --git a/setup.py b/setup.py index 195bf1d42..6c0456cfd 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,29 @@ +import sys +import os + from setuptools import setup, Extension import numpy from Cython.Build import cythonize +# Compile MoT-Voellmy binary at build time (best-effort). +# Insert the module directory into sys.path so we can import it. +_mot_dir = os.path.join(os.path.dirname(__file__), "avaframe", "com9MoTVoellmy") +_mot_script = os.path.join(_mot_dir, "_buildMoTVoellmy.py") +if os.path.isfile(_mot_script): + sys.path.insert(0, _mot_dir) + try: + from _buildMoTVoellmy import buildMoTVoellmy + + if not buildMoTVoellmy(): + print("Warning: MoT-Voellmy compilation failed. " + "com9MoTVoellmy will not be functional.") + except ImportError: + print("Warning: Could not import MoT-Voellmy build script. " + "com9MoTVoellmy will not be functional.") + finally: + sys.path.pop(0) + # Define extension modules conditionally # ext_modules = []