diff --git a/.github/workflows/test-windows.yml b/.github/workflows/test-windows.yml new file mode 100644 index 0000000..38bb662 --- /dev/null +++ b/.github/workflows/test-windows.yml @@ -0,0 +1,99 @@ +name: Windows tests + +on: + push: + branches: [main, "ci/**"] + pull_request: + workflow_dispatch: + +concurrency: + group: windows-tests-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +env: + FW_VERSION: "9.3.9.1" + FW_BUILD: "1439" + FW_CODE_DIR: "C:\\Program Files\\FieldWorks9" + FW_PROJECTS_DIR: "C:\\ProgramData\\SIL\\FieldWorks\\Projects" + FW_INSTALLER_DIR: "C:\\fw-ci\\installer" + FW_SAMPLE_DIR: "C:\\fw-ci\\samples" + +jobs: + test-nuget-overlay: + name: FieldWorks + NuGet overlay + runs-on: windows-latest + timeout-minutes: 120 + + steps: + - name: Checkout + uses: actions/checkout@v7 + + - name: Set up Python + uses: actions/setup-python@v7 + with: + python-version: "3.12" + architecture: x64 + + - name: Set up .NET SDK + uses: actions/setup-dotnet@v6 + with: + dotnet-version: "8.0.x" + + - name: Cache FieldWorks install + installer + sample + id: cache-fw + uses: actions/cache@v6 + with: + path: | + ${{ env.FW_CODE_DIR }} + ${{ env.FW_PROJECTS_DIR }} + ${{ env.FW_INSTALLER_DIR }} + ${{ env.FW_SAMPLE_DIR }} + C:\ProgramData\SIL\FieldWorks + C:\fw-ci\fw-code-dir.txt + key: fw-${{ env.FW_VERSION }}-${{ env.FW_BUILD }}-win-x64-v3 + + - name: Install FieldWorks (or refresh registry after cache restore) + shell: pwsh + run: | + ./scripts/ci/Install-FieldWorks.ps1 ` + -Version $env:FW_VERSION ` + -Build $env:FW_BUILD ` + -CodeDir $env:FW_CODE_DIR ` + -ProjectsDir $env:FW_PROJECTS_DIR ` + -DownloadDir $env:FW_INSTALLER_DIR + + - name: Upload FieldWorks install logs + if: failure() + uses: actions/upload-artifact@v7 + with: + name: fw-install-logs + path: | + C:\fw-ci\fw-install*.log + C:\fw-ci\fw-code-dir.txt + if-no-files-found: ignore + + - name: Set up sample FieldWorks project + shell: pwsh + run: | + ./scripts/ci/Setup-SampleProject.ps1 ` + -ProjectsDir $env:FW_PROJECTS_DIR ` + -DownloadDir $env:FW_SAMPLE_DIR + + - name: Install Python test dependencies + run: | + python -m pip install --upgrade pip + python -m pip install -r requirements.txt + python -m pip install -e . + + - name: Restore NuGet overlay and run tests + shell: cmd + env: + PYTHON: python + FLEXLIBS_FW_CODE_DIR: ${{ env.FW_CODE_DIR }} + FLEXLIBS_FW_PROJECTS_DIR: ${{ env.FW_PROJECTS_DIR }} + run: | + if exist C:\fw-ci\fw-code-dir.txt ( + set /p FLEXLIBS_FW_CODE_DIR==3.8,<3.14). +REM Override with: set PYTHON=py -3.11 +if not defined PYTHON ( + call :FindPython + if not defined PYTHON ( + echo No supported Python found ^(need 3.8-3.13^). Install one or set PYTHON=. + exit /b 1 + ) +) +echo Using PYTHON=%PYTHON% REM Check that the argument is a valid command, and do it. /I ignores case. FOR %%C IN ("Init" "Test" + "Test-nuget" "Clean" "Build" "Publish") DO ( @@ -17,40 +27,75 @@ FOR %%C IN ("Init" echo Usage: echo make init - Install the libraries for building echo make test - Run the unit tests + echo make test-nuget - Restore latest SIL NuGet packages and run tests with overlay echo make clean - Clean out build files echo make build - Build the project echo make publish - Publish the project to PyPI - goto :End + echo. + echo Extra args after test / test-nuget are passed to pytest, e.g.: + echo .\make.bat test-nuget -k "not CustomFields" -q + exit /b 1 :DoInit %PYTHON% -m pip install -r requirements.txt - goto :End + exit /b !ERRORLEVEL! :DoTest - %PYTHON% -m pytest - goto :End + call :CollectPytestArgs %* + %PYTHON% -m pytest !PYTEST_ARGS! + exit /b !ERRORLEVEL! + +:DoTest-nuget + call :CollectPytestArgs %* + powershell -NoProfile -ExecutionPolicy Bypass -File ".\tests\nuget-overlay\restore.ps1" + if errorlevel 1 exit /b 1 + set "FLEXLIBS_ASSEMBLY_DIR=%CD%\.fw-nuget-overlay" + %PYTHON% -m pytest !PYTEST_ARGS! + exit /b !ERRORLEVEL! :DoClean - rmdir /s /q ".\build" - rmdir /s /q ".\dist" - rmdir /s /q ".\flexlibs\docs" - goto :End + if exist ".\build" rmdir /s /q ".\build" + if exist ".\dist" rmdir /s /q ".\dist" + if exist ".\flexlibs\docs" rmdir /s /q ".\flexlibs\docs" + if exist ".\.fw-nuget-overlay" rmdir /s /q ".\.fw-nuget-overlay" + if exist ".\tests\nuget-overlay\bin" rmdir /s /q ".\tests\nuget-overlay\bin" + if exist ".\tests\nuget-overlay\obj" rmdir /s /q ".\tests\nuget-overlay\obj" + exit /b 0 :DoBuild @REM Build the Sphinx docs sphinx-build docs/sphinx flexlibs/docs/flexlibsAPI + if errorlevel 1 exit /b 1 @REM Build the wheel with setuptools %PYTHON% -m build -w -nx + if errorlevel 1 exit /b 1 @REM Check for package errors %PYTHON% -m twine check .\dist\* - goto :End + exit /b !ERRORLEVEL! :DoPublish echo Publishing wheel to PyPI %PYTHON% -m twine upload .\dist\flexlibs* - goto :End + exit /b !ERRORLEVEL! +:FindPython + for %%V in (3.13 3.12 3.11 3.10 3.9 3.8) do ( + py -%%V -c "import sys" >nul 2>&1 + if !ERRORLEVEL! EQU 0 ( + set "PYTHON=py -%%V" + goto :eof + ) + ) + goto :eof -:End +:CollectPytestArgs + REM Drop the make command (%1), keep the rest for pytest. + set "PYTEST_ARGS=" + shift +:CollectPytestArgsLoop + if "%~1"=="" goto :eof + set "PYTEST_ARGS=!PYTEST_ARGS! %1" + shift + goto CollectPytestArgsLoop diff --git a/scripts/ci/Install-FieldWorks.ps1 b/scripts/ci/Install-FieldWorks.ps1 new file mode 100644 index 0000000..773fde0 --- /dev/null +++ b/scripts/ci/Install-FieldWorks.ps1 @@ -0,0 +1,166 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Download and silently install FieldWorks for CI, or restore registry after a cache hit. +#> +[CmdletBinding()] +param( + [string]$Version = "9.3.9.1", + [string]$Build = "1439", + [string]$InstallerUrl = "", + [string]$CodeDir = "C:\Program Files\FieldWorks9", + [string]$ProjectsDir = "C:\ProgramData\SIL\FieldWorks\Projects", + [string]$DownloadDir = "C:\fw-ci\installer", + [string]$InstallLog = "C:\fw-ci\fw-install.log", + [string]$CodeDirOutFile = "C:\fw-ci\fw-code-dir.txt" +) + +$ErrorActionPreference = "Stop" + +function Ensure-Registry { + param([string]$CodeDir, [string]$ProjectsDir) + + $regPath = "HKLM:\SOFTWARE\SIL\FieldWorks\9" + if (-not (Test-Path $regPath)) { + New-Item -Path $regPath -Force | Out-Null + } + New-ItemProperty -Path $regPath -Name "RootCodeDir" -Value $CodeDir -PropertyType String -Force | Out-Null + New-ItemProperty -Path $regPath -Name "ProjectsDir" -Value $ProjectsDir -PropertyType String -Force | Out-Null + New-ItemProperty -Path $regPath -Name "Data_Directory" -Value (Split-Path $ProjectsDir -Parent) -PropertyType String -Force | Out-Null + Write-Host "Registry ready: RootCodeDir=$CodeDir ProjectsDir=$ProjectsDir" +} + +function Test-FieldWorksInstalled { + param([string]$CodeDir) + if (-not $CodeDir) { return $false } + return (Test-Path (Join-Path $CodeDir "FieldWorks.exe")) -and (Test-Path (Join-Path $CodeDir "FwUtils.dll")) +} + +function Find-FieldWorksCodeDir { + param([string]$Preferred) + + if (Test-FieldWorksInstalled -CodeDir $Preferred) { + return $Preferred + } + + foreach ($hive in @("HKLM:\SOFTWARE\SIL\FieldWorks\9", "HKLM:\SOFTWARE\WOW6432Node\SIL\FieldWorks\9")) { + if (Test-Path $hive) { + $fromReg = (Get-ItemProperty -Path $hive -ErrorAction SilentlyContinue).RootCodeDir + if (Test-FieldWorksInstalled -CodeDir $fromReg) { + Write-Host "Found FieldWorks via registry $hive => $fromReg" + return $fromReg + } + } + } + + $searchRoots = @( + "C:\Program Files\FieldWorks9", + "C:\Program Files\SIL", + "C:\Program Files (x86)\SIL", + "C:\Program Files", + "C:\Program Files (x86)" + ) + foreach ($root in $searchRoots) { + if (-not (Test-Path $root)) { continue } + $exe = Get-ChildItem -Path $root -Filter "FieldWorks.exe" -Recurse -ErrorAction SilentlyContinue | + Select-Object -First 1 + if ($exe) { + Write-Host "Found FieldWorks.exe at $($exe.FullName)" + return $exe.DirectoryName + } + } + return $null +} + +function Write-Diagnostics { + Write-Host "---- Diagnostics ----" + foreach ($p in @( + "C:\Program Files\FieldWorks9", + "C:\Program Files\SIL", + "C:\Program Files (x86)\SIL", + "C:\ProgramData\SIL" + )) { + if (Test-Path $p) { + Write-Host "Listing $p" + Get-ChildItem $p -ErrorAction SilentlyContinue | ForEach-Object { Write-Host " $($_.FullName)" } + } else { + Write-Host "Missing: $p" + } + } + foreach ($hive in @("HKLM:\SOFTWARE\SIL", "HKLM:\SOFTWARE\WOW6432Node\SIL")) { + if (Test-Path $hive) { + Write-Host "Registry under $hive :" + Get-ChildItem $hive -Recurse -ErrorAction SilentlyContinue | ForEach-Object { Write-Host " $($_.Name)" } + } + } + Get-ChildItem "C:\fw-ci\fw-install*.log" -ErrorAction SilentlyContinue | ForEach-Object { + Write-Host "---- $($_.Name) (last 40 lines) ----" + Get-Content $_.FullName -Tail 40 + } +} + +function Write-CodeDirOut { + param([string]$Resolved) + + Set-Content -Path $CodeDirOutFile -Value $Resolved -Encoding UTF8 + if ($env:GITHUB_ENV) { + Add-Content -Path $env:GITHUB_ENV -Value "FW_CODE_DIR=$Resolved" + } +} + +if (-not $InstallerUrl) { + $InstallerUrl = "https://downloads.languagetechnology.org/fieldworks/9.3.9/$Build/FieldWorks_${Version}_Offline_x64.exe" +} + +New-Item -ItemType Directory -Force -Path $DownloadDir | Out-Null +New-Item -ItemType Directory -Force -Path (Split-Path $InstallLog) | Out-Null +New-Item -ItemType Directory -Force -Path $ProjectsDir | Out-Null + +$existing = Find-FieldWorksCodeDir -Preferred $CodeDir +if ($existing) { + Write-Host "FieldWorks already present at $existing (cache hit or prior install)" + Ensure-Registry -CodeDir $existing -ProjectsDir $ProjectsDir + Write-CodeDirOut -Resolved $existing + exit 0 +} + +$installerName = Split-Path $InstallerUrl -Leaf +$installerPath = Join-Path $DownloadDir $installerName + +if (-not (Test-Path $installerPath)) { + Write-Host "Downloading FieldWorks installer..." + Write-Host " URL: $InstallerUrl" + Write-Host " To: $installerPath" + & curl.exe -L --retry 3 --retry-delay 5 -o $installerPath $InstallerUrl + if ($LASTEXITCODE -ne 0) { + throw "Download failed with exit code $LASTEXITCODE" + } +} else { + Write-Host "Using cached installer: $installerPath" +} + +$sizeMB = [math]::Round((Get-Item $installerPath).Length / 1MB, 1) +Write-Host "Installer size: ${sizeMB} MB" +Write-Host "Silent installing FieldWorks (/quiet /norestart ADDLOCAL=ALL)..." +Write-Host "Log: $InstallLog" + +$proc = Start-Process -FilePath $installerPath ` + -ArgumentList @("/quiet", "/norestart", "/log", $InstallLog, "ADDLOCAL=ALL") ` + -Wait -PassThru -NoNewWindow + +Write-Host "Installer exit code: $($proc.ExitCode)" +if ($proc.ExitCode -notin @(0, 3010, 1641)) { + Write-Diagnostics + throw "FieldWorks installer failed with exit code $($proc.ExitCode)" +} + +$resolved = Find-FieldWorksCodeDir -Preferred $CodeDir +if (-not $resolved) { + Write-Diagnostics + throw "FieldWorks.exe not found after install (checked Program Files and registry)" +} + +Write-Host "Resolved FieldWorks code dir: $resolved" +Ensure-Registry -CodeDir $resolved -ProjectsDir $ProjectsDir +Write-CodeDirOut -Resolved $resolved +Write-Host "FieldWorks install complete." diff --git a/scripts/ci/Setup-SampleProject.ps1 b/scripts/ci/Setup-SampleProject.ps1 new file mode 100644 index 0000000..b1c7674 --- /dev/null +++ b/scripts/ci/Setup-SampleProject.ps1 @@ -0,0 +1,63 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Download a sample FieldWorks backup and extract it into the projects directory. +#> +[CmdletBinding()] +param( + [string]$ProjectsDir = "C:\ProgramData\SIL\FieldWorks\Projects", + [string]$BackupUrl = "https://downloads.languagetechnology.org/fieldworks/9.0.4/Sena%202%202026-06-09%201659.fwbackup", + [string]$ProjectName = "Sena 2", + [string]$DownloadDir = "C:\fw-ci\samples" +) + +$ErrorActionPreference = "Stop" +Add-Type -AssemblyName System.IO.Compression.FileSystem + +New-Item -ItemType Directory -Force -Path $DownloadDir | Out-Null +New-Item -ItemType Directory -Force -Path $ProjectsDir | Out-Null + +$projectDir = Join-Path $ProjectsDir $ProjectName +$fwdata = Join-Path $projectDir ($ProjectName + ".fwdata") +if (Test-Path $fwdata) { + Write-Host "Sample project already present: $fwdata" + exit 0 +} + +$backupPath = Join-Path $DownloadDir ($ProjectName + ".fwbackup") +if (-not (Test-Path $backupPath)) { + Write-Host "Downloading sample project backup..." + Write-Host " URL: $BackupUrl" + & curl.exe -L --retry 3 --retry-delay 5 -o $backupPath $BackupUrl + if ($LASTEXITCODE -ne 0) { + throw "Sample project download failed with exit code $LASTEXITCODE" + } +} + +if (Test-Path $projectDir) { + Remove-Item -Recurse -Force $projectDir +} +New-Item -ItemType Directory -Force -Path $projectDir | Out-Null + +Write-Host "Extracting $backupPath -> $projectDir" +[System.IO.Compression.ZipFile]::ExtractToDirectory($backupPath, $projectDir) + +# fwbackup often has ProjectName.fwdata at the archive root +if (-not (Test-Path $fwdata)) { + $found = Get-ChildItem -Path $projectDir -Filter "*.fwdata" -Recurse | Select-Object -First 1 + if (-not $found) { + throw "No .fwdata found after extracting sample project" + } + if ($found.DirectoryName -ne $projectDir) { + Move-Item -Force $found.FullName (Join-Path $projectDir $found.Name) + } + if ($found.Name -ne ($ProjectName + ".fwdata")) { + Rename-Item -Force (Join-Path $projectDir $found.Name) ($ProjectName + ".fwdata") + } +} + +if (-not (Test-Path $fwdata)) { + throw "Expected project file missing: $fwdata" +} + +Write-Host "Sample project ready: $fwdata" diff --git a/tests/nuget-overlay/Marker.cs b/tests/nuget-overlay/Marker.cs new file mode 100644 index 0000000..6181577 --- /dev/null +++ b/tests/nuget-overlay/Marker.cs @@ -0,0 +1,7 @@ +// Placeholder so the SDK builds a library; only the package DLLs matter. +namespace FlexlibsNugetOverlay +{ + public static class Marker + { + } +} diff --git a/tests/nuget-overlay/Overlay.csproj b/tests/nuget-overlay/Overlay.csproj new file mode 100644 index 0000000..0e8d7e2 --- /dev/null +++ b/tests/nuget-overlay/Overlay.csproj @@ -0,0 +1,19 @@ + + + net462 + Library + FlexlibsNugetOverlay + disable + disable + + true + + + + + + + + + + diff --git a/tests/nuget-overlay/restore.ps1 b/tests/nuget-overlay/restore.ps1 new file mode 100644 index 0000000..91257d3 --- /dev/null +++ b/tests/nuget-overlay/restore.ps1 @@ -0,0 +1,55 @@ +#Requires -Version 5.1 +<# +.SYNOPSIS + Restore latest SIL NuGet packages into a flat folder for flexlibs overlay tests. + +.DESCRIPTION + Publishes tests/nuget-overlay to .fw-nuget-overlay at the repo root and writes + versions.txt listing resolved package versions. +#> +$ErrorActionPreference = "Stop" + +$RepoRoot = Resolve-Path (Join-Path $PSScriptRoot "..\..") +$Project = Join-Path $PSScriptRoot "Overlay.csproj" +$OutDir = Join-Path $RepoRoot ".fw-nuget-overlay" + +Write-Host "Publishing NuGet overlay to $OutDir" + +if (Test-Path $OutDir) { + Remove-Item -Recurse -Force $OutDir +} + +dotnet publish $Project -c Release -o $OutDir --nologo +if ($LASTEXITCODE -ne 0) { + throw "dotnet publish failed with exit code $LASTEXITCODE" +} + +$DllCount = @(Get-ChildItem -Path $OutDir -Filter "*.dll" -File).Count +if ($DllCount -eq 0) { + throw "No DLLs published to $OutDir" +} + +# Record resolved package versions from the assets file for debugging. +$Assets = Join-Path $PSScriptRoot "obj\project.assets.json" +$VersionsPath = Join-Path $OutDir "versions.txt" +$Lines = New-Object System.Collections.Generic.List[string] +$Lines.Add("Published: $(Get-Date -Format o)") +$Lines.Add("DLL count: $DllCount") +$Lines.Add("") + +if (Test-Path $Assets) { + $Lines.Add("Resolved packages (from project.assets.json):") + $json = Get-Content -Raw $Assets | ConvertFrom-Json + $libs = $json.libraries.PSObject.Properties + foreach ($lib in ($libs | Sort-Object Name)) { + if ($lib.Name -match '^(SIL\.(LCModel|Core|WritingSystems))') { + $Lines.Add(" $($lib.Name)") + } + } +} else { + $Lines.Add("(project.assets.json not found; versions unknown)") +} + +$Lines | Set-Content -Path $VersionsPath -Encoding UTF8 +Write-Host "Wrote $VersionsPath ($DllCount DLLs)" +Write-Host "Set FLEXLIBS_ASSEMBLY_DIR=$OutDir to use this overlay."