From b632b14308135efa1795c77494fb3299148fb913 Mon Sep 17 00:00:00 2001 From: Ugochukwu Mmaduekwe Date: Mon, 8 Jun 2026 00:28:41 +0100 Subject: [PATCH 1/5] big endian related fixes --- .../Delphi.Tests/SimpleBaseLib.Tests.dpr | 1 - .../src/Base32/Base32Tests.pas | 23 --- SimpleBaseLib/src/Bases/SbpBase32.pas | 57 +------ .../Delphi/SimpleBaseLib4PascalPackage.dpk | 1 - .../FPC/SimpleBaseLib4PascalPackage.lpk | 140 +++++++++--------- .../FPC/SimpleBaseLib4PascalPackage.pas | 2 +- .../src/Utilities/SbpPlatformUtilities.pas | 23 --- 7 files changed, 72 insertions(+), 175 deletions(-) delete mode 100644 SimpleBaseLib/src/Utilities/SbpPlatformUtilities.pas diff --git a/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.dpr b/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.dpr index eeea21c..7f50c77 100644 --- a/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.dpr +++ b/SimpleBaseLib.Tests/Delphi.Tests/SimpleBaseLib.Tests.dpr @@ -72,7 +72,6 @@ uses SbpBase45 in '..\..\SimpleBaseLib\src\Bases\SbpBase45.pas', SbpIBase45Alphabet in '..\..\SimpleBaseLib\src\Interfaces\Alphabets\SbpIBase45Alphabet.pas', SbpIBase45 in '..\..\SimpleBaseLib\src\Interfaces\Bases\SbpIBase45.pas', - SbpPlatformUtilities in '..\..\SimpleBaseLib\src\Utilities\SbpPlatformUtilities.pas', SbpAliasedBase32Alphabet in '..\..\SimpleBaseLib\src\Alphabets\SbpAliasedBase32Alphabet.pas', SbpBase32Alphabet in '..\..\SimpleBaseLib\src\Alphabets\SbpBase32Alphabet.pas', SbpBase32 in '..\..\SimpleBaseLib\src\Bases\SbpBase32.pas', diff --git a/SimpleBaseLib.Tests/src/Base32/Base32Tests.pas b/SimpleBaseLib.Tests/src/Base32/Base32Tests.pas index 5bcd575..50034c5 100644 --- a/SimpleBaseLib.Tests/src/Base32/Base32Tests.pas +++ b/SimpleBaseLib.Tests/src/Base32/Base32Tests.pas @@ -21,7 +21,6 @@ interface SbpIBase32, SbpBase32, SbpBase32Alphabet, - SbpBitOperations, SimpleBaseLibTestBase; type @@ -52,8 +51,6 @@ TTestBase32 = class(TSimpleBaseLibTestCase) procedure Test_Rfc4648_DecodeUInt64_ReturnsExpectedValues; procedure Test_Rfc4648_TryDecodeUInt64_ReturnsExpectedValues; procedure Test_Rfc4648_TryDecodeUInt64_InvalidInput_ReturnsFalse; - procedure Test_Rfc4648_Encode_BigEndianUInt64_ReturnsExpectedValues; - procedure Test_Rfc4648_DecodeUInt64_BigEndian_ReturnsExpectedValues; procedure Test_Rfc4648_EncodeInt64_Negative_Throws; procedure Test_Rfc4648_DecodeInt64_ReturnsExpectedValues; procedure Test_Rfc4648_DecodeInt64_OutOfRange_Throws; @@ -239,26 +236,6 @@ procedure TTestBase32.Test_Rfc4648_TryDecodeUInt64_InvalidInput_ReturnsFalse; CheckFalse(TBase32.Rfc4648.TryDecodeUInt64('!@#!@#invalid alphabet!@#!@#', LValue)); end; -procedure TTestBase32.Test_Rfc4648_Encode_BigEndianUInt64_ReturnsExpectedValues; -var - LBigEndian: IBase32; - LValue: UInt64; -begin - LBigEndian := TBase32.Create(TBase32Alphabet.Rfc4648, True); - LValue := TBitOperations.ReverseBytesUInt64($1122334455667788); - CheckEquals('RB3WMVKEGMRBC', LBigEndian.EncodeUInt64(LValue)); -end; - -procedure TTestBase32.Test_Rfc4648_DecodeUInt64_BigEndian_ReturnsExpectedValues; -var - LBigEndian: IBase32; - LExpected: UInt64; -begin - LBigEndian := TBase32.Create(TBase32Alphabet.Rfc4648, True); - LExpected := TBitOperations.ReverseBytesUInt64($1122334455667788); - CheckEquals(LExpected, LBigEndian.DecodeUInt64('RB3WMVKEGMRBC')); -end; - procedure TTestBase32.Test_Rfc4648_EncodeInt64_Negative_Throws; begin try diff --git a/SimpleBaseLib/src/Bases/SbpBase32.pas b/SimpleBaseLib/src/Bases/SbpBase32.pas index 88f8944..c37ed8d 100644 --- a/SimpleBaseLib/src/Bases/SbpBase32.pas +++ b/SimpleBaseLib/src/Bases/SbpBase32.pas @@ -18,7 +18,6 @@ interface SbpStreamUtilities, SbpCodingAlphabet, SbpBitOperations, - SbpPlatformUtilities, SbpBinaryPrimitives; type @@ -39,7 +38,6 @@ TBase32 = class(TInterfacedObject, IBase32, IBaseStreamCoder, INonAllocatingBa var FAlphabet: IBase32Alphabet; - FIsBigEndian: Boolean; class var FCrockford: IBase32; class var FRfc4648: IBase32; @@ -77,8 +75,7 @@ TBase32 = class(TInterfacedObject, IBase32, IBaseStreamCoder, INonAllocatingBa public class constructor Create; - constructor Create(const AAlphabet: IBase32Alphabet); overload; - constructor Create(const AAlphabet: IBase32Alphabet; AIsBigEndian: Boolean); overload; + constructor Create(const AAlphabet: IBase32Alphabet); class property Crockford: IBase32 read GetCrockford; class property Rfc4648: IBase32 read GetRfc4648; @@ -139,11 +136,6 @@ implementation end; constructor TBase32.Create(const AAlphabet: IBase32Alphabet); -begin - Create(AAlphabet, not TPlatformUtilities.IsLittleEndian); -end; - -constructor TBase32.Create(const AAlphabet: IBase32Alphabet; AIsBigEndian: Boolean); begin inherited Create; if AAlphabet.PaddingPosition <> TPaddingPosition.&End then @@ -153,7 +145,6 @@ constructor TBase32.Create(const AAlphabet: IBase32Alphabet; AIsBigEndian: Boole end; FAlphabet := AAlphabet; - FIsBigEndian := AIsBigEndian; end; function TBase32.GetAlphabet: IBase32Alphabet; @@ -327,9 +318,8 @@ function TBase32.EncodeUInt64(const ANumber: UInt64): String; const NumBytes = 8; var - LBuffer, LSpan: TSimpleBaseLibByteArray; + LBuffer: TSimpleBaseLibByteArray; LI: Int32; - LTmp: Byte; begin if ANumber = 0 then begin @@ -340,24 +330,6 @@ function TBase32.EncodeUInt64(const ANumber: UInt64): String; System.SetLength(LBuffer, NumBytes); TBinaryPrimitives.WriteUInt64LittleEndian(LBuffer, 0, ANumber); - if FIsBigEndian then - begin - LI := 0; - while (LI < NumBytes) and (LBuffer[LI] = 0) do - begin - Inc(LI); - end; - LSpan := System.Copy(LBuffer, LI, NumBytes - LI); - for LI := 0 to (System.Length(LSpan) div 2) - 1 do - begin - LTmp := LSpan[LI]; - LSpan[LI] := LSpan[System.Length(LSpan) - 1 - LI]; - LSpan[System.Length(LSpan) - 1 - LI] := LTmp; - end; - Result := Encode(LSpan); - Exit; - end; - LI := NumBytes - 1; while (LI > 0) and (LBuffer[LI] = 0) do begin @@ -369,8 +341,6 @@ function TBase32.EncodeUInt64(const ANumber: UInt64): String; function TBase32.DecodeUInt64(const AText: String): UInt64; var LBuffer, LNewSpan: TSimpleBaseLibByteArray; - LI: Int32; - LTmp: Byte; begin LBuffer := Decode(AText); if System.Length(LBuffer) = 0 then @@ -388,24 +358,13 @@ function TBase32.DecodeUInt64(const AText: String): UInt64; TArrayUtilities.Fill(LNewSpan, 0, 8, Byte(0)); Move(LBuffer[0], LNewSpan[0], System.Length(LBuffer)); - if FIsBigEndian then - begin - for LI := 0 to 3 do - begin - LTmp := LNewSpan[LI]; - LNewSpan[LI] := LNewSpan[7 - LI]; - LNewSpan[7 - LI] := LTmp; - end; - end; - Result := TBinaryPrimitives.ReadUInt64LittleEndian(LNewSpan, 0); end; function TBase32.TryDecodeUInt64(const AText: String; out ANumber: UInt64): Boolean; var LOutput: TSimpleBaseLibByteArray; - LBytesWritten, LI: Int32; - LTmp: Byte; + LBytesWritten: Int32; begin System.SetLength(LOutput, 8); TArrayUtilities.Fill(LOutput, 0, 8, Byte(0)); @@ -416,16 +375,6 @@ function TBase32.TryDecodeUInt64(const AText: String; out ANumber: UInt64): Bool Exit; end; - if FIsBigEndian then - begin - for LI := 0 to 3 do - begin - LTmp := LOutput[LI]; - LOutput[LI] := LOutput[7 - LI]; - LOutput[7 - LI] := LTmp; - end; - end; - ANumber := TBinaryPrimitives.ReadUInt64LittleEndian(LOutput, 0); Result := True; end; diff --git a/SimpleBaseLib/src/Packages/Delphi/SimpleBaseLib4PascalPackage.dpk b/SimpleBaseLib/src/Packages/Delphi/SimpleBaseLib4PascalPackage.dpk index 5d87889..3779883 100644 --- a/SimpleBaseLib/src/Packages/Delphi/SimpleBaseLib4PascalPackage.dpk +++ b/SimpleBaseLib/src/Packages/Delphi/SimpleBaseLib4PascalPackage.dpk @@ -79,7 +79,6 @@ contains SbpBase45 in '..\..\Bases\SbpBase45.pas', SbpIBase45Alphabet in '..\..\Interfaces\Alphabets\SbpIBase45Alphabet.pas', SbpIBase45 in '..\..\Interfaces\Bases\SbpIBase45.pas', - SbpPlatformUtilities in '..\..\Utilities\SbpPlatformUtilities.pas', SbpAliasedBase32Alphabet in '..\..\Alphabets\SbpAliasedBase32Alphabet.pas', SbpBase32Alphabet in '..\..\Alphabets\SbpBase32Alphabet.pas', SbpBase32 in '..\..\Bases\SbpBase32.pas', diff --git a/SimpleBaseLib/src/Packages/FPC/SimpleBaseLib4PascalPackage.lpk b/SimpleBaseLib/src/Packages/FPC/SimpleBaseLib4PascalPackage.lpk index 45ac31d..6897fd8 100644 --- a/SimpleBaseLib/src/Packages/FPC/SimpleBaseLib4PascalPackage.lpk +++ b/SimpleBaseLib/src/Packages/FPC/SimpleBaseLib4PascalPackage.lpk @@ -28,7 +28,7 @@ - + @@ -158,145 +158,141 @@ - - - - - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + diff --git a/SimpleBaseLib/src/Packages/FPC/SimpleBaseLib4PascalPackage.pas b/SimpleBaseLib/src/Packages/FPC/SimpleBaseLib4PascalPackage.pas index e543c72..832bd2a 100644 --- a/SimpleBaseLib/src/Packages/FPC/SimpleBaseLib4PascalPackage.pas +++ b/SimpleBaseLib/src/Packages/FPC/SimpleBaseLib4PascalPackage.pas @@ -15,7 +15,7 @@ interface SbpSimpleBaseLibConstants, SbpBase85Alphabet, SbpIBase85Alphabet, SbpBase58, SbpINumericBaseCoder, SbpBase32Alphabet, SbpINonAllocatingBaseCoder, SbpIBaseStreamCoder, SbpIBaseCoder, SbpAliasedBase32Alphabet, - SbpIAliasedBase32Alphabet, SbpPlatformUtilities, SbpIBase32Alphabet, + SbpIAliasedBase32Alphabet, SbpIBase32Alphabet, SbpCharMap, SbpPaddingPosition, SbpCodingAlphabet, SbpBase45Alphabet, SbpIBase45Alphabet, SbpIDividingCoder, SbpBase62Alphabet, SbpBase58Alphabet, SbpBase36Alphabet, SbpBase10Alphabet, SbpMoneroBase58, SbpBase62, SbpBase36, diff --git a/SimpleBaseLib/src/Utilities/SbpPlatformUtilities.pas b/SimpleBaseLib/src/Utilities/SbpPlatformUtilities.pas deleted file mode 100644 index 593b647..0000000 --- a/SimpleBaseLib/src/Utilities/SbpPlatformUtilities.pas +++ /dev/null @@ -1,23 +0,0 @@ -unit SbpPlatformUtilities; - -{$I ..\Include\SimpleBaseLib.inc} - -interface - -type - TPlatformUtilities = class sealed(TObject) - public - class function IsLittleEndian: Boolean; static; inline; - end; - -implementation - -class function TPlatformUtilities.IsLittleEndian: Boolean; -var - LValue: UInt16; -begin - LValue := 1; - Result := PByte(@LValue)^ = 1; -end; - -end. From 3bbd3bdbc1ae506995b654cc859f76bb8a3eb308 Mon Sep 17 00:00:00 2001 From: Ugochukwu Mmaduekwe Date: Mon, 8 Jun 2026 00:40:17 +0100 Subject: [PATCH 2/5] add powerpc64 big endian to ci --- .github/workflows/ci/arm32-install.sh | 9 + .github/workflows/ci/arm32-run.sh | 14 + .github/workflows/ci/native-build.sh | 9 + .../ci/openssl-libssl11-shim-macos.sh | 8 + .../ci/openssl-libssl11-shim-unix.sh | 40 + .github/workflows/ci/ppc64-be-build.sh | 43 + .github/workflows/ci/ppc64-be-images.env | 11 + .github/workflows/ci/ppc64-be-inner.sh | 10 + .github/workflows/ci/ppc64-qemu-setup.sh | 26 + .github/workflows/ci/resolve-targets.sh | 37 + .github/workflows/ci/shared/common.sh | 195 ++ .github/workflows/ci/shared/csu-stubs.c | 5 + .../workflows/ci/shared/lazarus-bootstrap.sh | 50 + .../ci/shared/runtime-endian-probe.c | 7 + .github/workflows/ci/vm-dragonfly-prepare.sh | 17 + .github/workflows/ci/vm-freebsd-prepare.sh | 25 + .github/workflows/ci/vm-freebsd-run.sh | 16 + .github/workflows/ci/vm-netbsd-prepare.sh | 6 + .github/workflows/ci/vm-run-shared.sh | 14 + .github/workflows/ci/vm-solaris-prepare.sh | 17 + .github/workflows/install-fpc-lazarus.sh | 212 +- .github/workflows/make.pas | 1882 ++++++++++++++--- .github/workflows/make.yml | 400 +--- .../src/Include/SimpleBaseLibFPC.inc | 4 - 24 files changed, 2319 insertions(+), 738 deletions(-) create mode 100644 .github/workflows/ci/arm32-install.sh create mode 100644 .github/workflows/ci/arm32-run.sh create mode 100644 .github/workflows/ci/native-build.sh create mode 100644 .github/workflows/ci/openssl-libssl11-shim-macos.sh create mode 100644 .github/workflows/ci/openssl-libssl11-shim-unix.sh create mode 100644 .github/workflows/ci/ppc64-be-build.sh create mode 100644 .github/workflows/ci/ppc64-be-images.env create mode 100644 .github/workflows/ci/ppc64-be-inner.sh create mode 100644 .github/workflows/ci/ppc64-qemu-setup.sh create mode 100644 .github/workflows/ci/resolve-targets.sh create mode 100644 .github/workflows/ci/shared/common.sh create mode 100644 .github/workflows/ci/shared/csu-stubs.c create mode 100644 .github/workflows/ci/shared/lazarus-bootstrap.sh create mode 100644 .github/workflows/ci/shared/runtime-endian-probe.c create mode 100644 .github/workflows/ci/vm-dragonfly-prepare.sh create mode 100644 .github/workflows/ci/vm-freebsd-prepare.sh create mode 100644 .github/workflows/ci/vm-freebsd-run.sh create mode 100644 .github/workflows/ci/vm-netbsd-prepare.sh create mode 100644 .github/workflows/ci/vm-run-shared.sh create mode 100644 .github/workflows/ci/vm-solaris-prepare.sh diff --git a/.github/workflows/ci/arm32-install.sh b/.github/workflows/ci/arm32-install.sh new file mode 100644 index 0000000..e2139a5 --- /dev/null +++ b/.github/workflows/ci/arm32-install.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +# shellcheck source=shared/common.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/shared/common.sh" +ci_init_paths + +export OPENSSL_ARCH_DIR=/usr/lib/arm-linux-gnueabihf +ci_debian_container_bootstrap diff --git a/.github/workflows/ci/arm32-run.sh b/.github/workflows/ci/arm32-run.sh new file mode 100644 index 0000000..0cb0b4c --- /dev/null +++ b/.github/workflows/ci/arm32-run.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +CI_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# run-on-arch-action `install` runs at docker build time (no repo mount). +# Install deps here in `run` so arm32-install.sh stays the single source of truth. +bash "$CI_ROOT/arm32-install.sh" + +# shellcheck source=shared/common.sh +source "$CI_ROOT/shared/common.sh" +ci_init_paths + +ci_build_standard diff --git a/.github/workflows/ci/native-build.sh b/.github/workflows/ci/native-build.sh new file mode 100644 index 0000000..73e0f53 --- /dev/null +++ b/.github/workflows/ci/native-build.sh @@ -0,0 +1,9 @@ +#!/usr/bin/env bash +set -euo pipefail + +# shellcheck source=shared/common.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/shared/common.sh" +ci_init_paths + +ci_openssl_hack +ci_build_standard diff --git a/.github/workflows/ci/openssl-libssl11-shim-macos.sh b/.github/workflows/ci/openssl-libssl11-shim-macos.sh new file mode 100644 index 0000000..51c2271 --- /dev/null +++ b/.github/workflows/ci/openssl-libssl11-shim-macos.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +# FPC 3.2.2 hardcodes libssl.1.1.dylib; symlink Homebrew OpenSSL 3 on macOS. +set -euo pipefail + +OSSL_LIB="$(brew --prefix openssl@3)/lib" +sudo mkdir -p /usr/local/lib +sudo ln -sf "$OSSL_LIB/libssl.3.dylib" /usr/local/lib/libssl.1.1.dylib +sudo ln -sf "$OSSL_LIB/libcrypto.3.dylib" /usr/local/lib/libcrypto.1.1.dylib diff --git a/.github/workflows/ci/openssl-libssl11-shim-unix.sh b/.github/workflows/ci/openssl-libssl11-shim-unix.sh new file mode 100644 index 0000000..4f5fcc9 --- /dev/null +++ b/.github/workflows/ci/openssl-libssl11-shim-unix.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# FPC 3.2.2 hardcodes libssl.so.1.1; symlink OpenSSL 3.x ELF libraries on Linux/BSD. +set -euo pipefail + +ARCH_DIR="${1:-}" +if [ -z "$ARCH_DIR" ] && command -v gcc >/dev/null 2>&1; then + multiarch="$(gcc -print-multiarch 2>/dev/null || true)" + if [ -n "$multiarch" ] && [ -f "/usr/lib/$multiarch/libssl.so.3" ]; then + ARCH_DIR="/usr/lib/$multiarch" + fi +fi +if [ -z "$ARCH_DIR" ] || [ ! -f "$ARCH_DIR/libssl.so.3" ]; then + for candidate in \ + /usr/lib/powerpc64-linux-gnu \ + /usr/lib/ppc64-linux-gnu \ + /usr/lib/arm-linux-gnueabihf \ + /usr/lib/aarch64-linux-gnu \ + /usr/lib/x86_64-linux-gnu \ + /usr/lib; do + if [ -f "$candidate/libssl.so.3" ]; then + ARCH_DIR="$candidate" + break + fi + done +fi + +if [ ! -f "$ARCH_DIR/libssl.so.3" ]; then + echo "openssl-libssl11-shim-unix: libssl.so.3 not found under $ARCH_DIR" >&2 + exit 1 +fi + +ln_cmd=(ln -sf) +if [ "${OPENSSL_USE_SUDO:-1}" = "1" ] && [ "$(id -u)" -ne 0 ]; then + if command -v sudo >/dev/null 2>&1; then + ln_cmd=(sudo ln -sf) + fi +fi + +"${ln_cmd[@]}" "$ARCH_DIR/libssl.so.3" "$ARCH_DIR/libssl.so.1.1" +"${ln_cmd[@]}" "$ARCH_DIR/libcrypto.so.3" "$ARCH_DIR/libcrypto.so.1.1" diff --git a/.github/workflows/ci/ppc64-be-build.sh b/.github/workflows/ci/ppc64-be-build.sh new file mode 100644 index 0000000..9f24c39 --- /dev/null +++ b/.github/workflows/ci/ppc64-be-build.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${GITHUB_WORKSPACE:?GITHUB_WORKSPACE is required}" +: "${FPC_VERSION:?FPC_VERSION is required}" +: "${FPC_TARGET:?FPC_TARGET is required}" +: "${MAKE_BUILD_BACKEND:?MAKE_BUILD_BACKEND is required}" + +CI_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +CI_SHARED="$CI_ROOT/shared" +# shellcheck source=ppc64-be-images.env +source "$CI_ROOT/ppc64-be-images.env" + +# Cross-compile glibc csu stubs on the x86 host. gcc inside QEMU ppc64 +# user-mode often SIGSEGVs; install-fpc-lazarus.sh expects CSU_STUBS_PREBUILT. +STUB_C="$(mktemp --suffix=.c)" +STUB_OBJ="$(mktemp --suffix=.o)" +CSU_STUBS_IN_CONTAINER=/csu_stubs_prebuilt.o +trap 'rm -f "$STUB_C" "$STUB_OBJ"' EXIT + +cp "$CI_SHARED/csu-stubs.c" "$STUB_C" + +sudo apt-get update -qq +sudo apt-get install -y -qq gcc-powerpc64-linux-gnu +powerpc64-linux-gnu-gcc -c -fPIC -o "$STUB_OBJ" "$STUB_C" +if [ ! -s "$STUB_OBJ" ]; then + echo "::error::cross-compile did not produce csu stubs object at $STUB_OBJ" >&2 + exit 1 +fi +echo "csu stubs cross-compiled: $(wc -c < "$STUB_OBJ") bytes" + +docker run --rm --platform linux/ppc64 \ + --security-opt seccomp=unconfined \ + -v "${GITHUB_WORKSPACE}:/work" -w /work \ + -v "${STUB_OBJ}:${CSU_STUBS_IN_CONTAINER}:ro" \ + -e FPC_VERSION \ + -e FPC_TARGET \ + -e MAKE_BUILD_BACKEND \ + -e DEBIAN_FRONTEND=noninteractive \ + -e QEMU_CPU=power8 \ + -e CSU_STUBS_PREBUILT="${CSU_STUBS_IN_CONTAINER}" \ + "$PPC64_RUNTIME_IMAGE" \ + bash .github/workflows/ci/ppc64-be-inner.sh diff --git a/.github/workflows/ci/ppc64-be-images.env b/.github/workflows/ci/ppc64-be-images.env new file mode 100644 index 0000000..6d19f3e --- /dev/null +++ b/.github/workflows/ci/ppc64-be-images.env @@ -0,0 +1,11 @@ +# Pinned images for linux-powerpc64-be CI (source from ppc64-qemu-setup.sh / ppc64-be-build.sh). +PPC64_QEMU_VERSION=7.2.0-1 +# One-shot privileged host binfmt setup (ppc64-qemu-setup.sh --reset -p yes -c yes). +# Use multiarch/qemu-user-static:$version (full image): includes the register script +# and all qemu-*-static binaries copied onto the host. Do not use x86_64-ppc64-$version +# here — that tag ships only the qemu-ppc64-static binary (no register script). +# See https://github.com/multiarch/qemu-user-static#multiarchqemu-user-static-images +PPC64_QEMU_REGISTER_IMAGE=multiarch/qemu-user-static:${PPC64_QEMU_VERSION} +# Full (non-slim) debian-ports runtime — qemu-ppc64-static embedded by upstream. +# https://github.com/urbanogilson/debian-debootstrap-ports +PPC64_RUNTIME_IMAGE=urbanogilson/debian-debootstrap-ports:ppc64-forky-sid diff --git a/.github/workflows/ci/ppc64-be-inner.sh b/.github/workflows/ci/ppc64-be-inner.sh new file mode 100644 index 0000000..069b7fc --- /dev/null +++ b/.github/workflows/ci/ppc64-be-inner.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +set -euo pipefail + +# shellcheck source=shared/common.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/shared/common.sh" +ci_init_paths + +export OPENSSL_ARCH_DIR=/usr/lib/powerpc64-linux-gnu +ci_debian_container_bootstrap gcc binutils +ci_build_standard diff --git a/.github/workflows/ci/ppc64-qemu-setup.sh b/.github/workflows/ci/ppc64-qemu-setup.sh new file mode 100644 index 0000000..f2cf051 --- /dev/null +++ b/.github/workflows/ci/ppc64-qemu-setup.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +set -euo pipefail + +CI_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=ppc64-be-images.env +source "$CI_ROOT/ppc64-be-images.env" + +docker run --rm --privileged \ + "$PPC64_QEMU_REGISTER_IMAGE" \ + --reset -p yes -c yes + +if ! ls /proc/sys/fs/binfmt_misc/qemu-ppc64* >/dev/null 2>&1; then + echo "::error::qemu-ppc64 binfmt handler not registered" + ls /proc/sys/fs/binfmt_misc/ + exit 1 +fi + +BINFMT_FILE="$(ls /proc/sys/fs/binfmt_misc/qemu-ppc64* | head -1)" +echo "binfmt handler ${BINFMT_FILE}:" +cat "$BINFMT_FILE" + +if ! grep -q 'flags:.*F' "$BINFMT_FILE"; then + echo "::error::qemu-ppc64 binfmt flags missing F (fix-binary mode); got:" >&2 + cat "$BINFMT_FILE" >&2 + exit 1 +fi diff --git a/.github/workflows/ci/resolve-targets.sh b/.github/workflows/ci/resolve-targets.sh new file mode 100644 index 0000000..a2afcdf --- /dev/null +++ b/.github/workflows/ci/resolve-targets.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Stable targets run on every push/PR (DEFAULT). Opt-in targets (netbsd, +# dragonflybsd) are excluded from DEFAULT — pass them explicitly via +# workflow_dispatch enabled_targets. +STABLE_TARGETS="linux-arm32,linux-powerpc64-be,linux-x64,linux-arm64,windows-x64,macos-arm64,macos-x64,freebsd,solaris" +OPT_IN_TARGETS="netbsd,dragonflybsd" +VALID_TARGETS="${STABLE_TARGETS},${OPT_IN_TARGETS}" +DEFAULT="$STABLE_TARGETS" + +if [ -z "${INPUT_TARGETS// /}" ]; then + TARGETS="$DEFAULT" + SOURCE="default" +else + TARGETS="${INPUT_TARGETS// /}" + SOURCE="workflow_dispatch input" +fi + +IFS=',' read -r -a _selected <<< "$TARGETS" +IFS=',' read -r -a _valid <<< "$VALID_TARGETS" +for _id in "${_selected[@]}"; do + [ -z "$_id" ] && continue + _found=0 + for _v in "${_valid[@]}"; do + if [ "$_id" = "$_v" ]; then + _found=1 + break + fi + done + if [ "$_found" -eq 0 ]; then + echo "::warning::Unknown target id \"${_id}\" (valid: ${VALID_TARGETS})" + fi +done + +echo "enabled_targets=${TARGETS}" >> "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" +echo "Enabled targets (${SOURCE}): ${TARGETS}" diff --git a/.github/workflows/ci/shared/common.sh b/.github/workflows/ci/shared/common.sh new file mode 100644 index 0000000..2c59f15 --- /dev/null +++ b/.github/workflows/ci/shared/common.sh @@ -0,0 +1,195 @@ +#!/usr/bin/env bash +# Shared CI helpers — source from scripts under .github/workflows/ci/ + +set -euo pipefail + +ci_init_paths() { + CI_SHARED="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + CI_ROOT="$(cd "$CI_SHARED/.." && pwd)" + WORKFLOWS_DIR="$(cd "$CI_ROOT/.." && pwd)" + REPO_ROOT="$(cd "$WORKFLOWS_DIR/../.." && pwd)" +} + +ci_install_toolchain() { + : "${FPC_TARGET:?FPC_TARGET is required}" + bash "$WORKFLOWS_DIR/install-fpc-lazarus.sh" +} + +ci_export_toolchain_path() { + local prefix="${INSTALL_PREFIX:-$HOME/fpc-install}" + export PATH="${LAZARUS_DIR:-$HOME/lazarus-src}:$prefix/bin:${PATH}" + if [ -n "${FPC_TARGET:-}" ] && [ -d "$prefix/bin/$FPC_TARGET" ]; then + export PATH="$prefix/bin/$FPC_TARGET:$PATH" + fi +} + +# Run fpc with an -i* info flag; retry when stdout is empty (QEMU/subprocess flake). +# Tuning: CI_FPC_PROBE_ATTEMPTS (default 3), CI_FPC_PROBE_DELAY_SECS (default 2). +# Prints the probe value to stdout; diagnostics go to stderr. +ci_fpc_info_probe() { + local flag="$1" + local max_attempts="${CI_FPC_PROBE_ATTEMPTS:-3}" + local delay="${CI_FPC_PROBE_DELAY_SECS:-2}" + local attempt=1 value + + while [ "$attempt" -le "$max_attempts" ]; do + value="$(fpc "$flag" 2>/dev/null | head -1 | tr -d '\r\n' || true)" + if [ -n "$value" ]; then + if [ "$attempt" -gt 1 ]; then + echo "fpc ${flag} succeeded on attempt ${attempt}/${max_attempts}" >&2 + fi + printf '%s\n' "$value" + return 0 + fi + if [ "$attempt" -lt "$max_attempts" ]; then + echo "::warning::fpc ${flag} returned empty (attempt ${attempt}/${max_attempts}), retrying..." >&2 + sleep "$delay" + fi + attempt=$((attempt + 1)) + done + echo "::error::fpc ${flag} returned empty after ${max_attempts} attempts" >&2 + return 1 +} + +# Prints a C compiler path to stdout; returns 1 if none found. +ci_find_c_compiler() { + local c + for c in cc gcc g++; do + if command -v "$c" >/dev/null 2>&1; then + command -v "$c" + return 0 + fi + done + for c in /usr/bin/gcc /usr/sfw/bin/gcc /opt/csw/bin/gcc /usr/gcc/*/bin/gcc; do + if [ -x "$c" ]; then + printf '%s\n' "$c" + return 0 + fi + done + return 1 +} + +# Prints little | big | unknown to stdout (for capture). +ci_runtime_endian() { + local shared_dir probe_src tmp value="unknown" cc_cmd + + shared_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + probe_src="${shared_dir}/runtime-endian-probe.c" + + cc_cmd="$(ci_find_c_compiler 2>/dev/null || true)" + + if [ -n "$cc_cmd" ] && [ -f "$probe_src" ]; then + tmp="$(mktemp /tmp/ci-runtime-endian.XXXXXX 2>/dev/null || true)" + if [ -z "$tmp" ]; then + tmp="$(mktemp -t ci-runtime-endian 2>/dev/null || true)" + fi + if [ -n "$tmp" ]; then + if "$cc_cmd" -O2 -o "$tmp" "$probe_src" 2>/dev/null; then + value="$("$tmp" 2>/dev/null | tr -d '\r\n' || true)" + case "$value" in + little|big) ;; + *) value="unknown" ;; + esac + fi + rm -f "$tmp" + fi + fi + + printf '%s\n' "$value" +} + +ci_preflight() { + local tp to endian target + + ci_fpc_info_probe -iV + tp="$(ci_fpc_info_probe -iTP)" || exit 1 + to="$(ci_fpc_info_probe -iTO)" || exit 1 + endian="$(ci_runtime_endian)" + target="${tp}-${to}" + echo "preflight: target=${target} endian=${endian}" + if [ "$endian" = "unknown" ]; then + echo "::warning::runtime endian probe returned unknown (no usable C compiler or probe failed)" >&2 + fi + if command -v lazbuild >/dev/null 2>&1; then + lazbuild --version + fi +} + +ci_run_make() { + instantfpc "$WORKFLOWS_DIR/make.pas" +} + +ci_build_standard() { + ci_install_toolchain + ci_export_toolchain_path + ci_preflight + ci_run_make +} + +ci_openssl_hack() { + case "$(uname -s)" in + Linux) bash "$CI_ROOT/openssl-libssl11-shim-unix.sh" ;; + Darwin) bash "$CI_ROOT/openssl-libssl11-shim-macos.sh" ;; + DragonFly) OPENSSL_USE_SUDO=0 bash "$CI_ROOT/openssl-libssl11-shim-unix.sh" /usr/local/lib ;; + esac +} + +ci_debian_container_bootstrap() { + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y curl ca-certificates git build-essential openssl "$@" + OPENSSL_USE_SUDO=0 bash "$CI_ROOT/openssl-libssl11-shim-unix.sh" "${OPENSSL_ARCH_DIR:-}" +} + +ci_github_path_append() { + local dir="$1" + [ -n "${GITHUB_PATH:-}" ] || return 0 + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + cygpath -w "$dir" >> "$GITHUB_PATH" + ;; + *) + echo "$dir" >> "$GITHUB_PATH" + ;; + esac +} + +ci_write_lazarus_environmentoptions() { + local laz_dir="$1" + local fpc_exe="$2" + local laz_cfg_dir laz_dir_native fpc_exe_native + + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + local win_local="${LOCALAPPDATA:-$USERPROFILE/AppData/Local}" + laz_cfg_dir="$(cygpath -u "$win_local")/lazarus" + laz_dir_native="$(cygpath -w "$laz_dir")" + fpc_exe_native="$(cygpath -w "$fpc_exe")" + ;; + *) + laz_cfg_dir="${HOME}/.lazarus" + laz_dir_native="$laz_dir" + fpc_exe_native="$fpc_exe" + ;; + esac + + mkdir -p "$laz_cfg_dir" + cat > "$laz_cfg_dir/environmentoptions.xml" < + + + + + + +EOF +} + +freebsd_pkg_bootstrap() { + export ASSUME_ALWAYS_YES=yes + export IGNORE_OSVERSION=yes + pkg bootstrap -f + pkg upgrade -Fqy || true + pkg update -f + pkg upgrade -y +} diff --git a/.github/workflows/ci/shared/csu-stubs.c b/.github/workflows/ci/shared/csu-stubs.c new file mode 100644 index 0000000..f185c83 --- /dev/null +++ b/.github/workflows/ci/shared/csu-stubs.c @@ -0,0 +1,5 @@ +/* glibc 2.34+ removed __libc_csu_init / __libc_csu_fini. FPC 3.2.2's + RTL still references them. Provide empty stubs so the linker + is satisfied. */ +void __libc_csu_init(int argc, char **argv, char **envp) { (void)argc; (void)argv; (void)envp; } +void __libc_csu_fini(void) {} diff --git a/.github/workflows/ci/shared/lazarus-bootstrap.sh b/.github/workflows/ci/shared/lazarus-bootstrap.sh new file mode 100644 index 0000000..802787c --- /dev/null +++ b/.github/workflows/ci/shared/lazarus-bootstrap.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Clone Lazarus, build lazbuild, write environmentoptions.xml, update PATH. +# Requires ci_init_paths and ci_write_lazarus_environmentoptions from common.sh. + +set -euo pipefail + +: "${LAZARUS_BRANCH:?LAZARUS_BRANCH is required}" +: "${LAZARUS_REPO:?LAZARUS_REPO is required}" +: "${FPC_EXE:?FPC_EXE is required (path to fpc binary for environmentoptions.xml)}" + +: "${LAZARUS_DIR:=$HOME/lazarus-src}" + +if [ -z "${MAKE_CMD:-}" ]; then + case "$(uname -s)" in + *BSD|DragonFly|SunOS) MAKE_CMD="gmake" ;; + MINGW*|MSYS*|CYGWIN*) MAKE_CMD="gmake" ;; + *) MAKE_CMD="make" ;; + esac +fi + +git clone --depth 1 --branch "$LAZARUS_BRANCH" "$LAZARUS_REPO" "$LAZARUS_DIR" + +if [ "$(uname -s)" = "DragonFly" ]; then + df_inc="$LAZARUS_DIR/ide/packages/ideconfig/include/dragonfly" + if [ ! -f "$df_inc/lazconf.inc" ]; then + mkdir -p "$df_inc" + cp "$LAZARUS_DIR/ide/packages/ideconfig/include/freebsd/lazconf.inc" \ + "$df_inc/lazconf.inc" + fi +fi + +case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) + "$MAKE_CMD" -C "$(cygpath -w "$LAZARUS_DIR")" lazbuild + ;; + *) + "$MAKE_CMD" -C "$LAZARUS_DIR" lazbuild + ;; +esac + +ci_write_lazarus_environmentoptions "$LAZARUS_DIR" "$FPC_EXE" + +export PATH="$LAZARUS_DIR:$PATH" +ci_github_path_append "$LAZARUS_DIR" + +lazbuild --version + +if [ "${MAKE_BUILD_BACKEND:-}" = "fpc" ]; then + echo "MAKE_BUILD_BACKEND=fpc — lazbuild was built but make.pas will use the fpc backend" +fi diff --git a/.github/workflows/ci/shared/runtime-endian-probe.c b/.github/workflows/ci/shared/runtime-endian-probe.c new file mode 100644 index 0000000..4e5c526 --- /dev/null +++ b/.github/workflows/ci/shared/runtime-endian-probe.c @@ -0,0 +1,7 @@ +#include + +int main(void) { + unsigned x = 1; + puts(((const unsigned char *)&x)[0] ? "little" : "big"); + return 0; +} diff --git a/.github/workflows/ci/vm-dragonfly-prepare.sh b/.github/workflows/ci/vm-dragonfly-prepare.sh new file mode 100644 index 0000000..994341d --- /dev/null +++ b/.github/workflows/ci/vm-dragonfly-prepare.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu + +# shellcheck source=shared/common.sh +. "$(cd "$(dirname "$0")" && pwd)/shared/common.sh" +ci_init_paths + +pkg install -y bash curl git gmake openssl + +for h in github.com packages.lazarus-ide.org downloads.freepascal.org; do + ip=$(drill "$h" 2>/dev/null | awk '/^'"$h"'/{print $5; exit}') + if [ -n "$ip" ]; then + echo "$ip $h" >> /etc/hosts + fi +done + +OPENSSL_USE_SUDO=0 bash "$CI_ROOT/openssl-libssl11-shim-unix.sh" /usr/local/lib diff --git a/.github/workflows/ci/vm-freebsd-prepare.sh b/.github/workflows/ci/vm-freebsd-prepare.sh new file mode 100644 index 0000000..44c83e7 --- /dev/null +++ b/.github/workflows/ci/vm-freebsd-prepare.sh @@ -0,0 +1,25 @@ +#!/bin/sh +# FreeBSD VM prepare — see FREEBSD_INSTALL_MODE (interim|preferred). +# Invoked with /bin/sh (bash is not installed until this script runs). +set -eu + +: "${FREEBSD_INSTALL_MODE:?FREEBSD_INSTALL_MODE is required (interim|preferred)}" + +CI_ROOT="$(cd "$(dirname "$0")" && pwd)" +# shellcheck source=shared/common.sh +. "$CI_ROOT/shared/common.sh" + +if [ "$FREEBSD_INSTALL_MODE" = "preferred" ]; then + freebsd_pkg_bootstrap + pkg install -y bash curl git gmake binutils + exit 0 +fi + +# INTERIM: pkg-installed FPC until FPC 3.2.4 dist tarball works on FreeBSD 15+. +freebsd_pkg_bootstrap +pkg install -y bash fpc git wget gmake + +export FPC_EXE="$(which fpc)" +export LAZARUS_DIR="$HOME/lazarus-src" +# shellcheck source=shared/lazarus-bootstrap.sh +. "$CI_ROOT/shared/lazarus-bootstrap.sh" diff --git a/.github/workflows/ci/vm-freebsd-run.sh b/.github/workflows/ci/vm-freebsd-run.sh new file mode 100644 index 0000000..eff2051 --- /dev/null +++ b/.github/workflows/ci/vm-freebsd-run.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +set -euo pipefail + +# shellcheck source=shared/common.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/shared/common.sh" +ci_init_paths + +: "${FREEBSD_INSTALL_MODE:?FREEBSD_INSTALL_MODE is required (interim|preferred)}" + +if [ "$FREEBSD_INSTALL_MODE" = "preferred" ]; then + ci_build_standard +else + export PATH="$HOME/lazarus-src:$PATH" + ci_preflight + ci_run_make +fi diff --git a/.github/workflows/ci/vm-netbsd-prepare.sh b/.github/workflows/ci/vm-netbsd-prepare.sh new file mode 100644 index 0000000..894a69d --- /dev/null +++ b/.github/workflows/ci/vm-netbsd-prepare.sh @@ -0,0 +1,6 @@ +#!/bin/sh +set -eu + +export PKG_PATH="https://cdn.NetBSD.org/pub/pkgsrc/packages/NetBSD/$(uname -p)/$(uname -r | cut -d_ -f1)/All" +pkg_add -uu pcre2 || true +pkg_add bash curl git gmake mozilla-rootcerts-openssl diff --git a/.github/workflows/ci/vm-run-shared.sh b/.github/workflows/ci/vm-run-shared.sh new file mode 100644 index 0000000..737ee90 --- /dev/null +++ b/.github/workflows/ci/vm-run-shared.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +set -euo pipefail + +: "${FPC_TARGET:?FPC_TARGET is required}" + +# shellcheck source=shared/common.sh +source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/shared/common.sh" +ci_init_paths + +if [ -n "${LD_LIBRARY_PATH_EXTRA:-}" ]; then + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH_EXTRA}${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +fi + +ci_build_standard diff --git a/.github/workflows/ci/vm-solaris-prepare.sh b/.github/workflows/ci/vm-solaris-prepare.sh new file mode 100644 index 0000000..75f90c1 --- /dev/null +++ b/.github/workflows/ci/vm-solaris-prepare.sh @@ -0,0 +1,17 @@ +#!/bin/sh +set -eu + +# 11.4-gcc images install GCC under /usr/gcc//bin (not always as cc on PATH). +_path="/opt/csw/bin:/usr/local/bin" +if [ -d /usr/gcc ]; then + for _gcc_bin in /usr/gcc/*/bin; do + if [ -d "$_gcc_bin" ]; then + _path="$_gcc_bin:$_path" + break + fi + done +fi +export PATH="$_path" +unset _path _gcc_bin + +pkgutil -y -i bash curl git gmake diff --git a/.github/workflows/install-fpc-lazarus.sh b/.github/workflows/install-fpc-lazarus.sh index 57b7710..39ed9a1 100644 --- a/.github/workflows/install-fpc-lazarus.sh +++ b/.github/workflows/install-fpc-lazarus.sh @@ -18,8 +18,13 @@ # (default: $HOME/lazarus-src) # LAZARUS_BRANCH branch/tag to clone, e.g. lazarus_4_4 # LAZARUS_REPO git URL -# MAKE_CMD 'make' on Linux/Windows/macOS, 'gmake' on BSD/Solaris -# (auto-detected if unset) +# MAKE_CMD 'make' on Linux/Windows/macOS, 'gmake' on BSD/Solaris +# (auto-detected if unset) +# MAKE_BUILD_BACKEND auto | lazbuild | fpc +# fpc — install FPC only; skip Lazarus/lazbuild +# lazbuild | auto — also clone Lazarus and build lazbuild +# (auto here always installs Lazarus; make.pas auto probes +# lazbuild on PATH at runtime and may still choose fpc) # # Outputs (appended to $GITHUB_PATH if set): # $INSTALL_PREFIX/bin and $LAZARUS_DIR are added to PATH @@ -27,10 +32,25 @@ set -xeuo pipefail +INSTALL_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck source=ci/shared/common.sh +source "$INSTALL_SCRIPT_DIR/ci/shared/common.sh" + : "${FPC_VERSION:?FPC_VERSION is required (e.g. 3.2.2)}" : "${FPC_TARGET:?FPC_TARGET is required (e.g. x86_64-linux)}" -: "${LAZARUS_BRANCH:?LAZARUS_BRANCH is required}" -: "${LAZARUS_REPO:?LAZARUS_REPO is required}" +: "${MAKE_BUILD_BACKEND:?MAKE_BUILD_BACKEND is required (lazbuild|fpc|auto)}" +case "$MAKE_BUILD_BACKEND" in + fpc) INSTALL_LAZARUS=0 ;; + auto|lazbuild) INSTALL_LAZARUS=1 ;; + *) + echo "unknown MAKE_BUILD_BACKEND: $MAKE_BUILD_BACKEND" >&2 + exit 1 + ;; +esac +if [ "$INSTALL_LAZARUS" = "1" ]; then + : "${LAZARUS_BRANCH:?LAZARUS_BRANCH is required}" + : "${LAZARUS_REPO:?LAZARUS_REPO is required}" +fi case "$(uname -s)" in MINGW*|MSYS*|CYGWIN*) IS_WINDOWS=1 ;; @@ -145,7 +165,7 @@ fi # ── Linux glibc 2.34+ workaround ───────────────────────────────────── # -# FPC 3.2.2 was built against glibc < 2.34 and its cprt0.o references +# FPC 3.2.2 was built against glibc < 2.34 and its RTL references # __libc_csu_init / __libc_csu_fini, which glibc 2.34 (Aug 2021) made # private. Linking anything FPC produces against current glibc fails: # undefined reference to `__libc_csu_init' @@ -153,33 +173,70 @@ fi # # Ubuntu 22.04+ ships glibc 2.34+ and is affected. The fix shipped in # FPC 3.2.4+ (and Debian/Fedora patched their packages); for our -# pre-built tarball we patch in place by merging stub object code -# into cprt0.o, satisfying the symbols at link time. +# pre-built tarball we provide empty stub symbols ourselves. +# +# Affected: every Linux target (x86_64, aarch64, arm/armhf, powerpc64). # -# Affected: every Linux target (x86_64, aarch64, arm/armhf). The -# 'cprt0' name is consistent across architectures. +# Strategy differs by target: +# • x86_64 / aarch64 / arm — merge csu_stubs.o into rtl/cprt0.o only. +# Do not add -k to fpc.cfg (duplicate symbols if both are used). +# • powerpc64-linux — si_c.o also references these symbols, so append +# -k to fpc.cfg and do not merge into cprt0.o. The stub object +# is cross-compiled on the x86 host (CSU_STUBS_PREBUILT); running cc +# inside QEMU user-mode for ppc64 is unreliable. +# +# This hack can be removed once we move to FPC 3.2.4+. # # See https://gitlab.com/freepascal.org/fpc/source/-/issues/39295 if [ "$(uname -s)" = "Linux" ]; then + STUB_INSTALL="$INSTALL_PREFIX/lib/fpc/$FPC_VERSION/csu_stubs.o" + STUB_C="$WORK_DIR/csu_stubs.c" RTL_DIR="$INSTALL_PREFIX/lib/fpc/$FPC_VERSION/units/$FPC_TARGET/rtl" CPRT0="$RTL_DIR/cprt0.o" - if [ -f "$CPRT0" ]; then - STUB_C="$WORK_DIR/csu_stubs.c" - STUB_O="$WORK_DIR/csu_stubs.o" - cat > "$STUB_C" <<'EOF' -/* glibc 2.34+ removed __libc_csu_init / __libc_csu_fini. FPC 3.2.2's - cprt0.o still references them. Provide empty stubs so the linker - is satisfied. */ -void __libc_csu_init(int argc, char **argv, char **envp) { (void)argc; (void)argv; (void)envp; } -void __libc_csu_fini(void) {} -EOF - cc -c -fPIC -o "$STUB_O" "$STUB_C" - # Merge stubs into cprt0.o using `ld -r` (relocatable link). - # cprt0.o references __libc_csu_*; the stub provides them; the - # merged object resolves both within itself. - ld -r -o "$CPRT0.new" "$CPRT0" "$STUB_O" - mv "$CPRT0.new" "$CPRT0" - echo "Patched $CPRT0 with glibc 2.34+ stubs." + + CSU_STUBS_SRC="$INSTALL_SCRIPT_DIR/ci/shared/csu-stubs.c" + FPC_CFG_MARKER="# glibc 2.34+ csu stubs (install-fpc-lazarus.sh)" + append_fpc_cfg() { + local marker="$1" + shift + local cfg + for cfg in /etc/fpc.cfg "${HOME}/.fpc.cfg"; do + if [ -f "$cfg" ] && ! grep -qF "$marker" "$cfg" 2>/dev/null; then + { + echo "" + echo "$marker" + printf '%s\n' "$@" + } >> "$cfg" + echo "Updated $cfg" + fi + done + } + + if [ "$FPC_TARGET" = "powerpc64-linux" ]; then + if [ -z "${CSU_STUBS_PREBUILT:-}" ] || [ ! -f "$CSU_STUBS_PREBUILT" ]; then + echo "ERROR: CSU_STUBS_PREBUILT is required for powerpc64-linux" >&2 + echo " (host must cross-compile csu stubs before QEMU docker run)" >&2 + exit 1 + fi + cp "$CSU_STUBS_PREBUILT" "$STUB_INSTALL" + append_fpc_cfg "$FPC_CFG_MARKER" "-k$STUB_INSTALL" + + if command -v gcc >/dev/null 2>&1; then + CRT_DIR="$(dirname "$(gcc -print-file-name=crti.o)")" + if [ -f "$CRT_DIR/crti.o" ]; then + CRT_MARKER="# powerpc64-linux crt paths (install-fpc-lazarus.sh)" + append_fpc_cfg "$CRT_MARKER" "-Fl$CRT_DIR" "-k-L$CRT_DIR" + echo "Updated fpc.cfg with crt search path $CRT_DIR" + fi + fi + else + cp "$CSU_STUBS_SRC" "$STUB_C" + cc -c -fPIC -o "$WORK_DIR/csu_stubs.o" "$STUB_C" + if [ -f "$CPRT0" ]; then + ld -r -o "$CPRT0.new" "$CPRT0" "$WORK_DIR/csu_stubs.o" + mv "$CPRT0.new" "$CPRT0" + echo "Patched $CPRT0 with glibc 2.34+ stubs." + fi fi fi @@ -205,102 +262,25 @@ fi # not MSYS/Git Bash paths (/c/foo/bin). cygpath converts both ways. # Both bin/ and bin/ go on PATH so subsequent steps find both # fpc.exe (the compiler) and instantfpc.exe (the utility). -if [ -n "${GITHUB_PATH:-}" ]; then - if [ "$IS_WINDOWS" = "1" ]; then - cygpath -w "$FPC_BIN_DIR" >> "$GITHUB_PATH" - cygpath -w "$FPC_UTIL_DIR" >> "$GITHUB_PATH" - else - echo "$FPC_BIN_DIR" >> "$GITHUB_PATH" - fi +ci_github_path_append "$FPC_BIN_DIR" +if [ -n "$FPC_UTIL_DIR" ]; then + ci_github_path_append "$FPC_UTIL_DIR" fi fpc -iV -fpc -iSO -fpc -iSP - -# ── Build Lazarus from source ──────────────────────────────────────── -# -# We always build lazbuild from source rather than using a packaged -# Lazarus. Reasons: -# - Cross-platform consistency: same code path on all 10 targets. -# - The packaged Lazarus on Linux/Windows pulls in the full IDE -# (GTK / Qt / native widgets), which we don't need. -# - lazbuild builds in ~1–2 minutes; cheap relative to the FPC fetch. - -git clone --depth 1 --branch "$LAZARUS_BRANCH" "$LAZARUS_REPO" "$LAZARUS_DIR" -# DragonFlyBSD: Lazarus is missing include/dragonfly/lazconf.inc. -# DragonFlyBSD is a FreeBSD derivative, so the FreeBSD include works -# as-is. Patch it in before building. -if [ "$(uname -s)" = "DragonFly" ]; then - DF_INC="$LAZARUS_DIR/ide/packages/ideconfig/include/dragonfly" - if [ ! -f "$DF_INC/lazconf.inc" ]; then - mkdir -p "$DF_INC" - cp "$LAZARUS_DIR/ide/packages/ideconfig/include/freebsd/lazconf.inc" \ - "$DF_INC/lazconf.inc" - fi -fi - -# On Windows, gmake.exe is a native PE binary that may not understand -# Git Bash's /c/Users/... path style. Pass a Windows-format path. -if [ "$IS_WINDOWS" = "1" ]; then - $MAKE_CMD -C "$(cygpath -w "$LAZARUS_DIR")" lazbuild -else - $MAKE_CMD -C "$LAZARUS_DIR" lazbuild -fi - -# ── Write Lazarus environmentoptions.xml ───────────────────────────── -# lazbuild reads this on startup to locate the Lazarus source tree -# and the FPC compiler. Without it, lazbuild emits: -# Error: (lazbuild) Invalid Lazarus directory "": directory lcl not found -# -# The location lazbuild looks at is platform-specific: -# Unix: $HOME/.lazarus/ (dotted) -# Windows: %LOCALAPPDATA%\lazarus\ — NOT %APPDATA%, NOT dotted -# -# The Windows path comes from Lazarus's lazbaseconf.inc: -# PrimaryConfigPath := ExtractFilePath(ChompPathDelim( -# GetAppConfigDirUTF8(False))) + 'lazarus'; -# where GetAppConfigDir(False) on Windows resolves via -# CSIDL_LOCAL_APPDATA to %LOCALAPPDATA%\\, so the parent + -# 'lazarus' is %LOCALAPPDATA%\lazarus\ (no leading dot). +# ── Build Lazarus from source (lazbuild | auto only) ───────────────── # -# On Windows, lazbuild is a native PE binary and expects Windows- -# style paths in the XML values, so we cygpath them to backslash form. -if [ "$IS_WINDOWS" = "1" ]; then - # On Windows runners $LOCALAPPDATA is set; fall back to the - # well-known location if it isn't. - WIN_LOCALAPPDATA="${LOCALAPPDATA:-$USERPROFILE/AppData/Local}" - LAZ_CFG_DIR="$(cygpath -u "$WIN_LOCALAPPDATA")/lazarus" - LAZ_DIR_NATIVE="$(cygpath -w "$LAZARUS_DIR")" - FPC_EXE_NATIVE="$(cygpath -w "$FPC_EXE")" -else - LAZ_CFG_DIR="${HOME}/.lazarus" - LAZ_DIR_NATIVE="$LAZARUS_DIR" - FPC_EXE_NATIVE="$FPC_EXE" -fi - -mkdir -p "$LAZ_CFG_DIR" +# When INSTALL_LAZARUS=1, clone Lazarus and build lazbuild (~1–2 min). +# Packaged Lazarus on Linux/Windows pulls in the full IDE; we only need +# lazbuild for CI package/project builds. -cat > "$LAZ_CFG_DIR/environmentoptions.xml" < - - - - - - -EOF - -# Put lazbuild on PATH for subsequent steps. -export PATH="$LAZARUS_DIR:$PATH" -if [ -n "${GITHUB_PATH:-}" ]; then - if [ "$IS_WINDOWS" = "1" ]; then - cygpath -w "$LAZARUS_DIR" >> "$GITHUB_PATH" - else - echo "$LAZARUS_DIR" >> "$GITHUB_PATH" - fi +if [ "$INSTALL_LAZARUS" = "0" ]; then + echo "MAKE_BUILD_BACKEND=fpc — FPC install complete (lazbuild skipped)." + exit 0 fi -lazbuild --version +export FPC_EXE +# shellcheck source=ci/shared/lazarus-bootstrap.sh +source "$INSTALL_SCRIPT_DIR/ci/shared/lazarus-bootstrap.sh" echo "FPC + Lazarus installation complete." \ No newline at end of file diff --git a/.github/workflows/make.pas b/.github/workflows/make.pas index fa4492d..ba4c094 100644 --- a/.github/workflows/make.pas +++ b/.github/workflows/make.pas @@ -3,45 +3,27 @@ {$SCOPEDENUMS ON} uses - Classes, - SysUtils, - StrUtils, - Zipper, - fphttpclient, - RegExpr, - openssl, - opensslsockets, - Process; + SysUtils, Classes, Generics.Collections, StrUtils, Process, RegExpr, + Zipper, fphttpclient, openssl, opensslsockets; -const - Target: string = 'SimpleBaseLib.Tests'; - - // ANSI color codes - CSI_Reset = #27'[0m'; - CSI_Red = #27'[31m'; - CSI_Green = #27'[32m'; - CSI_Yellow = #27'[33m'; - CSI_Cyan = #27'[36m'; +type + TMakeRunner = class; + TLpkPathProc = procedure(const ALpkPath: string) of object; - // Package path filter — skip platform-incompatible and template packages - PackageExcludePattern = - {$IF DEFINED(MSWINDOWS)} - '(cocoa|x11|_template)' - {$ELSEIF DEFINED(DARWIN)} - '(gdi|x11|_template)' - {$ELSE} - '(cocoa|gdi|_template)' - {$IFEND} - ; + // --------------------------------------------------------------------------- + // Build backend + // --------------------------------------------------------------------------- - OPMBaseUrl = 'https://packages.lazarus-ide.org/'; - GitHubArchiveBaseUrl = 'https://github.com/'; + TBuildBackend = ( + Auto, + Lazbuild, + Fpc + ); -// --------------------------------------------------------------------------- -// Dependency configuration -// --------------------------------------------------------------------------- + // --------------------------------------------------------------------------- + // Dependency configuration + // --------------------------------------------------------------------------- -type TDependencyKind = (OPM, GitHub); TDependency = record @@ -50,39 +32,265 @@ TDependency = record Ref: string; // GitHub: branch, tag or commit (ignored for OPM) end; + // --------------------------------------------------------------------------- + // Lazarus project / package XML + // --------------------------------------------------------------------------- + + TLazCompilerOptions = record + CompilerMode: string; + OptLevel: string; + UseDwarfSets: Boolean; + CustomConfigFile: string; + IncludePaths: string; + UnitPaths: string; + UnitOutputDirTemplate: string; + end; + + TLazXml = class + public + class function ReadFile(const AFileName: string): string; + class function ExtractBlock(const AContent, AOpenTag: string): string; + class function ExtractAttr(const AContent, ATag: string): string; + class function ParseCompilerOptions(const AContent: string): TLazCompilerOptions; + class function ResolveUnitOutputDir(const AOptions: TLazCompilerOptions; + const AProjDir, ATargetCpu, ATargetOs: string): string; + class procedure AppendCompilerOptionsToArgv(const AOptions: TLazCompilerOptions; + const AProjDir, AUnitOutDir, APkgOutDir, ATargetCpu, ATargetOs: string; + AArgs: TStrings); + class function ResolvePath(const AValue, AProjDir, AUnitOutDir, APkgOutDir, + ATargetCpu, ATargetOs: string): string; + class function CollectFileSourceDirs(const AContent, APkgDir, ATargetCpu, + ATargetOs: string): TStringList; + class function ExtractPackageNames(const ABlock: string): TStringList; + class function ExtractPackageNamesFromContent(const AContent, + ABlockTag: string): TStringList; + class function ContentRequiresPackage(const AContent, APackageName, + ABlockTag: string): Boolean; + class function ExtractUnitNames(const AContent: string): TStringList; + class procedure AppendPackageBuildArgs(AArgs: TStrings; + const AStubFileName, AUnitOutDir: string); + class procedure AppendProjectBuildArgs(AArgs: TStrings; + const AMainSource, AUnitOutDir, ATargetBinary: string); + private + class function IsAbsolutePath(const S: string): Boolean; + class function ExpandMacros(const S, AProjDir, AUnitOutDir, APkgOutDir, + ATargetCpu, ATargetOs: string): string; + class procedure AppendSearchPathArgs(const APaths, AProjDir, AUnitOutDir, + APkgOutDir, ATargetCpu, ATargetOs, APrefix: string; AArgs: TStrings); + class function ArgsHasFuPath(const AArgs: TStrings; const APath: string): Boolean; + class procedure AppendFuIfMissing(const APath: string; AArgs: TStrings); + end; + + TProjectFiles = class + public + class function FindAll(const ASearchDir, AMask: string): TStringList; + class procedure RemoveRecursive(const ADir: string); + private + class function MatchesMask(const AFileName, AMask: string): Boolean; + class function IsBackupDir(const ADirName: string): Boolean; + class function ShouldExcludePath(const AFilePath: string): Boolean; + class procedure FindRecursive(const ADir, AMask: string; AList: TStrings); + end; + + TLpiProject = class + private + FLpiPath: string; + FProjDir: string; + FMainLpr: string; + FUnitOutDir: string; + FTargetBinary: string; + FOptions: TLazCompilerOptions; + FRequiredPackageNames: TStringList; + public + constructor CreateFromFile(const ALpiPath, ATargetCpu, ATargetOs: string); + destructor Destroy; override; + function IsValid: Boolean; + function BuildFpcArgv(const AExtraUnitPaths: TStrings; + ATargetCpu, ATargetOs: string): TStringList; + property RequiredPackageNames: TStringList read FRequiredPackageNames; + property TargetBinary: string read FTargetBinary; + property UnitOutDir: string read FUnitOutDir; + property ProjDir: string read FProjDir; + end; + + TLpkPackage = class + private + FLpkPath: string; + FPkgDir: string; + FPackageName: string; + FStubPas: string; + FUnitOutDir: string; + FOptions: TLazCompilerOptions; + FRequiredNames: TStringList; + FHasLclDependency: Boolean; + FSourceDirs: TStringList; + public + constructor CreateFromFile(const ALpkPath, ATargetCpu, ATargetOs: string); + destructor Destroy; override; + function IsValid: Boolean; + function ResolveUnitOutDir(const ATargetCpu, ATargetOs: string): string; + property PackageName: string read FPackageName; + property UnitOutDir: string read FUnitOutDir; + property PkgDir: string read FPkgDir; + property Options: TLazCompilerOptions read FOptions; + property RequiredNames: TStringList read FRequiredNames; + property SourceDirs: TStringList read FSourceDirs; + property StubPas: string read FStubPas; + property HasLclDependency: Boolean read FHasLclDependency; + class function HasLclDependencyInFile(const ALpkPath: string): Boolean; + end; + + TDepVisitKind = (BuildOrder, UnitPaths); + + TPackageGraph = class + private + FRunner: TMakeRunner; + FItems: specialize TObjectList; + FNameToIndex: TStringList; + function GetPackage(Index: Integer): TLpkPackage; + function FindIndexByName(const AName: string): Integer; + function IsBuiltinPackage(const AName: string): Boolean; + function ResolveDepIndex(const APackageName, AContext: string): Integer; + procedure VisitPackageDeps(const AIndex: Integer; AVisited: specialize TList; + AKind: TDepVisitKind; AOrder: specialize TList; APaths: TStrings); + procedure CollectBuildOrder(const AIndex: Integer; AOrder: specialize TList); + procedure CollectUnitPaths(const AIndex: Integer; AVisited: specialize TList; + APaths: TStrings); + public + constructor Create(ARunner: TMakeRunner); + destructor Destroy; override; + procedure DiscoverUnder(const ARoot: string); + procedure RegisterLpk(const ALpkPath: string); + function BuildAll: Boolean; + function UnitPathFor(const APackageName: string): string; + function UnitPathsForRequired(const ANames: TStrings): TStringList; + function PackageCount: Integer; + class function ExcludePattern: string; + class function ShouldExcludeLpkPath(const ALpkPath: string): Boolean; + class function ShouldSkipLpk(const ALpkPath: string): Boolean; + end; + + // --------------------------------------------------------------------------- + // Main orchestrator + // --------------------------------------------------------------------------- + + TMakeRunner = class + private + FBackend: TBuildBackend; + FBackendResolved: Boolean; + FTargetCpu: string; + FTargetOs: string; + FErrorCount: Integer; + FGraph: TPackageGraph; + function ParseBackendEnv: TBuildBackend; + function ResolveAutoBackend: TBuildBackend; + procedure InitEnvironment; + procedure UpdateSubmodules; + procedure InstallDependencies; + procedure BuildAllProjects; + function BuildProject(const ALpiPath: string): string; + function BuildProjectWithLazbuild(const APath: string): string; + function BuildProjectWithFpc(const APath: string): string; + function ExtractBinaryFromBuildLog(const AOutput, AFallback: string): string; + function IsGUIProject(const ALpiPath: string): Boolean; + function IsTestProject(const ALpiPath: string): Boolean; + procedure RunTestProject(const APath: string); + procedure RunSampleProject(const APath: string); + procedure InitSslForDownloads; + procedure DownloadAndExtract(const AUrl, ADestDir: string); + function GetDepsBaseDir(const ASubDir: string): string; + function InstallOPMPackage(const APackageName: string): string; + function InstallGitHubPackage(const AOwnerRepo, ARef: string): string; + function ResolveDependency(const ADep: TDependency): string; + procedure RegisterPackageLazbuild(const APath: string); + procedure RegisterAllPackagesLazbuild(const ASearchDir: string); + function UsesLazbuild: Boolean; + function RunCommandEx(const AExecutable: string; const AArgs: TStrings; + const AWorkingDir: string; AStreamToStderr: Boolean; + out AOutput: string): Boolean; overload; + function RunCommandEx(const AExecutable: string; + const AArgs: array of string; const AWorkingDir: string; + AStreamToStderr: Boolean; out AOutput: string): Boolean; overload; + function RepoRoot: string; + function TargetDirectory: string; + procedure ForEachLpkInDir(const ARoot: string; ACallback: TLpkPathProc); + procedure RunBuiltBinary(const ABinaryPath: string; + const AArgs: array of string; const AFailMessage: string); + procedure NormalizeFpcTarget(var AValue: string); + function RunFpcInfoProbeWithRetry(const AInfoFlag: string; + out AValue: string): Boolean; + procedure PrepareProjectBuild(Proj: TLpiProject); + public + constructor Create; + destructor Destroy; override; + function Execute: Integer; + procedure Log(const AColor, AMessage: string); + procedure LogInline(const AColor, AMessage: string); + procedure ReportBuildErrors(const ABuildOutput: string); + procedure ReportSummary; + procedure IncError; + property TargetCpu: string read FTargetCpu; + property TargetOs: string read FTargetOs; + end; + +// --------------------------------------------------------------------------- +// Configuration constants +// --------------------------------------------------------------------------- + const + Target: string = 'SimpleBaseLib.Tests'; + + CSI_Reset = #27'[0m'; + CSI_Red = #27'[31m'; + CSI_Green = #27'[32m'; + CSI_Yellow = #27'[33m'; + CSI_Cyan = #27'[36m'; + + OPMBaseUrl = 'https://packages.lazarus-ide.org/'; + GitHubArchiveBaseUrl = 'https://github.com/'; + Dependencies: array of TDependency = ( // Examples: - // (Kind: TDependencyKind.OPM; Name: 'HashLib'; Ref: ''), - // (Kind: TDependencyKind.GitHub; Name: 'Xor-el/SimpleBaseLib4Pascal'; Ref: 'master'), + // (Kind: TDependencyKind.OPM; Name: 'HashLib'; Ref: ''), + // (Kind: TDependencyKind.GitHub; Name: 'Xor-el/SimpleBaseLib4Pascal'; Ref: 'master'), ); // --------------------------------------------------------------------------- -// Helpers for building TDependency records (optional convenience) +// Dependency helpers // --------------------------------------------------------------------------- function OPM(const AName: string): TDependency; begin Result.Kind := TDependencyKind.OPM; Result.Name := AName; - Result.Ref := ''; + Result.Ref := ''; end; function GitHub(const AOwnerRepo, ARef: string): TDependency; begin Result.Kind := TDependencyKind.GitHub; Result.Name := AOwnerRepo; - Result.Ref := ARef; + Result.Ref := ARef; end; -var - ErrorCount: Integer = 0; - // --------------------------------------------------------------------------- -// FCL/RTL-only helpers (replace FileUtil usage) +// TLazXml // --------------------------------------------------------------------------- -function ReadFileToString(const AFileName: string): string; +{ TLazXml } + +class function TLazXml.IsAbsolutePath(const S: string): Boolean; +begin + {$IFDEF MSWINDOWS} + Result := (Length(S) >= 2) and ( + ((UpCase(S[1]) >= 'A') and (UpCase(S[1]) <= 'Z') and (S[2] = ':')) or + (S[1] = '\')); + {$ELSE} + Result := (Length(S) > 0) and (S[1] = '/'); + {$ENDIF} +end; + +class function TLazXml.ReadFile(const AFileName: string): string; var Stream: TFileStream; Size: Int64; @@ -101,40 +309,343 @@ function ReadFileToString(const AFileName: string): string; end; end; -function MatchesMaskSimple(const AFileName, AMask: string): Boolean; +class function TLazXml.ExtractBlock(const AContent, AOpenTag: string): string; +var + P, Q, TagLen: Integer; + CloseTag: string; + NextCh: Char; +begin + Result := ''; + TagLen := Length(AOpenTag) + 1; + P := Pos('<' + AOpenTag, AContent); + while P > 0 do + begin + if P + TagLen > Length(AContent) then + Break; + NextCh := AContent[P + TagLen]; + if (NextCh = '>') or (NextCh = ' ') or (NextCh = #9) or (NextCh = '/') then + Break; + P := PosEx('<' + AOpenTag, AContent, P + 1); + end; + if P = 0 then + Exit; + CloseTag := ''; + Q := PosEx(CloseTag, AContent, P); + if Q = 0 then + Result := Copy(AContent, P, MaxInt) + else + Result := Copy(AContent, P, Q - P + Length(CloseTag)); +end; + +class function TLazXml.ExtractAttr(const AContent, ATag: string): string; +var + Needle: string; + P, Q: Integer; +begin + Result := ''; + Needle := '<' + ATag + ' Value="'; + P := Pos(Needle, AContent); + if P = 0 then + Exit; + Inc(P, Length(Needle)); + Q := PosEx('"', AContent, P); + if Q = 0 then + Exit; + Result := Copy(AContent, P, Q - P); +end; + +class function TLazXml.ExpandMacros(const S, AProjDir, AUnitOutDir, APkgOutDir, + ATargetCpu, ATargetOs: string): string; +begin + Result := S; + Result := StringReplace(Result, '$(ProjOutDir)', AUnitOutDir, [rfReplaceAll]); + Result := StringReplace(Result, '$(PkgOutDir)', APkgOutDir, [rfReplaceAll]); + Result := StringReplace(Result, '$(TargetCPU)', ATargetCpu, [rfReplaceAll]); + Result := StringReplace(Result, '$(TargetOS)', ATargetOs, [rfReplaceAll]); + Result := StringReplace(Result, '\', PathDelim, [rfReplaceAll]); + if Result = '' then + Exit; + if not IsAbsolutePath(Result) then + Result := ExpandFileName(IncludeTrailingPathDelimiter(AProjDir) + Result); +end; + +class function TLazXml.ResolvePath(const AValue, AProjDir, AUnitOutDir, + APkgOutDir, ATargetCpu, ATargetOs: string): string; +begin + Result := ExpandMacros(Trim(AValue), AProjDir, AUnitOutDir, APkgOutDir, + ATargetCpu, ATargetOs); +end; + +class procedure TLazXml.AppendSearchPathArgs(const APaths, AProjDir, + AUnitOutDir, APkgOutDir, ATargetCpu, ATargetOs, APrefix: string; + AArgs: TStrings); +var + Parts: TStringArray; + I: Integer; + PathItem: string; +begin + Parts := SplitString(APaths, ';'); + for I := 0 to High(Parts) do + begin + PathItem := Trim(Parts[I]); + if PathItem = '' then + Continue; + AArgs.Add(APrefix + ResolvePath(PathItem, AProjDir, AUnitOutDir, APkgOutDir, + ATargetCpu, ATargetOs)); + end; +end; + +class function TLazXml.ArgsHasFuPath(const AArgs: TStrings; const APath: string): Boolean; +var + I: Integer; + Norm, ArgPath: string; +begin + Result := False; + if APath = '' then + Exit; + Norm := LowerCase(ExpandFileName(ExcludeTrailingPathDelimiter(APath))); + for I := 0 to AArgs.Count - 1 do + begin + if not StartsText('-Fu', AArgs[I]) then + Continue; + ArgPath := Copy(AArgs[I], 4, MaxInt); + if SameText(Norm, LowerCase(ExpandFileName(ExcludeTrailingPathDelimiter(ArgPath)))) then + Exit(True); + end; +end; + +class procedure TLazXml.AppendFuIfMissing(const APath: string; AArgs: TStrings); +begin + if (APath <> '') and not ArgsHasFuPath(AArgs, APath) then + AArgs.Add('-Fu' + IncludeTrailingPathDelimiter(APath)); +end; + +class function TLazXml.CollectFileSourceDirs(const AContent, APkgDir, ATargetCpu, + ATargetOs: string): TStringList; +var + Block, FilePath, DirPath, Ext: string; + Filter: TRegExpr; +begin + Result := TStringList.Create; + Result.Sorted := True; + Result.Duplicates := dupIgnore; + Block := ExtractBlock(AContent, 'Files'); + if Block = '' then + Exit; + Filter := TRegExpr.Create(''); + try + if Filter.Exec(Block) then + repeat + FilePath := Filter.Match[1]; + Ext := LowerCase(ExtractFileExt(FilePath)); + if (Ext = '.pas') or (Ext = '.pp') or (Ext = '.p') then + begin + DirPath := ResolvePath(ExtractFilePath(FilePath), APkgDir, '', '', + ATargetCpu, ATargetOs); + if DirPath <> '' then + Result.Add(ExcludeTrailingPathDelimiter(DirPath)); + end; + until not Filter.ExecNext; + finally + Filter.Free; + end; +end; + +class function TLazXml.ParseCompilerOptions(const AContent: string): TLazCompilerOptions; +var + Block: string; +begin + Result.CompilerMode := 'delphi'; + Result.OptLevel := '2'; + Result.UseDwarfSets := Pos('dsDwarf3', AContent) > 0; + Result.CustomConfigFile := ''; + Result.IncludePaths := ''; + Result.UnitPaths := ''; + Result.UnitOutputDirTemplate := 'lib\$(TargetCPU)-$(TargetOS)'; + + Block := ExtractBlock(AContent, 'CompilerOptions'); + if Block = '' then + Exit; + + if ExtractAttr(Block, 'OptimizationLevel') <> '' then + Result.OptLevel := ExtractAttr(Block, 'OptimizationLevel'); + Result.IncludePaths := ExtractAttr(Block, 'IncludeFiles'); + Result.UnitPaths := ExtractAttr(Block, 'OtherUnitFiles'); + if ExtractAttr(Block, 'UnitOutputDirectory') <> '' then + Result.UnitOutputDirTemplate := ExtractAttr(Block, 'UnitOutputDirectory'); + + if Pos(' 0 then + Result.CustomConfigFile := ExtractAttr(Block, 'ConfigFilePath'); + + if ExtractAttr(Block, 'SyntaxMode') <> '' then + Result.CompilerMode := LowerCase(ExtractAttr(Block, 'SyntaxMode')) + else if ExtractAttr(Block, 'CompilerMode') <> '' then + Result.CompilerMode := LowerCase(ExtractAttr(Block, 'CompilerMode')); + if Result.CompilerMode = '' then + Result.CompilerMode := 'delphi'; +end; + +class function TLazXml.ResolveUnitOutputDir(const AOptions: TLazCompilerOptions; + const AProjDir, ATargetCpu, ATargetOs: string): string; +begin + Result := ResolvePath(AOptions.UnitOutputDirTemplate, AProjDir, '', '', + ATargetCpu, ATargetOs); + ForceDirectories(Result); +end; + +class procedure TLazXml.AppendCompilerOptionsToArgv(const AOptions: TLazCompilerOptions; + const AProjDir, AUnitOutDir, APkgOutDir, ATargetCpu, ATargetOs: string; + AArgs: TStrings); +var + ConfigPath: string; +begin + if AOptions.CompilerMode <> '' then + AArgs.Add('-M' + AOptions.CompilerMode) + else + AArgs.Add('-Mdelphi'); + AArgs.Add('-O' + AOptions.OptLevel); + if AOptions.UseDwarfSets then + AArgs.Add('-godwarfsets'); + AppendSearchPathArgs(AOptions.IncludePaths, AProjDir, AUnitOutDir, APkgOutDir, + ATargetCpu, ATargetOs, '-Fi', AArgs); + AppendSearchPathArgs(AOptions.UnitPaths, AProjDir, AUnitOutDir, APkgOutDir, + ATargetCpu, ATargetOs, '-Fu', AArgs); + if AOptions.CustomConfigFile <> '' then + begin + ConfigPath := ResolvePath(AOptions.CustomConfigFile, AProjDir, AUnitOutDir, + APkgOutDir, ATargetCpu, ATargetOs); + if FileExists(ConfigPath) then + AArgs.Add('@' + ConfigPath); + end; +end; + +class function TLazXml.ExtractPackageNames(const ABlock: string): TStringList; +var + Filter: TRegExpr; +begin + Result := TStringList.Create; + if ABlock = '' then + Exit; + Filter := TRegExpr.Create(''); + try + if Filter.Exec(ABlock) then + repeat + Result.Add(Filter.Match[1]); + until not Filter.ExecNext; + finally + Filter.Free; + end; +end; + +class function TLazXml.ExtractPackageNamesFromContent(const AContent, + ABlockTag: string): TStringList; +begin + Result := ExtractPackageNames(ExtractBlock(AContent, ABlockTag)); +end; + +class function TLazXml.ContentRequiresPackage(const AContent, APackageName, + ABlockTag: string): Boolean; +var + Names: TStringList; + I: Integer; +begin + Result := False; + Names := ExtractPackageNamesFromContent(AContent, ABlockTag); + try + for I := 0 to Names.Count - 1 do + if SameText(Names[I], APackageName) then + Exit(True); + finally + Names.Free; + end; +end; + +class function TLazXml.ExtractUnitNames(const AContent: string): TStringList; +var + Filter: TRegExpr; +begin + Result := TStringList.Create; + Filter := TRegExpr.Create(''); + try + if Filter.Exec(AContent) then + repeat + Result.Add(Filter.Match[1]); + until not Filter.ExecNext; + finally + Filter.Free; + end; +end; + +class procedure TLazXml.AppendPackageBuildArgs(AArgs: TStrings; + const AStubFileName, AUnitOutDir: string); +begin + AArgs.Add('-FU' + IncludeTrailingPathDelimiter(AUnitOutDir)); + AArgs.Add('-B'); + AArgs.Add(AStubFileName); +end; + +class procedure TLazXml.AppendProjectBuildArgs(AArgs: TStrings; + const AMainSource, AUnitOutDir, ATargetBinary: string); +begin + AArgs.Add('-FU' + AUnitOutDir); + AArgs.Add('-FE' + ExtractFilePath(ATargetBinary)); + AArgs.Add('-B'); + AArgs.Add('-o' + ATargetBinary); + AArgs.Add(AMainSource); +end; + +// --------------------------------------------------------------------------- +// TProjectFiles +// --------------------------------------------------------------------------- + +{ TProjectFiles } + +class function TProjectFiles.MatchesMask(const AFileName, AMask: string): Boolean; var LExt: string; begin LExt := LowerCase(ExtractFileExt(AFileName)); - if AMask = '*.lpk' then Exit(LExt = '.lpk'); - if AMask = '*.lpi' then Exit(LExt = '.lpi'); - Result := False; end; -procedure FindAllFilesRecursive(const ADir, AMask: string; AList: TStrings); +class function TProjectFiles.IsBackupDir(const ADirName: string): Boolean; +begin + Result := SameText(ADirName, 'backup'); +end; + +class function TProjectFiles.ShouldExcludePath(const AFilePath: string): Boolean; +var + Norm: string; +begin + Norm := LowerCase(ExpandFileName(AFilePath)); + Result := Pos(PathDelim + 'backup' + PathDelim, Norm) > 0; +end; + +class procedure TProjectFiles.FindRecursive(const ADir, AMask: string; + AList: TStrings); var Search: TSearchRec; - DirPath: string; - EntryPath: string; + DirPath, EntryPath: string; begin DirPath := IncludeTrailingPathDelimiter(ExpandFileName(ADir)); - if FindFirst(DirPath + '*', faAnyFile, Search) = 0 then try repeat if (Search.Name = '.') or (Search.Name = '..') then Continue; - - EntryPath := DirPath + Search.Name; - if (Search.Attr and faDirectory) <> 0 then - FindAllFilesRecursive(EntryPath, AMask, AList) - else if MatchesMaskSimple(Search.Name, AMask) then + begin + if IsBackupDir(Search.Name) then + Continue; + FindRecursive(DirPath + Search.Name, AMask, AList); + Continue; + end; + EntryPath := DirPath + Search.Name; + if MatchesMask(Search.Name, AMask) and not ShouldExcludePath(EntryPath) then AList.Add(EntryPath); until FindNext(Search) <> 0; finally @@ -142,203 +653,869 @@ procedure FindAllFilesRecursive(const ADir, AMask: string; AList: TStrings); end; end; -function FindAllFilesList(const ASearchDir, AMask: string): TStringList; +class function TProjectFiles.FindAll(const ASearchDir, AMask: string): TStringList; begin Result := TStringList.Create; - FindAllFilesRecursive(ASearchDir, AMask, Result); + FindRecursive(ASearchDir, AMask, Result); +end; + +class procedure TProjectFiles.RemoveRecursive(const ADir: string); +var + Search: TSearchRec; + NormDir, DirPath, EntryPath: string; +begin + if (ADir = '') or not DirectoryExists(ADir) then + Exit; + NormDir := ExpandFileName(ADir); + DirPath := IncludeTrailingPathDelimiter(NormDir); + if FindFirst(DirPath + '*', faAnyFile, Search) = 0 then + try + repeat + if (Search.Name = '.') or (Search.Name = '..') then + Continue; + EntryPath := DirPath + Search.Name; + if (Search.Attr and faDirectory) <> 0 then + RemoveRecursive(EntryPath) + else + DeleteFile(EntryPath); + until FindNext(Search) <> 0; + finally + FindClose(Search); + end; + RemoveDir(NormDir); end; // --------------------------------------------------------------------------- -// Logging helpers +// TLpiProject // --------------------------------------------------------------------------- -procedure Log(const AColor, AMessage: string); +{ TLpiProject } + +constructor TLpiProject.CreateFromFile(const ALpiPath, ATargetCpu, + ATargetOs: string); +var + Content, Block, Name: string; + PkgNames: TStringList; begin - WriteLn(stderr, AColor, AMessage, CSI_Reset); + inherited Create; + FRequiredPackageNames := TStringList.Create; + FLpiPath := ALpiPath; + FProjDir := ExtractFilePath(ALpiPath); + + if not FileExists(ALpiPath) then + Exit; + + Content := TLazXml.ReadFile(ALpiPath); + FOptions := TLazXml.ParseCompilerOptions(Content); + FUnitOutDir := TLazXml.ResolveUnitOutputDir(FOptions, FProjDir, ATargetCpu, ATargetOs); + + Block := TLazXml.ExtractBlock(Content, 'Unit0'); + if Block = '' then + Block := Content; + Name := TLazXml.ExtractAttr(Block, 'Filename'); + if Name <> '' then + FMainLpr := TLazXml.ResolvePath(Name, FProjDir, '', '', ATargetCpu, ATargetOs); + + Block := TLazXml.ExtractBlock(Content, 'Target'); + if Block <> '' then + begin + Name := TLazXml.ExtractAttr(Block, 'Filename'); + if Name <> '' then + FTargetBinary := TLazXml.ResolvePath(Name, FProjDir, FUnitOutDir, '', + ATargetCpu, ATargetOs); + end; + if FTargetBinary = '' then + FTargetBinary := ChangeFileExt(FMainLpr, ''); + + PkgNames := TLazXml.ExtractPackageNamesFromContent(Content, 'RequiredPackages'); + try + FRequiredPackageNames.Assign(PkgNames); + finally + PkgNames.Free; + end; end; -procedure LogInline(const AColor, AMessage: string); +destructor TLpiProject.Destroy; begin - Write(stderr, AColor, AMessage, CSI_Reset); + FRequiredPackageNames.Free; + inherited Destroy; +end; + +function TLpiProject.IsValid: Boolean; +begin + Result := (FLpiPath <> '') and FileExists(FLpiPath) and (FMainLpr <> '') and + FileExists(FMainLpr); +end; + +function TLpiProject.BuildFpcArgv(const AExtraUnitPaths: TStrings; + ATargetCpu, ATargetOs: string): TStringList; +var + I: Integer; +begin + Result := TStringList.Create; + TLazXml.AppendCompilerOptionsToArgv(FOptions, FProjDir, FUnitOutDir, FUnitOutDir, + ATargetCpu, ATargetOs, Result); + if Assigned(AExtraUnitPaths) then + for I := 0 to AExtraUnitPaths.Count - 1 do + TLazXml.AppendFuIfMissing(AExtraUnitPaths[I], Result); + TLazXml.AppendProjectBuildArgs(Result, FMainLpr, FUnitOutDir, FTargetBinary); end; // --------------------------------------------------------------------------- -// Git submodules +// TLpkPackage // --------------------------------------------------------------------------- -procedure UpdateSubmodules; +{ TLpkPackage } + +constructor TLpkPackage.CreateFromFile(const ALpkPath, ATargetCpu, + ATargetOs: string); var - CommandOutput: ansistring; + Content, Block: string; + Units: string; + UnitNames, PkgNames: TStringList; + I: Integer; + SL: TStringList; begin - if not FileExists('.gitmodules') then + inherited Create; + FRequiredNames := TStringList.Create; + FLpkPath := ALpkPath; + FPkgDir := ExtractFilePath(ALpkPath); + + if not FileExists(ALpkPath) then + begin + FSourceDirs := TStringList.Create; Exit; - if RunCommand('git', ['submodule', 'update', '--init', '--recursive', - '--force', '--remote'], CommandOutput) then - Log(CSI_Yellow, Trim(CommandOutput)); + end; + + Content := TLazXml.ReadFile(ALpkPath); + FSourceDirs := TLazXml.CollectFileSourceDirs(Content, FPkgDir, ATargetCpu, ATargetOs); + Block := TLazXml.ExtractBlock(Content, 'Package'); + if Block <> '' then + FPackageName := TLazXml.ExtractAttr(Block, 'Name') + else + FPackageName := TLazXml.ExtractAttr(Content, 'Name'); + FOptions := TLazXml.ParseCompilerOptions(Content); + FUnitOutDir := TLazXml.ResolveUnitOutputDir(FOptions, FPkgDir, ATargetCpu, ATargetOs); + + PkgNames := TLazXml.ExtractPackageNamesFromContent(Content, 'RequiredPkgs'); + try + for I := 0 to PkgNames.Count - 1 do + begin + if SameText(PkgNames[I], 'LCL') then + FHasLclDependency := True; + FRequiredNames.Add(PkgNames[I]); + end; + finally + PkgNames.Free; + end; + + FStubPas := IncludeTrailingPathDelimiter(FPkgDir) + FPackageName + '.pas'; + if not FileExists(FStubPas) then + begin + Units := ''; + UnitNames := TLazXml.ExtractUnitNames(Content); + try + for I := 0 to UnitNames.Count - 1 do + begin + if Units <> '' then + Units := Units + ', '; + Units := Units + UnitNames[I]; + end; + finally + UnitNames.Free; + end; + + SL := TStringList.Create; + try + SL.Add('{ Auto-generated by Make for package compile }'); + SL.Add(''); + SL.Add('unit ' + FPackageName + ';'); + SL.Add(''); + SL.Add('{$warn 5023 off : no warning about unused units}'); + SL.Add('interface'); + SL.Add(''); + SL.Add('uses'); + SL.Add(' ' + Units + ';'); + SL.Add(''); + SL.Add('implementation'); + SL.Add(''); + SL.Add('end.'); + SL.SaveToFile(FStubPas); + finally + SL.Free; + end; + end; +end; + +destructor TLpkPackage.Destroy; +begin + FSourceDirs.Free; + FRequiredNames.Free; + inherited Destroy; +end; + +function TLpkPackage.ResolveUnitOutDir(const ATargetCpu, ATargetOs: string): string; +begin + FUnitOutDir := TLazXml.ResolveUnitOutputDir(FOptions, FPkgDir, ATargetCpu, ATargetOs); + Result := FUnitOutDir; +end; + +function TLpkPackage.IsValid: Boolean; +begin + Result := (FLpkPath <> '') and FileExists(FLpkPath) and (FPackageName <> '') and + (FStubPas <> '') and FileExists(FStubPas); +end; + +class function TLpkPackage.HasLclDependencyInFile(const ALpkPath: string): Boolean; +var + Content: string; +begin + Result := False; + if not FileExists(ALpkPath) then + Exit; + Content := TLazXml.ReadFile(ALpkPath); + Result := TLazXml.ContentRequiresPackage(Content, 'LCL', 'RequiredPkgs'); end; // --------------------------------------------------------------------------- -// Package registration +// TPackageGraph // --------------------------------------------------------------------------- -procedure RegisterPackage(const APath: string); +{ TPackageGraph } + +class function TPackageGraph.ExcludePattern: string; +begin + {$IF DEFINED(MSWINDOWS)} + Result := '(cocoa|x11|_template)'; + {$ELSEIF DEFINED(DARWIN)} + Result := '(gdi|x11|_template)'; + {$ELSE} + Result := '(cocoa|gdi|_template)'; + {$IFEND} +end; + +class function TPackageGraph.ShouldExcludeLpkPath(const ALpkPath: string): Boolean; var Filter: TRegExpr; - CommandOutput: ansistring; begin - Filter := TRegExpr.Create(PackageExcludePattern); + Filter := TRegExpr.Create(ExcludePattern); try - if Filter.Exec(APath) then - Exit; - if RunCommand('lazbuild', ['--add-package-link', APath], CommandOutput) then - Log(CSI_Yellow, 'added ' + APath); + Result := Filter.Exec(ALpkPath); finally Filter.Free; end; end; -// --------------------------------------------------------------------------- -// Extract linked binary path from lazbuild output -// --------------------------------------------------------------------------- +class function TPackageGraph.ShouldSkipLpk(const ALpkPath: string): Boolean; +begin + Result := ShouldExcludeLpkPath(ALpkPath) or + TLpkPackage.HasLclDependencyInFile(ALpkPath); +end; -function ExtractLinkedBinary(const ABuildOutput: string): string; +constructor TPackageGraph.Create(ARunner: TMakeRunner); +begin + inherited Create; + FRunner := ARunner; + FItems := specialize TObjectList.Create(True); + FNameToIndex := TStringList.Create; + FNameToIndex.Sorted := True; + FNameToIndex.Duplicates := dupError; +end; + +destructor TPackageGraph.Destroy; +begin + FNameToIndex.Free; + FItems.Free; + inherited Destroy; +end; + +function TPackageGraph.GetPackage(Index: Integer): TLpkPackage; +begin + Result := FItems[Index]; +end; + +function TPackageGraph.PackageCount: Integer; +begin + Result := FItems.Count; +end; + +function TPackageGraph.IsBuiltinPackage(const AName: string): Boolean; +begin + Result := SameText(AName, 'FCL') or SameText(AName, 'RTL') or + SameText(AName, 'FCLBase'); +end; + +function TPackageGraph.FindIndexByName(const AName: string): Integer; var - Line: string; - Parts: TStringArray; + Idx: Integer; +begin + Result := -1; + Idx := FNameToIndex.IndexOf(AName); + if Idx >= 0 then + Result := Integer(PtrInt(FNameToIndex.Objects[Idx])); +end; + +procedure TPackageGraph.RegisterLpk(const ALpkPath: string); +var + Pkg: TLpkPackage; +begin + if ShouldSkipLpk(ALpkPath) then + begin + if not ShouldExcludeLpkPath(ALpkPath) then + FRunner.Log(CSI_Yellow, 'skip LCL-dependent package ' + ALpkPath); + Exit; + end; + + Pkg := TLpkPackage.CreateFromFile(ALpkPath, FRunner.TargetCpu, FRunner.TargetOs); + if not Pkg.IsValid then + begin + FRunner.Log(CSI_Red, 'failed to load package: ' + ALpkPath); + FRunner.IncError; + Pkg.Free; + Exit; + end; + + if FindIndexByName(Pkg.PackageName) >= 0 then + begin + Pkg.Free; + Exit; + end; + + FNameToIndex.AddObject(Pkg.PackageName, TObject(PtrInt(FItems.Count))); + FItems.Add(Pkg); +end; + +procedure TPackageGraph.DiscoverUnder(const ARoot: string); +var + List: TStringList; + Each: string; +begin + if not DirectoryExists(ARoot) then + Exit; + List := TProjectFiles.FindAll(ARoot, '*.lpk'); + try + for Each in List do + RegisterLpk(Each); + finally + List.Free; + end; +end; + +function TPackageGraph.ResolveDepIndex(const APackageName, + AContext: string): Integer; +begin + Result := FindIndexByName(APackageName); + if Result < 0 then + begin + FRunner.Log(CSI_Red, Format('%s requires unknown package "%s"', + [AContext, APackageName])); + FRunner.IncError; + end; +end; + +procedure TPackageGraph.VisitPackageDeps(const AIndex: Integer; + AVisited: specialize TList; AKind: TDepVisitKind; AOrder: specialize TList; + APaths: TStrings); +var + Pkg: TLpkPackage; + Path: string; + I, DepIdx: Integer; + DepName: string; +begin + if (AIndex < 0) or (AIndex >= FItems.Count) then + Exit; + + case AKind of + TDepVisitKind.BuildOrder: + if AOrder.IndexOf(AIndex) >= 0 then + Exit; + TDepVisitKind.UnitPaths: + begin + if AVisited.IndexOf(AIndex) >= 0 then + Exit; + AVisited.Add(AIndex); + end; + end; + + Pkg := GetPackage(AIndex); + for I := 0 to Pkg.RequiredNames.Count - 1 do + begin + DepName := Pkg.RequiredNames[I]; + if IsBuiltinPackage(DepName) then + Continue; + DepIdx := ResolveDepIndex(DepName, 'package "' + Pkg.PackageName + '"'); + if DepIdx < 0 then + Continue; + VisitPackageDeps(DepIdx, AVisited, AKind, AOrder, APaths); + end; + + case AKind of + TDepVisitKind.BuildOrder: + AOrder.Add(AIndex); + TDepVisitKind.UnitPaths: + begin + Path := Pkg.ResolveUnitOutDir(FRunner.TargetCpu, FRunner.TargetOs); + if (Path <> '') and (APaths.IndexOf(Path) < 0) then + APaths.Add(Path); + end; + end; +end; + +procedure TPackageGraph.CollectBuildOrder(const AIndex: Integer; + AOrder: specialize TList); +begin + VisitPackageDeps(AIndex, nil, TDepVisitKind.BuildOrder, AOrder, nil); +end; + +procedure TPackageGraph.CollectUnitPaths(const AIndex: Integer; + AVisited: specialize TList; APaths: TStrings); +begin + VisitPackageDeps(AIndex, AVisited, TDepVisitKind.UnitPaths, nil, APaths); +end; + +function TPackageGraph.BuildAll: Boolean; +var + Order: specialize TList; + I, Idx, J: Integer; + Pkg: TLpkPackage; + Args: TStringList; + BuildOutput: string; + OutDir: string; + DepPath: string; +begin + Result := True; + if FItems.Count = 0 then + Exit; + + Order := specialize TList.Create; + try + for I := 0 to FItems.Count - 1 do + CollectBuildOrder(I, Order); + + for I := 0 to Order.Count - 1 do + begin + Idx := Order[I]; + Pkg := GetPackage(Idx); + + OutDir := Pkg.ResolveUnitOutDir(FRunner.TargetCpu, FRunner.TargetOs); + + FRunner.LogInline(CSI_Yellow, 'build package ' + Pkg.PackageName); + TProjectFiles.RemoveRecursive(OutDir); + ForceDirectories(OutDir); + + Args := TStringList.Create; + try + TLazXml.AppendCompilerOptionsToArgv(Pkg.Options, Pkg.PkgDir, OutDir, + OutDir, FRunner.TargetCpu, FRunner.TargetOs, Args); + for J := 0 to Pkg.SourceDirs.Count - 1 do + TLazXml.AppendFuIfMissing(Pkg.SourceDirs[J], Args); + TLazXml.AppendFuIfMissing(Pkg.PkgDir, Args); + for J := 0 to Pkg.RequiredNames.Count - 1 do + begin + if IsBuiltinPackage(Pkg.RequiredNames[J]) then + Continue; + DepPath := UnitPathFor(Pkg.RequiredNames[J]); + TLazXml.AppendFuIfMissing(DepPath, Args); + end; + TLazXml.AppendPackageBuildArgs(Args, ExtractFileName(Pkg.StubPas), OutDir); + + if FRunner.RunCommandEx('fpc', Args, Pkg.PkgDir, True, BuildOutput) then + FRunner.Log(CSI_Green, ' -> ' + OutDir) + else + begin + FRunner.IncError; + FRunner.ReportBuildErrors(BuildOutput); + Result := False; + end; + finally + Args.Free; + end; + end; + finally + Order.Free; + end; +end; + +function TPackageGraph.UnitPathFor(const APackageName: string): string; +var + Idx: Integer; begin Result := ''; - for Line in SplitString(ABuildOutput, LineEnding) do - if ContainsStr(Line, 'Linking') then + Idx := FindIndexByName(APackageName); + if (Idx < 0) or (Idx >= FItems.Count) then + Exit; + Result := GetPackage(Idx).ResolveUnitOutDir(FRunner.TargetCpu, FRunner.TargetOs); +end; + +function TPackageGraph.UnitPathsForRequired(const ANames: TStrings): TStringList; +var + I, Idx: Integer; + Visited: specialize TList; +begin + Result := TStringList.Create; + if not Assigned(ANames) then + Exit; + + Visited := specialize TList.Create; + try + for I := 0 to ANames.Count - 1 do begin - Parts := SplitString(Line, ' '); - if Length(Parts) >= 3 then - Result := Parts[2]; - Exit; + if IsBuiltinPackage(ANames[I]) then + Continue; + Idx := FindIndexByName(ANames[I]); + if Idx < 0 then + begin + FRunner.Log(CSI_Red, Format('project requires unknown package "%s"', + [ANames[I]])); + FRunner.IncError; + Continue; + end; + CollectUnitPaths(Idx, Visited, Result); end; + finally + Visited.Free; + end; end; // --------------------------------------------------------------------------- -// Report build errors from lazbuild output +// TMakeRunner // --------------------------------------------------------------------------- -procedure ReportBuildErrors(const ABuildOutput: string); +{ TMakeRunner } + +constructor TMakeRunner.Create; +begin + inherited Create; + FBackend := TBuildBackend.Auto; + FBackendResolved := False; + FErrorCount := 0; + FGraph := TPackageGraph.Create(Self); +end; + +destructor TMakeRunner.Destroy; +begin + FGraph.Free; + inherited Destroy; +end; + +procedure TMakeRunner.Log(const AColor, AMessage: string); +begin + WriteLn(stderr, AColor, AMessage, CSI_Reset); +end; + +procedure TMakeRunner.LogInline(const AColor, AMessage: string); +begin + Write(stderr, AColor, AMessage, CSI_Reset); +end; + +procedure TMakeRunner.IncError; +begin + Inc(FErrorCount); +end; + +procedure TMakeRunner.ReportBuildErrors(const ABuildOutput: string); var Line: string; ErrorFilter: TRegExpr; begin - ErrorFilter := TRegExpr.Create('(Fatal|Error):'); + ErrorFilter := TRegExpr.Create('(Fatal|Error):'); + try + for Line in SplitString(ABuildOutput, LineEnding) do + if ErrorFilter.Exec(Line) then + Log(CSI_Red, Line); + finally + ErrorFilter.Free; + end; +end; + +procedure TMakeRunner.ReportSummary; +begin + WriteLn(stderr); + if FErrorCount > 0 then + Log(CSI_Red, 'Errors: ' + IntToStr(FErrorCount)) + else + Log(CSI_Green, 'Errors: 0'); +end; + +function TMakeRunner.ParseBackendEnv: TBuildBackend; +var + Env: string; +begin + Env := LowerCase(Trim(GetEnvironmentVariable('MAKE_BUILD_BACKEND'))); + if (Env = '') or (Env = 'auto') then + Exit(TBuildBackend.Auto); + if Env = 'lazbuild' then + Exit(TBuildBackend.Lazbuild); + if Env = 'fpc' then + Exit(TBuildBackend.Fpc); + raise Exception.CreateFmt('unknown MAKE_BUILD_BACKEND: "%s"', [Env]); +end; + +function TMakeRunner.ResolveAutoBackend: TBuildBackend; +var + Output: string; +begin + if RunCommandEx('lazbuild', ['--version'], '', False, Output) then + Exit(TBuildBackend.Lazbuild); + Result := TBuildBackend.Fpc; +end; + +function TMakeRunner.UsesLazbuild: Boolean; +begin + if not FBackendResolved then + InitEnvironment; + Result := FBackend = TBuildBackend.Lazbuild; +end; + +function TMakeRunner.RunCommandEx(const AExecutable: string; const AArgs: TStrings; + const AWorkingDir: string; AStreamToStderr: Boolean; + out AOutput: string): Boolean; overload; +var + Proc: TProcess; + OutStream: TStringStream; + Count: LongInt; + Buffer: array[0..8191] of Byte; + Chunk: string; + + procedure DrainOutput; + begin + while Proc.Output.NumBytesAvailable > 0 do + begin + Count := Proc.Output.Read(Buffer, SizeOf(Buffer)); + if Count <= 0 then + Break; + OutStream.WriteBuffer(Buffer, Count); + if AStreamToStderr then + begin + SetLength(Chunk, Count); + Move(Buffer[0], Chunk[1], Count); + Write(stderr, Chunk); + end; + end; + end; + +begin + AOutput := ''; + Proc := TProcess.Create(nil); + OutStream := TStringStream.Create(''); + try + Proc.Executable := AExecutable; + Proc.Parameters.Assign(AArgs); + if AWorkingDir <> '' then + Proc.CurrentDirectory := AWorkingDir; + Proc.Options := [poUsePipes, poStderrToOutPut]; + Proc.ShowWindow := swoHide; + Proc.Execute; + repeat + DrainOutput; + if Proc.Running then + Sleep(10); + until not Proc.Running; + DrainOutput; + Proc.WaitOnExit; + AOutput := OutStream.DataString; + Result := Proc.ExitStatus = 0; + finally + OutStream.Free; + Proc.Free; + end; +end; + +function TMakeRunner.RunCommandEx(const AExecutable: string; + const AArgs: array of string; const AWorkingDir: string; + AStreamToStderr: Boolean; out AOutput: string): Boolean; overload; +var + SL: TStringList; + I: Integer; +begin + SL := TStringList.Create; try - for Line in SplitString(ABuildOutput, LineEnding) do - if ErrorFilter.Exec(Line) then - Log(CSI_Red, Line); + for I := 0 to High(AArgs) do + SL.Add(AArgs[I]); + Result := RunCommandEx(AExecutable, SL, AWorkingDir, AStreamToStderr, AOutput); finally - ErrorFilter.Free; + SL.Free; end; end; -// --------------------------------------------------------------------------- -// Build a single .lpi project -// Returns the path to the linked binary on success, empty string on failure -// --------------------------------------------------------------------------- +procedure TMakeRunner.NormalizeFpcTarget(var AValue: string); +begin + AValue := StringReplace(AValue, #13, '', [rfReplaceAll]); + AValue := StringReplace(AValue, #10, '', [rfReplaceAll]); + AValue := Trim(AValue); +end; -function BuildProject(const APath: string): string; +function TMakeRunner.RunFpcInfoProbeWithRetry(const AInfoFlag: string; + out AValue: string): Boolean; var - BuildOutput: string; - Success: Boolean; + Attempt, MaxAttempts, DelayMs: Integer; + Env, Output: string; begin - Result := ''; - LogInline(CSI_Yellow, 'build from ' + APath); - try - Success := RunCommand('lazbuild', ['--build-all', '--recursive', - '--no-write-project', APath], BuildOutput); - if Success then + MaxAttempts := 3; + DelayMs := 2000; + Env := Trim(GetEnvironmentVariable('CI_FPC_PROBE_ATTEMPTS')); + if Env <> '' then + MaxAttempts := StrToIntDef(Env, MaxAttempts); + Env := Trim(GetEnvironmentVariable('CI_FPC_PROBE_DELAY_MS')); + if Env <> '' then + DelayMs := StrToIntDef(Env, DelayMs); + + AValue := ''; + for Attempt := 1 to MaxAttempts do + begin + if RunCommandEx('fpc', [AInfoFlag], '', False, Output) then begin - Result := ExtractLinkedBinary(BuildOutput); - if Result <> '' then - Log(CSI_Green, ' -> ' + Result) - else - WriteLn(stderr); - end - else + NormalizeFpcTarget(Output); + if Output <> '' then + begin + AValue := Output; + if Attempt > 1 then + Log(CSI_Yellow, Format('fpc %s succeeded on attempt %d/%d', + [AInfoFlag, Attempt, MaxAttempts])); + Exit(True); + end; + end; + if Attempt < MaxAttempts then begin - WriteLn(stderr); - Inc(ErrorCount); - ReportBuildErrors(BuildOutput); + Log(CSI_Yellow, Format('fpc %s empty or failed (attempt %d/%d), retrying...', + [AInfoFlag, Attempt, MaxAttempts])); + Sleep(DelayMs); end; - except - on E: Exception do + end; + Result := False; +end; + +function TMakeRunner.RepoRoot: string; +var + Seeds: array[0..1] of string; + I: Integer; + Candidate, Parent, DemoDir: string; +begin + Seeds[0] := ExpandFileName(ExtractFilePath(ParamStr(0))); + Seeds[1] := ExpandFileName(GetCurrentDir); + for I := 0 to High(Seeds) do + begin + Candidate := Seeds[I]; + while Candidate <> '' do begin - WriteLn(stderr); - Inc(ErrorCount); - Log(CSI_Red, E.ClassName + ': ' + E.Message); + DemoDir := IncludeTrailingPathDelimiter(ConcatPaths([Candidate, Target])); + if DirectoryExists(DemoDir) then + Exit(ExcludeTrailingPathDelimiter(Candidate)); + Parent := ExpandFileName(IncludeTrailingPathDelimiter(Candidate) + '..'); + if SameText(Parent, Candidate) then + Break; + Candidate := Parent; end; end; + Result := GetCurrentDir; end; -// --------------------------------------------------------------------------- -// Build and run a test project -// --------------------------------------------------------------------------- +function TMakeRunner.TargetDirectory: string; +begin + Result := IncludeTrailingPathDelimiter(ConcatPaths([RepoRoot, Target])); +end; -procedure RunTestProject(const APath: string); +procedure TMakeRunner.ForEachLpkInDir(const ARoot: string; + ACallback: TLpkPathProc); var - BinaryPath, TestOutput: string; + List: TStringList; + Each: string; begin - BinaryPath := BuildProject(APath); - if BinaryPath = '' then + if not Assigned(ACallback) or not DirectoryExists(ARoot) then Exit; + List := TProjectFiles.FindAll(ARoot, '*.lpk'); try - if RunCommand(BinaryPath, ['--all', '--format=plain', '--progress'], - TestOutput) then - WriteLn(stderr, TestOutput) - else - begin - Inc(ErrorCount); - WriteLn(stderr, TestOutput); - end; - except - on E: Exception do - begin - Inc(ErrorCount); - Log(CSI_Red, E.ClassName + ': ' + E.Message); - end; + for Each in List do + ACallback(Each); + finally + List.Free; end; end; -// --------------------------------------------------------------------------- -// Build and run a sample (non-test) project -// --------------------------------------------------------------------------- +procedure TMakeRunner.RunBuiltBinary(const ABinaryPath: string; + const AArgs: array of string; const AFailMessage: string); +var + Output: string; +begin + if RunCommandEx(ABinaryPath, AArgs, '', False, Output) then + WriteLn(Output) + else + begin + IncError; + if AFailMessage <> '' then + Log(CSI_Red, AFailMessage); + WriteLn(stderr, Output); + end; +end; -procedure RunSampleProject(const APath: string); +procedure TMakeRunner.InitEnvironment; var - BinaryPath, SampleOutput: string; + Requested: TBuildBackend; + Output: string; begin - BinaryPath := BuildProject(APath); - if BinaryPath = '' then + if FBackendResolved then Exit; - try - Log(CSI_Yellow, 'run ' + BinaryPath); - if RunCommand(BinaryPath, [], SampleOutput) then - WriteLn(SampleOutput) - else - begin - Inc(ErrorCount); - Log(CSI_Red, 'sample execution failed: ' + BinaryPath); - WriteLn(stderr, SampleOutput); - end; - except - on E: Exception do - begin - Inc(ErrorCount); - Log(CSI_Red, E.ClassName + ': ' + E.Message); - end; + + if not RunFpcInfoProbeWithRetry('-iTP', FTargetCpu) then + raise Exception.Create('fpc -iTP returned empty TargetCPU'); + if not RunFpcInfoProbeWithRetry('-iTO', FTargetOs) then + raise Exception.Create('fpc -iTO returned empty TargetOS'); + + Requested := ParseBackendEnv; + case Requested of + TBuildBackend.Lazbuild: + begin + if not RunCommandEx('lazbuild', ['--version'], '', False, Output) then + raise Exception.Create('MAKE_BUILD_BACKEND=lazbuild but lazbuild not found'); + FBackend := TBuildBackend.Lazbuild; + end; + TBuildBackend.Fpc: + FBackend := TBuildBackend.Fpc; + TBuildBackend.Auto: + FBackend := ResolveAutoBackend; + end; + + FBackendResolved := True; + case FBackend of + TBuildBackend.Lazbuild: + Log(CSI_Yellow, 'build backend: lazbuild'); + TBuildBackend.Fpc: + Log(CSI_Yellow, 'build backend: fpc'); + TBuildBackend.Auto: + ; end; end; -// --------------------------------------------------------------------------- -// Shared download + extract -// --------------------------------------------------------------------------- +procedure TMakeRunner.UpdateSubmodules; +var + CommandOutput: string; +begin + if not FileExists('.gitmodules') then + Exit; + if RunCommandEx('git', ['submodule', 'update', '--init', '--recursive', + '--force', '--remote'], '', False, CommandOutput) then + Log(CSI_Yellow, Trim(CommandOutput)); +end; -procedure DownloadAndExtract(const AUrl, ADestDir: string); +// FPC 3.2.2 hardcodes OpenSSL 1.1 DLL names on Windows, but +// modern CI runners ship OpenSSL 3.x. Override so FPC can find +// the libraries. This hack can be removed once we move to +// FPC 3.2.4+ which natively includes OpenSSL 3.x DLL names. +procedure TMakeRunner.InitSslForDownloads; +begin + {$IFDEF MSWINDOWS} + {$IFDEF WIN64} + DLLSSLName := 'libssl-3-x64.dll'; + DLLUtilName := 'libcrypto-3-x64.dll'; + {$ELSE} + DLLSSLName := 'libssl-3.dll'; + DLLUtilName := 'libcrypto-3.dll'; + {$ENDIF} + {$ENDIF} + InitSSLInterface; +end; + +procedure TMakeRunner.DownloadAndExtract(const AUrl, ADestDir: string); var TempFile: string; Stream: TFileStream; @@ -375,11 +1552,7 @@ procedure DownloadAndExtract(const AUrl, ADestDir: string); end; end; -// --------------------------------------------------------------------------- -// Dependency providers -// --------------------------------------------------------------------------- - -function GetDepsBaseDir(const ASubDir: string): string; +function TMakeRunner.GetDepsBaseDir(const ASubDir: string): string; var BaseDir: string; begin @@ -388,24 +1561,22 @@ function GetDepsBaseDir(const ASubDir: string): string; {$ELSE} BaseDir := GetEnvironmentVariable('HOME'); {$ENDIF} - Result := IncludeTrailingPathDelimiter( - ConcatPaths([BaseDir, '.lazarus', ASubDir])); + Result := IncludeTrailingPathDelimiter(ConcatPaths([BaseDir, '.lazarus', ASubDir])); end; -function InstallOPMPackage(const APackageName: string): string; +function TMakeRunner.InstallOPMPackage(const APackageName: string): string; begin - Result := GetDepsBaseDir(ConcatPaths(['onlinepackagemanager', 'packages'])) - + APackageName; + Result := GetDepsBaseDir(ConcatPaths(['onlinepackagemanager', 'packages'])) + + APackageName; if DirectoryExists(Result) then Exit; DownloadAndExtract(OPMBaseUrl + APackageName + '.zip', Result); end; -function InstallGitHubPackage(const AOwnerRepo, ARef: string): string; +function TMakeRunner.InstallGitHubPackage(const AOwnerRepo, ARef: string): string; var SafeName, EffectiveRef: string; begin - // Flatten 'owner/repo' to 'owner--repo' for a safe directory name SafeName := StringReplace(AOwnerRepo, '/', '--', [rfReplaceAll]); EffectiveRef := ARef; if EffectiveRef = '' then @@ -415,56 +1586,223 @@ function InstallGitHubPackage(const AOwnerRepo, ARef: string): string; if DirectoryExists(Result) then Exit; - // https://github.com/{owner}/{repo}/archive/refs/heads/{branch}.zip - // also works for tags: refs/tags/{tag}.zip and commits: {sha}.zip DownloadAndExtract( GitHubArchiveBaseUrl + AOwnerRepo + '/archive/' + EffectiveRef + '.zip', Result); end; -function ResolveDependency(const ADep: TDependency): string; +function TMakeRunner.ResolveDependency(const ADep: TDependency): string; begin case ADep.Kind of - TDependencyKind.OPM: Result := InstallOPMPackage(ADep.Name); - TDependencyKind.GitHub: Result := InstallGitHubPackage(ADep.Name, ADep.Ref); + TDependencyKind.OPM: + Result := InstallOPMPackage(ADep.Name); + TDependencyKind.GitHub: + Result := InstallGitHubPackage(ADep.Name, ADep.Ref); else raise Exception.CreateFmt('Unknown dependency kind for "%s"', [ADep.Name]); end; end; -// --------------------------------------------------------------------------- -// Project classification helpers -// --------------------------------------------------------------------------- +procedure TMakeRunner.RegisterPackageLazbuild(const APath: string); +var + CommandOutput: string; +begin + if TPackageGraph.ShouldSkipLpk(APath) then + begin + if not TPackageGraph.ShouldExcludeLpkPath(APath) then + Log(CSI_Yellow, 'skip LCL-dependent package ' + APath); + Exit; + end; + if RunCommandEx('lazbuild', ['--add-package-link', APath], '', False, + CommandOutput) then + Log(CSI_Yellow, 'added ' + APath); +end; -// --------------------------------------------------------------------------- -// Determine whether an .lpi project is GUI -// --------------------------------------------------------------------------- +procedure TMakeRunner.RegisterAllPackagesLazbuild(const ASearchDir: string); +begin + ForEachLpkInDir(ASearchDir, @RegisterPackageLazbuild); +end; -// A project is considered GUI if its .lpi lists LCL as a required package. -// GUI projects cannot run headless in CI, so we skip them entirely. -function IsGUIProject(const ALpiPath: string): Boolean; +procedure TMakeRunner.InstallDependencies; var - Content: string; - Filter: TRegExpr; + DepDirs: TStringList; + I: Integer; begin - Result := False; - if not FileExists(ALpiPath) then - Exit; - Content := ReadFileToString(ALpiPath); - Filter := TRegExpr.Create(''); + DepDirs := TStringList.Create; try - Result := Filter.Exec(Content); + if Length(Dependencies) > 0 then + begin + InitSslForDownloads; + for I := 0 to High(Dependencies) do + DepDirs.Add(ResolveDependency(Dependencies[I])); + end; + + if UsesLazbuild then + begin + for I := 0 to DepDirs.Count - 1 do + RegisterAllPackagesLazbuild(DepDirs[I]); + RegisterAllPackagesLazbuild(RepoRoot); + end + else + begin + for I := 0 to DepDirs.Count - 1 do + FGraph.DiscoverUnder(DepDirs[I]); + FGraph.DiscoverUnder(RepoRoot); + if FGraph.PackageCount > 0 then + FGraph.BuildAll; + end; finally - Filter.Free; + DepDirs.Free; end; end; -// --------------------------------------------------------------------------- -// Determine whether an .lpi project is a test runner -// --------------------------------------------------------------------------- +function TMakeRunner.ExtractBinaryFromBuildLog(const AOutput, + AFallback: string): string; +var + Line: string; + Parts: TStringArray; + I: Integer; +begin + Result := AFallback; + for Line in SplitString(AOutput, LineEnding) do + if ContainsStr(Line, 'Linking') then + begin + Parts := SplitString(Line, ' '); + for I := High(Parts) downto 0 do + begin + if Trim(Parts[I]) <> '' then + begin + Result := Trim(Parts[I]); + Break; + end; + end; + Exit; + end; +end; + +procedure TMakeRunner.PrepareProjectBuild(Proj: TLpiProject); +begin + TProjectFiles.RemoveRecursive(Proj.UnitOutDir); + if FileExists(Proj.TargetBinary) then + DeleteFile(Proj.TargetBinary); + ForceDirectories(ExtractFilePath(Proj.TargetBinary)); + ForceDirectories(Proj.UnitOutDir); +end; + +function TMakeRunner.BuildProjectWithLazbuild(const APath: string): string; +var + Proj: TLpiProject; + BuildOutput: string; +begin + Result := ''; + Proj := TLpiProject.CreateFromFile(APath, FTargetCpu, FTargetOs); + try + if not Proj.IsValid then + begin + Log(CSI_Red, 'invalid project: ' + APath); + IncError; + Exit; + end; + PrepareProjectBuild(Proj); + if RunCommandEx('lazbuild', ['--build-all', '--recursive', + '--no-write-project', APath], '', True, BuildOutput) then + begin + Result := ExtractBinaryFromBuildLog(BuildOutput, Proj.TargetBinary); + if Result <> '' then + Log(CSI_Green, ' -> ' + Result) + else + WriteLn(stderr, BuildOutput); + end + else + begin + WriteLn(stderr, BuildOutput); + IncError; + ReportBuildErrors(BuildOutput); + end; + finally + Proj.Free; + end; +end; + +function TMakeRunner.BuildProjectWithFpc(const APath: string): string; +var + Proj: TLpiProject; + ExtraPaths, Args: TStringList; + BuildOutput: string; +begin + Result := ''; + Proj := TLpiProject.CreateFromFile(APath, FTargetCpu, FTargetOs); + try + if not Proj.IsValid then + begin + Log(CSI_Red, 'invalid project: ' + APath); + IncError; + Exit; + end; + + PrepareProjectBuild(Proj); + ExtraPaths := FGraph.UnitPathsForRequired(Proj.RequiredPackageNames); + try + Args := Proj.BuildFpcArgv(ExtraPaths, FTargetCpu, FTargetOs); + try + if RunCommandEx('fpc', Args, Proj.ProjDir, True, BuildOutput) then + begin + Result := ExtractBinaryFromBuildLog(BuildOutput, Proj.TargetBinary); + if FileExists(Result) then + Log(CSI_Green, ' -> ' + Result) + else + begin + Log(CSI_Red, 'fpc reported success but binary missing: ' + Proj.TargetBinary); + IncError; + end; + end + else + begin + IncError; + ReportBuildErrors(BuildOutput); + end; + finally + Args.Free; + end; + finally + ExtraPaths.Free; + end; + finally + Proj.Free; + end; +end; + +function TMakeRunner.BuildProject(const ALpiPath: string): string; +begin + Result := ''; + LogInline(CSI_Yellow, 'build from ' + ALpiPath); + try + if UsesLazbuild then + Result := BuildProjectWithLazbuild(ALpiPath) + else + Result := BuildProjectWithFpc(ALpiPath); + except + on E: Exception do + begin + WriteLn(stderr); + IncError; + Log(CSI_Red, E.ClassName + ': ' + E.Message); + end; + end; +end; + +function TMakeRunner.IsGUIProject(const ALpiPath: string): Boolean; +var + Content: string; +begin + Result := False; + if not FileExists(ALpiPath) then + Exit; + Content := TLazXml.ReadFile(ALpiPath); + Result := TLazXml.ContentRequiresPackage(Content, 'LCL', 'RequiredPackages'); +end; -// A console project is a test runner if its .lpr uses consoletestrunner. -function IsTestProject(const ALpiPath: string): Boolean; +function TMakeRunner.IsTestProject(const ALpiPath: string): Boolean; var LprPath, Content: string; begin @@ -472,38 +1810,53 @@ function IsTestProject(const ALpiPath: string): Boolean; LprPath := ChangeFileExt(ALpiPath, '.lpr'); if not FileExists(LprPath) then Exit; - Content := ReadFileToString(LprPath); - Result := ContainsStr(Content, 'consoletestrunner'); + Content := TLazXml.ReadFile(LprPath); + Result := Pos('consoletestrunner', Content) > 0; end; -// --------------------------------------------------------------------------- -// Register all .lpk packages found under a directory -// --------------------------------------------------------------------------- - -procedure RegisterAllPackages(const ASearchDir: string); +procedure TMakeRunner.RunTestProject(const APath: string); var - List: TStringList; - Each: string; + BinaryPath: string; begin - List := FindAllFilesList(ASearchDir, '*.lpk'); + BinaryPath := BuildProject(APath); + if BinaryPath = '' then + Exit; try - for Each in List do - RegisterPackage(Each); - finally - List.Free; + RunBuiltBinary(BinaryPath, ['--all', '--format=plain', '--progress'], ''); + except + on E: Exception do + begin + IncError; + Log(CSI_Red, E.ClassName + ': ' + E.Message); + end; end; end; -// --------------------------------------------------------------------------- -// Build (and optionally test/run) all .lpi projects found under Target -// --------------------------------------------------------------------------- +procedure TMakeRunner.RunSampleProject(const APath: string); +var + BinaryPath: string; +begin + BinaryPath := BuildProject(APath); + if BinaryPath = '' then + Exit; + try + Log(CSI_Yellow, 'run ' + BinaryPath); + RunBuiltBinary(BinaryPath, [], 'sample execution failed: ' + BinaryPath); + except + on E: Exception do + begin + IncError; + Log(CSI_Red, E.ClassName + ': ' + E.Message); + end; + end; +end; -procedure BuildAllProjects; +procedure TMakeRunner.BuildAllProjects; var List: TStringList; Each: string; begin - List := FindAllFilesList(Target, '*.lpi'); + List := TProjectFiles.FindAll(TargetDirectory, '*.lpi'); try for Each in List do begin @@ -523,53 +1876,28 @@ procedure BuildAllProjects; end; end; -// --------------------------------------------------------------------------- -// Entry point -// --------------------------------------------------------------------------- - -procedure Main; -var - I: Integer; +function TMakeRunner.Execute: Integer; begin + InitEnvironment; + Log(CSI_Cyan, 'using target directory: ' + TargetDirectory); UpdateSubmodules; - - // Install and register dependencies (safe when array is empty) - if Length(Dependencies) > 0 then - begin - // FPC 3.2.2 hardcodes OpenSSL 1.1 DLL names on Windows, but - // modern CI runners ship OpenSSL 3.x. Override so FPC can find - // the libraries. This hack can be removed once we move to - // FPC 3.2.4+ which natively includes OpenSSL 3.x DLL names. - {$IFDEF MSWINDOWS} - {$IFDEF WIN64} - DLLSSLName := 'libssl-3-x64.dll'; - DLLUtilName := 'libcrypto-3-x64.dll'; - {$ELSE} - DLLSSLName := 'libssl-3.dll'; - DLLUtilName := 'libcrypto-3.dll'; - {$ENDIF} - {$ENDIF} - InitSSLInterface; - for I := 0 to High(Dependencies) do - RegisterAllPackages(ResolveDependency(Dependencies[I])); - end; - - // Register all local packages - RegisterAllPackages(GetCurrentDir); - - // Build and test + InstallDependencies; BuildAllProjects; - - // Summary - WriteLn(stderr); - if ErrorCount > 0 then - Log(CSI_Red, 'Errors: ' + IntToStr(ErrorCount)) - else - Log(CSI_Green, 'Errors: 0'); - - ExitCode := ErrorCount; + ReportSummary; + Result := FErrorCount; end; +// --------------------------------------------------------------------------- +// Program entry +// --------------------------------------------------------------------------- + +var + Runner: TMakeRunner; begin - Main; -end. \ No newline at end of file + Runner := TMakeRunner.Create; + try + ExitCode := Runner.Execute; + finally + Runner.Free; + end; +end. diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml index ec0bc35..dce928e 100644 --- a/.github/workflows/make.yml +++ b/.github/workflows/make.yml @@ -14,9 +14,11 @@ on: inputs: enabled_targets: description: >- - Comma-separated list of targets to run (leave empty for default). + Comma-separated list of targets to run (leave empty for default stable + set). Opt-in only: netbsd, dragonflybsd — pass explicitly here. Valid IDs: linux-x64, linux-arm64, windows-x64, macos-arm64, - macos-x64, linux-arm32, freebsd, netbsd, dragonflybsd, solaris + macos-x64, linux-arm32, linux-powerpc64-be, freebsd, solaris, + netbsd, dragonflybsd default: "" type: string @@ -24,85 +26,28 @@ concurrency: group: ${{ github.workflow }}-${{ github.ref }} cancel-in-progress: true -# ═══════════════════════════════════════════════════════════════════════ -# Shared configuration -# -# FPC and Lazarus are installed identically on every target by -# .github/workflows/install-fpc-lazarus.sh, which fetches FPC from the -# official freepascal.org dist mirror and builds lazbuild from source. -# No native package manager dependency, no SourceForge dependency. -# ═══════════════════════════════════════════════════════════════════════ - env: FPC_VERSION: 3.2.2 LAZARUS_BRANCH: lazarus_4_4 LAZARUS_REPO: https://github.com/fpc/Lazarus.git - -# ═══════════════════════════════════════════════════════════════════════ -# Jobs -# ═══════════════════════════════════════════════════════════════════════ + MAKE_BUILD_BACKEND: fpc jobs: - # ───────────────────────────────────────────────────────────────────── - # Target gating — single source of truth for which jobs run. - # - # This job resolves the effective target list exactly once and - # exposes it as an output. Every other job gates on that output, - # so the default list lives in exactly one place: the DEFAULT - # variable below. - # - # To disable a target permanently, remove its ID from DEFAULT. - # Currently disabled (absent from DEFAULT): - # - netbsd : package server intermittently times out - # - dragonflybsd : FPC 3.2.x TLS broken (see job comments) - # - # For ad-hoc runs with a different set, use the Run Workflow - # button in the Actions tab — the workflow_dispatch input - # overrides DEFAULT when non-empty. - # - # Valid IDs: linux-x64, linux-arm64, windows-x64, macos-arm64, - # macos-x64, linux-arm32, freebsd, netbsd, - # dragonflybsd, solaris - # ───────────────────────────────────────────────────────────────────── - setup: name: Resolve target list runs-on: ubuntu-latest outputs: enabled_targets: ${{ steps.resolve.outputs.enabled_targets }} steps: + - uses: actions/checkout@v6 + - name: Resolve enabled targets id: resolve shell: bash env: INPUT_TARGETS: ${{ github.event.inputs.enabled_targets }} - run: | - set -euo pipefail - DEFAULT="linux-arm32,linux-x64,linux-arm64,windows-x64,macos-arm64,macos-x64,freebsd,solaris" - # workflow_dispatch with an empty textbox still sends an empty - # string (the declared `default:` is suppressed in that case), - # and push/PR/schedule runs have no inputs context at all, so - # both paths land here as empty. Fall through to DEFAULT. - if [ -z "${INPUT_TARGETS// /}" ]; then - TARGETS="$DEFAULT" - SOURCE="default" - else - # Strip any whitespace the user may have pasted. - TARGETS="${INPUT_TARGETS// /}" - SOURCE="workflow_dispatch input" - fi - echo "enabled_targets=${TARGETS}" >> "$GITHUB_OUTPUT" - echo "::notice::Enabled targets (${SOURCE}): ${TARGETS}" - - # ───────────────────────────────────────────────────────────────────── - # Tier 1 — Native GitHub-hosted runners (Linux, macOS, Windows) - # - # Each entry sets only the bits that vary: runner image, FPC - # target triple, and a per-OS dependency-install command. The - # shared installer script handles everything else identically - # across these five targets. - # ───────────────────────────────────────────────────────────────────── + run: bash .github/workflows/ci/resolve-targets.sh native: needs: setup @@ -133,19 +78,19 @@ jobs: runner: macos-15-intel name: macOS x86_64 (Intel) fpc_target: x86_64-darwin - steps: - name: Check if target is enabled id: gate shell: bash env: ENABLED_TARGETS: ${{ needs.setup.outputs.enabled_targets }} + TARGET_ID: ${{ matrix.id }} run: | - if [[ ",${ENABLED_TARGETS}," == *",${{ matrix.id }},"* ]]; then - echo "enabled=true" >> "$GITHUB_OUTPUT" + if [[ ",${ENABLED_TARGETS}," == *",${TARGET_ID},"* ]]; then + echo "enabled=true" >> "$GITHUB_OUTPUT" else echo "enabled=false" >> "$GITHUB_OUTPUT" - echo "::notice::Skipping ${{ matrix.id }} (not in enabled targets)" + echo "Skipping ${TARGET_ID} (not in enabled targets)" fi - name: Checkout @@ -154,57 +99,12 @@ jobs: with: submodules: true - # ── OpenSSL libssl.1.1 symlink hack (Linux + macOS) ────────────── - # - # FPC 3.2.2 hardcodes libssl.1.1 in its DLLVersions array, but - # current Linux distros and Homebrew ship OpenSSL 3.x only. - # Symlink so FPC's openssl unit can find the libraries. - # Removable once we move to FPC 3.2.4+ which includes '.3' in - # DLLVersions natively. - - name: OpenSSL symlink hack (Linux) - if: steps.gate.outputs.enabled == 'true' && runner.os == 'Linux' - shell: bash - run: | - set -xeuo pipefail - ARCH_DIR="/usr/lib/$(gcc -print-multiarch)" - sudo ln -sf "$ARCH_DIR/libssl.so.3" "$ARCH_DIR/libssl.so.1.1" - sudo ln -sf "$ARCH_DIR/libcrypto.so.3" "$ARCH_DIR/libcrypto.so.1.1" - - - name: OpenSSL symlink hack (macOS) - if: steps.gate.outputs.enabled == 'true' && runner.os == 'macOS' - shell: bash - run: | - set -xeuo pipefail - OSSL_LIB="$(brew --prefix openssl@3)/lib" - sudo mkdir -p /usr/local/lib - sudo ln -sf "$OSSL_LIB/libssl.3.dylib" /usr/local/lib/libssl.1.1.dylib - sudo ln -sf "$OSSL_LIB/libcrypto.3.dylib" /usr/local/lib/libcrypto.1.1.dylib - - # ── Install FPC + Lazarus from upstream tarball ────────────────── - # - # Single shared script across all native targets. On Windows it - # runs under Git Bash (pre-installed on windows-latest), which - # provides bash + GNU coreutils + tar — everything install.sh - # needs. - - name: Install FPC + Lazarus + - name: Build if: steps.gate.outputs.enabled == 'true' shell: bash env: FPC_TARGET: ${{ matrix.fpc_target }} - run: bash .github/workflows/install-fpc-lazarus.sh - - - name: Build - if: steps.gate.outputs.enabled == 'true' - shell: bash - run: | - set -xeuo pipefail - fpc -iV - lazbuild --version - instantfpc .github/workflows/make.pas - - # ───────────────────────────────────────────────────────────────────── - # Tier 2 — Linux ARM32 via QEMU user-mode emulation - # ───────────────────────────────────────────────────────────────────── + run: bash .github/workflows/ci/native-build.sh linux-arm32: name: Linux ARMv7 (QEMU) @@ -223,48 +123,35 @@ jobs: with: arch: armv7 distro: ubuntu24.04 - # Pass through the env vars our shared script expects. - # YAML map format with literal '|' is what this action expects. env: | FPC_VERSION: ${{ env.FPC_VERSION }} + FPC_TARGET: arm-linux LAZARUS_BRANCH: ${{ env.LAZARUS_BRANCH }} LAZARUS_REPO: ${{ env.LAZARUS_REPO }} - install: | - apt-get update - apt-get install -y curl ca-certificates git build-essential \ - openssl + MAKE_BUILD_BACKEND: ${{ env.MAKE_BUILD_BACKEND }} + run: bash .github/workflows/ci/arm32-run.sh + + linux-powerpc64-be: + name: Linux PowerPC64 BE (QEMU) + runs-on: ubuntu-latest + timeout-minutes: 180 + needs: setup + if: contains(format(',{0},', needs.setup.outputs.enabled_targets), ',linux-powerpc64-be,') + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + submodules: true - # FPC 3.2.2 hardcodes libssl.1.1; symlink to the OpenSSL 3.x - # libraries Ubuntu 24.04 ships. Removable once on FPC 3.2.4+. - ARCH_DIR="/usr/lib/arm-linux-gnueabihf" - ln -sf "$ARCH_DIR/libssl.so.3" "$ARCH_DIR/libssl.so.1.1" - ln -sf "$ARCH_DIR/libcrypto.so.3" "$ARCH_DIR/libcrypto.so.1.1" - run: | - set -xeuo pipefail - export FPC_TARGET=arm-linux - bash .github/workflows/install-fpc-lazarus.sh - export PATH="$HOME/lazarus-src:$HOME/fpc-install/bin:$PATH" - fpc -iV - instantfpc .github/workflows/make.pas + - name: Set up QEMU (ppc64 BE, pinned) + run: bash .github/workflows/ci/ppc64-qemu-setup.sh - # ───────────────────────────────────────────────────────────────────── - # Tier 3 — BSD family via vmactions QEMU system VMs - # - # All BSD jobs reuse the shared installer script. The VM image's - # package manager only needs to provide the script's prerequisites - # (curl, git, tar, gmake, bash). FPC itself comes from the - # freepascal.org mirror, identically to native runners. - # - # Not supported (removed): - # - FreeBSD aarch64: fpc-devel exists but is experimental. - # - OpenBSD: pre-built FPC binary links against older libc; - # incompatible with current OpenBSD. No usable package either. - # - NetBSD aarch64: no FPC package available. - # - # Disabled (commented out below): - # - NetBSD x86_64: package server intermittently times out. - # - DragonFlyBSD x86_64: FPC 3.2.x TLS broken (see comment below). - # ───────────────────────────────────────────────────────────────────── + - name: Build (PowerPC64 BE via QEMU) + env: + FPC_VERSION: ${{ env.FPC_VERSION }} + FPC_TARGET: powerpc64-linux + MAKE_BUILD_BACKEND: fpc + run: bash .github/workflows/ci/ppc64-be-build.sh freebsd: name: FreeBSD x86_64 @@ -272,36 +159,6 @@ jobs: timeout-minutes: 120 needs: setup if: contains(format(',{0},', needs.setup.outputs.enabled_targets), ',freebsd,') - # ───────────────────────────────────────────────────────────────── - # INTERIM: install FPC from FreeBSD's pkg system instead of using - # the shared install-fpc-lazarus.sh script. - # - # Why: FPC 3.2.2's official tarball at downloads.freepascal.org is - # built on FreeBSD 11 (filename: fpc-3.2.2.x86_64-freebsd11.tar). - # FreeBSD's ABI is not stable across major versions, and binaries - # linked against FPC 3.2.2's freebsd11 RTL units segfault on any - # FreeBSD ≥12. The compat11x/12x/13x ports that historically - # smoothed this over are no longer available on FreeBSD 14+. - # - # Bootstrapping FPC from source (which is what every FreeBSD pkg - # maintainer does internally to ship `fpc`) gets close — the - # build completes — but FPC 3.2.2's source itself contains - # FreeBSD ≥12 incompatibilities (struct stat layout, etc., see - # FPC issue #37784). fixes_3_2 was tried; build still produced - # binaries that couldn't exec ppcx64 due to fpc.cfg path / version - # detection issues. - # - # Rather than chase those, we use FreeBSD's pkg-built fpc, which - # is freshly compiled against the running FreeBSD's libc and just - # works. This re-introduces a FreeBSD-specific code path; that's - # the deliberate tradeoff. - # - # Plan to remove this branch: when FPC 3.2.4 is released, the - # dist-mirror tarball will be FreeBSD ≥13-built and/or the source - # tarball will have all the FreeBSD compat backports. At that - # point, comment the INTERIM block below and uncomment the - # PREFERRED block — no other changes needed. - # ───────────────────────────────────────────────────────────────── steps: - name: Checkout uses: actions/checkout@v6 @@ -310,85 +167,15 @@ jobs: - name: Build (FreeBSD x86_64) uses: vmactions/freebsd-vm@v1 + env: + FPC_TARGET: x86_64-freebsd + FREEBSD_INSTALL_MODE: interim with: - envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO + envs: FPC_VERSION FPC_TARGET LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FREEBSD_INSTALL_MODE release: "15.0" usesh: true - prepare: | - # ─── INTERIM: pkg install fpc ────────────────────────── - # Workaround for periodic FreeBSD pkg cluster breakage - # (see https://github.com/freebsd/pkg/issues/2653): - # bootstrap pkg fresh, then prime the repo metadata with - # a fetch-only upgrade before the real catalog refresh — - # this sidesteps transient broken-cluster states on the - # FreeBSD:15:amd64/latest repo. After that, force-refresh - # the catalog and align pre-installed VM packages with - # the current repo before installing new ones — otherwise - # newly-installed packages may link against newer libs - # than the VM image ships. - export ASSUME_ALWAYS_YES=yes - export IGNORE_OSVERSION=yes - pkg bootstrap -f - # Prime repo metadata (fetch-only) before the real update. - pkg upgrade -Fqy || true - pkg update -f - pkg upgrade -y - - pkg install -y fpc git wget gmake - - LAZARUS_DIR="$HOME/lazarus-src" - git clone --depth 1 --branch "$LAZARUS_BRANCH" \ - "$LAZARUS_REPO" "$LAZARUS_DIR" - gmake -C "$LAZARUS_DIR" lazbuild - - mkdir -p "$HOME/.lazarus" - cat > "$HOME/.lazarus/environmentoptions.xml" < - - - - - - - EOF - - export PATH="$LAZARUS_DIR:$PATH" - lazbuild --version - - # ─── PREFERRED: shared installer (currently disabled) ── - # When FPC 3.2.4 ships, comment out the INTERIM block - # above and uncomment this. The shared script handles - # FreeBSD identically to the other 9 platforms. - # - # Add binutils to the pkg install line above (FPC source - # build needs GNU as / ld.bfd from /usr/local/bin/) and - # also add: bash curl - # - # export ASSUME_ALWAYS_YES=yes - # export IGNORE_OSVERSION=yes - # pkg bootstrap -f - # pkg update -f - # pkg upgrade -y - # pkg install -y bash curl git gmake binutils - run: | - set -xeuo pipefail - export PATH="$HOME/lazarus-src:$PATH" - fpc -iV - lazbuild --version - instantfpc .github/workflows/make.pas - - # ─── PREFERRED: shared installer (currently disabled) ── - # When the INTERIM block in `prepare:` is removed, - # uncomment this. It runs the same shared script the - # other 9 platforms use — FPC_TARGET selects the dist- - # mirror tarball; INSTALL_PREFIX/LAZARUS_DIR default to - # $HOME/fpc-install and $HOME/lazarus-src. - # - # export FPC_TARGET=x86_64-freebsd - # bash .github/workflows/install-fpc-lazarus.sh - # export PATH="$HOME/lazarus-src:$HOME/fpc-install/bin:$PATH" - # fpc -iV - # instantfpc .github/workflows/make.pas + prepare: sh .github/workflows/ci/vm-freebsd-prepare.sh + run: bash .github/workflows/ci/vm-freebsd-run.sh netbsd: name: NetBSD x86_64 @@ -396,8 +183,6 @@ jobs: timeout-minutes: 120 needs: setup if: contains(format(',{0},', needs.setup.outputs.enabled_targets), ',netbsd,') - # Disabled: NetBSD package server (cdn.NetBSD.org) intermittently - # times out, causing CI failures. Re-enable when server is stable. steps: - name: Checkout uses: actions/checkout@v6 @@ -406,22 +191,12 @@ jobs: - name: Build (NetBSD x86_64) uses: vmactions/netbsd-vm@v1 + env: + FPC_TARGET: x86_64-netbsd with: - envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO - prepare: | - export PKG_PATH="https://cdn.NetBSD.org/pub/pkgsrc/packages/NetBSD/$(uname -p)/$(uname -r | cut -d_ -f1)/All" - - # Force-update pcre2 to resolve version conflict with git - pkg_add -uu pcre2 || true - pkg_add bash curl git gmake mozilla-rootcerts-openssl - run: | - set -xeuo pipefail - export PATH="/usr/pkg/bin:/usr/pkg/sbin:$PATH" - export FPC_TARGET=x86_64-netbsd - bash .github/workflows/install-fpc-lazarus.sh - export PATH="$HOME/lazarus-src:$HOME/fpc-install/bin:$PATH" - fpc -iV - instantfpc .github/workflows/make.pas + envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FPC_TARGET + prepare: sh .github/workflows/ci/vm-netbsd-prepare.sh + run: bash .github/workflows/ci/vm-run-shared.sh dragonflybsd: name: DragonFlyBSD x86_64 @@ -429,18 +204,6 @@ jobs: timeout-minutes: 120 needs: setup if: contains(format(',{0},', needs.setup.outputs.enabled_targets), ',dragonflybsd,') - # Disabled: FPC 3.2.x cannot establish TLS connections on - # DragonFlyBSD — base LibreSSL is ABI-incompatible and DPorts - # OpenSSL is 3.x which FPC 3.2.x doesn't support. FPC's - # pure-Pascal DNS resolver is also broken (same as mono/mono#8168). - # - # FPC 3.2.4+ fixes OpenSSL 3.x loading (adds '.3' to DLLVersions) - # but will NOT fix the DNS resolver bug. The /etc/hosts workaround - # and LD_LIBRARY_PATH below will still be needed. - # - # Lazarus has no DragonFlyBSD lazconf.inc, but DragonFlyBSD is a - # FreeBSD derivative so the FreeBSD include works as-is. The - # shared installer script patches it in after cloning Lazarus. steps: - name: Checkout uses: actions/checkout@v6 @@ -449,49 +212,14 @@ jobs: - name: Build (DragonFlyBSD x86_64) uses: vmactions/dragonflybsd-vm@v1 + env: + FPC_TARGET: x86_64-dragonfly + LD_LIBRARY_PATH_EXTRA: /usr/local/lib with: - envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO + envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FPC_TARGET LD_LIBRARY_PATH_EXTRA usesh: true - prepare: | - pkg install -y bash curl git gmake openssl - - # FPC's pure-Pascal DNS resolver (netdb unit) is broken on - # DragonFlyBSD — it fails to resolve hostnames even though - # system tools (host, drill, wget, git) work fine. This is - # the same class of bug as mono/mono#8168. - # - # Workaround: resolve dependency hostnames via system DNS - # and add them to /etc/hosts. FPC's netdb checks /etc/hosts - # first (via gethostbyname), bypassing the broken resolver. - for h in github.com packages.lazarus-ide.org downloads.freepascal.org; do - ip=$(drill "$h" 2>/dev/null | awk '/^'"$h"'/{print $5; exit}') - if [ -n "$ip" ]; then - echo "$ip $h" >> /etc/hosts - fi - done - - # DragonFlyBSD base ships LibreSSL in /usr/lib. Real OpenSSL - # 3.x from DPorts installs to /usr/local/lib. FPC 3.2.4+ - # adds '.3' to DLLVersions — once upgraded, remove these - # symlinks but keep LD_LIBRARY_PATH in the run step. - ln -sf libssl.so.3 /usr/local/lib/libssl.so.1.1 - ln -sf libcrypto.so.3 /usr/local/lib/libcrypto.so.1.1 - run: | - set -xeuo pipefail - export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - export FPC_TARGET=x86_64-dragonfly - bash .github/workflows/install-fpc-lazarus.sh - export PATH="$HOME/lazarus-src:$HOME/fpc-install/bin:$PATH" - fpc -iV - instantfpc .github/workflows/make.pas - - # ───────────────────────────────────────────────────────────────────── - # Tier 4 — Solaris via vmactions QEMU system VM - # - # Solaris uses pkgutil (OpenCSW) for community packages which install - # to /opt/csw/bin. FPC comes from the official freepascal.org mirror - # via the shared installer script. - # ───────────────────────────────────────────────────────────────────── + prepare: sh .github/workflows/ci/vm-dragonfly-prepare.sh + run: bash .github/workflows/ci/vm-run-shared.sh solaris: name: Solaris x86_64 @@ -507,21 +235,11 @@ jobs: - name: Build (Solaris x86_64) uses: vmactions/solaris-vm@v1 + env: + FPC_TARGET: x86_64-solaris with: - envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO + envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FPC_TARGET release: "11.4-gcc" usesh: true - prepare: | - # CSW packages install to /opt/csw — must be in PATH - # before any CSW-installed tool can be used. - export PATH="/opt/csw/bin:/usr/local/bin:$PATH" - - pkgutil -y -i bash curl git gmake - run: | - set -xeuo pipefail - export PATH="/opt/csw/bin:/usr/local/bin:$PATH" - export FPC_TARGET=x86_64-solaris - bash .github/workflows/install-fpc-lazarus.sh - export PATH="$HOME/lazarus-src:$HOME/fpc-install/bin:$PATH" - fpc -iV - instantfpc .github/workflows/make.pas \ No newline at end of file + prepare: sh .github/workflows/ci/vm-solaris-prepare.sh + run: bash .github/workflows/ci/vm-run-shared.sh diff --git a/SimpleBaseLib/src/Include/SimpleBaseLibFPC.inc b/SimpleBaseLib/src/Include/SimpleBaseLibFPC.inc index 153c660..dbfc05d 100644 --- a/SimpleBaseLib/src/Include/SimpleBaseLibFPC.inc +++ b/SimpleBaseLib/src/Include/SimpleBaseLibFPC.inc @@ -11,10 +11,6 @@ (* &&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&&& *) -{$IFDEF ENDIAN_BIG} - {$MESSAGE FATAL 'This Library does not support "Big Endian" processors yet.'} -{$ENDIF} - {$IF FPC_FULLVERSION < 30202} {$MESSAGE ERROR 'This Library requires FreePascal 3.2.2 or higher.'} {$IFEND} From 00b1cbbcb7e36a73ab1953788ad156699b3f8dfd Mon Sep 17 00:00:00 2001 From: Ugochukwu Mmaduekwe Date: Tue, 9 Jun 2026 01:07:02 +0100 Subject: [PATCH 3/5] refactor ci --- .../ci/openssl-libssl11-shim-macos.sh | 3 +- .../ci/openssl-libssl11-shim-unix.sh | 3 +- .github/workflows/ci/ppc64-be-build.sh | 1 + .github/workflows/ci/ppc64-be-images.env | 28 +++++--- .github/workflows/ci/ppc64-qemu-setup.sh | 30 ++++++--- .github/workflows/ci/resolve-targets.sh | 41 +++++++++--- .github/workflows/ci/shared/common.sh | 63 ++++++++++++------ .../workflows/ci/shared/lazarus-bootstrap.sh | 24 +++---- .github/workflows/ci/targets.json | 16 +++++ .github/workflows/ci/vm-dragonfly-prepare.sh | 11 +++- .github/workflows/ci/vm-freebsd-prepare.sh | 3 +- .github/workflows/ci/vm-freebsd-run.sh | 6 +- .github/workflows/ci/vm-netbsd-prepare.sh | 4 ++ .github/workflows/install-fpc-lazarus.sh | 34 ++++------ .github/workflows/make.pas | 66 ++++++++++++++----- .github/workflows/make.yml | 62 ++++++----------- 16 files changed, 243 insertions(+), 152 deletions(-) create mode 100644 .github/workflows/ci/targets.json diff --git a/.github/workflows/ci/openssl-libssl11-shim-macos.sh b/.github/workflows/ci/openssl-libssl11-shim-macos.sh index 51c2271..539505d 100644 --- a/.github/workflows/ci/openssl-libssl11-shim-macos.sh +++ b/.github/workflows/ci/openssl-libssl11-shim-macos.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# FPC 3.2.2 hardcodes libssl.1.1.dylib; symlink Homebrew OpenSSL 3 on macOS. +# TODO(FPC 3.2.4): remove this shim. FPC 3.2.2 hardcodes libssl.1.1.dylib, so +# symlink Homebrew's OpenSSL 3 dylibs to the 1.1 names on macOS. set -euo pipefail OSSL_LIB="$(brew --prefix openssl@3)/lib" diff --git a/.github/workflows/ci/openssl-libssl11-shim-unix.sh b/.github/workflows/ci/openssl-libssl11-shim-unix.sh index 4f5fcc9..a01bc73 100644 --- a/.github/workflows/ci/openssl-libssl11-shim-unix.sh +++ b/.github/workflows/ci/openssl-libssl11-shim-unix.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash -# FPC 3.2.2 hardcodes libssl.so.1.1; symlink OpenSSL 3.x ELF libraries on Linux/BSD. +# TODO(FPC 3.2.4): remove this shim. FPC 3.2.2 hardcodes libssl.so.1.1, so +# symlink the OpenSSL 3.x ELF libraries to the 1.1 sonames on Linux/BSD. set -euo pipefail ARCH_DIR="${1:-}" diff --git a/.github/workflows/ci/ppc64-be-build.sh b/.github/workflows/ci/ppc64-be-build.sh index 9f24c39..2677745 100644 --- a/.github/workflows/ci/ppc64-be-build.sh +++ b/.github/workflows/ci/ppc64-be-build.sh @@ -36,6 +36,7 @@ docker run --rm --platform linux/ppc64 \ -e FPC_VERSION \ -e FPC_TARGET \ -e MAKE_BUILD_BACKEND \ + -e CI_DEBUG \ -e DEBIAN_FRONTEND=noninteractive \ -e QEMU_CPU=power8 \ -e CSU_STUBS_PREBUILT="${CSU_STUBS_IN_CONTAINER}" \ diff --git a/.github/workflows/ci/ppc64-be-images.env b/.github/workflows/ci/ppc64-be-images.env index 6d19f3e..ddacffb 100644 --- a/.github/workflows/ci/ppc64-be-images.env +++ b/.github/workflows/ci/ppc64-be-images.env @@ -1,11 +1,21 @@ -# Pinned images for linux-powerpc64-be CI (source from ppc64-qemu-setup.sh / ppc64-be-build.sh). -PPC64_QEMU_VERSION=7.2.0-1 -# One-shot privileged host binfmt setup (ppc64-qemu-setup.sh --reset -p yes -c yes). -# Use multiarch/qemu-user-static:$version (full image): includes the register script -# and all qemu-*-static binaries copied onto the host. Do not use x86_64-ppc64-$version -# here — that tag ships only the qemu-ppc64-static binary (no register script). -# See https://github.com/multiarch/qemu-user-static#multiarchqemu-user-static-images -PPC64_QEMU_REGISTER_IMAGE=multiarch/qemu-user-static:${PPC64_QEMU_VERSION} -# Full (non-slim) debian-ports runtime — qemu-ppc64-static embedded by upstream. +# Pinned image for the linux-powerpc64-be CI flow (sourced by ppc64-be-build.sh). +# +# QEMU (the ppc64 big-endian emulator) is NOT pinned here: it is installed from +# the Ubuntu runner's apt as qemu-user-static (currently QEMU ~8.2) by +# ppc64-qemu-setup.sh. Rationale: +# - multiarch/qemu-user-static is abandoned (frozen at QEMU 7.2.0, Jan 2023). +# - tonistiigi/binfmt ships only little-endian ppc64le, not big-endian ppc64. +# The distro package gives us a current big-endian qemu-ppc64 and auto-registers +# its binfmt handler with the F (fix-binary) flag. See ppc64-qemu-setup.sh. +# +# Runtime rootfs: Debian-ports ppc64 (big-endian) exists only in sid, so there is +# no stable release to track. We currently track the rolling tag and let the +# floating distro QEMU (see above) keep pace with the userland. If sid ever drifts +# ahead of the emulator again (intermittent SIGSEGV / "ppcXXX can't be executed"), +# re-pin by digest: swap the active line below for the commented one and refresh via +# docker buildx imagetools inspect urbanogilson/debian-debootstrap-ports:ppc64-forky-sid +# (full variant — qemu-ppc64-static embedded upstream). # https://github.com/urbanogilson/debian-debootstrap-ports PPC64_RUNTIME_IMAGE=urbanogilson/debian-debootstrap-ports:ppc64-forky-sid +# Last-known-good digest (2026-05-15, gcc 15 userland) — uncomment to pin: +# PPC64_RUNTIME_IMAGE=urbanogilson/debian-debootstrap-ports:ppc64-forky-sid@sha256:5a0b62beaeb64dec7ed941f3a28d30827ea0f773b9d10a3f66c79dce48f36841 diff --git a/.github/workflows/ci/ppc64-qemu-setup.sh b/.github/workflows/ci/ppc64-qemu-setup.sh index f2cf051..e4de570 100644 --- a/.github/workflows/ci/ppc64-qemu-setup.sh +++ b/.github/workflows/ci/ppc64-qemu-setup.sh @@ -1,24 +1,36 @@ #!/usr/bin/env bash set -euo pipefail -CI_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -# shellcheck source=ppc64-be-images.env -source "$CI_ROOT/ppc64-be-images.env" +# Register the ppc64 (big-endian) binfmt handler on the runner host so that +# `docker run --platform linux/ppc64 ...` (ppc64-be-build.sh) transparently +# executes big-endian binaries under QEMU user-mode. +# +# We install QEMU from the Ubuntu runner's apt (qemu-user-static, currently +# ~8.2) rather than multiarch/qemu-user-static (abandoned at 7.2.0) or +# tonistiigi/binfmt (little-endian ppc64le only). The package's postinst +# registers each handler with the F (fix-binary) flag via update-binfmts, so +# the interpreter fd is preserved into the container and qemu does not need to +# exist inside the rootfs. See ppc64-be-images.env for the rationale. -docker run --rm --privileged \ - "$PPC64_QEMU_REGISTER_IMAGE" \ - --reset -p yes -c yes +export DEBIAN_FRONTEND=noninteractive +sudo apt-get update -qq +sudo apt-get install -y -qq qemu-user-static binfmt-support -if ! ls /proc/sys/fs/binfmt_misc/qemu-ppc64* >/dev/null 2>&1; then +# Idempotent: enable the handler in case it was installed but left disabled. +sudo update-binfmts --enable qemu-ppc64 2>/dev/null || true + +BINFMT_FILE=/proc/sys/fs/binfmt_misc/qemu-ppc64 +if [ ! -e "$BINFMT_FILE" ]; then echo "::error::qemu-ppc64 binfmt handler not registered" - ls /proc/sys/fs/binfmt_misc/ + ls /proc/sys/fs/binfmt_misc/ || true exit 1 fi -BINFMT_FILE="$(ls /proc/sys/fs/binfmt_misc/qemu-ppc64* | head -1)" +echo "qemu-ppc64-static: $(qemu-ppc64-static --version 2>/dev/null | head -1 || echo 'unknown')" echo "binfmt handler ${BINFMT_FILE}:" cat "$BINFMT_FILE" +# F (fix-binary) is required so the interpreter works inside the container. if ! grep -q 'flags:.*F' "$BINFMT_FILE"; then echo "::error::qemu-ppc64 binfmt flags missing F (fix-binary mode); got:" >&2 cat "$BINFMT_FILE" >&2 diff --git a/.github/workflows/ci/resolve-targets.sh b/.github/workflows/ci/resolve-targets.sh index a2afcdf..4be5ae5 100644 --- a/.github/workflows/ci/resolve-targets.sh +++ b/.github/workflows/ci/resolve-targets.sh @@ -1,13 +1,24 @@ #!/usr/bin/env bash set -euo pipefail -# Stable targets run on every push/PR (DEFAULT). Opt-in targets (netbsd, -# dragonflybsd) are excluded from DEFAULT — pass them explicitly via -# workflow_dispatch enabled_targets. -STABLE_TARGETS="linux-arm32,linux-powerpc64-be,linux-x64,linux-arm64,windows-x64,macos-arm64,macos-x64,freebsd,solaris" -OPT_IN_TARGETS="netbsd,dragonflybsd" -VALID_TARGETS="${STABLE_TARGETS},${OPT_IN_TARGETS}" -DEFAULT="$STABLE_TARGETS" +# Resolve which targets CI should run, from the registry in targets.json. +# +# Inputs: +# INPUT_TARGETS workflow_dispatch CSV. Empty => default set (default=true). +# Outputs (GITHUB_OUTPUT): +# enabled_targets CSV of selected ids; gates the qemu/vm jobs in make.yml. +# native_matrix JSON array of enabled kind=native entries; consumed as the +# native job's strategy.matrix.include (empty => job skipped). +# +# targets.json is the single source of truth. Opt-in targets (default=false, +# e.g. netbsd, dragonflybsd) are excluded from the default and must be named +# explicitly via INPUT_TARGETS. + +CI_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +REGISTRY="$CI_ROOT/targets.json" + +VALID_TARGETS="$(jq -r '[.targets[].id] | join(",")' "$REGISTRY")" +DEFAULT="$(jq -r '[.targets[] | select(.default) | .id] | join(",")' "$REGISTRY")" if [ -z "${INPUT_TARGETS// /}" ]; then TARGETS="$DEFAULT" @@ -17,6 +28,8 @@ else SOURCE="workflow_dispatch input" fi +# Warn (don't fail) on unknown ids so a typo is visible but harmless: an +# unrecognised id simply matches no job. IFS=',' read -r -a _selected <<< "$TARGETS" IFS=',' read -r -a _valid <<< "$VALID_TARGETS" for _id in "${_selected[@]}"; do @@ -33,5 +46,17 @@ for _id in "${_selected[@]}"; do fi done -echo "enabled_targets=${TARGETS}" >> "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" +# Filter the registry to enabled native entries. Bind .id before switching the +# pipe context to the split list; index() returns null when absent (falsy) and +# an integer otherwise (0 is truthy in jq). +NATIVE_MATRIX="$(jq -c --arg ids "$TARGETS" \ + '[.targets[] | select(.kind == "native") | select(.id as $i | ($ids | split(",") | index($i)))]' \ + "$REGISTRY")" + +{ + echo "enabled_targets=${TARGETS}" + echo "native_matrix=${NATIVE_MATRIX}" +} >> "${GITHUB_OUTPUT:?GITHUB_OUTPUT is required}" + echo "Enabled targets (${SOURCE}): ${TARGETS}" +echo "Native matrix: ${NATIVE_MATRIX}" diff --git a/.github/workflows/ci/shared/common.sh b/.github/workflows/ci/shared/common.sh index 2c59f15..4dccd76 100644 --- a/.github/workflows/ci/shared/common.sh +++ b/.github/workflows/ci/shared/common.sh @@ -10,6 +10,25 @@ ci_init_paths() { REPO_ROOT="$(cd "$WORKFLOWS_DIR/../.." && pwd)" } +# True when running under a Windows POSIX layer (Git Bash / MSYS / Cygwin). +ci_is_windows() { + case "$(uname -s)" in + MINGW*|MSYS*|CYGWIN*) return 0 ;; + *) return 1 ;; + esac +} + +# Prints the GNU make command name for this OS. 'make' is GNU make on Linux, +# but BSD make on the *BSDs/Solaris and absent on Windows runners (only +# Strawberry Perl's gmake), so those need 'gmake'. +ci_default_make_cmd() { + case "$(uname -s)" in + *BSD|DragonFly|SunOS) echo "gmake" ;; + MINGW*|MSYS*|CYGWIN*) echo "gmake" ;; + *) echo "make" ;; + esac +} + ci_install_toolchain() { : "${FPC_TARGET:?FPC_TARGET is required}" bash "$WORKFLOWS_DIR/install-fpc-lazarus.sh" @@ -126,6 +145,13 @@ ci_build_standard() { ci_run_make } +# Build when the toolchain is already installed (e.g. a distro/pkg FPC). +# Skips ci_install_toolchain; the caller is responsible for PATH. +ci_build_prebuilt() { + ci_preflight + ci_run_make +} + ci_openssl_hack() { case "$(uname -s)" in Linux) bash "$CI_ROOT/openssl-libssl11-shim-unix.sh" ;; @@ -144,14 +170,12 @@ ci_debian_container_bootstrap() { ci_github_path_append() { local dir="$1" [ -n "${GITHUB_PATH:-}" ] || return 0 - case "$(uname -s)" in - MINGW*|MSYS*|CYGWIN*) - cygpath -w "$dir" >> "$GITHUB_PATH" - ;; - *) - echo "$dir" >> "$GITHUB_PATH" - ;; - esac + # GITHUB_PATH expects native Windows paths (C:\foo), not MSYS (/c/foo). + if ci_is_windows; then + cygpath -w "$dir" >> "$GITHUB_PATH" + else + echo "$dir" >> "$GITHUB_PATH" + fi } ci_write_lazarus_environmentoptions() { @@ -159,19 +183,16 @@ ci_write_lazarus_environmentoptions() { local fpc_exe="$2" local laz_cfg_dir laz_dir_native fpc_exe_native - case "$(uname -s)" in - MINGW*|MSYS*|CYGWIN*) - local win_local="${LOCALAPPDATA:-$USERPROFILE/AppData/Local}" - laz_cfg_dir="$(cygpath -u "$win_local")/lazarus" - laz_dir_native="$(cygpath -w "$laz_dir")" - fpc_exe_native="$(cygpath -w "$fpc_exe")" - ;; - *) - laz_cfg_dir="${HOME}/.lazarus" - laz_dir_native="$laz_dir" - fpc_exe_native="$fpc_exe" - ;; - esac + if ci_is_windows; then + local win_local="${LOCALAPPDATA:-$USERPROFILE/AppData/Local}" + laz_cfg_dir="$(cygpath -u "$win_local")/lazarus" + laz_dir_native="$(cygpath -w "$laz_dir")" + fpc_exe_native="$(cygpath -w "$fpc_exe")" + else + laz_cfg_dir="${HOME}/.lazarus" + laz_dir_native="$laz_dir" + fpc_exe_native="$fpc_exe" + fi mkdir -p "$laz_cfg_dir" cat > "$laz_cfg_dir/environmentoptions.xml" </dev/null | awk '/^'"$h"'/{print $5; exit}') if [ -n "$ip" ]; then @@ -14,4 +18,5 @@ for h in github.com packages.lazarus-ide.org downloads.freepascal.org; do fi done +# TODO(FPC 3.2.4): drop the OpenSSL 1.1 shim once FPC links against OpenSSL 3. OPENSSL_USE_SUDO=0 bash "$CI_ROOT/openssl-libssl11-shim-unix.sh" /usr/local/lib diff --git a/.github/workflows/ci/vm-freebsd-prepare.sh b/.github/workflows/ci/vm-freebsd-prepare.sh index 44c83e7..2045445 100644 --- a/.github/workflows/ci/vm-freebsd-prepare.sh +++ b/.github/workflows/ci/vm-freebsd-prepare.sh @@ -15,7 +15,8 @@ if [ "$FREEBSD_INSTALL_MODE" = "preferred" ]; then exit 0 fi -# INTERIM: pkg-installed FPC until FPC 3.2.4 dist tarball works on FreeBSD 15+. +# TODO(FPC 3.2.4): remove the interim path; use pkg-installed FPC until the +# FPC 3.2.4 dist tarball works on FreeBSD 15+ (see vm-freebsd-run.sh). freebsd_pkg_bootstrap pkg install -y bash fpc git wget gmake diff --git a/.github/workflows/ci/vm-freebsd-run.sh b/.github/workflows/ci/vm-freebsd-run.sh index eff2051..0d9606c 100644 --- a/.github/workflows/ci/vm-freebsd-run.sh +++ b/.github/workflows/ci/vm-freebsd-run.sh @@ -10,7 +10,9 @@ ci_init_paths if [ "$FREEBSD_INSTALL_MODE" = "preferred" ]; then ci_build_standard else + # TODO(FPC 3.2.4): remove the interim path once the FreeBSD 15+ dist tarball + # installs cleanly. Until then the toolchain is pkg-installed in prepare, so + # build against it directly without re-running the installer. export PATH="$HOME/lazarus-src:$PATH" - ci_preflight - ci_run_make + ci_build_prebuilt fi diff --git a/.github/workflows/ci/vm-netbsd-prepare.sh b/.github/workflows/ci/vm-netbsd-prepare.sh index 894a69d..3c3b10c 100644 --- a/.github/workflows/ci/vm-netbsd-prepare.sh +++ b/.github/workflows/ci/vm-netbsd-prepare.sh @@ -1,6 +1,10 @@ #!/bin/sh set -eu +# pkgsrc binary repo is keyed by the base release, so strip any _STABLE/_PATCH +# suffix from uname -r (e.g. 10.0_STABLE -> 10.0). export PKG_PATH="https://cdn.NetBSD.org/pub/pkgsrc/packages/NetBSD/$(uname -p)/$(uname -r | cut -d_ -f1)/All" +# Upgrade the base pcre2 first so the new tools link against it; tolerated +# because a fresh image may have nothing to upgrade. pkg_add -uu pcre2 || true pkg_add bash curl git gmake mozilla-rootcerts-openssl diff --git a/.github/workflows/install-fpc-lazarus.sh b/.github/workflows/install-fpc-lazarus.sh index 39ed9a1..ad7d13c 100644 --- a/.github/workflows/install-fpc-lazarus.sh +++ b/.github/workflows/install-fpc-lazarus.sh @@ -30,7 +30,10 @@ # $INSTALL_PREFIX/bin and $LAZARUS_DIR are added to PATH # ───────────────────────────────────────────────────────────────────── -set -xeuo pipefail +set -euo pipefail +# Opt-in command tracing (this installer is verbose); set CI_DEBUG=1 to enable. +# Match only truthy values so a flat CI_DEBUG=0 (the CI default) stays quiet. +case "${CI_DEBUG:-}" in 1|true|yes|on) set -x ;; esac INSTALL_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" # shellcheck source=ci/shared/common.sh @@ -52,26 +55,12 @@ if [ "$INSTALL_LAZARUS" = "1" ]; then : "${LAZARUS_REPO:?LAZARUS_REPO is required}" fi -case "$(uname -s)" in - MINGW*|MSYS*|CYGWIN*) IS_WINDOWS=1 ;; - *) IS_WINDOWS=0 ;; -esac +if ci_is_windows; then IS_WINDOWS=1; else IS_WINDOWS=0; fi : "${INSTALL_PREFIX:=$HOME/fpc-install}" : "${LAZARUS_DIR:=$HOME/lazarus-src}" - -# Pick GNU make. On most platforms 'make' is GNU make. On BSDs and -# Solaris, 'make' is BSD make and we need 'gmake'. On Windows -# (windows-latest runner) the only GNU make pre-installed is -# Strawberry Perl's 'gmake.exe' — there is no 'make' on PATH. -# Caller can override MAKE_CMD if they have a different setup. -if [ -z "${MAKE_CMD:-}" ]; then - case "$(uname -s)" in - *BSD|DragonFly|SunOS) MAKE_CMD="gmake" ;; - MINGW*|MSYS*|CYGWIN*) MAKE_CMD="gmake" ;; - *) MAKE_CMD="make" ;; - esac -fi +# MAKE_CMD (gmake on BSD/Solaris/Windows) is defaulted by lazarus-bootstrap.sh, +# which is the only consumer; a caller-provided value propagates via sourcing. # ── Fetch + extract ────────────────────────────────────────────────── @@ -165,6 +154,8 @@ fi # ── Linux glibc 2.34+ workaround ───────────────────────────────────── # +# TODO(FPC 3.2.4): remove this whole glibc stub block (see end of comment). +# # FPC 3.2.2 was built against glibc < 2.34 and its RTL references # __libc_csu_init / __libc_csu_fini, which glibc 2.34 (Aug 2021) made # private. Linking anything FPC produces against current glibc fails: @@ -185,7 +176,7 @@ fi # is cross-compiled on the x86 host (CSU_STUBS_PREBUILT); running cc # inside QEMU user-mode for ppc64 is unreliable. # -# This hack can be removed once we move to FPC 3.2.4+. +# Removable once we move to FPC 3.2.4+ (see TODO above). # # See https://gitlab.com/freepascal.org/fpc/source/-/issues/39295 if [ "$(uname -s)" = "Linux" ]; then @@ -267,7 +258,10 @@ if [ -n "$FPC_UTIL_DIR" ]; then ci_github_path_append "$FPC_UTIL_DIR" fi -fpc -iV +# Probe the freshly installed compiler. Under QEMU user-mode (notably ppc64 +# big-endian) the compiler binary can SIGSEGV intermittently, so retry via +# ci_fpc_info_probe instead of letting one emulation hiccup fail the install. +ci_fpc_info_probe -iV # ── Build Lazarus from source (lazbuild | auto only) ───────────────── # diff --git a/.github/workflows/make.pas b/.github/workflows/make.pas index ba4c094..9245579 100644 --- a/.github/workflows/make.pas +++ b/.github/workflows/make.pas @@ -14,6 +14,9 @@ TMakeRunner = class; // Build backend // --------------------------------------------------------------------------- + // Selected via MAKE_BUILD_BACKEND. Auto probes for lazbuild on PATH and + // falls back to Fpc. Lazbuild drives builds through the IDE tool; Fpc builds + // packages/projects by invoking the compiler directly (see TPackageGraph). TBuildBackend = ( Auto, Lazbuild, @@ -46,6 +49,9 @@ TLazCompilerOptions = record UnitOutputDirTemplate: string; end; + // Lightweight extractor for the subset of Lazarus .lpi/.lpk XML we need + // (block ranges, Value="" attributes, search paths). Not a general XML + // parser; it relies on the well-formed, tool-generated Lazarus schema. TLazXml = class public class function ReadFile(const AFileName: string): string; @@ -142,6 +148,10 @@ TLpkPackage = class TDepVisitKind = (BuildOrder, UnitPaths); + // Dependency graph of discovered .lpk packages. Used by the fpc backend to + // compute a topological build order and to collect each package's unit + // output directory (-Fu paths) for dependents. The lazbuild backend does + // not need this; it registers package links and lets the IDE resolve deps. TPackageGraph = class private FRunner: TMakeRunner; @@ -168,6 +178,10 @@ TPackageGraph = class class function ExcludePattern: string; class function ShouldExcludeLpkPath(const ALpkPath: string): Boolean; class function ShouldSkipLpk(const ALpkPath: string): Boolean; + // Returns True if the .lpk should be skipped, logging the reason via + // ARunner when it is skipped solely for an LCL (GUI) dependency. + class function ShouldSkipLpkLogged(ARunner: TMakeRunner; + const ALpkPath: string): Boolean; end; // --------------------------------------------------------------------------- @@ -181,6 +195,7 @@ TMakeRunner = class FTargetCpu: string; FTargetOs: string; FErrorCount: Integer; + FUseColor: Boolean; FGraph: TPackageGraph; function ParseBackendEnv: TBuildBackend; function ResolveAutoBackend: TBuildBackend; @@ -251,8 +266,8 @@ TMakeRunner = class Dependencies: array of TDependency = ( // Examples: - // (Kind: TDependencyKind.OPM; Name: 'HashLib'; Ref: ''), - // (Kind: TDependencyKind.GitHub; Name: 'Xor-el/SimpleBaseLib4Pascal'; Ref: 'master'), + // (Kind: TDependencyKind.OPM; Name: 'SimpleBaseLib'; Ref: ''), + // (Kind: TDependencyKind.GitHub; Name: 'Xor-el/HashLib4Pascal'; Ref: 'master'), ); // --------------------------------------------------------------------------- @@ -809,6 +824,9 @@ constructor TLpkPackage.CreateFromFile(const ALpkPath, ATargetCpu, PkgNames.Free; end; + // fpc compiles a unit, not an .lpk. When the package has no real unit named + // after it, synthesize a stub unit that `uses` every listed unit so a single + // `fpc ` builds the whole package. (lazbuild reads the .lpk directly.) FStubPas := IncludeTrailingPathDelimiter(FPkgDir) + FPackageName + '.pas'; if not FileExists(FStubPas) then begin @@ -912,6 +930,16 @@ class function TPackageGraph.ShouldSkipLpk(const ALpkPath: string): Boolean; TLpkPackage.HasLclDependencyInFile(ALpkPath); end; +class function TPackageGraph.ShouldSkipLpkLogged(ARunner: TMakeRunner; + const ALpkPath: string): Boolean; +begin + Result := ShouldSkipLpk(ALpkPath); + // Platform/template packages are excluded silently; only the LCL skip is + // worth a note since it is the reason a console-only CI drops a package. + if Result and not ShouldExcludeLpkPath(ALpkPath) then + ARunner.Log(CSI_Yellow, 'skip LCL-dependent package ' + ALpkPath); +end; + constructor TPackageGraph.Create(ARunner: TMakeRunner); begin inherited Create; @@ -959,12 +987,8 @@ procedure TPackageGraph.RegisterLpk(const ALpkPath: string); var Pkg: TLpkPackage; begin - if ShouldSkipLpk(ALpkPath) then - begin - if not ShouldExcludeLpkPath(ALpkPath) then - FRunner.Log(CSI_Yellow, 'skip LCL-dependent package ' + ALpkPath); + if ShouldSkipLpkLogged(FRunner, ALpkPath) then Exit; - end; Pkg := TLpkPackage.CreateFromFile(ALpkPath, FRunner.TargetCpu, FRunner.TargetOs); if not Pkg.IsValid then @@ -1189,6 +1213,9 @@ constructor TMakeRunner.Create; FBackend := TBuildBackend.Auto; FBackendResolved := False; FErrorCount := 0; + // Honor the NO_COLOR convention (https://no-color.org): any value disables + // ANSI colors. GitHub Actions renders ANSI in its log viewer, so default on. + FUseColor := GetEnvironmentVariable('NO_COLOR') = ''; FGraph := TPackageGraph.Create(Self); end; @@ -1200,12 +1227,18 @@ destructor TMakeRunner.Destroy; procedure TMakeRunner.Log(const AColor, AMessage: string); begin - WriteLn(stderr, AColor, AMessage, CSI_Reset); + if FUseColor then + WriteLn(stderr, AColor, AMessage, CSI_Reset) + else + WriteLn(stderr, AMessage); end; procedure TMakeRunner.LogInline(const AColor, AMessage: string); begin - Write(stderr, AColor, AMessage, CSI_Reset); + if FUseColor then + Write(stderr, AColor, AMessage, CSI_Reset) + else + Write(stderr, AMessage); end; procedure TMakeRunner.IncError; @@ -1497,10 +1530,11 @@ procedure TMakeRunner.UpdateSubmodules; Log(CSI_Yellow, Trim(CommandOutput)); end; -// FPC 3.2.2 hardcodes OpenSSL 1.1 DLL names on Windows, but -// modern CI runners ship OpenSSL 3.x. Override so FPC can find -// the libraries. This hack can be removed once we move to -// FPC 3.2.4+ which natively includes OpenSSL 3.x DLL names. +// TODO(FPC 3.2.4): drop this Windows override. FPC 3.2.2's openssl unit +// hardcodes the OpenSSL 1.1 DLL names (libssl-1_1*.dll / libeay32.dll), but +// modern Windows CI runners ship only OpenSSL 3.x. Point FPC at the 3.x DLLs +// so HTTPS downloads work. FPC 3.2.4+ already knows the OpenSSL 3 names, so +// this whole procedure can become a plain InitSSLInterface call then. procedure TMakeRunner.InitSslForDownloads; begin {$IFDEF MSWINDOWS} @@ -1607,12 +1641,8 @@ procedure TMakeRunner.RegisterPackageLazbuild(const APath: string); var CommandOutput: string; begin - if TPackageGraph.ShouldSkipLpk(APath) then - begin - if not TPackageGraph.ShouldExcludeLpkPath(APath) then - Log(CSI_Yellow, 'skip LCL-dependent package ' + APath); + if TPackageGraph.ShouldSkipLpkLogged(Self, APath) then Exit; - end; if RunCommandEx('lazbuild', ['--add-package-link', APath], '', False, CommandOutput) then Log(CSI_Yellow, 'added ' + APath); diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml index dce928e..3e8c87e 100644 --- a/.github/workflows/make.yml +++ b/.github/workflows/make.yml @@ -14,13 +14,17 @@ on: inputs: enabled_targets: description: >- - Comma-separated list of targets to run (leave empty for default stable + Comma-separated list of targets to run (leave empty for the default set). Opt-in only: netbsd, dragonflybsd — pass explicitly here. Valid IDs: linux-x64, linux-arm64, windows-x64, macos-arm64, macos-x64, linux-arm32, linux-powerpc64-be, freebsd, solaris, netbsd, dragonflybsd default: "" type: string + debug: + description: Verbose install tracing (CI_DEBUG=1 in install-fpc-lazarus.sh) + default: false + type: boolean concurrency: group: ${{ github.workflow }}-${{ github.ref }} @@ -31,6 +35,9 @@ env: LAZARUS_BRANCH: lazarus_4_4 LAZARUS_REPO: https://github.com/fpc/Lazarus.git MAKE_BUILD_BACKEND: fpc + # Set by the workflow_dispatch "debug" toggle; '1' enables `set -x` tracing in + # install-fpc-lazarus.sh, '0' otherwise. Forwarded into the sandboxed jobs below. + CI_DEBUG: ${{ github.event.inputs.debug == 'true' && '1' || '0' }} jobs: @@ -39,6 +46,7 @@ jobs: runs-on: ubuntu-latest outputs: enabled_targets: ${{ steps.resolve.outputs.enabled_targets }} + native_matrix: ${{ steps.resolve.outputs.native_matrix }} steps: - uses: actions/checkout@v6 @@ -51,56 +59,23 @@ jobs: native: needs: setup + # native_matrix is built from targets.json filtered to enabled native ids. + # An empty array would be an invalid matrix, so skip the job in that case. + if: needs.setup.outputs.native_matrix != '[]' name: ${{ matrix.name }} runs-on: ${{ matrix.runner }} timeout-minutes: 120 strategy: fail-fast: false matrix: - include: - - id: linux-x64 - runner: ubuntu-latest - name: Linux x86_64 - fpc_target: x86_64-linux - - id: linux-arm64 - runner: ubuntu-24.04-arm - name: Linux AArch64 - fpc_target: aarch64-linux - - id: windows-x64 - runner: windows-latest - name: Windows x86_64 - fpc_target: x86_64-win64 - - id: macos-arm64 - runner: macos-latest - name: macOS AArch64 (Apple Silicon) - fpc_target: aarch64-darwin - - id: macos-x64 - runner: macos-15-intel - name: macOS x86_64 (Intel) - fpc_target: x86_64-darwin + include: ${{ fromJSON(needs.setup.outputs.native_matrix) }} steps: - - name: Check if target is enabled - id: gate - shell: bash - env: - ENABLED_TARGETS: ${{ needs.setup.outputs.enabled_targets }} - TARGET_ID: ${{ matrix.id }} - run: | - if [[ ",${ENABLED_TARGETS}," == *",${TARGET_ID},"* ]]; then - echo "enabled=true" >> "$GITHUB_OUTPUT" - else - echo "enabled=false" >> "$GITHUB_OUTPUT" - echo "Skipping ${TARGET_ID} (not in enabled targets)" - fi - - name: Checkout - if: steps.gate.outputs.enabled == 'true' uses: actions/checkout@v6 with: submodules: true - name: Build - if: steps.gate.outputs.enabled == 'true' shell: bash env: FPC_TARGET: ${{ matrix.fpc_target }} @@ -129,6 +104,7 @@ jobs: LAZARUS_BRANCH: ${{ env.LAZARUS_BRANCH }} LAZARUS_REPO: ${{ env.LAZARUS_REPO }} MAKE_BUILD_BACKEND: ${{ env.MAKE_BUILD_BACKEND }} + CI_DEBUG: ${{ env.CI_DEBUG }} run: bash .github/workflows/ci/arm32-run.sh linux-powerpc64-be: @@ -143,7 +119,7 @@ jobs: with: submodules: true - - name: Set up QEMU (ppc64 BE, pinned) + - name: Set up QEMU (ppc64 big-endian) run: bash .github/workflows/ci/ppc64-qemu-setup.sh - name: Build (PowerPC64 BE via QEMU) @@ -171,7 +147,7 @@ jobs: FPC_TARGET: x86_64-freebsd FREEBSD_INSTALL_MODE: interim with: - envs: FPC_VERSION FPC_TARGET LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FREEBSD_INSTALL_MODE + envs: FPC_VERSION FPC_TARGET LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FREEBSD_INSTALL_MODE CI_DEBUG release: "15.0" usesh: true prepare: sh .github/workflows/ci/vm-freebsd-prepare.sh @@ -194,7 +170,7 @@ jobs: env: FPC_TARGET: x86_64-netbsd with: - envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FPC_TARGET + envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FPC_TARGET CI_DEBUG prepare: sh .github/workflows/ci/vm-netbsd-prepare.sh run: bash .github/workflows/ci/vm-run-shared.sh @@ -216,7 +192,7 @@ jobs: FPC_TARGET: x86_64-dragonfly LD_LIBRARY_PATH_EXTRA: /usr/local/lib with: - envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FPC_TARGET LD_LIBRARY_PATH_EXTRA + envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FPC_TARGET LD_LIBRARY_PATH_EXTRA CI_DEBUG usesh: true prepare: sh .github/workflows/ci/vm-dragonfly-prepare.sh run: bash .github/workflows/ci/vm-run-shared.sh @@ -238,7 +214,7 @@ jobs: env: FPC_TARGET: x86_64-solaris with: - envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FPC_TARGET + envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FPC_TARGET CI_DEBUG release: "11.4-gcc" usesh: true prepare: sh .github/workflows/ci/vm-solaris-prepare.sh From 9d4b27354e8304cb5992f3158acc7c74cac8f852 Mon Sep 17 00:00:00 2001 From: Ugochukwu Mmaduekwe Date: Tue, 9 Jun 2026 04:11:27 +0100 Subject: [PATCH 4/5] ci updates --- .github/workflows/ci/ppc64-be-build.sh | 3 + .github/workflows/install-fpc-lazarus.sh | 19 +- .github/workflows/make.pas | 318 +++++++++++++++++------ .github/workflows/make.yml | 16 +- 4 files changed, 256 insertions(+), 100 deletions(-) diff --git a/.github/workflows/ci/ppc64-be-build.sh b/.github/workflows/ci/ppc64-be-build.sh index 2677745..d17cd60 100644 --- a/.github/workflows/ci/ppc64-be-build.sh +++ b/.github/workflows/ci/ppc64-be-build.sh @@ -36,6 +36,9 @@ docker run --rm --platform linux/ppc64 \ -e FPC_VERSION \ -e FPC_TARGET \ -e MAKE_BUILD_BACKEND \ + -e MAKE_PACKAGE_SCOPE \ + -e LAZARUS_BRANCH \ + -e LAZARUS_REPO \ -e CI_DEBUG \ -e DEBIAN_FRONTEND=noninteractive \ -e QEMU_CPU=power8 \ diff --git a/.github/workflows/install-fpc-lazarus.sh b/.github/workflows/install-fpc-lazarus.sh index ad7d13c..f2ea290 100644 --- a/.github/workflows/install-fpc-lazarus.sh +++ b/.github/workflows/install-fpc-lazarus.sh @@ -20,11 +20,9 @@ # LAZARUS_REPO git URL # MAKE_CMD 'make' on Linux/Windows/macOS, 'gmake' on BSD/Solaris # (auto-detected if unset) -# MAKE_BUILD_BACKEND auto | lazbuild | fpc -# fpc — install FPC only; skip Lazarus/lazbuild -# lazbuild | auto — also clone Lazarus and build lazbuild -# (auto here always installs Lazarus; make.pas auto probes -# lazbuild on PATH at runtime and may still choose fpc) +# MAKE_BUILD_BACKEND lazbuild | fpc (default: fpc) +# fpc — install FPC only; skip Lazarus/lazbuild +# lazbuild — also clone Lazarus and build lazbuild # # Outputs (appended to $GITHUB_PATH if set): # $INSTALL_PREFIX/bin and $LAZARUS_DIR are added to PATH @@ -41,12 +39,11 @@ source "$INSTALL_SCRIPT_DIR/ci/shared/common.sh" : "${FPC_VERSION:?FPC_VERSION is required (e.g. 3.2.2)}" : "${FPC_TARGET:?FPC_TARGET is required (e.g. x86_64-linux)}" -: "${MAKE_BUILD_BACKEND:?MAKE_BUILD_BACKEND is required (lazbuild|fpc|auto)}" -case "$MAKE_BUILD_BACKEND" in - fpc) INSTALL_LAZARUS=0 ;; - auto|lazbuild) INSTALL_LAZARUS=1 ;; +case "${MAKE_BUILD_BACKEND:-fpc}" in + fpc) INSTALL_LAZARUS=0 ;; + lazbuild) INSTALL_LAZARUS=1 ;; *) - echo "unknown MAKE_BUILD_BACKEND: $MAKE_BUILD_BACKEND" >&2 + echo "unknown MAKE_BUILD_BACKEND: $MAKE_BUILD_BACKEND (expected lazbuild|fpc)" >&2 exit 1 ;; esac @@ -263,7 +260,7 @@ fi # ci_fpc_info_probe instead of letting one emulation hiccup fail the install. ci_fpc_info_probe -iV -# ── Build Lazarus from source (lazbuild | auto only) ───────────────── +# ── Build Lazarus from source (lazbuild backend only) ──────────────── # # When INSTALL_LAZARUS=1, clone Lazarus and build lazbuild (~1–2 min). # Packaged Lazarus on Linux/Windows pulls in the full IDE; we only need diff --git a/.github/workflows/make.pas b/.github/workflows/make.pas index 9245579..78bf06e 100644 --- a/.github/workflows/make.pas +++ b/.github/workflows/make.pas @@ -14,15 +14,24 @@ TMakeRunner = class; // Build backend // --------------------------------------------------------------------------- - // Selected via MAKE_BUILD_BACKEND. Auto probes for lazbuild on PATH and - // falls back to Fpc. Lazbuild drives builds through the IDE tool; Fpc builds - // packages/projects by invoking the compiler directly (see TPackageGraph). + // Selected via MAKE_BUILD_BACKEND (defaults to Fpc when unset). Lazbuild + // drives builds through the IDE tool; Fpc builds packages/projects by + // invoking the compiler directly (see TPackageGraph). TBuildBackend = ( - Auto, Lazbuild, Fpc ); + // Selected via MAKE_PACKAGE_SCOPE (defaults to Required when unset). 'all' + // compiles every discovered dependency package, so a package that fails to + // compile on the target is caught even when no built project references it; + // 'required' compiles only the packages the buildable projects transitively + // depend on. Honoured by both backends. + TPackageScope = ( + All, + Required + ); + // --------------------------------------------------------------------------- // Dependency configuration // --------------------------------------------------------------------------- @@ -166,12 +175,15 @@ TPackageGraph = class procedure CollectBuildOrder(const AIndex: Integer; AOrder: specialize TList); procedure CollectUnitPaths(const AIndex: Integer; AVisited: specialize TList; APaths: TStrings); + function BuildPackageAt(const AIndex: Integer): Boolean; + function BuildOrder(AOrder: specialize TList): Boolean; public constructor Create(ARunner: TMakeRunner); destructor Destroy; override; procedure DiscoverUnder(const ARoot: string); procedure RegisterLpk(const ALpkPath: string); function BuildAll: Boolean; + function BuildRequired(const ANames: TStrings): Boolean; function UnitPathFor(const APackageName: string): string; function UnitPathsForRequired(const ANames: TStrings): TStringList; function PackageCount: Integer; @@ -192,16 +204,19 @@ TMakeRunner = class private FBackend: TBuildBackend; FBackendResolved: Boolean; + FPackageScope: TPackageScope; FTargetCpu: string; FTargetOs: string; FErrorCount: Integer; FUseColor: Boolean; FGraph: TPackageGraph; function ParseBackendEnv: TBuildBackend; - function ResolveAutoBackend: TBuildBackend; + function ParsePackageScopeEnv: TPackageScope; procedure InitEnvironment; procedure UpdateSubmodules; procedure InstallDependencies; + procedure BuildDiscoveredPackagesFpc; + function CollectProjectRequiredNames: TStringList; procedure BuildAllProjects; function BuildProject(const ALpiPath: string): string; function BuildProjectWithLazbuild(const APath: string): string; @@ -219,6 +234,8 @@ TMakeRunner = class function ResolveDependency(const ADep: TDependency): string; procedure RegisterPackageLazbuild(const APath: string); procedure RegisterAllPackagesLazbuild(const ASearchDir: string); + procedure BuildPackageLazbuild(const APath: string); + procedure BuildAllPackagesLazbuild(const ASearchDir: string); function UsesLazbuild: Boolean; function RunCommandEx(const AExecutable: string; const AArgs: TStrings; const AWorkingDir: string; AStreamToStderr: Boolean; @@ -1097,64 +1114,107 @@ procedure TPackageGraph.CollectUnitPaths(const AIndex: Integer; VisitPackageDeps(AIndex, AVisited, TDepVisitKind.UnitPaths, nil, APaths); end; -function TPackageGraph.BuildAll: Boolean; +// Compile a single discovered package by graph index. Wipes and recreates its +// unit output dir, then invokes fpc with the package's options plus the unit +// paths of its dependencies. Returns False (and records an error) on failure. +function TPackageGraph.BuildPackageAt(const AIndex: Integer): Boolean; var - Order: specialize TList; - I, Idx, J: Integer; Pkg: TLpkPackage; Args: TStringList; - BuildOutput: string; - OutDir: string; - DepPath: string; + BuildOutput, OutDir, DepPath: string; + J: Integer; begin Result := True; - if FItems.Count = 0 then - Exit; + Pkg := GetPackage(AIndex); + OutDir := Pkg.ResolveUnitOutDir(FRunner.TargetCpu, FRunner.TargetOs); - Order := specialize TList.Create; + FRunner.LogInline(CSI_Yellow, 'build package ' + Pkg.PackageName); + TProjectFiles.RemoveRecursive(OutDir); + ForceDirectories(OutDir); + + Args := TStringList.Create; try - for I := 0 to FItems.Count - 1 do - CollectBuildOrder(I, Order); + TLazXml.AppendCompilerOptionsToArgv(Pkg.Options, Pkg.PkgDir, OutDir, + OutDir, FRunner.TargetCpu, FRunner.TargetOs, Args); + for J := 0 to Pkg.SourceDirs.Count - 1 do + TLazXml.AppendFuIfMissing(Pkg.SourceDirs[J], Args); + TLazXml.AppendFuIfMissing(Pkg.PkgDir, Args); + for J := 0 to Pkg.RequiredNames.Count - 1 do + begin + if IsBuiltinPackage(Pkg.RequiredNames[J]) then + Continue; + DepPath := UnitPathFor(Pkg.RequiredNames[J]); + TLazXml.AppendFuIfMissing(DepPath, Args); + end; + TLazXml.AppendPackageBuildArgs(Args, ExtractFileName(Pkg.StubPas), OutDir); - for I := 0 to Order.Count - 1 do + if FRunner.RunCommandEx('fpc', Args, Pkg.PkgDir, True, BuildOutput) then + FRunner.Log(CSI_Green, ' -> ' + OutDir) + else begin - Idx := Order[I]; - Pkg := GetPackage(Idx); + FRunner.IncError; + FRunner.ReportBuildErrors(BuildOutput); + Result := False; + end; + finally + Args.Free; + end; +end; - OutDir := Pkg.ResolveUnitOutDir(FRunner.TargetCpu, FRunner.TargetOs); +// Compile the packages in AOrder (already topologically sorted, dependencies +// first). Builds every entry even when one fails, accumulating errors, and +// returns False if any package failed. +function TPackageGraph.BuildOrder(AOrder: specialize TList): Boolean; +var + I: Integer; +begin + Result := True; + for I := 0 to AOrder.Count - 1 do + if not BuildPackageAt(AOrder[I]) then + Result := False; +end; + +// MAKE_PACKAGE_SCOPE=all: compile every discovered package, in build order. +function TPackageGraph.BuildAll: Boolean; +var + Order: specialize TList; + I: Integer; +begin + if FItems.Count = 0 then + Exit(True); - FRunner.LogInline(CSI_Yellow, 'build package ' + Pkg.PackageName); - TProjectFiles.RemoveRecursive(OutDir); - ForceDirectories(OutDir); + Order := specialize TList.Create; + try + for I := 0 to FItems.Count - 1 do + CollectBuildOrder(I, Order); + Result := BuildOrder(Order); + finally + Order.Free; + end; +end; - Args := TStringList.Create; - try - TLazXml.AppendCompilerOptionsToArgv(Pkg.Options, Pkg.PkgDir, OutDir, - OutDir, FRunner.TargetCpu, FRunner.TargetOs, Args); - for J := 0 to Pkg.SourceDirs.Count - 1 do - TLazXml.AppendFuIfMissing(Pkg.SourceDirs[J], Args); - TLazXml.AppendFuIfMissing(Pkg.PkgDir, Args); - for J := 0 to Pkg.RequiredNames.Count - 1 do - begin - if IsBuiltinPackage(Pkg.RequiredNames[J]) then - Continue; - DepPath := UnitPathFor(Pkg.RequiredNames[J]); - TLazXml.AppendFuIfMissing(DepPath, Args); - end; - TLazXml.AppendPackageBuildArgs(Args, ExtractFileName(Pkg.StubPas), OutDir); +// MAKE_PACKAGE_SCOPE=required: compile only the dependency closure of ANames, +// in build order. Unknown names are ignored here; the project build reports +// them via UnitPathsForRequired. +function TPackageGraph.BuildRequired(const ANames: TStrings): Boolean; +var + Order: specialize TList; + I, Idx: Integer; +begin + if (ANames = nil) or (ANames.Count = 0) or (FItems.Count = 0) then + Exit(True); - if FRunner.RunCommandEx('fpc', Args, Pkg.PkgDir, True, BuildOutput) then - FRunner.Log(CSI_Green, ' -> ' + OutDir) - else - begin - FRunner.IncError; - FRunner.ReportBuildErrors(BuildOutput); - Result := False; - end; - finally - Args.Free; - end; + Order := specialize TList.Create; + try + for I := 0 to ANames.Count - 1 do + begin + if IsBuiltinPackage(ANames[I]) then + Continue; + Idx := FindIndexByName(ANames[I]); + if Idx >= 0 then + CollectBuildOrder(Idx, Order); end; + Result := BuildOrder(Order); finally Order.Free; end; @@ -1210,8 +1270,9 @@ function TPackageGraph.UnitPathsForRequired(const ANames: TStrings): TStringList constructor TMakeRunner.Create; begin inherited Create; - FBackend := TBuildBackend.Auto; + FBackend := TBuildBackend.Fpc; FBackendResolved := False; + FPackageScope := TPackageScope.Required; FErrorCount := 0; // Honor the NO_COLOR convention (https://no-color.org): any value disables // ANSI colors. GitHub Actions renders ANSI in its log viewer, so default on. @@ -1275,8 +1336,8 @@ function TMakeRunner.ParseBackendEnv: TBuildBackend; Env: string; begin Env := LowerCase(Trim(GetEnvironmentVariable('MAKE_BUILD_BACKEND'))); - if (Env = '') or (Env = 'auto') then - Exit(TBuildBackend.Auto); + if Env = '' then + Exit(TBuildBackend.Fpc); if Env = 'lazbuild' then Exit(TBuildBackend.Lazbuild); if Env = 'fpc' then @@ -1284,13 +1345,18 @@ function TMakeRunner.ParseBackendEnv: TBuildBackend; raise Exception.CreateFmt('unknown MAKE_BUILD_BACKEND: "%s"', [Env]); end; -function TMakeRunner.ResolveAutoBackend: TBuildBackend; +function TMakeRunner.ParsePackageScopeEnv: TPackageScope; var - Output: string; + Env: string; begin - if RunCommandEx('lazbuild', ['--version'], '', False, Output) then - Exit(TBuildBackend.Lazbuild); - Result := TBuildBackend.Fpc; + Env := LowerCase(Trim(GetEnvironmentVariable('MAKE_PACKAGE_SCOPE'))); + if Env = '' then + Exit(TPackageScope.Required); + if Env = 'all' then + Exit(TPackageScope.All); + if Env = 'required' then + Exit(TPackageScope.Required); + raise Exception.CreateFmt('unknown MAKE_PACKAGE_SCOPE: "%s"', [Env]); end; function TMakeRunner.UsesLazbuild: Boolean; @@ -1483,7 +1549,6 @@ procedure TMakeRunner.RunBuiltBinary(const ABinaryPath: string; procedure TMakeRunner.InitEnvironment; var - Requested: TBuildBackend; Output: string; begin if FBackendResolved then @@ -1494,19 +1559,10 @@ procedure TMakeRunner.InitEnvironment; if not RunFpcInfoProbeWithRetry('-iTO', FTargetOs) then raise Exception.Create('fpc -iTO returned empty TargetOS'); - Requested := ParseBackendEnv; - case Requested of - TBuildBackend.Lazbuild: - begin - if not RunCommandEx('lazbuild', ['--version'], '', False, Output) then - raise Exception.Create('MAKE_BUILD_BACKEND=lazbuild but lazbuild not found'); - FBackend := TBuildBackend.Lazbuild; - end; - TBuildBackend.Fpc: - FBackend := TBuildBackend.Fpc; - TBuildBackend.Auto: - FBackend := ResolveAutoBackend; - end; + FBackend := ParseBackendEnv; + if (FBackend = TBuildBackend.Lazbuild) + and not RunCommandEx('lazbuild', ['--version'], '', False, Output) then + raise Exception.Create('MAKE_BUILD_BACKEND=lazbuild but lazbuild not found'); FBackendResolved := True; case FBackend of @@ -1514,8 +1570,14 @@ procedure TMakeRunner.InitEnvironment; Log(CSI_Yellow, 'build backend: lazbuild'); TBuildBackend.Fpc: Log(CSI_Yellow, 'build backend: fpc'); - TBuildBackend.Auto: - ; + end; + + FPackageScope := ParsePackageScopeEnv; + case FPackageScope of + TPackageScope.All: + Log(CSI_Yellow, 'package scope: all'); + TPackageScope.Required: + Log(CSI_Yellow, 'package scope: required'); end; end; @@ -1653,36 +1715,124 @@ procedure TMakeRunner.RegisterAllPackagesLazbuild(const ASearchDir: string); ForEachLpkInDir(ASearchDir, @RegisterPackageLazbuild); end; +procedure TMakeRunner.BuildPackageLazbuild(const APath: string); +var + BuildOutput: string; +begin + // Parity with the fpc backend's TPackageGraph.BuildAll: --add-package-link only + // registers a package; it never compiles it, so a dependency that fails to + // compile on this target goes unnoticed unless a built project happens to use + // it. Compile every registered package explicitly to catch that. + // Uses the non-logging skip so LCL packages are not re-announced (registration + // already logged them). + if TPackageGraph.ShouldSkipLpk(APath) then + Exit; + LogInline(CSI_Yellow, 'build package ' + APath); + if RunCommandEx('lazbuild', ['--build-all', '--recursive', APath], '', True, + BuildOutput) then + Log(CSI_Green, ' -> ok') + else + begin + WriteLn(stderr, BuildOutput); + IncError; + ReportBuildErrors(BuildOutput); + end; +end; + +procedure TMakeRunner.BuildAllPackagesLazbuild(const ASearchDir: string); +begin + ForEachLpkInDir(ASearchDir, @BuildPackageLazbuild); +end; + procedure TMakeRunner.InstallDependencies; var - DepDirs: TStringList; + Roots: TStringList; I: Integer; begin - DepDirs := TStringList.Create; + // Search roots = each resolved dependency directory plus the repo itself. + Roots := TStringList.Create; try if Length(Dependencies) > 0 then begin InitSslForDownloads; for I := 0 to High(Dependencies) do - DepDirs.Add(ResolveDependency(Dependencies[I])); + Roots.Add(ResolveDependency(Dependencies[I])); end; + Roots.Add(RepoRoot); if UsesLazbuild then begin - for I := 0 to DepDirs.Count - 1 do - RegisterAllPackagesLazbuild(DepDirs[I]); - RegisterAllPackagesLazbuild(RepoRoot); + // Register every package first so cross-package dependencies resolve. + for I := 0 to Roots.Count - 1 do + RegisterAllPackagesLazbuild(Roots[I]); + // Scope=all: also compile each registered package so a package that fails + // to build on this target is caught even when no project uses it (parity + // with the fpc backend — see BuildPackageLazbuild). Scope=required: + // lazbuild compiles the packages a project needs while building it. + if FPackageScope = TPackageScope.All then + for I := 0 to Roots.Count - 1 do + BuildAllPackagesLazbuild(Roots[I]); end else begin - for I := 0 to DepDirs.Count - 1 do - FGraph.DiscoverUnder(DepDirs[I]); - FGraph.DiscoverUnder(RepoRoot); + for I := 0 to Roots.Count - 1 do + FGraph.DiscoverUnder(Roots[I]); if FGraph.PackageCount > 0 then - FGraph.BuildAll; + BuildDiscoveredPackagesFpc; end; finally - DepDirs.Free; + Roots.Free; + end; +end; + +// fpc backend: compile discovered packages according to MAKE_PACKAGE_SCOPE. +procedure TMakeRunner.BuildDiscoveredPackagesFpc; +var + Names: TStringList; +begin + if FPackageScope = TPackageScope.All then + FGraph.BuildAll + else + begin + Names := CollectProjectRequiredNames; + try + FGraph.BuildRequired(Names); + finally + Names.Free; + end; + end; +end; + +// Union of RequiredPackages across the buildable (non-GUI) projects under the +// target directory. Drives the fpc backend's 'required' scope so it compiles +// only the dependency closure those projects need. +function TMakeRunner.CollectProjectRequiredNames: TStringList; +var + List: TStringList; + Each: string; + Proj: TLpiProject; + I: Integer; +begin + Result := TStringList.Create; + Result.Sorted := True; + Result.Duplicates := dupIgnore; + List := TProjectFiles.FindAll(TargetDirectory, '*.lpi'); + try + for Each in List do + begin + if IsGUIProject(Each) then + Continue; + Proj := TLpiProject.CreateFromFile(Each, FTargetCpu, FTargetOs); + try + if Proj.IsValid then + for I := 0 to Proj.RequiredPackageNames.Count - 1 do + Result.Add(Proj.RequiredPackageNames[I]); + finally + Proj.Free; + end; + end; + finally + List.Free; end; end; diff --git a/.github/workflows/make.yml b/.github/workflows/make.yml index 3e8c87e..29fd5ee 100644 --- a/.github/workflows/make.yml +++ b/.github/workflows/make.yml @@ -35,6 +35,10 @@ env: LAZARUS_BRANCH: lazarus_4_4 LAZARUS_REPO: https://github.com/fpc/Lazarus.git MAKE_BUILD_BACKEND: fpc + # make.pas package build scope: 'all' compiles every discovered package (catches + # a package that fails on the target even if unused); 'required' only compiles + # the projects' dependency closure. Override per job below, like MAKE_BUILD_BACKEND. + MAKE_PACKAGE_SCOPE: required # Set by the workflow_dispatch "debug" toggle; '1' enables `set -x` tracing in # install-fpc-lazarus.sh, '0' otherwise. Forwarded into the sandboxed jobs below. CI_DEBUG: ${{ github.event.inputs.debug == 'true' && '1' || '0' }} @@ -104,6 +108,7 @@ jobs: LAZARUS_BRANCH: ${{ env.LAZARUS_BRANCH }} LAZARUS_REPO: ${{ env.LAZARUS_REPO }} MAKE_BUILD_BACKEND: ${{ env.MAKE_BUILD_BACKEND }} + MAKE_PACKAGE_SCOPE: ${{ env.MAKE_PACKAGE_SCOPE }} CI_DEBUG: ${{ env.CI_DEBUG }} run: bash .github/workflows/ci/arm32-run.sh @@ -126,7 +131,8 @@ jobs: env: FPC_VERSION: ${{ env.FPC_VERSION }} FPC_TARGET: powerpc64-linux - MAKE_BUILD_BACKEND: fpc + MAKE_BUILD_BACKEND: ${{ env.MAKE_BUILD_BACKEND }} + MAKE_PACKAGE_SCOPE: ${{ env.MAKE_PACKAGE_SCOPE }} run: bash .github/workflows/ci/ppc64-be-build.sh freebsd: @@ -147,7 +153,7 @@ jobs: FPC_TARGET: x86_64-freebsd FREEBSD_INSTALL_MODE: interim with: - envs: FPC_VERSION FPC_TARGET LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FREEBSD_INSTALL_MODE CI_DEBUG + envs: FPC_VERSION FPC_TARGET LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND MAKE_PACKAGE_SCOPE FREEBSD_INSTALL_MODE CI_DEBUG release: "15.0" usesh: true prepare: sh .github/workflows/ci/vm-freebsd-prepare.sh @@ -170,7 +176,7 @@ jobs: env: FPC_TARGET: x86_64-netbsd with: - envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FPC_TARGET CI_DEBUG + envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND MAKE_PACKAGE_SCOPE FPC_TARGET CI_DEBUG prepare: sh .github/workflows/ci/vm-netbsd-prepare.sh run: bash .github/workflows/ci/vm-run-shared.sh @@ -192,7 +198,7 @@ jobs: FPC_TARGET: x86_64-dragonfly LD_LIBRARY_PATH_EXTRA: /usr/local/lib with: - envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FPC_TARGET LD_LIBRARY_PATH_EXTRA CI_DEBUG + envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND MAKE_PACKAGE_SCOPE FPC_TARGET LD_LIBRARY_PATH_EXTRA CI_DEBUG usesh: true prepare: sh .github/workflows/ci/vm-dragonfly-prepare.sh run: bash .github/workflows/ci/vm-run-shared.sh @@ -214,7 +220,7 @@ jobs: env: FPC_TARGET: x86_64-solaris with: - envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND FPC_TARGET CI_DEBUG + envs: FPC_VERSION LAZARUS_BRANCH LAZARUS_REPO MAKE_BUILD_BACKEND MAKE_PACKAGE_SCOPE FPC_TARGET CI_DEBUG release: "11.4-gcc" usesh: true prepare: sh .github/workflows/ci/vm-solaris-prepare.sh From 7537118cbeb9f0715b7384425428f91d4c083392 Mon Sep 17 00:00:00 2001 From: Ugochukwu Mmaduekwe Date: Tue, 9 Jun 2026 10:27:14 +0100 Subject: [PATCH 5/5] update to make.pas --- .github/workflows/make.pas | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/make.pas b/.github/workflows/make.pas index 78bf06e..c73c9cd 100644 --- a/.github/workflows/make.pas +++ b/.github/workflows/make.pas @@ -364,7 +364,7 @@ class function TLazXml.ExtractBlock(const AContent, AOpenTag: string): string; CloseTag := ''; Q := PosEx(CloseTag, AContent, P); if Q = 0 then - Result := Copy(AContent, P, MaxInt) + Result := Copy(AContent, P, Length(AContent) - P + 1) else Result := Copy(AContent, P, Q - P + Length(CloseTag)); end; @@ -440,7 +440,7 @@ class function TLazXml.ArgsHasFuPath(const AArgs: TStrings; const APath: string) begin if not StartsText('-Fu', AArgs[I]) then Continue; - ArgPath := Copy(AArgs[I], 4, MaxInt); + ArgPath := Copy(AArgs[I], 4, Length(AArgs[I]) - 3); if SameText(Norm, LowerCase(ExpandFileName(ExcludeTrailingPathDelimiter(ArgPath)))) then Exit(True); end;