diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index aee107e..38abae8 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -14,11 +14,11 @@ jobs: with: fetch-depth: 0 - - name: Install linters + - name: Install linters and test runner run: | sudo snap install ruff sudo apt-get update - sudo apt-get install -y mypy + sudo apt-get install -y mypy python3-pytest - name: Install qcom-ptool run: | @@ -29,6 +29,7 @@ jobs: PTOOL_SEED: qcom-ptool-ci run: | make lint + make unit-test make all integration check-checksums - name: Verify checksum manifest is up to date diff --git a/Makefile b/Makefile index a6a6d54..c2c4487 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,21 @@ TOPDIR := $(PWD) -PARTITIONS := $(wildcard platforms/*/*/partitions.conf) -PARTITIONS_XML := $(patsubst %.conf,%.xml, $(PARTITIONS)) -PLATFORMS := $(patsubst %/partitions.conf,%/gpt, $(PARTITIONS)) + +# Partition sources come in two forms: +# * legacy boards - platforms///partitions.conf +# * migrated boards - a YAML board under platforms/boards/ that emits one +# partitions.xml per storage (see the grouped rules below). + +# Migrated board: Glymur-CRD (YAML board, Debian HLOS variant). +GLYMUR_XML := platforms/glymur-crd/nvme/partitions.xml \ + platforms/glymur-crd/spinor/partitions.xml + +YAML_PARTITIONS_XML := $(GLYMUR_XML) + +# Legacy .conf sources, excluding any board already migrated to YAML. +YAML_CONF_EXCLUDE := $(patsubst %.xml,%.conf, $(YAML_PARTITIONS_XML)) +PARTITIONS := $(filter-out $(YAML_CONF_EXCLUDE), $(wildcard platforms/*/*/partitions.conf)) +PARTITIONS_XML := $(patsubst %.conf,%.xml, $(PARTITIONS)) $(YAML_PARTITIONS_XML) +PLATFORMS := $(patsubst %/partitions.xml,%/gpt, $(PARTITIONS_XML)) CONTENTS_XML_IN := $(wildcard platforms/*/*/contents.xml.in) CONTENTS_XML := $(patsubst %.xml.in,%.xml, $(CONTENTS_XML_IN)) @@ -12,7 +26,7 @@ QCOM_PTOOL ?= qcom-ptool # optional build_id for Axiom contents.xml files BUILD_ID ?= -.PHONY: all check check-checksums clean generate-checksums install lint integration +.PHONY: all check check-checksums clean generate-checksums install lint integration unit-test all: $(PLATFORMS) $(PARTITIONS_XML) $(CONTENTS_XML) @@ -22,6 +36,15 @@ all: $(PLATFORMS) $(PARTITIONS_XML) $(CONTENTS_XML) %/partitions.xml: %/partitions.conf $(QCOM_PTOOL) gen_partition -i $^ -o $@ +# Glymur-CRD: both storage XMLs are emitted by one board resolution. +# The storage output dirs may not exist on a fresh checkout (a migrated +# storage need not carry any tracked file), so create them first. +$(GLYMUR_XML) &: platforms/boards/glymur-crd.yaml \ + platforms/variants/hlos/qcom-deb-images.yaml + @mkdir -p $(dir $(GLYMUR_XML)) + $(QCOM_PTOOL) gen_partition --board platforms/boards/glymur-crd.yaml \ + --hlos qcom-deb-images $(addprefix -o ,$(GLYMUR_XML)) + %/contents.xml: %/partitions.xml %/contents.xml.in $(QCOM_PTOOL) gen_contents -p $< -t $@.in -o $@ $${BUILD_ID:+ -b $(BUILD_ID)} @@ -29,6 +52,9 @@ lint: ruff check qcom_ptool mypy qcom_ptool +unit-test: + pytest + integration: all # make sure generated output has created expected files tests/integration/check-missing-files platforms/*/*/*.xml @@ -46,7 +72,7 @@ generate-checksums: all ! -name '*.xml.in' -print0 | LC_ALL=C sort -z | xargs -0 sha256sum \ > tests/integration/checksums.sha256 -check: lint integration +check: lint unit-test integration install: pip install . diff --git a/README.md b/README.md index 4bed9c8..8146df0 100644 --- a/README.md +++ b/README.md @@ -30,17 +30,19 @@ subcommand. ## Dependencies -At runtime, the scripts use only the Python standard library (Python 3.8+), -so no runtime dependencies need to be installed beyond the package itself. +At runtime the tool targets Python 3.8+ and depends on two third-party +libraries, `PyYAML` and `jsonschema`, used to load and validate the YAML +partition source. Both are declared in `pyproject.toml` and pulled in +automatically by `pip install .`. -For development, `make lint` invokes `ruff` and `mypy` directly from the -command line. On Debian/Ubuntu, install them as follows (ruff is not -packaged in apt on all releases/architectures, so we install it from -snap): +For development, `make lint` invokes `ruff` and `mypy` and `make unit-test` +runs the `pytest` suite under `tests/unit/`. On Debian/Ubuntu, install +them as follows (ruff is not packaged in apt on all releases/architectures, +so we install it from snap): ```sh sudo snap install ruff -sudo apt install mypy +sudo apt install mypy python3-pytest ``` ## Makefile targets @@ -49,8 +51,9 @@ sudo apt install mypy |---------------|------------------------------------------------------------| | `all` | Generate partition XML and GPT binaries for all platforms | | `lint` | Run ruff (linter) and mypy (type checker) on the package | +| `unit-test` | Run the pytest suite under `tests/unit/` | | `integration` | Build all platforms and verify generated files are present | -| `check` | Run both `lint` and `integration` | +| `check` | Run `lint`, `unit-test`, and `integration` | | `install` | Install the package (`pip install .`) | | `clean` | Remove generated XML and binary files from platforms/ | @@ -63,12 +66,13 @@ The Makefile invokes `qcom-ptool` from `PATH`. Install the package (or # install the tool pip install -e . -# install linters (Debian/Ubuntu) +# install linters and test runner (Debian/Ubuntu) sudo snap install ruff -sudo apt install mypy +sudo apt install mypy python3-pytest -# run linters +# run linters and unit tests make lint +make unit-test # build all platforms and run tests make check diff --git a/platforms/boards/glymur-crd.yaml b/platforms/boards/glymur-crd.yaml new file mode 100644 index 0000000..36c8873 --- /dev/null +++ b/platforms/boards/glymur-crd.yaml @@ -0,0 +1,93 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# +# Glymur-CRD board definition (structured YAML source, proof of concept). +# Multi-storage: NVMe (HLOS) + SPI-NOR (boot / platform-config). +# Partition order is preserved verbatim from the legacy .conf files so +# the emitted XML and GPT are byte-identical. The NVMe rootfs size and +# filename are supplied by an --hlos variant (variants/hlos/*.yaml). +platform: + name: glymur-crd + +storage: + - id: nvme0 + type: nvme + size: 68719476736 # 64 GiB + sector-size: 512 + write-protect-boundary: 65536 + grow-last-partition: true + partitions: + - {name: efi, size: "524288KB", type-guid: "C12A7328-F81F-11D2-BA4B-00A0C93EC93B", filename: efi.bin} + - {name: rootfs, type-guid: "B921B045-1DF0-41C3-AF44-4C6F280D3FAE"} + + - id: spinor0 + type: spinor + size: 67108864 # 64 MiB + sector-size: 4096 + write-protect-boundary: 0 + partitions: + - {name: cdt, size: "4KB", type-guid: "A19F205F-CCD8-4B6D-8F1E-2D9BC24CFFB1", filename: cdt.bin} + - {name: SD_MGR, size: "528KB", type-guid: "5E463172-D0AC-4DD4-91B8-CD3EE1281579"} + - {name: VarStore, size: "728KB", type-guid: "165BD6BC-9250-4AC8-95A7-A93F4A440066"} + - {name: QWESLICCACHE, size: "36KB", type-guid: "7DDC813A-E88A-491A-95DC-4811A869D313"} + - {name: QWESLICSTORE, size: "128KB", type-guid: "7BAB3C93-5F73-4D02-B8CB-5B9F899D29A8"} + - {name: QWESLICSTORE_HMAC, size: "4KB", type-guid: "C1AA9678-A187-4FC4-A04E-EAFA96CA007E"} + - {name: QWESLICSTORE_BACKUP, size: "128KB", type-guid: "1A702852-77D3-40E6-BE75-884E0E601787"} + - {name: QWESLICSTORE_BACKUP_HMAC, size: "4KB", type-guid: "206E91D6-C375-4A82-8234-FD5FF6F46AA5"} + - {name: SOCCP_DATA, size: "4KB", type-guid: "b716b66d-3c3f-4c56-9602-31a1cf375cd7"} + - {name: SOCCP_DATA_HMAC, size: "4KB", type-guid: "3b6d8a27-8ed6-4036-aad4-6b33f5395b64"} + - {name: SOCCP_DATA_BACKUP, size: "4KB", type-guid: "ca6c26b9-229c-4baa-b50f-10bf04fef77c"} + - {name: SOCCP_DATA_BACKUP_HMAC, size: "4KB", type-guid: "f85b1281-9024-4899-89c6-1484fa60188d"} + - {name: DPP, size: "600KB", type-guid: "F97B8793-3ABF-4719-896B-7C3E9B85E104"} + - {name: DPP_HMAC, size: "8KB", type-guid: "EA672D1E-D692-476D-B58C-049F7552F166"} + - {name: DPP_BACKUP, size: "600KB", type-guid: "BBB2113E-27EF-47BD-9DF1-E6082380A928"} + - {name: DPP_BACKUP_HMAC, size: "8KB", type-guid: "AF97B2E7-B593-4FEE-A752-1D9257FE4111"} + - {name: SSD, size: "8KB", type-guid: "2C86E742-745E-4FDD-BFD8-B6A7AC638772"} + - {name: SSD_HMAC, size: "4KB", type-guid: "C958B529-DEF9-4FDD-AB17-8C73669C89ED"} + - {name: SSD_BACKUP, size: "8KB", type-guid: "AD8BB1AC-598A-4B71-A9AE-78AB515D2DCC"} + - {name: SSD_BACKUP_HMAC, size: "4KB", type-guid: "525600C9-9B85-4F1A-8842-648C7797B33B"} + - {name: SMBIOS, size: "16KB", type-guid: "04A856B8-84C1-4075-8391-9A994235E5F0"} + - {name: SMBIOS_HMAC, size: "4KB", type-guid: "F0D69946-3FAE-4A1A-8BDE-80BCD9F257A4"} + - {name: SMBIOS_BACKUP, size: "16KB", type-guid: "72D243D7-D540-48FA-AAB8-03D6E05CA358"} + - {name: SMBIOS_BACKUP_HMAC, size: "4KB", type-guid: "582A7680-0F07-40C5-B308-67F1B94B96DB"} + - {name: ddr, size: "240KB", type-guid: "20A0C19C-286A-42FA-9CE7-F64C3226A794"} + - {name: ddr_HMAC, size: "4KB", type-guid: "FFCD927B-74E1-4597-A863-D9AC803DF191"} + - {name: ddr_BACKUP, size: "240KB", type-guid: "78A54EF3-D7D7-4518-91D5-1E972C82B282"} + - {name: ddr_BACKUP_HMAC, size: "4KB", type-guid: "5CC13258-DE5A-479A-8ECA-6B349294A5E9"} + - {name: limits, size: "4KB", type-guid: "10A0C19C-516A-5444-5CE3-664C3226A794"} + - {name: limits_HMAC, size: "4KB", type-guid: "D9307477-9E76-44C2-9BFB-27151CB08C39"} + - {name: limits_BACKUP, size: "4KB", type-guid: "C8C25968-A2CD-4DC4-9E17-07FC0F0D99DA"} + - {name: limits_BACKUP_HMAC, size: "4KB", type-guid: "36E332BE-BE07-4190-AB98-75CAB2815179"} + - {name: SYSFW_VERSION, size: "4KB", type-guid: "3C44F88B-1878-4C29-B122-EE78766442A7"} + - {name: SYSFW_VERSION_HMAC, size: "4KB", type-guid: "DD7B74CE-A009-45CB-8A95-41863216B447"} + - {name: SYSFW_VERSION_BACKUP, size: "4KB", type-guid: "A94B037C-EBD2-477F-8BD2-3AF4A30FEDE7"} + - {name: SYSFW_VERSION_BACKUP_HMAC, size: "4KB", type-guid: "DE7930E6-FFA1-44EB-9F81-D896974FC2D2"} + - {name: SPU_NVM, size: "512KB", type-guid: "E42E2B4C-33B0-429B-B1EF-D341C547022C"} + - {name: Resiliency_Log, size: "16KB", type-guid: "3BB99F72-E524-4128-A815-7194CC190A3D"} + - {name: xbl_sc_test_mode, size: "4KB", type-guid: "91FDD2B9-8ED3-4176-BC42-260F2E34D04A"} + - {name: xbl_sc_logs, size: "80KB", type-guid: "F7EECB66-781A-439A-8955-70E12ED4A7A0"} + - {name: recoveryinfo, size: "4KB", type-guid: "7374B391-291C-49FA-ABC2-0463AB5F713F"} + - {name: resilience_driver, size: "8KB", type-guid: "4A2864F7-CB02-49F2-BA75-485E49C00966"} + - {name: RecoveryGPT, size: "48KB", type-guid: "452E8C3B-B67F-4C66-80D9-AD457F74CB0A"} + - {name: SECDATA, size: "28KB", type-guid: "76CFC7EF-039D-4E2C-B81E-4DD8C2CB2A93"} + - {name: ddr_debug, size: "2160KB", type-guid: "2D58205E-BA35-4BF9-B6C1-C6FDC80A373B"} + - {name: APDP, size: "64KB", type-guid: "E6E98DA2-E22A-4D12-AB33-169E7DEAA507"} + - {name: XBL_SC, size: "2520KB", type-guid: "DEA0BA2C-CBDD-4805-B4F9-F428251C3E98", filename: xbl_s.melf} + - {name: BOOT_FW1, size: "12288KB", type-guid: "CD6CDFAB-B3F7-46C6-BFFE-1A1D2B8B7BA0", filename: bootfw1.bin} + - {name: BOOT_FW2, size: "6200KB", type-guid: "FA99213C-D1C9-4EA7-8689-EBC1C3EFAB7E", filename: bootfw2.bin} + - {name: XBL_RAMDUMP, size: "512KB", type-guid: "0382F197-E41F-4E84-B18B-0B564AEAD875"} + - {name: TZAPPS, size: "768KB", type-guid: "14D11C40-2A3D-4F97-882D-103A1EC09333"} + - {name: MULTIIMGQTI, size: "32KB", type-guid: "846C6F05-EB46-4C0A-A1A3-3648EF3F9D0E"} + - {name: dtb_a, size: "4096KB", type-guid: "2A1A52FC-AA0B-401C-A808-5EA0F91068F8"} + - {name: APDP_BACKUP, size: "64KB", type-guid: "110F198D-8174-4193-9AF1-5DA94CDC59C9"} + - {name: XBL_SC_BACKUP, size: "2520KB", type-guid: "7A3DF1A3-A31A-454D-BD78-DF259ED486BE", filename: xbl_s.melf} + - {name: BOOT_FW1_BACKUP, size: "12288KB", type-guid: "08E98E12-0ACF-4D3F-B881-E7A2D87949DF", filename: bootfw1.bin} + - {name: BOOT_FW2_BACKUP, size: "6200KB", type-guid: "0A849567-3DDF-409C-A9BB-26BC8CF8D079", filename: bootfw2.bin} + - {name: QC_RESERVED, size: "64KB", type-guid: "5A62D5E4-2E26-4560-95DA-E86CDC7CC0D4"} + - {name: QC_RESERVED_BACKUP, size: "64KB", type-guid: "8CF0B012-6EF8-44D9-ADF6-3892140979DA"} + - {name: OEM_RESERVED, size: "2048KB", type-guid: "A9CE4DC1-9004-49A2-A8C7-A4C060B35D8E"} + - {name: XBL_RAMDUMP_BACKUP, size: "512KB", type-guid: "FF608BF6-AEDF-4084-BEC5-C92AB4E4534D"} + - {name: TZAPPS_BACKUP, size: "768KB", type-guid: "BE3719E5-48A7-4ABC-B494-304864D02148"} + - {name: MULTIIMGQTI_BACKUP, size: "32KB", type-guid: "D30C8B21-DDD9-45B6-8DE0-3165D34395C9"} + - {name: SPU_PROD_BACKUP, size: "1024KB", type-guid: "C759F596-F8FE-4A1A-851E-5BF0F291793E"} + - {name: dtb_b, size: "4096KB", type-guid: "A166F11A-2B39-4FAA-B7E7-F8AA080D0587"} diff --git a/platforms/glymur-crd/nvme/partitions.conf b/platforms/glymur-crd/nvme/partitions.conf deleted file mode 100644 index 88757d3..0000000 --- a/platforms/glymur-crd/nvme/partitions.conf +++ /dev/null @@ -1,24 +0,0 @@ -# Copyright (c) 2026 Qualcomm Innovation Center, Inc. All rights reserved. -# SPDX-License-Identifier: BSD-3-Clause-Clear - -# select disk type emmc | nand | nvme | ufs Mandatory -# disk size in bytes Mandatory -# options if not explicitly provide - ---disk --type=nvme --size=68719476736 --write-protect-boundary=65536 --sector-size-in-bytes=512 --grow-last-partition - -# per partition entry -# mandatory options: -# --lun (mandatory for UFS, emmc no need this) -# --name -# --size in bytes -# --type-guid -# optional options: (defaults used if not provided) -# --attributes 1000000000000004 -# --filename "" -# --readonly true -# --sparse false - -# This is physical partition 0 ---partition --name=efi --size=524288KB --type-guid=C12A7328-F81F-11D2-BA4B-00A0C93EC93B --filename=efi.bin ---partition --name=rootfs --size=33554432KB --type-guid=B921B045-1DF0-41C3-AF44-4C6F280D3FAE --filename=rootfs.img diff --git a/platforms/glymur-crd/spinor/partitions.conf b/platforms/glymur-crd/spinor/partitions.conf deleted file mode 100644 index 0d6f263..0000000 --- a/platforms/glymur-crd/spinor/partitions.conf +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (c) 2026 Qualcomm Innovation Center, Inc. All rights reserved. -# SPDX-License-Identifier: BSD-3-Clause-Clear -# select disk --type-guid emmc | nand | nvme | spinor | ufs Mandatory -# disk size in bytes Mandatory -# options if not explicitly provide -# ---disk --type=spinor --size=67108864 --write-protect-boundary=0 --sector-size-in-bytes=4096 -# -# per partition entry -# mandatory options: -# --lun (mandatory for UFS, emmc no need this) -# --name -# --size in bytes -# --type-guid -# optional options: (defaults used if not provided) -# --attributes 1000000000000004 -# --filename -# --readonly true -# --sparse false -# -# This is physical partition 0 ---partition --name=cdt --size=4KB --type-guid=A19F205F-CCD8-4B6D-8F1E-2D9BC24CFFB1 --filename=cdt.bin ---partition --name=SD_MGR --size=528KB --type-guid=5E463172-D0AC-4DD4-91B8-CD3EE1281579 ---partition --name=VarStore --size=728KB --type-guid=165BD6BC-9250-4AC8-95A7-A93F4A440066 ---partition --name=QWESLICCACHE --size=36KB --type-guid=7DDC813A-E88A-491A-95DC-4811A869D313 ---partition --name=QWESLICSTORE --size=128KB --type-guid=7BAB3C93-5F73-4D02-B8CB-5B9F899D29A8 ---partition --name=QWESLICSTORE_HMAC --size=4KB --type-guid=C1AA9678-A187-4FC4-A04E-EAFA96CA007E ---partition --name=QWESLICSTORE_BACKUP --size=128KB --type-guid=1A702852-77D3-40E6-BE75-884E0E601787 ---partition --name=QWESLICSTORE_BACKUP_HMAC --size=4KB --type-guid=206E91D6-C375-4A82-8234-FD5FF6F46AA5 ---partition --name=SOCCP_DATA --size=4KB --type-guid=b716b66d-3c3f-4c56-9602-31a1cf375cd7 ---partition --name=SOCCP_DATA_HMAC --size=4KB --type-guid=3b6d8a27-8ed6-4036-aad4-6b33f5395b64 ---partition --name=SOCCP_DATA_BACKUP --size=4KB --type-guid=ca6c26b9-229c-4baa-b50f-10bf04fef77c ---partition --name=SOCCP_DATA_BACKUP_HMAC --size=4KB --type-guid=f85b1281-9024-4899-89c6-1484fa60188d ---partition --name=DPP --size=600KB --type-guid=F97B8793-3ABF-4719-896B-7C3E9B85E104 ---partition --name=DPP_HMAC --size=8KB --type-guid=EA672D1E-D692-476D-B58C-049F7552F166 ---partition --name=DPP_BACKUP --size=600KB --type-guid=BBB2113E-27EF-47BD-9DF1-E6082380A928 ---partition --name=DPP_BACKUP_HMAC --size=8KB --type-guid=AF97B2E7-B593-4FEE-A752-1D9257FE4111 ---partition --name=SSD --size=8KB --type-guid=2C86E742-745E-4FDD-BFD8-B6A7AC638772 ---partition --name=SSD_HMAC --size=4KB --type-guid=C958B529-DEF9-4FDD-AB17-8C73669C89ED ---partition --name=SSD_BACKUP --size=8KB --type-guid=AD8BB1AC-598A-4B71-A9AE-78AB515D2DCC ---partition --name=SSD_BACKUP_HMAC --size=4KB --type-guid=525600C9-9B85-4F1A-8842-648C7797B33B ---partition --name=SMBIOS --size=16KB --type-guid=04A856B8-84C1-4075-8391-9A994235E5F0 ---partition --name=SMBIOS_HMAC --size=4KB --type-guid=F0D69946-3FAE-4A1A-8BDE-80BCD9F257A4 ---partition --name=SMBIOS_BACKUP --size=16KB --type-guid=72D243D7-D540-48FA-AAB8-03D6E05CA358 ---partition --name=SMBIOS_BACKUP_HMAC --size=4KB --type-guid=582A7680-0F07-40C5-B308-67F1B94B96DB ---partition --name=ddr --size=240KB --type-guid=20A0C19C-286A-42FA-9CE7-F64C3226A794 ---partition --name=ddr_HMAC --size=4KB --type-guid=FFCD927B-74E1-4597-A863-D9AC803DF191 ---partition --name=ddr_BACKUP --size=240KB --type-guid=78A54EF3-D7D7-4518-91D5-1E972C82B282 ---partition --name=ddr_BACKUP_HMAC --size=4KB --type-guid=5CC13258-DE5A-479A-8ECA-6B349294A5E9 ---partition --name=limits --size=4KB --type-guid=10A0C19C-516A-5444-5CE3-664C3226A794 ---partition --name=limits_HMAC --size=4KB --type-guid=D9307477-9E76-44C2-9BFB-27151CB08C39 ---partition --name=limits_BACKUP --size=4KB --type-guid=C8C25968-A2CD-4DC4-9E17-07FC0F0D99DA ---partition --name=limits_BACKUP_HMAC --size=4KB --type-guid=36E332BE-BE07-4190-AB98-75CAB2815179 ---partition --name=SYSFW_VERSION --size=4KB --type-guid=3C44F88B-1878-4C29-B122-EE78766442A7 ---partition --name=SYSFW_VERSION_HMAC --size=4KB --type-guid=DD7B74CE-A009-45CB-8A95-41863216B447 ---partition --name=SYSFW_VERSION_BACKUP --size=4KB --type-guid=A94B037C-EBD2-477F-8BD2-3AF4A30FEDE7 ---partition --name=SYSFW_VERSION_BACKUP_HMAC --size=4KB --type-guid=DE7930E6-FFA1-44EB-9F81-D896974FC2D2 ---partition --name=SPU_NVM --size=512KB --type-guid=E42E2B4C-33B0-429B-B1EF-D341C547022C ---partition --name=Resiliency_Log --size=16KB --type-guid=3BB99F72-E524-4128-A815-7194CC190A3D ---partition --name=xbl_sc_test_mode --size=4KB --type-guid=91FDD2B9-8ED3-4176-BC42-260F2E34D04A ---partition --name=xbl_sc_logs --size=80KB --type-guid=F7EECB66-781A-439A-8955-70E12ED4A7A0 ---partition --name=recoveryinfo --size=4KB --type-guid=7374B391-291C-49FA-ABC2-0463AB5F713F ---partition --name=resilience_driver --size=8KB --type-guid=4A2864F7-CB02-49F2-BA75-485E49C00966 ---partition --name=RecoveryGPT --size=48KB --type-guid=452E8C3B-B67F-4C66-80D9-AD457F74CB0A ---partition --name=SECDATA --size=28KB --type-guid=76CFC7EF-039D-4E2C-B81E-4DD8C2CB2A93 ---partition --name=ddr_debug --size=2160KB --type-guid=2D58205E-BA35-4BF9-B6C1-C6FDC80A373B ---partition --name=APDP --size=64KB --type-guid=E6E98DA2-E22A-4D12-AB33-169E7DEAA507 ---partition --name=XBL_SC --size=2520KB --type-guid=DEA0BA2C-CBDD-4805-B4F9-F428251C3E98 --filename=xbl_s.melf ---partition --name=BOOT_FW1 --size=12288KB --type-guid=CD6CDFAB-B3F7-46C6-BFFE-1A1D2B8B7BA0 --filename=bootfw1.bin ---partition --name=BOOT_FW2 --size=6200KB --type-guid=FA99213C-D1C9-4EA7-8689-EBC1C3EFAB7E --filename=bootfw2.bin ---partition --name=XBL_RAMDUMP --size=512KB --type-guid=0382F197-E41F-4E84-B18B-0B564AEAD875 ---partition --name=TZAPPS --size=768KB --type-guid=14D11C40-2A3D-4F97-882D-103A1EC09333 ---partition --name=MULTIIMGQTI --size=32KB --type-guid=846C6F05-EB46-4C0A-A1A3-3648EF3F9D0E ---partition --name=dtb_a --size=4096KB --type-guid=2A1A52FC-AA0B-401C-A808-5EA0F91068F8 ---partition --name=APDP_BACKUP --size=64KB --type-guid=110F198D-8174-4193-9AF1-5DA94CDC59C9 ---partition --name=XBL_SC_BACKUP --size=2520KB --type-guid=7A3DF1A3-A31A-454D-BD78-DF259ED486BE --filename=xbl_s.melf ---partition --name=BOOT_FW1_BACKUP --size=12288KB --type-guid=08E98E12-0ACF-4D3F-B881-E7A2D87949DF --filename=bootfw1.bin ---partition --name=BOOT_FW2_BACKUP --size=6200KB --type-guid=0A849567-3DDF-409C-A9BB-26BC8CF8D079 --filename=bootfw2.bin ---partition --name=QC_RESERVED --size=64KB --type-guid=5A62D5E4-2E26-4560-95DA-E86CDC7CC0D4 ---partition --name=QC_RESERVED_BACKUP --size=64KB --type-guid=8CF0B012-6EF8-44D9-ADF6-3892140979DA ---partition --name=OEM_RESERVED --size=2048KB --type-guid=A9CE4DC1-9004-49A2-A8C7-A4C060B35D8E ---partition --name=XBL_RAMDUMP_BACKUP --size=512KB --type-guid=FF608BF6-AEDF-4084-BEC5-C92AB4E4534D ---partition --name=TZAPPS_BACKUP --size=768KB --type-guid=BE3719E5-48A7-4ABC-B494-304864D02148 ---partition --name=MULTIIMGQTI_BACKUP --size=32KB --type-guid=D30C8B21-DDD9-45B6-8DE0-3165D34395C9 ---partition --name=SPU_PROD_BACKUP --size=1024KB --type-guid=C759F596-F8FE-4A1A-851E-5BF0F291793E ---partition --name=dtb_b --size=4096KB --type-guid=A166F11A-2B39-4FAA-B7E7-F8AA080D0587 diff --git a/platforms/variants/hlos/qcom-deb-images.yaml b/platforms/variants/hlos/qcom-deb-images.yaml new file mode 100644 index 0000000..3e426ff --- /dev/null +++ b/platforms/variants/hlos/qcom-deb-images.yaml @@ -0,0 +1,10 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# +# Debian HLOS variant (qcom-deb-images): 32 GiB rootfs, stock filename. +storage: + - id: nvme0 + partitions: + - name: rootfs + size: "33554432KB" + filename: rootfs.img diff --git a/pyproject.toml b/pyproject.toml index 82b97cd..5b768c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,6 +9,10 @@ description = "Qualcomm partition tool - GPT partition table generator and mass readme = "README.md" license = "BSD-3-Clause" requires-python = ">=3.8" +dependencies = [ + "PyYAML>=5.1", + "jsonschema>=3.2", +] [project.scripts] qcom-ptool = "qcom_ptool.cli:main" @@ -16,6 +20,9 @@ qcom-ptool = "qcom_ptool.cli:main" [tool.setuptools.packages.find] include = ["qcom_ptool*"] +[tool.setuptools.package-data] +"qcom_ptool.schema" = ["*.json"] + [tool.ruff] target-version = "py38" line-length = 120 @@ -53,3 +60,10 @@ ignore_missing_imports = true module = ["qcom_ptool.msp", "qcom_ptool.ptool"] # Legacy scripts - enable gradually check_untyped_defs = false + +[tool.pytest.ini_options] +# Allow running `pytest` from the repo root without requiring an editable +# install of the package; tests under tests/unit/ import qcom_ptool directly +# from the source tree. +pythonpath = ["."] +testpaths = ["tests/unit"] diff --git a/qcom_ptool/board.py b/qcom_ptool/board.py new file mode 100644 index 0000000..dce0686 --- /dev/null +++ b/qcom_ptool/board.py @@ -0,0 +1,298 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +""" +Multi-storage board model and composition resolver. + +A board file declares a ``platform`` and a list of ``storage`` devices, each +with its own partitions. Three composition mechanisms layer on top: + +* board-level ``extends:`` - inherit a single base board and override by + stable identifier (storage ``id``, partition ``name``); +* storage-level ``includes:`` - concatenate shared partition fragments from + ``_common/`` into a storage's partition list; +* variant overlays (``--hlos`` / ``--boot-fw``) - applied last, merged by the + same identifiers. + +Resolution order is: base board -> derived board -> storage includes -> +variant overlays. Merges are deep and field-by-field; entries are matched by +``id`` (storage) or ``name`` (partition), and anything unmatched is appended +in file order. There is no partition deletion in v1. + +The resolver operates on raw YAML dicts. Each fully-resolved storage is then +validated against the strict single-storage ``partitions.schema.json`` and +reduced to a :class:`qcom_ptool.spec.LoadedSpec` through the very same +normalisation helpers the YAML loader uses, so a resolved board emits XML +byte-identical to an equivalent hand-written single-storage file. +""" + +from __future__ import annotations + +import copy +import json +import os +from collections.abc import Mapping, Sequence +from importlib import resources +from typing import Any, TypedDict + +import jsonschema +import yaml # type: ignore[import-untyped] + +from qcom_ptool.loaders.yaml import _disk_from_node, _partition_from_node +from qcom_ptool.spec import LoadedSpec, PartitionsByLun + +_SCHEMA_PACKAGE = "qcom_ptool.schema" + +# Storage keys that describe the disk geometry (everything except id / includes +# / partitions). Used to reduce a resolved storage into the single-storage +# document shape that partitions.schema.json validates. +_DISK_KEYS = ( + "type", + "size", + "sector-size", + "write-protect-boundary", + "grow-last-partition", + "align-partitions", +) + + +class BoardResolveError(ValueError): + """Raised when a board file cannot be loaded, composed or validated.""" + + +class ResolvedBoard(TypedDict): + """A board with every extends/includes/variant applied.""" + + platform: dict[str, Any] + storage: list[dict[str, Any]] + + +# --------------------------------------------------------------------------- +# Schema / file helpers +# --------------------------------------------------------------------------- + + +def _load_schema(name: str) -> dict[str, Any]: + text = resources.files(_SCHEMA_PACKAGE).joinpath(name).read_text(encoding="utf-8") + schema: dict[str, Any] = json.loads(text) + return schema + + +def _read_mapping(path: str, schema: str) -> dict[str, Any]: + """Load ``path`` as YAML, validate against ``schema`` and return a copy.""" + try: + with open(path) as f: + document = yaml.safe_load(f) + except OSError as exc: + raise BoardResolveError("cannot read %s: %s" % (path, exc)) from exc + if not isinstance(document, Mapping): + raise BoardResolveError( + "%s: expected a top-level mapping, got %s" % (path, type(document).__name__) + ) + try: + jsonschema.validate(document, _load_schema(schema)) + except jsonschema.exceptions.ValidationError as exc: + raise BoardResolveError("%s: schema validation failed: %s" % (path, exc.message)) from exc + return copy.deepcopy(dict(document)) + + +# --------------------------------------------------------------------------- +# Deep-merge primitives (by stable identifier) +# --------------------------------------------------------------------------- + + +def _merge_partitions(base: list[dict[str, Any]], overlay: Sequence[Mapping[str, Any]]) -> None: + """Merge ``overlay`` partitions into ``base`` by ``name``. + + A same-named partition is deep-merged field-by-field in place (so an + override can set only ``size`` and inherit the rest); a new partition is + appended in overlay order. Nothing is ever moved or removed. + """ + index = {p["name"]: p for p in base if "name" in p} + for entry in overlay: + name = entry.get("name") + target = index.get(name) + if target is not None: + target.update(copy.deepcopy(dict(entry))) + else: + new_entry = copy.deepcopy(dict(entry)) + base.append(new_entry) + if name is not None: + index[name] = new_entry + + +def _merge_storage(base: dict[str, Any], overlay: Mapping[str, Any]) -> None: + """Merge one storage ``overlay`` into ``base`` (matched by id upstream).""" + for key, value in overlay.items(): + if key == "id": + continue + if key == "partitions": + base.setdefault("partitions", []) + _merge_partitions(base["partitions"], value) + elif key == "includes": + merged = base.setdefault("includes", []) + for inc in value: + if inc not in merged: + merged.append(inc) + else: + base[key] = copy.deepcopy(value) + + +def _merge_storage_list(base: list[dict[str, Any]], overlay: Sequence[Mapping[str, Any]]) -> None: + """Merge ``overlay`` storage entries into ``base`` by ``id``.""" + index = {s["id"]: s for s in base if "id" in s} + for entry in overlay: + sid = entry.get("id") + target = index.get(sid) + if target is not None: + _merge_storage(target, entry) + else: + new_entry = copy.deepcopy(dict(entry)) + base.append(new_entry) + if sid is not None: + index[sid] = new_entry + + +# --------------------------------------------------------------------------- +# Board-level extends +# --------------------------------------------------------------------------- + + +def _load_board_with_extends(path: str, root: str, chain: tuple[str, ...]) -> dict[str, Any]: + """Load a board file and fold its single base (if any) underneath it.""" + real = os.path.realpath(path) + if real in chain: + raise BoardResolveError("extends cycle detected at %s" % path) + board = _read_mapping(path, "board.schema.json") + base_ref = board.pop("extends", None) + if base_ref is None: + return board + base_path = os.path.join(root, base_ref) + base = _load_board_with_extends(base_path, root, (*chain, real)) + _merge_storage_list(base.setdefault("storage", []), board.get("storage", [])) + if "platform" in board: + base["platform"] = board["platform"] + return base + + +# --------------------------------------------------------------------------- +# Storage-level includes +# --------------------------------------------------------------------------- + + +def _fragment_partitions(path: str, root: str, chain: tuple[str, ...]) -> list[dict[str, Any]]: + """Resolve a ``_common/`` fragment (transitively) into a partition list.""" + real = os.path.realpath(path) + if real in chain: + raise BoardResolveError("includes cycle detected at %s" % path) + fragment = _read_mapping(path, "include.schema.json") + partitions: list[dict[str, Any]] = [] + for inc in fragment.get("includes", []): + inc_parts = _fragment_partitions(os.path.join(root, inc), root, (*chain, real)) + _merge_partitions(partitions, inc_parts) + _merge_partitions(partitions, fragment.get("partitions", [])) + return partitions + + +def _expand_includes(storage: dict[str, Any], root: str) -> None: + """Replace a storage's ``includes:`` + inline ``partitions:`` with one list. + + Ordering: included fragments in listed order first, then the storage's own + inline partitions, with same-named entries merged in place. + """ + includes = storage.pop("includes", []) + expanded: list[dict[str, Any]] = [] + for inc in includes: + inc_parts = _fragment_partitions(os.path.join(root, inc), root, ()) + _merge_partitions(expanded, inc_parts) + _merge_partitions(expanded, storage.get("partitions", [])) + storage["partitions"] = expanded + + +# --------------------------------------------------------------------------- +# Public resolution +# --------------------------------------------------------------------------- + + +def _variant_path(root: str, axis: str, name: str) -> str: + return os.path.join(root, "variants", axis, name + ".yaml") + + +def _validate_resolved_storage(storage: Mapping[str, Any]) -> None: + """Validate a fully-resolved storage against the strict single-storage schema.""" + disk = {k: storage[k] for k in _DISK_KEYS if k in storage} + document = {"disk": disk, "partitions": storage.get("partitions", [])} + try: + jsonschema.validate(document, _load_schema("partitions.schema.json")) + except jsonschema.exceptions.ValidationError as exc: + raise BoardResolveError( + "storage %r resolved to an invalid spec: %s" + % (storage.get("id", "?"), exc.message) + ) from exc + + +def resolve_board( + path: str, + root: str | None = None, + hlos: str | None = None, + boot_fw: str | None = None, +) -> ResolvedBoard: + """Resolve ``path`` into a fully-composed :class:`ResolvedBoard`. + + ``root`` is the platforms directory that ``extends:`` / ``includes:`` paths + and variant names resolve against; it defaults to the parent of the board + file's directory when that directory is ``boards/``, else the board file's + own directory. + """ + if root is None: + parent = os.path.dirname(os.path.abspath(path)) + root = os.path.dirname(parent) if os.path.basename(parent) == "boards" else parent + + board = _load_board_with_extends(path, root, ()) + storages = board.get("storage", []) + if not storages: + raise BoardResolveError("%s: board declares no storage" % path) + + for storage in storages: + _expand_includes(storage, root) + + for axis, name in (("hlos", hlos), ("boot-fw", boot_fw)): + if name is None: + continue + overlay = _read_mapping(_variant_path(root, axis, name), "board.schema.json") + _merge_storage_list(storages, overlay.get("storage", [])) + + for storage in storages: + _validate_resolved_storage(storage) + + return {"platform": board.get("platform", {}), "storage": storages} + + +# --------------------------------------------------------------------------- +# Reduction to the emitter's LoadedSpec +# --------------------------------------------------------------------------- + + +def storage_to_loaded_spec( + storage: Mapping[str, Any], + image_map: Mapping[str, str] | None = None, +) -> LoadedSpec: + """Reduce one resolved storage to a LoadedSpec via the YAML loader helpers.""" + if image_map is None: + image_map = {} + disk = _disk_from_node(storage) + partitions: PartitionsByLun = {} + for node in storage.get("partitions", []): + phys_part, entry = _partition_from_node(node, image_map) + partitions.setdefault(phys_part, []).append(entry) + return {"disk": disk, "partitions": partitions} + + +def board_to_specs( + board: ResolvedBoard, + image_map: Mapping[str, str] | None = None, +) -> list[tuple[str, LoadedSpec]]: + """Reduce a resolved board to ``[(storage_id, LoadedSpec), ...]`` in order.""" + return [ + (storage.get("id", ""), storage_to_loaded_spec(storage, image_map)) + for storage in board["storage"] + ] diff --git a/qcom_ptool/cli.py b/qcom_ptool/cli.py index bb011d4..f319656 100644 --- a/qcom_ptool/cli.py +++ b/qcom_ptool/cli.py @@ -12,6 +12,7 @@ "gen_contents": "qcom_ptool.gen_contents", "ptool": "qcom_ptool.ptool", "msp": "qcom_ptool.msp", + "show": "qcom_ptool.show", } diff --git a/qcom_ptool/gen_partition.py b/qcom_ptool/gen_partition.py index b6449d0..051d17e 100755 --- a/qcom_ptool/gen_partition.py +++ b/qcom_ptool/gen_partition.py @@ -27,176 +27,34 @@ # OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN # IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +from __future__ import annotations + import getopt -import re import sys import xml.etree.ElementTree as ET -from collections import OrderedDict from typing import NoReturn from xml.dom import minidom +from qcom_ptool.loaders import load as load_spec +from qcom_ptool.spec import DiskParams, PartitionsByLun + def usage() -> NoReturn: print( - "\n\tUsage: %s -i -o -m [partition_name1=image_filename1,partition_name2=image_filename2,...]\n\tVersion 1.0\n" - % (sys.argv[0]) + "\n\tUsage:\n" + "\t %s -i -o " + "[-m name1=image1,name2=image2,...]\n" + "\t %s --board [--hlos ] [--boot-fw ] " + "[--root ] -o [-o ...]\n" + "\n\tIn --board mode one -o is given per storage, in declared order.\n" + "\tVersion 1.0\n" % (sys.argv[0], sys.argv[0]) ) sys.exit(1) -################################################################## -# defaults to be used -disk_params_defaults = OrderedDict( - { - "type": "", - "size": "", - "SECTOR_SIZE_IN_BYTES": "512", - "WRITE_PROTECT_BOUNDARY_IN_KB": "65536", - "GROW_LAST_PARTITION_TO_FILL_DISK": "false", - "ALIGN_PARTITIONS_TO_PERFORMANCE_BOUNDARY": "true", - "PERFORMANCE_BOUNDARY_IN_KB": "4", - } -) - -partition_entry_defaults = { - "label": "", - "size_in_kb": "", - "type": "00000000-0000-0000-0000-000000000000", - "bootable": "false", - "readonly": "true", - "filename": "", - "sparse": "false", -} - -################################################################## -# store entries read from input file -disk_entry = None -partition_entries = [] -# store partition image map passed from command line -partition_image_map: dict[str, str] = {} -input_file = None -output_xml = None - - -def disk_options(argv): - disk_params = disk_params_defaults.copy() - for opt, arg in argv: - if opt in ["--type"]: - disk_params["type"] = arg - elif opt in ["--size"]: - disk_params["size"] = arg - elif opt in ["--sector-size-in-bytes"]: - disk_params["SECTOR_SIZE_IN_BYTES"] = arg - elif opt in ["--write-protect-boundary"]: - disk_params["WRITE_PROTECT_BOUNDARY_IN_KB"] = arg - elif opt in ["--grow-last-partition"]: - disk_params["GROW_LAST_PARTITION_TO_FILL_DISK"] = "true" - elif opt in ["--align-partitions"]: - disk_params["ALIGN_PARTITIONS_TO_PERFORMANCE_BOUNDARY"] = "true" - disk_params["PERFORMANCE_BOUNDARY_IN_KB"] = str(int(arg) // 1024) - return disk_params - - -def partition_size_in_kb(size): - if not re.search("[a-zA-Z]+", size): - return int(size) // 1024 - m = re.search("([0-9]+)(?=[Kk][Bb]?)", size) - if m: - return int(m.group(0)) - m = re.search("([0-9]+)(?=[Mm][Bb]?)", size) - if m: - return int(m.group(0)) * 1024 - m = re.search("([0-9]+)(?=[Gg][Bb]?)", size) - if m: - return int(m.group(0)) * 1024 * 1024 - raise ValueError("Unrecognized size format: '%s'" % size) - - -def partition_options(argv): - partition_entry = partition_entry_defaults.copy() - phys_part = 0 - for opt, arg in argv: - if opt in ["--lun", "--phys-part"]: - phys_part = arg - elif opt in ["--name"]: - partition_entry["label"] = arg - elif opt in ["--size"]: - kbytes = partition_size_in_kb(arg) - partition_entry["size_in_kb"] = str(kbytes) - elif opt in ["--type-guid"]: - partition_entry["type"] = arg - elif opt in ["--attributes"]: - attribute_bits = int(arg, 16) - if attribute_bits & (1 << 2): - partition_entry["bootable"] = "true" - else: - partition_entry["bootable"] = "false" - if attribute_bits & (1 << 60): - partition_entry["readonly"] = "true" - else: - partition_entry["readonly"] = "false" - elif opt in ["--filename"]: - partition_entry["filename"] = arg - elif opt in ["--sparse"]: - partition_entry["sparse"] = arg - if partition_entry["label"] in partition_image_map: - partition_entry["filename"] = partition_image_map[partition_entry["label"]] - return phys_part, partition_entry - - -def parse_partition_entries(partition_entries): - partitions_params: dict[int, list[dict[str, str]]] = {} - - for partition_entry in partition_entries: - opts_list = list(partition_entry.split(" ")) - if opts_list[0] == "--partition": - try: - options, _remainders = getopt.gnu_getopt( - opts_list[1:], - "", - [ - "lun=", - "phys-part=", - "name=", - "size=", - "type-guid=", - "filename=", - "attributes=", - "sparse=", - ], - ) - phys_part, partition = partition_options(options) - partitions_params.setdefault(phys_part, []).append(partition) - except Exception as e: - print(str(e)) - usage() - - return partitions_params - - -def parse_disk_entry(disk_entry): - opts_list = list(disk_entry.split(" ")) - if opts_list[0] == "--disk": - try: - options, _remainders = getopt.gnu_getopt( - opts_list[1:], - "", - [ - "type=", - "size=", - "sector-size-in-bytes=", - "write-protect-boundary=", - "grow-last-partition", - "align-partitions=", - ], - ) - return disk_options(options) - except Exception as e: - print(str(e)) - usage() - - -def generate_multi_lun_xml(disk_params, partitions, output_xml): +def generate_multi_lun_xml( + disk_params: DiskParams, partitions: PartitionsByLun, output_xml: str +) -> None: root = ET.Element("configuration") parser_instruction_text = "" @@ -222,7 +80,9 @@ def generate_multi_lun_xml(disk_params, partitions, output_xml): f.write(xmlstr) -def generate_partition_xml(disk_params, partitions, output_xml): +def generate_partition_xml( + disk_params: DiskParams, partitions: PartitionsByLun, output_xml: str +) -> None: print("Generating %s XML %s" % (disk_params["type"].upper(), output_xml)) if disk_params["type"] in ("emmc", "nvme", "spinor", "ufs"): @@ -236,58 +96,106 @@ def generate_partition_xml(disk_params, partitions, output_xml): ############################################################################### # main -disk_entry_err_msg = "contains more than one --disk entries" -if len(sys.argv) < 3: - usage() -try: - if sys.argv[1] == "-h" or sys.argv[1] == "--help": +def main(argv: list[str] | None = None) -> int: + if argv is None: + argv = sys.argv + + if len(argv) < 3: + usage() + + input_file: str | None = None + board_file: str | None = None + root: str | None = None + hlos: str | None = None + boot_fw: str | None = None + outputs: list[str] = [] + image_map: dict[str, str] = {} + + if argv[1] == "-h" or argv[1] == "--help": usage() try: - opts, rem = getopt.getopt(sys.argv[1:], "i:o:m:") + opts, _rem = getopt.getopt( + argv[1:], "i:o:m:", ["board=", "hlos=", "boot-fw=", "root="] + ) for opt, arg in opts: - if opt in ["-i"]: + if opt == "-i": input_file = arg - elif opt in ["-o"]: - output_xml = arg - elif opt in ["-m"]: + elif opt == "-o": + outputs.append(arg) + elif opt == "--board": + board_file = arg + elif opt == "--hlos": + hlos = arg + elif opt == "--boot-fw": + boot_fw = arg + elif opt == "--root": + root = arg + elif opt == "-m": for mapping in arg.split(","): tags = mapping.split("=") if len(tags) > 1: - partition_image_map[tags[0]] = tags[1] + image_map[tags[0]] = tags[1] else: usage() - except Exception as argerr: print(str(argerr)) usage() - if input_file is None or output_xml is None: + + if (board_file is None) == (input_file is None): + print("Error: pass exactly one of -i or --board ") usage() - f = open(input_file) - line = f.readline() - while line: - if not re.search(r"^\s*#", line) and not re.search(r"^\s*$", line): - line = line.strip() - if re.search("^--disk", line): - if disk_entry is None: - disk_entry = line - else: - print("%s %s" % (sys.argv[1], disk_entry_err_msg)) - print("%s\n%s" % (disk_entry, line)) - sys.exit(1) - elif re.search("^--partition", line): - partition_entries.append(line) - else: - print("Ignoring %s" % (line)) - line = f.readline() - f.close() -except Exception as e: - print("Error: ", e) - sys.exit(1) + if not outputs: + usage() + + if board_file is not None: + return _run_board(board_file, root, hlos, boot_fw, outputs, image_map) + + if input_file is None or len(outputs) != 1: + print("Error: -i single-storage mode takes exactly one -o") + usage() + try: + spec = load_spec(input_file, image_map=image_map) + except Exception as e: + print("Error: ", e) + return 1 + + generate_partition_xml(spec["disk"], spec["partitions"], outputs[0]) + return 0 + + +def _run_board( + board_file: str, + root: str | None, + hlos: str | None, + boot_fw: str | None, + outputs: list[str], + image_map: dict[str, str], +) -> int: + # Late import so the -i path never pulls in PyYAML / jsonschema. + from qcom_ptool.board import board_to_specs, resolve_board + + try: + board = resolve_board(board_file, root=root, hlos=hlos, boot_fw=boot_fw) + specs = board_to_specs(board, image_map=image_map) + except Exception as e: + print("Error: ", e) + return 1 + + if len(outputs) != len(specs): + ids = ", ".join(sid for sid, _ in specs) + print( + "Error: board declares %d storage(s) [%s] but %d -o output(s) given" + % (len(specs), ids, len(outputs)) + ) + return 1 + + for (sid, spec), output in zip(specs, outputs): + print("Storage %s -> %s" % (sid, output)) + generate_partition_xml(spec["disk"], spec["partitions"], output) + return 0 -disk_params = parse_disk_entry(disk_entry) -partitions = parse_partition_entries(partition_entries) -generate_partition_xml(disk_params, partitions, output_xml) -sys.exit(0) +if __name__ == "__main__": + sys.exit(main()) diff --git a/qcom_ptool/loaders/__init__.py b/qcom_ptool/loaders/__init__.py new file mode 100644 index 0000000..a372170 --- /dev/null +++ b/qcom_ptool/loaders/__init__.py @@ -0,0 +1,45 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +""" +Source-format loaders for partition specs. + +Each loader exposes ``load(path, image_map=None) -> LoadedSpec`` so the rest +of the package never has to care which on-disk format produced the +:mod:`qcom_ptool.spec` representation it operates on. Adding a new format +(e.g. YAML) means dropping a new module here and registering its suffix in +the dispatcher below — no other module needs to change. +""" + +from __future__ import annotations + +import os +from collections.abc import Mapping + +from qcom_ptool.spec import LoadedSpec + + +class UnsupportedFormatError(ValueError): + """Raised when no loader is registered for a given path's extension.""" + + +def load(path: str, image_map: Mapping[str, str] | None = None) -> LoadedSpec: + """Dispatch to the appropriate loader based on file extension. + + ``image_map`` overrides the ``--filename`` (or equivalent) for partitions + whose name appears as a key. It's applied uniformly by every loader so + the CLI ``-m`` flag works the same regardless of source format. + """ + suffix = os.path.splitext(path)[1].lower() + if suffix in ("", ".conf"): + # Late import: keeps this dispatcher dependency-free until a format + # is actually requested, which matters once optional formats (YAML) + # land with their own third-party imports. + from qcom_ptool.loaders import conf + + return conf.load(path, image_map=image_map) + if suffix in (".yaml", ".yml"): + # Late import so the .conf path never pays for PyYAML / jsonschema. + from qcom_ptool.loaders import yaml as yaml_loader + + return yaml_loader.load(path, image_map=image_map) + raise UnsupportedFormatError(f"No loader registered for suffix {suffix!r}") diff --git a/qcom_ptool/loaders/conf.py b/qcom_ptool/loaders/conf.py new file mode 100644 index 0000000..9c905e1 --- /dev/null +++ b/qcom_ptool/loaders/conf.py @@ -0,0 +1,219 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +""" +Loader for the legacy ``--disk`` / ``--partition`` line-format files under +``platforms/``. + +A ``.conf`` file is an unstructured stream of comment lines, blank lines, +exactly one ``--disk`` line, and any number of ``--partition`` lines. Each +significant line is itself a getopt-style option list. This module parses +both layers and returns a :class:`qcom_ptool.spec.LoadedSpec`. + +The parsing helpers are kept individually addressable so the unit tests +under ``tests/unit/`` can characterise each layer in isolation. +""" + +from __future__ import annotations + +import getopt +import re +from collections.abc import Mapping + +from qcom_ptool.spec import ( + DISK_PARAMS_DEFAULTS, + PARTITION_ENTRY_DEFAULTS, + DiskParams, + LoadedSpec, + PartitionEntry, + PartitionsByLun, +) + + +class ConfParseError(ValueError): + """Raised when a ``.conf`` file cannot be parsed.""" + + +# --------------------------------------------------------------------------- +# Size string parsing +# --------------------------------------------------------------------------- + + +def partition_size_in_kb(size: str) -> int: + """Convert a size string ("1024", "1KB", "2MB", "1GB") to KB. + + Bare integers are interpreted as bytes and divided by 1024. Strings with a + K/M/G suffix (any case, with optional 'B') return the kilobyte equivalent. + Anything else raises ``ValueError``; callers may catch and reformat. + """ + if not re.search("[a-zA-Z]+", size): + return int(size) // 1024 + m = re.search("([0-9]+)(?=[Kk][Bb]?)", size) + if m: + return int(m.group(0)) + m = re.search("([0-9]+)(?=[Mm][Bb]?)", size) + if m: + return int(m.group(0)) * 1024 + m = re.search("([0-9]+)(?=[Gg][Bb]?)", size) + if m: + return int(m.group(0)) * 1024 * 1024 + raise ValueError("Unrecognized size format: '%s'" % size) + + +# --------------------------------------------------------------------------- +# Option-list -> normalised dict +# --------------------------------------------------------------------------- + + +def disk_options(argv: list[tuple[str, str]]) -> DiskParams: + """Translate parsed ``--disk`` options into the canonical disk dict.""" + disk = DISK_PARAMS_DEFAULTS.copy() + for opt, arg in argv: + if opt == "--type": + disk["type"] = arg + elif opt == "--size": + disk["size"] = arg + elif opt == "--sector-size-in-bytes": + disk["SECTOR_SIZE_IN_BYTES"] = arg + elif opt == "--write-protect-boundary": + disk["WRITE_PROTECT_BOUNDARY_IN_KB"] = arg + elif opt == "--grow-last-partition": + disk["GROW_LAST_PARTITION_TO_FILL_DISK"] = "true" + elif opt == "--align-partitions": + disk["ALIGN_PARTITIONS_TO_PERFORMANCE_BOUNDARY"] = "true" + disk["PERFORMANCE_BOUNDARY_IN_KB"] = str(int(arg) // 1024) + return disk + + +def partition_options( + argv: list[tuple[str, str]], + image_map: Mapping[str, str] | None = None, +) -> tuple[str, PartitionEntry]: + """Translate parsed ``--partition`` options into ``(phys_part, entry)``. + + ``image_map`` (if provided) overrides the entry's ``filename`` when the + partition name matches a key. The override is applied after all options + have been processed, so it always wins over an explicit ``--filename``. + """ + if image_map is None: + image_map = {} + entry: PartitionEntry = PARTITION_ENTRY_DEFAULTS.copy() + phys_part: str = "0" + for opt, arg in argv: + if opt in ("--lun", "--phys-part"): + phys_part = arg + elif opt == "--name": + entry["label"] = arg + elif opt == "--size": + entry["size_in_kb"] = str(partition_size_in_kb(arg)) + elif opt == "--type-guid": + entry["type"] = arg + elif opt == "--attributes": + attribute_bits = int(arg, 16) + entry["bootable"] = "true" if attribute_bits & (1 << 2) else "false" + entry["readonly"] = "true" if attribute_bits & (1 << 60) else "false" + elif opt == "--filename": + entry["filename"] = arg + elif opt == "--sparse": + entry["sparse"] = arg + if entry["label"] in image_map: + entry["filename"] = image_map[entry["label"]] + return phys_part, entry + + +# --------------------------------------------------------------------------- +# Line-level parsing +# --------------------------------------------------------------------------- + + +_DISK_LONG_OPTS = [ + "type=", + "size=", + "sector-size-in-bytes=", + "write-protect-boundary=", + "grow-last-partition", + "align-partitions=", +] + +_PARTITION_LONG_OPTS = [ + "lun=", + "phys-part=", + "name=", + "size=", + "type-guid=", + "filename=", + "attributes=", + "sparse=", +] + + +def parse_disk_line(line: str) -> DiskParams | None: + """Parse a single ``--disk ...`` line; return None if the line isn't one.""" + opts_list = line.split(" ") + if not opts_list or opts_list[0] != "--disk": + return None + options, _rem = getopt.gnu_getopt(opts_list[1:], "", _DISK_LONG_OPTS) + return disk_options(options) + + +def parse_partition_lines( + lines: list[str], + image_map: Mapping[str, str] | None = None, +) -> PartitionsByLun: + """Parse a list of ``--partition ...`` lines, grouped by LUN/phys-part.""" + if image_map is None: + image_map = {} + partitions: PartitionsByLun = {} + for line in lines: + opts_list = line.split(" ") + if not opts_list or opts_list[0] != "--partition": + continue + options, _rem = getopt.gnu_getopt(opts_list[1:], "", _PARTITION_LONG_OPTS) + phys_part, entry = partition_options(options, image_map) + partitions.setdefault(phys_part, []).append(entry) + return partitions + + +# --------------------------------------------------------------------------- +# File-level parsing +# --------------------------------------------------------------------------- + + +def read_conf(path: str) -> tuple[str, list[str]]: + """Read ``path`` and return ``(disk_line, [partition_lines])``. + + Comments (``#``-prefixed) and blank lines are skipped. Multiple ``--disk`` + lines are an error. Lines that aren't ``--disk`` or ``--partition`` are + printed as "Ignoring ..." for parity with the original behaviour. + """ + disk_line: str | None = None + partition_lines: list[str] = [] + with open(path) as f: + for raw in f: + if re.search(r"^\s*#", raw) or re.search(r"^\s*$", raw): + continue + line = raw.strip() + if line.startswith("--disk"): + if disk_line is not None: + raise ConfParseError( + "%s contains more than one --disk entries:\n%s\n%s" + % (path, disk_line, line) + ) + disk_line = line + elif line.startswith("--partition"): + partition_lines.append(line) + else: + print("Ignoring %s" % line) + if disk_line is None: + raise ConfParseError("%s contains no --disk entry" % path) + return disk_line, partition_lines + + +def load(path: str, image_map: Mapping[str, str] | None = None) -> LoadedSpec: + """Public entry point: read ``path`` and return a normalised ``LoadedSpec``.""" + disk_line, partition_lines = read_conf(path) + disk = parse_disk_line(disk_line) + if disk is None: + # Should be unreachable: read_conf guarantees disk_line starts with --disk. + raise ConfParseError("%s: failed to parse --disk line" % path) + partitions = parse_partition_lines(partition_lines, image_map) + return {"disk": disk, "partitions": partitions} diff --git a/qcom_ptool/loaders/yaml.py b/qcom_ptool/loaders/yaml.py new file mode 100644 index 0000000..13d00f3 --- /dev/null +++ b/qcom_ptool/loaders/yaml.py @@ -0,0 +1,137 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +""" +Loader for the structured YAML partition source (single storage). + +A YAML document describes one ``disk`` mapping and a ``partitions`` list; this +module validates it against ``qcom_ptool/schema/partitions.schema.json`` and +normalises it into the exact same :class:`qcom_ptool.spec.LoadedSpec` the +``.conf`` loader produces, so both formats emit byte-identical XML. + +Normalisation mirrors ``loaders/conf.py`` field for field: the disk dict is +built by copying ``DISK_PARAMS_DEFAULTS`` and each partition by copying +``PARTITION_ENTRY_DEFAULTS``, so key order (and therefore XML attribute order) +matches the legacy loader. ``partition_size_in_kb`` and the attribute-bit +decoding are reused / replicated identically. +""" + +from __future__ import annotations + +import json +from collections.abc import Mapping +from importlib import resources +from typing import Any + +import jsonschema +import yaml # type: ignore[import-untyped] + +from qcom_ptool.loaders.conf import partition_size_in_kb +from qcom_ptool.spec import ( + DISK_PARAMS_DEFAULTS, + PARTITION_ENTRY_DEFAULTS, + DiskParams, + LoadedSpec, + PartitionEntry, + PartitionsByLun, +) + +_SCHEMA_PACKAGE = "qcom_ptool.schema" +_SCHEMA_RESOURCE = "partitions.schema.json" + + +class YamlParseError(ValueError): + """Raised when a YAML partition source is malformed or fails validation.""" + + +def _bool_str(value: Any) -> str: + """Render a YAML scalar as the lowercase "true"/"false" the .conf loader + stores. ``str(True)`` would yield "True", which the emitter would then + write verbatim and break parity, so booleans are normalised explicitly.""" + if isinstance(value, bool): + return "true" if value else "false" + return str(value) + + +def load_schema() -> dict[str, Any]: + """Return the packaged partition JSON Schema as a dict.""" + text = resources.files(_SCHEMA_PACKAGE).joinpath(_SCHEMA_RESOURCE).read_text( + encoding="utf-8" + ) + schema: dict[str, Any] = json.loads(text) + return schema + + +def _disk_from_node(node: Mapping[str, Any]) -> DiskParams: + """Normalise the ``disk`` mapping into the canonical disk dict. + + Starts from ``DISK_PARAMS_DEFAULTS`` and overrides only the keys present, + exactly like ``conf.disk_options``, so unset fields keep their defaults and + the key order is identical. + """ + disk = DISK_PARAMS_DEFAULTS.copy() + disk["type"] = str(node["type"]) + disk["size"] = str(node["size"]) + if "sector-size" in node: + disk["SECTOR_SIZE_IN_BYTES"] = str(node["sector-size"]) + if "write-protect-boundary" in node: + disk["WRITE_PROTECT_BOUNDARY_IN_KB"] = str(node["write-protect-boundary"]) + if node.get("grow-last-partition"): + disk["GROW_LAST_PARTITION_TO_FILL_DISK"] = "true" + if "align-partitions" in node: + disk["ALIGN_PARTITIONS_TO_PERFORMANCE_BOUNDARY"] = "true" + disk["PERFORMANCE_BOUNDARY_IN_KB"] = str(int(node["align-partitions"]) // 1024) + return disk + + +def _partition_from_node( + node: Mapping[str, Any], + image_map: Mapping[str, str], +) -> tuple[str, PartitionEntry]: + """Normalise one partition mapping into ``(phys_part, entry)``. + + Mirrors ``conf.partition_options``: same defaults, same attribute-bit + decoding, and the image-map override applied last so it always wins. + """ + entry: PartitionEntry = PARTITION_ENTRY_DEFAULTS.copy() + phys_part = "0" + if "lun" in node: + phys_part = str(node["lun"]) + elif "phys-part" in node: + phys_part = str(node["phys-part"]) + entry["label"] = str(node["name"]) + entry["size_in_kb"] = str(partition_size_in_kb(str(node["size"]))) + entry["type"] = str(node["type-guid"]) + if "attributes" in node: + attribute_bits = int(str(node["attributes"]), 16) + entry["bootable"] = "true" if attribute_bits & (1 << 2) else "false" + entry["readonly"] = "true" if attribute_bits & (1 << 60) else "false" + if "filename" in node: + entry["filename"] = str(node["filename"]) + if "sparse" in node: + entry["sparse"] = _bool_str(node["sparse"]) + if entry["label"] in image_map: + entry["filename"] = image_map[entry["label"]] + return phys_part, entry + + +def load(path: str, image_map: Mapping[str, str] | None = None) -> LoadedSpec: + """Public entry point: read ``path`` and return a normalised ``LoadedSpec``.""" + if image_map is None: + image_map = {} + with open(path) as f: + document = yaml.safe_load(f) + if not isinstance(document, Mapping): + raise YamlParseError( + "%s: expected a top-level mapping, got %s" % (path, type(document).__name__) + ) + try: + jsonschema.validate(document, load_schema()) + except jsonschema.exceptions.ValidationError as exc: + raise YamlParseError("%s: schema validation failed: %s" % (path, exc.message)) from exc + + disk = _disk_from_node(document["disk"]) + partitions: PartitionsByLun = {} + for node in document.get("partitions", []): + phys_part, entry = _partition_from_node(node, image_map) + partitions.setdefault(phys_part, []).append(entry) + return {"disk": disk, "partitions": partitions} diff --git a/qcom_ptool/schema/__init__.py b/qcom_ptool/schema/__init__.py new file mode 100644 index 0000000..c4c2d13 --- /dev/null +++ b/qcom_ptool/schema/__init__.py @@ -0,0 +1,8 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +""" +Packaged JSON Schema documents for the structured (YAML) partition source. + +The ``.json`` files here are shipped as package data (see ``pyproject.toml``) +and loaded at runtime via ``importlib.resources`` by the YAML loader. +""" diff --git a/qcom_ptool/schema/board.schema.json b/qcom_ptool/schema/board.schema.json new file mode 100644 index 0000000..9217434 --- /dev/null +++ b/qcom_ptool/schema/board.schema.json @@ -0,0 +1,65 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/qualcomm-linux/qcom-ptool/board.schema.json", + "title": "qcom-ptool board / variant source", + "description": "Structural schema for a board file, a derived (extends) board, or a variant overlay. Storage entries and partitions are permissive here because overrides are partial (e.g. a derived board may set only a partition size); the fully-resolved storage is validated strictly against partitions.schema.json.", + "type": "object", + "additionalProperties": false, + "required": ["storage"], + "properties": { + "platform": { + "type": "object", + "additionalProperties": false, + "properties": { + "name": {"type": "string"} + } + }, + "extends": { + "description": "Path (relative to the platforms root) of a single base board to inherit.", + "type": "string" + }, + "storage": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["id"], + "properties": { + "id": {"type": "string", "minLength": 1}, + "type": {"type": "string", "enum": ["emmc", "nand", "nvme", "spinor", "ufs"]}, + "size": {"type": "integer", "minimum": 1}, + "sector-size": {"type": "integer", "minimum": 1}, + "write-protect-boundary": {"type": "integer", "minimum": 0}, + "grow-last-partition": {"type": "boolean"}, + "align-partitions": {"type": "integer", "minimum": 0}, + "includes": { + "description": "Shared partition fragments (relative to the platforms root) concatenated into this storage's partition list.", + "type": "array", + "items": {"type": "string"} + }, + "partitions": { + "type": "array", + "items": {"$ref": "#/definitions/partition"} + } + } + } + } + }, + "definitions": { + "partition": { + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "size": {"type": "string", "pattern": "^[0-9]+([KkMmGg][Bb]?)?$"}, + "type-guid": {"type": "string", "pattern": "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$"}, + "filename": {"type": "string"}, + "attributes": {"type": "string", "pattern": "^[0-9A-Fa-f]+$"}, + "sparse": {"type": "boolean"}, + "lun": {"type": "integer", "minimum": 0}, + "phys-part": {"type": "integer", "minimum": 0} + } + } + } +} diff --git a/qcom_ptool/schema/include.schema.json b/qcom_ptool/schema/include.schema.json new file mode 100644 index 0000000..904e26e --- /dev/null +++ b/qcom_ptool/schema/include.schema.json @@ -0,0 +1,32 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/qualcomm-linux/qcom-ptool/include.schema.json", + "title": "qcom-ptool shared partition fragment", + "description": "A reusable partition group under platforms/_common/, pulled into a storage via 'includes:'. May itself pull in further fragments.", + "type": "object", + "additionalProperties": false, + "properties": { + "includes": { + "type": "array", + "items": {"type": "string"} + }, + "partitions": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name"], + "properties": { + "name": {"type": "string", "minLength": 1}, + "size": {"type": "string", "pattern": "^[0-9]+([KkMmGg][Bb]?)?$"}, + "type-guid": {"type": "string", "pattern": "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$"}, + "filename": {"type": "string"}, + "attributes": {"type": "string", "pattern": "^[0-9A-Fa-f]+$"}, + "sparse": {"type": "boolean"}, + "lun": {"type": "integer", "minimum": 0}, + "phys-part": {"type": "integer", "minimum": 0} + } + } + } + } +} diff --git a/qcom_ptool/schema/partitions.schema.json b/qcom_ptool/schema/partitions.schema.json new file mode 100644 index 0000000..9bed21f --- /dev/null +++ b/qcom_ptool/schema/partitions.schema.json @@ -0,0 +1,91 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "https://github.com/qualcomm-linux/qcom-ptool/partitions.schema.json", + "title": "qcom-ptool single-storage partition spec", + "description": "Structured YAML source for one storage device and its partitions. Reduces to a LoadedSpec (see qcom_ptool/spec.py).", + "type": "object", + "additionalProperties": false, + "required": ["disk", "partitions"], + "properties": { + "disk": { + "type": "object", + "additionalProperties": false, + "required": ["type", "size"], + "properties": { + "type": { + "description": "Storage class.", + "type": "string", + "enum": ["emmc", "nand", "nvme", "spinor", "ufs"] + }, + "size": { + "description": "Total device size in bytes.", + "type": "integer", + "minimum": 1 + }, + "sector-size": { + "description": "Sector size in bytes.", + "type": "integer", + "minimum": 1 + }, + "write-protect-boundary": { + "description": "Write-protect boundary in KB.", + "type": "integer", + "minimum": 0 + }, + "grow-last-partition": { + "description": "Grow the last partition to fill the device.", + "type": "boolean" + }, + "align-partitions": { + "description": "Performance alignment boundary in bytes.", + "type": "integer", + "minimum": 0 + } + } + }, + "partitions": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "required": ["name", "size", "type-guid"], + "properties": { + "name": { + "type": "string", + "minLength": 1 + }, + "size": { + "description": "Partition size as a quoted string: bytes ('1024') or with a K/M/G[B] suffix ('524288KB'). Quoting is required so values are never misparsed as numbers.", + "type": "string", + "pattern": "^[0-9]+([KkMmGg][Bb]?)?$" + }, + "type-guid": { + "description": "Partition type GUID; must be a quoted string to avoid integer misparsing.", + "type": "string", + "pattern": "^[0-9A-Fa-f]{8}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{4}-[0-9A-Fa-f]{12}$" + }, + "filename": { + "type": "string" + }, + "attributes": { + "description": "GPT attribute bits as a hexadecimal string.", + "type": "string", + "pattern": "^[0-9A-Fa-f]+$" + }, + "sparse": { + "type": "boolean" + }, + "lun": { + "description": "Physical partition / LUN index (mandatory for multi-LUN UFS).", + "type": "integer", + "minimum": 0 + }, + "phys-part": { + "type": "integer", + "minimum": 0 + } + } + } + } + } +} diff --git a/qcom_ptool/show.py b/qcom_ptool/show.py new file mode 100644 index 0000000..9509b66 --- /dev/null +++ b/qcom_ptool/show.py @@ -0,0 +1,70 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +""" +``qcom-ptool show`` - print a board's fully-resolved spec. + +Derived boards and heavy ``includes:`` use make the effective partition +layout impossible to read from a single file. This subcommand resolves all +``extends`` / ``includes`` / variant overlays and prints the result as YAML, +so reviewers and integrators can see exactly what will be emitted without +chasing the composition chain by hand. +""" + +from __future__ import annotations + +import getopt +import sys +from typing import NoReturn + +import yaml # type: ignore[import-untyped] + +from qcom_ptool.board import resolve_board + + +def usage() -> NoReturn: + print( + "\n\tUsage: %s --board [--hlos ] " + "[--boot-fw ] [--root ]\n" % sys.argv[0] + ) + sys.exit(1) + + +def main(argv: list[str] | None = None) -> int: + if argv is None: + argv = sys.argv + + board_file: str | None = None + root: str | None = None + hlos: str | None = None + boot_fw: str | None = None + + try: + opts, _rem = getopt.getopt(argv[1:], "", ["board=", "hlos=", "boot-fw=", "root="]) + for opt, arg in opts: + if opt == "--board": + board_file = arg + elif opt == "--hlos": + hlos = arg + elif opt == "--boot-fw": + boot_fw = arg + elif opt == "--root": + root = arg + except Exception as argerr: + print(str(argerr)) + usage() + + if board_file is None: + usage() + + try: + board = resolve_board(board_file, root=root, hlos=hlos, boot_fw=boot_fw) + except Exception as e: + print("Error: ", e) + return 1 + + yaml.safe_dump(dict(board), sys.stdout, sort_keys=False, default_flow_style=False) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/qcom_ptool/spec.py b/qcom_ptool/spec.py new file mode 100644 index 0000000..580bf54 --- /dev/null +++ b/qcom_ptool/spec.py @@ -0,0 +1,73 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +""" +Internal representation shared by all input loaders. + +Every loader (``loaders/conf.py`` today, ``loaders/yaml.py`` tomorrow) must +return a ``LoadedSpec`` so the rest of ``gen_partition.py`` is independent +of the source format. + +``DiskParams`` and ``PartitionEntry`` are type aliases for ``dict[str, str]`` +rather than ``TypedDict``s; the XML emitter passes the dicts straight to +``ET.SubElement(..., attrib=...)`` which requires a plain ``dict[str, str]`` +at runtime, and aliasing keeps that contract precise without TypedDict's +``total=False`` ``object`` widening. Migrating to dataclasses (with explicit +``to_attrib()`` methods) is a clean follow-up if stronger validation becomes +worth its blast radius. +""" + +from __future__ import annotations + +from collections import OrderedDict +from typing import TypedDict + +# Parsed ``--disk`` line normalised into the keys ``ptool`` consumes. +# Expected keys: type, size, SECTOR_SIZE_IN_BYTES, WRITE_PROTECT_BOUNDARY_IN_KB, +# GROW_LAST_PARTITION_TO_FILL_DISK, ALIGN_PARTITIONS_TO_PERFORMANCE_BOUNDARY, +# PERFORMANCE_BOUNDARY_IN_KB. See DISK_PARAMS_DEFAULTS for the canonical set. +DiskParams = dict[str, str] + +# Parsed ``--partition`` line normalised for the XML attribute set. +# Expected keys: label, size_in_kb, type, bootable, readonly, filename, sparse. +# See PARTITION_ENTRY_DEFAULTS for the canonical set. +PartitionEntry = dict[str, str] + +# Mapping of physical partition / LUN id (kept as a string for backward +# compatibility with the existing line parser) to its partition entries. +PartitionsByLun = dict[str, list[PartitionEntry]] + + +# Defaults used by the conf loader (and any future loader that wants to +# inherit the same baseline). Kept as module-level constants so loaders can +# ``.copy()`` from them rather than rebuilding the structure each call. +# ``OrderedDict`` is preserved here because the XML emitter relies on its +# iteration order to produce stable output across Python versions. +DISK_PARAMS_DEFAULTS: DiskParams = OrderedDict( + { + "type": "", + "size": "", + "SECTOR_SIZE_IN_BYTES": "512", + "WRITE_PROTECT_BOUNDARY_IN_KB": "65536", + "GROW_LAST_PARTITION_TO_FILL_DISK": "false", + "ALIGN_PARTITIONS_TO_PERFORMANCE_BOUNDARY": "true", + "PERFORMANCE_BOUNDARY_IN_KB": "4", + } +) + + +PARTITION_ENTRY_DEFAULTS: PartitionEntry = { + "label": "", + "size_in_kb": "", + "type": "00000000-0000-0000-0000-000000000000", + "bootable": "false", + "readonly": "true", + "filename": "", + "sparse": "false", +} + + +class LoadedSpec(TypedDict): + """What every loader returns from ``load(path)``.""" + + disk: DiskParams + partitions: PartitionsByLun diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/data/glymur-crd-nvme.conf b/tests/unit/data/glymur-crd-nvme.conf new file mode 100644 index 0000000..a94efa7 --- /dev/null +++ b/tests/unit/data/glymur-crd-nvme.conf @@ -0,0 +1,9 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# +# Legacy .conf equivalent of glymur-crd-nvme.yaml, kept as a self-contained +# fixture so the YAML/.conf parity tests do not depend on any board under +# platforms/ (which migrate to YAML over time). +--disk --type=nvme --size=68719476736 --write-protect-boundary=65536 --sector-size-in-bytes=512 --grow-last-partition +--partition --name=efi --size=524288KB --type-guid=C12A7328-F81F-11D2-BA4B-00A0C93EC93B --filename=efi.bin +--partition --name=rootfs --size=33554432KB --type-guid=B921B045-1DF0-41C3-AF44-4C6F280D3FAE --filename=rootfs.img diff --git a/tests/unit/data/glymur-crd-nvme.yaml b/tests/unit/data/glymur-crd-nvme.yaml new file mode 100644 index 0000000..a8d3004 --- /dev/null +++ b/tests/unit/data/glymur-crd-nvme.yaml @@ -0,0 +1,21 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +# +# Single-storage YAML equivalent of platforms/glymur-crd/nvme/partitions.conf. +# Used by tests/unit/test_loaders_yaml.py to assert byte-for-byte parity +# between the .conf and .yaml loaders. +disk: + type: nvme + size: 68719476736 + write-protect-boundary: 65536 + sector-size: 512 + grow-last-partition: true +partitions: + - name: efi + size: "524288KB" + type-guid: "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" + filename: efi.bin + - name: rootfs + size: "33554432KB" + type-guid: "B921B045-1DF0-41C3-AF44-4C6F280D3FAE" + filename: rootfs.img diff --git a/tests/unit/test_board.py b/tests/unit/test_board.py new file mode 100644 index 0000000..5971f4f --- /dev/null +++ b/tests/unit/test_board.py @@ -0,0 +1,437 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +""" +Unit tests for the multi-storage board resolver (qcom_ptool/board.py). + +These pin the composition contract: board-level ``extends``, +storage-level ``includes`` (with transitivity), variant overlays, deep-merge +by stable identifier, the deterministic ordering rule, cycle detection, and +the reduction to a byte-parity LoadedSpec. +""" + +from __future__ import annotations + +import os +import textwrap + +import pytest + +from qcom_ptool.board import ( + BoardResolveError, + board_to_specs, + resolve_board, +) +from qcom_ptool.gen_partition import generate_partition_xml +from qcom_ptool.loaders import load as load_spec + +GUID_A = "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" +GUID_B = "B921B045-1DF0-41C3-AF44-4C6F280D3FAE" +GUID_C = "DEA0BA2C-CBDD-4805-B4F9-F428251C3E98" +GUID_D = "CD6CDFAB-B3F7-46C6-BFFE-1A1D2B8B7BA0" + + +def _tree(tmp_path, files: dict[str, str]) -> str: + """Write a mini platforms tree and return its root path.""" + root = tmp_path / "platforms" + for rel, text in files.items(): + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(textwrap.dedent(text)) + return str(root) + + +def _names(storage: dict) -> list[str]: + return [p["name"] for p in storage["partitions"]] + + +def _by_id(board, sid: str) -> dict: + return next(s for s in board["storage"] if s["id"] == sid) + + +_BASE = """ + platform: {name: base} + storage: + - id: nvme0 + type: nvme + size: 68719476736 + sector-size: 512 + write-protect-boundary: 65536 + grow-last-partition: true + partitions: + - name: efi + size: "524288KB" + type-guid: "%s" + filename: efi.bin + - name: rootfs + size: "1024KB" + type-guid: "%s" +""" % (GUID_A, GUID_B) + + +# --------------------------------------------------------------------------- +# extends +# --------------------------------------------------------------------------- + + +def test_extends_inherits_and_overrides_by_name(tmp_path): + root = _tree( + tmp_path, + { + "boards/base.yaml": _BASE, + "boards/derived.yaml": """ + extends: boards/base.yaml + platform: {name: derived} + storage: + - id: nvme0 + partitions: + - name: rootfs + size: "2048KB" + """, + }, + ) + board = resolve_board(os.path.join(root, "boards/derived.yaml")) + assert board["platform"] == {"name": "derived"} + nvme = _by_id(board, "nvme0") + rootfs = next(p for p in nvme["partitions"] if p["name"] == "rootfs") + # size overridden, type-guid inherited field-by-field. + assert rootfs["size"] == "2048KB" + assert rootfs["type-guid"] == GUID_B + # order preserved: overridden entry stays in place. + assert _names(nvme) == ["efi", "rootfs"] + + +def test_extends_appends_new_storage_and_partition(tmp_path): + root = _tree( + tmp_path, + { + "boards/base.yaml": _BASE, + "boards/derived.yaml": """ + extends: boards/base.yaml + storage: + - id: nvme0 + partitions: + - name: extra + size: "8KB" + type-guid: "%s" + - id: emmc0 + type: emmc + size: 1024 + partitions: + - name: boot + size: "8KB" + type-guid: "%s" + """ % (GUID_C, GUID_D), + }, + ) + board = resolve_board(os.path.join(root, "boards/derived.yaml")) + assert [s["id"] for s in board["storage"]] == ["nvme0", "emmc0"] + assert _names(_by_id(board, "nvme0")) == ["efi", "rootfs", "extra"] + + +def test_extends_cycle_is_rejected(tmp_path): + root = _tree( + tmp_path, + { + "boards/a.yaml": "extends: boards/b.yaml\nstorage: []\n", + "boards/b.yaml": "extends: boards/a.yaml\nstorage: []\n", + }, + ) + with pytest.raises(BoardResolveError, match="cycle"): + resolve_board(os.path.join(root, "boards/a.yaml")) + + +# --------------------------------------------------------------------------- +# includes +# --------------------------------------------------------------------------- + + +def test_includes_concatenate_then_inline_last(tmp_path): + root = _tree( + tmp_path, + { + "_common/boot.yaml": """ + partitions: + - name: XBL_SC + size: "2520KB" + type-guid: "%s" + - name: BOOT_FW1 + size: "12288KB" + type-guid: "%s" + """ % (GUID_C, GUID_D), + "boards/b.yaml": """ + platform: {name: b} + storage: + - id: spinor0 + type: spinor + size: 67108864 + sector-size: 4096 + write-protect-boundary: 0 + includes: [_common/boot.yaml] + partitions: + - name: cdt + size: "4KB" + type-guid: "%s" + """ % GUID_A, + }, + ) + board = resolve_board(os.path.join(root, "boards/b.yaml")) + # included fragments first (in order), inline partitions last. + assert _names(_by_id(board, "spinor0")) == ["XBL_SC", "BOOT_FW1", "cdt"] + + +def test_includes_are_transitive(tmp_path): + root = _tree( + tmp_path, + { + "_common/leaf.yaml": """ + partitions: + - name: LEAF + size: "4KB" + type-guid: "%s" + """ % GUID_A, + "_common/mid.yaml": """ + includes: [_common/leaf.yaml] + partitions: + - name: MID + size: "4KB" + type-guid: "%s" + """ % GUID_B, + "boards/b.yaml": """ + storage: + - id: spinor0 + type: spinor + size: 67108864 + sector-size: 4096 + includes: [_common/mid.yaml] + partitions: [] + """, + }, + ) + board = resolve_board(os.path.join(root, "boards/b.yaml")) + assert _names(_by_id(board, "spinor0")) == ["LEAF", "MID"] + + +def test_includes_cycle_is_rejected(tmp_path): + root = _tree( + tmp_path, + { + "_common/a.yaml": "includes: [_common/b.yaml]\n", + "_common/b.yaml": "includes: [_common/a.yaml]\n", + "boards/b.yaml": """ + storage: + - id: s0 + type: spinor + size: 67108864 + includes: [_common/a.yaml] + """, + }, + ) + with pytest.raises(BoardResolveError, match="cycle"): + resolve_board(os.path.join(root, "boards/b.yaml")) + + +# --------------------------------------------------------------------------- +# variant overlays +# --------------------------------------------------------------------------- + + +def test_variants_apply_last_over_includes(tmp_path): + root = _tree( + tmp_path, + { + "_common/boot.yaml": """ + partitions: + - name: BOOT_FW1 + size: "12288KB" + type-guid: "%s" + filename: bootfw1.bin + """ % GUID_D, + "boards/b.yaml": """ + storage: + - id: nvme0 + type: nvme + size: 68719476736 + sector-size: 512 + partitions: + - name: rootfs + size: "1024KB" + type-guid: "%s" + - id: spinor0 + type: spinor + size: 67108864 + sector-size: 4096 + includes: [_common/boot.yaml] + partitions: + - name: cdt + size: "4KB" + type-guid: "%s" + """ % (GUID_B, GUID_A), + "variants/hlos/debian.yaml": """ + storage: + - id: nvme0 + partitions: + - name: rootfs + size: "33554432KB" + filename: rootfs.img + """, + "variants/boot-fw/xbl-v3.yaml": """ + storage: + - id: spinor0 + partitions: + - name: BOOT_FW1 + size: "16384KB" + filename: bootfw1-v3.bin + - name: vm-data + size: "8192KB" + type-guid: "%s" + """ % GUID_C, + }, + ) + board = resolve_board( + os.path.join(root, "boards/b.yaml"), hlos="debian", boot_fw="xbl-v3" + ) + nvme = _by_id(board, "nvme0") + rootfs = nvme["partitions"][0] + assert rootfs["size"] == "33554432KB" + assert rootfs["filename"] == "rootfs.img" + assert rootfs["type-guid"] == GUID_B # inherited through the overlay + + spinor = _by_id(board, "spinor0") + # BOOT_FW1 overridden in place; vm-data appended after inline cdt. + assert _names(spinor) == ["BOOT_FW1", "cdt", "vm-data"] + boot_fw1 = spinor["partitions"][0] + assert boot_fw1["size"] == "16384KB" + assert boot_fw1["filename"] == "bootfw1-v3.bin" + + +# --------------------------------------------------------------------------- +# final validation +# --------------------------------------------------------------------------- + + +def test_resolved_storage_missing_type_is_rejected(tmp_path): + root = _tree( + tmp_path, + { + "boards/b.yaml": """ + storage: + - id: nvme0 + size: 68719476736 + partitions: + - name: efi + size: "1KB" + type-guid: "%s" + """ % GUID_A, + }, + ) + with pytest.raises(BoardResolveError, match="invalid spec"): + resolve_board(os.path.join(root, "boards/b.yaml")) + + +def test_partition_missing_type_guid_after_resolution_is_rejected(tmp_path): + # rootfs never gets a type-guid from base or any overlay -> invalid. + root = _tree( + tmp_path, + { + "boards/b.yaml": """ + storage: + - id: nvme0 + type: nvme + size: 68719476736 + partitions: + - name: rootfs + size: "1KB" + """, + }, + ) + with pytest.raises(BoardResolveError, match="invalid spec"): + resolve_board(os.path.join(root, "boards/b.yaml")) + + +# --------------------------------------------------------------------------- +# reduction / byte parity +# --------------------------------------------------------------------------- + + +def test_board_reduces_to_byte_identical_xml(tmp_path): + """A resolved single-storage board emits the same XML as the equivalent + hand-written single-storage YAML loaded via the Phase 2 loader.""" + root = _tree( + tmp_path, + { + "boards/b.yaml": """ + platform: {name: b} + storage: + - id: nvme0 + type: nvme + size: 68719476736 + write-protect-boundary: 65536 + sector-size: 512 + grow-last-partition: true + partitions: + - name: efi + size: "524288KB" + type-guid: "%s" + filename: efi.bin + - name: rootfs + size: "33554432KB" + type-guid: "%s" + filename: rootfs.img + """ % (GUID_A, GUID_B), + }, + ) + single = tmp_path / "single.yaml" + single.write_text( + textwrap.dedent( + """ + disk: + type: nvme + size: 68719476736 + write-protect-boundary: 65536 + sector-size: 512 + grow-last-partition: true + partitions: + - name: efi + size: "524288KB" + type-guid: "%s" + filename: efi.bin + - name: rootfs + size: "33554432KB" + type-guid: "%s" + filename: rootfs.img + """ % (GUID_A, GUID_B) + ) + ) + + board = resolve_board(os.path.join(root, "boards/b.yaml")) + (sid, spec) = board_to_specs(board)[0] + assert sid == "nvme0" + + board_xml = tmp_path / "board.xml" + single_xml = tmp_path / "single.xml" + generate_partition_xml(spec["disk"], spec["partitions"], str(board_xml)) + single_spec = load_spec(str(single)) + generate_partition_xml(single_spec["disk"], single_spec["partitions"], str(single_xml)) + assert board_xml.read_bytes() == single_xml.read_bytes() + + +def test_image_map_flows_through_reduction(tmp_path): + root = _tree( + tmp_path, + { + "boards/b.yaml": """ + storage: + - id: nvme0 + type: nvme + size: 68719476736 + partitions: + - name: rootfs + size: "1KB" + type-guid: "%s" + filename: rootfs.img + """ % GUID_B, + }, + ) + board = resolve_board(os.path.join(root, "boards/b.yaml")) + specs = board_to_specs(board, image_map={"rootfs": "override.img"}) + assert specs[0][1]["partitions"]["0"][0]["filename"] == "override.img" diff --git a/tests/unit/test_cli_board.py b/tests/unit/test_cli_board.py new file mode 100644 index 0000000..98913bd --- /dev/null +++ b/tests/unit/test_cli_board.py @@ -0,0 +1,139 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +""" +CLI tests for board-mode ``gen_partition`` and the ``show`` subcommand. +""" + +from __future__ import annotations + +import textwrap + +import pytest + +from qcom_ptool import gen_partition, show + +GUID_A = "C12A7328-F81F-11D2-BA4B-00A0C93EC93B" +GUID_B = "B921B045-1DF0-41C3-AF44-4C6F280D3FAE" + +_BOARD = """ + platform: {name: demo} + storage: + - id: nvme0 + type: nvme + size: 68719476736 + sector-size: 512 + write-protect-boundary: 65536 + grow-last-partition: true + partitions: + - name: efi + size: "524288KB" + type-guid: "%s" + filename: efi.bin + - id: spinor0 + type: spinor + size: 67108864 + sector-size: 4096 + write-protect-boundary: 0 + partitions: + - name: cdt + size: "4KB" + type-guid: "%s" + filename: cdt.bin +""" % (GUID_A, GUID_B) + + +def _board(tmp_path) -> str: + root = tmp_path / "platforms" + (root / "boards").mkdir(parents=True) + path = root / "boards" / "demo.yaml" + path.write_text(textwrap.dedent(_BOARD)) + return str(path) + + +def test_board_mode_emits_one_xml_per_storage(tmp_path): + board = _board(tmp_path) + nvme = tmp_path / "nvme.xml" + spinor = tmp_path / "spinor.xml" + rc = gen_partition.main( + ["gen_partition", "--board", board, "-o", str(nvme), "-o", str(spinor)] + ) + assert rc == 0 + assert 'label="efi"' in nvme.read_text() + assert 'label="cdt"' in spinor.read_text() + + +def test_board_mode_output_count_mismatch_returns_1(tmp_path): + board = _board(tmp_path) + rc = gen_partition.main( + ["gen_partition", "--board", board, "-o", str(tmp_path / "only.xml")] + ) + assert rc == 1 + + +def test_requires_exactly_one_source(tmp_path): + board = _board(tmp_path) + # both -i and --board -> usage() exits. + with pytest.raises(SystemExit): + gen_partition.main( + ["gen_partition", "-i", "x.conf", "--board", board, "-o", "out.xml"] + ) + + +def test_legacy_single_storage_still_works(tmp_path): + src = tmp_path / "single.yaml" + src.write_text( + textwrap.dedent( + """ + disk: + type: nvme + size: 68719476736 + partitions: + - name: efi + size: "524288KB" + type-guid: "%s" + filename: efi.bin + """ % GUID_A + ) + ) + out = tmp_path / "out.xml" + rc = gen_partition.main(["gen_partition", "-i", str(src), "-o", str(out)]) + assert rc == 0 + assert 'label="efi"' in out.read_text() + + +def test_show_prints_resolved_spec(tmp_path, capsys): + board = _board(tmp_path) + rc = show.main(["show", "--board", board]) + assert rc == 0 + out = capsys.readouterr().out + assert "nvme0" in out + assert "spinor0" in out + assert "cdt" in out + + +def test_show_applies_variants(tmp_path, capsys): + root = tmp_path / "platforms" + (root / "boards").mkdir(parents=True) + (root / "variants" / "hlos").mkdir(parents=True) + (root / "boards" / "demo.yaml").write_text(textwrap.dedent(_BOARD)) + (root / "variants" / "hlos" / "debian.yaml").write_text( + textwrap.dedent( + """ + storage: + - id: nvme0 + partitions: + - name: efi + filename: efi-debian.bin + """ + ) + ) + rc = show.main( + ["show", "--board", str(root / "boards" / "demo.yaml"), "--hlos", "debian"] + ) + assert rc == 0 + assert "efi-debian.bin" in capsys.readouterr().out + + +def test_show_requires_board(): + with pytest.raises(SystemExit): + show.main(["show"]) diff --git a/tests/unit/test_glymur_poc.py b/tests/unit/test_glymur_poc.py new file mode 100644 index 0000000..6f9e93e --- /dev/null +++ b/tests/unit/test_glymur_poc.py @@ -0,0 +1,46 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +""" +Structural regression test for the migrated Glymur-CRD YAML board. + +Byte-for-byte fidelity of the emitted artifacts is enforced by the pinned +manifest (tests/integration/checksums.sha256, checked by `make +check-checksums`). This test guards the board's shape at the unit level: it +resolves with the Debian HLOS variant and asserts the two storages, their +partition order, and that the rootfs size/filename come from the variant. +""" + +from __future__ import annotations + +import os + +from qcom_ptool.board import board_to_specs, resolve_board + +REPO = os.path.abspath(os.path.join(os.path.dirname(__file__), "..", "..")) +BOARD = os.path.join(REPO, "platforms", "boards", "glymur-crd.yaml") + + +def test_glymur_resolves_two_storages_in_order(): + board = resolve_board(BOARD, hlos="qcom-deb-images") + specs = board_to_specs(board) + assert [sid for sid, _ in specs] == ["nvme0", "spinor0"] + + +def test_glymur_nvme_layout_and_variant_rootfs(): + board = resolve_board(BOARD, hlos="qcom-deb-images") + nvme = dict(board_to_specs(board))["nvme0"] + labels = [p["label"] for p in nvme["partitions"]["0"]] + assert labels == ["efi", "rootfs"] + rootfs = nvme["partitions"]["0"][1] + # Supplied by variants/hlos/qcom-deb-images.yaml. + assert rootfs["size_in_kb"] == "33554432" + assert rootfs["filename"] == "rootfs.img" + + +def test_glymur_spinor_has_full_partition_list(): + board = resolve_board(BOARD, hlos="qcom-deb-images") + spinor = dict(board_to_specs(board))["spinor0"] + labels = [p["label"] for p in spinor["partitions"]["0"]] + assert len(labels) == 65 + assert labels[0] == "cdt" + assert labels[-1] == "dtb_b" diff --git a/tests/unit/test_loaders_conf.py b/tests/unit/test_loaders_conf.py new file mode 100644 index 0000000..a5e02a6 --- /dev/null +++ b/tests/unit/test_loaders_conf.py @@ -0,0 +1,381 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +""" +Tests for ``qcom_ptool.loaders.conf`` plus a small main() smoke test. + +The bulk of these started life as characterization tests against the inline +parser in ``gen_partition.py`` and were rewritten to target the extracted +loader. They pin the contract that any future loader (e.g. YAML) must match +to remain byte-for-byte compatible with the .conf path. +""" + +from __future__ import annotations + +import pytest + +from qcom_ptool import gen_partition as gp +from qcom_ptool import spec +from qcom_ptool.loaders import UnsupportedFormatError, conf, load + + +# --------------------------------------------------------------------------- +# partition_size_in_kb +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "size,expected", + [ + # bare bytes -> divided by 1024 + ("1024", 1), + ("2048", 2), + ("0", 0), + # KB / Kb / Kk + ("1KB", 1), + ("128KB", 128), + ("4Kb", 4), + ("4K", 4), + # MB / Mb / M -> *1024 + ("1MB", 1024), + ("2MB", 2048), + ("4M", 4096), + # GB / Gb / G -> *1024*1024 + ("1GB", 1024 * 1024), + ("64GB", 64 * 1024 * 1024), + # mixed-prefix strings: regex returns the first numeric run before the suffix + ("foo123KB", 123), + ], +) +def test_partition_size_in_kb_recognised_forms(size: str, expected: int) -> None: + assert conf.partition_size_in_kb(size) == expected + + +@pytest.mark.parametrize("size", ["abc", "1TB", "MB", "1PB"]) +def test_partition_size_in_kb_unrecognised_suffix_raises(size: str) -> None: + """Strings with letters that don't match K/M/G hit the explicit raise.""" + with pytest.raises(ValueError, match="Unrecognized size format"): + conf.partition_size_in_kb(size) + + +def test_partition_size_in_kb_empty_string_raises_int_error() -> None: + """Empty strings take the "no letters -> int(size)" branch and surface + int()'s native ValueError rather than the explicit "Unrecognized" message. + Pinning this asymmetry so any future loader normalises empty strings + consistently.""" + with pytest.raises(ValueError, match="invalid literal for int"): + conf.partition_size_in_kb("") + + +# --------------------------------------------------------------------------- +# disk_options +# --------------------------------------------------------------------------- + + +def test_disk_options_returns_defaults_when_no_options() -> None: + result = conf.disk_options([]) + assert result == spec.DISK_PARAMS_DEFAULTS + # ensure caller receives a copy, not the shared default + assert result is not spec.DISK_PARAMS_DEFAULTS + + +def test_disk_options_basic_ufs() -> None: + opts = [ + ("--type", "ufs"), + ("--size", "137438953472"), + ("--sector-size-in-bytes", "4096"), + ("--write-protect-boundary", "0"), + ("--grow-last-partition", ""), + ] + result = conf.disk_options(opts) + assert result["type"] == "ufs" + assert result["size"] == "137438953472" + assert result["SECTOR_SIZE_IN_BYTES"] == "4096" + assert result["WRITE_PROTECT_BOUNDARY_IN_KB"] == "0" + assert result["GROW_LAST_PARTITION_TO_FILL_DISK"] == "true" + + +def test_disk_options_align_partitions_converts_bytes_to_kb() -> None: + # --align-partitions takes a value in bytes; stored value is in KB + result = conf.disk_options([("--align-partitions", "4096")]) + assert result["ALIGN_PARTITIONS_TO_PERFORMANCE_BOUNDARY"] == "true" + assert result["PERFORMANCE_BOUNDARY_IN_KB"] == "4" + + +def test_disk_options_unknown_flag_silently_ignored() -> None: + # The function intentionally only matches known flags; unknown flags + # leave the params at their defaults. + result = conf.disk_options([("--bogus", "x")]) + assert result == spec.DISK_PARAMS_DEFAULTS + + +# --------------------------------------------------------------------------- +# partition_options +# --------------------------------------------------------------------------- + + +def test_partition_options_basic() -> None: + opts = [ + ("--lun", "0"), + ("--name", "rootfs"), + ("--size", "33554432KB"), + ("--type-guid", "B921B045-1DF0-41C3-AF44-4C6F280D3FAE"), + ("--filename", "rootfs.img"), + ] + phys_part, entry = conf.partition_options(opts) + assert phys_part == "0" + assert entry["label"] == "rootfs" + assert entry["size_in_kb"] == "33554432" + assert entry["type"] == "B921B045-1DF0-41C3-AF44-4C6F280D3FAE" + assert entry["filename"] == "rootfs.img" + + +def test_partition_options_phys_part_alias() -> None: + # --phys-part and --lun are equivalent + _, entry = conf.partition_options([("--name", "x"), ("--phys-part", "3")]) + assert entry["label"] == "x" + phys_part, _ = conf.partition_options([("--phys-part", "3")]) + assert phys_part == "3" + + +def test_partition_options_size_with_suffix_normalises_to_kb() -> None: + _, entry = conf.partition_options([("--size", "2MB")]) + assert entry["size_in_kb"] == "2048" + + _, entry = conf.partition_options([("--size", "1GB")]) + assert entry["size_in_kb"] == str(1024 * 1024) + + +def test_partition_options_attributes_bootable_and_readonly_bits() -> None: + # bit 2 (0x4) -> bootable=true; bit 60 (1<<60) -> readonly=true + _, entry = conf.partition_options([("--attributes", "1000000000000004")]) + assert entry["bootable"] == "true" + assert entry["readonly"] == "true" + + # bit 2 alone -> bootable=true, readonly=false + _, entry = conf.partition_options([("--attributes", "4")]) + assert entry["bootable"] == "true" + assert entry["readonly"] == "false" + + # bit 60 alone -> bootable=false, readonly=true + _, entry = conf.partition_options([("--attributes", "1000000000000000")]) + assert entry["bootable"] == "false" + assert entry["readonly"] == "true" + + # neither bit -> bootable=false, readonly=false + _, entry = conf.partition_options([("--attributes", "0")]) + assert entry["bootable"] == "false" + assert entry["readonly"] == "false" + + +def test_partition_options_image_map_overrides_filename() -> None: + _, entry = conf.partition_options( + [("--name", "rootfs"), ("--filename", "default.img")], + image_map={"rootfs": "custom-rootfs.img"}, + ) + assert entry["filename"] == "custom-rootfs.img" + + +def test_partition_options_image_map_no_match_keeps_filename() -> None: + _, entry = conf.partition_options( + [("--name", "rootfs"), ("--filename", "default.img")], + image_map={"other": "wont-match.img"}, + ) + assert entry["filename"] == "default.img" + + +def test_partition_options_defaults_applied() -> None: + _, entry = conf.partition_options([("--name", "x")]) + assert entry["type"] == "00000000-0000-0000-0000-000000000000" + assert entry["bootable"] == "false" + assert entry["readonly"] == "true" + assert entry["sparse"] == "false" + assert entry["filename"] == "" + + +# --------------------------------------------------------------------------- +# parse_disk_line (line-level) +# --------------------------------------------------------------------------- + + +def test_parse_disk_line_full_line() -> None: + line = ( + "--disk --type=ufs --size=137438953472 " + "--sector-size-in-bytes=4096 --write-protect-boundary=0 " + "--grow-last-partition" + ) + result = conf.parse_disk_line(line) + assert result is not None + assert result["type"] == "ufs" + assert result["size"] == "137438953472" + assert result["SECTOR_SIZE_IN_BYTES"] == "4096" + assert result["GROW_LAST_PARTITION_TO_FILL_DISK"] == "true" + + +def test_parse_disk_line_returns_none_when_not_disk_line() -> None: + # A line that doesn't start with --disk is silently dropped. + assert conf.parse_disk_line("--partition --name=x --size=1KB") is None + + +# --------------------------------------------------------------------------- +# parse_partition_lines (line-level) +# --------------------------------------------------------------------------- + + +def test_parse_partition_lines_single_partition() -> None: + lines = [ + "--partition --lun=0 --name=rootfs --size=1KB " + "--type-guid=B921B045-1DF0-41C3-AF44-4C6F280D3FAE", + ] + result = conf.parse_partition_lines(lines) + assert "0" in result + assert len(result["0"]) == 1 + assert result["0"][0]["label"] == "rootfs" + assert result["0"][0]["size_in_kb"] == "1" + + +def test_parse_partition_lines_groups_by_lun() -> None: + lines = [ + "--partition --lun=0 --name=a --size=1KB " + "--type-guid=00000000-0000-0000-0000-000000000001", + "--partition --lun=0 --name=b --size=2KB " + "--type-guid=00000000-0000-0000-0000-000000000002", + "--partition --lun=1 --name=c --size=4KB " + "--type-guid=00000000-0000-0000-0000-000000000003", + ] + result = conf.parse_partition_lines(lines) + assert sorted(result.keys()) == ["0", "1"] + assert [p["label"] for p in result["0"]] == ["a", "b"] + assert [p["label"] for p in result["1"]] == ["c"] + + +def test_parse_partition_lines_skips_non_partition_lines() -> None: + # Lines not starting with --partition are silently ignored by this helper. + lines = ["--disk --type=ufs --size=1024", "--something-else"] + assert conf.parse_partition_lines(lines) == {} + + +# --------------------------------------------------------------------------- +# loader public API: load() + dispatcher +# --------------------------------------------------------------------------- + + +def test_conf_load_full(tmp_path) -> None: + p = tmp_path / "p.conf" + p.write_text( + "# header\n" + "--disk --type=ufs --size=1073741824 --sector-size-in-bytes=4096\n" + "--partition --lun=0 --name=rootfs --size=1KB " + "--type-guid=B921B045-1DF0-41C3-AF44-4C6F280D3FAE --filename=r.img\n" + ) + result = conf.load(str(p)) + assert result["disk"]["type"] == "ufs" + assert result["disk"]["size"] == "1073741824" + assert result["partitions"]["0"][0]["label"] == "rootfs" + assert result["partitions"]["0"][0]["filename"] == "r.img" + + +def test_conf_load_image_map_overrides_filename(tmp_path) -> None: + p = tmp_path / "p.conf" + p.write_text( + "--disk --type=ufs --size=1073741824 --sector-size-in-bytes=4096\n" + "--partition --lun=0 --name=rootfs --size=1KB " + "--type-guid=B921B045-1DF0-41C3-AF44-4C6F280D3FAE --filename=default.img\n" + ) + result = conf.load(str(p), image_map={"rootfs": "override.img"}) + assert result["partitions"]["0"][0]["filename"] == "override.img" + + +def test_conf_load_rejects_two_disk_lines(tmp_path) -> None: + p = tmp_path / "p.conf" + p.write_text("--disk --type=ufs --size=1\n--disk --type=ufs --size=2\n") + with pytest.raises(conf.ConfParseError, match="more than one --disk"): + conf.load(str(p)) + + +def test_conf_load_rejects_missing_disk_line(tmp_path) -> None: + p = tmp_path / "p.conf" + p.write_text( + "--partition --lun=0 --name=x --size=1KB " + "--type-guid=00000000-0000-0000-0000-000000000001\n" + ) + with pytest.raises(conf.ConfParseError, match="no --disk entry"): + conf.load(str(p)) + + +def test_dispatcher_routes_conf_extension(tmp_path) -> None: + p = tmp_path / "p.conf" + p.write_text( + "--disk --type=ufs --size=1073741824 --sector-size-in-bytes=4096\n" + "--partition --lun=0 --name=x --size=1KB " + "--type-guid=00000000-0000-0000-0000-000000000001\n" + ) + result = load(str(p)) + assert result["disk"]["type"] == "ufs" + + +def test_dispatcher_rejects_unknown_extension(tmp_path) -> None: + p = tmp_path / "p.toml" + p.write_text("not actually parsed") + with pytest.raises(UnsupportedFormatError, match="No loader registered"): + load(str(p)) + + +# --------------------------------------------------------------------------- +# main() smoke test (end-to-end via gen_partition.main) +# --------------------------------------------------------------------------- + + +def test_main_produces_xml_for_minimal_conf(tmp_path) -> None: + conf_path = tmp_path / "partitions.conf" + conf_path.write_text( + "# comment line, should be ignored\n" + "\n" + "--disk --type=ufs --size=1073741824 --sector-size-in-bytes=4096\n" + "--partition --lun=0 --name=rootfs --size=1KB " + "--type-guid=B921B045-1DF0-41C3-AF44-4C6F280D3FAE --filename=r.img\n" + ) + out = tmp_path / "partitions.xml" + + rc = gp.main(["gen_partition", "-i", str(conf_path), "-o", str(out)]) + + assert rc == 0 + content = out.read_text() + assert "physical_partition" in content + assert 'label="rootfs"' in content + assert 'size_in_kb="1"' in content + + +def test_main_image_map_override_applied(tmp_path) -> None: + conf_path = tmp_path / "partitions.conf" + conf_path.write_text( + "--disk --type=ufs --size=1073741824 --sector-size-in-bytes=4096\n" + "--partition --lun=0 --name=rootfs --size=1KB " + "--type-guid=B921B045-1DF0-41C3-AF44-4C6F280D3FAE --filename=default.img\n" + ) + out = tmp_path / "partitions.xml" + + rc = gp.main( + [ + "gen_partition", + "-i", str(conf_path), + "-o", str(out), + "-m", "rootfs=override.img", + ] + ) + + assert rc == 0 + assert 'filename="override.img"' in out.read_text() + + +def test_main_rejects_two_disk_lines(tmp_path, capsys) -> None: + conf_path = tmp_path / "partitions.conf" + conf_path.write_text( + "--disk --type=ufs --size=1\n" + "--disk --type=ufs --size=2\n" + ) + out = tmp_path / "partitions.xml" + + rc = gp.main(["gen_partition", "-i", str(conf_path), "-o", str(out)]) + + assert rc == 1 + assert "more than one --disk" in capsys.readouterr().out diff --git a/tests/unit/test_loaders_yaml.py b/tests/unit/test_loaders_yaml.py new file mode 100644 index 0000000..73afe2b --- /dev/null +++ b/tests/unit/test_loaders_yaml.py @@ -0,0 +1,202 @@ +# Copyright (c) Qualcomm Technologies, Inc. and/or its subsidiaries. +# SPDX-License-Identifier: BSD-3-Clause +""" +Unit tests for the YAML partition loader. + +The load-bearing property is that the YAML loader produces the *same* +LoadedSpec as the .conf loader, so both emit byte-identical XML. These tests +pin that equivalence on a real board, on a synthetic multi-LUN spec with +attribute bits, and at the XML-bytes level, plus the schema rejections that +protect against the documented YAML footguns. +""" + +from __future__ import annotations + +import os + +import pytest + +from qcom_ptool.gen_partition import generate_partition_xml +from qcom_ptool.loaders import load +from qcom_ptool.loaders import yaml as yaml_loader + +DATA_DIR = os.path.join(os.path.dirname(__file__), "data") +# Self-contained fixtures (not a live board): glymur-crd migrated to YAML, +# so the parity reference lives here rather than under platforms/. +GLYMUR_NVME_CONF = os.path.join(DATA_DIR, "glymur-crd-nvme.conf") +GLYMUR_NVME_YAML = os.path.join(DATA_DIR, "glymur-crd-nvme.yaml") + + +def _write(tmp_path, name: str, text: str) -> str: + path = tmp_path / name + path.write_text(text) + return str(path) + + +# --------------------------------------------------------------------------- +# Equivalence with the .conf loader +# --------------------------------------------------------------------------- + + +def test_yaml_matches_conf_for_glymur_nvme(): + """The committed YAML fixture and the real board .conf load identically.""" + assert load(GLYMUR_NVME_YAML) == load(GLYMUR_NVME_CONF) + + +def test_yaml_matches_conf_multi_lun_and_attributes(tmp_path): + """LUN grouping and attribute-bit decoding match the .conf loader.""" + conf = _write( + tmp_path, + "s.conf", + "--disk --type=ufs --size=137438953472 --write-protect-boundary=0 " + "--sector-size-in-bytes=4096 --grow-last-partition\n" + "--partition --lun=0 --name=rootfs --size=79691776KB " + "--type-guid=1B81E7E6-F50D-419B-A739-2AEEF8DA3335 --filename=rootfs.img\n" + "--partition --lun=1 --name=xbl_a --size=3584KB " + "--type-guid=DEA0BA2C-CBDD-4805-B4F9-F428251C3E98 --filename=xbl.elf " + "--attributes=1000000000000004\n", + ) + yml = _write( + tmp_path, + "s.yaml", + "disk:\n" + " type: ufs\n" + " size: 137438953472\n" + " write-protect-boundary: 0\n" + " sector-size: 4096\n" + " grow-last-partition: true\n" + "partitions:\n" + " - name: rootfs\n" + " lun: 0\n" + ' size: "79691776KB"\n' + ' type-guid: "1B81E7E6-F50D-419B-A739-2AEEF8DA3335"\n' + " filename: rootfs.img\n" + " - name: xbl_a\n" + " lun: 1\n" + ' size: "3584KB"\n' + ' type-guid: "DEA0BA2C-CBDD-4805-B4F9-F428251C3E98"\n' + " filename: xbl.elf\n" + ' attributes: "1000000000000004"\n', + ) + spec = load(yml) + assert spec == load(conf) + # attribute-bit decode: bit 2 -> bootable, bit 60 -> readonly. + xbl = spec["partitions"]["1"][0] + assert xbl["bootable"] == "true" + assert xbl["readonly"] == "true" + + +def test_yaml_and_conf_emit_byte_identical_xml(tmp_path): + """End-to-end: the emitter output is identical for both source formats.""" + conf_xml = tmp_path / "from_conf.xml" + yaml_xml = tmp_path / "from_yaml.xml" + conf_spec = load(GLYMUR_NVME_CONF) + yaml_spec = load(GLYMUR_NVME_YAML) + generate_partition_xml(conf_spec["disk"], conf_spec["partitions"], str(conf_xml)) + generate_partition_xml(yaml_spec["disk"], yaml_spec["partitions"], str(yaml_xml)) + assert yaml_xml.read_bytes() == conf_xml.read_bytes() + + +# --------------------------------------------------------------------------- +# Loader behaviour +# --------------------------------------------------------------------------- + + +def test_dispatcher_routes_yaml_and_yml(tmp_path): + """The dispatcher sends .yaml and .yml to the YAML loader.""" + body = ( + "disk:\n" + " type: nvme\n" + " size: 68719476736\n" + "partitions:\n" + " - name: efi\n" + ' size: "524288KB"\n' + ' type-guid: "C12A7328-F81F-11D2-BA4B-00A0C93EC93B"\n' + ) + for name in ("s.yaml", "s.yml"): + spec = load(_write(tmp_path, name, body)) + assert spec["disk"]["type"] == "nvme" + assert spec["partitions"]["0"][0]["label"] == "efi" + + +def test_sparse_boolean_normalised_to_lowercase(tmp_path): + """A YAML boolean must serialise as "true", not Python's "True".""" + spec = load( + _write( + tmp_path, + "s.yaml", + "disk:\n type: nvme\n size: 68719476736\n" + "partitions:\n" + " - name: cache\n" + ' size: "1024KB"\n' + ' type-guid: "C12A7328-F81F-11D2-BA4B-00A0C93EC93B"\n' + " sparse: true\n", + ) + ) + assert spec["partitions"]["0"][0]["sparse"] == "true" + + +def test_image_map_overrides_filename(tmp_path): + """The -m image map overrides filename, matching the .conf loader.""" + spec = load( + _write( + tmp_path, + "s.yaml", + "disk:\n type: nvme\n size: 68719476736\n" + "partitions:\n" + " - name: rootfs\n" + ' size: "1024KB"\n' + ' type-guid: "C12A7328-F81F-11D2-BA4B-00A0C93EC93B"\n' + " filename: rootfs.img\n", + ), + image_map={"rootfs": "override.img"}, + ) + assert spec["partitions"]["0"][0]["filename"] == "override.img" + + +# --------------------------------------------------------------------------- +# Schema validation (footgun rejection) +# --------------------------------------------------------------------------- + +_VALID_DISK = "disk:\n type: nvme\n size: 68719476736\n" + + +def _partition(**overrides) -> str: + fields = { + "name": "efi", + "size": '"524288KB"', + "type-guid": '"C12A7328-F81F-11D2-BA4B-00A0C93EC93B"', + } + fields.update(overrides) + lines = "".join(f" {k}: {v}\n" for k, v in fields.items()) + return _VALID_DISK + "partitions:\n" + " -\n" + lines + + +@pytest.mark.parametrize( + "text", + [ + # GUID left unquoted and all-digit -> parsed as int -> type error. + _VALID_DISK + + 'partitions:\n - name: efi\n size: "1KB"\n type-guid: 0123456789012345678901234567\n', + # size as a bare integer (unquoted) -> not a string -> rejected. + _VALID_DISK + + "partitions:\n - name: efi\n size: 524288\n" + + ' type-guid: "C12A7328-F81F-11D2-BA4B-00A0C93EC93B"\n', + # unknown partition key -> additionalProperties: false. + _partition(typo="1"), + # missing required type-guid. + _VALID_DISK + 'partitions:\n - name: efi\n size: "1KB"\n', + # malformed size string. + _partition(size='"12XB"'), + # unknown disk type. + "disk:\n type: floppy\n size: 1024\npartitions: []\n", + ], +) +def test_schema_rejects_invalid(tmp_path, text): + with pytest.raises(yaml_loader.YamlParseError): + load(_write(tmp_path, "bad.yaml", text)) + + +def test_top_level_must_be_mapping(tmp_path): + with pytest.raises(yaml_loader.YamlParseError): + load(_write(tmp_path, "list.yaml", "- 1\n- 2\n"))