From e96e9c06af87b3a8e74d6561729e9396f8524f58 Mon Sep 17 00:00:00 2001 From: Jin Xiao Date: Mon, 5 Oct 2026 01:49:50 +0800 Subject: [PATCH] fix: align UTF-8 decoding across payload utilities --- docs/CHECKLIST.md | 2 + docs/JMESPATH.md | 2 + docs/MODULE_ACCEPTANCE_SCOPED.json | 68 ++++- docs/MODULE_REVIEW_2026_10_05.json | 278 +++++++++++++++++++++ docs/MODULE_REVIEW_2026_10_05.md | 39 +++ docs/PARAMETERS.md | 2 + docs/PARSER.md | 2 + jmespath/envelopes.go | 3 +- jmespath/testdata/utf8-v2.35.0.json | 125 +++++++++ jmespath/utf8_test.go | 36 +++ mkdocs.yml | 1 + parameters/testdata/utf8-v2.35.0.json | 103 ++++++++ parameters/transform.go | 7 +- parameters/utf8_test.go | 53 ++++ parser/helpers.go | 5 +- parser/testdata/utf8-v2.35.0.json | 127 ++++++++++ parser/utf8_test.go | 35 +++ tools/reference/generate-utf8-decoding.mjs | 46 ++++ tools/reference/package.json | 1 + 19 files changed, 923 insertions(+), 12 deletions(-) create mode 100644 docs/MODULE_REVIEW_2026_10_05.json create mode 100644 docs/MODULE_REVIEW_2026_10_05.md create mode 100644 jmespath/testdata/utf8-v2.35.0.json create mode 100644 jmespath/utf8_test.go create mode 100644 parameters/testdata/utf8-v2.35.0.json create mode 100644 parameters/utf8_test.go create mode 100644 parser/testdata/utf8-v2.35.0.json create mode 100644 parser/utf8_test.go create mode 100644 tools/reference/generate-utf8-decoding.mjs diff --git a/docs/CHECKLIST.md b/docs/CHECKLIST.md index cbbacda..7de5dfa 100644 --- a/docs/CHECKLIST.md +++ b/docs/CHECKLIST.md @@ -276,3 +276,5 @@ Race-detector execution is intentionally excluded because CGO must remain disabl - [x] REL-PUBLIC: Publish all 27 maintained public modules as v0.1.0 from merged preparation PR #16 at immutable SHA 34fcc751355a2b29bca393748139ff96dd8bbf2a. The hosted GoReleaser OSS v2.18.2 run finalized every Release only after independent public download, tidy, build, module/checksum verification, and rejection of local replacements, with CGO disabled, GOWORK=off and fresh session caches. Release issue #15 closed automatically after completion. The frozen tracer/xray adapter was excluded. See [RELEASE_ACCEPTANCE.json](RELEASE_ACCEPTANCE.json) (2026-10-04). Hosted preparation controls and native PR/main checks were exercised; browser form rendering and default-token authorization remain separate rollout checks. - [x] REL-UNIFIED: Replace new component-scoped releases with one version for all maintained public modules, preserving independent Go paths and prefixed tags. Add project release_version, aggregate auto-increment policy, cohort-wide reserved-version avoidance, synchronized internal requirements, schema-2 full-cohort validation, component/repository grouped root notes, and root finalization last with the stable Latest label. Twenty-one isolated automation and 37 release regressions passed, including Logger-only cohort bumps, divergent published baselines/reserved nested tags, partial/mixed-plan rejection, aligned requirements, failure/resume ordering, legacy schema-1 recovery, and real GoReleaser OSS v2.18.2 root/component/prerelease draft/finalization and Latest configuration. Eighteen contribution/form regressions and actionlint v1.7.12 passed (2026-10-05). This records implementation acceptance; no new public version was prepared or published. See issue #23 and [Releasing modules](RELEASING.md). - [x] SIGNER-BODY-HANDOFF: Preserve asynchronous base-transport body ownership for custom signers returning the input request or a clone sharing its body. Four deterministic success/error regressions reproduced premature closure before the fix; packaged Signer tests, vet, tidy verification and its independent consumer build pass with CGO disabled. Distinct/empty bodies and nil signing results retain cleanup coverage. See issue #25 and [Signer ownership](SIGNER.md) (2026-10-05). +- [x] MODULE-REVIEW-20261005: Review the main entry points and critical state/ownership/error/decoding paths of the 26 maintained public modules excluding Logger. Twenty-five native Windows packaged checks and standalone consumers passed with CGO disabled; the exact-baseline hosted Linux run passed all 31 repository modules, including KMS and frozen X-Ray. Register issues #25–#28 before production changes; merge the verified Signer fix before starting the shared decoding fix. Local Docker runtime acceptance was unavailable. See [the review scope and evidence](MODULE_REVIEW_2026_10_05.md). +- [x] UTF8-DECODING: Reuse Commons.DecodeUTF8 in Parser, Parameters and JMESPath while preserving each reference path's BOM and gzip fallback rules. Generate 56 expected cases from the pinned TypeScript 2.35.0 packages: 20 Parser, 16 Parameters and 20 JMESPath cases. New regressions failed before the fix; all three affected packaged modules passed tests, vet, tidy and standalone consumer builds afterward, with CGO disabled and GOWORK=off. Parameters additionally verifies Auto transforms and untransformed raw bytes. See issues #26–#28 and [the review report](MODULE_REVIEW_2026_10_05.md) (2026-10-05). diff --git a/docs/JMESPATH.md b/docs/JMESPATH.md index a336aa8..bb5aaf4 100644 --- a/docs/JMESPATH.md +++ b/docs/JMESPATH.md @@ -83,6 +83,8 @@ Pass a compiled expression to `logger.HandlerOptions.CorrelationExtractor`. Logg Seventy-eight cases execute the actual TypeScript v2.35.0 package: standard functions, projections/filters/slices/pipes, numeric/null behavior, all thirteen envelopes, Unicode/BOM handling, and errors. Additional Go tests cover custom functions, signature errors, result/definition isolation, typed events, and 100 concurrent searches. Docker covers decoded projections and Logger correlation alongside Signer and OTel. +Twenty additional pinned reference cases verify UTF-8 replacement and BOM boundaries for the Powertools Base64 functions. Both replace each maximal malformed subpart using Commons. `powertools_base64` strips exactly one leading BOM, matching `TextDecoder`; `powertools_base64_gzip` retains BOM, matching `Buffer.toString`. See `testdata/utf8-v2.35.0.json` and `tools/reference/generate-utf8-decoding.mjs`. + `*Error` carries `Kind`, `Expression`, optional `Function`, and an unwrap cause. Syntax errors are grouped instead of reproducing every TypeScript lexer/parser exception class or message; empty expressions have a distinct kind. Function type/arity and unknown-function failures remain distinguishable. The pinned interpreter silently swallows ordinary decoder and custom-function failures, returning JavaScript `undefined`. Go intentionally returns an error and preserves the cause. Fixtures explicitly record the three decoder `undefined` results rather than treating them as successful null values. JSON object ordering, `to_string` serialization details, malformed UTF-8 replacement boundaries, and extreme numeric behavior still require exhaustive cross-language coverage. This implementation does not claim a complete specification compliance audit or performance budgets. diff --git a/docs/MODULE_ACCEPTANCE_SCOPED.json b/docs/MODULE_ACCEPTANCE_SCOPED.json index 17eed76..aa717d7 100644 --- a/docs/MODULE_ACCEPTANCE_SCOPED.json +++ b/docs/MODULE_ACCEPTANCE_SCOPED.json @@ -5,24 +5,82 @@ "completed": true, "modules": [ { - "path": "github.com/rambow-cloud/powertools-lambda-go/signer", - "directory": "signer", + "path": "github.com/rambow-cloud/powertools-lambda-go/jmespath", + "directory": "jmespath", "public": true, "version": "v0.1.0", - "archive_sha256": "7e7bc65590a5f255d3a4381394ec15d75848455196dab3a5a6aa3f3884e96726", + "archive_sha256": "175484f2b375c7f22875d4e687ea002dc8dbfd3dfe4ab3612a8f716c1b802be0", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true, + "external_modules": [ + "github.com/davecgh/go-spew", + "github.com/google/go-cmp", + "github.com/jmespath-community/go-jmespath", + "github.com/pmezard/go-difflib", + "github.com/stretchr/testify", + "golang.org/x/exp", + "golang.org/x/mod", + "golang.org/x/sys", + "golang.org/x/tools", + "gopkg.in/yaml.v3" + ], + "binary_bytes": 3223040 + }, + { + "path": "github.com/rambow-cloud/powertools-lambda-go/parser", + "directory": "parser", + "public": true, + "version": "v0.1.0", + "archive_sha256": "750c955357a32433e4163bbefc03f2c1fa339b05490731b463c60c937386945a", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true, + "external_modules": [], + "binary_bytes": 4545024 + }, + { + "path": "github.com/rambow-cloud/powertools-lambda-go/parameters", + "directory": "parameters", + "public": true, + "version": "v0.1.0", + "archive_sha256": "3adfda169361b436e67c5ae13e08caf4f3e244be587bb808e387358c959ed13c", "tests": true, "vet": true, "tidy": true, "consumer_build": true, "external_modules": [ "github.com/aws/aws-sdk-go-v2", + "github.com/aws/aws-sdk-go-v2/config", + "github.com/aws/aws-sdk-go-v2/credentials", + "github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue", + "github.com/aws/aws-sdk-go-v2/feature/ec2/imds", + "github.com/aws/aws-sdk-go-v2/internal/configsources", + "github.com/aws/aws-sdk-go-v2/internal/endpoints/v2", + "github.com/aws/aws-sdk-go-v2/internal/v4a", + "github.com/aws/aws-sdk-go-v2/service/appconfigdata", + "github.com/aws/aws-sdk-go-v2/service/dynamodb", + "github.com/aws/aws-sdk-go-v2/service/dynamodbstreams", + "github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding", + "github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery", + "github.com/aws/aws-sdk-go-v2/service/internal/presigned-url", + "github.com/aws/aws-sdk-go-v2/service/secretsmanager", + "github.com/aws/aws-sdk-go-v2/service/signin", + "github.com/aws/aws-sdk-go-v2/service/ssm", + "github.com/aws/aws-sdk-go-v2/service/sso", + "github.com/aws/aws-sdk-go-v2/service/ssooidc", + "github.com/aws/aws-sdk-go-v2/service/sts", "github.com/aws/smithy-go" ], - "binary_bytes": 5602304 + "binary_bytes": 3157504 } ], "scope": "Selected modules only; this report does not replace full-workspace acceptance", "selected_directories": [ - "signer" + "jmespath", + "parameters", + "parser" ] } diff --git a/docs/MODULE_REVIEW_2026_10_05.json b/docs/MODULE_REVIEW_2026_10_05.json new file mode 100644 index 0000000..550a350 --- /dev/null +++ b/docs/MODULE_REVIEW_2026_10_05.json @@ -0,0 +1,278 @@ +{ + "date": "2026-10-05", + "baseline_commit": "6d4157d24f84f7fa2b8a1cdeb94ad5bac8359a93", + "scope": "26 maintained public modules excluding Logger; frozen tracer/xray regression evidence retained", + "local": { + "go": "1.27.1", + "platform": "windows/amd64", + "cgo_enabled": false, + "workspace": "off", + "module_count": 25, + "modules": [ + { + "directory": ".", + "archive_sha256": "480e2dbebef7cdb9947e999a313533d1b582c88a38d2585f62ac10894b3cbe79", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "commons/regex", + "archive_sha256": "cd229f91182c2ff3015384faaa4e3214dbb0896f09fe66eaa9c07004fd99ec55", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "signer", + "archive_sha256": "473cc7dced621fa191f12cc4f5105646203ee3d4d46e5667bc9ccbe434c82217", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "batch", + "archive_sha256": "c4bbc2f4e8c8a0e69ae9b10dcfd41a348bd60cda532706018a868d79922f0bc3", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "commons/awssdk", + "archive_sha256": "ec1411bd3d705bfcc03ae078b4adb85a57ae6e601ff0c7310d20075c84077ec5", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "commons/dynamodb", + "archive_sha256": "01a09d86bb4c90995ca10f6237ce0103a48fbe665cf966aabf8aace9e6264dca", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "commons/metadata", + "archive_sha256": "8aac8b9fe97edfe705d22cb62f297453d4a88bcae11adcafbdfa47cfc79ced9e", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "datamasking", + "archive_sha256": "a0af3350e9bde0c4d46a521e1274c938afc3cb24fab9257409955dfc16e122cb", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "eventhandler/appsyncevents", + "archive_sha256": "d873f72f5496c14c6bcf669def78a88c97c2ae9d48f7cad90c2cae14842f1ded", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "eventhandler/appsyncgraphql", + "archive_sha256": "859a311c3cc428a27358c7ab2a156d94b3a6f4c78e69950e419cb5e42fd01ba4", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "eventhandler/bedrock", + "archive_sha256": "a2a0c89936007fb9396a57932d18a0c95ee99163584f6b6ee9bf83f53e497cf7", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "eventhandler/http", + "archive_sha256": "5e62d66311aabcd8a4941c8baa046746067caf7b201d531c5cfc89c244b137c8", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "jmespath", + "archive_sha256": "7bf2ec8c850435531fa6f460aea4bba17c01d66954999d07d22e8bf8c1534f9f", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "kafka", + "archive_sha256": "63cce9360d920e296350e6880204455d805b6608384a9adf5a7a06ad1946f6f4", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "metrics", + "archive_sha256": "0e54bf02fc1c175c726d29397fd1eb728053e3513d1f6a1bb9f45e86b3cc1a88", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "parser", + "archive_sha256": "ad0a9d640a88d0b01049749730d9b04d4eadef06be22bd4a60f5fe683947d8af", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "eventhandler/http/metrics", + "archive_sha256": "f3ec4b843a955435986a62d760f7c84c290d50339ff3e7a00a97cb890905f565", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "idempotency", + "archive_sha256": "6e89d4fb28b8a6ff10f9b49f4e53cb89c94634ec724a0449fc061e27d6581d67", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "kafka/avro", + "archive_sha256": "09f7d1ceae9e8a043b29fb890e5dfc21589394b3c579389989cdd29265fe1083", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "kafka/protobuf", + "archive_sha256": "0037dad4873823fc7cc2cf8ca70a55a2468ad661af4a8ce7902d7b4acfab252b", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "parameters", + "archive_sha256": "bd19af4a40ad71ba1dd8f249b68ab00d365eb286482a753d5df8e78ddae1bec2", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "tracer", + "archive_sha256": "3dd558ab92d7b3f441ae1673014c17f5bdc6acf8af73ae43ea93edf5fa93bc5f", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "validation", + "archive_sha256": "c3124787bce16f480efd7e207ecd3998e3408f9b3ea4a7881c3fc9ed89313eec", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "eventhandler/http/tracer", + "archive_sha256": "dd0927202bafa506137ca91910c9b520eea67dedfc486cdcb7ef40bebf820fe5", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + }, + { + "directory": "idempotency/cache", + "archive_sha256": "bb252f2f266ad78a4131f14ea7ad16ce4fe023716a618449811e9058696f055e", + "tests": true, + "vet": true, + "tidy": true, + "consumer_build": true + } + ] + }, + "linux_baseline": { + "run": "https://github.com/rambow-cloud/powertools-lambda-go/actions/runs/37219502507", + "completed": true, + "module_count": 31, + "kms": true, + "frozen_xray": true + }, + "findings": [ + { + "issue": 25, + "module": "signer", + "problem": "premature shared request-body closure" + }, + { + "issue": 26, + "module": "parser", + "problem": "UTF-8 replacement and BOM handling" + }, + { + "issue": 27, + "module": "parameters", + "problem": "UTF-8 replacement and BOM handling" + }, + { + "issue": 28, + "module": "jmespath", + "problem": "UTF-8 replacement" + } + ], + "local_limitations": [ + "Linux Docker engine did not respond within a bounded probe.", + "Frozen X-Ray workspace test could not download a selected external dependency." + ], + "reference_case_count": 53, + "pre_fix_differences": { + "parser": 13, + "parameters": 8, + "jmespath": 6 + }, + "signer_fix": { + "pr": 29, + "merge_commit": "b02d8dc47e8529ca6b6b61e217bf37b2e819df34", + "required_checks": "passed before merge" + }, + "utf8_regression_cases": { + "parser": 20, + "parameters": 16, + "jmespath": 20, + "total": 56 + }, + "utf8_packaged_verification": { + "selected_modules": [ + "parser", + "parameters", + "jmespath" + ], + "completed": true, + "tests": true, + "vet": true, + "tidy": true, + "standalone_consumers": 3, + "cgo_enabled": false, + "workspace": "off" + } +} diff --git a/docs/MODULE_REVIEW_2026_10_05.md b/docs/MODULE_REVIEW_2026_10_05.md new file mode 100644 index 0000000..ea0562b --- /dev/null +++ b/docs/MODULE_REVIEW_2026_10_05.md @@ -0,0 +1,39 @@ +# Module review, 2026-10-05 + +The review used main commit `6d4157d24f84f7fa2b8a1cdeb94ad5bac8359a93`, following the unified-version release change. It covered the main entry points and critical state, ownership, error and decoding paths of the 26 maintained public modules other than Logger. Existing compatibility boundaries and pinned TypeScript fixtures were checked before classifying findings. + +## Scope and baseline evidence + +| Area | Reviewed paths | +| --- | --- | +| Commons and adapters | Environment/number conversion, cloning/merging, cache primitives, invocation identity, SDK user agent, DynamoDB decoding, metadata retrieval and regex matching/replacement | +| Metrics and tracing | Scoped state, serialization/publication, wrapper cleanup, context propagation, OTel ownership and HTTP adapters | +| Parameters and Signer | Cache invalidation, provider retrieval/batching, transforms, signing and transport body ownership | +| Batch and Idempotency | Partial/FIFO failures, acquisition/replay/cleanup, DynamoDB persistence and cache recovery | +| Parser, Validation and JMESPath | Schema composition, envelope extraction, compilation, operational errors, reference fixtures and text decoding | +| Kafka and event handlers | Record decoding, Avro/Protobuf adapters, AppSync/Bedrock dispatch, HTTP conversion, routing, middleware and streaming cleanup | + +On Windows amd64 with Go 1.27.1 and CGO disabled, `tools/modules.py check --only ...` passed for all 25 selected modules that can execute natively on this host. Each packaged archive passed tests, vet, tidy verification and an independent consumer build with `GOWORK=off`. The exact selected modules and archive digests are recorded in [the baseline summary](MODULE_REVIEW_2026_10_05.json). + +The remaining maintained module, `datamasking/kms`, requires Linux. The local Docker Linux engine did not respond within a bounded probe. A local workspace attempt to run the frozen `tracer/xray` regressions also stopped at an external dependency download. The [complete Linux CI run at the exact baseline commit](https://github.com/rambow-cloud/powertools-lambda-go/actions/runs/37219502507) passed all 31 repository modules, including KMS and frozen X-Ray tests, vet, tidy and consumer builds. Its Lambda example builds and static binary packaging passed for Linux amd64 and arm64. These hosted checks supply Linux baseline evidence; a fresh local Docker runtime suite was unavailable. No AWS cloud tests were run. + +## Registered findings + +All four issues were created before changing production code. The ordinary baseline tests passed; additional reproductions exposed gaps in body-lifetime and malformed-text coverage. + +| Issue | Module | Confirmed behavior | +| --- | --- | --- | +| [#25](https://github.com/rambow-cloud/powertools-lambda-go/issues/25) | Signer | An identity or cloned custom signer shares the input body, which the wrapper closed before an asynchronous base transport finished using it. | +| [#26](https://github.com/rambow-cloud/powertools-lambda-go/issues/26) | Parser | Base64 decoding collapsed malformed byte runs and retained the plain-path BOM; gzip JSON decoding split truncated sequences incorrectly. | +| [#27](https://github.com/rambow-cloud/powertools-lambda-go/issues/27) | Parameters | Binary transforms collapsed malformed byte runs and retained BOM; BOM-prefixed byte JSON failed to parse. | +| [#28](https://github.com/rambow-cloud/powertools-lambda-go/issues/28) | JMESPath | Plain and gzip Base64 functions collapsed consecutive malformed subparts into one replacement character. | + +Four deterministic Signer cases failed before its fix. [PR #29](https://github.com/rambow-cloud/powertools-lambda-go/pull/29) passed the packaged Signer check and all three required native PR checks, then was merged using `gh` at `b02d8dc47e8529ca6b6b61e217bf37b2e819df34`. Only afterward did the shared text-decoding fix begin, mapping issues #26–#28 to one problem-focused PR. + +The text audit initially compared 53 inputs against the actual TypeScript 2.35.0 npm packages. It found 13 Parser, eight Parameters and six JMESPath differences. The committed regression corpus adds three byte-input cases: 20 Parser, 16 Parameters and 20 JMESPath cases, 56 total. The generator retains the reference's separate `TextDecoder` and `Buffer.toString` BOM behavior. All three affected packaged modules pass after replacing the divergent conversions with the existing Commons UTF-8 primitive. Parameters also checks Auto transforms and preservation of untransformed raw bytes. + +## Reproduction and acceptance boundaries + +After installing the pinned development dependencies, run `npm run fixtures:utf8` from `tools/reference` to regenerate the three UTF-8 fixtures. Run `CGO_ENABLED=0 uv run --no-project python tools/modules.py check --only parser --only parameters --only jmespath` from the repository root for their complete packaged verification. The Signer regression is `TestTransportBodyHandoff` in `signer/transport_body_test.go`. + +This is a review of the stated paths with executable baseline and targeted regressions. Existing unfinished compatibility, service-integration and performance gates remain tracked in their module documents. Frozen X-Ray retains its migration warning and existing regression scope. Module versions stay at the current release baseline; the PR release-note entries will be accumulated by the release workflow. diff --git a/docs/PARAMETERS.md b/docs/PARAMETERS.md index be1f663..553ceab 100644 --- a/docs/PARAMETERS.md +++ b/docs/PARAMETERS.md @@ -177,6 +177,8 @@ The cache protects maps with a mutex and copies supported mutable decoded values Transform names ignore case. Single-value failures return `*parameters.TransformParameterError`. Multiple retrievals retain failed entries as nil by default; `ThrowOnTransformError` returns the error. Non-string/non-byte values pass through. JSON null and missing values both map to nil; successfully decoded JSON null can still be cached by ordinary `Get`. +Byte input and Base64-decoded binary output follow `TextDecoder`: each maximal malformed UTF-8 subpart becomes a separate replacement character, and exactly one leading BOM is stripped at each decoding boundary. BOM-prefixed byte JSON therefore parses normally. Unset transforms preserve raw bytes. Sixteen pinned reference cases cover these boundaries, including Auto transforms, in `testdata/utf8-v2.35.0.json`. + `ThrowOnMissing` returns `*parameters.ParameterNotFoundError` for absent raw values. SSM/Secrets Manager normalize SDK not-found exceptions only with this option enabled; otherwise those exceptions remain available under `*parameters.GetParameterError`. Use `errors.As` for types and `errors.Is` for cancellation/deadlines. Writes return `*parameters.SetParameterError` on failure. ## Other providers diff --git a/docs/PARSER.md b/docs/PARSER.md index 0005c04..9f8d2d5 100644 --- a/docs/PARSER.md +++ b/docs/PARSER.md @@ -95,6 +95,8 @@ orders, err := parser.Parse(ctx, event, `JSONStringified` requires a string and then validates decoded JSON. `Base64Encoded` accepts Base64 JSON, gzip-compressed Base64 JSON, or decoded text, matching the reference helper's fallback sequence. Base64 decoding reuses Commons. Malformed Base64 can produce an operational error. A malformed JSON issue retains the `Invalid JSON` prefix; the detailed syntax diagnostic comes from Go's decoder. +`Base64Encoded` reuses Commons UTF-8 decoding, replacing each maximal malformed subpart separately. Plain input and the original-byte fallback strip exactly one leading BOM, matching `TextDecoder`. The gzip JSON path retains BOM, matching `Buffer.toString`; unsuccessful gzip JSON parsing still falls back to the original compressed bytes. Twenty pinned reference cases cover these paths, malformed byte runs, truncated sequences, single/double BOMs and valid Unicode in `testdata/utf8-v2.35.0.json`. + ## Initial event contracts | Reference export | Go equivalent | Behavior | diff --git a/jmespath/envelopes.go b/jmespath/envelopes.go index ed26865..668eede 100644 --- a/jmespath/envelopes.go +++ b/jmespath/envelopes.go @@ -5,7 +5,6 @@ import ( "compress/gzip" "encoding/json" "io" - "strings" "github.com/rambow-cloud/powertools-lambda-go/commons" ) @@ -58,7 +57,7 @@ func WithPowertoolsFunctions() Option { // TextDecoder strips a leading UTF-8 BOM; Buffer.toString does not. decoded = bytes.TrimPrefix(decoded, []byte{0xef, 0xbb, 0xbf}) } - return strings.ToValidUTF8(string(decoded), "\ufffd"), nil + return commons.DecodeUTF8(decoded), nil }}) } return WithFunctions(functions...) diff --git a/jmespath/testdata/utf8-v2.35.0.json b/jmespath/testdata/utf8-v2.35.0.json new file mode 100644 index 0000000..5244a83 --- /dev/null +++ b/jmespath/testdata/utf8-v2.35.0.json @@ -0,0 +1,125 @@ +{ + "reference": "2.35.0", + "cases": [ + { + "name": "plain", + "kind": "base64", + "input": "aGVsbG8=", + "expected": "hello" + }, + { + "name": "plain", + "kind": "gzip", + "input": "H4sIAAAAAAAACstIzcnJBwCGphA2BQAAAA==", + "expected": "hello" + }, + { + "name": "valid-unicode", + "kind": "base64", + "input": "5L2g5aW98J+YgA==", + "expected": "你好😀" + }, + { + "name": "valid-unicode", + "kind": "gzip", + "input": "H4sIAAAAAAAACnuyd8HTpXs/zJ/RAAArqvIaCgAAAA==", + "expected": "你好😀" + }, + { + "name": "invalid-bytes", + "kind": "base64", + "input": "//4=", + "expected": "��" + }, + { + "name": "invalid-bytes", + "kind": "gzip", + "input": "H4sIAAAAAAAACvv/DwCWMPiIAgAAAA==", + "expected": "��" + }, + { + "name": "truncated-sequence", + "kind": "base64", + "input": "4oI=", + "expected": "�" + }, + { + "name": "truncated-sequence", + "kind": "gzip", + "input": "H4sIAAAAAAAACnvUBACdYScuAgAAAA==", + "expected": "�" + }, + { + "name": "surrogate", + "kind": "base64", + "input": "7aCA", + "expected": "���" + }, + { + "name": "surrogate", + "kind": "gzip", + "input": "H4sIAAAAAAAACnu7oAEAKKXEHQMAAAA=", + "expected": "���" + }, + { + "name": "bom-text", + "kind": "base64", + "input": "77u/aGVsbG8=", + "expected": "hello" + }, + { + "name": "bom-text", + "kind": "gzip", + "input": "H4sIAAAAAAAACnu/e39Gak5OPgDFAw4GCAAAAA==", + "expected": "hello" + }, + { + "name": "double-bom", + "kind": "base64", + "input": "77u/77u/aGVsbG8=", + "expected": "hello" + }, + { + "name": "double-bom", + "kind": "gzip", + "input": "H4sIAAAAAAAACnu/e//73fszUnNy8gGjRa1dCwAAAA==", + "expected": "hello" + }, + { + "name": "bom-json", + "kind": "base64", + "input": "77u/eyJpZCI6MX0=", + "expected": "{\"id\":1}" + }, + { + "name": "bom-json", + "kind": "gzip", + "input": "H4sIAAAAAAAACnu/e3+1UmaKkpVhLQCjvv4fCwAAAA==", + "expected": "{\"id\":1}" + }, + { + "name": "invalid-json-string", + "kind": "base64", + "input": "Iv/+Ig==", + "expected": "\"��\"" + }, + { + "name": "invalid-json-string", + "kind": "gzip", + "input": "H4sIAAAAAAAAClP6/08JAJMBuMoEAAAA", + "expected": "\"��\"" + }, + { + "name": "partial-json-string", + "kind": "base64", + "input": "IuKCIg==", + "expected": "\"�\"" + }, + { + "name": "partial-json-string", + "kind": "gzip", + "input": "H4sIAAAAAAAAClN61KQEAEoHzF0EAAAA", + "expected": "\"�\"" + } + ] +} diff --git a/jmespath/utf8_test.go b/jmespath/utf8_test.go new file mode 100644 index 0000000..27f93e2 --- /dev/null +++ b/jmespath/utf8_test.go @@ -0,0 +1,36 @@ +package jmespath + +import ( + "encoding/json" + "os" + "reflect" + "testing" +) + +func TestPowertoolsUTF8Reference(t *testing.T) { + raw, err := os.ReadFile("testdata/utf8-v2.35.0.json") + if err != nil { + t.Fatal(err) + } + var fixture struct { + Cases []struct { + Name, Kind, Input string + Expected any + } + } + if err := json.Unmarshal(raw, &fixture); err != nil { + t.Fatal(err) + } + for _, item := range fixture.Cases { + t.Run(item.Kind+"/"+item.Name, func(t *testing.T) { + expression := "powertools_base64(@)" + if item.Kind == "gzip" { + expression = "powertools_base64_gzip(@)" + } + got, err := Search(expression, item.Input, WithPowertoolsFunctions()) + if err != nil || !reflect.DeepEqual(got, item.Expected) { + t.Fatalf("got %#v (%v), want %#v", got, err, item.Expected) + } + }) + } +} diff --git a/mkdocs.yml b/mkdocs.yml index 0eb77e7..abaea9d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -148,6 +148,7 @@ nav: - Documentation and CI: DOCUMENTATION.md - Verification: - Overview: VALIDATION.md + - Module review (2026-10-05): MODULE_REVIEW_2026_10_05.md - Local integration runner: LOCAL_INTEGRATION.md - Local acceptance: LOCAL_VALIDATION.md - Historical AWS acceptance: AWS_VALIDATION.md diff --git a/parameters/testdata/utf8-v2.35.0.json b/parameters/testdata/utf8-v2.35.0.json new file mode 100644 index 0000000..5ab2adf --- /dev/null +++ b/parameters/testdata/utf8-v2.35.0.json @@ -0,0 +1,103 @@ +{ + "reference": "2.35.0", + "cases": [ + { + "name": "plain", + "kind": "binary", + "input": "aGVsbG8=", + "expected": "hello" + }, + { + "name": "valid-unicode", + "kind": "binary", + "input": "5L2g5aW98J+YgA==", + "expected": "你好😀" + }, + { + "name": "invalid-bytes", + "kind": "binary", + "input": "//4=", + "expected": "��" + }, + { + "name": "truncated-sequence", + "kind": "binary", + "input": "4oI=", + "expected": "�" + }, + { + "name": "surrogate", + "kind": "binary", + "input": "7aCA", + "expected": "���" + }, + { + "name": "bom-text", + "kind": "binary", + "input": "77u/aGVsbG8=", + "expected": "hello" + }, + { + "name": "double-bom", + "kind": "binary", + "input": "77u/77u/aGVsbG8=", + "expected": "hello" + }, + { + "name": "bom-json", + "kind": "binary", + "input": "77u/eyJpZCI6MX0=", + "expected": "{\"id\":1}" + }, + { + "name": "bom-json", + "kind": "json-bytes", + "input": "77u/eyJpZCI6MX0=", + "expected": { + "id": 1 + } + }, + { + "name": "invalid-json-string", + "kind": "binary", + "input": "Iv/+Ig==", + "expected": "\"��\"" + }, + { + "name": "invalid-json-string", + "kind": "json-bytes", + "input": "Iv/+Ig==", + "expected": "��" + }, + { + "name": "partial-json-string", + "kind": "binary", + "input": "IuKCIg==", + "expected": "\"�\"" + }, + { + "name": "partial-json-string", + "kind": "json-bytes", + "input": "IuKCIg==", + "expected": "�" + }, + { + "name": "bom-base64-input-invalid-bytes", + "kind": "binary-bytes", + "input": "77u/Ly80PQ==", + "expected": "��" + }, + { + "name": "bom-base64-input-bom-text", + "kind": "binary-bytes", + "input": "77u/Nzd1L2FHVnNiRzg9", + "expected": "hello" + }, + { + "name": "bom-base64-input-double-bom", + "kind": "binary-bytes", + "input": "77u/Nzd1Lzc3dS9hR1ZzYkc4PQ==", + "expected": "hello" + } + ] +} diff --git a/parameters/transform.go b/parameters/transform.go index d204bb8..f709416 100644 --- a/parameters/transform.go +++ b/parameters/transform.go @@ -3,8 +3,9 @@ package parameters import ( "encoding/json" "fmt" - "github.com/rambow-cloud/powertools-lambda-go/commons" "strings" + + "github.com/rambow-cloud/powertools-lambda-go/commons" ) type Transform string @@ -36,7 +37,7 @@ func TransformValue(name string, value any, transform Transform) (any, error) { case string: raw = v case []byte: - raw = strings.ToValidUTF8(string(v), "\ufffd") + raw = strings.TrimPrefix(commons.DecodeUTF8(v), "\ufeff") default: return value, nil } @@ -59,7 +60,7 @@ func TransformValue(name string, value any, transform Transform) (any, error) { case string(Binary): var decoded []byte decoded, err = commons.FromBase64(raw, "base64") - result = strings.ToValidUTF8(string(decoded), "\ufffd") + result = strings.TrimPrefix(commons.DecodeUTF8(decoded), "\ufeff") default: err = fmt.Errorf("unsupported transform %q", transform) } diff --git a/parameters/utf8_test.go b/parameters/utf8_test.go new file mode 100644 index 0000000..987be29 --- /dev/null +++ b/parameters/utf8_test.go @@ -0,0 +1,53 @@ +package parameters_test + +import ( + "encoding/base64" + "encoding/json" + "os" + "reflect" + "testing" + + "github.com/rambow-cloud/powertools-lambda-go/parameters" +) + +func TestTransformUTF8Reference(t *testing.T) { + raw, err := os.ReadFile("testdata/utf8-v2.35.0.json") + if err != nil { + t.Fatal(err) + } + var fixture struct { + Cases []struct { + Name, Kind, Input string + Expected any + } + } + if err := json.Unmarshal(raw, &fixture); err != nil { + t.Fatal(err) + } + for _, item := range fixture.Cases { + t.Run(item.Kind+"/"+item.Name, func(t *testing.T) { + var input any = item.Input + transform, name := parameters.Binary, "fixture.binary" + if item.Kind == "json-bytes" { + transform, name = parameters.JSON, "fixture.json" + } + if item.Kind == "json-bytes" || item.Kind == "binary-bytes" { + var err error + input, err = base64.StdEncoding.DecodeString(item.Input) + if err != nil { + t.Fatal(err) + } + got, err := parameters.TransformValue(name, input, "") + if err != nil || !reflect.DeepEqual(got, input) { + t.Fatalf("raw byte input changed: %#v, %v", got, err) + } + } + for _, mode := range []parameters.Transform{transform, parameters.Auto} { + got, err := parameters.TransformValue(name, input, mode) + if err != nil || !reflect.DeepEqual(got, item.Expected) { + t.Errorf("%s: got %#v (%v), want %#v", mode, got, err, item.Expected) + } + } + }) + } +} diff --git a/parser/helpers.go b/parser/helpers.go index c40ee8b..6f362fd 100644 --- a/parser/helpers.go +++ b/parser/helpers.go @@ -38,11 +38,12 @@ var base64Value = Pipe(String(), SchemaFunc[any](func(_ context.Context, input a if reader, err := gzip.NewReader(bytes.NewReader(data)); err == nil { decompressed, readErr := io.ReadAll(reader) closeErr := reader.Close() - if readErr == nil && closeErr == nil && json.Unmarshal(decompressed, &decoded) == nil { + if readErr == nil && closeErr == nil && json.Unmarshal([]byte(commons.DecodeUTF8(decompressed)), &decoded) == nil { return decoded, nil, nil } } - text := strings.ToValidUTF8(string(data), "\ufffd") + // TextDecoder strips one leading BOM; gzip's Buffer.toString above retains it. + text := strings.TrimPrefix(commons.DecodeUTF8(data), "\ufeff") if json.Unmarshal([]byte(text), &decoded) != nil { decoded = text } diff --git a/parser/testdata/utf8-v2.35.0.json b/parser/testdata/utf8-v2.35.0.json new file mode 100644 index 0000000..02bcc1c --- /dev/null +++ b/parser/testdata/utf8-v2.35.0.json @@ -0,0 +1,127 @@ +{ + "reference": "2.35.0", + "cases": [ + { + "name": "plain", + "kind": "plain", + "input": "aGVsbG8=", + "expected": "hello" + }, + { + "name": "plain", + "kind": "gzip", + "input": "H4sIAAAAAAAACstIzcnJBwCGphA2BQAAAA==", + "expected": "\u001f�\b\u0000\u0000\u0000\u0000\u0000\u0000\n�H���\u0007\u0000��\u00106\u0005\u0000\u0000\u0000" + }, + { + "name": "valid-unicode", + "kind": "plain", + "input": "5L2g5aW98J+YgA==", + "expected": "你好😀" + }, + { + "name": "valid-unicode", + "kind": "gzip", + "input": "H4sIAAAAAAAACnuyd8HTpXs/zJ/RAAArqvIaCgAAAA==", + "expected": "\u001f�\b\u0000\u0000\u0000\u0000\u0000\u0000\n{�w�ӥ{?̟�\u0000\u0000+��\u001a\n\u0000\u0000\u0000" + }, + { + "name": "invalid-bytes", + "kind": "plain", + "input": "//4=", + "expected": "��" + }, + { + "name": "invalid-bytes", + "kind": "gzip", + "input": "H4sIAAAAAAAACvv/DwCWMPiIAgAAAA==", + "expected": "\u001f�\b\u0000\u0000\u0000\u0000\u0000\u0000\n��\u000f\u0000�0��\u0002\u0000\u0000\u0000" + }, + { + "name": "truncated-sequence", + "kind": "plain", + "input": "4oI=", + "expected": "�" + }, + { + "name": "truncated-sequence", + "kind": "gzip", + "input": "H4sIAAAAAAAACnvUBACdYScuAgAAAA==", + "expected": "\u001f�\b\u0000\u0000\u0000\u0000\u0000\u0000\n{�\u0004\u0000�a'.\u0002\u0000\u0000\u0000" + }, + { + "name": "surrogate", + "kind": "plain", + "input": "7aCA", + "expected": "���" + }, + { + "name": "surrogate", + "kind": "gzip", + "input": "H4sIAAAAAAAACnu7oAEAKKXEHQMAAAA=", + "expected": "\u001f�\b\u0000\u0000\u0000\u0000\u0000\u0000\n{��\u0001\u0000(��\u001d\u0003\u0000\u0000\u0000" + }, + { + "name": "bom-text", + "kind": "plain", + "input": "77u/aGVsbG8=", + "expected": "hello" + }, + { + "name": "bom-text", + "kind": "gzip", + "input": "H4sIAAAAAAAACnu/e39Gak5OPgDFAw4GCAAAAA==", + "expected": "\u001f�\b\u0000\u0000\u0000\u0000\u0000\u0000\n{�{FjNN>\u0000�\u0003\u000e\u0006\b\u0000\u0000\u0000" + }, + { + "name": "double-bom", + "kind": "plain", + "input": "77u/77u/aGVsbG8=", + "expected": "hello" + }, + { + "name": "double-bom", + "kind": "gzip", + "input": "H4sIAAAAAAAACnu/e//73fszUnNy8gGjRa1dCwAAAA==", + "expected": "\u001f�\b\u0000\u0000\u0000\u0000\u0000\u0000\n{�{����3Rsr�\u0001�E�]\u000b\u0000\u0000\u0000" + }, + { + "name": "bom-json", + "kind": "plain", + "input": "77u/eyJpZCI6MX0=", + "expected": { + "id": 1 + } + }, + { + "name": "bom-json", + "kind": "gzip", + "input": "H4sIAAAAAAAACnu/e3+1UmaKkpVhLQCjvv4fCwAAAA==", + "expected": "\u001f�\b\u0000\u0000\u0000\u0000\u0000\u0000\n{�{�Rf���a-\u0000���\u001f\u000b\u0000\u0000\u0000" + }, + { + "name": "invalid-json-string", + "kind": "plain", + "input": "Iv/+Ig==", + "expected": "��" + }, + { + "name": "invalid-json-string", + "kind": "gzip", + "input": "H4sIAAAAAAAAClP6/08JAJMBuMoEAAAA", + "expected": "��" + }, + { + "name": "partial-json-string", + "kind": "plain", + "input": "IuKCIg==", + "expected": "�" + }, + { + "name": "partial-json-string", + "kind": "gzip", + "input": "H4sIAAAAAAAAClN61KQEAEoHzF0EAAAA", + "expected": "�" + } + ] +} diff --git a/parser/utf8_test.go b/parser/utf8_test.go new file mode 100644 index 0000000..ac7d71b --- /dev/null +++ b/parser/utf8_test.go @@ -0,0 +1,35 @@ +package parser_test + +import ( + "context" + "encoding/json" + "os" + "reflect" + "testing" + + "github.com/rambow-cloud/powertools-lambda-go/parser" +) + +func TestBase64UTF8Reference(t *testing.T) { + raw, err := os.ReadFile("testdata/utf8-v2.35.0.json") + if err != nil { + t.Fatal(err) + } + var fixture struct { + Cases []struct { + Name, Kind, Input string + Expected any + } + } + if err := json.Unmarshal(raw, &fixture); err != nil { + t.Fatal(err) + } + for _, item := range fixture.Cases { + t.Run(item.Kind+"/"+item.Name, func(t *testing.T) { + got, err := parser.Parse(context.Background(), item.Input, parser.Base64Encoded(parser.Unknown())) + if err != nil || !reflect.DeepEqual(got, item.Expected) { + t.Fatalf("got %#v (%v), want %#v", got, err, item.Expected) + } + }) + } +} diff --git a/tools/reference/generate-utf8-decoding.mjs b/tools/reference/generate-utf8-decoding.mjs new file mode 100644 index 0000000..8dd3223 --- /dev/null +++ b/tools/reference/generate-utf8-decoding.mjs @@ -0,0 +1,46 @@ +// Capture the pinned UTF-8 contracts across the three text-decoding utilities. +import { writeFileSync } from 'node:fs'; +import { resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; +import { gzipSync } from 'node:zlib'; +import { z } from 'zod'; +import { Base64Encoded } from '@aws-lambda-powertools/parser/helpers'; +import { search } from '@aws-lambda-powertools/jmespath'; +import { PowertoolsFunctions } from '@aws-lambda-powertools/jmespath/functions'; +import { transformValue } from './node_modules/@aws-lambda-powertools/parameters/lib/esm/base/transformValue.js'; + +// An optional output root allows reference dependencies to stay in an isolated directory. +const root = process.argv[2] ? resolve(process.argv[2]) : fileURLToPath(new URL('../..', import.meta.url)); +const samples = [ + ['plain', Buffer.from('hello')], + ['valid-unicode', Buffer.from('你好😀')], + ['invalid-bytes', Buffer.from([0xff, 0xfe])], + ['truncated-sequence', Buffer.from([0xe2, 0x82])], + ['surrogate', Buffer.from([0xed, 0xa0, 0x80])], + ['bom-text', Buffer.from('\ufeffhello')], + ['double-bom', Buffer.from('\ufeff\ufeffhello')], + ['bom-json', Buffer.from('\ufeff{"id":1}')], + ['invalid-json-string', Buffer.from([34, 0xff, 0xfe, 34])], + ['partial-json-string', Buffer.from([34, 0xe2, 0x82, 34])], +]; +const cases = {parser: [], jmespath: [], parameters: []}; +for (const [name, bytes] of samples) { + const input = bytes.toString('base64'); + const gzip = gzipSync(bytes).toString('base64'); + cases.parser.push({name, kind: 'plain', input, expected: Base64Encoded(z.unknown()).parse(input)}); + cases.parser.push({name, kind: 'gzip', input: gzip, expected: Base64Encoded(z.unknown()).parse(gzip)}); + cases.jmespath.push({name, kind: 'base64', input, expected: search('powertools_base64(@)', input, {customFunctions: new PowertoolsFunctions()})}); + cases.jmespath.push({name, kind: 'gzip', input: gzip, expected: search('powertools_base64_gzip(@)', gzip, {customFunctions: new PowertoolsFunctions()})}); + cases.parameters.push({name, kind: 'binary', input, expected: transformValue(input, 'binary', true, 'fixture')}); + if (name.endsWith('json-string') || name === 'bom-json') { + cases.parameters.push({name, kind: 'json-bytes', input, expected: transformValue(new Uint8Array(bytes), 'json', true, 'fixture')}); + } +} +for (const [name, bytes] of samples.filter(([name]) => ['invalid-bytes', 'bom-text', 'double-bom'].includes(name))) { + const raw = Buffer.from('\ufeff' + bytes.toString('base64')); + cases.parameters.push({name: 'bom-base64-input-' + name, kind: 'binary-bytes', input: raw.toString('base64'), expected: transformValue(new Uint8Array(raw), 'binary', true, 'fixture')}); +} +for (const [module, entries] of Object.entries(cases)) { + writeFileSync(resolve(root, module, 'testdata', 'utf8-v2.35.0.json'), JSON.stringify({reference: '2.35.0', cases: entries}, null, 2) + '\n'); + console.log(`${module}: ${entries.length} pinned UTF-8 cases`); +} diff --git a/tools/reference/package.json b/tools/reference/package.json index 47b7069..45a6f9a 100644 --- a/tools/reference/package.json +++ b/tools/reference/package.json @@ -3,6 +3,7 @@ "private": true, "type": "module", "scripts": { + "fixtures:utf8": "node generate-utf8-decoding.mjs", "fixtures:logger:parity": "node generate-logger-parity.mjs", "fixtures:graphql": "node generate-appsync-graphql.mjs", "fixtures:bedrock": "node generate-bedrock.mjs",