diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md new file mode 100644 index 0000000..d8b84b4 --- /dev/null +++ b/.github/CONTRIBUTING.md @@ -0,0 +1,110 @@ +# Contributing to evmsync + +This page covers how to build and test evmsync, where code goes, the rules that keep a consumer's +state equal to the final canonical chain, and how to send a change. + +For a bug fix or a small change, send a pull request. For a new feature, a new package or a change +in behavior, open an issue first, so we can agree on the design before you write the code. + +## Development setup + +You need Go 1.26 or newer and golangci-lint v2.14.0. golangci-lint v2.7.2 and other releases built +with Go 1.25 refuse to load a `go 1.26.0` module. + +```shell +go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.14.0 +``` + +Before every commit, run the gate from the repository root: + +```shell +scripts/gate.sh +``` + +It runs the same checks as CI on your working tree: gofmt, `go mod tidy -diff`, `go vet`, the +build and tests with `CGO_ENABLED=0`, the tests with the race detector, the nested modules and +golangci-lint. It prints one `PASS`, `FAIL` or `SKIP` line per step and ends with `RESULT GREEN` (exit 0) +or `RESULT RED` (exit 1). Three environment variables change what it runs: + +| Variable | Default | Meaning | +|---|---|---| +| `GATE_TCS` | `local` | Space-separated `GOTOOLCHAIN` values to run the steps with, for example `local go1.26.0` | +| `GATE_LOGS` | `tmp/gate` under the repository root | Directory for each step's log, `-.log` | +| `GATE_LINT` | `golangci-lint` | The golangci-lint binary; the gate checks that it is v2.14.0 | + +CI runs golangci-lint, a build with `CGO_ENABLED=0` on the latest Go release, and the tests with the +race detector on the Go version in `go.mod`. It fails when total test coverage drops below 95%, the +threshold in `.testcoverage.yml`. + +## Repository layout + +Each directory is one package with one job; `pgstore`, `e2e` and `examples` are nested modules +with their own `go.mod`. The version is the release a package first ships in. + +| Package | Holds | Version | +|---|---|---| +| `chain` | Block references, headers, logs, filters, blooms and the `Source` interface | v0.1 | +| `chainlog` | The canonical log: entries, offsets, the `Log` interface, the validator `State`, the record codec and an in-memory log | v0.1 | +| `chainlog/wal` | A `chainlog.Log` stored in segment files | v0.1 | +| `chainlog/logtest` | The conformance suite for `chainlog.Log` implementations | v0.1 | +| `tracker` | Reorg detection and finality | v0.1 | +| `consumer` | The `Consumer` contract, `Run` and `Memory` | v0.1 | +| `internal/archtest` | Tests for the import rules, license headers and apply-path rules | v0.1 | +| `internal/fakechain` | A seeded fake chain and a fake `Source` for tests | v0.1 | +| `internal/invariant` | The reorg invariant property test | v0.1 | +| `rpc` | The JSON-RPC client that implements `chain.Source` | v0.2 | +| `ingest` | Backfill, realtime sync and validation | v0.2 | +| `evmsync` (root) | `App`, typed handler dispatch and the `Store` interface | v0.3 | +| `memstore` | An in-memory store with an undo journal | v0.3 | +| `cmd/evmsync` | The `evmsync gen` command | v0.3 | +| `internal/gen` | ABI to Go source generation | v0.3 | +| `pgstore` | The PostgreSQL store, a nested module | v0.4 | +| `e2e` | End-to-end tests against Anvil, a nested module | v0.2 | +| `examples` | Runnable examples, a nested module | v0.4 | + +Which package may import which is fixed, and `internal/archtest` fails the tests when an import +breaks the rules. A new package needs a design change first: open an issue that proposes it. The +change that adds it also adds it to this table and to the inventory in `internal/archtest`. + +## Rules for apply-path code + +The apply path is the code that turns log entries into state: `chain`, `chainlog`, `tracker`, +`consumer` and `memstore`. Replaying the same log must always build the same state, so this code +follows four rules, and `internal/archtest` checks them. The dispatch code of the root `evmsync` +package and `pgstore` follows the same rules; review checks those by hand. + +- **No wall clock.** Read time from the block (`chain.BlockRef.Time`), never from `time.Now`, + `time.Since`, `time.Until`, `time.Sleep`, timers or tickers. +- **No global randomness.** No functions of `math/rand` or `math/rand/v2`. +- **No goroutines.** No `go` statement, except inside `consumer.Run`. +- **No map order.** Range over sorted keys, never over a map directly. Wrap `maps.Keys` and + `maps.Values` in `slices.Sorted`; instead of `maps.All`, range over the sorted keys. + +When the order of a map range never reaches state, an entry, an error or a log line, annotate the +loop with a reason: + +```go +for _, v := range balances { //evmsync:maporder addition does not depend on order + total += v +} +``` + +The directive applies to its own line when code precedes it, otherwise to the next line. It needs a +reason, it must suppress a finding, and it has no space after `//`. + +## License header + +Every `.go` file, generated files included, starts with these two lines and a blank line. Files +under `testdata/` are the exception. + +```go +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 +``` + +## Commits + +Commit messages follow [Conventional Commits](https://www.conventionalcommits.org/): a type, the +package as the scope, and a short subject, for example `feat(chainlog): add the record codec` or +`fix(tracker): stop at the finalized block`. Changes to `go.mod` and the lint config use the +`build` type, changes to workflows the `ci` type. diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml new file mode 100644 index 0000000..96a904e --- /dev/null +++ b/.github/workflows/build.yml @@ -0,0 +1,80 @@ +name: build +on: [ push, pull_request ] +permissions: + contents: read +jobs: + golangci: + name: lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + - name: Run golangci-lint + uses: golangci/golangci-lint-action@v9 + with: + version: v2.14.0 + + build: + name: Go build + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version: stable # the test job covers the minimum version in go.mod + - name: Build + env: + CGO_ENABLED: '0' + run: go build -v ./... + + test: + name: Go test + runs-on: ubuntu-latest + outputs: + coverage: ${{ steps.coverage.outputs.total-coverage }} + steps: + - uses: actions/checkout@v7 + - uses: actions/setup-go@v7 + with: + go-version-file: go.mod + - name: Test + run: go test -race -covermode=atomic -coverprofile=cover.out ./... + + - name: Check coverage + id: coverage + uses: vladopajic/go-test-coverage@f94bcf0d6b9fa5fb8b783830b22648f6c17475e2 # v2.19.0 + with: + config: ./.testcoverage.yml + + - name: Coverage summary + if: ${{ !cancelled() && steps.coverage.outputs.report != '' }} + env: + REPORT: ${{ fromJSON(steps.coverage.outputs.report) }} + run: | + { + echo '```' + echo "$REPORT" + echo '```' + } >> "$GITHUB_STEP_SUMMARY" + + coverage-badge: + name: Coverage badge + needs: test + if: github.event_name == 'push' && github.ref_name == 'main' + runs-on: ubuntu-latest + permissions: + issues: write # the README coverage badge shows the title of issue #95 + steps: + - name: Update coverage badge + env: + GH_TOKEN: ${{ github.token }} + COVERAGE: ${{ needs.test.outputs.coverage }} + run: | + test -n "$COVERAGE" + title="$COVERAGE%" + current=$(gh issue view 95 --repo "$GITHUB_REPOSITORY" --json title --jq .title) + if [ "$current" != "$title" ]; then + gh issue edit 95 --repo "$GITHUB_REPOSITORY" --title "$title" + fi diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml new file mode 100644 index 0000000..7175b4e --- /dev/null +++ b/.github/workflows/nightly.yml @@ -0,0 +1,29 @@ +name: nightly + +on: + schedule: + - cron: '0 3 * * *' + workflow_dispatch: + +permissions: + contents: read + +env: + GOTOOLCHAIN: local + +jobs: + invariant: + name: reorg invariant (100,000 seeds) + runs-on: ubuntu-latest + timeout-minutes: 150 + env: + GOWORK: 'off' + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: '1.27.x' + - name: reorg invariant + run: | + EVMSYNC_INVARIANT_SEEDS=100000 EVMSYNC_INVARIANT_BASE=$(date -u +%Y%m%d)000000 \ + go test ./internal/invariant -run '^TestReorgInvariant$' -count=1 -timeout 120m diff --git a/.gitignore b/.gitignore index 67c87ef..50f3a44 100644 --- a/.gitignore +++ b/.gitignore @@ -30,11 +30,6 @@ go.work.sum *.swo *~ -# AI coding tools (local settings) -.cursor/ -.claude/settings.local.json -.claude/worktrees/ - # OS files .DS_Store Thumbs.db diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..3bdf094 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,50 @@ +version: "2" +linters: + enable: + - bodyclose + - dogsled + - gochecknoinits + - gocritic + - gosec + - nakedret + - staticcheck + - unconvert + - unparam + - whitespace + disable: + - dupl + - gochecknoglobals + - lll + settings: + errcheck: + check-type-assertions: true + goconst: + min-len: 2 + min-occurrences: 2 + gocritic: + disabled-checks: + - ifElseChain + gocyclo: + min-complexity: 25 + nakedret: + max-func-lines: 15 + exclusions: + generated: lax + presets: + - comments + - common-false-positives + - legacy + - std-error-handling + paths: + - third_party$ + - builtin$ + - examples$ +formatters: + enable: + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/.testcoverage.yml b/.testcoverage.yml new file mode 100644 index 0000000..8b1eebe --- /dev/null +++ b/.testcoverage.yml @@ -0,0 +1,7 @@ +# Settings for go-test-coverage, used by CI (.github/workflows/build.yml). +# See https://github.com/vladopajic/go-test-coverage for all options. +profile: cover.out + +threshold: + # Minimum coverage of the whole module, in percent. + total: 95 diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..261eeb9 --- /dev/null +++ b/LICENSE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/README.md b/README.md index a33d296..38a7a25 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,12 @@ only handle forward. your database. - **It's read-only.** It never sends transactions. +## Roadmap + +evmsync is built in versions, and v0.4.0 is the first public release. +[The roadmap issue](https://github.com/hmdsefi/evmsync/issues/84) lists each +version's epics and shows their progress. + ## Built with [gograph](https://github.com/hmdsefi/gograph) orders handlers by their table diff --git a/chain/bench_test.go b/chain/bench_test.go new file mode 100644 index 0000000..e09e299 --- /dev/null +++ b/chain/bench_test.go @@ -0,0 +1,90 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chain + +import ( + "os" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" +) + +func BenchmarkFilterMatch(b *testing.B) { + f := Filter{ + Addresses: []common.Address{usdc}, + Topics: [][]common.Hash{{transfer}, nil, {topA, topB}}, + } + l := Log{Address: usdc, Topics: []common.Hash{transfer, topX, topB}} + if !f.Match(l) { + b.Fatal("benchmark log does not match") + } + b.ReportAllocs() + for b.Loop() { + f.Match(l) + } +} + +func BenchmarkBloomTest(b *testing.B) { + bl := bloomVector() + b.ReportAllocs() + for b.Loop() { + bl.Test(transfer[:]) + } +} + +func BenchmarkBloomMayContain(b *testing.B) { + bl := bloomVector() + f := Filter{Addresses: []common.Address{usdc}, Topics: [][]common.Hash{{transfer}}} + b.ReportAllocs() + for b.Loop() { + bl.MayContain(f) + } +} + +func BenchmarkBloomOf(b *testing.B) { + logs := make([]Log, 10) + for i := range logs { + logs[i] = Log{ + Address: usdc, + Topics: []common.Hash{transfer, one32, common.BytesToHash([]byte{byte(i)})}, + } + } + b.ReportAllocs() + for b.Loop() { + BloomOf(logs) + } +} + +func TestPerf(t *testing.T) { + if os.Getenv("EVMSYNC_PERF") != "1" || raceEnabled { + t.Skip("set EVMSYNC_PERF=1 and run without -race to measure the targets") + } + tests := []struct { + name string + bench func(*testing.B) + maxTime time.Duration + maxAllocs int64 // -1: not asserted + }{ + {"FilterMatch", BenchmarkFilterMatch, 30 * time.Nanosecond, 0}, + {"BloomTest", BenchmarkBloomTest, time.Microsecond, 1}, + {"BloomMayContain", BenchmarkBloomMayContain, 2 * time.Microsecond, 2}, + {"BloomOf", BenchmarkBloomOf, 40 * time.Microsecond, -1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := testing.Benchmark(tt.bench) + if r.N == 0 { + t.Fatal("benchmark did not run") + } + t.Logf("%d ns/op, %d allocs/op", r.NsPerOp(), r.AllocsPerOp()) + if got := time.Duration(r.NsPerOp()); got > tt.maxTime { + t.Errorf("%v/op, want at most %v", got, tt.maxTime) + } + if tt.maxAllocs >= 0 && r.AllocsPerOp() > tt.maxAllocs { + t.Errorf("%d allocs/op, want at most %d", r.AllocsPerOp(), tt.maxAllocs) + } + }) + } +} diff --git a/chain/block.go b/chain/block.go new file mode 100644 index 0000000..797616e --- /dev/null +++ b/chain/block.go @@ -0,0 +1,50 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chain + +import ( + "strconv" + + "github.com/ethereum/go-ethereum/common" +) + +// BlockRef identifies a block and links it to its parent. Two refs name the same block only when +// their hashes are equal. The zero BlockRef means "no block". +type BlockRef struct { + Number uint64 + Hash common.Hash + Parent common.Hash + Time uint64 // unix seconds, from the block header +} + +// String returns the decimal number, one space and Hash.Hex(), for example +// "123 0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef". +func (r BlockRef) String() string { + return strconv.FormatUint(r.Number, 10) + " " + r.Hash.Hex() +} + +// Header is the part of a block header evmsync uses. +type Header struct { + Ref BlockRef + Bloom Bloom // the block's logsBloom; all zero when the source does not know it +} + +// Log is one event log of a block. Removed logs never appear: a reorg is a chainlog Revert entry. +type Log struct { + Address common.Address + Topics []common.Hash // 0 to 4 topics; nil when there are none + Data []byte // nil when empty + BlockNumber uint64 // the number of the block holding the log + BlockHash common.Hash // the hash of the block holding the log + TxHash common.Hash + TxIndex uint32 // index of the transaction in the block + Index uint32 // index of the log in the block +} + +// Block is a block reference and the logs of that block that match the ingest filter, in +// increasing Index order. Logs is nil when no log matches. +type Block struct { + Ref BlockRef + Logs []Log +} diff --git a/chain/block_test.go b/chain/block_test.go new file mode 100644 index 0000000..0cb1091 --- /dev/null +++ b/chain/block_test.go @@ -0,0 +1,79 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chain + +import ( + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +var ( + usdc = common.HexToAddress("0x2791Bca1f2de4661ED88A30C99A7a9449Aa84174") + transfer = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") + one32 = common.HexToHash("0x01") + zero32 = common.Hash{} +) + +func TestBlockRefString(t *testing.T) { + tests := []struct { + name string + ref BlockRef + want string + }{ + { + name: "number and hash", + ref: BlockRef{Number: 123, Hash: transfer}, + want: "123 0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + }, + { + name: "zero ref", + ref: BlockRef{}, + want: "0 0x" + strings.Repeat("0", 64), + }, + { + name: "parent and time not printed", + ref: BlockRef{Number: 123, Hash: transfer, Parent: one32, Time: 1_700_000_000}, + want: "123 0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.ref.String(); got != tt.want { + t.Errorf("String() = %q, want %q", got, tt.want) + } + }) + } +} + +func TestComparable(t *testing.T) { + h, p := transfer, one32 + base := BlockRef{1, h, p, 5} + if base != (BlockRef{1, h, p, 5}) { + t.Fatal("equal refs compare unequal") + } + changed := []BlockRef{ + {2, h, p, 5}, + {1, p, p, 5}, + {1, h, h, 5}, + {1, h, p, 6}, + } + for i, r := range changed { + if base == r { + t.Errorf("ref with field %d changed compares equal", i) + } + } + var h1, h2 Header + if h1 != h2 { + t.Error("zero headers compare unequal") + } + h2.Bloom[0] = 1 + if h1 == h2 { + t.Error("headers with different blooms compare equal") + } + if h1.Bloom == h2.Bloom { + t.Error("different blooms compare equal") + } +} diff --git a/chain/bloom.go b/chain/bloom.go new file mode 100644 index 0000000..681fcf8 --- /dev/null +++ b/chain/bloom.go @@ -0,0 +1,117 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chain + +import ( + "hash" + "sync" + + "github.com/ethereum/go-ethereum/common" + "golang.org/x/crypto/sha3" +) + +// Bloom is the 2048-bit logsBloom of a block header, in header byte order. Test, MayContain and +// IsZero only read b and are safe for concurrent use; Add writes b. +type Bloom [256]byte + +// keccak is a reusable Keccak-256 state with its output buffer. +type keccak struct { + h hash.Hash + sum [32]byte +} + +// keccakPool keeps Add and Test free of per-call allocations. +var keccakPool = sync.Pool{ + New: func() any { return &keccak{h: sha3.NewLegacyKeccak256()} }, +} + +// bloomBits returns the byte index and bit mask of the three bloom bits for data, as go-ethereum +// computes them: three 11-bit values from the first six bytes of Keccak-256(data). +func bloomBits(data []byte) (idx [3]int, mask [3]byte) { + k := keccakPool.Get().(*keccak) //nolint:errcheck // the pool only holds *keccak + k.h.Reset() + k.h.Write(data) + h := k.h.Sum(k.sum[:0]) + for i := range 3 { + j := 2 * i + mask[i] = 1 << (h[j+1] & 0x7) + idx[i] = 255 - int((uint16(h[j])<<8|uint16(h[j+1]))&0x7ff)>>3 + } + keccakPool.Put(k) + return idx, mask +} + +// Add sets the three bits of data, as Ethereum does for each log address and topic. +func (b *Bloom) Add(data []byte) { + idx, mask := bloomBits(data) + for i := range idx { + b[idx[i]] |= mask[i] + } +} + +// Test reports whether the three bits of data are all set. A false result proves that data was +// never added. +func (b Bloom) Test(data []byte) bool { + idx, mask := bloomBits(data) + for i := range idx { + if b[idx[i]]&mask[i] == 0 { + return false + } + } + return true +} + +// MayContain reports whether a block with bloom b may hold a log that passes f. false proves that +// no log of the block passes f; true may be a false positive. Callers that treat an all-zero bloom +// as "unknown" check IsZero first: MayContain does not. +func (b Bloom) MayContain(f Filter) bool { + if len(f.Addresses) > 0 && !b.anyAddress(f.Addresses) { + return false + } + for _, pos := range f.Topics { + if len(pos) > 0 && !b.anyTopic(pos) { + return false + } + } + return true +} + +// anyAddress reports whether Test is true for one of as. It indexes the slice instead of ranging +// over copies, so the hashed bytes do not escape to the heap. +func (b *Bloom) anyAddress(as []common.Address) bool { + for i := range as { + if b.Test(as[i][:]) { + return true + } + } + return false +} + +// anyTopic reports whether Test is true for one of ts. +func (b *Bloom) anyTopic(ts []common.Hash) bool { + for i := range ts { + if b.Test(ts[i][:]) { + return true + } + } + return false +} + +// IsZero reports whether all 2048 bits are zero. +func (b Bloom) IsZero() bool { + return b == Bloom{} +} + +// BloomOf returns the bloom of logs: Add of every log's address and of each of its topics. +func BloomOf(logs []Log) Bloom { + var b Bloom + for i := range logs { + l := &logs[i] + b.Add(l.Address[:]) + for j := range l.Topics { + b.Add(l.Topics[j][:]) + } + } + return b +} diff --git a/chain/bloom_test.go b/chain/bloom_test.go new file mode 100644 index 0000000..1e1bff6 --- /dev/null +++ b/chain/bloom_test.go @@ -0,0 +1,260 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chain + +import ( + "encoding/hex" + "math/rand/v2" + "reflect" + "sync" + "testing" + + "github.com/ethereum/go-ethereum/common" + "golang.org/x/crypto/sha3" +) + +// nonZero returns the bloom's non-zero bytes by index. +func nonZero(b Bloom) map[int]byte { + m := map[int]byte{} + for i, v := range b { + if v != 0 { + m[i] = v + } + } + return m +} + +// bloomVector returns the bloom of usdc with topics transfer and one32. +func bloomVector() Bloom { + return BloomOf([]Log{{Address: usdc, Topics: []common.Hash{transfer, one32}}}) +} + +func TestBloomVectors(t *testing.T) { + tests := []struct { + name string + data []byte + prefix string + want map[int]byte + }{ + {"empty", nil, "c5d2460186f7", map[int]byte{33: 0x80, 63: 0x02, 69: 0x04}}, + {"usdc", usdc[:], "80e7ec40fbf0", map[int]byte{119: 0x01, 129: 0x01, 227: 0x80}}, + {"transfer", transfer[:], "ada389e1fc24", map[int]byte{75: 0x08, 123: 0x10, 195: 0x02}}, + {"address 0x01", other[:], "146828805631", map[int]byte{57: 0x02, 114: 0x01, 239: 0x01}}, + {"zero32", zero32[:], "290decd9548b", map[int]byte{100: 0x02, 110: 0x08, 222: 0x20}}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + h := sha3.NewLegacyKeccak256() + h.Write(tt.data) + if got := hex.EncodeToString(h.Sum(nil)[:6]); got != tt.prefix { + t.Fatalf("Keccak-256 prefix = %s, want %s", got, tt.prefix) + } + var b Bloom + b.Add(tt.data) + if got := nonZero(b); !reflect.DeepEqual(got, tt.want) { + t.Errorf("non-zero bytes after Add = %v, want %v", got, tt.want) + } + if !b.Test(tt.data) { + t.Error("Test after Add = false, want true") + } + }) + } + + b := bloomVector() + want := map[int]byte{ + 61: 0x04, 75: 0x08, 85: 0x04, 119: 0x01, 123: 0x10, + 129: 0x01, 195: 0x02, 222: 0x40, 227: 0x80, + } + if got := nonZero(b); !reflect.DeepEqual(got, want) { + t.Errorf("BloomOf non-zero bytes = %v, want %v", got, want) + } + if BloomOf(nil) != (Bloom{}) { + t.Error("BloomOf(nil) is not the zero bloom") + } +} + +func TestBloomOfTest(t *testing.T) { + b := bloomVector() + tests := []struct { + name string + data []byte + want bool + }{ + {"usdc", usdc[:], true}, + {"transfer", transfer[:], true}, + {"one32", one32[:], true}, + {"zero32", zero32[:], false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := b.Test(tt.data); got != tt.want { + t.Errorf("Test = %v, want %v", got, tt.want) + } + }) + } +} + +func TestBloomTestNeedsAllBits(t *testing.T) { + // usdc sets bit masks 0x01, 0x01 and 0x80 in bytes 119, 129 and 227. + for _, idx := range []int{119, 129, 227} { + var b Bloom + b.Add(usdc[:]) + b[idx] = 0 + if b.Test(usdc[:]) { + t.Errorf("Test(usdc) with byte %d cleared = true, want false", idx) + } + } +} + +func TestMayContain(t *testing.T) { + b := bloomVector() + tests := []struct { + name string + bloom Bloom + f Filter + want bool + }{ + {"zero filter", b, Filter{}, true}, + {"address", b, Filter{Addresses: []common.Address{usdc}}, true}, + {"other address", b, Filter{Addresses: []common.Address{other}}, false}, + {"other or address", b, Filter{Addresses: []common.Address{other, usdc}}, true}, + {"topic", b, Filter{Topics: [][]common.Hash{{transfer}}}, true}, + {"topic in position 1", b, Filter{Topics: [][]common.Hash{nil, {one32}}}, true}, + {"absent topic", b, Filter{Topics: [][]common.Hash{{zero32}}}, false}, + {"absent topic in position 1", b, Filter{Topics: [][]common.Hash{nil, {zero32}}}, false}, + { + "two absent addresses", + b, + Filter{Addresses: []common.Address{other, other2}}, + false, + }, + {"absent or present topic", b, Filter{Topics: [][]common.Hash{{zero32, transfer}}}, true}, + {"four wildcards", b, Filter{Topics: [][]common.Hash{nil, nil, nil, nil}}, true}, + { + "address and absent topic", + b, + Filter{Addresses: []common.Address{usdc}, Topics: [][]common.Hash{{zero32}}}, + false, + }, + {"zero bloom, zero filter", Bloom{}, Filter{}, true}, + {"zero bloom, address", Bloom{}, Filter{Addresses: []common.Address{usdc}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.bloom.MayContain(tt.f); got != tt.want { + t.Errorf("MayContain = %v, want %v", got, tt.want) + } + }) + } +} + +func TestIsZero(t *testing.T) { + if !(Bloom{}).IsZero() { + t.Error("Bloom{}.IsZero() = false, want true") + } + for i := range len(Bloom{}) { + var b Bloom + b[i] = 1 + if b.IsZero() { + t.Errorf("bloom with byte %d set: IsZero() = true, want false", i) + } + } +} + +func TestBloomAllocs(t *testing.T) { + if raceEnabled { + t.Skip("the race detector makes sync.Pool drop items") + } + b := bloomVector() + logs := []Log{{Address: usdc, Topics: []common.Hash{transfer}}} + f := Filter{Addresses: []common.Address{usdc}, Topics: [][]common.Hash{{transfer}}} + tests := []struct { + name string + fn func() + }{ + {"Add", func() { + var nb Bloom + nb.Add(logs[0].Address[:]) + }}, + {"Test", func() { b.Test(logs[0].Topics[0][:]) }}, + {"MayContain", func() { b.MayContain(f) }}, + {"BloomOf", func() { BloomOf(logs) }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if allocs := testing.AllocsPerRun(100, tt.fn); allocs != 0 { + t.Errorf("%v allocs per call, want 0", allocs) + } + }) + } +} + +func TestBloomConcurrent(t *testing.T) { + logs := []Log{{Address: usdc, Topics: []common.Hash{transfer, one32}}} + want := bloomVector() + errs := make(chan string, 8) + var wg sync.WaitGroup + for range 8 { + wg.Go(func() { + for range 500 { + b := BloomOf(logs) + switch { + case b != want: + errs <- "BloomOf differs from the sequential result" + return + case !b.Test(usdc[:]): + errs <- "Test(usdc) = false, want true" + return + case b.Test(zero32[:]): + errs <- "Test(zero32) = true, want false" + return + } + } + }) + } + wg.Wait() + close(errs) + for e := range errs { + t.Error(e) + } +} + +func TestBloomNoFalseNegatives(t *testing.T) { + r := rand.New(rand.NewPCG(1, 2)) //nolint:gosec // a fixed seed for reproducible cases, not for secrets + addrs := []common.Address{usdc, other, common.HexToAddress("0x02"), common.HexToAddress("0x03")} + topics := []common.Hash{transfer, one32, zero32, topA, topB, topX} + matched := 0 + for range 10_000 { + logs := make([]Log, 1+r.IntN(5)) + for i := range logs { + logs[i].Address = addrs[r.IntN(len(addrs))] + for range r.IntN(5) { + logs[i].Topics = append(logs[i].Topics, topics[r.IntN(len(topics))]) + } + } + var f Filter + for range r.IntN(3) { + f.Addresses = append(f.Addresses, addrs[r.IntN(len(addrs))]) + } + for range r.IntN(5) { + var pos []common.Hash + for range r.IntN(3) { + pos = append(pos, topics[r.IntN(len(topics))]) + } + f.Topics = append(f.Topics, pos) + } + for _, l := range logs { + if f.Match(l) { + matched++ + if !BloomOf(logs).MayContain(f) { + t.Fatalf("false negative: logs %v, filter %v", logs, f) + } + break + } + } + } + if matched == 0 { + t.Fatal("no generated case matched; the test checks nothing") + } +} diff --git a/chain/doc.go b/chain/doc.go new file mode 100644 index 0000000..42d2866 --- /dev/null +++ b/chain/doc.go @@ -0,0 +1,6 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +// Package chain holds the values that describe an EVM chain (block references, headers, logs, +// blocks, log filters and blooms) and the Source interface that reads them. +package chain diff --git a/chain/filter.go b/chain/filter.go new file mode 100644 index 0000000..d2a3be6 --- /dev/null +++ b/chain/filter.go @@ -0,0 +1,63 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chain + +import ( + "fmt" + "slices" + + "github.com/ethereum/go-ethereum/common" +) + +// MaxTopicPositions is the number of topic positions a filter may constrain (LOG0 to LOG4 give +// logs at most four topics). +const MaxTopicPositions = 4 + +// MaxFilterValues is the largest number of addresses, and of topics in one position, a filter may +// list. It is go-ethereum's default LogQueryLimit. +const MaxFilterValues = 1000 + +// Filter selects logs with eth_getLogs semantics. A nil or empty Addresses matches every address. +// Topics[i] constrains topic i: a nil or empty Topics[i] matches any topic, otherwise the log's +// topic i must equal one of the listed hashes. A log with fewer topics than len(Topics) never +// matches, even when the extra positions are wildcards. Match and Validate only read f. +type Filter struct { + Addresses []common.Address + Topics [][]common.Hash +} + +// Match reports whether l passes f. It reads only l.Address and l.Topics. +func (f Filter) Match(l Log) bool { + if len(f.Addresses) > 0 && !slices.Contains(f.Addresses, l.Address) { + return false + } + if len(f.Topics) > len(l.Topics) { + return false + } + for i, pos := range f.Topics { + if len(pos) > 0 && !slices.Contains(pos, l.Topics[i]) { + return false + } + } + return true +} + +// Validate returns an error wrapping ErrInvalidFilter when f has more than MaxTopicPositions +// topic positions, more than MaxFilterValues addresses, or more than MaxFilterValues topics in one +// position. Duplicates are allowed. +func (f Filter) Validate() error { + if len(f.Topics) > MaxTopicPositions { + return fmt.Errorf("%w: %d topic positions, at most %d", ErrInvalidFilter, len(f.Topics), MaxTopicPositions) + } + if len(f.Addresses) > MaxFilterValues { + return fmt.Errorf("%w: %d addresses, at most %d", ErrInvalidFilter, len(f.Addresses), MaxFilterValues) + } + for i, pos := range f.Topics { + if len(pos) > MaxFilterValues { + return fmt.Errorf("%w: topic position %d lists %d topics, at most %d", + ErrInvalidFilter, i, len(pos), MaxFilterValues) + } + } + return nil +} diff --git a/chain/filter_test.go b/chain/filter_test.go new file mode 100644 index 0000000..608cbf3 --- /dev/null +++ b/chain/filter_test.go @@ -0,0 +1,183 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chain + +import ( + "errors" + "reflect" + "slices" + "testing" + + "github.com/ethereum/go-ethereum/common" +) + +var ( + other = common.HexToAddress("0x01") + other2 = common.HexToAddress("0x02") + topA = common.HexToHash("0xaa") + topB = common.HexToHash("0xbb") + topX = common.HexToHash("0xcc") +) + +func cloneFilter(f Filter) Filter { + c := Filter{Addresses: slices.Clone(f.Addresses)} + if f.Topics != nil { + c.Topics = make([][]common.Hash, len(f.Topics)) + for i, pos := range f.Topics { + c.Topics[i] = slices.Clone(pos) + } + } + return c +} + +func cloneLog(l Log) Log { + l.Topics = slices.Clone(l.Topics) + l.Data = slices.Clone(l.Data) + return l +} + +func TestFilterMatch(t *testing.T) { + full := Log{ + Address: usdc, + Topics: []common.Hash{transfer, topA, topB}, + Data: []byte{1, 2, 3}, + BlockNumber: 7, + BlockHash: one32, + TxHash: topX, + TxIndex: 2, + Index: 5, + } + bare := Log{Address: usdc} + tests := []struct { + name string + f Filter + want bool + wantBare bool + }{ + {"zero filter", Filter{}, true, true}, + {"empty addresses", Filter{Addresses: []common.Address{}}, true, true}, + {"address", Filter{Addresses: []common.Address{usdc}}, true, true}, + {"other address", Filter{Addresses: []common.Address{other}}, false, false}, + {"other or address", Filter{Addresses: []common.Address{other, usdc}}, true, true}, + {"topic 0", Filter{Topics: [][]common.Hash{{transfer}}}, true, false}, + {"nil then topic 1", Filter{Topics: [][]common.Hash{nil, {topA}}}, true, false}, + {"empty then topic 1", Filter{Topics: [][]common.Hash{{}, {topA}}}, true, false}, + {"topic 0 alternatives", Filter{Topics: [][]common.Hash{{topX, transfer}}}, true, false}, + {"wrong topic 0", Filter{Topics: [][]common.Hash{{topX}}}, false, false}, + {"three wildcards", Filter{Topics: [][]common.Hash{nil, nil, nil}}, true, false}, + {"four wildcards", Filter{Topics: [][]common.Hash{nil, nil, nil, nil}}, false, false}, + {"topic in wrong position", Filter{Topics: [][]common.Hash{nil, {topB}}}, false, false}, + { + "address and topics", + Filter{Addresses: []common.Address{usdc}, Topics: [][]common.Hash{{transfer}, nil, {topB}}}, + true, false, + }, + {"five wildcards", Filter{Topics: [][]common.Hash{nil, nil, nil, nil, nil}}, false, false}, + {"two other addresses", Filter{Addresses: []common.Address{other, other2}}, false, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fc, lc := cloneFilter(tt.f), cloneLog(full) + if got := tt.f.Match(full); got != tt.want { + t.Errorf("Match(L) = %v, want %v", got, tt.want) + } + if got := tt.f.Match(bare); got != tt.wantBare { + t.Errorf("Match(log without topics) = %v, want %v", got, tt.wantBare) + } + allocs := testing.AllocsPerRun(100, func() { tt.f.Match(full) }) + if allocs != 0 { + t.Errorf("Match allocates %v times per call, want 0", allocs) + } + if !reflect.DeepEqual(tt.f, fc) { + t.Error("Match modified the filter") + } + if !reflect.DeepEqual(full, lc) { + t.Error("Match modified the log") + } + }) + } +} + +func TestFilterMatchFourthTopic(t *testing.T) { + l := Log{Address: usdc, Topics: []common.Hash{transfer, topA, topB, topX}} + tests := []struct { + name string + f Filter + want bool + }{ + {"topic 3 listed", Filter{Topics: [][]common.Hash{nil, nil, nil, {topX}}}, true}, + {"topic 3 not listed", Filter{Topics: [][]common.Hash{nil, nil, nil, {topA}}}, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tt.f.Match(l); got != tt.want { + t.Errorf("Match = %v, want %v", got, tt.want) + } + }) + } +} + +func TestFilterValidate(t *testing.T) { + positions := func(n int) [][]common.Hash { return make([][]common.Hash, n) } + tests := []struct { + name string + f Filter + wantErr string + }{ + {"zero filter", Filter{}, ""}, + {"4 positions", Filter{Topics: positions(4)}, ""}, + {"5 positions", Filter{Topics: positions(5)}, "chain: invalid filter: 5 topic positions, at most 4"}, + {"1000 addresses", Filter{Addresses: make([]common.Address, 1000)}, ""}, + { + "1001 addresses", + Filter{Addresses: make([]common.Address, 1001)}, + "chain: invalid filter: 1001 addresses, at most 1000", + }, + { + "1000 topics in a position", + Filter{Topics: [][]common.Hash{make([]common.Hash, 1000)}}, + "", + }, + { + "positions 0 and 2 with 1001 topics", + Filter{Topics: [][]common.Hash{make([]common.Hash, 1001), nil, make([]common.Hash, 1001)}}, + "chain: invalid filter: topic position 0 lists 1001 topics, at most 1000", + }, + { + "position 2 with 1001 topics", + Filter{Topics: [][]common.Hash{nil, {topA}, make([]common.Hash, 1001)}}, + "chain: invalid filter: topic position 2 lists 1001 topics, at most 1000", + }, + { + "5 positions and 1001 addresses", + Filter{Addresses: make([]common.Address, 1001), Topics: positions(5)}, + "chain: invalid filter: 5 topic positions, at most 4", + }, + { + "1001 addresses and 1001 topics", + Filter{Addresses: make([]common.Address, 1001), Topics: [][]common.Hash{make([]common.Hash, 1001)}}, + "chain: invalid filter: 1001 addresses, at most 1000", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.f.Validate() + if tt.wantErr == "" { + if err != nil { + t.Fatalf("Validate() = %v, want nil", err) + } + return + } + if err == nil { + t.Fatalf("Validate() = nil, want %q", tt.wantErr) + } + if !errors.Is(err, ErrInvalidFilter) { + t.Errorf("Validate() = %v, want an error wrapping ErrInvalidFilter", err) + } + if err.Error() != tt.wantErr { + t.Errorf("Validate() = %q, want %q", err.Error(), tt.wantErr) + } + }) + } +} diff --git a/chain/norace_test.go b/chain/norace_test.go new file mode 100644 index 0000000..2cc3a26 --- /dev/null +++ b/chain/norace_test.go @@ -0,0 +1,8 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +//go:build !race + +package chain + +const raceEnabled = false diff --git a/chain/race_test.go b/chain/race_test.go new file mode 100644 index 0000000..08979e9 --- /dev/null +++ b/chain/race_test.go @@ -0,0 +1,8 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +//go:build race + +package chain + +const raceEnabled = true diff --git a/chain/source.go b/chain/source.go new file mode 100644 index 0000000..40b296f --- /dev/null +++ b/chain/source.go @@ -0,0 +1,66 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chain + +import ( + "context" + "errors" + + "github.com/ethereum/go-ethereum/common" +) + +// Source is the read-only view of one chain that the tracker and ingest use. Implementations are +// safe for concurrent use. +// +// Every implementation follows this contract, and callers may rely on it: +// +// - A method that fails because the block or tag does not exist returns an error wrapping +// ErrNotFound; because the chain lacks a feature, ErrUnsupported; because from > to, +// ErrInvalidRange. Other failures (transport, provider errors, decoding) wrap none of the three. +// - When ctx is done before or during a call, the method returns an error that wraps ctx.Err() +// and none of ErrNotFound, ErrUnsupported and ErrInvalidRange. +// - A source answers from its current view, which can change between two calls (a reorg, a load +// balancer). Two Latest calls may return decreasing numbers, and Finalized may move backwards. +// - Results follow the "empty is nil" rule (nil Topics, Data and Logs when empty), give every log +// the BlockNumber and BlockHash of the block that holds it, list a block's logs in strictly +// increasing Index order with TxIndex non-decreasing, and belong to the caller: mutating a +// returned slice never changes a later result. +// - Every method that takes a Filter treats it as read-only and accepts every filter that +// Filter.Validate accepts. A source may return an error for a filter that Validate rejects. +type Source interface { + // Latest returns the head block as the source sees it now. Its error never wraps ErrNotFound. + Latest(ctx context.Context) (Header, error) + // Finalized returns the newest finalized block. It returns an error wrapping ErrUnsupported + // when the chain has no finalized tag, and ErrNotFound when no block is finalized yet. + Finalized(ctx context.Context) (Header, error) + // HeaderByNumber returns the canonical header at number n, or an error wrapping ErrNotFound + // when n is above the source's head. Number 0 is the genesis block. + HeaderByNumber(ctx context.Context, n uint64) (Header, error) + // HeaderByHash returns the header of the block with hash h, canonical or not, or an error + // wrapping ErrNotFound when the source does not know h. The zero hash is never known. + HeaderByHash(ctx context.Context, h common.Hash) (Header, error) + // Logs returns the logs of the block with hash block that pass f, in increasing Index order + // (EIP-234). It returns nil when none passes, and an error wrapping ErrNotFound when the source + // does not know the block. For a known block that is not canonical, it may return the block's + // logs or ErrNotFound. + Logs(ctx context.Context, block common.Hash, f Filter) ([]Log, error) + // LogsInRange returns the logs of the canonical blocks from..to (inclusive) that pass f, in + // increasing (BlockNumber, Index) order, or nil when none passes. It returns an error wrapping + // ErrInvalidRange when from > to, without contacting a provider, and ErrNotFound when to is + // above the source's head. The blocks are canonical in one view of the source; callers that + // need more check the log block hashes against headers. + LogsInRange(ctx context.Context, from, to uint64, f Filter) ([]Log, error) +} + +var ( + // ErrNotFound reports that no block has the requested number or hash. + ErrNotFound = errors.New("chain: not found") + // ErrUnsupported reports that the chain does not support a request, for example the + // finalized tag. + ErrUnsupported = errors.New("chain: unsupported") + // ErrInvalidRange reports a block range whose start is above its end. + ErrInvalidRange = errors.New("chain: invalid block range") + // ErrInvalidFilter reports a filter that breaks the limits of Filter.Validate. + ErrInvalidFilter = errors.New("chain: invalid filter") +) diff --git a/chain/source_test.go b/chain/source_test.go new file mode 100644 index 0000000..a356829 --- /dev/null +++ b/chain/source_test.go @@ -0,0 +1,34 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chain + +import ( + "errors" + "testing" +) + +func TestErrors(t *testing.T) { + tests := []struct { + name string + err error + want string + }{ + {"ErrNotFound", ErrNotFound, "chain: not found"}, + {"ErrUnsupported", ErrUnsupported, "chain: unsupported"}, + {"ErrInvalidRange", ErrInvalidRange, "chain: invalid block range"}, + {"ErrInvalidFilter", ErrInvalidFilter, "chain: invalid filter"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.err.Error() != tt.want { + t.Errorf("Error() = %q, want %q", tt.err.Error(), tt.want) + } + for _, o := range tests { + if o.name != tt.name && errors.Is(tt.err, o.err) { + t.Errorf("errors.Is(%s, %s) = true, want false", tt.name, o.name) + } + } + }) + } +} diff --git a/chainlog/bench_test.go b/chainlog/bench_test.go new file mode 100644 index 0000000..d1e8840 --- /dev/null +++ b/chainlog/bench_test.go @@ -0,0 +1,95 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import ( + "testing" + + "github.com/hmdsefi/evmsync/chain" +) + +// benchBlock is a block with 10 logs of 3 topics and 64 bytes of data each. +func benchBlock() Entry { + ref := blk(7, 0) + return Entry{Offset: 1 << 20, Kind: KindBlock, Block: chain.Block{Ref: ref, Logs: mkLogs(ref, 10, 3, 64)}} +} + +func BenchmarkEncode(b *testing.B) { + e := benchBlock() + dst := make([]byte, 0, 4096) + b.ReportAllocs() + for b.Loop() { + dst = Encode(dst[:0], e) + } +} + +func BenchmarkDecode(b *testing.B) { + rec := Encode(nil, benchBlock()) + b.ReportAllocs() + for b.Loop() { + if _, _, err := Decode(rec); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkStateBlock checks and applies one linked block and a Finalize of the window's first +// ref per iteration, so the window keeps 99,999 refs. +func BenchmarkStateBlock(b *testing.B) { + const window = 99_999 + s := NewState(0) + s.Apply(finalize(blk(0, 0))) + for n := uint64(1); n <= window; n++ { + s.Apply(block(blk(n, 0))) + } + next := uint64(window + 1) + b.ReportAllocs() + for b.Loop() { + e := block(blk(next, 0)) + f := finalize(blk(next-window, 0)) + if err := s.Check(e); err != nil { + b.Fatal(err) + } + s.Apply(e) + if err := s.Check(f); err != nil { + b.Fatal(err) + } + s.Apply(f) + next++ + } +} + +// BenchmarkMemoryAppend appends one block of 10 logs (3 topics, 64 bytes of data) and a Finalize +// of the window's first ref per iteration, so the window keeps 50,000 refs. ns/op covers both +// appends, so it can only overstate the cost of the block. +func BenchmarkMemoryAppend(b *testing.B) { + const window = 50_000 + ctx := b.Context() + m := NewMemory(Meta{}, 0) + if _, err := m.Append(ctx, blocks(1, window)...); err != nil { + b.Fatal(err) + } + next := uint64(window + 1) + logs := mkLogs(blk(next, 0), 10, 3, 64) // Append copies them, so each block reuses them + b.ReportAllocs() + for b.Loop() { + ref := blk(next, 0) + for i := range logs { + logs[i].BlockNumber, logs[i].BlockHash = ref.Number, ref.Hash + } + e := Entry{Kind: KindBlock, Block: chain.Block{Ref: ref, Logs: logs}} + if _, err := m.Append(ctx, e); err != nil { + b.Fatal(err) + } + if _, err := m.Append(ctx, finalize(blk(next-window, 0))); err != nil { + b.Fatal(err) + } + if next%1024 == 0 { + if err := m.Truncate(ctx, m.head); err != nil { // bound the benchmark's memory + b.Fatal(err) + } + } + next++ + } +} diff --git a/chainlog/codec.go b/chainlog/codec.go new file mode 100644 index 0000000..a37e144 --- /dev/null +++ b/chainlog/codec.go @@ -0,0 +1,380 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import ( + "encoding/binary" + "fmt" + "hash/crc32" + "math" + "math/bits" + "strconv" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" +) + +const ( + recordVersion = 1 + headerSize = 8 // length:uint32le crc:uint32le + maxTopics = 4 + flagFinal = 0x01 + // minLogSize is the smallest encoded log: address, ntopics, an empty data length, tx hash, + // one-byte tx index and index. + minLogSize = 20 + 1 + 1 + 32 + 1 + 1 +) + +var crcTable = crc32.MakeTable(crc32.Castagnoli) + +// Encode appends the record of e to dst and returns the extended slice. e.Offset must be >= 1 +// and e.Kind valid; Encode panics otherwise. It writes only the fields of e.Kind. +// +// A record is an 8-byte header (body length and CRC-32C of the body, both little-endian uint32) +// followed by the body: version 1, the kind, the offset and the kind's payload, with integers as +// unsigned LEB128 varints. Encode does not check the body against MaxRecordSize. +func Encode(dst []byte, e Entry) []byte { + if e.Kind < KindBlock || e.Kind > KindFinalize { + panic("evmsync: chainlog.Encode: unknown kind " + strconv.Itoa(int(e.Kind))) + } + if e.Offset == 0 { + panic("evmsync: chainlog.Encode: offset 0") + } + logs := e.Block.Logs + if e.Kind == KindBlock { + for i := range logs { + if n := len(logs[i].Topics); n > maxTopics { + panic("evmsync: chainlog.Encode: log " + strconv.Itoa(i) + " has " + strconv.Itoa(n) + " topics") + } + } + } + + start := len(dst) + dst = append(dst, 0, 0, 0, 0, 0, 0, 0, 0, recordVersion, byte(e.Kind)) + dst = binary.AppendUvarint(dst, uint64(e.Offset)) + if e.Kind != KindBlock { + dst = appendRef(dst, e.To) + } else { + dst = appendRef(dst, e.Block.Ref) + var flags byte + if e.Final { + flags = flagFinal + } + dst = append(dst, flags) + dst = binary.AppendUvarint(dst, uint64(len(logs))) + for i := range logs { + l := &logs[i] + dst = append(dst, l.Address[:]...) + dst = append(dst, byte(len(l.Topics))) //nolint:gosec // at most maxTopics, checked above + for j := range l.Topics { + dst = append(dst, l.Topics[j][:]...) + } + dst = binary.AppendUvarint(dst, uint64(len(l.Data))) + dst = append(dst, l.Data...) + dst = append(dst, l.TxHash[:]...) + dst = binary.AppendUvarint(dst, uint64(l.TxIndex)) + dst = binary.AppendUvarint(dst, uint64(l.Index)) + } + } + + body := dst[start+headerSize:] + binary.LittleEndian.PutUint32(dst[start:], uint32(len(body))) //nolint:gosec // only a body above 4 GiB overflows; Append rejects bodies above MaxRecordSize + binary.LittleEndian.PutUint32(dst[start+4:], crc32.Checksum(body, crcTable)) + return dst +} + +func appendRef(dst []byte, r chain.BlockRef) []byte { + dst = binary.AppendUvarint(dst, r.Number) + dst = append(dst, r.Hash[:]...) + dst = append(dst, r.Parent[:]...) + return binary.AppendUvarint(dst, r.Time) +} + +// Decode decodes the first record in src. It returns the entry and the record's length in +// bytes. On failure it returns a zero Entry, 0 and an error wrapping one of the codec errors. +// +// Decode accepts exactly the records Encode writes. The entry does not alias src. A block's logs +// get BlockNumber and BlockHash from the block's ref, and empty Logs, Topics and Data are nil. +func Decode(src []byte) (Entry, int, error) { + if len(src) < headerSize { + return Entry{}, 0, fmt.Errorf("%w: have %d bytes, need %d", ErrIncomplete, len(src), headerSize) + } + length := binary.LittleEndian.Uint32(src) + if length > MaxRecordSize { + return Entry{}, 0, fmt.Errorf("%w: body of %d bytes exceeds %d", ErrRecordTooLarge, length, MaxRecordSize) + } + n := headerSize + int(length) + if len(src) < n { + return Entry{}, 0, fmt.Errorf("%w: have %d bytes, need %d", ErrIncomplete, len(src), n) + } + body := src[headerSize:n] + if stored, computed := binary.LittleEndian.Uint32(src[4:]), crc32.Checksum(body, crcTable); stored != computed { + return Entry{}, 0, fmt.Errorf("%w: stored 0x%08x, computed 0x%08x", ErrChecksum, stored, computed) + } + if len(body) < 3 { + return Entry{}, 0, fmt.Errorf("%w: body of %d bytes is shorter than 3", ErrPayload, len(body)) + } + if body[0] != recordVersion { + return Entry{}, 0, fmt.Errorf("%w: %d", ErrVersion, body[0]) + } + kind := Kind(body[1]) + if kind < KindBlock || kind > KindFinalize { + return Entry{}, 0, fmt.Errorf("%w: %d", ErrKind, body[1]) + } + e, err := decodePayload(kind, body[2:]) + if err != nil { + return Entry{}, 0, err + } + return e, n, nil +} + +// decodePayload decodes the body after its version and kind bytes. +func decodePayload(kind Kind, b []byte) (Entry, error) { + d := decoder{b: b} + off, err := d.uvarint("offset", -1) + if err != nil { + return Entry{}, err + } + if off == 0 { + return Entry{}, fmt.Errorf("%w: offset is 0", ErrPayload) + } + e := Entry{Offset: Offset(off), Kind: kind} + if kind != KindBlock { + if e.To, err = d.ref(); err != nil { + return Entry{}, err + } + return e, d.end() + } + + if e.Block.Ref, err = d.ref(); err != nil { + return Entry{}, err + } + flags, err := d.byte("flags", -1) + if err != nil { + return Entry{}, err + } + if flags&^flagFinal != 0 { + return Entry{}, fmt.Errorf("%w: flags 0x%02x has unknown bits", ErrPayload, flags) + } + e.Final = flags == flagFinal + nlogs, err := d.uvarint("nlogs", -1) + if err != nil { + return Entry{}, err + } + // Bound nlogs by the remaining bytes before anything is allocated for the logs. + if nlogs > uint64(len(d.b)/minLogSize) { + return Entry{}, fmt.Errorf("%w: %d logs cannot fit in %d bytes", ErrPayload, nlogs, len(d.b)) + } + if nlogs == 0 { + return e, d.end() + } + + // Two passes over the logs: the first validates them and counts their topics and data bytes, + // the second fills one []Log, one []common.Hash and one []byte. + logStart := d + ntopics, ndata, err := d.logs(int(nlogs), nil, nil, nil) //nolint:gosec // nlogs <= MaxRecordSize/minLogSize, checked above + if err != nil { + return Entry{}, err + } + if err := d.end(); err != nil { + return Entry{}, err + } + logs := make([]chain.Log, nlogs) + var topics []common.Hash + if ntopics > 0 { + topics = make([]common.Hash, ntopics) + } + var data []byte + if ndata > 0 { + data = make([]byte, ndata) + } + d = logStart + if _, _, err := d.logs(len(logs), logs, topics, data); err != nil { + return Entry{}, err // unreachable: the first pass validated these bytes + } + for i := range logs { + logs[i].BlockNumber = e.Block.Ref.Number + logs[i].BlockHash = e.Block.Ref.Hash + } + e.Block.Logs = logs + return e, nil +} + +// decoder reads the fields of a record body in order. Its errors wrap ErrPayload. +type decoder struct { + b []byte // the bytes not read yet +} + +// logs reads n logs. With logs nil it only validates them and returns their total number of +// topics and data bytes. Otherwise it fills logs[i], sub-slicing its Topics from topics and its +// Data from data, which must have the totals of a validating pass over the same bytes. +func (d *decoder) logs(n int, logs []chain.Log, topics []common.Hash, data []byte) (ntopics, ndata int, err error) { + for i := range n { + addr, err := d.take(common.AddressLength, "address", i) + if err != nil { + return 0, 0, err + } + nt, err := d.byte("ntopics", i) + if err != nil { + return 0, 0, err + } + if nt > maxTopics { + return 0, 0, fmt.Errorf("%w: log %d: %d topics, at most %d", ErrPayload, i, nt, maxTopics) + } + tb, err := d.take(common.HashLength*int(nt), "topics", i) + if err != nil { + return 0, 0, err + } + dl, err := d.uvarint("data", i) + if err != nil { + return 0, 0, err + } + if dl > uint64(len(d.b)) { + return 0, 0, errTruncated("data", i) + } + db, _ := d.take(int(dl), "data", i) //nolint:gosec // dl <= len(d.b), checked above + txHash, err := d.take(common.HashLength, "txhash", i) + if err != nil { + return 0, 0, err + } + txIndex, err := d.uvarint("txindex", i) + if err != nil { + return 0, 0, err + } + if txIndex > math.MaxUint32 { + return 0, 0, fmt.Errorf("%w: log %d: tx index %d exceeds %d", ErrPayload, i, txIndex, uint64(math.MaxUint32)) + } + index, err := d.uvarint("index", i) + if err != nil { + return 0, 0, err + } + if index > math.MaxUint32 { + return 0, 0, fmt.Errorf("%w: log %d: index %d exceeds %d", ErrPayload, i, index, uint64(math.MaxUint32)) + } + + if logs != nil { + l := &logs[i] + copy(l.Address[:], addr) + if nt > 0 { + l.Topics = topics[ntopics : ntopics+int(nt) : ntopics+int(nt)] + for j := range l.Topics { + copy(l.Topics[j][:], tb[j*common.HashLength:]) + } + } + if len(db) > 0 { + l.Data = data[ndata : ndata+len(db) : ndata+len(db)] + copy(l.Data, db) + } + copy(l.TxHash[:], txHash) + l.TxIndex = uint32(txIndex) + l.Index = uint32(index) + } + ntopics += int(nt) + ndata += len(db) + } + return ntopics, ndata, nil +} + +func (d *decoder) ref() (chain.BlockRef, error) { + var r chain.BlockRef + var err error + if r.Number, err = d.uvarint("ref.number", -1); err != nil { + return chain.BlockRef{}, err + } + hash, err := d.take(common.HashLength, "ref.hash", -1) + if err != nil { + return chain.BlockRef{}, err + } + parent, err := d.take(common.HashLength, "ref.parent", -1) + if err != nil { + return chain.BlockRef{}, err + } + if r.Time, err = d.uvarint("ref.time", -1); err != nil { + return chain.BlockRef{}, err + } + copy(r.Hash[:], hash) + copy(r.Parent[:], parent) + return r, nil +} + +// take returns the next n bytes. field and log name the field in the error (log -1: not a log +// field). +func (d *decoder) take(n int, field string, log int) ([]byte, error) { + if n > len(d.b) { + return nil, errTruncated(field, log) + } + p := d.b[:n] + d.b = d.b[n:] + return p, nil +} + +func (d *decoder) byte(field string, log int) (byte, error) { + p, err := d.take(1, field, log) + if err != nil { + return 0, err + } + return p[0], nil +} + +// uvarint reads a minimal unsigned LEB128 varint. +func (d *decoder) uvarint(field string, log int) (uint64, error) { + x, n := binary.Uvarint(d.b) + switch { + case n == 0: + return 0, errTruncated(field, log) + case n < 0: + return 0, fmt.Errorf("%w: %s: varint overflows uint64", ErrPayload, fieldName(field, log)) + case n >= 2 && d.b[n-1] == 0: + // binary.Uvarint accepts a padded encoding such as 80 00; Encode never writes one. + return 0, fmt.Errorf("%w: %s: non-minimal varint", ErrPayload, fieldName(field, log)) + } + d.b = d.b[n:] + return x, nil +} + +// end fails when bytes remain after the payload. +func (d *decoder) end() error { + if len(d.b) > 0 { + return fmt.Errorf("%w: %d bytes after the payload", ErrTrailing, len(d.b)) + } + return nil +} + +func errTruncated(field string, log int) error { + return fmt.Errorf("%w: %s: truncated", ErrPayload, fieldName(field, log)) +} + +// fieldName returns field, or "log[]." for a field of log number log >= 0. +func fieldName(field string, log int) string { + if log < 0 { + return field + } + return "log[" + strconv.Itoa(log) + "]." + field +} + +// recordBodySize returns the length of e's record body as Encode writes it, counting the offset +// as 10 bytes (the longest varint), so the result does not depend on the offset Append assigns. +// It allocates nothing and encodes nothing. +func recordBodySize(e Entry) int { + const head = 2 + binary.MaxVarintLen64 // version, kind, offset + if e.Kind != KindBlock { + return head + refSize(e.To) + } + logs := e.Block.Logs + n := head + refSize(e.Block.Ref) + 1 + uvarintLen(uint64(len(logs))) + for i := range logs { + l := &logs[i] + n += common.AddressLength + 1 + common.HashLength*len(l.Topics) + + uvarintLen(uint64(len(l.Data))) + len(l.Data) + common.HashLength + + uvarintLen(uint64(l.TxIndex)) + uvarintLen(uint64(l.Index)) + } + return n +} + +func refSize(r chain.BlockRef) int { + return uvarintLen(r.Number) + 2*common.HashLength + uvarintLen(r.Time) +} + +// uvarintLen returns the number of bytes binary.AppendUvarint writes for x. +func uvarintLen(x uint64) int { return (bits.Len64(x|1) + 6) / 7 } diff --git a/chainlog/codec_test.go b/chainlog/codec_test.go new file mode 100644 index 0000000..885e10d --- /dev/null +++ b/chainlog/codec_test.go @@ -0,0 +1,527 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "math" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" +) + +var codecErrors = []error{ + ErrIncomplete, ErrRecordTooLarge, ErrChecksum, ErrVersion, ErrKind, ErrPayload, ErrTrailing, +} + +// record returns the 8-byte header for body (length, CRC-32C) followed by body. +func record(body []byte) []byte { + r := binary.LittleEndian.AppendUint32(nil, uint32(len(body))) //nolint:gosec // test bodies are small + r = binary.LittleEndian.AppendUint32(r, crc32.Checksum(body, crc32.MakeTable(crc32.Castagnoli))) + return append(r, body...) +} + +func refBytes(r chain.BlockRef) []byte { + b := binary.AppendUvarint(nil, r.Number) + b = append(b, r.Hash[:]...) + b = append(b, r.Parent[:]...) + return binary.AppendUvarint(b, r.Time) +} + +func cat(parts ...[]byte) []byte { + var b []byte + for _, p := range parts { + b = append(b, p...) + } + return b +} + +func rep(b byte, n int) []byte { return bytes.Repeat([]byte{b}, n) } + +func uv(x uint64) []byte { return binary.AppendUvarint(nil, x) } + +func panicText(f func()) (s string) { + defer func() { s = fmt.Sprint(recover()) }() + f() + return "" +} + +// mkLogs returns n logs of block ref, each with ntopics topics and dataLen bytes of data. Each log +// has its own Topics and Data arrays. +func mkLogs(ref chain.BlockRef, n, ntopics, dataLen int) []chain.Log { + logs := make([]chain.Log, n) + for i := range logs { + l := chain.Log{ + Address: common.Address{0: byte(i), 19: 0xaa}, + BlockNumber: ref.Number, + BlockHash: ref.Hash, + TxHash: common.Hash{0: 0x77, 31: byte(i)}, + TxIndex: uint32(i / 2), + Index: uint32(i), + } + for j := range ntopics { + l.Topics = append(l.Topics, common.Hash{0: byte(j + 1), 31: byte(i)}) + } + if dataLen > 0 { + l.Data = rep(byte(i+1), dataLen) + } + logs[i] = l + } + return logs +} + +// normalize returns e as Decode returns it: the fields of its kind only, the logs' block fields +// set from the ref and empty slices nil. +func normalize(e Entry) Entry { + if e.Kind != KindBlock { + return Entry{Offset: e.Offset, Kind: e.Kind, To: e.To} + } + out := Entry{Offset: e.Offset, Kind: e.Kind, Final: e.Final, Block: chain.Block{Ref: e.Block.Ref}} + for _, l := range e.Block.Logs { + l.BlockNumber = e.Block.Ref.Number + l.BlockHash = e.Block.Ref.Hash + if len(l.Topics) == 0 { + l.Topics = nil + } + if len(l.Data) == 0 { + l.Data = nil + } + out.Block.Logs = append(out.Block.Logs, l) + } + return out +} + +var ( + hash11 = common.Hash(rep(0x11, 32)) + hash22 = common.Hash(rep(0x22, 32)) +) + +// revertR is the revert of the layout test; its record is 78 bytes. +func revertR() Entry { + return Entry{Offset: 1, Kind: KindRevert, To: chain.BlockRef{Number: 5, Hash: hash11, Parent: hash22, Time: 1000}} +} + +// finalBlock is the block of the layout test: Final, one log with address 20×aa, one topic, data 01 02, +// tx hash 32×bb, tx index 1, index 300. +func finalBlock() (Entry, common.Hash) { + ref := blk(7, 0) + topic := common.Hash{31: 1} + return Entry{Offset: 200, Kind: KindBlock, Final: true, Block: chain.Block{Ref: ref, Logs: []chain.Log{{ + Address: common.Address(rep(0xaa, 20)), + Topics: []common.Hash{topic}, + Data: []byte{1, 2}, + BlockNumber: ref.Number, + BlockHash: ref.Hash, + TxHash: common.Hash(rep(0xbb, 32)), + TxIndex: 1, + Index: 300, + }}}}, topic +} + +func TestEncodeLayout(t *testing.T) { + revertBody := cat([]byte{0x01, 0x02, 0x01, 0x05}, rep(0x11, 32), rep(0x22, 32), []byte{0xe8, 0x07}) + fb, topic := finalBlock() + ref := fb.Block.Ref + plain := Entry{Offset: 128, Kind: KindBlock, Block: chain.Block{Ref: ref}} + fin := Entry{Offset: 3, Kind: KindFinalize, To: ref} + // A revert with block fields set writes only To. + noisyRevert := revertR() + noisyRevert.Final = true + noisyRevert.Block = chain.Block{Ref: ref, Logs: fb.Block.Logs} + + tests := []struct { + name string + e Entry + body []byte + }{ + {"revert", revertR(), revertBody}, + {"revert ignores block fields", noisyRevert, revertBody}, + {"final block with one log", fb, cat( + []byte{0x01, 0x01, 0xc8, 0x01}, refBytes(ref), []byte{0x01, 0x01}, + rep(0xaa, 20), []byte{0x01}, topic[:], []byte{0x02, 0x01, 0x02}, + rep(0xbb, 32), []byte{0x01, 0xac, 0x02}, + )}, + {"block without logs", plain, cat([]byte{0x01, 0x01, 0x80, 0x01}, refBytes(ref), []byte{0x00, 0x00})}, + {"finalize", fin, cat([]byte{0x01, 0x03, 0x03}, refBytes(ref))}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := Encode(nil, tt.e) + if want := record(tt.body); !bytes.Equal(got, want) { + t.Errorf("Encode =\n%x\nwant\n%x", got, want) + } + }) + } + + r := Encode(nil, revertR()) + if len(r) != 78 || !bytes.Equal(r[:4], []byte{0x46, 0x00, 0x00, 0x00}) { + t.Errorf("revert record: %d bytes, header %x; want 78 bytes, header 46000000", len(r), r[:4]) + } + if got, want := binary.LittleEndian.Uint32(r[4:8]), crc32.Checksum(r[8:], crc32.MakeTable(crc32.Castagnoli)); got != want { + t.Errorf("revert record CRC = %#08x, want %#08x", got, want) + } + + prefix := []byte("xyz") + got := Encode(prefix, revertR()) + if want := cat([]byte("xyz"), r); !bytes.Equal(got, want) { + t.Errorf("Encode with a prefix =\n%x\nwant\n%x", got, want) + } +} + +func TestEncodePanics(t *testing.T) { + five := mkLogs(blk(7, 0), 2, 4, 0) + five[1].Topics = append(five[1].Topics, common.Hash{1}) + tests := []struct { + name string + e Entry + want string + }{ + {"kind 0", Entry{Offset: 1, To: blk(1, 0)}, "evmsync: chainlog.Encode: unknown kind 0"}, + {"kind 4", Entry{Offset: 1, Kind: 4, To: blk(1, 0)}, "evmsync: chainlog.Encode: unknown kind 4"}, + {"kind 0 and offset 0", Entry{}, "evmsync: chainlog.Encode: unknown kind 0"}, + {"offset 0", Entry{Kind: KindBlock, Block: chain.Block{Ref: blk(7, 0)}}, "evmsync: chainlog.Encode: offset 0"}, + {"offset 0 revert", Entry{Kind: KindRevert, To: blk(7, 0)}, "evmsync: chainlog.Encode: offset 0"}, + {"5 topics", Entry{Offset: 1, Kind: KindBlock, Block: chain.Block{Ref: blk(7, 0), Logs: five}}, "evmsync: chainlog.Encode: log 1 has 5 topics"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := panicText(func() { Encode(nil, tt.e) }); got != tt.want { + t.Errorf("panic = %q, want %q", got, tt.want) + } + }) + } + + // A revert's unused block fields are not checked. + r := Entry{Offset: 1, Kind: KindRevert, To: blk(7, 0), Block: chain.Block{Logs: five}} + if got := panicText(func() { Encode(nil, r) }); got != "" { + t.Errorf("Encode of a revert with 5-topic block logs panicked: %s", got) + } + + e := Entry{Offset: 1 << 40, Kind: KindBlock, Block: chain.Block{Ref: blk(7, 0), Logs: mkLogs(blk(7, 0), 10, 3, 64)}} + dst := make([]byte, 0, 4096) + if n := testing.AllocsPerRun(100, func() { dst = Encode(dst[:0], e) }); n != 0 { + t.Errorf("Encode into a large enough dst: %v allocs, want 0", n) + } +} + +func TestDecodeErrors(t *testing.T) { + r := Encode(nil, revertR()) + rb := r[8:] // the 70-byte revert body + afterOffset := rb[3:] + ref := blk(7, 0) + blockHead := cat([]byte{0x01, 0x01, 0x01}, refBytes(ref)) + + withLen := func(n uint32) []byte { + c := bytes.Clone(r) + binary.LittleEndian.PutUint32(c, n) + return c + } + flipped := bytes.Clone(r) + flipped[8+10] ^= 0xff + stored := binary.LittleEndian.Uint32(flipped[4:8]) + computed := crc32.Checksum(flipped[8:], crc32.MakeTable(crc32.Castagnoli)) + setByte := func(i int, v byte) []byte { + b := bytes.Clone(rb) + b[i] = v + return record(b) + } + nlogs1000 := cat(blockHead, []byte{0x00}, uv(1000)) + nlogs1000Body := cat(nlogs1000, make([]byte, 200-len(nlogs1000))) + minLog := cat(rep(0xaa, 20), []byte{0x00, 0x00}, rep(0xbb, 32), []byte{0x00}) + fb, _ := finalBlock() + fbBody := Encode(nil, fb)[8:] + shortFlipped := record([]byte{0x01, 0x02}) + shortFlipped[4] ^= 0xff + shortStored := binary.LittleEndian.Uint32(shortFlipped[4:8]) + shortComputed := crc32.Checksum([]byte{0x01, 0x02}, crc32.MakeTable(crc32.Castagnoli)) + versionAndKind := bytes.Clone(rb) + versionAndKind[0], versionAndKind[1] = 2, 4 + log0data100 := cat(rep(0xaa, 20), []byte{0x00}, uv(100), rep(0x01, 100), rep(0xbb, 32), []byte{0x00, 0x00}) + + tests := []struct { + name string + src []byte + err error + detail string + }{ + {"5 bytes", r[:5], ErrIncomplete, "have 5 bytes, need 8"}, + {"empty", nil, ErrIncomplete, "have 0 bytes, need 8"}, + {"length 16777217", withLen(16777217), ErrRecordTooLarge, "body of 16777217 bytes exceeds 16777216"}, + {"length max uint32", withLen(math.MaxUint32), ErrRecordTooLarge, "body of 4294967295 bytes exceeds 16777216"}, + {"77 bytes", r[:77], ErrIncomplete, "have 77 bytes, need 78"}, + {"length 100", withLen(100), ErrIncomplete, "have 78 bytes, need 108"}, + {"body byte 10 flipped", flipped, ErrChecksum, fmt.Sprintf("stored 0x%08x, computed 0x%08x", stored, computed)}, + {"body of 2 bytes", record([]byte{0x01, 0x02}), ErrPayload, "body of 2 bytes is shorter than 3"}, + {"8 zero bytes", make([]byte, 8), ErrPayload, "body of 0 bytes is shorter than 3"}, + {"version 2", setByte(0, 2), ErrVersion, "2"}, + {"version 0", setByte(0, 0), ErrVersion, "0"}, + {"kind 4", setByte(1, 4), ErrKind, "4"}, + {"kind 0", setByte(1, 0), ErrKind, "0"}, + {"offset 0", record(cat([]byte{0x01, 0x02, 0x00}, afterOffset)), ErrPayload, "offset is 0"}, + {"offset 80 00", record(cat([]byte{0x01, 0x02, 0x80, 0x00}, afterOffset)), ErrPayload, "offset: non-minimal varint"}, + {"offset 11 ff", record(cat([]byte{0x01, 0x02}, rep(0xff, 11), afterOffset)), ErrPayload, "offset: varint overflows uint64"}, + {"offset cut", record([]byte{0x01, 0x02, 0x80}), ErrPayload, "offset: truncated"}, + {"ref.number cut", record([]byte{0x01, 0x02, 0x01}), ErrPayload, "ref.number: truncated"}, + {"ref.hash cut", record(cat([]byte{0x01, 0x02, 0x01, 0x05}, rep(0x11, 31))), ErrPayload, "ref.hash: truncated"}, + {"ref.parent cut", record(rb[:3+1+32+10]), ErrPayload, "ref.parent: truncated"}, + {"ref.time cut", record(rb[:3+1+64]), ErrPayload, "ref.time: truncated"}, + {"ref.time overflows", record(cat(rb[:3+1+64], rep(0xff, 11))), ErrPayload, "ref.time: varint overflows uint64"}, + {"trailing byte", record(cat(rb, []byte{0x00})), ErrTrailing, "1 bytes after the payload"}, + {"trailing bytes after a block", record(cat(blockHead, []byte{0x00, 0x00, 0x07, 0x07})), ErrTrailing, "2 bytes after the payload"}, + {"trailing byte after logs", record(cat(fbBody, []byte{0x00})), ErrTrailing, "1 bytes after the payload"}, + {"flags cut", record(blockHead), ErrPayload, "flags: truncated"}, + {"flags 02", record(cat(blockHead, []byte{0x02, 0x00})), ErrPayload, "flags 0x02 has unknown bits"}, + {"flags 80", record(cat(blockHead, []byte{0x80, 0x00})), ErrPayload, "flags 0x80 has unknown bits"}, + {"nlogs cut", record(cat(blockHead, []byte{0x00})), ErrPayload, "nlogs: truncated"}, + {"nlogs non-minimal", record(cat(blockHead, []byte{0x00, 0x80, 0x00})), ErrPayload, "nlogs: non-minimal varint"}, + {"nlogs 1000 in 200 bytes", record(nlogs1000Body), ErrPayload, fmt.Sprintf("1000 logs cannot fit in %d bytes", 200-len(nlogs1000))}, + {"nlogs max", record(cat(blockHead, []byte{0x00}, uv(math.MaxUint64), make([]byte, 100))), ErrPayload, "18446744073709551615 logs cannot fit in 100 bytes"}, + {"one log in 55 bytes", record(cat(blockHead, []byte{0x00, 0x01}, minLog[:55])), ErrPayload, "1 logs cannot fit in 55 bytes"}, + {"ntopics 5", record(cat(blockHead, []byte{0x00, 0x01}, rep(0xaa, 20), []byte{0x05}, make([]byte, 160))), ErrPayload, "log 0: 5 topics, at most 4"}, + {"topics cut", record(cat(blockHead, []byte{0x00, 0x01}, rep(0xaa, 20), []byte{0x04}, make([]byte, 60))), ErrPayload, "log[0].topics: truncated"}, + {"data cut", record(cat(blockHead, []byte{0x00, 0x01}, rep(0xaa, 20), []byte{0x00}, uv(100), make([]byte, 40))), ErrPayload, "log[0].data: truncated"}, + {"data length overflows", record(cat(blockHead, []byte{0x00, 0x01}, rep(0xaa, 20), []byte{0x00}, rep(0xff, 11), make([]byte, 40))), ErrPayload, "log[0].data: varint overflows uint64"}, + {"txindex overflows", record(cat(blockHead, []byte{0x00, 0x01}, rep(0xaa, 20), []byte{0x00, 0x00}, rep(0xbb, 32), rep(0xff, 11))), ErrPayload, "log[0].txindex: varint overflows uint64"}, + {"tx index 2^32", record(cat(blockHead, []byte{0x00, 0x01}, rep(0xaa, 20), []byte{0x00, 0x00}, rep(0xbb, 32), uv(1<<32), []byte{0x00})), ErrPayload, "log 0: tx index 4294967296 exceeds 4294967295"}, + {"index 2^32", record(cat(blockHead, []byte{0x00, 0x01}, rep(0xaa, 20), []byte{0x00, 0x00}, rep(0xbb, 32), []byte{0x00}, uv(1<<32))), ErrPayload, "log 0: index 4294967296 exceeds 4294967295"}, + {"index cut", record(cat(blockHead, []byte{0x00, 0x01}, rep(0xaa, 20), []byte{0x00, 0x05}, rep(0x01, 5), rep(0xbb, 32), []byte{0x00, 0x80})), ErrPayload, "log[0].index: truncated"}, + // Two faults: the one earlier in the layout is reported. + {"checksum before body length", shortFlipped, ErrChecksum, fmt.Sprintf("stored 0x%08x, computed 0x%08x", shortStored, shortComputed)}, + {"version before kind", record(versionAndKind), ErrVersion, "2"}, + {"offset 0 before a cut ref", record([]byte{0x01, 0x02, 0x00}), ErrPayload, "offset is 0"}, + {"flags before nlogs", record(cat(blockHead, []byte{0x02})), ErrPayload, "flags 0x02 has unknown bits"}, + {"ntopics bound before topics", record(cat(blockHead, []byte{0x00, 0x01}, rep(0xaa, 20), []byte{0x05}, make([]byte, 40))), ErrPayload, "log 0: 5 topics, at most 4"}, + {"tx index bound before index", record(cat(blockHead, []byte{0x00, 0x01}, rep(0xaa, 20), []byte{0x00, 0x00}, rep(0xbb, 32), uv(1<<32))), ErrPayload, "log 0: tx index 4294967296 exceeds 4294967295"}, + {"data length equals the rest", record(cat(blockHead, []byte{0x00, 0x01}, rep(0xaa, 20), []byte{0x00}, uv(40), make([]byte, 40))), ErrPayload, "log[0].txhash: truncated"}, + {"log 1 address cut", record(cat(blockHead, []byte{0x00, 0x02}, log0data100, rep(0xaa, 10))), ErrPayload, "log[1].address: truncated"}, + {"log 1 ntopics cut", record(cat(blockHead, []byte{0x00, 0x02}, log0data100, rep(0xaa, 20))), ErrPayload, "log[1].ntopics: truncated"}, + {"log 1 txhash cut", record(cat( + blockHead, []byte{0x00, 0x02}, + rep(0xaa, 20), []byte{0x00}, uv(60), rep(0x01, 60), rep(0xbb, 32), []byte{0x00, 0x00}, + rep(0xaa, 20), []byte{0x00, 0x00}, rep(0xbb, 10), + )), ErrPayload, "log[1].txhash: truncated"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e, n, err := Decode(tt.src) + if !errors.Is(err, tt.err) { + t.Fatalf("Decode error = %v, want one wrapping %v", err, tt.err) + } + if got, want := err.Error(), tt.err.Error()+": "+tt.detail; got != want { + t.Errorf("error text = %q, want %q", got, want) + } + matched := 0 + for _, ce := range codecErrors { + if errors.Is(err, ce) { + matched++ + } + } + if matched != 1 { + t.Errorf("error wraps %d codec errors, want 1", matched) + } + if n != 0 || !reflect.DeepEqual(e, Entry{}) { + t.Errorf("Decode = %+v, %d; want a zero entry and 0", e, n) + } + }) + } +} + +func TestDecodeValues(t *testing.T) { + fb, _ := finalBlock() + rec := Encode(nil, fb) + src := cat(rec, []byte{0xde, 0xad}) // bytes after the record are not read + got, n, err := Decode(src) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if n != len(rec) { + t.Errorf("n = %d, want %d", n, len(rec)) + } + if !reflect.DeepEqual(got, fb) { + t.Errorf("Decode =\n%+v\nwant\n%+v", got, fb) + } + if got.To != (chain.BlockRef{}) || len(got.Block.Logs) != 1 || + got.Block.Logs[0].BlockNumber != 7 || got.Block.Logs[0].BlockHash != blk(7, 0).Hash { + t.Errorf("Decode: To %v, logs %+v; want zero To and one log of block 7", got.To, got.Block.Logs) + } + clear(src) + if !reflect.DeepEqual(got, fb) { + t.Error("zeroing src changed the decoded entry") + } + + empty := Entry{Offset: 9, Kind: KindBlock, Block: chain.Block{Ref: blk(9, 0), Logs: []chain.Log{}}} + got, _, err = Decode(Encode(nil, empty)) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if got.Block.Logs != nil { + t.Errorf("block without logs: Logs = %#v, want nil", got.Block.Logs) + } + + bare := Entry{Offset: 9, Kind: KindBlock, Block: chain.Block{Ref: blk(9, 0), Logs: []chain.Log{{ + Topics: []common.Hash{}, Data: []byte{}, Index: 4, + }}}} + got, _, err = Decode(Encode(nil, bare)) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if l := got.Block.Logs[0]; l.Topics != nil || l.Data != nil || l.Index != 4 { + t.Errorf("log without topics and data: Topics %#v, Data %#v, Index %d; want nil, nil, 4", l.Topics, l.Data, l.Index) + } + + mixed := mkLogs(blk(9, 0), 3, 2, 5) + mixed[1].Topics, mixed[1].Data = nil, nil + got, _, err = Decode(Encode(nil, Entry{Offset: 9, Kind: KindBlock, Block: chain.Block{Ref: blk(9, 0), Logs: mixed}})) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if l := got.Block.Logs[1]; l.Topics != nil || l.Data != nil { + t.Errorf("bare log between logs with topics and data: Topics %#v, Data %#v; want nil, nil", l.Topics, l.Data) + } + if l := got.Block.Logs[2]; len(l.Topics) != 2 || len(l.Data) != 5 { + t.Errorf("log 2: %d topics, %d data bytes; want 2 and 5", len(l.Topics), len(l.Data)) + } + + for _, kind := range []Kind{KindRevert, KindFinalize} { + e := Entry{Offset: 77, Kind: kind, To: blk(12, 3)} + got, _, err := Decode(Encode(nil, e)) + if err != nil { + t.Fatalf("Decode %v: %v", kind, err) + } + if !reflect.DeepEqual(got, e) { + t.Errorf("Decode %v = %+v, want %+v", kind, got, e) + } + } + + ten := Encode(nil, Entry{Offset: 5, Kind: KindBlock, Block: chain.Block{Ref: blk(7, 0), Logs: mkLogs(blk(7, 0), 10, 3, 64)}}) + if a := testing.AllocsPerRun(100, func() { _, _, _ = Decode(ten) }); a > 3 { + t.Errorf("Decode of a 10-log block: %v allocs, want at most 3", a) + } + rv := Encode(nil, revertR()) + if a := testing.AllocsPerRun(100, func() { _, _, _ = Decode(rv) }); a != 0 { + t.Errorf("Decode of a revert: %v allocs, want 0", a) + } + + two, _, err := Decode(Encode(nil, Entry{Offset: 5, Kind: KindBlock, Block: chain.Block{Ref: blk(7, 0), Logs: mkLogs(blk(7, 0), 2, 2, 3)}})) + if err != nil { + t.Fatalf("Decode: %v", err) + } + logs := two.Block.Logs + t1, d1 := logs[1].Topics[0], logs[1].Data[0] + logs[0].Topics = append(logs[0].Topics, common.Hash{0xee}) + logs[0].Data = append(logs[0].Data, 0xee) + if logs[1].Topics[0] != t1 || logs[1].Data[0] != d1 { + t.Error("appending to log 0's Topics or Data changed log 1's") + } +} + +func TestDecodeMaxRecordSize(t *testing.T) { + ref := blk(7, 0) + sized := func(dataLen int) Entry { + l := chain.Log{Data: make([]byte, dataLen)} + return Entry{Offset: 1, Kind: KindBlock, Block: chain.Block{Ref: ref, Logs: []chain.Log{l}}} + } + fixed := len(Encode(nil, sized(0))) - 8 - 1 // the body without its one-byte data length + exact := sized(MaxRecordSize - fixed - 4) // a 4-byte data length + rec := Encode(nil, exact) + if len(rec) != 8+MaxRecordSize { + t.Fatalf("test setup: record of %d bytes, want %d", len(rec), 8+MaxRecordSize) + } + if _, n, err := Decode(rec); err != nil || n != 8+MaxRecordSize { + t.Errorf("Decode of a body of exactly MaxRecordSize = %d, %v; want %d, nil", n, err, 8+MaxRecordSize) + } + over := Encode(nil, sized(MaxRecordSize-fixed-3)) + if _, _, err := Decode(over); !errors.Is(err, ErrRecordTooLarge) { + t.Errorf("Decode of a body of MaxRecordSize+1: %v, want ErrRecordTooLarge", err) + } +} + +// roundTripEntries cover offsets and numbers at varint boundaries, 0 to 10 logs, 0 to 4 topics, +// data up to 70,000 bytes, the largest indexes, both Final values, a revert and a finalize. +func roundTripEntries() []Entry { + ref := blk(7, 0) + maxRef := chain.BlockRef{Number: math.MaxUint64, Hash: h(1, 1), Parent: h(2, 2), Time: math.MaxUint64} + var es []Entry + for _, off := range []Offset{1, 127, 128, math.MaxUint64} { + es = append(es, Entry{Offset: off, Kind: KindRevert, To: ref}) + } + for _, n := range []uint64{0, math.MaxUint64} { + r := ref + r.Number = n + es = append(es, + Entry{Offset: 2, Kind: KindBlock, Block: chain.Block{Ref: r}}, + Entry{Offset: 2, Kind: KindFinalize, To: r}, + ) + } + es = append(es, Entry{Offset: 3, Kind: KindRevert, To: maxRef}) + for _, nlogs := range []int{0, 1, 10} { + for _, final := range []bool{false, true} { + es = append(es, Entry{Offset: 4, Kind: KindBlock, Final: final, Block: chain.Block{Ref: ref, Logs: mkLogs(ref, nlogs, 2, 5)}}) + } + } + for nt := range 5 { + es = append(es, Entry{Offset: 5, Kind: KindBlock, Block: chain.Block{Ref: ref, Logs: mkLogs(ref, 2, nt, 1)}}) + } + for _, dl := range []int{0, 1, 70_000} { + es = append(es, Entry{Offset: 6, Kind: KindBlock, Block: chain.Block{Ref: ref, Logs: mkLogs(ref, 3, 1, dl)}}) + } + mixed := mkLogs(ref, 4, 2, 5) + mixed[1].Topics, mixed[1].Data = nil, nil + mixed[3].Topics, mixed[3].Data = []common.Hash{}, []byte{} + es = append(es, Entry{Offset: 6, Kind: KindBlock, Block: chain.Block{Ref: ref, Logs: mixed}}) + big := mkLogs(ref, 1, 4, 3) + big[0].TxIndex, big[0].Index = math.MaxUint32, math.MaxUint32 + es = append(es, + Entry{Offset: 7, Kind: KindBlock, Final: true, Block: chain.Block{Ref: maxRef, Logs: big}}, + Entry{Offset: 8, Kind: KindBlock, Block: chain.Block{Ref: ref, Logs: []chain.Log{}}}, + Entry{Offset: 9, Kind: KindBlock, Block: chain.Block{Ref: ref, Logs: []chain.Log{{Topics: []common.Hash{}, Data: []byte{}}}}}, + Entry{Offset: 10, Kind: KindFinalize, To: maxRef}, + ) + return es +} + +func TestRoundTrip(t *testing.T) { + for i, e := range roundTripEntries() { + t.Run(fmt.Sprintf("%d-%v", i, e.Kind), func(t *testing.T) { + rec := Encode(nil, e) + got, n, err := Decode(rec) + if err != nil { + t.Fatalf("Decode: %v", err) + } + if n != len(rec) { + t.Errorf("n = %d, want %d", n, len(rec)) + } + if want := normalize(e); !reflect.DeepEqual(got, want) { + t.Errorf("Decode(Encode(e)) =\n%+v\nwant\n%+v", got, want) + } + if again := Encode(nil, got); !bytes.Equal(again, rec) { + t.Error("Encode of the decoded entry differs from the record") + } + }) + } +} + +func TestRecordBodySize(t *testing.T) { + for i, e := range roundTripEntries() { + e.Offset = math.MaxUint64 + want := len(Encode(nil, e)) - 8 + for _, off := range []Offset{1, 300, math.MaxUint64} { + e.Offset = off + if got := recordBodySize(e); got != want { + t.Errorf("entry %d (%v), offset %d: recordBodySize = %d, want %d", i, e.Kind, off, got, want) + } + } + } + e := Entry{Offset: 1, Kind: KindBlock, Block: chain.Block{Ref: blk(7, 0), Logs: mkLogs(blk(7, 0), 10, 3, 64)}} + if a := testing.AllocsPerRun(100, func() { recordBodySize(e) }); a != 0 { + t.Errorf("recordBodySize: %v allocs, want 0", a) + } +} diff --git a/chainlog/conformance_test.go b/chainlog/conformance_test.go new file mode 100644 index 0000000..d3a1d53 --- /dev/null +++ b/chainlog/conformance_test.go @@ -0,0 +1,17 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog_test + +import ( + "testing" + + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/chainlog/logtest" +) + +func TestMemoryConformance(t *testing.T) { + logtest.Run(t, func(t *testing.T) chainlog.Log { + return chainlog.NewMemory(logtest.Meta(), logtest.WindowCap) + }) +} diff --git a/chainlog/doc.go b/chainlog/doc.go new file mode 100644 index 0000000..fa396f9 --- /dev/null +++ b/chainlog/doc.go @@ -0,0 +1,6 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +// Package chainlog defines the canonical log of one chain: entries, offsets, metadata, the Log +// interface, the validator State, the record codec and an in-memory Log. +package chainlog diff --git a/chainlog/entry.go b/chainlog/entry.go new file mode 100644 index 0000000..516bf32 --- /dev/null +++ b/chainlog/entry.go @@ -0,0 +1,72 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import ( + "encoding/hex" + "strconv" + + "github.com/hmdsefi/evmsync/chain" +) + +// Offset is an entry's position in a log. The first entry has offset 1; 0 means "before the +// first entry". +type Offset uint64 + +// Kind is an entry's type. +type Kind uint8 + +// The three entry kinds. +const ( + KindBlock Kind = iota + 1 // apply a block + KindRevert // undo every block above To + KindFinalize // every block at or below To is final +) + +// String returns "block", "revert", "finalize", or "Kind()" for other values. +func (k Kind) String() string { + switch k { + case KindBlock: + return "block" + case KindRevert: + return "revert" + case KindFinalize: + return "finalize" + default: + return "Kind(" + strconv.Itoa(int(k)) + ")" + } +} + +// Entry is one entry of the log. Only the fields of its Kind are set; the others are zero. +type Entry struct { + Offset Offset // assigned by Append; ignored on input + Kind Kind + Block chain.Block // KindBlock + Final bool // KindBlock: ingest knew the block was final when it appended it + To chain.BlockRef // KindRevert, KindFinalize +} + +// ID identifies one log instance. It is random, fixed at creation, and changes when a log is +// deleted and rebuilt, which renumbers offsets. +type ID [16]byte + +// String returns the 32 lowercase hex digits of id. +func (id ID) String() string { return hex.EncodeToString(id[:]) } + +// IsZero reports whether every byte of id is zero. +func (id ID) IsZero() bool { return id == ID{} } + +// Meta is a log's metadata, fixed when the log is created. +type Meta struct { + ID ID + ChainID uint64 + Start uint64 // the first block ingest indexes + Filter chain.Filter // the ingest filter +} + +// DefaultWindowCap is the window cap used when a cap of 0 is given. +const DefaultWindowCap = 100_000 + +// MaxRecordSize is the largest record body, in bytes, that Decode accepts (16 MiB). +const MaxRecordSize = 16 << 20 diff --git a/chainlog/entry_test.go b/chainlog/entry_test.go new file mode 100644 index 0000000..ae1cdc2 --- /dev/null +++ b/chainlog/entry_test.go @@ -0,0 +1,104 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import ( + "encoding/binary" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" +) + +// h returns the hash with byte 0 = 0xb0|f, bytes 24..31 = n big-endian and all others zero. +func h(n uint64, f byte) common.Hash { + var x common.Hash + x[0] = 0xb0 | f + binary.BigEndian.PutUint64(x[24:], n) + return x +} + +// blk returns block n of chain f, linked to block n-1 of the same chain. +func blk(n uint64, f byte) chain.BlockRef { + r := chain.BlockRef{Number: n, Hash: h(n, f), Time: 1_700_000_000 + 12*n} + if n > 0 { + r.Parent = h(n-1, f) + } + return r +} + +func TestKindString(t *testing.T) { + tests := []struct { + kind Kind + want string + }{ + {KindBlock, "block"}, + {KindRevert, "revert"}, + {KindFinalize, "finalize"}, + {Kind(0), "Kind(0)"}, + {Kind(9), "Kind(9)"}, + {Kind(255), "Kind(255)"}, + } + for _, tt := range tests { + t.Run(tt.want, func(t *testing.T) { + if got := tt.kind.String(); got != tt.want { + t.Errorf("Kind(%d).String() = %q, want %q", uint8(tt.kind), got, tt.want) + } + }) + } +} + +func TestIDString(t *testing.T) { + id := ID{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f} + if got, want := id.String(), "000102030405060708090a0b0c0d0e0f"; got != want { + t.Errorf("String() = %q, want %q", got, want) + } + if !(ID{}).IsZero() { + t.Error("ID{}.IsZero() = false, want true") + } + for i := range len(ID{}) { + var id ID + id[i] = 1 + if id.IsZero() { + t.Errorf("ID{%d: 1}.IsZero() = true, want false", i) + } + } +} + +func TestConstants(t *testing.T) { + if DefaultWindowCap != 100_000 { + t.Errorf("DefaultWindowCap = %d, want 100000", DefaultWindowCap) + } + if MaxRecordSize != 16_777_216 { + t.Errorf("MaxRecordSize = %d, want 16777216", MaxRecordSize) + } +} + +func TestErrorTexts(t *testing.T) { + tests := []struct { + err error + want string + }{ + {ErrMalformed, "chainlog: malformed entry"}, + {ErrClosed, "chainlog: log closed"}, + {ErrTruncated, "chainlog: offset truncated"}, + {ErrAhead, "chainlog: offset beyond head"}, + {ErrBadState, "chainlog: bad state encoding"}, + {ErrIncomplete, "chainlog: incomplete record"}, + {ErrRecordTooLarge, "chainlog: record too large"}, + {ErrChecksum, "chainlog: record checksum mismatch"}, + {ErrVersion, "chainlog: unknown record version"}, + {ErrKind, "chainlog: unknown entry kind"}, + {ErrPayload, "chainlog: bad record payload"}, + {ErrTrailing, "chainlog: trailing bytes in record"}, + } + for _, tt := range tests { + t.Run(tt.want, func(t *testing.T) { + if got := tt.err.Error(); got != tt.want { + t.Errorf("error text = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/chainlog/errors.go b/chainlog/errors.go new file mode 100644 index 0000000..c518473 --- /dev/null +++ b/chainlog/errors.go @@ -0,0 +1,39 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import "errors" + +var ( + // ErrMalformed is returned by State.Check and Log.Append for an entry that may not follow + // the entries before it. + ErrMalformed = errors.New("chainlog: malformed entry") + // ErrClosed is returned by every Log method except Close after the log is closed. + ErrClosed = errors.New("chainlog: log closed") + // ErrTruncated is returned by Log.Read for an offset below Oldest. + ErrTruncated = errors.New("chainlog: offset truncated") + // ErrAhead is returned by Log.Read and Log.Wait for an offset beyond the head: the caller is + // reading a different log than the one it committed against. + ErrAhead = errors.New("chainlog: offset beyond head") + // ErrBadState is returned by State.UnmarshalBinary for malformed input. + ErrBadState = errors.New("chainlog: bad state encoding") +) + +// Decode returns one of these errors, wrapped with a detail, for each way a record can fail. +var ( + // ErrIncomplete means the input ends before the record does. + ErrIncomplete = errors.New("chainlog: incomplete record") + // ErrRecordTooLarge means the record's length is above MaxRecordSize. + ErrRecordTooLarge = errors.New("chainlog: record too large") + // ErrChecksum means the record's CRC-32C does not match its body. + ErrChecksum = errors.New("chainlog: record checksum mismatch") + // ErrVersion means the record has a version other than 1. + ErrVersion = errors.New("chainlog: unknown record version") + // ErrKind means the record's entry kind is not KindBlock, KindRevert or KindFinalize. + ErrKind = errors.New("chainlog: unknown entry kind") + // ErrPayload means a field of the record's body is cut short or out of range. + ErrPayload = errors.New("chainlog: bad record payload") + // ErrTrailing means bytes remain in the record's body after its payload. + ErrTrailing = errors.New("chainlog: trailing bytes in record") +) diff --git a/chainlog/fuzz_test.go b/chainlog/fuzz_test.go new file mode 100644 index 0000000..2dc17ca --- /dev/null +++ b/chainlog/fuzz_test.go @@ -0,0 +1,121 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import ( + "bytes" + "encoding/binary" + "errors" + "math" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" +) + +func FuzzDecode(f *testing.F) { + ref := blk(7, 0) + three := Encode(nil, Entry{Offset: 3, Kind: KindBlock, Block: chain.Block{Ref: ref, Logs: mkLogs(ref, 3, 2, 5)}}) + flipped := bytes.Clone(three) + flipped[len(flipped)-10] ^= 0x01 + tooLarge := make([]byte, 8) + binary.LittleEndian.PutUint32(tooLarge, MaxRecordSize+1) + + f.Add(Encode(nil, Entry{Offset: 1, Kind: KindBlock, Block: chain.Block{Ref: ref}})) + f.Add(Encode(nil, revertR())) + f.Add(Encode(nil, Entry{Offset: 2, Kind: KindFinalize, To: ref})) + f.Add(three) + f.Add(three[:len(three)-1]) + f.Add(flipped) + f.Add(make([]byte, 8)) + f.Add(tooLarge) + f.Add(record(cat(Encode(nil, revertR())[8:], []byte{0x00}))) + f.Add(record(cat(three[8:], []byte{0x00}))) + + f.Fuzz(func(t *testing.T, data []byte) { + e, n, err := Decode(data) + if err != nil { + matched := 0 + for _, ce := range codecErrors { + if errors.Is(err, ce) { + matched++ + } + } + if matched != 1 { + t.Fatalf("error %q wraps %d codec errors, want 1", err, matched) + } + if n != 0 || !reflect.DeepEqual(e, Entry{}) { + t.Fatalf("failed Decode returned %+v, %d; want a zero entry and 0", e, n) + } + return + } + length := int(binary.LittleEndian.Uint32(data)) + if n != 8+length || n > len(data) { + t.Fatalf("n = %d, want 8+%d and at most %d", n, length, len(data)) + } + if again := Encode(nil, e); !bytes.Equal(again, data[:n]) { + t.Fatalf("Encode(Decode(data)) =\n%x\nwant\n%x", again, data[:n]) + } + }) +} + +func FuzzRoundTrip(f *testing.F) { + h1, h2 := h(7, 0), h(6, 0) + // Offsets 1, 127, 128 and the largest; numbers 0 and the largest; 0, 1 and 7 logs; 0 to 4 + // topics; 0, 1 and 70,000 bytes of data; the largest tx index and index; Final both ways; + // revert and finalize. + f.Add(uint8(1), uint64(1), uint64(7), h1[:], h2[:], uint64(1000), false, uint8(0), uint8(0), []byte(nil), uint32(0), uint32(0)) + f.Add(uint8(1), uint64(127), uint64(0), h1[:], h2[:], uint64(0), false, uint8(0), uint8(0), []byte(nil), uint32(0), uint32(0)) + f.Add(uint8(2), uint64(128), uint64(math.MaxUint64), h1[:], h2[:], uint64(math.MaxUint64), false, uint8(0), uint8(0), []byte(nil), uint32(0), uint32(0)) + f.Add(uint8(0), uint64(math.MaxUint64), uint64(7), h1[:], h2[:], uint64(1000), true, uint8(1), uint8(4), []byte{1}, uint32(math.MaxUint32), uint32(math.MaxUint32)) + f.Add(uint8(0), uint64(5), uint64(7), h1[:], h2[:], uint64(1000), false, uint8(7), uint8(3), make([]byte, 70_000), uint32(2), uint32(10)) + f.Add(uint8(0), uint64(5), uint64(math.MaxUint64), h1[:], h2[:], uint64(1000), true, uint8(1), uint8(1), []byte{}, uint32(0), uint32(0)) + f.Add(uint8(0), uint64(5), uint64(0), []byte{}, []byte{}, uint64(0), false, uint8(2), uint8(2), []byte{9, 9}, uint32(1), uint32(1)) + f.Add(uint8(0), uint64(6), uint64(9), h1[:], h2[:], uint64(1000), false, uint8(3), uint8(0), []byte{}, uint32(0), uint32(4)) + + f.Fuzz(func(t *testing.T, kind uint8, offset, number uint64, hash, parent []byte, time uint64, + final bool, nlogs, ntopics uint8, data []byte, txindex, index uint32, + ) { + ref := chain.BlockRef{Number: number, Time: time} + copy(ref.Hash[:], hash) + ref.Hash[0] = 1 + copy(ref.Parent[:], parent) + e := Entry{Offset: Offset(max(offset, 1)), Kind: Kind(1 + kind%3)} + if e.Kind != KindBlock { + e.To = ref + } else { + e.Final = final + e.Block.Ref = ref + for i := range uint32(nlogs % 8) { + if index > math.MaxUint32-i { + t.Skip("index + i overflows uint32") + } + l := chain.Log{ + Address: common.Address(ref.Parent[:20]), + Data: data, + TxHash: ref.Hash, + TxIndex: txindex, + Index: index + i, + } + for j := range ntopics % 5 { + l.Topics = append(l.Topics, common.Hash{0: j, 31: byte(i)}) + } + e.Block.Logs = append(e.Block.Logs, l) + } + } + rec := Encode(nil, e) + got, n, err := Decode(rec) + if err != nil { + t.Fatalf("Decode(Encode(e)): %v", err) + } + if n != len(rec) { + t.Fatalf("n = %d, want %d", n, len(rec)) + } + if want := normalize(e); !reflect.DeepEqual(got, want) { + t.Fatalf("Decode(Encode(e)) =\n%+v\nwant\n%+v", got, want) + } + }) +} diff --git a/chainlog/log.go b/chainlog/log.go new file mode 100644 index 0000000..85c0c35 --- /dev/null +++ b/chainlog/log.go @@ -0,0 +1,44 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import "context" + +// Log is one chain's canonical log. Implementations are safe for concurrent use; one goroutine +// appends (ingest), any number read. +// +// Every method except Close first returns ErrClosed if the log is closed, then ctx.Err() if ctx +// is done, both unwrapped, and only then checks its arguments. Offsets are dense from Oldest to +// Head and never reused. +type Log interface { + // Meta returns the log's metadata. + Meta(ctx context.Context) (Meta, error) + // Append validates entries in order against the log's State and appends all of them, or none + // with an error wrapping ErrMalformed. It assigns offsets Head+1, Head+2, ... and returns the + // last one; with no entries it returns Head. + Append(ctx context.Context, entries ...Entry) (Offset, error) + // Read returns up to max entries starting at offset from, in offset order. It returns nil + // when from is Head+1, an error wrapping ErrTruncated when from is below Oldest, and an error + // wrapping ErrAhead when from is above Head+1. A from of 0 reads from 1. max must be >= 1. + Read(ctx context.Context, from Offset, max int) ([]Entry, error) + // Wait returns nil once Head > after, ctx.Err() if ctx ends first, ErrClosed if the log + // closes first, and an error wrapping ErrAhead when after > Head. + Wait(ctx context.Context, after Offset) error + // Head returns the offset of the newest entry, 0 if none was ever appended. + Head(ctx context.Context) (Offset, error) + // Oldest returns the offset of the first retained entry, Head+1 when none is retained. + Oldest(ctx context.Context) (Offset, error) + // State returns a copy of the validator state after the newest entry. + State(ctx context.Context) (State, error) + // Truncate drops retained entries with offsets below min(before, L+1), where L is the offset + // of the newest Finalize entry (0 if none). Implementations may keep more (the WAL drops + // whole segments only). The caller passes the lowest offset any consumer still needs, and + // never truncates a durable log while a consumer that keeps its state only in memory + // (consumer.Memory, memstore) is in use: after a restart such a consumer replays the whole + // log. A log that is itself in memory may be truncated, since a restart loses both. + Truncate(ctx context.Context, before Offset) error + // Close releases the log's resources. Later calls of other methods return ErrClosed; later + // calls of Close return nil. + Close() error +} diff --git a/chainlog/logtest/cases.go b/chainlog/logtest/cases.go new file mode 100644 index 0000000..65eeb92 --- /dev/null +++ b/chainlog/logtest/cases.go @@ -0,0 +1,803 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package logtest + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "math" + "reflect" + "sync" + "testing" + "time" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +// testCase is one subtest of Run. fresh opens another empty log for the cases that need two. +type testCase struct { + name string + run func(t *testing.T, l chainlog.Log, fresh func() chainlog.Log) +} + +func cases() []testCase { + return []testCase{ + {"Empty", testEmpty}, + {"Meta", testMeta}, + {"AppendOffsets", testAppendOffsets}, + {"AppendIgnoresOffset", testAppendIgnoresOffset}, + {"AppendEmpty", testAppendEmpty}, + {"RoundTrip", testRoundTrip}, + {"ReadBounds", testReadBounds}, + {"Ownership", testOwnership}, + {"Rule1Increasing", testRule1Increasing}, + {"Rule2Linked", testRule2Linked}, + {"Rule3RevertTarget", testRule3RevertTarget}, + {"Rule4Finality", testRule4Finality}, + {"Rule5FinalBlocks", testRule5FinalBlocks}, + {"Rule6Window", testRule6Window}, + {"Shape", testShape}, + {"TooLarge", testTooLarge}, + {"BatchAtomic", testBatchAtomic}, + {"StateCopy", testStateCopy}, + {"Wait", testWait}, + {"Truncate", testTruncate}, + {"TruncateBound", testTruncateBound}, + {"Closed", testClosed}, + {"CancelledContext", testCancelledContext}, + {"Concurrent", testConcurrent}, + {"BatchVisibility", testBatchVisibility}, + } +} + +func block(r chain.BlockRef) chainlog.Entry { + return chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: r}} +} + +func finalBlock(r chain.BlockRef) chainlog.Entry { + e := block(r) + e.Final = true + return e +} + +func revert(r chain.BlockRef) chainlog.Entry { + return chainlog.Entry{Kind: chainlog.KindRevert, To: r} +} + +func finalize(r chain.BlockRef) chainlog.Entry { + return chainlog.Entry{Kind: chainlog.KindFinalize, To: r} +} + +// blocks returns the linked blocks from..to of chain 0. +func blocks(from, to uint64) []chainlog.Entry { + var es []chainlog.Entry + for n := from; n <= to; n++ { + es = append(es, block(blk(n, 0))) + } + return es +} + +// numbered returns copies of es with offsets 1, 2, ... +func numbered(es ...chainlog.Entry) []chainlog.Entry { + out := make([]chainlog.Entry, len(es)) + for i, e := range es { + e.Offset = chainlog.Offset(i + 1) //nolint:gosec // i < len(es) + out[i] = e + } + return out +} + +// noErr stops the test when err is not nil. +func noErr(t *testing.T, err error, what string) { + t.Helper() + if err != nil { + t.Fatalf("%s: %v", what, err) + } +} + +// eq reports got when it does not deep-equal want. +func eq(t *testing.T, what string, got, want any) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Errorf("%s =\n%+v\nwant\n%+v", what, got, want) + } +} + +// wantErr checks that err wraps target and, unless text is empty, reads text. +func wantErr(t *testing.T, what string, err, target error, text string) { + t.Helper() + if !errors.Is(err, target) { + t.Fatalf("%s error = %v, want one wrapping %v", what, err, target) + } + if text != "" { + eq(t, what+" error", err.Error(), text) + } +} + +// same checks that err is exactly want, unwrapped, as the contract returns ErrClosed and ctx.Err(). +func same(t *testing.T, what string, err, want error) { + t.Helper() + if err != want { //nolint:errorlint // the contract returns these errors unwrapped + t.Errorf("%s = %v (%T), want %v unwrapped", what, err, err, want) + } +} + +func appendOK(t *testing.T, l chainlog.Log, es ...chainlog.Entry) chainlog.Offset { + t.Helper() + off, err := l.Append(t.Context(), es...) + noErr(t, err, "Append") + return off +} + +// wantRejected checks that appending the single entry e fails with the malformed detail and +// leaves Head unchanged. +func wantRejected(t *testing.T, l chainlog.Log, e chainlog.Entry, detail string) { + t.Helper() + before := head(t, l) + _, err := l.Append(t.Context(), e) + wantErr(t, "Append", err, chainlog.ErrMalformed, "chainlog: append entry 1 of 1: chainlog: malformed entry: "+detail) + eq(t, "Head after a rejected Append", head(t, l), before) +} + +func head(t *testing.T, l chainlog.Log) chainlog.Offset { + t.Helper() + h, err := l.Head(t.Context()) + noErr(t, err, "Head") + return h +} + +func oldest(t *testing.T, l chainlog.Log) chainlog.Offset { + t.Helper() + o, err := l.Oldest(t.Context()) + noErr(t, err, "Oldest") + return o +} + +func state(t *testing.T, l chainlog.Log) chainlog.State { + t.Helper() + s, err := l.State(t.Context()) + noErr(t, err, "State") + return s +} + +func read(t *testing.T, l chainlog.Log, from chainlog.Offset, max int) []chainlog.Entry { + t.Helper() + es, err := l.Read(t.Context(), from, max) + noErr(t, err, fmt.Sprintf("Read(%d, %d)", from, max)) + return es +} + +// wantEntries checks that got deep-equals want. +func wantEntries(t *testing.T, got, want []chainlog.Entry) { + t.Helper() + eq(t, "entries", got, want) +} + +// view is what a state shows through its accessors. +type view struct { + tip, fin chain.BlockRef + win []chain.BlockRef +} + +func viewOf(s chainlog.State) view { return view{s.Tip(), s.Finalized(), s.Window()} } + +func wantView(t *testing.T, l chainlog.Log, want view) { + t.Helper() + eq(t, "state", viewOf(state(t, l)), want) +} + +func refs(rs ...chain.BlockRef) []chain.BlockRef { return rs } + +func testEmpty(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + eq(t, "Head", head(t, l), chainlog.Offset(0)) + eq(t, "Oldest", oldest(t, l), chainlog.Offset(1)) + wantView(t, l, view{}) + for _, from := range []chainlog.Offset{0, 1} { + eq(t, fmt.Sprintf("Read(%d, 10)", from), read(t, l, from, 10), []chainlog.Entry(nil)) + } +} + +func testMeta(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + m, err := l.Meta(t.Context()) + noErr(t, err, "Meta") + eq(t, "Meta", m, Meta()) + m.ID[0] = 0xff + m.Filter.Addresses[0][0] = 0xff + m.Filter.Topics[0][0][0] = 0xff + m.Filter.Topics[0] = append(m.Filter.Topics[0], [32]byte{1}) + again, err := l.Meta(t.Context()) + noErr(t, err, "Meta") + eq(t, "Meta after mutating an earlier result", again, Meta()) +} + +func testAppendOffsets(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + eq(t, "Append of 3 entries", appendOK(t, l, blocks(101, 103)...), chainlog.Offset(3)) + wantEntries(t, read(t, l, 1, 10), numbered(blocks(101, 103)...)) + eq(t, "Append of 2 more entries", appendOK(t, l, blocks(104, 105)...), chainlog.Offset(5)) +} + +func testAppendIgnoresOffset(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + es := blocks(101, 102) + es[0].Offset, es[1].Offset = 99, 99 + eq(t, "Append", appendOK(t, l, es...), chainlog.Offset(2)) + wantEntries(t, read(t, l, 1, 10), numbered(blocks(101, 102)...)) +} + +func testAppendEmpty(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + eq(t, "Append() on an empty log", appendOK(t, l), chainlog.Offset(0)) + appendOK(t, l, blocks(101, 102)...) + before := viewOf(state(t, l)) + eq(t, "Append()", appendOK(t, l), chainlog.Offset(2)) + eq(t, "Head", head(t, l), chainlog.Offset(2)) + wantView(t, l, before) +} + +// logOf returns a log of block r with ntopics topics and dataLen bytes of data. Empty Topics and +// Data are non-nil when empty is set. +func logOf(r chain.BlockRef, index uint32, ntopics, dataLen int, empty bool) chain.Log { + l := chain.Log{BlockNumber: r.Number, BlockHash: r.Hash, TxIndex: index / 2, Index: index} + l.Address[19] = 0xaa + binary.BigEndian.PutUint32(l.Address[:4], index) + l.TxHash = h(uint64(index), 9) + for i := range ntopics { + var topic [32]byte + topic[0] = byte(i + 1) + binary.BigEndian.PutUint32(topic[28:], index) + l.Topics = append(l.Topics, topic) + } + if dataLen > 0 { + l.Data = make([]byte, dataLen) + for i := range l.Data { + l.Data[i] = byte(i) + } + } + if empty { + l.Topics = append(l.Topics, [32]byte{})[:0] + l.Data = []byte{} + } + return l +} + +func testRoundTrip(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + b101, b102, b103 := blk(101, 0), blk(102, 0), blk(103, 0) + in := []chainlog.Entry{ + block(b101), + {Kind: chainlog.KindBlock, Block: chain.Block{Ref: b102, Logs: []chain.Log{logOf(b102, 0, 4, 300, false)}}}, + {Kind: chainlog.KindBlock, Block: chain.Block{Ref: b103, Logs: []chain.Log{ + logOf(b103, 0, 0, 0, true), logOf(b103, 1, 1, 1, false), logOf(b103, 3, 2, 7, false), logOf(b103, 5, 3, 0, false), + }}}, + revert(b102), + finalize(b101), + } + appendOK(t, l, in...) + + want := numbered(in...) + want[2].Block.Logs = []chain.Log{logOf(b103, 0, 0, 0, false), logOf(b103, 1, 1, 1, false), logOf(b103, 3, 2, 7, false), logOf(b103, 5, 3, 0, false)} + got := read(t, l, 1, 10) + wantEntries(t, got, want) + l0 := got[2].Block.Logs[0] + eq(t, "Topics read back from an empty slice", l0.Topics == nil, true) + eq(t, "Data read back from an empty slice", l0.Data, []byte(nil)) +} + +func testReadBounds(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + all := numbered(blocks(101, 105)...) + appendOK(t, l, blocks(101, 105)...) + wantEntries(t, read(t, l, 2, 2), all[1:3]) + wantEntries(t, read(t, l, 5, 100), all[4:5]) + wantEntries(t, read(t, l, 0, 1), all[0:1]) + wantEntries(t, read(t, l, 1, math.MaxInt), all) + eq(t, "Read(6, 1)", read(t, l, 6, 1), []chainlog.Entry(nil)) + _, err := l.Read(t.Context(), 7, 1) + wantErr(t, "Read(7, 1)", err, chainlog.ErrAhead, "chainlog: offset beyond head: read from 7, head is 5") + for _, max := range []int{0, -1} { + eq(t, fmt.Sprintf("Read(1, %d) panic", max), panicText(func() { _, _ = l.Read(t.Context(), 1, max) }), + fmt.Sprintf("evmsync: chainlog: Read max must be at least 1, got %d", max)) + } +} + +func panicText(f func()) (s string) { + defer func() { s = fmt.Sprint(recover()) }() + f() + return "" +} + +func testOwnership(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + r := blk(101, 0) + e := chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: r, Logs: []chain.Log{logOf(r, 0, 1, 3, false)}}} + want := numbered(chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: r, Logs: []chain.Log{logOf(r, 0, 1, 3, false)}}}) + appendOK(t, l, e) + e.Block.Logs[0].Data[0] = 0xee + e.Block.Logs[0].Topics[0][0] = 0xee + got := read(t, l, 1, 1) + wantEntries(t, got, want) + got[0].Block.Logs[0].Data[0] = 0xdd + got[0].Block.Logs[0].Topics[0][0] = 0xdd + wantEntries(t, read(t, l, 1, 1), want) +} + +func testRule1Increasing(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + appendOK(t, l, block(blk(101, 0))) + wantRejected(t, l, block(blk(101, 0)), "block 101 is not above tip "+blk(101, 0).String()) + appendOK(t, l, block(blk(102, 0))) + wantRejected(t, l, block(blk(101, 0)), "block 101 is not above tip "+blk(102, 0).String()) + wantRejected(t, l, block(blk(102, 1)), "block 102 is not above tip "+blk(102, 0).String()) +} + +func testRule2Linked(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + appendOK(t, l, block(blk(101, 0))) + bad := blk(102, 0) + bad.Parent = h(101, 1) + wantRejected(t, l, block(bad), "block 102: parent "+bad.Parent.Hex()+" is not tip hash "+blk(101, 0).Hash.Hex()) + wantRejected(t, l, block(blk(105, 0)), "block 105: non-final block is not the child of tip "+blk(101, 0).String()) + appendOK(t, l, finalize(blk(101, 0))) + unlinked := blk(105, 0) + unlinked.Parent = h(99, 5) + eq(t, "Append of a Final block 105", appendOK(t, l, finalBlock(unlinked)), chainlog.Offset(3)) + wantView(t, l, view{tip: unlinked, fin: unlinked}) +} + +func testRule3RevertTarget(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + wantRejected(t, l, revert(blk(101, 0)), "revert to "+blk(101, 0).String()+": target is not below tip "+chain.BlockRef{}.String()) + appendOK(t, l, finalize(blk(100, 0))) + appendOK(t, l, blocks(101, 103)...) + tip := blk(103, 0).String() + wantRejected(t, l, revert(blk(103, 0)), "revert to "+blk(103, 0).String()+": target is not below tip "+tip) + wantRejected(t, l, revert(blk(104, 0)), "revert to "+blk(104, 0).String()+": target is not below tip "+tip) + wantRejected(t, l, revert(blk(102, 1)), "revert to "+blk(102, 1).String()+": target is neither the finalized block nor in the window") + appendOK(t, l, revert(blk(102, 0))) + wantView(t, l, view{tip: blk(102, 0), fin: blk(100, 0), win: refs(blk(101, 0), blk(102, 0))}) + appendOK(t, l, revert(blk(100, 0))) + wantView(t, l, view{tip: blk(100, 0), fin: blk(100, 0)}) +} + +func testRule4Finality(t *testing.T, l chainlog.Log, fresh func() chainlog.Log) { + appendOK(t, l, blocks(101, 104)...) + wantRejected(t, l, finalize(blk(100, 1)), "finalize to "+blk(100, 1).String()+": window block "+blk(101, 0).String()+" does not link to it") + appendOK(t, l, finalize(blk(102, 0))) + wantRejected(t, l, finalize(blk(101, 0)), "finalize to "+blk(101, 0).String()+": below finalized "+blk(102, 0).String()) + wantRejected(t, l, finalize(blk(102, 1)), "finalize to "+blk(102, 1).String()+": conflicts with finalized "+blk(102, 0).String()) + wantRejected(t, l, finalize(blk(103, 1)), "finalize to "+blk(103, 1).String()+": conflicts with window block "+blk(103, 0).String()) + child := fork(105, 0, 1) + wantRejected(t, l, finalize(child), "finalize to "+child.String()+": parent "+child.Parent.Hex()+" is not tip hash "+blk(104, 0).Hash.Hex()) + + before := viewOf(state(t, l)) + appendOK(t, l, finalize(blk(102, 0))) + wantView(t, l, before) + appendOK(t, l, finalize(blk(110, 0))) + wantView(t, l, view{tip: blk(110, 0), fin: blk(110, 0)}) + + empty := fresh() + appendOK(t, empty, finalize(blk(100, 0))) + wantView(t, empty, view{tip: blk(100, 0), fin: blk(100, 0)}) +} + +func testRule5FinalBlocks(t *testing.T, l chainlog.Log, fresh func() chainlog.Log) { + appendOK(t, l, blocks(101, 104)...) + wantRejected(t, l, finalBlock(blk(105, 0)), "block 105: final block with 4 unfinalized blocks in the window") + + b := fresh() + appendOK(t, b, finalize(blk(100, 0))) + for _, n := range []uint64{103, 105} { + appendOK(t, b, finalBlock(blk(n, 0))) + wantView(t, b, view{tip: blk(n, 0), fin: blk(n, 0)}) + } + appendOK(t, b, blocks(106, 107)...) + wantRejected(t, b, revert(blk(103, 0)), "revert to "+blk(103, 0).String()+": target is neither the finalized block nor in the window") + appendOK(t, b, revert(blk(105, 0))) + wantView(t, b, view{tip: blk(105, 0), fin: blk(105, 0)}) +} + +func testRule6Window(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + appendOK(t, l, blocks(101, 100+WindowCap)...) + next := uint64(101 + WindowCap) + wantRejected(t, l, block(blk(next, 0)), fmt.Sprintf("block %d: window is full (%d blocks)", next, WindowCap)) + appendOK(t, l, finalize(blk(102, 0))) + appendOK(t, l, block(blk(next, 0))) +} + +func testShape(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + r := blk(101, 0) + lg := func(index, txIndex uint32) chain.Log { + return chain.Log{BlockNumber: r.Number, BlockHash: r.Hash, Index: index, TxIndex: txIndex} + } + withLogs := func(logs ...chain.Log) chainlog.Entry { + return chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: r, Logs: logs}} + } + withTo := block(r) + withTo.To = blk(99, 0) + revertFinal := revert(blk(99, 0)) + revertFinal.Final = true + finalizeLogs := finalize(blk(99, 0)) + finalizeLogs.Block.Logs = []chain.Log{lg(0, 0)} + wrongNumber := lg(0, 0) + wrongNumber.BlockNumber = 102 + wrongHash := lg(0, 0) + wrongHash.BlockHash = blk(102, 0).Hash + five := lg(0, 0) + for range 5 { + five.Topics = append(five.Topics, [32]byte{1}) + } + tests := []struct { + name string + e chainlog.Entry + detail string + }{ + {"unknown kind", chainlog.Entry{Kind: 7, Block: chain.Block{Ref: r}}, "unknown kind 7"}, + {"block with To", withTo, "block 101: To must be zero"}, + {"revert with Final", revertFinal, "revert entry: Block and Final must be zero"}, + {"finalize with logs", finalizeLogs, "finalize entry: Block and Final must be zero"}, + {"block with zero hash", block(chain.BlockRef{Number: 101}), "block 101: zero hash"}, + {"revert to zero hash", revert(chain.BlockRef{Number: 99}), "revert to 99: zero hash"}, + {"log block number", withLogs(wrongNumber), "block 101: log 0: block number 102"}, + {"log block hash", withLogs(wrongHash), "block 101: log 0: block hash " + blk(102, 0).Hash.Hex()}, + {"log topics", withLogs(five), "block 101: log 0: 5 topics, at most 4"}, + {"log index", withLogs(lg(3, 0), lg(3, 0)), "block 101: log 1: index 3 not above 3"}, + {"log tx index", withLogs(lg(0, 2), lg(1, 1)), "block 101: log 1: tx index 1 below 2"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + wantRejected(t, l, tt.e, tt.detail) + }) + } +} + +func testTooLarge(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + r := blk(101, 0) + big := chain.Log{BlockNumber: r.Number, BlockHash: r.Hash, Data: make([]byte, chainlog.MaxRecordSize)} + e := chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: r, Logs: []chain.Log{big}}} + sized := e + sized.Offset = math.MaxUint64 + body := len(chainlog.Encode(nil, sized)) - 8 + wantRejected(t, l, e, fmt.Sprintf("block 101: record body of %d bytes exceeds 16777216", body)) + wantView(t, l, view{}) +} + +func testBatchAtomic(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + appendOK(t, l, block(blk(101, 0))) + before := viewOf(state(t, l)) + _, err := l.Append(t.Context(), block(blk(102, 0)), block(blk(102, 0)), block(blk(103, 0))) + wantErr(t, "Append", err, chainlog.ErrMalformed, "chainlog: append entry 2 of 3: chainlog: malformed entry: block 102 is not above tip "+blk(102, 0).String()) + eq(t, "Head", head(t, l), chainlog.Offset(1)) + wantView(t, l, before) + wantEntries(t, read(t, l, 1, 10), numbered(block(blk(101, 0)))) +} + +func testStateCopy(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + appendOK(t, l, blocks(101, 103)...) + want := view{tip: blk(103, 0), win: refs(blk(101, 0), blk(102, 0), blk(103, 0))} + // The revert first: the block then writes into whatever array the copy's window shares. + s := state(t, l) + s.Apply(revert(blk(101, 0))) + s.Apply(block(blk(102, 3))) + w := state(t, l).Window() + w[0] = chain.BlockRef{} + w = append(w, blk(9, 9)) + w[1] = w[len(w)-1] + wantView(t, l, want) +} + +func testWait(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + ctx := t.Context() + appendOK(t, l, blocks(101, 102)...) + bounded, cancelBound := context.WithTimeout(ctx, waitLimit) + defer cancelBound() + eq(t, "Wait(1) with Head 2", l.Wait(bounded, 1), error(nil)) + + done := blockedWait(t, l, ctx, 2) + appendOK(t, l, block(blk(103, 0))) + eq(t, "blocked Wait(2) after an Append", receive(t, done, "Wait(2)"), error(nil)) + + cctx, cancel := context.WithCancel(ctx) + done = blockedWait(t, l, cctx, 3) + cancel() + wantErr(t, "blocked Wait(3) after cancel", receive(t, done, "Wait(3)"), context.Canceled, "") + + wantErr(t, "Wait(10) with Head 3", l.Wait(bounded, 10), chainlog.ErrAhead, "chainlog: offset beyond head: wait after 10, head is 3") + + const waiters = 16 + results := make([]<-chan error, waiters) + for i := range results { + results[i] = blockedWait(t, l, ctx, 3) + } + appendOK(t, l, block(blk(104, 0))) + for _, done := range results { + eq(t, "blocked waiter after one Append", receive(t, done, "Wait(3)"), error(nil)) + } + + // An Append that appends nothing, empty or rejected, leaves a waiter blocked. + cctx, cancel = context.WithCancel(ctx) + done = blockedWait(t, l, cctx, 4) + appendOK(t, l) + _, err := l.Append(ctx, block(blk(104, 0))) + wantErr(t, "rejected Append", err, chainlog.ErrMalformed, "") + head(t, l) // give a wrongly woken waiter time to return + head(t, l) + cancel() + wantErr(t, "blocked Wait(4) after an empty and a rejected Append", receive(t, done, "Wait(4)"), context.Canceled, "") +} + +// waitLimit bounds every wait of the suite, so a broken log fails a subtest instead of hanging. +const waitLimit = 30 * time.Second + +// signalCtx is a context that reports, once, that a caller asked for its Done channel. A Wait that +// blocks asks for it. +type signalCtx struct { + context.Context + asked chan struct{} + once sync.Once +} + +func (c *signalCtx) Done() <-chan struct{} { + c.once.Do(func() { close(c.asked) }) + return c.Context.Done() +} + +// blockedWait starts Wait(ctx, after) and returns its result channel once Wait is blocked, that +// is, once it asked for ctx.Done. +func blockedWait(t *testing.T, l chainlog.Log, ctx context.Context, after chainlog.Offset) <-chan error { + t.Helper() + sc := &signalCtx{Context: ctx, asked: make(chan struct{})} + done := make(chan error, 1) + go func() { done <- l.Wait(sc, after) }() + select { + case <-sc.asked: + case err := <-done: + t.Fatalf("Wait(%d) returned %v instead of blocking", after, err) + case <-time.After(waitLimit): + t.Fatalf("Wait(%d) neither blocked nor returned within %v", after, waitLimit) + } + return done +} + +// receive returns the result of a waiter, failing the test when it takes longer than waitLimit. +func receive(t *testing.T, done <-chan error, what string) error { + t.Helper() + select { + case err := <-done: + return err + case <-time.After(waitLimit): + t.Fatalf("%s did not return within %v", what, waitLimit) + return nil + } +} + +func testTruncate(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + ctx := t.Context() + in := append(blocks(101, 106), finalize(blk(104, 0))) + in = append(in, blocks(107, 108)...) + all := numbered(in...) + appendOK(t, l, in...) + meta, err := l.Meta(ctx) + noErr(t, err, "Meta") + before := viewOf(state(t, l)) + + for _, b := range []chainlog.Offset{0, 1} { + noErr(t, l.Truncate(ctx, b), fmt.Sprintf("Truncate(%d)", b)) + eq(t, fmt.Sprintf("Oldest after Truncate(%d)", b), oldest(t, l), chainlog.Offset(1)) + } + // The consumer's bound: offsets from 5 on stay. + noErr(t, l.Truncate(ctx, 5), "Truncate(5)") + if o := oldest(t, l); o < 1 || o > 5 { + t.Errorf("Oldest after Truncate(5) = %d, want 1 to 5", o) + } + wantEntries(t, read(t, l, 5, 100), all[4:]) + // A rejected batch moves no bound, also when it holds a Finalize. + _, err = l.Append(ctx, finalize(blk(108, 0)), block(blk(108, 0))) + wantErr(t, "Append of a rejected batch with a Finalize", err, chainlog.ErrMalformed, "") + noErr(t, l.Truncate(ctx, 100), "Truncate(100)") + o := oldest(t, l) + // The newest Finalize is offset 7, so offsets 8 and 9 stay. + if o < 1 || o > 8 { + t.Errorf("Oldest after Truncate(100) = %d, want 1 to 8", o) + } + wantEntries(t, read(t, l, o, 100), all[o-1:]) + if o > 1 { + _, err := l.Read(ctx, 1, 1) + wantErr(t, "Read(1, 1)", err, chainlog.ErrTruncated, fmt.Sprintf("chainlog: offset truncated: read from 1, oldest is %d", o)) + } + eq(t, "Head", head(t, l), chainlog.Offset(9)) + wantView(t, l, before) + after, err := l.Meta(ctx) + noErr(t, err, "Meta") + eq(t, "Meta after Truncate", after, meta) + eq(t, "Append after Truncate", appendOK(t, l, block(blk(109, 0))), chainlog.Offset(10)) + eq(t, "Oldest after Append", oldest(t, l), o) +} + +func testTruncateBound(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + appendOK(t, l, blocks(101, 105)...) + noErr(t, l.Truncate(t.Context(), 10), "Truncate(10)") + eq(t, "Oldest without a Finalize", oldest(t, l), chainlog.Offset(1)) + wantEntries(t, read(t, l, 1, 10), numbered(blocks(101, 105)...)) +} + +func testClosed(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + ctx := t.Context() + appendOK(t, l, block(blk(101, 0))) + done := blockedWait(t, l, ctx, 1) + noErr(t, l.Close(), "Close") + // The contract returns ErrClosed unwrapped, so compare with ==. + same(t, "Wait blocked before Close", receive(t, done, "Wait(1)"), chainlog.ErrClosed) + for _, c := range calls() { + same(t, c.name+" after Close", c.call(ctx, l), chainlog.ErrClosed) + } + // Closed comes before a cancelled context and before the arguments. + cancelled, cancel := context.WithCancel(ctx) + cancel() + for _, c := range calls() { + same(t, c.name+" after Close with a cancelled context", c.call(cancelled, l), chainlog.ErrClosed) + } + eq(t, "second Close", l.Close(), error(nil)) +} + +// call is one method call of l with ctx, reduced to its error. +type call struct { + name string + call func(ctx context.Context, l chainlog.Log) error +} + +// calls lists a call of every method of l except Close. The last four have bad arguments, which a +// log checks only after the closed state and the context. +func calls() []call { + return []call{ + {"Meta", func(ctx context.Context, l chainlog.Log) error { _, err := l.Meta(ctx); return err }}, + {"Append", func(ctx context.Context, l chainlog.Log) error { + _, err := l.Append(ctx, block(blk(102, 0))) + return err + }}, + {"Read", func(ctx context.Context, l chainlog.Log) error { _, err := l.Read(ctx, 1, 1); return err }}, + {"Wait", func(ctx context.Context, l chainlog.Log) error { return l.Wait(ctx, 0) }}, + {"Head", func(ctx context.Context, l chainlog.Log) error { _, err := l.Head(ctx); return err }}, + {"Oldest", func(ctx context.Context, l chainlog.Log) error { _, err := l.Oldest(ctx); return err }}, + {"State", func(ctx context.Context, l chainlog.Log) error { _, err := l.State(ctx); return err }}, + {"Truncate", func(ctx context.Context, l chainlog.Log) error { return l.Truncate(ctx, 1) }}, + {"Append()", func(ctx context.Context, l chainlog.Log) error { _, err := l.Append(ctx); return err }}, + {"Append(malformed)", func(ctx context.Context, l chainlog.Log) error { _, err := l.Append(ctx, chainlog.Entry{}); return err }}, + {"Wait(99)", func(ctx context.Context, l chainlog.Log) error { return l.Wait(ctx, 99) }}, + {"Read(1, 0)", readRecovered}, + } +} + +// readRecovered calls Read(ctx, 1, 0) and turns its panic into an error. +func readRecovered(ctx context.Context, l chainlog.Log) (err error) { + defer func() { + if r := recover(); r != nil { + err = fmt.Errorf("panic: %v", r) + } + }() + _, err = l.Read(ctx, 1, 0) + return err +} + +func testCancelledContext(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + appendOK(t, l, block(blk(101, 0))) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + for _, c := range calls() { + same(t, c.name+" with a cancelled context", c.call(ctx, l), context.Canceled) + } + eq(t, "Head after an Append with a cancelled context", head(t, l), chainlog.Offset(1)) +} + +func testConcurrent(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + const first, last = 101, 2100 // 2,000 linked blocks + var batches [][]chainlog.Entry + var want []chainlog.Entry + window := 0 + for n, i := uint64(first), 0; n <= last; i++ { + end := min(n+uint64(i%3), last) // batch sizes cycle 1, 2, 3 + b := blocks(n, end) + window += len(b) + batches = append(batches, b) + n = end + 1 + if window >= 5 { + batches = append(batches, []chainlog.Entry{finalize(blk(end, 0))}) + window = 0 + } + } + for _, b := range batches { + want = append(want, b...) + } + want = numbered(want...) + total := chainlog.Offset(len(want)) //nolint:gosec // a few thousand entries + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + const readers = 8 + got := make([][]chainlog.Entry, readers) + errs := make([]error, readers) + var wg sync.WaitGroup + for r := range readers { + wg.Go(func() { got[r], errs[r] = follow(ctx, l, total) }) + } + var appendErr error + for _, b := range batches { + if _, appendErr = l.Append(ctx, b...); appendErr != nil { + cancel() // stop the readers + break + } + } + finished := make(chan struct{}) + go func() { + wg.Wait() + close(finished) + }() + select { + case <-finished: + case <-time.After(waitLimit): + cancel() + t.Fatalf("readers did not finish within %v", waitLimit) + } + noErr(t, appendErr, "Append") + for r := range readers { + noErr(t, errs[r], "reader") + eq(t, "a reader's entries", got[r], want) + } +} + +// follow reads l with Wait and Read(from, 7), as a consumer does, until it holds offset last. +func follow(ctx context.Context, l chainlog.Log, last chainlog.Offset) ([]chainlog.Entry, error) { + var got []chainlog.Entry + for from := chainlog.Offset(1); from <= last; { + if err := l.Wait(ctx, from-1); err != nil { + return got, err + } + es, err := l.Read(ctx, from, 7) + if err != nil { + return got, err + } + got = append(got, es...) + from += chainlog.Offset(len(es)) //nolint:gosec // at most 7 + } + return got, nil +} + +func testBatchVisibility(t *testing.T, l chainlog.Log, _ func() chainlog.Log) { + const batches = 300 + ctx, cancel := context.WithCancel(t.Context()) + appended := make(chan error, 1) + defer func() { + cancel() + if appended != nil { + <-appended // the appender ends: Append returns once ctx is cancelled + } + }() + go func() { + for i := range uint64(batches) { + a, b := blk(101+2*i, 0), blk(102+2*i, 0) + if _, err := l.Append(ctx, block(a), block(b), finalize(b)); err != nil { + appended <- err + return + } + } + appended <- nil + }() + var inside chainlog.Offset // a Head that is not a multiple of 3, if any + deadline := time.Now().Add(waitLimit) + for { + if time.Now().After(deadline) { + t.Fatalf("Head did not reach %d within %v", 3*batches, waitLimit) + } + h, err := l.Head(ctx) + noErr(t, err, "Head") + if h%3 != 0 { + inside = h + break + } + if h == 3*batches { + break + } + select { + case err := <-appended: + noErr(t, err, "Append") + appended = nil // keep polling: Head reaches 900 + default: + } + } + eq(t, "a Head inside a batch of 3", inside, chainlog.Offset(0)) +} diff --git a/chainlog/logtest/doc.go b/chainlog/logtest/doc.go new file mode 100644 index 0000000..9b31e6c --- /dev/null +++ b/chainlog/logtest/doc.go @@ -0,0 +1,5 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +// Package logtest is the conformance suite for chainlog.Log implementations. +package logtest diff --git a/chainlog/logtest/logtest.go b/chainlog/logtest/logtest.go new file mode 100644 index 0000000..215610a --- /dev/null +++ b/chainlog/logtest/logtest.go @@ -0,0 +1,72 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package logtest + +import ( + "encoding/binary" + "testing" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +// WindowCap is the window cap every log under test must have. +const WindowCap = 8 + +// Meta returns the metadata every log under test must be created with: ID +// 000102030405060708090a0b0c0d0e0f, ChainID 31337, Start 100, Filter with one address +// 0x00000000000000000000000000000000000000aa and Topics [[0x00…01]] (last byte 1). +func Meta() chainlog.Meta { + m := chainlog.Meta{ + ID: chainlog.ID{0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x09, 0x0a, 0x0b, 0x0c, 0x0d, 0x0e, 0x0f}, + ChainID: 31337, + Start: 100, + } + m.Filter.Addresses = append(m.Filter.Addresses, [20]byte{19: 0xaa}) + m.Filter.Topics = append(m.Filter.Topics, nil) + m.Filter.Topics[0] = append(m.Filter.Topics[0], [32]byte{31: 0x01}) + return m +} + +// Run runs the conformance suite. open must return a new, empty log created with Meta() and +// WindowCap; Run calls it once per subtest (twice in the subtests that need a second empty log) +// and closes each log when the subtest ends. Wait must select on ctx.Done() while it blocks: Run +// detects a blocked Wait that way. +func Run(t *testing.T, open func(t *testing.T) chainlog.Log) { + t.Helper() + for _, c := range cases() { + t.Run(c.name, func(t *testing.T) { + fresh := func() chainlog.Log { + l := open(t) + t.Cleanup(func() { _ = l.Close() }) + return l + } + c.run(t, fresh(), fresh) + }) + } +} + +// h returns the hash with byte 0 = 0xb0|f, bytes 24..31 = n big-endian and all others zero. +func h(n uint64, f byte) [32]byte { + var x [32]byte + x[0] = 0xb0 | f + binary.BigEndian.PutUint64(x[24:], n) + return x +} + +// blk returns block n of chain f, linked to block n-1 of the same chain (no parent for n = 0). +func blk(n uint64, f byte) chain.BlockRef { + r := chain.BlockRef{Number: n, Hash: h(n, f), Time: 1_700_000_000 + 12*n} + if n > 0 { + r.Parent = h(n-1, f) + } + return r +} + +// fork returns blk(n, f) with the parent of chain base: the first block of a fork off base. +func fork(n uint64, f, base byte) chain.BlockRef { + r := blk(n, f) + r.Parent = h(n-1, base) + return r +} diff --git a/chainlog/logtest/logtest_test.go b/chainlog/logtest/logtest_test.go new file mode 100644 index 0000000..ad4f365 --- /dev/null +++ b/chainlog/logtest/logtest_test.go @@ -0,0 +1,18 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package logtest_test + +import ( + "testing" + + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/chainlog/logtest" +) + +// TestRun runs the suite on the in-memory log, so the suite's own code is measured by coverage. +func TestRun(t *testing.T) { + logtest.Run(t, func(t *testing.T) chainlog.Log { + return chainlog.NewMemory(logtest.Meta(), logtest.WindowCap) + }) +} diff --git a/chainlog/memory.go b/chainlog/memory.go new file mode 100644 index 0000000..0593746 --- /dev/null +++ b/chainlog/memory.go @@ -0,0 +1,257 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import ( + "context" + "crypto/rand" + "fmt" + "slices" + "strconv" + "sync" + + "github.com/ethereum/go-ethereum/common" +) + +// Memory is a Log held in memory. It never truncates on its own. It is safe for concurrent use. +// +// Memory stores each entry as its encoded record, so it rejects exactly the entries a file log +// rejects, and every Read returns fresh copies. +type Memory struct { + meta Meta // normalized; never changes + + mu sync.Mutex + recs [][]byte // the records of offsets oldest..head; guarded by mu + oldest Offset // guarded by mu + head Offset // guarded by mu + lastFin Offset // offset of the newest Finalize entry, 0 if none; guarded by mu + state State // guarded by mu + closed bool // guarded by mu + wake chan struct{} // closed and replaced by every Append and by Close; guarded by mu +} + +var _ Log = (*Memory)(nil) + +// NewMemory returns an empty Memory log with metadata meta and window cap cap (0 means +// DefaultWindowCap). A zero meta.ID is replaced by 16 bytes from crypto/rand. It panics if +// cap < 0 or meta.Filter.Validate fails. +func NewMemory(meta Meta, cap int) *Memory { + if cap < 0 { + panic("evmsync: chainlog.NewMemory: cap must not be negative, got " + strconv.Itoa(cap)) + } + if err := meta.Filter.Validate(); err != nil { + panic("evmsync: chainlog.NewMemory: " + err.Error()) + } + meta = normalizeMeta(meta) + if meta.ID.IsZero() { + _, _ = rand.Read(meta.ID[:]) // never fails since Go 1.24 + } + return &Memory{meta: meta, oldest: 1, state: NewState(cap), wake: make(chan struct{})} +} + +// normalizeMeta returns a deep copy of m whose empty filter lists are nil. +func normalizeMeta(m Meta) Meta { + m.Filter.Addresses = cloneOrNil(m.Filter.Addresses) + if len(m.Filter.Topics) == 0 { + m.Filter.Topics = nil + return m + } + topics := make([][]common.Hash, len(m.Filter.Topics)) + for i, pos := range m.Filter.Topics { + topics[i] = cloneOrNil(pos) + } + m.Filter.Topics = topics + return m +} + +func cloneOrNil[S ~[]E, E any](s S) S { + if len(s) == 0 { + return nil + } + return slices.Clone(s) +} + +// check returns ErrClosed or ctx.Err(), in that order. The caller holds m.mu. +func (m *Memory) check(ctx context.Context) error { + if m.closed { + return ErrClosed + } + return ctx.Err() +} + +// Meta returns the log's metadata. +func (m *Memory) Meta(ctx context.Context) (Meta, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.check(ctx); err != nil { + return Meta{}, err + } + return normalizeMeta(m.meta), nil +} + +// Append validates entries in order against the log's State and appends all of them, or none +// with an error wrapping ErrMalformed. It assigns offsets Head+1, Head+2, ... and returns the +// last one; with no entries it returns Head. +func (m *Memory) Append(ctx context.Context, entries ...Entry) (Offset, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.check(ctx); err != nil { + return 0, err + } + if len(entries) == 0 { + return m.head, nil + } + if i, err := m.state.CheckBatch(entries); err != nil { + return 0, fmt.Errorf("chainlog: append entry %d of %d: %w", i+1, len(entries), err) + } + for i := range entries { + e := entries[i] + m.head++ + e.Offset = m.head + m.recs = append(m.recs, Encode(make([]byte, 0, headerSize+recordBodySize(e)), e)) + m.state.Apply(e) + if e.Kind == KindFinalize { + m.lastFin = e.Offset + } + } + close(m.wake) + m.wake = make(chan struct{}) + return m.head, nil +} + +// Read returns up to max entries starting at offset from, in offset order. It returns nil when +// from is Head+1, an error wrapping ErrTruncated when from is below Oldest, and an error wrapping +// ErrAhead when from is above Head+1. A from of 0 reads from 1. It panics if max < 1. +func (m *Memory) Read(ctx context.Context, from Offset, max int) ([]Entry, error) { + recs, err := m.records(ctx, from, max) + if len(recs) == 0 { + return nil, err + } + // Records never change once appended, so they are decoded without the lock. + es := make([]Entry, len(recs)) + for i, rec := range recs { + e, _, err := Decode(rec) + if err != nil { + // Unreachable: every record was encoded by Append. + return nil, fmt.Errorf("chainlog: read record %d of %d: %w", i+1, len(recs), err) + } + es[i] = e + } + return es, nil +} + +// records returns the records Read decodes. It copies the slots, since Truncate clears them. +func (m *Memory) records(ctx context.Context, from Offset, max int) ([][]byte, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.check(ctx); err != nil { + return nil, err + } + if max < 1 { + panic("evmsync: chainlog: Read max must be at least 1, got " + strconv.Itoa(max)) + } + if from == 0 { + from = 1 + } + switch { + case from < m.oldest: + return nil, fmt.Errorf("%w: read from %d, oldest is %d", ErrTruncated, from, m.oldest) + case from == m.head+1: + return nil, nil + case from > m.head+1: + return nil, fmt.Errorf("%w: read from %d, head is %d", ErrAhead, from, m.head) + } + n := min(uint64(max), uint64(m.head-from)+1) + return slices.Clone(m.recs[from-m.oldest:][:n]), nil +} + +// Wait returns nil once Head > after, ctx.Err() if ctx ends first, ErrClosed if the log closes +// first, and an error wrapping ErrAhead when after > Head. +func (m *Memory) Wait(ctx context.Context, after Offset) error { + for { + m.mu.Lock() + if err := m.check(ctx); err != nil { + m.mu.Unlock() + return err + } + head, wake := m.head, m.wake + m.mu.Unlock() + switch { + case head > after: + return nil + case after > head: + return fmt.Errorf("%w: wait after %d, head is %d", ErrAhead, after, head) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-wake: + } + } +} + +// Head returns the offset of the newest entry, 0 if none was ever appended. +func (m *Memory) Head(ctx context.Context) (Offset, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.check(ctx); err != nil { + return 0, err + } + return m.head, nil +} + +// Oldest returns the offset of the first retained entry, Head+1 when none is retained. +func (m *Memory) Oldest(ctx context.Context) (Offset, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.check(ctx); err != nil { + return 0, err + } + return m.oldest, nil +} + +// State returns a copy of the validator state after the newest entry. +func (m *Memory) State(ctx context.Context) (State, error) { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.check(ctx); err != nil { + return State{}, err + } + return m.state.Clone(), nil +} + +// Truncate drops the retained entries with offsets below min(before, L+1), where L is the offset +// of the newest Finalize entry (0 if none), and releases their memory. If that bound is above +// Oldest, Oldest becomes the bound. +func (m *Memory) Truncate(ctx context.Context, before Offset) error { + m.mu.Lock() + defer m.mu.Unlock() + if err := m.check(ctx); err != nil { + return err + } + b := min(before, m.lastFin+1) + if b <= m.oldest { + return nil + } + drop := b - m.oldest + clear(m.recs[:drop]) + m.recs = m.recs[drop:] + m.oldest = b + return nil +} + +// Close releases the log's memory and wakes every Wait with ErrClosed. Later calls of other +// methods return ErrClosed; later calls of Close return nil. +func (m *Memory) Close() error { + m.mu.Lock() + defer m.mu.Unlock() + if m.closed { + return nil + } + m.closed = true + m.recs = nil + m.state = State{} + close(m.wake) + return nil +} diff --git a/chainlog/memory_test.go b/chainlog/memory_test.go new file mode 100644 index 0000000..7b29f85 --- /dev/null +++ b/chainlog/memory_test.go @@ -0,0 +1,167 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import ( + "errors" + "reflect" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" +) + +func TestMemoryMeta(t *testing.T) { + ctx := t.Context() + topic := common.Hash{31: 7} + addrs := []common.Address{} + topics := [][]common.Hash{{}, {topic}} + m := NewMemory(Meta{ChainID: 1, Filter: chain.Filter{Addresses: addrs, Topics: topics}}, 0) + topics[1][0] = common.Hash{1} // the log keeps its own copy + got, err := m.Meta(ctx) + if err != nil { + t.Fatalf("Meta: %v", err) + } + if got.ID.IsZero() { + t.Error("Meta().ID is zero, want a random ID") + } + if got.ChainID != 1 || got.Filter.Addresses != nil || !reflect.DeepEqual(got.Filter.Topics, [][]common.Hash{nil, {topic}}) { + t.Errorf("Meta() = %+v, want ChainID 1, nil Addresses, Topics [nil [topic]]", got) + } + + other, _ := NewMemory(Meta{}, 0).Meta(ctx) + if other.ID == got.ID { + t.Errorf("two logs created with a zero ID share ID %s", got.ID) + } + id := ID{1, 2, 3} + kept, _ := NewMemory(Meta{ID: id}, 0).Meta(ctx) + if kept.ID != id { + t.Errorf("Meta().ID = %s, want the given %s", kept.ID, id) + } + + five := Meta{Filter: chain.Filter{Topics: make([][]common.Hash, 5)}} + if got, want := panicText(func() { NewMemory(five, 0) }), "evmsync: chainlog.NewMemory: chain: invalid filter: 5 topic positions, at most 4"; got != want { + t.Errorf("NewMemory with 5 topic positions: panic %q, want %q", got, want) + } + if got, want := panicText(func() { NewMemory(Meta{}, -1) }), "evmsync: chainlog.NewMemory: cap must not be negative, got -1"; got != want { + t.Errorf("NewMemory with cap -1: panic %q, want %q", got, want) + } +} + +func TestMemoryTruncateExact(t *testing.T) { + ctx := t.Context() + m := NewMemory(Meta{}, 0) + in := append(blocks(1, 10), finalize(blk(8, 0))) + in = append(in, blocks(11, 12)...) + if _, err := m.Append(ctx, in...); err != nil { + t.Fatalf("Append: %v", err) + } + all := m.recs // the backing store before any Truncate + + if err := m.Truncate(ctx, 6); err != nil { + t.Fatalf("Truncate(6): %v", err) + } + if o, _ := m.Oldest(ctx); o != 6 { + t.Errorf("Oldest after Truncate(6) = %d, want 6", o) + } + if err := m.Truncate(ctx, 100); err != nil { + t.Fatalf("Truncate(100): %v", err) + } + if o, _ := m.Oldest(ctx); o != 12 { + t.Errorf("Oldest after Truncate(100) = %d, want 12 (the Finalize is offset 11)", o) + } + _, err := m.Read(ctx, 11, 1) + if !errors.Is(err, ErrTruncated) { + t.Fatalf("Read(11, 1) = %v, want one wrapping ErrTruncated", err) + } + if got, want := err.Error(), "chainlog: offset truncated: read from 11, oldest is 12"; got != want { + t.Errorf("Read(11, 1) error = %q, want %q", got, want) + } + + if uint64(len(m.recs)) != uint64(m.head-m.oldest)+1 { + t.Errorf("backing store holds %d records, want Head - Oldest + 1 = %d", len(m.recs), m.head-m.oldest+1) + } + for i, rec := range all[:11] { + if rec != nil { + t.Errorf("dropped record %d is still referenced", i+1) + } + } + for i, rec := range all[11:] { + if rec == nil { + t.Errorf("retained record %d was cleared", i+12) + } + } + + // A Finalize as the newest entry lets Truncate drop everything. + if _, err := m.Append(ctx, finalize(blk(12, 0))); err != nil { + t.Fatalf("Append: %v", err) + } + if err := m.Truncate(ctx, 100); err != nil { + t.Fatalf("Truncate(100): %v", err) + } + if o, _ := m.Oldest(ctx); o != 15 { + t.Errorf("Oldest with nothing retained = %d, want Head+1 = 15", o) + } + if len(m.recs) != 0 { + t.Errorf("backing store holds %d records, want 0", len(m.recs)) + } + if es, err := m.Read(ctx, 15, 1); es != nil || err != nil { + t.Errorf("Read(15, 1) = %v, %v; want nil, nil", es, err) + } +} + +func TestMemoryClosedReleases(t *testing.T) { + m := NewMemory(Meta{}, 0) + if _, err := m.Append(t.Context(), blocks(1, 3)...); err != nil { + t.Fatalf("Append: %v", err) + } + if err := m.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + if m.recs != nil { + t.Errorf("Close kept %d records", len(m.recs)) + } +} + +// TestAppendCostFlat compares the cost of Append with a window of 99,000 refs and of 10 refs. Each +// of the 1,000 steps appends a block and a Finalize of the window's first ref, so the window keeps +// its length. +func TestAppendCostFlat(t *testing.T) { + if raceEnabled { + t.Skip("the race detector distorts the timing") + } + cost := func(window uint64) time.Duration { + ctx := t.Context() + m := NewMemory(Meta{}, 0) + if _, err := m.Append(ctx, blocks(1, window)...); err != nil { + t.Fatalf("Append: %v", err) + } + next, first := window+1, uint64(1) + best := time.Duration(-1) + for range 3 { + start := time.Now() + for range 1000 { + // One block in, the window's first ref finalized: the window keeps its length. + if _, err := m.Append(ctx, block(blk(next, 0))); err != nil { + t.Fatalf("Append: %v", err) + } + if _, err := m.Append(ctx, finalize(blk(first, 0))); err != nil { + t.Fatalf("Append: %v", err) + } + next++ + first++ + } + if d := time.Since(start); best < 0 || d < best { + best = d + } + } + return best + } + small, large := cost(10), cost(99_000) + if ratio := float64(large) / float64(small); ratio > 2 { + t.Errorf("1,000 appends: %v with a window of 99,000, %v with 10; ratio %.2f, want at most 2", large, small, ratio) + } +} diff --git a/chainlog/norace_test.go b/chainlog/norace_test.go new file mode 100644 index 0000000..a9726f2 --- /dev/null +++ b/chainlog/norace_test.go @@ -0,0 +1,8 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +//go:build !race + +package chainlog + +const raceEnabled = false diff --git a/chainlog/perf_test.go b/chainlog/perf_test.go new file mode 100644 index 0000000..9cb5240 --- /dev/null +++ b/chainlog/perf_test.go @@ -0,0 +1,42 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import ( + "os" + "testing" + "time" +) + +func TestPerf(t *testing.T) { + if os.Getenv("EVMSYNC_PERF") != "1" || raceEnabled { + t.Skip("set EVMSYNC_PERF=1 and run without -race to measure the targets") + } + tests := []struct { + name string + bench func(*testing.B) + maxTime time.Duration + maxAllocs int64 // -1: not asserted + }{ + {"Encode", BenchmarkEncode, time.Microsecond, 0}, + {"Decode", BenchmarkDecode, 2 * time.Microsecond, 3}, + {"StateBlock", BenchmarkStateBlock, 300 * time.Nanosecond, 0}, + {"MemoryAppend", BenchmarkMemoryAppend, 5 * time.Microsecond, -1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := testing.Benchmark(tt.bench) + if r.N == 0 { + t.Fatal("benchmark did not run") + } + t.Logf("%d ns/op, %d allocs/op", r.NsPerOp(), r.AllocsPerOp()) + if got := time.Duration(r.NsPerOp()); got > tt.maxTime { + t.Errorf("%v/op, want at most %v", got, tt.maxTime) + } + if tt.maxAllocs >= 0 && r.AllocsPerOp() > tt.maxAllocs { + t.Errorf("%d allocs/op, want at most %d", r.AllocsPerOp(), tt.maxAllocs) + } + }) + } +} diff --git a/chainlog/race_test.go b/chainlog/race_test.go new file mode 100644 index 0000000..4fd7de2 --- /dev/null +++ b/chainlog/race_test.go @@ -0,0 +1,8 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +//go:build race + +package chainlog + +const raceEnabled = true diff --git a/chainlog/state.go b/chainlog/state.go new file mode 100644 index 0000000..0ed8a70 --- /dev/null +++ b/chainlog/state.go @@ -0,0 +1,326 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import ( + "cmp" + "fmt" + "math" + "slices" + "strconv" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" +) + +// State validates entries against the rules of the log and tracks the tip, the finalized ref F +// and the window: the refs of the non-reverted, non-final blocks above F, ascending by number. +// The zero State is an empty state with cap DefaultWindowCap. A State is a value: copies made +// with Clone are independent; plain assignment shares the window and must not be used after +// either copy changes. A State is not safe for concurrent use. +// +// Check accepts an entry when its fields fit its kind and: +// - a block is above the tip, and a block one above the tip has the tip's hash as parent. A +// non-final block is always one above the tip. A Final block needs an empty window, may be +// further above the tip, and becomes the tip and F; +// - a revert is below the tip and targets F or a window ref; +// - a finalize does not go below F, does not conflict with F or a window ref at its number, and +// links to the window ref above it and to the tip when it is the tip's child; +// - a non-final block does not grow the window beyond the cap; +// - every log of a block carries the block's number and hash, at most 4 topics, increasing +// indexes and non-decreasing transaction indexes, and the block's record fits MaxRecordSize. +type State struct { + cap int // the window cap; 0 means DefaultWindowCap + tip chain.BlockRef // the newest non-reverted block ref + fin chain.BlockRef // F, the finalized ref + win []chain.BlockRef // the window; Finalize re-slices its front, Block appends at its back +} + +// NewState returns an empty State whose window holds at most cap refs. cap 0 means +// DefaultWindowCap. It panics if cap < 0. +func NewState(cap int) State { + if cap < 0 { + panic("evmsync: chainlog.NewState: cap must not be negative, got " + strconv.Itoa(cap)) + } + return State{cap: cap} +} + +// Tip returns the newest non-reverted block ref, or the zero ref. +func (s State) Tip() chain.BlockRef { return s.tip } + +// Finalized returns F, or the zero ref. +func (s State) Finalized() chain.BlockRef { return s.fin } + +// Window returns a new slice of the window's refs in ascending number order, nil when empty. +func (s State) Window() []chain.BlockRef { + if len(s.win) == 0 { + return nil + } + return slices.Clone(s.win) +} + +// Clone returns an independent copy of s. +func (s State) Clone() State { + s.win = s.Window() + return s +} + +// Check returns nil if e may follow the entries already applied, and otherwise an error wrapping +// ErrMalformed. It does not change s. +func (s *State) Check(e Entry) error { + if err := checkShape(&e); err != nil { + return err + } + switch e.Kind { + case KindBlock: + if n := recordBodySize(e); n > MaxRecordSize { + return malformed("block %d: record body of %d bytes exceeds %d", e.Block.Ref.Number, n, MaxRecordSize) + } + return s.checkBlock(&e) + case KindRevert: + return s.checkRevert(e.To) + default: + return s.checkFinalize(e.To) + } +} + +// CheckBatch checks entries in order, each against the state after the ones before it. It +// returns -1 and nil when all pass, and otherwise the index of the first failing entry and its +// Check error. s is unchanged when CheckBatch returns; it changes s while it runs, so callers +// hold the lock that guards s. +// +// Its cost grows with len(entries) and with the refs a Revert in the batch drops, never with the +// window's length. +func (s *State) CheckBatch(entries []Entry) (int, error) { + // Room for every block of the batch, made once: an append that grows the window inside the + // loop would copy all of it, and the undo would throw the copy away. + blocks := 0 + for i := range entries { + if entries[i].Kind == KindBlock && !entries[i].Final { + blocks++ + } + } + if cap(s.win)-len(s.win) < blocks { + s.win = slices.Grow(s.win, blocks) + } + var buf [8]undo + undos := buf[:0] + for i := range entries { + e := &entries[i] + if err := s.Check(*e); err != nil { + s.undo(undos) + return i, err + } + u := undo{tip: s.tip, fin: s.fin, win: s.win} + if e.Kind == KindRevert { + // Later blocks of the batch may overwrite the dropped refs in the backing array. + if dropped := s.win[s.above(e.To.Number):]; len(dropped) > 0 { + u.dropped = slices.Clone(dropped) + } + } + undos = append(undos, u) + s.Apply(*e) + } + s.undo(undos) + return -1, nil +} + +// undo is what CheckBatch needs to take back one applied entry. +type undo struct { + tip, fin chain.BlockRef + win []chain.BlockRef // the window's slice header before the entry + dropped []chain.BlockRef // a copy of the refs a Revert dropped from the window's end +} + +// undo takes back the entries of us, newest first. +func (s *State) undo(us []undo) { + for i := len(us) - 1; i >= 0; i-- { + u := &us[i] + s.tip, s.fin, s.win = u.tip, u.fin, u.win + copy(s.win[len(s.win)-len(u.dropped):], u.dropped) + } +} + +// Apply records e. It is meant for entries that passed Check; for any other entry it does not +// panic, and the resulting state is unspecified. +func (s *State) Apply(e Entry) { + switch e.Kind { + case KindBlock: + r := e.Block.Ref + s.tip = r + if e.Final { + s.fin = r + s.win = s.win[:0] + return + } + // append copies only the live refs when it grows, which drops the prefix Finalize + // re-sliced away. + s.win = append(s.win, r) + case KindRevert: + s.tip = e.To + s.win = s.win[:s.above(e.To.Number)] + case KindFinalize: + s.fin = e.To + s.win = s.win[s.above(e.To.Number):] + if isZero(s.tip) || e.To.Number > s.tip.Number { + s.tip = e.To + } + } +} + +func (s *State) windowCap() int { + if s.cap == 0 { + return DefaultWindowCap + } + return s.cap +} + +// find returns the index of the window ref numbered n, or the index where it would be and false. +func (s *State) find(n uint64) (int, bool) { + if len(s.win) == 0 || n < s.win[0].Number { + return 0, false + } + if n > s.win[len(s.win)-1].Number { + return len(s.win), false + } + // In a window Check built, numbers are contiguous, so the ref numbered n is at n - first. + // Binary search handles the windows with gaps that only Apply builds. + if d := n - s.win[0].Number; d < uint64(len(s.win)) && s.win[d].Number == n { + return int(d), true //nolint:gosec // d < len(s.win) + } + return slices.BinarySearchFunc(s.win, n, func(r chain.BlockRef, n uint64) int { return cmp.Compare(r.Number, n) }) +} + +// above returns the index of the first window ref numbered above n. +func (s *State) above(n uint64) int { + i, ok := s.find(n) + if ok { + i++ + } + return i +} + +func (s *State) checkBlock(e *Entry) error { + r := e.Block.Ref + if tip := s.tip; !isZero(tip) { + if r.Number <= tip.Number { + return malformed("block %d is not above tip %s", r.Number, tip) + } + if r.Number-1 == tip.Number { + if r.Parent != tip.Hash { + return malformed("block %d: parent %s is not tip hash %s", r.Number, r.Parent.Hex(), tip.Hash.Hex()) + } + } else if !e.Final { + return malformed("block %d: non-final block is not the child of tip %s", r.Number, tip) + } + } + if e.Final { + if len(s.win) > 0 { + return malformed("block %d: final block with %d unfinalized blocks in the window", r.Number, len(s.win)) + } + return nil + } + if c := s.windowCap(); len(s.win) >= c { + return malformed("block %d: window is full (%d blocks)", r.Number, c) + } + return nil +} + +func (s *State) checkRevert(to chain.BlockRef) error { + if isZero(s.tip) || to.Number >= s.tip.Number { + return malformed("revert to %s: target is not below tip %s", to, s.tip) + } + if to == s.fin { + return nil // To has a non-zero hash, so it never matches a zero F + } + if i, ok := s.find(to.Number); ok && s.win[i] == to { + return nil + } + return malformed("revert to %s: target is neither the finalized block nor in the window", to) +} + +func (s *State) checkFinalize(to chain.BlockRef) error { + if f := s.fin; !isZero(f) { + if to.Number < f.Number { + return malformed("finalize to %s: below finalized %s", to, f) + } + if to.Number == f.Number { + if to != f { + return malformed("finalize to %s: conflicts with finalized %s", to, f) + } + return nil // a repeated Finalize + } + } + if i, ok := s.find(to.Number); ok && s.win[i] != to { + return malformed("finalize to %s: conflicts with window block %s", to, s.win[i]) + } + if to.Number < math.MaxUint64 { + if i, ok := s.find(to.Number + 1); ok && s.win[i].Parent != to.Hash { + return malformed("finalize to %s: window block %s does not link to it", to, s.win[i]) + } + } + if t := s.tip; !isZero(t) && to.Number > 0 && to.Number-1 == t.Number && to.Parent != t.Hash { + return malformed("finalize to %s: parent %s is not tip hash %s", to, to.Parent.Hex(), t.Hash.Hex()) + } + return nil +} + +// checkShape checks the fields of e alone: its kind, the fields its kind leaves zero, non-zero +// hashes and the logs of a block. +func checkShape(e *Entry) error { + switch e.Kind { + case KindBlock: + r := e.Block.Ref + if !isZero(e.To) { + return malformed("block %d: To must be zero", r.Number) + } + if r.Hash == (common.Hash{}) { + return malformed("block %d: zero hash", r.Number) + } + return checkLogs(r, e.Block.Logs) + case KindRevert, KindFinalize: + if !isZero(e.Block.Ref) || len(e.Block.Logs) > 0 || e.Final { + return malformed("%s entry: Block and Final must be zero", e.Kind) + } + if e.To.Hash == (common.Hash{}) { + return malformed("%s to %d: zero hash", e.Kind, e.To.Number) + } + return nil + default: + return malformed("unknown kind %d", uint8(e.Kind)) + } +} + +func checkLogs(r chain.BlockRef, logs []chain.Log) error { + for i := range logs { + l := &logs[i] + if l.BlockNumber != r.Number { + return malformed("block %d: log %d: block number %d", r.Number, i, l.BlockNumber) + } + if l.BlockHash != r.Hash { + return malformed("block %d: log %d: block hash %s", r.Number, i, l.BlockHash.Hex()) + } + if len(l.Topics) > maxTopics { + return malformed("block %d: log %d: %d topics, at most %d", r.Number, i, len(l.Topics), maxTopics) + } + if i == 0 { + continue + } + prev := &logs[i-1] + if l.Index <= prev.Index { + return malformed("block %d: log %d: index %d not above %d", r.Number, i, l.Index, prev.Index) + } + if l.TxIndex < prev.TxIndex { + return malformed("block %d: log %d: tx index %d below %d", r.Number, i, l.TxIndex, prev.TxIndex) + } + } + return nil +} + +func isZero(r chain.BlockRef) bool { return r == chain.BlockRef{} } + +func malformed(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrMalformed, fmt.Sprintf(format, args...)) +} diff --git a/chainlog/state_binary.go b/chainlog/state_binary.go new file mode 100644 index 0000000..f99f40b --- /dev/null +++ b/chainlog/state_binary.go @@ -0,0 +1,174 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import ( + "encoding/binary" + "fmt" + "strconv" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" +) + +const ( + stateVersion = 1 + // minRefSize is the smallest encoded ref: two one-byte varints around the two hashes. + minRefSize = 1 + 2*common.HashLength + 1 +) + +// MarshalBinary encodes s: version 1, a flags byte of 0, the tip and F, the window's length as +// a varint and its refs. A ref is its number and time as varints around its hash and parent; the +// zero ref is 66 bytes. The cap is not encoded. MarshalBinary never returns an error. +func (s State) MarshalBinary() ([]byte, error) { + size := 2 + refSize(s.tip) + refSize(s.fin) + binary.MaxVarintLen64 + for _, r := range s.win { + size += refSize(r) + } + b := make([]byte, 0, size) + b = append(b, stateVersion, 0) + b = appendRef(b, s.tip) + b = appendRef(b, s.fin) + b = binary.AppendUvarint(b, uint64(len(s.win))) + for _, r := range s.win { + b = appendRef(b, r) + } + return b, nil +} + +// UnmarshalBinary replaces s's tip, F and window with the decoded ones and keeps s's cap. It +// returns an error wrapping ErrBadState on malformed input, including a window that is not +// strictly ascending, not above F, or whose last ref (F when empty) is not the tip, and then +// leaves s unchanged. A window longer than s's cap is accepted; the next non-final block then +// fails Check. +func (s *State) UnmarshalBinary(data []byte) error { + d := stateDecoder{b: data} + version, err := d.byte("version") + if err != nil { + return err + } + if version != stateVersion { + return badState("version %d", version) + } + flags, err := d.byte("flags") + if err != nil { + return err + } + if flags != 0 { + return badState("flags 0x%02x", flags) + } + tip, err := d.ref("tip", -1) + if err != nil { + return err + } + fin, err := d.ref("finalized", -1) + if err != nil { + return err + } + n, err := d.uvarint("nwindow", -1) + if err != nil { + return err + } + // Bound the window by the remaining bytes before allocating it. + if n > uint64(len(d.b)/minRefSize) { + return badState("%d window refs cannot fit in %d bytes", n, len(d.b)) + } + var win []chain.BlockRef + if n > 0 { + win = make([]chain.BlockRef, n) + for i := range win { + if win[i], err = d.ref("window", i); err != nil { + return err + } + } + } + if len(d.b) > 0 { + return badState("%d trailing bytes", len(d.b)) + } + if err := checkDecoded(tip, fin, win); err != nil { + return err + } + s.tip, s.fin, s.win = tip, fin, win + return nil +} + +// checkDecoded checks the invariants every applied sequence keeps: the window is strictly +// ascending and above F, and the tip is its last ref, or F when it is empty. +func checkDecoded(tip, fin chain.BlockRef, win []chain.BlockRef) error { + for i := 1; i < len(win); i++ { + if win[i].Number <= win[i-1].Number { + return badState("window is not ascending") + } + } + if !isZero(fin) && len(win) > 0 && win[0].Number <= fin.Number { + return badState("window ref %d is not above finalized %d", win[0].Number, fin.Number) + } + if len(win) > 0 { + if tip != win[len(win)-1] { + return badState("tip %s is not the last window ref", tip) + } + } else if tip != fin { + return badState("tip %s is not the finalized ref", tip) + } + return nil +} + +// stateDecoder reads the fields of a state encoding in order. Its errors wrap ErrBadState. +type stateDecoder struct { + b []byte // the bytes not read yet +} + +func (d *stateDecoder) byte(field string) (byte, error) { + if len(d.b) == 0 { + return 0, badState("%s: truncated", field) + } + c := d.b[0] + d.b = d.b[1:] + return c, nil +} + +// uvarint reads a minimal unsigned varint. field and index name the field in the error (index -1: +// not a window ref). +func (d *stateDecoder) uvarint(field string, index int) (uint64, error) { + x, n := binary.Uvarint(d.b) + switch { + case n == 0: + return 0, badState("%s: truncated", stateField(field, index)) + case n < 0 || (n >= 2 && d.b[n-1] == 0): + return 0, badState("%s: bad varint", stateField(field, index)) + } + d.b = d.b[n:] + return x, nil +} + +func (d *stateDecoder) ref(field string, index int) (chain.BlockRef, error) { + var r chain.BlockRef + var err error + if r.Number, err = d.uvarint(field, index); err != nil { + return chain.BlockRef{}, err + } + if len(d.b) < 2*common.HashLength { + return chain.BlockRef{}, badState("%s: truncated", stateField(field, index)) + } + copy(r.Hash[:], d.b) + copy(r.Parent[:], d.b[common.HashLength:]) + d.b = d.b[2*common.HashLength:] + if r.Time, err = d.uvarint(field, index); err != nil { + return chain.BlockRef{}, err + } + return r, nil +} + +// stateField returns field, or "[]" for index >= 0. +func stateField(field string, index int) string { + if index < 0 { + return field + } + return field + "[" + strconv.Itoa(index) + "]" +} + +func badState(format string, args ...any) error { + return fmt.Errorf("%w: %s", ErrBadState, fmt.Sprintf(format, args...)) +} diff --git a/chainlog/state_binary_test.go b/chainlog/state_binary_test.go new file mode 100644 index 0000000..deda5bf --- /dev/null +++ b/chainlog/state_binary_test.go @@ -0,0 +1,143 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import ( + "bytes" + "errors" + "reflect" + "testing" + + "github.com/hmdsefi/evmsync/chain" +) + +// stateBytes is the binary form of a state with these refs, built by hand. +func stateBytes(tip, fin chain.BlockRef, win ...chain.BlockRef) []byte { + b := cat([]byte{0x01, 0x00}, refBytes(tip), refBytes(fin), uv(uint64(len(win)))) + for _, r := range win { + b = append(b, refBytes(r)...) + } + return b +} + +func TestStateBinary(t *testing.T) { + empty, _ := State{}.MarshalBinary() + if want := cat([]byte{0x01}, make([]byte, 134)); !bytes.Equal(empty, want) { + t.Errorf("empty state = %x, want 01 and 134 zero bytes", empty) + } + + // The empty state, every state of the apply sequence, and one after a Final block. + var states []State + var s State + states = append(states, s) + for _, st := range applySteps() { + s.Apply(st.e) + states = append(states, s.Clone()) + } + s.Apply(finalBlk(blk(12, 1))) + states = append(states, s.Clone()) + for i, s := range states { + b, err := s.MarshalBinary() + if err != nil { + t.Fatalf("state %d: MarshalBinary: %v", i, err) + } + if want := stateBytes(s.Tip(), s.Finalized(), s.Window()...); !bytes.Equal(b, want) { + t.Errorf("state %d: MarshalBinary =\n%x\nwant\n%x", i, b, want) + } + u := NewState(3) + if err := u.UnmarshalBinary(b); err != nil { + t.Fatalf("state %d: UnmarshalBinary: %v", i, err) + } + wantState(t, u, s.Tip(), s.Finalized(), s.Window()) + } + + b1, b2, b3 := blk(1, 0), blk(2, 0), blk(3, 0) + zero := chain.BlockRef{} + valid := stateBytes(b2, b1, b2) + late := b2 + late.Time++ + big := chain.BlockRef{Number: 300, Hash: h(300, 0)} // a two-byte number and time 0: 67 bytes + tests := []struct { + name string + data []byte + detail string + }{ + {"empty input", nil, "version: truncated"}, + {"flags cut", []byte{0x01}, "flags: truncated"}, + {"tip cut", []byte{0x01, 0x00}, "tip: truncated"}, + {"tip hash cut", cat([]byte{0x01, 0x00, 0x05}, rep(0x11, 10)), "tip: truncated"}, + {"tip varint non-minimal", cat([]byte{0x01, 0x00, 0x80, 0x00}, make([]byte, 70)), "tip: bad varint"}, + {"tip varint overflows", cat([]byte{0x01, 0x00}, rep(0xff, 11), make([]byte, 70)), "tip: bad varint"}, + {"finalized cut", cat([]byte{0x01, 0x00}, refBytes(zero), []byte{0x00}, make([]byte, 64)), "finalized: truncated"}, + {"finalized time non-minimal", cat([]byte{0x01, 0x00}, refBytes(zero), make([]byte, 65), []byte{0x81, 0x00}), "finalized: bad varint"}, + {"nwindow cut", cat([]byte{0x01, 0x00}, refBytes(zero), refBytes(zero)), "nwindow: truncated"}, + {"nwindow non-minimal", cat([]byte{0x01, 0x00}, refBytes(zero), refBytes(zero), []byte{0x80, 0x00}), "nwindow: bad varint"}, + {"version 2", cat([]byte{0x02}, valid[1:]), "version 2"}, + {"version 0", cat([]byte{0x00}, valid[1:]), "version 0"}, + {"flags 01", cat([]byte{0x01, 0x01}, valid[2:]), "flags 0x01"}, + {"flags 80", cat([]byte{0x01, 0x80}, valid[2:]), "flags 0x80"}, + {"3 refs in 100 bytes", cat([]byte{0x01, 0x00}, refBytes(zero), refBytes(zero), []byte{0x03}, make([]byte, 100)), "3 window refs cannot fit in 100 bytes"}, + {"2 refs in 131 bytes", cat([]byte{0x01, 0x00}, refBytes(zero), refBytes(zero), []byte{0x02}, make([]byte, 131)), "2 window refs cannot fit in 131 bytes"}, + {"window[1] cut", cat([]byte{0x01, 0x00}, refBytes(zero), refBytes(zero), []byte{0x02}, refBytes(big), refBytes(b2)[:65]), "window[1]: truncated"}, + {"window[0] varint", cat([]byte{0x01, 0x00}, refBytes(zero), refBytes(zero), []byte{0x01, 0x80, 0x00}, make([]byte, 66)), "window[0]: bad varint"}, + {"trailing byte", cat(valid, []byte{0x00}), "1 trailing bytes"}, + {"window not ascending", stateBytes(b2, zero, b3, b2), "window is not ascending"}, + {"window repeats a number", stateBytes(b2, zero, b2, b2), "window is not ascending"}, + {"window at F", stateBytes(b3, b2, b2, b3), "window ref 2 is not above finalized 2"}, + {"window below F", stateBytes(b3, b3, b1, b2), "window ref 1 is not above finalized 3"}, + {"tip not last", stateBytes(b1, zero, b1, b2), "tip " + b1.String() + " is not the last window ref"}, + {"tip not F", stateBytes(b3, b2), "tip " + b3.String() + " is not the finalized ref"}, + {"tip without F", stateBytes(b3, zero), "tip " + b3.String() + " is not the finalized ref"}, + {"tip differs from the last ref in time only", stateBytes(late, zero, b1, b2), "tip " + late.String() + " is not the last window ref"}, + {"tip differs from F in time only", stateBytes(late, b2), "tip " + late.String() + " is not the finalized ref"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + u := build(t, 0, blocks(4, 6)...) + before := snap(u) + err := u.UnmarshalBinary(tt.data) + if !errors.Is(err, ErrBadState) { + t.Fatalf("UnmarshalBinary error = %v, want one wrapping ErrBadState", err) + } + if got, want := err.Error(), "chainlog: bad state encoding: "+tt.detail; got != want { + t.Errorf("error text =\n%q\nwant\n%q", got, want) + } + if after := snap(u); !reflect.DeepEqual(after, before) { + t.Errorf("a failed UnmarshalBinary changed the state") + } + }) + } + + // A window that starts at block 0 with no F, as on a chain indexed from genesis. + genesis := build(t, 0, block(blk(0, 0)), block(blk(1, 0))) + gb, _ := genesis.MarshalBinary() + var gu State + if err := gu.UnmarshalBinary(gb); err != nil { + t.Fatalf("UnmarshalBinary of a window from block 0: %v", err) + } + wantState(t, gu, blk(1, 0), zero, refs(blk(0, 0), blk(1, 0))) + + // The smallest refs are 66 bytes: two of them fit in exactly 132. + r1 := chain.BlockRef{Number: 1, Hash: h(1, 0), Parent: h(0, 0)} + r2 := chain.BlockRef{Number: 2, Hash: h(2, 0), Parent: h(1, 0)} + small := stateBytes(r2, zero, r1, r2) + if rest := len(small) - (2 + 66 + 66 + 1); rest != 132 { + t.Fatalf("test setup: window of %d bytes, want 132", rest) + } + var u2 State + if err := u2.UnmarshalBinary(small); err != nil { + t.Fatalf("UnmarshalBinary of two 66-byte refs: %v", err) + } + wantState(t, u2, r2, zero, refs(r1, r2)) + + // A window longer than the receiver's cap is accepted; the next block then fails. + five := build(t, 0, blocks(1, 5)...) + b, _ := five.MarshalBinary() + u := NewState(3) + if err := u.UnmarshalBinary(b); err != nil { + t.Fatalf("UnmarshalBinary of a window of 5 into NewState(3): %v", err) + } + wantState(t, u, blk(5, 0), chain.BlockRef{}, five.Window()) + wantMalformed(t, u.Check(block(blk(6, 0))), "block 6: window is full (3 blocks)") +} diff --git a/chainlog/state_test.go b/chainlog/state_test.go new file mode 100644 index 0000000..f2604cd --- /dev/null +++ b/chainlog/state_test.go @@ -0,0 +1,698 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package chainlog + +import ( + "errors" + "math" + "math/rand/v2" + "reflect" + "slices" + "strconv" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" +) + +// fork returns blk(n, f) with the parent of chain base: the first block of a fork off base. +func fork(n uint64, f, base byte) chain.BlockRef { + r := blk(n, f) + r.Parent = h(n-1, base) + return r +} + +func block(r chain.BlockRef) Entry { return Entry{Kind: KindBlock, Block: chain.Block{Ref: r}} } + +func finalBlk(r chain.BlockRef) Entry { + return Entry{Kind: KindBlock, Final: true, Block: chain.Block{Ref: r}} +} + +func revert(r chain.BlockRef) Entry { return Entry{Kind: KindRevert, To: r} } +func finalize(r chain.BlockRef) Entry { return Entry{Kind: KindFinalize, To: r} } + +// blocks returns the linked blocks from..to of chain 0. +func blocks(from, to uint64) []Entry { + var es []Entry + for n := from; n <= to; n++ { + es = append(es, block(blk(n, 0))) + } + return es +} + +func refs(rs ...chain.BlockRef) []chain.BlockRef { return rs } + +// build returns NewState(cap) after es, each checked and applied. +func build(t testing.TB, cap int, es ...Entry) State { + t.Helper() + s := NewState(cap) + for i, e := range es { + if err := s.Check(e); err != nil { + t.Fatalf("entry %d (%v): %v", i, e.Kind, err) + } + s.Apply(e) + } + return s +} + +// applyOnly returns NewState(0) after es, applied without Check, as WAL replay does. +func applyOnly(es ...Entry) State { + s := NewState(0) + for _, e := range es { + s.Apply(e) + } + return s +} + +func wantMalformed(t *testing.T, err error, detail string) { + t.Helper() + if !errors.Is(err, ErrMalformed) { + t.Fatalf("error = %v, want one wrapping ErrMalformed", err) + } + if got, want := err.Error(), "chainlog: malformed entry: "+detail; got != want { + t.Errorf("error text =\n%q\nwant\n%q", got, want) + } +} + +func wantPass(t *testing.T, s *State, e Entry) { + t.Helper() + if err := s.Check(e); err != nil { + t.Errorf("Check(%v) = %v, want nil", e.Kind, err) + } +} + +func wantState(t *testing.T, s State, tip, fin chain.BlockRef, win []chain.BlockRef) { + t.Helper() + if s.Tip() != tip { + t.Errorf("Tip() = %v, want %v", s.Tip(), tip) + } + if s.Finalized() != fin { + t.Errorf("Finalized() = %v, want %v", s.Finalized(), fin) + } + if !reflect.DeepEqual(s.Window(), win) { + t.Errorf("Window() = %v, want %v", s.Window(), win) + } +} + +// wantInvariants checks that the window is strictly ascending, above F when F is set, and that +// the tip is the last window ref, or F when the window is empty. +func wantInvariants(t *testing.T, s State) { + t.Helper() + w := s.Window() + for i := 1; i < len(w); i++ { + if w[i].Number <= w[i-1].Number { + t.Fatalf("window not ascending: %v", w) + } + } + if fin := s.Finalized(); fin != (chain.BlockRef{}) && len(w) > 0 && w[0].Number <= fin.Number { + t.Fatalf("window %v not above finalized %v", w, fin) + } + if len(w) > 0 && s.Tip() != w[len(w)-1] { + t.Fatalf("tip %v is not the last window ref of %v", s.Tip(), w) + } + if len(w) == 0 && s.Tip() != s.Finalized() { + t.Fatalf("empty window: tip %v, finalized %v", s.Tip(), s.Finalized()) + } +} + +// snapshot is everything a State shows: its accessors and its binary form. +type snapshot struct { + tip, fin chain.BlockRef + win []chain.BlockRef + bin []byte +} + +func snap(s State) snapshot { + b, _ := s.MarshalBinary() + return snapshot{s.Tip(), s.Finalized(), s.Window(), b} +} + +func TestNewState(t *testing.T) { + type named struct { + name string + s State + } + for _, tt := range []named{{"NewState(0)", NewState(0)}, {"State{}", State{}}, {"NewState(5)", NewState(5)}} { + t.Run(tt.name, func(t *testing.T) { + wantState(t, tt.s, chain.BlockRef{}, chain.BlockRef{}, nil) + }) + } + + for _, tt := range []named{{"NewState(0)", NewState(0)}, {"State{}", State{}}} { + s := tt.s + t.Run(tt.name+" cap", func(t *testing.T) { + for n := uint64(1); n <= DefaultWindowCap; n++ { + e := block(blk(n, 0)) + if err := s.Check(e); err != nil { + t.Fatalf("block %d: %v", n, err) + } + s.Apply(e) + } + wantMalformed(t, s.Check(block(blk(DefaultWindowCap+1, 0))), "block 100001: window is full (100000 blocks)") + }) + } + + s := build(t, 5, blocks(1, 5)...) + wantMalformed(t, s.Check(block(blk(6, 0))), "block 6: window is full (5 blocks)") + + if got, want := panicText(func() { NewState(-1) }), "evmsync: chainlog.NewState: cap must not be negative, got -1"; got != want { + t.Errorf("NewState(-1) panic = %q, want %q", got, want) + } +} + +func TestRule1Increasing(t *testing.T) { + var empty State + wantPass(t, &empty, block(blk(0, 0))) + wantPass(t, &empty, block(blk(7, 0))) + + s := build(t, 0, block(blk(7, 0))) + wantMalformed(t, s.Check(block(blk(7, 0))), "block 7 is not above tip "+blk(7, 0).String()) + wantMalformed(t, s.Check(block(blk(6, 0))), "block 6 is not above tip "+blk(7, 0).String()) + wantMalformed(t, s.Check(finalBlk(blk(6, 0))), "block 6 is not above tip "+blk(7, 0).String()) + wantPass(t, &s, block(blk(8, 0))) + wantMalformed(t, s.Check(block(blk(9, 0))), "block 9: non-final block is not the child of tip "+blk(7, 0).String()) + + s = build(t, 0, block(blk(7, 0)), block(blk(8, 0)), block(blk(9, 0)), revert(blk(7, 0))) + wantPass(t, &s, block(blk(8, 0))) + wantPass(t, &s, block(fork(8, 1, 0))) + wantMalformed(t, s.Check(block(blk(7, 1))), "block 7 is not above tip "+blk(7, 0).String()) + + // After a Finalize above the tip, the next block is above its number. + s = build(t, 0, block(blk(7, 0)), finalize(blk(12, 0))) + wantMalformed(t, s.Check(block(blk(12, 0))), "block 12 is not above tip "+blk(12, 0).String()) + wantPass(t, &s, block(blk(13, 0))) +} + +func TestRule2Linked(t *testing.T) { + s := build(t, 0, block(blk(7, 0))) + bad := blk(8, 0) + bad.Parent = blk(7, 1).Hash + wantMalformed(t, s.Check(block(bad)), "block 8: parent "+blk(7, 1).Hash.Hex()+" is not tip hash "+blk(7, 0).Hash.Hex()) + wantPass(t, &s, block(blk(8, 0))) + wantMalformed(t, s.Check(block(blk(12, 0))), "block 12: non-final block is not the child of tip "+blk(7, 0).String()) + + s = build(t, 0, finalize(blk(7, 0))) + free := blk(12, 0) + free.Parent = h(99, 5) + wantPass(t, &s, finalBlk(free)) + // A Final block right above the tip still links to it. + wantMalformed(t, s.Check(finalBlk(bad)), "block 8: parent "+blk(7, 1).Hash.Hex()+" is not tip hash "+blk(7, 0).Hash.Hex()) + wantPass(t, &s, finalBlk(blk(8, 0))) +} + +func TestRule3RevertTarget(t *testing.T) { + var empty State + wantMalformed(t, empty.Check(revert(blk(5, 0))), "revert to "+blk(5, 0).String()+": target is not below tip "+chain.BlockRef{}.String()) + + s := build(t, 0, append([]Entry{finalize(blk(5, 0))}, blocks(6, 8)...)...) + tip := blk(8, 0).String() + wantMalformed(t, s.Check(revert(blk(8, 0))), "revert to "+blk(8, 0).String()+": target is not below tip "+tip) + wantMalformed(t, s.Check(revert(blk(9, 0))), "revert to "+blk(9, 0).String()+": target is not below tip "+tip) + wantMalformed(t, s.Check(revert(blk(7, 1))), "revert to "+blk(7, 1).String()+": target is neither the finalized block nor in the window") + wantPass(t, &s, revert(blk(7, 0))) + wantPass(t, &s, revert(blk(6, 0))) + wantPass(t, &s, revert(blk(5, 0))) + late := blk(6, 0) + late.Time++ + wantMalformed(t, s.Check(revert(late)), "revert to "+late.String()+": target is neither the finalized block nor in the window") + otherParent := blk(5, 0) + otherParent.Parent = h(4, 1) + wantMalformed(t, s.Check(revert(otherParent)), "revert to "+otherParent.String()+": target is neither the finalized block nor in the window") + wantMalformed(t, s.Check(revert(blk(4, 0))), "revert to "+blk(4, 0).String()+": target is neither the finalized block nor in the window") + + s.Apply(revert(blk(6, 0))) + wantState(t, s, blk(6, 0), blk(5, 0), refs(blk(6, 0))) + s.Apply(revert(blk(5, 0))) + wantState(t, s, blk(5, 0), blk(5, 0), nil) +} + +func TestRule4Finality(t *testing.T) { + base := append([]Entry{finalize(blk(5, 0))}, blocks(6, 9)...) + s := build(t, 0, base...) + wantMalformed(t, s.Check(finalize(blk(4, 0))), "finalize to "+blk(4, 0).String()+": below finalized "+blk(5, 0).String()) + wantMalformed(t, s.Check(finalize(blk(5, 1))), "finalize to "+blk(5, 1).String()+": conflicts with finalized "+blk(5, 0).String()) + wantMalformed(t, s.Check(finalize(blk(7, 1))), "finalize to "+blk(7, 1).String()+": conflicts with window block "+blk(7, 0).String()) + late := blk(7, 0) + late.Time++ + wantMalformed(t, s.Check(finalize(late)), "finalize to "+late.String()+": conflicts with window block "+blk(7, 0).String()) + tipChild := fork(10, 0, 1) + wantMalformed(t, s.Check(finalize(tipChild)), "finalize to "+tipChild.String()+": parent "+h(9, 1).Hex()+" is not tip hash "+h(9, 0).Hex()) + + // A repeated Finalize passes and changes nothing. + wantPass(t, &s, finalize(blk(5, 0))) + before := snap(s) + s.Apply(finalize(blk(5, 0))) + if after := snap(s); !reflect.DeepEqual(after, before) { + t.Errorf("repeated Finalize changed the state: %+v, want %+v", after, before) + } + + // A Finalize inside the window prunes it. + wantPass(t, &s, finalize(blk(7, 0))) + s.Apply(finalize(blk(7, 0))) + wantState(t, s, blk(9, 0), blk(7, 0), refs(blk(8, 0), blk(9, 0))) + + // A Finalize above the tip moves the tip and empties the window. + s = build(t, 0, base...) + wantPass(t, &s, finalize(blk(12, 0))) + s.Apply(finalize(blk(12, 0))) + wantState(t, s, blk(12, 0), blk(12, 0), nil) + + // Check 4 on a sparse window, which only Apply can build. + sparse := applyOnly(finalize(blk(10, 0)), block(fork(20, 0, 3)), block(fork(30, 0, 3))) + wantMalformed(t, sparse.Check(finalize(blk(29, 1))), "finalize to "+blk(29, 1).String()+": window block "+fork(30, 0, 3).String()+" does not link to it") + linked := blk(29, 3) + wantPass(t, &sparse, finalize(linked)) + + // A sparse window has a ref at index n - first that is not numbered n. + wantPass(t, &sparse, finalize(blk(21, 3))) + + var empty State + wantPass(t, &empty, finalize(blk(3, 0))) + empty.Apply(finalize(blk(3, 0))) + wantState(t, empty, blk(3, 0), blk(3, 0), nil) + + // Genesis: a Finalize of block 0 sets the tip, so later blocks link to it. + var genesis State + wantPass(t, &genesis, finalize(blk(0, 0))) + genesis.Apply(finalize(blk(0, 0))) + wantState(t, genesis, blk(0, 0), blk(0, 0), nil) + wantMalformed(t, genesis.Check(block(blk(5, 0))), "block 5: non-final block is not the child of tip "+blk(0, 0).String()) + + // Numbers at the ends of uint64. + maxRef := chain.BlockRef{Number: math.MaxUint64, Hash: h(math.MaxUint64, 0), Parent: h(9, 9)} + low := build(t, 0, block(blk(0, 0))) + wantPass(t, &low, finalize(maxRef)) + low.Apply(finalize(maxRef)) + wantState(t, low, maxRef, maxRef, nil) + high := build(t, 0, block(maxRef)) + wantPass(t, &high, finalize(blk(0, 0))) + high.Apply(finalize(blk(0, 0))) + wantState(t, high, maxRef, blk(0, 0), refs(maxRef)) +} + +func TestRule5FinalBlocks(t *testing.T) { + s := build(t, 0, finalize(blk(5, 0)), block(blk(6, 0))) + wantMalformed(t, s.Check(finalBlk(blk(8, 0))), "block 8: final block with 1 unfinalized blocks in the window") + + s = build(t, 0, finalize(blk(5, 0)), finalBlk(blk(8, 0))) + wantState(t, s, blk(8, 0), blk(8, 0), nil) + for _, e := range blocks(9, 10) { + wantPass(t, &s, e) + s.Apply(e) + } + wantMalformed(t, s.Check(finalBlk(blk(12, 0))), "block 12: final block with 2 unfinalized blocks in the window") + wantPass(t, &s, revert(blk(8, 0))) + wantMalformed(t, s.Check(revert(blk(5, 0))), "revert to "+blk(5, 0).String()+": target is neither the finalized block nor in the window") + + var empty State + wantPass(t, &empty, finalBlk(blk(3, 0))) + empty.Apply(finalBlk(blk(3, 0))) + wantState(t, empty, blk(3, 0), blk(3, 0), nil) + + // Apply empties the window for a Final block even without Check, as WAL replay applies. + s = applyOnly(block(blk(1, 0)), block(blk(2, 0)), finalBlk(blk(5, 0))) + wantState(t, s, blk(5, 0), blk(5, 0), nil) +} + +func TestRule6Window(t *testing.T) { + s := build(t, 3, blocks(1, 3)...) + wantMalformed(t, s.Check(block(blk(4, 0))), "block 4: window is full (3 blocks)") + wantPass(t, &s, finalize(blk(2, 0))) + wantPass(t, &s, revert(blk(2, 0))) + s.Apply(finalize(blk(2, 0))) + wantPass(t, &s, block(blk(4, 0))) +} + +// lg returns a log of block r with the given index and tx index. +func lg(r chain.BlockRef, index, txIndex uint32) chain.Log { + return chain.Log{BlockNumber: r.Number, BlockHash: r.Hash, Index: index, TxIndex: txIndex} +} + +func withLogs(r chain.BlockRef, logs ...chain.Log) Entry { + return Entry{Kind: KindBlock, Block: chain.Block{Ref: r, Logs: logs}} +} + +func TestShape(t *testing.T) { + r := blk(7, 0) + wrongNumber := lg(r, 0, 0) + wrongNumber.BlockNumber = 8 + wrongHash := lg(r, 0, 0) + wrongHash.BlockHash = blk(8, 0).Hash + five := lg(r, 0, 0) + five.Topics = make([]common.Hash, 5) + blockWithTo := block(r) + blockWithTo.To = blk(3, 0) + revertFinal := revert(blk(3, 0)) + revertFinal.Final = true + finalizeLogs := finalize(blk(3, 0)) + finalizeLogs.Block.Logs = []chain.Log{lg(r, 0, 0)} + revertRef := revert(blk(3, 0)) + revertRef.Block.Ref = r + kind0 := Entry{Block: chain.Block{Ref: chain.BlockRef{Number: 7}}} + kindAndTo := Entry{Kind: 9, To: blk(3, 0)} + toAndZeroHash := block(chain.BlockRef{Number: 7}) + toAndZeroHash.To = blk(3, 0) + revertFinalZero := revert(chain.BlockRef{Number: 3}) + revertFinalZero.Final = true + wrongBoth := lg(r, 0, 0) + wrongBoth.BlockNumber, wrongBoth.BlockHash = 8, blk(8, 0).Hash + + tests := []struct { + name string + e Entry + detail string + }{ + {"kind 0", Entry{}, "unknown kind 0"}, + {"kind 4", Entry{Kind: 4, To: blk(3, 0)}, "unknown kind 4"}, + {"kind row wins", kind0, "unknown kind 0"}, + {"kind row wins over To", kindAndTo, "unknown kind 9"}, + {"block with To", blockWithTo, "block 7: To must be zero"}, + {"To row wins over zero hash", toAndZeroHash, "block 7: To must be zero"}, + {"revert with Final", revertFinal, "revert entry: Block and Final must be zero"}, + {"revert with a block ref", revertRef, "revert entry: Block and Final must be zero"}, + {"finalize with logs", finalizeLogs, "finalize entry: Block and Final must be zero"}, + {"block with zero hash", block(chain.BlockRef{Number: 7, Parent: h(6, 0)}), "block 7: zero hash"}, + {"revert to zero hash", revert(chain.BlockRef{Number: 4}), "revert to 4: zero hash"}, + {"finalize to zero hash", finalize(chain.BlockRef{Number: 4, Time: 9}), "finalize to 4: zero hash"}, + {"log block number", withLogs(r, wrongNumber), "block 7: log 0: block number 8"}, + {"log block hash", withLogs(r, lg(r, 0, 0), wrongHash), "block 7: log 1: block hash " + blk(8, 0).Hash.Hex()}, + {"5 topics", withLogs(r, five), "block 7: log 0: 5 topics, at most 4"}, + {"index 3 then 3", withLogs(r, lg(r, 3, 0), lg(r, 3, 0)), "block 7: log 1: index 3 not above 3"}, + {"index 3 then 2", withLogs(r, lg(r, 3, 0), lg(r, 2, 0)), "block 7: log 1: index 2 not above 3"}, + {"tx index 2 then 1", withLogs(r, lg(r, 0, 2), lg(r, 1, 1)), "block 7: log 1: tx index 1 below 2"}, + // Entries that fail two adjacent rows report the first. + {"Final row wins over zero hash", revertFinalZero, "revert entry: Block and Final must be zero"}, + {"block number row wins over hash", withLogs(r, wrongBoth), "block 7: log 0: block number 8"}, + {"index row wins over tx index", withLogs(r, lg(r, 3, 2), lg(r, 3, 1)), "block 7: log 1: index 3 not above 3"}, + } + var empty State + // A state whose rules every one of these entries would also fail: shape is checked first. + s := build(t, 0, blocks(7, 9)...) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + wantMalformed(t, empty.Check(tt.e), tt.detail) + wantMalformed(t, s.Check(tt.e), tt.detail) + }) + } + + ok := withLogs(r, lg(r, 0, 0), lg(r, 1, 0), lg(r, 5, 3)) + ok.Block.Logs[2].Topics = make([]common.Hash, 4) + wantPass(t, &empty, ok) + emptyLogs := revert(blk(3, 0)) + emptyLogs.Block.Logs = []chain.Log{} + wantMalformed(t, empty.Check(emptyLogs), "revert to "+blk(3, 0).String()+": target is not below tip "+chain.BlockRef{}.String()) +} + +func TestSize(t *testing.T) { + r := blk(7, 0) + sized := func(dataLen int) Entry { + l := lg(r, 0, 0) + l.Data = make([]byte, dataLen) + return withLogs(r, l) + } + bodySize := func(e Entry) int { + e.Offset = math.MaxUint64 + return len(Encode(nil, e)) - 8 + } + var empty State + wantPass(t, &empty, sized(MaxRecordSize-200)) + + // The size rule comes after the shape rules and before the block rules. + misnumbered := sized(MaxRecordSize) + misnumbered.Block.Logs[0].BlockNumber = 8 + wantMalformed(t, empty.Check(misnumbered), "block 7: log 0: block number 8") + afterSeven := build(t, 0, block(blk(7, 0))) + wantMalformed(t, afterSeven.Check(sized(MaxRecordSize)), "block 7: record body of "+strconv.Itoa(bodySize(sized(MaxRecordSize)))+" bytes exceeds 16777216") + tooBig := sized(MaxRecordSize) + wantMalformed(t, empty.Check(tooBig), "block 7: record body of "+strconv.Itoa(bodySize(tooBig))+" bytes exceeds 16777216") + + // The largest data that fits gives a body of exactly MaxRecordSize. + fixed := bodySize(sized(0)) - 1 // without the one-byte data length + exact := sized(MaxRecordSize - fixed - 4) + if got := bodySize(exact); got != MaxRecordSize { + t.Fatalf("test setup: body of %d bytes, want %d", got, MaxRecordSize) + } + wantPass(t, &empty, exact) + over := sized(MaxRecordSize - fixed - 3) + wantMalformed(t, empty.Check(over), "block 7: record body of 16777217 bytes exceeds 16777216") + + if a := testing.AllocsPerRun(10, func() { _ = empty.Check(exact) }); a != 0 { + t.Errorf("Check of a 16 MiB block: %v allocs, want 0", a) + } + s := build(t, 0, blocks(1, 1000)...) + next := block(blk(1001, 0)) + if a := testing.AllocsPerRun(100, func() { _ = s.Check(next) }); a != 0 { + t.Errorf("Check of a block: %v allocs, want 0", a) + } +} + +// step is one entry of the apply sequence and the state after it. +type step struct { + e Entry + tip, fin chain.BlockRef + win []chain.BlockRef +} + +// applySteps is a sequence through every kind: blocks, a revert, a fork, a finalize inside the +// window. +func applySteps() []step { + f4, f5, f6, f7 := fork(4, 1, 0), blk(5, 1), blk(6, 1), blk(7, 1) + b := func(n uint64) chain.BlockRef { return blk(n, 0) } + zero := chain.BlockRef{} + return []step{ + {block(b(1)), b(1), zero, refs(b(1))}, + {block(b(2)), b(2), zero, refs(b(1), b(2))}, + {block(b(3)), b(3), zero, refs(b(1), b(2), b(3))}, + {block(b(4)), b(4), zero, refs(b(1), b(2), b(3), b(4))}, + {block(b(5)), b(5), zero, refs(b(1), b(2), b(3), b(4), b(5))}, + {revert(b(3)), b(3), zero, refs(b(1), b(2), b(3))}, + {block(f4), f4, zero, refs(b(1), b(2), b(3), f4)}, + {block(f5), f5, zero, refs(b(1), b(2), b(3), f4, f5)}, + {block(f6), f6, zero, refs(b(1), b(2), b(3), f4, f5, f6)}, + {finalize(f4), f6, f4, refs(f5, f6)}, + {block(f7), f7, f4, refs(f5, f6, f7)}, + } +} + +func TestApplySequence(t *testing.T) { + var s State + for i, st := range applySteps() { + if err := s.Check(st.e); err != nil { + t.Fatalf("step %d: %v", i, err) + } + s.Apply(st.e) + wantState(t, s, st.tip, st.fin, st.win) + wantInvariants(t, s) + } + + r := rand.New(rand.NewPCG(3, 4)) //nolint:gosec // a seeded, reproducible sequence, not a secret + var fuzz State + for range 1000 { + n := r.Uint64N(40) + forks := [...]byte{0, 1, 2} + kinds := [...]Kind{0, KindBlock, KindRevert, KindFinalize, 4} + ref := chain.BlockRef{Number: n, Hash: h(n, forks[r.IntN(3)]), Parent: h(n-1, forks[r.IntN(3)]), Time: r.Uint64()} + e := Entry{Kind: kinds[r.IntN(len(kinds))], Final: r.IntN(4) == 0} + if e.Kind == KindBlock { + e.Block.Ref = ref + } else { + e.To = ref + } + _ = fuzz.Check(e) + fuzz.Apply(e) + _ = fuzz.Window() + _, _ = fuzz.MarshalBinary() + _ = fuzz.Clone() + } +} + +// TestCheckedWalk applies a seeded walk of entries that pass Check and checks the invariants and +// the binary round trip after each one. +func TestCheckedWalk(t *testing.T) { + r := rand.New(rand.NewPCG(1, 2)) //nolint:gosec // a seeded, reproducible sequence, not a secret + s := NewState(16) + applied := 0 + for range 20_000 { + e := randomEntry(r, s) + if s.Check(e) != nil { + continue + } + s.Apply(e) + applied++ + wantInvariants(t, s) + b, _ := s.MarshalBinary() + u := NewState(16) + if err := u.UnmarshalBinary(b); err != nil { + t.Fatalf("after %d entries: UnmarshalBinary: %v", applied, err) + } + if !reflect.DeepEqual(snap(u), snap(s)) { + t.Fatalf("after %d entries: round trip %+v, want %+v", applied, snap(u), snap(s)) + } + } + if applied < 1000 { + t.Errorf("only %d of 20,000 drawn entries passed Check, want at least 1,000", applied) + } +} + +// randomEntry draws an entry near s's tip: a block on one of three forks, or a revert or finalize +// whose target is a window ref half the time. +func randomEntry(r *rand.Rand, s State) Entry { + forks := [...]byte{0, 1, 2} + tip := s.Tip() + n := tip.Number + r.Uint64N(3) + ref := chain.BlockRef{Number: n, Hash: h(n, forks[r.IntN(3)]), Parent: tip.Hash, Time: 12 * n} + if w := s.Window(); len(w) > 0 && r.IntN(2) == 0 { + ref = w[r.IntN(len(w))] + } + switch r.IntN(4) { + case 0: + return revert(ref) + case 1: + return finalize(ref) + default: + return Entry{Kind: KindBlock, Final: r.IntN(8) == 0, Block: chain.Block{Ref: ref}} + } +} + +func TestIsolation(t *testing.T) { + five := refs(blk(1, 0), blk(2, 0), blk(3, 0), blk(4, 0), blk(5, 0)) + s := build(t, 0, blocks(1, 5)...) + w := s.Window() + w[0] = chain.BlockRef{} + if !reflect.DeepEqual(s.Window(), five) { + t.Errorf("changing Window()'s result changed the state: %v", s.Window()) + } + + c := s.Clone() + c.Apply(revert(blk(3, 0))) + c.Apply(block(fork(4, 1, 0))) + c.Apply(block(blk(5, 1))) + if !reflect.DeepEqual(s.Window(), five) { + t.Errorf("applying to a clone changed the original: %v", s.Window()) + } + + c = s.Clone() + s.Apply(revert(blk(3, 0))) + s.Apply(block(fork(4, 1, 0))) + s.Apply(block(blk(5, 1))) + if !reflect.DeepEqual(c.Window(), five) { + t.Errorf("applying to the original changed a clone: %v", c.Window()) + } + wantState(t, c, blk(5, 0), chain.BlockRef{}, five) + + // Clone keeps the cap. + small := build(t, 2, blocks(1, 2)...) + sc := small.Clone() + wantMalformed(t, sc.Check(block(blk(3, 0))), "block 3: window is full (2 blocks)") + + var empty State + if w := empty.Clone().Window(); w != nil { + t.Errorf("clone of an empty state: Window() = %v, want nil", w) + } +} + +func TestCheckBatch(t *testing.T) { + stateA := func() State { return build(t, 0, append([]Entry{finalize(blk(0, 0))}, blocks(1, 5)...)...) } + stateB := build(t, 0, append([]Entry{finalize(blk(0, 0))}, blocks(1, 4)...)...) + f6 := finalBlk(blk(6, 0)) + // Longer than CheckBatch's 8-entry stack buffer: a revert, 15 blocks of a fork, a second + // revert, a block and a finalize. + longBatch := []Entry{revert(blk(3, 0)), block(fork(4, 1, 0))} + for n := uint64(5); n <= 18; n++ { + longBatch = append(longBatch, block(blk(n, 1))) + } + longBatch = append(longBatch, revert(blk(10, 1)), block(fork(11, 2, 1)), finalize(blk(5, 1))) + + tests := []struct { + name string + s State + batch []Entry + index int + detail string + }{ + {"a", stateA(), []Entry{revert(blk(3, 0)), block(fork(4, 1, 0)), block(blk(4, 1))}, 2, "block 4 is not above tip " + fork(4, 1, 0).String()}, + {"b", stateB, []Entry{block(blk(5, 0)), f6}, 1, "block 6: final block with 5 unfinalized blocks in the window"}, + {"c", stateA(), []Entry{revert(blk(3, 0)), block(fork(4, 1, 0))}, -1, ""}, + {"first fails", stateA(), []Entry{block(blk(9, 0)), block(blk(6, 0))}, 0, "block 9: non-final block is not the child of tip " + blk(5, 0).String()}, + {"finalize, revert, blocks, fail", stateA(), []Entry{ + finalize(blk(2, 0)), revert(blk(3, 0)), block(fork(4, 1, 0)), block(blk(5, 1)), block(blk(9, 9)), + }, 4, "block 9: non-final block is not the child of tip " + blk(5, 1).String()}, + {"revert to F, final blocks, fail", stateA(), []Entry{ + revert(blk(0, 0)), finalBlk(blk(2, 2)), block(blk(3, 2)), block(blk(4, 2)), revert(blk(1, 0)), + }, 4, "revert to " + blk(1, 0).String() + ": target is neither the finalized block nor in the window"}, + {"empty batch", stateA(), nil, -1, ""}, + {"one-ref revert", stateA(), []Entry{revert(blk(4, 0)), block(fork(5, 1, 0))}, -1, ""}, + {"19 entries", stateA(), longBatch, -1, ""}, + {"19 entries, then a failure", stateA(), append(slices.Clone(longBatch), block(blk(30, 0))), 19, "block 30: non-final block is not the child of tip " + fork(11, 2, 1).String()}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := tt.s + before := snap(s) + i, err := s.CheckBatch(tt.batch) + if i != tt.index { + t.Errorf("index = %d, want %d", i, tt.index) + } + if tt.index < 0 { + if err != nil { + t.Errorf("error = %v, want nil", err) + } + } else { + wantMalformed(t, err, tt.detail) + } + if after := snap(s); !reflect.DeepEqual(after, before) { + t.Errorf("CheckBatch changed the state:\n%+v\nwant\n%+v", after, before) + } + }) + } + + // A full backing array grows once, for good: later calls neither copy nor allocate. + full := NewState(0) + n := uint64(1) + for ; len(full.win) < 1000 || len(full.win) < cap(full.win); n++ { + full.Apply(block(blk(n, 0))) + } + one := []Entry{block(blk(n, 0))} + if a := testing.AllocsPerRun(100, func() { _, _ = full.CheckBatch(one) }); a != 0 { + t.Errorf("CheckBatch of one block on a full backing array: %v allocs, want 0", a) + } + + // The cost does not grow with the window: no clone, no allocation for a block or a finalize. + s := build(t, 0, blocks(1, 50_000)...) + batch := []Entry{block(blk(50_001, 0)), finalize(blk(10, 0)), block(blk(50_002, 0))} + if a := testing.AllocsPerRun(100, func() { _, _ = s.CheckBatch(batch) }); a != 0 { + t.Errorf("CheckBatch on a window of 50,000: %v allocs, want 0", a) + } +} + +func TestStateSparseBackfillPrefix(t *testing.T) { + batch := []Entry{finalBlk(blk(3, 0)), finalBlk(blk(4, 0)), finalBlk(blk(9, 0)), finalize(blk(11, 0))} + tips := []chain.BlockRef{blk(1, 0), blk(3, 0), blk(4, 0), blk(9, 0), blk(11, 0)} + for k := range len(batch) + 1 { + checked := build(t, 0, append([]Entry{finalize(blk(1, 0))}, batch[:k]...)...) + replayed := applyOnly(append([]Entry{finalize(blk(1, 0))}, batch[:k]...)...) + for _, tt := range []struct { + name string + s State + }{{"checked", checked}, {"applied", replayed}} { + s := tt.s + t.Run(tt.name+"/"+strconv.Itoa(k), func(t *testing.T) { + wantState(t, s, tips[k], tips[k], nil) + wantInvariants(t, s) + if i, err := s.CheckBatch(batch[k:]); i != -1 || err != nil { + t.Errorf("CheckBatch(rest) = %d, %v; want -1, nil", i, err) + } + wantPass(t, &s, block(blk(tips[k].Number+1, 0))) + + b, _ := s.MarshalBinary() + var u State + if err := u.UnmarshalBinary(b); err != nil { + t.Fatalf("UnmarshalBinary: %v", err) + } + wantState(t, u, tips[k], tips[k], nil) + }) + } + } +} diff --git a/chainlog/wal/bench_test.go b/chainlog/wal/bench_test.go new file mode 100644 index 0000000..a48a413 --- /dev/null +++ b/chainlog/wal/bench_test.go @@ -0,0 +1,138 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "testing" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +// benchBlock returns block n of chain 0 with 10 logs of 3 topics and 64 bytes of data each. +func benchBlock(n uint64) chainlog.Entry { + r := ref(n) + logs := make([]chain.Log, 10) + for i := range logs { + l := chain.Log{BlockNumber: r.Number, BlockHash: r.Hash, TxHash: r.Hash, TxIndex: uint32(i), Index: uint32(i)} //nolint:gosec // i < 10 + l.Address[19] = 0xaa + l.Topics = append(l.Topics, r.Hash, r.Parent, r.Hash) + l.Data = make([]byte, 64) + logs[i] = l + } + return chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: r, Logs: logs}} +} + +// fill appends blocks 1..n to l, finalizing every 1,024 blocks so the window stays small. +func fill(b *testing.B, l *Log, n uint64) { + b.Helper() + ctx := b.Context() + for num := uint64(1); num <= n; num++ { + es := []chainlog.Entry{benchBlock(num)} + if num%1024 == 0 { + es = append(es, chainlog.Entry{Kind: chainlog.KindFinalize, To: ref(num)}) + } + if _, err := l.Append(ctx, es...); err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkAppend(b *testing.B) { + b.Run("never", func(b *testing.B) { benchAppend(b, SyncNever) }) + b.Run("always", func(b *testing.B) { benchAppend(b, SyncAlways) }) +} + +// benchAppend appends one block per iteration, and a Finalize every 1,024 blocks. +func benchAppend(b *testing.B, sync SyncPolicy) { + ctx := b.Context() + l, err := Open(b.TempDir(), Options{Sync: sync}) + if err != nil { + b.Fatal(err) + } + defer l.Close() + next := uint64(1) + b.ReportAllocs() + for b.Loop() { + if _, err := l.Append(ctx, benchBlock(next)); err != nil { + b.Fatal(err) + } + if next%1024 == 0 { + if _, err := l.Append(ctx, chainlog.Entry{Kind: chainlog.KindFinalize, To: ref(next)}); err != nil { + b.Fatal(err) + } + } + next++ + } +} + +// BenchmarkRead reads 256 entries per call, sequentially through a log of 65,536 blocks. +func BenchmarkRead(b *testing.B) { + const n, batch = 1 << 16, 256 + ctx := b.Context() + l, err := Open(b.TempDir(), Options{Sync: SyncNever}) + if err != nil { + b.Fatal(err) + } + defer l.Close() + fill(b, l, n) + head, _ := l.Head(ctx) + from := chainlog.Offset(1) + b.ReportAllocs() + for b.Loop() { + es, err := l.Read(ctx, from, batch) + if err != nil { + b.Fatal(err) + } + from += chainlog.Offset(len(es)) //nolint:gosec // at most 256 + if from > head { + from = 1 + } + } +} + +// BenchmarkOpen opens a 256 MiB log with no checkpoint, so Open scans and replays all of it. +func BenchmarkOpen(b *testing.B) { + const size = 256 << 20 + dir := b.TempDir() + opts := Options{Sync: SyncNever, SegmentSize: 512 << 20} + l, err := Open(dir, opts) + if err != nil { + b.Fatal(err) + } + perBlock := uint64(len(chainlog.Encode(nil, chainlog.Entry{Offset: 1 << 20, Kind: chainlog.KindBlock, Block: benchBlock(1).Block}))) + fill(b, l, size/perBlock) + if err := l.Close(); err != nil { + b.Fatal(err) + } + b.SetBytes(size) + b.ReportAllocs() + for b.Loop() { + l, err := Open(dir, opts) + if err != nil { + b.Fatal(err) + } + if err := l.Close(); err != nil { + b.Fatal(err) + } + } +} + +// BenchmarkReadTail reads one entry 255 records past an indexed record, as a consumer following +// the tail does: Read skips the records before it by their headers. +func BenchmarkReadTail(b *testing.B) { + ctx := b.Context() + l, err := Open(b.TempDir(), Options{Sync: SyncNever}) + if err != nil { + b.Fatal(err) + } + defer l.Close() + fill(b, l, 1024) + b.ReportAllocs() + for b.Loop() { + if _, err := l.Read(ctx, 511, 1); err != nil { + b.Fatal(err) + } + } +} diff --git a/chainlog/wal/checkpoint.go b/chainlog/wal/checkpoint.go new file mode 100644 index 0000000..352eab4 --- /dev/null +++ b/chainlog/wal/checkpoint.go @@ -0,0 +1,75 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "encoding/binary" + "fmt" + + "github.com/hmdsefi/evmsync/chainlog" +) + +const ( + checkpointName = "checkpoint" + checkpointMagic = "EVMSCKPT" + checkpointVersion = 1 + maxCheckpointSize = 1 << 30 // the largest checkpoint body, in bytes +) + +// checkpoint is the validator state after offset x, so Open replays only the entries after x. +type checkpoint struct { + x, l chainlog.Offset // state after offset x; last finalize offset l <= x + state chainlog.State +} + +// encodeCheckpoint returns the checkpoint file for c: the same 8-byte header as meta, then a body +// of the magic, version 1, x and l as varints, and the state's binary form. +func encodeCheckpoint(c checkpoint) []byte { + state, _ := c.state.MarshalBinary() // never fails + b := make([]byte, fileHeader, fileHeader+len(checkpointMagic)+1+2*binary.MaxVarintLen64+len(state)) + b = append(b, checkpointMagic...) + b = append(b, checkpointVersion) + b = binary.AppendUvarint(b, uint64(c.x)) + b = binary.AppendUvarint(b, uint64(c.l)) + b = append(b, state...) + return sealFile(b) +} + +// decodeCheckpoint decodes a checkpoint file. The state gets window cap windowCap. Its errors wrap +// ErrCorrupt and read "wal: corrupt log: checkpoint: "; a bad state also wraps +// chainlog.ErrBadState. +func decodeCheckpoint(b []byte, windowCap int) (checkpoint, error) { + corrupt := func(detail string) (checkpoint, error) { + return checkpoint{}, fmt.Errorf("%w: checkpoint: %s", ErrCorrupt, detail) + } + body, detail := openFile(b, maxCheckpointSize) + if detail != "" { + return corrupt(detail) + } + d := reader{b: body} + magic, ok := d.take(len(checkpointMagic)) + if !ok { + return corrupt("truncated") + } + if string(magic) != checkpointMagic { + return corrupt("bad magic") + } + version, ok := d.take(1) + if !ok { + return corrupt("truncated") + } + if version[0] != checkpointVersion { + return corrupt(fmt.Sprintf("version %d", version[0])) + } + x, okX := d.uvarint() + l, okL := d.uvarint() + if !okX || !okL || l > x { + return corrupt("bad header") + } + c := checkpoint{x: chainlog.Offset(x), l: chainlog.Offset(l), state: chainlog.NewState(windowCap)} + if err := c.state.UnmarshalBinary(d.b); err != nil { + return checkpoint{}, fmt.Errorf("%w: checkpoint: %w", ErrCorrupt, err) + } + return c, nil +} diff --git a/chainlog/wal/checkpoint_test.go b/chainlog/wal/checkpoint_test.go new file mode 100644 index 0000000..1aef51a --- /dev/null +++ b/chainlog/wal/checkpoint_test.go @@ -0,0 +1,122 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "bytes" + "encoding/binary" + "errors" + "reflect" + "strings" + "testing" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +// ref returns block n of a chain whose hashes are as in logtest, linked to block n-1. +func ref(n uint64) chain.BlockRef { + r := chain.BlockRef{Number: n, Time: 1_700_000_000 + 12*n} + r.Hash[0] = 0xb0 + binary.BigEndian.PutUint64(r.Hash[24:], n) + if n > 0 { + r.Parent[0] = 0xb0 + binary.BigEndian.PutUint64(r.Parent[24:], n-1) + } + return r +} + +// testState is the state after Finalize 2 and blocks 3, 4, 5: a window of 3. +func testState() chainlog.State { + s := chainlog.NewState(0) + s.Apply(chainlog.Entry{Kind: chainlog.KindFinalize, To: ref(2)}) + for n := uint64(3); n <= 5; n++ { + s.Apply(chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: ref(n)}}) + } + return s +} + +type stateView struct { + tip, fin chain.BlockRef + win []chain.BlockRef +} + +func viewOf(s chainlog.State) stateView { return stateView{s.Tip(), s.Finalized(), s.Window()} } + +func TestCheckpointRoundTrip(t *testing.T) { + c := checkpoint{x: 300, l: 7, state: testState()} + b := encodeCheckpoint(c) + stateBin, _ := c.state.MarshalBinary() + if want := seal(cat([]byte("EVMSCKPT"), []byte{0x01}, uv(300), uv(7), stateBin)); !bytes.Equal(b, want) { + t.Fatalf("encodeCheckpoint =\n%x\nwant\n%x", b, want) + } + got, err := decodeCheckpoint(b, 0) + if err != nil { + t.Fatalf("decodeCheckpoint: %v", err) + } + if got.x != 300 || got.l != 7 || !reflect.DeepEqual(viewOf(got.state), viewOf(c.state)) { + t.Errorf("decodeCheckpoint = x %d, l %d, %+v; want 300, 7, %+v", got.x, got.l, viewOf(got.state), viewOf(c.state)) + } + + // The state gets the cap Open asks for, also when its window is already longer. + small, err := decodeCheckpoint(b, 2) + if err != nil { + t.Fatalf("decodeCheckpoint with cap 2: %v", err) + } + next := chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: ref(6)}} + if err := small.state.Check(next); err == nil || !strings.HasSuffix(err.Error(), "window is full (2 blocks)") { + t.Errorf("Check after decoding with cap 2 = %v, want a full window of 2 blocks", err) + } + + empty, err := decodeCheckpoint(encodeCheckpoint(checkpoint{state: chainlog.NewState(0)}), 0) + if err != nil || empty.x != 0 || empty.l != 0 || !reflect.DeepEqual(viewOf(empty.state), stateView{}) { + t.Errorf("empty checkpoint decoded as %+v, %v", empty, err) + } +} + +func TestCheckpointCorrupt(t *testing.T) { + stateBin, _ := testState().MarshalBinary() + valid := encodeCheckpoint(checkpoint{x: 9, l: 7, state: testState()}) + flipped := bytes.Clone(valid) + flipped[len(flipped)-3] ^= 0x01 + tooLarge := binary.LittleEndian.AppendUint32(nil, 1073741825) + tooLarge = append(tooLarge, 0, 0, 0, 0) + badState := bytes.Clone(stateBin) + badState[0] = 2 + head := cat([]byte("EVMSCKPT"), []byte{0x01}) + + tests := []struct { + name string + data []byte + detail string + }{ + {"cut", valid[:len(valid)-1], "truncated"}, + {"header cut", valid[:7], "truncated"}, + {"flipped byte", flipped, "checksum mismatch"}, + {"bad magic", seal(cat([]byte("EVMSMETA"), []byte{0x01}, uv(9), uv(7), stateBin)), "bad magic"}, + {"version 2", seal(cat([]byte("EVMSCKPT"), []byte{0x02}, uv(9), uv(7), stateBin)), "version 2"}, + {"body cut in the magic", seal([]byte("EVMS")), "truncated"}, + {"l above x", seal(cat(head, uv(7), uv(9), stateBin)), "bad header"}, + {"x non-minimal", seal(cat(head, []byte{0x89, 0x00}, uv(7), stateBin)), "bad header"}, + {"l cut", seal(cat(head, uv(9))), "bad header"}, + {"bad state", seal(cat(head, uv(9), uv(7), badState)), "chainlog: bad state encoding: version 2"}, + {"length 1073741825", tooLarge, "too large"}, + {"trailing byte after the body", cat(valid, []byte{0x00}), "trailing bytes"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := decodeCheckpoint(tt.data, 0) + if !errors.Is(err, ErrCorrupt) { + t.Fatalf("decodeCheckpoint error = %v, want one wrapping ErrCorrupt", err) + } + if got, want := err.Error(), "wal: corrupt log: checkpoint: "+tt.detail; got != want { + t.Errorf("error = %q, want %q", got, want) + } + }) + } + _, err := decodeCheckpoint(seal(cat(head, uv(9), uv(7), badState)), 0) + if !errors.Is(err, chainlog.ErrBadState) { + t.Errorf("bad state error = %v, want one also wrapping chainlog.ErrBadState", err) + } +} diff --git a/chainlog/wal/compact.go b/chainlog/wal/compact.go new file mode 100644 index 0000000..d9e4f66 --- /dev/null +++ b/chainlog/wal/compact.go @@ -0,0 +1,83 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "errors" + "log/slog" + "path/filepath" + + "github.com/hmdsefi/evmsync/chainlog" +) + +// compact deletes the segments, other than the active one, whose entries are all below b. It +// first writes a checkpoint at Head, so the deleted entries are never needed again, then drops +// the segments from the index and deletes their files oldest first. A crash at any point leaves a +// log that Open accepts. The caller holds appendMu. +func (l *Log) compact(b chainlog.Offset) error { + n := 0 + for n < len(l.segs)-1 && l.segs[n].last < b { + n++ + } + if n == 0 { + return nil + } + cp := encodeCheckpoint(checkpoint{x: l.head, l: l.lastFin, state: l.state}) + if err := writeFileAtomic(l.dir, checkpointName, cp, l.fsync); err != nil { + return err + } + + l.mu.Lock() + dropped := l.segs[:n:n] + l.segs = l.segs[n:] + l.oldest = l.segs[0].first + oldest := l.oldest + l.mu.Unlock() + + // A reader that copied a dropped segment fails its ReadAt once the file is closed, and then + // reports chainlog.ErrTruncated. + for _, s := range dropped { + _ = s.f.Close() // a read-only file; its close error tells nothing + } + for _, s := range dropped { + l.undelete = append(l.undelete, s.name) + } + if err := l.removeSegments(); err != nil { + if errors.Is(err, errStopped) { + return nil + } + return err + } + if err := syncDir(l.dir, l.fsync); err != nil { + return err + } + l.logger.Info("wal compacted", + slog.Uint64("before", uint64(b)), + slog.Int("segments_deleted", len(dropped)), + slog.Uint64("oldest", uint64(oldest)), + ) + return nil +} + +// errStopped is how removeSegments reports that hooks.removed stopped it, as a crash would. +var errStopped = errors.New("wal: compaction stopped") + +// removeSegments deletes the dropped segment files in l.undelete, oldest first. On a failure it +// stops and keeps the rest, so the next Truncate retries them before it deletes any other segment. +// The caller holds appendMu. +func (l *Log) removeSegments() error { + for len(l.undelete) > 0 { + name := l.undelete[0] + path := filepath.Join(l.dir, name) + if err := l.hooks.removeFile(path); err != nil { + return ioErr("remove", path, err) + } + l.undelete = l.undelete[1:] + if l.hooks.removed != nil && !l.hooks.removed(name) { + l.undelete = nil // a simulated crash: the process stops here + return errStopped + } + } + return nil +} diff --git a/chainlog/wal/compact_test.go b/chainlog/wal/compact_test.go new file mode 100644 index 0000000..8c2341d --- /dev/null +++ b/chainlog/wal/compact_test.go @@ -0,0 +1,449 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "context" + "errors" + "io/fs" + "log/slog" + "os" + "path/filepath" + "reflect" + "sync" + "syscall" + "testing" + + "github.com/hmdsefi/evmsync/chainlog" +) + +// compactOpts are testOpts with the default window cap and the given sync policy. +func compactOpts(segmentSize int64, sync SyncPolicy) Options { + o := testOpts(segmentSize) + o.WindowCap = 0 + o.Sync = sync + return o +} + +// finalizedBlocks returns linked blocks 1..n with one log each and a Finalize of every tenth. +func finalizedBlocks(n uint64) []chainlog.Entry { + var es []chainlog.Entry + for _, e := range linkedBlocks(1, n, 40) { + es = append(es, e) + if num := e.Block.Ref.Number; num%10 == 0 { + es = append(es, finalizeTo(ref(num))) + } + } + return es +} + +// lastFinalize returns the offset of the newest Finalize in es, numbered from 1. +func lastFinalize(es []chainlog.Entry) chainlog.Offset { + var l chainlog.Offset + for i, e := range es { + if e.Kind == chainlog.KindFinalize { + l = chainlog.Offset(i + 1) //nolint:gosec // a small test log + } + } + return l +} + +func TestCompaction(t *testing.T) { + ctx := t.Context() + logger, rec := newRecorder() + opts := compactOpts(512, SyncAlways) + opts.Logger = logger + dir := t.TempDir() + es := finalizedBlocks(60) + l := openLog(t, dir, opts, hooks{}) + for _, e := range es { + mustAppend(t, l, e) + } + before := segmentNames(t, dir) + bound := lastFinalize(es) + 1 + var wantKept []string + for i, name := range before { + last := chainlog.Offset(len(es)) + if i < len(before)-1 { + next, _ := parseSegmentName(before[i+1]) + last = next - 1 + } + if i == len(before)-1 || last >= bound { + wantKept = append(wantKept, name) + } + } + if len(wantKept) == len(before) { + t.Fatal("test setup: no segment lies below the bound") + } + + if err := l.Truncate(ctx, 1000); err != nil { + t.Fatalf("Truncate: %v", err) + } + wantSame(t, "segments after Truncate(1000)", segmentNames(t, dir), wantKept) + firstKept, _ := parseSegmentName(wantKept[0]) + if o, _ := l.Oldest(ctx); o != firstKept { + t.Errorf("Oldest = %d, want the first remaining segment's offset %d", o, firstKept) + } + b, err := os.ReadFile(filepath.Join(dir, checkpointName)) + if err != nil { + t.Fatal(err) + } + cp, err := decodeCheckpoint(b, 0) + if err != nil { + t.Fatalf("decodeCheckpoint: %v", err) + } + head, _ := l.Head(ctx) + if cp.x != head || cp.l != bound-1 { + t.Errorf("checkpoint at x %d, l %d; want Head %d and the last finalize %d", cp.x, cp.l, head, bound-1) + } + attrs, _ := rec.find(slog.LevelInfo, "wal compacted") + wantSame(t, "wal compacted", attrs, map[string]any{ + "before": uint64(bound), "segments_deleted": int64(len(before) - len(wantKept)), "oldest": uint64(firstKept), + }) + if _, err := l.Read(ctx, firstKept-1, 1); !errors.Is(err, chainlog.ErrTruncated) { + t.Errorf("Read below Oldest = %v, want ErrTruncated", err) + } + snapshot := snap(t, l) + mustClose(t, l) + + l = openLog(t, dir, compactOpts(512, SyncAlways), hooks{}) + wantSame(t, "log after reopen", snap(t, l), snapshot) + mustClose(t, l) +} + +// TestCompactionCrash stops compaction after its first deletion, as a crash would, and reopens. +func TestCompactionCrash(t *testing.T) { + ctx := t.Context() + dir := t.TempDir() + es := finalizedBlocks(60) + var deleted []string + stop := hooks{removed: func(name string) bool { + deleted = append(deleted, name) + return false + }} + l := openLog(t, dir, compactOpts(512, SyncAlways), stop) + for _, e := range es { + mustAppend(t, l, e) + } + segments := segmentNames(t, dir) + head, _ := l.Head(ctx) + st, _ := l.State(ctx) + if err := l.Truncate(ctx, 1000); err != nil { + t.Fatalf("Truncate: %v", err) + } + mustClose(t, l) + if len(deleted) != 1 || deleted[0] != segments[0] { + t.Fatalf("deleted %v before the crash, want only %s", deleted, segments[0]) + } + + l = openLog(t, dir, compactOpts(512, SyncAlways), hooks{}) + second, _ := parseSegmentName(segments[1]) + if o, _ := l.Oldest(ctx); o != second { + t.Errorf("Oldest after the crash = %d, want %d", o, second) + } + if h, _ := l.Head(ctx); h != head { + t.Errorf("Head after the crash = %d, want %d", h, head) + } + got, _ := l.State(ctx) + wantSame(t, "State after the crash", viewOf(got), viewOf(st)) + es2, err := l.Read(ctx, second, 1000) + if err != nil { + t.Fatalf("Read: %v", err) + } + if want := int(head - second + 1); len(es2) != want { //nolint:gosec // a small test log + t.Errorf("Read returned %d entries, want %d", len(es2), want) + } + // Compaction finishes on the next Truncate. + if err := l.Truncate(ctx, 1000); err != nil { + t.Fatalf("Truncate after reopen: %v", err) + } + names := segmentNames(t, dir) + first, _ := parseSegmentName(names[0]) + if o, _ := l.Oldest(ctx); o != first || len(names) >= len(segments)-1 { + t.Errorf("after the second Truncate: Oldest %d, %d segments; want the first remaining segment and fewer than %d", o, len(names), len(segments)-1) + } +} + +// TestCompactionBoundsDisk keeps truncating a long-running log and checks its directory size. +func TestCompactionBoundsDisk(t *testing.T) { + if testing.Short() { + t.Skip("appends 200,000 blocks") + } + ctx := t.Context() + dir := t.TempDir() + opts := compactOpts(65536, SyncNever) + opts.WindowCap = 1000 + l := openLog(t, dir, opts, hooks{}) + const blocks = 200_000 + for n := uint64(1); n <= blocks; n++ { + es := linkedBlocks(n, 1, 40) + if n%100 == 0 { + es = append(es, finalizeTo(ref(n))) + } + head := mustAppend(t, l, es...) + if n%1000 != 0 { + continue + } + if err := l.Truncate(ctx, head+1); err != nil { + t.Fatalf("Truncate: %v", err) + } + files := listing(t, dir) + var total int64 + for _, size := range files { + total += size + } + if limit := 4*65536 + files[checkpointName] + 4096; total > limit { + t.Fatalf("after %d blocks the directory holds %d bytes, more than %d", n, total, limit) + } + } +} + +// TestTruncateConcurrentRead reads from Oldest while another goroutine appends and truncates: +// every error is ErrTruncated and every entry read equals the one appended. +func TestTruncateConcurrentRead(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + l := openLog(t, t.TempDir(), compactOpts(512, SyncNever), hooks{}) + es := finalizedBlocks(1000) + want := make([][]byte, len(es)) + for i, e := range es { + e.Offset = chainlog.Offset(i + 1) //nolint:gosec // a small test log + want[i] = chainlog.Encode(nil, e) + } + + const readers = 4 + var wg sync.WaitGroup + errs := make([]error, readers) + for r := range readers { + wg.Go(func() { + for ctx.Err() == nil { + o, err := l.Oldest(ctx) + if err != nil { + return + } + got, err := l.Read(ctx, o, 64) + if errors.Is(err, chainlog.ErrTruncated) || errors.Is(err, context.Canceled) { + continue + } + if err != nil { + errs[r] = err + return + } + for i, e := range got { + if e.Offset != o+chainlog.Offset(i) { //nolint:gosec // i < 64 + errs[r] = errors.New("Read returned offsets that are not contiguous from its start") + return + } + if !reflect.DeepEqual(chainlog.Encode(nil, e), want[e.Offset-1]) { + errs[r] = errors.New("an entry read differs from the one appended") + return + } + } + } + }) + } + var appendErr error + for i, e := range es { + head, err := l.Append(ctx, e) + if err != nil { + appendErr = err + break + } + if i%50 == 49 { + if err := l.Truncate(ctx, head+1); err != nil { + appendErr = err + break + } + } + } + cancel() + wg.Wait() + if appendErr != nil { + t.Fatalf("Append or Truncate: %v", appendErr) + } + for r, err := range errs { + if err != nil { + t.Errorf("reader %d: %v", r, err) + } + } + if o, _ := l.Oldest(t.Context()); o == 1 { + t.Error("Oldest is still 1: nothing was truncated") + } +} + +// TestCompactionBound truncates a log whose newest entries follow its last Finalize: compaction +// honors the caller's bound and the last finalize offset plus 1, and checkpoints at Head. +func TestCompactionBound(t *testing.T) { + ctx := t.Context() + dir := t.TempDir() + es := finalizedBlocks(60) + es = append(es, linkedBlocks(61, 10, 40)...) // no Finalize after block 60 + l := openLog(t, dir, compactOpts(512, SyncNever), hooks{}) + for _, e := range es { + mustAppend(t, l, e) + } + bound := lastFinalize(es) + 1 + names := segmentNames(t, dir) + firsts := make([]chainlog.Offset, len(names)) + for i, name := range names { + firsts[i], _ = parseSegmentName(name) + } + + // A caller bound at a segment's last offset keeps that segment: it holds the entry at the bound. + if err := l.Truncate(ctx, firsts[2]-1); err != nil { + t.Fatalf("Truncate(%d): %v", firsts[2]-1, err) + } + wantSame(t, "segments after Truncate at a segment's last offset", segmentNames(t, dir), names[1:]) + first, _ := os.ReadFile(filepath.Join(dir, checkpointName)) + if cp, err := decodeCheckpoint(first, 0); err != nil || cp.l != bound-1 { + t.Errorf("checkpoint after the first Truncate: l %d, %v; want the last finalize %d", cp.l, err, bound-1) + } + + // A caller bound at a segment's first offset keeps that segment and drops the ones before. + keep := firsts[2] + if err := l.Truncate(ctx, keep); err != nil { + t.Fatalf("Truncate(%d): %v", keep, err) + } + if o, _ := l.Oldest(ctx); o != keep { + t.Errorf("Oldest after Truncate(%d) = %d, want %d", keep, o, keep) + } + wantSame(t, "segments after Truncate at a segment's start", segmentNames(t, dir), names[2:]) + + // A bound beyond the last finalize offset stops at L+1: the segment holding it stays. + if err := l.Truncate(ctx, 1000); err != nil { + t.Fatalf("Truncate(1000): %v", err) + } + var wantKept []string + for i, name := range names { + last := chainlog.Offset(len(es)) + if i < len(names)-1 { + last = firsts[i+1] - 1 + } + if i == len(names)-1 || last >= bound { + wantKept = append(wantKept, name) + } + } + wantSame(t, "segments after Truncate(1000)", segmentNames(t, dir), wantKept) + head, _ := l.Head(ctx) + b, _ := os.ReadFile(filepath.Join(dir, checkpointName)) + cp, err := decodeCheckpoint(b, 0) + if err != nil || cp.x != head || cp.l != bound-1 { + t.Errorf("checkpoint x %d, l %d, %v; want Head %d and the last finalize %d", cp.x, cp.l, err, head, bound-1) + } + before := snap(t, l) + mustClose(t, l) + l = openLog(t, dir, compactOpts(512, SyncNever), hooks{}) + wantSame(t, "log after reopen", snap(t, l), before) +} + +// TestCompactionOrder records the fsyncs and deletions of a compaction with SyncAlways: the +// checkpoint is durable before the first deletion, the directory fsync follows the last, and the +// dropped files are closed. +func TestCompactionOrder(t *testing.T) { + ctx := t.Context() + dir := t.TempDir() + var events []string + h := hooks{ + sync: func(name string) error { events = append(events, "sync "+filepath.Base(name)); return nil }, + removed: func(name string) bool { events = append(events, "remove "+name); return true }, + } + l := openLog(t, dir, compactOpts(512, SyncAlways), h) + for _, e := range finalizedBlocks(60) { + mustAppend(t, l, e) + } + names := segmentNames(t, dir) + firstFile := l.segs[0].f + events = nil + if err := l.Truncate(ctx, 1000); err != nil { + t.Fatalf("Truncate: %v", err) + } + kept := segmentNames(t, dir) + want := []string{"sync checkpoint.tmp", "sync " + filepath.Base(dir)} + for _, name := range names[:len(names)-len(kept)] { + want = append(want, "remove "+name) + } + want = append(want, "sync "+filepath.Base(dir)) + wantSame(t, "fsyncs and deletions of a compaction", events, want) + if _, err := firstFile.Read(make([]byte, 1)); !errors.Is(err, os.ErrClosed) { + t.Errorf("a dropped segment's file reads %v, want os.ErrClosed", err) + } +} + +// TestCompactionRemoveFailure fails the deletion of the oldest segment: Truncate reports it, and +// no later segment is deleted until a later Truncate deletes that one, so the directory never has +// a gap that Open would reject. +func TestCompactionRemoveFailure(t *testing.T) { + ctx := t.Context() + dir := t.TempDir() + fail := true + h := hooks{remove: func(path string) error { + if fail && filepath.Base(path) == firstSegment { + return syscall.EBUSY + } + return os.Remove(path) + }} + l := openLog(t, dir, compactOpts(512, SyncNever), h) + for _, e := range finalizedBlocks(60) { + mustAppend(t, l, e) + } + names := segmentNames(t, dir) + want := "wal: remove " + filepath.Join(dir, firstSegment) + ": " + syscall.EBUSY.Error() + if err := l.Truncate(ctx, 1000); err == nil || err.Error() != want { + t.Fatalf("Truncate with a failing deletion = %v, want %q", err, want) + } + wantSame(t, "segments after the failed deletion", segmentNames(t, dir), names) + + // More segments become deletable, but the stuck file still goes first. + mustAppend(t, l, linkedBlocks(61, 20, 40)...) + mustAppend(t, l, finalizeTo(ref(80))) + if err := l.Truncate(ctx, 1000); err == nil || err.Error() != want { + t.Fatalf("Truncate while the deletion still fails = %v, want %q", err, want) + } + if got := segmentNames(t, dir); got[0] != firstSegment || len(got) < len(names) { + t.Errorf("segments while the deletion fails = %v, want none deleted", got) + } + copied := copyDir(t, dir) + reopened, err := Open(copied, compactOpts(512, SyncNever)) + if err != nil { + t.Fatalf("Open of the directory while the deletion fails: %v", err) + } + _ = reopened.Close() + + fail = false + if err := l.Truncate(ctx, 1000); err != nil { + t.Fatalf("Truncate once the deletion works: %v", err) + } + got := segmentNames(t, dir) + if got[0] == firstSegment { + t.Errorf("the stuck segment is still on disk: %v", got) + } + first, _ := parseSegmentName(got[0]) + if o, _ := l.Oldest(ctx); o != first { + t.Errorf("Oldest = %d, want the first segment on disk %d", o, first) + } + before := snap(t, l) + mustClose(t, l) + l = openLog(t, dir, compactOpts(512, SyncNever), hooks{}) + wantSame(t, "log after reopen", snap(t, l), before) +} + +// TestCompactionNothingToDelete truncates a log whose only segment is the active one: nothing is +// deleted and no checkpoint is written. +func TestCompactionNothingToDelete(t *testing.T) { + ctx := t.Context() + dir := t.TempDir() + l := openLog(t, dir, compactOpts(0, SyncNever), hooks{}) + for _, e := range finalizedBlocks(20) { + mustAppend(t, l, e) + } + if err := l.Truncate(ctx, 1000); err != nil { + t.Fatalf("Truncate: %v", err) + } + if _, err := os.Stat(filepath.Join(dir, checkpointName)); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("a Truncate that deletes nothing wrote a checkpoint: %v", err) + } + if o, _ := l.Oldest(ctx); o != 1 { + t.Errorf("Oldest = %d, want 1", o) + } +} diff --git a/chainlog/wal/conformance_test.go b/chainlog/wal/conformance_test.go new file mode 100644 index 0000000..3e1de6a --- /dev/null +++ b/chainlog/wal/conformance_test.go @@ -0,0 +1,37 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal_test + +import ( + "testing" + + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/chainlog/logtest" + "github.com/hmdsefi/evmsync/chainlog/wal" +) + +func TestConformance(t *testing.T) { + tests := []struct { + name string + sync wal.SyncPolicy + segmentSize int64 + }{ + {"never-0", wal.SyncNever, 0}, + {"never-256", wal.SyncNever, 256}, + {"always-4096", wal.SyncAlways, 4096}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + logtest.Run(t, func(t *testing.T) chainlog.Log { + l, err := wal.Open(t.TempDir(), wal.Options{ + Meta: logtest.Meta(), WindowCap: logtest.WindowCap, Sync: tt.sync, SegmentSize: tt.segmentSize, + }) + if err != nil { + t.Fatalf("Open: %v", err) + } + return l + }) + }) + } +} diff --git a/chainlog/wal/doc.go b/chainlog/wal/doc.go new file mode 100644 index 0000000..0911654 --- /dev/null +++ b/chainlog/wal/doc.go @@ -0,0 +1,5 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +// Package wal is a chainlog.Log stored in segment files in one directory. +package wal diff --git a/chainlog/wal/errors.go b/chainlog/wal/errors.go new file mode 100644 index 0000000..839bef1 --- /dev/null +++ b/chainlog/wal/errors.go @@ -0,0 +1,18 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import "errors" + +var ( + // ErrCorrupt is returned by Open and Read when a file of the log holds bytes the WAL did not + // write: a bad record before the tail, a bad meta or checkpoint file, or segments that do not + // fit together. The WAL never repairs it; a corrupt log needs a new directory. + ErrCorrupt = errors.New("wal: corrupt log") + // ErrLocked is returned by Open when another open log holds the directory's lock. + ErrLocked = errors.New("wal: directory locked") + // ErrFailed is returned by Append and Truncate after an earlier write or fsync failed. The + // log stays readable up to Head; reopening it recovers the files. + ErrFailed = errors.New("wal: log failed") +) diff --git a/chainlog/wal/example_test.go b/chainlog/wal/example_test.go new file mode 100644 index 0000000..c503137 --- /dev/null +++ b/chainlog/wal/example_test.go @@ -0,0 +1,43 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal_test + +import ( + "context" + "fmt" + "os" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/chainlog/wal" +) + +// ExampleOpen creates a log, appends a block and reads it back. +func ExampleOpen() { + ctx := context.Background() + dir, err := os.MkdirTemp("", "wal-example") + if err != nil { + panic(err) + } + defer os.RemoveAll(dir) + + log, err := wal.Open(dir, wal.Options{Meta: chainlog.Meta{ChainID: 1}}) + if err != nil { + panic(err) + } + defer log.Close() + + ref := chain.BlockRef{Number: 100, Hash: [32]byte{1}, Time: 1_700_000_000} + if _, err := log.Append(ctx, chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: ref}}); err != nil { + panic(err) + } + entries, err := log.Read(ctx, 1, 10) + if err != nil { + panic(err) + } + for _, e := range entries { + fmt.Println(e.Offset, e.Kind, e.Block.Ref.Number) + } + // Output: 1 block 100 +} diff --git a/chainlog/wal/fileio.go b/chainlog/wal/fileio.go new file mode 100644 index 0000000..9a4c45c --- /dev/null +++ b/chainlog/wal/fileio.go @@ -0,0 +1,151 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "io/fs" + "os" + "path/filepath" +) + +const ( + fileMode = 0o600 + dirMode = 0o750 + fileHeader = 8 // the header of meta and checkpoint: length:uint32le crc:uint32le +) + +var crcTable = crc32.MakeTable(crc32.Castagnoli) + +// sealFile writes the length and CRC-32C of b[fileHeader:] into b's header and returns b. +func sealFile(b []byte) []byte { + body := b[fileHeader:] + binary.LittleEndian.PutUint32(b, uint32(len(body))) //nolint:gosec // meta and checkpoint bodies are far below 4 GiB + binary.LittleEndian.PutUint32(b[4:], crc32.Checksum(body, crcTable)) + return b +} + +// openFile checks the header of a meta or checkpoint file and returns its body. maxSize is the +// largest body the file may declare. Failures are details for ErrCorrupt. +func openFile(b []byte, maxSize uint32) ([]byte, string) { + if len(b) < fileHeader { + return nil, "truncated" + } + length := binary.LittleEndian.Uint32(b) + if length > maxSize { + return nil, "too large" + } + if uint64(len(b)) < fileHeader+uint64(length) { + return nil, "truncated" + } + body := b[fileHeader : fileHeader+int(length)] + if crc32.Checksum(body, crcTable) != binary.LittleEndian.Uint32(b[4:]) { + return nil, "checksum mismatch" + } + if len(b) > fileHeader+int(length) { + return nil, "trailing bytes" + } + return body, "" +} + +// reader reads the fields of a meta or checkpoint body in order. +type reader struct { + b []byte // the bytes not read yet +} + +func (d *reader) take(n int) ([]byte, bool) { + if n > len(d.b) { + return nil, false + } + p := d.b[:n] + d.b = d.b[n:] + return p, true +} + +// uvarint reads a minimal unsigned varint. It reports false for a cut, overlong or padded one. +func (d *reader) uvarint() (uint64, bool) { + x, n := binary.Uvarint(d.b) + if n <= 0 || (n >= 2 && d.b[n-1] == 0) { + return 0, false + } + d.b = d.b[n:] + return x, true +} + +// ioErr returns err as "wal: : ". An error of the os package already names its +// operation and path, so only its cause is kept; errors.Is still matches it. +func ioErr(op, path string, err error) error { + var pe *fs.PathError + var le *os.LinkError + switch { + case errors.As(err, &pe): + err = pe.Err + case errors.As(err, &le): + err = le.Err + } + return fmt.Errorf("wal: %s %s: %w", op, path, err) +} + +// syncer fsyncs a file or a directory. A nil syncer means no fsync (SyncNever). +type syncer func(f *os.File) error + +// writeFileAtomic replaces dir/name with data: it writes name.tmp, fsyncs it with fsync, closes +// it, renames it to name, and then fsyncs dir with fsync. With a non-nil fsync, a crash leaves +// either the old file or the new one, and possibly name.tmp, which Open deletes. On an error it +// removes name.tmp. +func writeFileAtomic(dir, name string, data []byte, fsync syncer) error { + tmp := filepath.Join(dir, name+".tmp") + if err := writeFile(tmp, data, fsync); err != nil { + _ = os.Remove(tmp) + return err + } + path := filepath.Join(dir, name) + if err := os.Rename(tmp, path); err != nil { + _ = os.Remove(tmp) + return ioErr("rename", path, err) + } + return syncDir(dir, fsync) +} + +// writeFile creates path with data, fsyncs it with fsync, and closes it. +func writeFile(path string, data []byte, fsync syncer) error { + f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, fileMode) //nolint:gosec // a file name the WAL owns + if err != nil { + return ioErr("open", path, err) + } + if _, err := f.Write(data); err != nil { + return errors.Join(ioErr("write", path, err), f.Close()) + } + if fsync != nil { + if err := fsync(f); err != nil { + return errors.Join(ioErr("sync", path, err), f.Close()) + } + } + if err := f.Close(); err != nil { + return ioErr("write", path, err) + } + return nil +} + +// syncDir fsyncs the directory dir with fsync, so the names created, renamed or removed in it are +// durable. With a nil fsync it does nothing. +func syncDir(dir string, fsync syncer) error { + if fsync == nil { + return nil + } + d, err := os.Open(dir) //nolint:gosec // dir is the log's own directory + if err != nil { + return ioErr("open", dir, err) + } + if err := fsync(d); err != nil { + return errors.Join(ioErr("sync", dir, err), d.Close()) + } + if err := d.Close(); err != nil { + return ioErr("sync", dir, err) + } + return nil +} diff --git a/chainlog/wal/fuzz_test.go b/chainlog/wal/fuzz_test.go new file mode 100644 index 0000000..8552215 --- /dev/null +++ b/chainlog/wal/fuzz_test.go @@ -0,0 +1,148 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "bytes" + "encoding/binary" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/hmdsefi/evmsync/chainlog" +) + +// FuzzRecover damages a closed log and reopens it. Open must never panic and may fail only with +// ErrCorrupt; whatever it keeps must be a prefix of what was appended. Mode 0 cuts the last +// segment, which Open always recovers; modes 1 and 2 flip a byte of the last or of any segment, +// which is corruption unless it hits the very last record. +func FuzzRecover(f *testing.F) { + f.Add(uint8(4), uint8(0), uint32(100), uint8(0)) + f.Add(uint8(4), uint8(1), uint32(20), uint8(1)) + f.Add(uint8(39), uint8(2), uint32(7), uint8(0x80)) + f.Add(uint8(0), uint8(0), uint32(0), uint8(0)) + + f.Fuzz(func(t *testing.T, n, mode uint8, pos uint32, x uint8) { + dir := t.TempDir() + es := linkedBlocks(1, 1+uint64(n%40), 40) + l, err := Open(dir, recoveryOpts(512)) + if err != nil { + t.Fatalf("Open: %v", err) + } + for _, e := range es { + mustAppend(t, l, e) + } + mustClose(t, l) + + names := segmentNames(t, dir) + seg := len(names) - 1 + if mode%3 == 2 { + seg = int(pos % uint32(len(names))) //nolint:gosec // fewer than 40 segments + } + path := filepath.Join(dir, names[seg]) + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + size := uint32(len(b)) //nolint:gosec // a segment of at most a few KiB + first, _ := parseSegmentName(names[seg]) + start, end := index(first)-1, len(es) // the segment's entries are es[start:end] + if seg < len(names)-1 { + next, _ := parseSegmentName(names[seg+1]) + end = index(next) - 1 + } + ends := recordEnds(es[start:end], first) + + const ( + anyPrefix = -1 // Open may keep any prefix of the entries + wantCorrupt = -2 // Open must fail with ErrCorrupt + ) + var wantHead int // the Head Open recovers to, or one of the two above + switch { + case mode%3 == 0: + cut := int64(pos % (size + 1)) + if err := os.Truncate(path, cut); err != nil { + t.Fatal(err) + } + kept := 0 + for kept < len(ends) && ends[kept] <= cut { + kept++ + } + wantHead = start + kept + case size == 0: + t.Skip("no byte to flip") + default: + i := pos % size + b[i] ^= x | 1 + if err := os.WriteFile(path, b, 0o600); err != nil { //nolint:gosec // a segment of the test's own directory + t.Fatal(err) + } + wantHead = anyPrefix + recStart := int64(0) + for k, recEnd := range ends { + if int64(i) < recEnd { + inLast := seg == len(names)-1 + if int64(i) < recStart+4 && inLast { + // A damaged length field: a torn tail when the declared end is at or past + // the end of the file, and corruption otherwise. + length := int64(binary.LittleEndian.Uint32(b[recStart:])) + if recStart+8+length >= int64(size) { + wantHead = start + k + } else { + wantHead = wantCorrupt + } + } else if inLast && k == len(ends)-1 { + wantHead = len(es) - 1 + } else { + wantHead = wantCorrupt + } + break + } + recStart = recEnd + } + } + + l, err = Open(dir, recoveryOpts(512)) + if err != nil { + if !errors.Is(err, ErrCorrupt) { + t.Fatalf("Open = %v, want success or ErrCorrupt", err) + } + if wantHead >= 0 { + t.Fatalf("Open = %v, want Head %d", err, wantHead) + } + return + } + defer l.Close() + // The active segment ends at its last whole record, where the next Append writes. + active := l.segs[len(l.segs)-1] + fi, err := os.Stat(filepath.Join(dir, active.name)) + if err != nil { + t.Fatal(err) + } + if fi.Size() != active.size { + t.Fatalf("active segment holds %d bytes on disk, the log counts %d", fi.Size(), active.size) + } + if wantHead == wantCorrupt { + t.Fatal("Open succeeded on a damaged record that is not the last one, want ErrCorrupt") + } + head, _ := l.Head(t.Context()) + if wantHead >= 0 && index(head) != wantHead { + t.Fatalf("Head = %d, want %d", head, wantHead) + } + if head == 0 { + return + } + got, err := l.Read(t.Context(), 1, len(es)) + if err != nil { + t.Fatalf("Read: %v", err) + } + if !bytes.Equal(asRead(got), encoded(es[:head])) { + t.Fatal("entries 1..Head differ from the originals") + } + }) +} + +// index returns offset o as an int; test logs hold far fewer than 2^63 entries. +func index(o chainlog.Offset) int { return int(o) } //nolint:gosec // a small test offset diff --git a/chainlog/wal/helpers_test.go b/chainlog/wal/helpers_test.go new file mode 100644 index 0000000..5af9d08 --- /dev/null +++ b/chainlog/wal/helpers_test.go @@ -0,0 +1,169 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "io" + "maps" + "os" + "path/filepath" + "reflect" + "testing" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/chainlog/logtest" +) + +// linkedBlocks returns n linked blocks first..first+n-1 of chain 0 (hashes as in logtest), each +// with one log of one topic and dataLen bytes of data. +func linkedBlocks(first, n uint64, dataLen int) []chainlog.Entry { + es := make([]chainlog.Entry, 0, n) + for num := first; num < first+n; num++ { + r := ref(num) + l := chain.Log{BlockNumber: r.Number, BlockHash: r.Hash, TxHash: r.Hash, Index: 1} + l.Address[19] = 0xaa + l.Topics = append(l.Topics, r.Parent) + if dataLen > 0 { + l.Data = make([]byte, dataLen) + l.Data[0] = 0x5a + } + es = append(es, chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: r, Logs: []chain.Log{l}}}) + } + return es +} + +func finalizeTo(r chain.BlockRef) chainlog.Entry { + return chainlog.Entry{Kind: chainlog.KindFinalize, To: r} +} + +func revertTo(r chain.BlockRef) chainlog.Entry { + return chainlog.Entry{Kind: chainlog.KindRevert, To: r} +} + +// testOpts are the options of most tests: the conformance suite's meta and cap, no fsync. +func testOpts(segmentSize int64) Options { + return Options{Meta: logtest.Meta(), WindowCap: logtest.WindowCap, Sync: SyncNever, SegmentSize: segmentSize} +} + +// openLog opens dir with opts and h and closes the log when the test ends. +func openLog(t *testing.T, dir string, opts Options, h hooks) *Log { + t.Helper() + l, err := openWithHooks(dir, opts, h) + if err != nil { + t.Fatalf("Open: %v", err) + } + t.Cleanup(func() { _ = l.Close() }) + return l +} + +func mustAppend(t *testing.T, l *Log, es ...chainlog.Entry) chainlog.Offset { + t.Helper() + off, err := l.Append(t.Context(), es...) + if err != nil { + t.Fatalf("Append: %v", err) + } + return off +} + +func mustClose(t *testing.T, l *Log) { + t.Helper() + if err := l.Close(); err != nil { + t.Fatalf("Close: %v", err) + } +} + +// snapshotOf is what a log shows: Head, Oldest, the state's accessors and every retained entry. +type snapshotOf struct { + head, oldest chainlog.Offset + state stateView + entries []chainlog.Entry +} + +func snap(t *testing.T, l chainlog.Log) snapshotOf { + t.Helper() + ctx := t.Context() + var s snapshotOf + var err error + if s.head, err = l.Head(ctx); err != nil { + t.Fatalf("Head: %v", err) + } + if s.oldest, err = l.Oldest(ctx); err != nil { + t.Fatalf("Oldest: %v", err) + } + st, err := l.State(ctx) + if err != nil { + t.Fatalf("State: %v", err) + } + s.state = viewOf(st) + if s.entries, err = l.Read(ctx, s.oldest, 1<<20); err != nil { + t.Fatalf("Read: %v", err) + } + return s +} + +func wantSame(t *testing.T, what string, got, want any) { + t.Helper() + if !reflect.DeepEqual(got, want) { + t.Errorf("%s =\n%+v\nwant\n%+v", what, got, want) + } +} + +// listing returns the names and sizes of the files in dir. +func listing(t *testing.T, dir string) map[string]int64 { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + m := make(map[string]int64, len(entries)) + for _, e := range entries { + fi, err := e.Info() + if err != nil { + t.Fatalf("Info: %v", err) + } + m[e.Name()] = fi.Size() + } + return m +} + +// copyDir copies the files of src into a new directory and returns it. +func copyDir(t *testing.T, src string) string { + t.Helper() + dst := t.TempDir() + for name := range maps.Keys(listing(t, src)) { + in, err := os.Open(filepath.Join(src, name)) + if err != nil { + t.Fatalf("Open: %v", err) + } + out, err := os.OpenFile(filepath.Join(dst, name), os.O_WRONLY|os.O_CREATE, 0o600) + if err != nil { + t.Fatalf("Create: %v", err) + } + if _, err := io.Copy(out, in); err != nil { + t.Fatalf("Copy: %v", err) + } + _ = in.Close() + if err := out.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + } + return dst +} + +// segmentNames returns the segment files of dir in offset order. +func segmentNames(t *testing.T, dir string) []string { + t.Helper() + var names []string + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + for _, e := range entries { + if _, ok := parseSegmentName(e.Name()); ok { + names = append(names, e.Name()) + } + } + return names +} diff --git a/chainlog/wal/hooks.go b/chainlog/wal/hooks.go new file mode 100644 index 0000000..f492b55 --- /dev/null +++ b/chainlog/wal/hooks.go @@ -0,0 +1,33 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "os" + + "github.com/hmdsefi/evmsync/chainlog" +) + +// hooks let tests inject failures and observe the WAL's work. Open uses the zero hooks. +type hooks struct { + write func(f *os.File, p []byte) (int, error) // nil means f.Write + replay func(off chainlog.Offset) // each entry Open applies after the checkpoint + removed func(name string) bool // after each segment deletion; false stops compaction there + sync func(name string) error // before each fsync; an error replaces the fsync + remove func(path string) error // nil means os.Remove +} + +func (h *hooks) writeTo(f *os.File, p []byte) (int, error) { + if h.write != nil { + return h.write(f, p) + } + return f.Write(p) +} + +func (h *hooks) removeFile(path string) error { + if h.remove != nil { + return h.remove(path) + } + return os.Remove(path) +} diff --git a/chainlog/wal/lock_other.go b/chainlog/wal/lock_other.go new file mode 100644 index 0000000..8cd41f4 --- /dev/null +++ b/chainlog/wal/lock_other.go @@ -0,0 +1,20 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +//go:build !unix + +package wal + +import ( + "errors" + "fmt" + "os" +) + +// errWouldBlock reports that another open file description holds the lock. +var errWouldBlock = errors.New("lock held") + +// lockFile fails: the WAL needs flock, which only Unix systems have. +func lockFile(path string) (*os.File, error) { + return nil, fmt.Errorf("wal: lock %s: %w", path, errors.ErrUnsupported) +} diff --git a/chainlog/wal/lock_test.go b/chainlog/wal/lock_test.go new file mode 100644 index 0000000..a4e55c4 --- /dev/null +++ b/chainlog/wal/lock_test.go @@ -0,0 +1,87 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +//go:build unix + +package wal + +import ( + "bufio" + "errors" + "os" + "os/exec" + "testing" +) + +func TestLock(t *testing.T) { + dir := t.TempDir() + l1, err := Open(dir, testOpts(0)) + if err != nil { + t.Fatalf("Open: %v", err) + } + _, err = Open(dir, testOpts(0)) + if !errors.Is(err, ErrLocked) { + t.Fatalf("second Open = %v, want one wrapping ErrLocked", err) + } + if got, want := err.Error(), "wal: directory locked: "+dir; got != want { + t.Errorf("second Open error = %q, want %q", got, want) + } + mustClose(t, l1) + l2, err := Open(dir, testOpts(0)) + if err != nil { + t.Fatalf("Open after Close: %v", err) + } + mustClose(t, l2) + + // A process killed while it holds the lock leaves the directory openable. + cmd := exec.Command(os.Args[0], "-test.run=^TestLockHelper$") //nolint:gosec // the test binary itself + cmd.Env = append(os.Environ(), "EVMSYNC_WAL_LOCK_CHILD="+dir) + stdin, err := cmd.StdinPipe() + if err != nil { + t.Fatal(err) + } + defer stdin.Close() + stdout, err := cmd.StdoutPipe() + if err != nil { + t.Fatal(err) + } + if err := cmd.Start(); err != nil { + t.Fatalf("starting the child: %v", err) + } + lines := bufio.NewScanner(stdout) + locked := false + for !locked && lines.Scan() { + locked = lines.Text() == "locked" + } + if !locked { + _ = cmd.Process.Kill() + _ = cmd.Wait() + t.Fatal("the child exited without taking the lock") + } + if err := cmd.Process.Kill(); err != nil { // SIGKILL + t.Fatalf("Kill: %v", err) + } + _ = cmd.Wait() + l3, err := Open(dir, testOpts(0)) + if err != nil { + t.Fatalf("Open after the child was killed: %v", err) + } + mustClose(t, l3) +} + +// TestLockHelper is the child process of TestLock: it opens the directory, prints "locked" and +// blocks on stdin until it is killed. +func TestLockHelper(t *testing.T) { + dir := os.Getenv("EVMSYNC_WAL_LOCK_CHILD") + if dir == "" { + t.Skip("runs only as TestLock's child process") + } + l, err := Open(dir, testOpts(0)) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer l.Close() + _, _ = os.Stdout.WriteString("locked\n") + buf := make([]byte, 1) + _, _ = os.Stdin.Read(buf) +} diff --git a/chainlog/wal/lock_unix.go b/chainlog/wal/lock_unix.go new file mode 100644 index 0000000..f01c911 --- /dev/null +++ b/chainlog/wal/lock_unix.go @@ -0,0 +1,32 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +//go:build unix + +package wal + +import ( + "errors" + "os" + "syscall" +) + +// errWouldBlock reports that another open file description holds the lock. +var errWouldBlock = errors.New("lock held") + +// lockFile opens path, creating it, and takes a non-blocking exclusive flock on it. The kernel +// releases the lock when the file is closed or the process dies. +func lockFile(path string) (*os.File, error) { + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, fileMode) //nolint:gosec // path is the log's own lock file + if err != nil { + return nil, ioErr("open", path, err) + } + if err := syscall.Flock(int(f.Fd()), syscall.LOCK_EX|syscall.LOCK_NB); err != nil { //nolint:gosec // a file descriptor fits an int + _ = f.Close() + if errors.Is(err, syscall.EWOULDBLOCK) { + return nil, errWouldBlock + } + return nil, ioErr("lock", path, err) + } + return f, nil +} diff --git a/chainlog/wal/meta.go b/chainlog/wal/meta.go new file mode 100644 index 0000000..2924083 --- /dev/null +++ b/chainlog/wal/meta.go @@ -0,0 +1,140 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "encoding/binary" + "fmt" + "slices" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +const ( + metaName = "meta" + metaMagic = "EVMSMETA" + metaVersion = 1 + maxMetaSize = 1 << 20 // the largest meta body, in bytes + addressLen = 20 + hashLen = 32 + maxFilterLen = chain.MaxFilterValues +) + +// encodeMeta returns the meta file for m: a header (body length and CRC-32C of the body, both +// little-endian uint32) and a body of the magic, version 1, the ID, the chain ID and start as +// varints, then the filter's addresses and topic positions, each list prefixed by its length. +// m's filter must already be valid. +func encodeMeta(m chainlog.Meta) []byte { + b := make([]byte, fileHeader, 64) + b = append(b, metaMagic...) + b = append(b, metaVersion) + b = append(b, m.ID[:]...) + b = binary.AppendUvarint(b, m.ChainID) + b = binary.AppendUvarint(b, m.Start) + b = binary.AppendUvarint(b, uint64(len(m.Filter.Addresses))) + for _, a := range m.Filter.Addresses { + b = append(b, a[:]...) + } + b = binary.AppendUvarint(b, uint64(len(m.Filter.Topics))) + for _, pos := range m.Filter.Topics { + b = binary.AppendUvarint(b, uint64(len(pos))) + for _, t := range pos { + b = append(b, t[:]...) + } + } + return sealFile(b) +} + +// decodeMeta decodes a meta file into a normalized meta: empty filter lists are nil. Its errors +// wrap ErrCorrupt and read "wal: corrupt log: meta: ". +func decodeMeta(b []byte) (chainlog.Meta, error) { + corrupt := func(detail string) (chainlog.Meta, error) { + return chainlog.Meta{}, fmt.Errorf("%w: meta: %s", ErrCorrupt, detail) + } + body, detail := openFile(b, maxMetaSize) + if detail != "" { + return corrupt(detail) + } + d := reader{b: body} + magic, ok := d.take(len(metaMagic)) + if !ok { + return corrupt("truncated") + } + if string(magic) != metaMagic { + return corrupt("bad magic") + } + version, ok := d.take(1) + if !ok { + return corrupt("truncated") + } + if version[0] != metaVersion { + return corrupt(fmt.Sprintf("version %d", version[0])) + } + var m chainlog.Meta + id, ok := d.take(len(m.ID)) + if !ok { + return corrupt("truncated") + } + copy(m.ID[:], id) + if m.ChainID, ok = d.uvarint(); !ok { + return corrupt("truncated") + } + if m.Start, ok = d.uvarint(); !ok { + return corrupt("truncated") + } + + naddr, ok := d.uvarint() + if !ok { + return corrupt("truncated") + } + if naddr > maxFilterLen { + return corrupt("filter out of range") + } + if naddr > 0 { + m.Filter.Addresses = slices.Grow(m.Filter.Addresses, int(naddr)) + } + for range naddr { + a, ok := d.take(addressLen) + if !ok { + return corrupt("truncated") + } + m.Filter.Addresses = append(m.Filter.Addresses, [addressLen]byte(a)) + } + + npos, ok := d.uvarint() + if !ok { + return corrupt("truncated") + } + if npos > chain.MaxTopicPositions { + return corrupt("filter out of range") + } + if npos > 0 { + m.Filter.Topics = slices.Grow(m.Filter.Topics, int(npos)) + } + for i := range npos { + ntopics, ok := d.uvarint() + if !ok { + return corrupt("truncated") + } + if ntopics > maxFilterLen { + return corrupt("filter out of range") + } + m.Filter.Topics = append(m.Filter.Topics, nil) + if ntopics > 0 { + m.Filter.Topics[i] = slices.Grow(m.Filter.Topics[i], int(ntopics)) + } + for range ntopics { + t, ok := d.take(hashLen) + if !ok { + return corrupt("truncated") + } + m.Filter.Topics[i] = append(m.Filter.Topics[i], [hashLen]byte(t)) + } + } + if len(d.b) > 0 { + return corrupt("trailing bytes") + } + return m, nil +} diff --git a/chainlog/wal/meta_test.go b/chainlog/wal/meta_test.go new file mode 100644 index 0000000..0721e0b --- /dev/null +++ b/chainlog/wal/meta_test.go @@ -0,0 +1,235 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "bytes" + "encoding/binary" + "errors" + "hash/crc32" + "io/fs" + "os" + "path/filepath" + "reflect" + "strings" + "syscall" + "testing" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +// seal returns body with the meta and checkpoint file header (length, CRC-32C) in front. +func seal(body []byte) []byte { + b := binary.LittleEndian.AppendUint32(nil, uint32(len(body))) //nolint:gosec // test bodies are small + b = binary.LittleEndian.AppendUint32(b, crc32.Checksum(body, crc32.MakeTable(crc32.Castagnoli))) + return append(b, body...) +} + +func cat(parts ...[]byte) []byte { + var b []byte + for _, p := range parts { + b = append(b, p...) + } + return b +} + +func uv(x uint64) []byte { return binary.AppendUvarint(nil, x) } + +func rep(b byte, n int) []byte { return bytes.Repeat([]byte{b}, n) } + +// testMeta has two addresses and three topic positions, the middle one empty. +func testMeta() chainlog.Meta { + m := chainlog.Meta{ID: chainlog.ID{0: 0x11, 15: 0xff}, ChainID: 31337, Start: 300} + m.Filter.Addresses = append(m.Filter.Addresses, [20]byte{19: 0xaa}, [20]byte{0: 0xbb}) + m.Filter.Topics = append(m.Filter.Topics, nil, nil, nil) + m.Filter.Topics[0] = append(m.Filter.Topics[0], [32]byte{31: 1}, [32]byte{31: 2}) + m.Filter.Topics[1] = m.Filter.Topics[1][:0:0] // empty, not nil + m.Filter.Topics[2] = append(m.Filter.Topics[2], [32]byte{0: 3}) + return m +} + +func TestMetaRoundTrip(t *testing.T) { + m := testMeta() + got, err := decodeMeta(encodeMeta(m)) + if err != nil { + t.Fatalf("decodeMeta: %v", err) + } + want := testMeta() + want.Filter.Topics[1] = nil + if !reflect.DeepEqual(got, want) { + t.Errorf("decodeMeta(encodeMeta(m)) =\n%+v\nwant\n%+v", got, want) + } + if got.Filter.Topics[1] != nil { + t.Errorf("empty topic position decoded as %#v, want nil", got.Filter.Topics[1]) + } + + empty := chainlog.Meta{ID: chainlog.ID{1}, Filter: chain.Filter{Addresses: testMeta().Filter.Addresses[:0], Topics: testMeta().Filter.Topics[:0]}} + got, err = decodeMeta(encodeMeta(empty)) + if err != nil { + t.Fatalf("decodeMeta: %v", err) + } + if got.Filter.Addresses != nil || got.Filter.Topics != nil || got.ID != empty.ID { + t.Errorf("meta with empty lists decoded as %+v, want nil Addresses and Topics", got) + } +} + +// metaBody is the body encodeMeta writes for testMeta, built by hand. +func metaBody() []byte { + m := testMeta() + return cat( + []byte("EVMSMETA"), []byte{0x01}, m.ID[:], uv(31337), uv(300), + uv(2), m.Filter.Addresses[0][:], m.Filter.Addresses[1][:], + uv(3), uv(2), m.Filter.Topics[0][0][:], m.Filter.Topics[0][1][:], uv(0), uv(1), m.Filter.Topics[2][0][:], + ) +} + +func TestMetaLayout(t *testing.T) { + b := encodeMeta(testMeta()) + if want := seal(metaBody()); !bytes.Equal(b, want) { + t.Fatalf("encodeMeta =\n%x\nwant\n%x", b, want) + } + id := testMeta().ID + if string(b[8:16]) != "EVMSMETA" || b[16] != 0x01 || !bytes.Equal(b[17:33], id[:]) { + t.Errorf("meta starts %x, want the header, EVMSMETA, 01 and the ID", b[:33]) + } +} + +func TestMetaCorrupt(t *testing.T) { + valid := encodeMeta(testMeta()) + body := metaBody() + flipped := bytes.Clone(valid) + flipped[20] ^= 0x01 + tooLarge := bytes.Clone(valid) + binary.LittleEndian.PutUint32(tooLarge, 1048577) + head := cat([]byte("EVMSMETA"), []byte{0x01}, make([]byte, 16), uv(1), uv(2)) // through start + + tests := []struct { + name string + data []byte + detail string + }{ + {"cut to 5 bytes", valid[:5], "truncated"}, + {"empty", nil, "truncated"}, + {"cut by one byte", valid[:len(valid)-1], "truncated"}, + {"flipped body byte", flipped, "checksum mismatch"}, + {"bad magic", seal(cat([]byte("XXXXXXXX"), body[8:])), "bad magic"}, + {"version 2", seal(cat(body[:8], []byte{0x02}, body[9:])), "version 2"}, + {"npos 5", seal(cat(head, uv(0), uv(5))), "filter out of range"}, + {"naddr 1001", seal(cat(head, uv(1001))), "filter out of range"}, + {"ntopics 1001", seal(cat(head, uv(0), uv(1), uv(1001))), "filter out of range"}, + {"length 1048577", tooLarge, "too large"}, + {"trailing byte in the body", seal(cat(body, []byte{0x00})), "trailing bytes"}, + {"trailing byte after the body", cat(valid, []byte{0x00}), "trailing bytes"}, + {"body cut in the ID", seal(body[:12]), "truncated"}, + {"body cut in an address", seal(cat(head, uv(1), rep(0xaa, 19))), "truncated"}, + {"body cut in a topic", seal(cat(head, uv(0), uv(1), uv(1), rep(0x01, 31))), "truncated"}, + {"chain ID non-minimal", seal(cat(body[:25], []byte{0x80, 0x00}, body[28:])), "truncated"}, + {"no positions count", seal(cat(head, uv(0))), "truncated"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, err := decodeMeta(tt.data) + if !errors.Is(err, ErrCorrupt) { + t.Fatalf("decodeMeta error = %v, want one wrapping ErrCorrupt", err) + } + if got, want := err.Error(), "wal: corrupt log: meta: "+tt.detail; got != want { + t.Errorf("error = %q, want %q", got, want) + } + }) + } +} + +func TestWriteFileAtomic(t *testing.T) { + for _, tt := range []struct { + name string + fsync syncer + }{{metaName, nil}, {checkpointName, (*os.File).Sync}} { + dir := t.TempDir() + if err := writeFileAtomic(dir, tt.name, []byte("first"), tt.fsync); err != nil { + t.Fatalf("writeFileAtomic: %v", err) + } + if err := writeFileAtomic(dir, tt.name, []byte("second"), tt.fsync); err != nil { + t.Fatalf("writeFileAtomic over an existing file: %v", err) + } + got, err := os.ReadFile(filepath.Join(dir, tt.name)) + if err != nil || string(got) != "second" { + t.Errorf("%s: content %q, %v; want \"second\"", tt.name, got, err) + } + fi, err := os.Stat(filepath.Join(dir, tt.name)) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if mode := fi.Mode().Perm(); mode != 0o600 { + t.Errorf("%s: mode %v, want 0600", tt.name, mode) + } + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("ReadDir: %v", err) + } + if len(entries) != 1 { + t.Errorf("%s: directory holds %v, want only the file", tt.name, entries) + } + } + + missing := filepath.Join(t.TempDir(), "absent") + err := writeFileAtomic(missing, "meta", []byte("x"), (*os.File).Sync) + if !errors.Is(err, fs.ErrNotExist) { + t.Fatalf("writeFileAtomic into a missing directory = %v, want one wrapping fs.ErrNotExist", err) + } + if want := "wal: open " + filepath.Join(missing, "meta.tmp") + ": " + syscall.ENOENT.Error(); err.Error() != want { + t.Errorf("error = %q, want %q", err, want) + } + + // A failed rename leaves no temporary file behind. + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "meta", "x"), 0o750); err != nil { + t.Fatal(err) + } + err = writeFileAtomic(dir, "meta", []byte("x"), nil) + if err == nil || !strings.HasPrefix(err.Error(), "wal: rename "+filepath.Join(dir, "meta")+": ") { + t.Errorf("writeFileAtomic onto a directory = %v, want a rename error", err) + } + if _, err := os.Stat(filepath.Join(dir, "meta.tmp")); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("meta.tmp after a failed rename: %v, want it removed", err) + } + if err := syncDir(missing, (*os.File).Sync); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("syncDir of a missing directory = %v, want one wrapping fs.ErrNotExist", err) + } +} + +// TestMetaLimits round-trips a meta at every filter limit and checks the body size limit. +func TestMetaLimits(t *testing.T) { + var m chainlog.Meta + m.ID[0] = 1 + for i := range 1000 { + var a [20]byte + binary.BigEndian.PutUint16(a[:], uint16(i)) //nolint:gosec // i < 1000 + m.Filter.Addresses = append(m.Filter.Addresses, a) + } + for range 4 { + m.Filter.Topics = append(m.Filter.Topics, nil) + } + for i := range 1000 { + var topic [32]byte + binary.BigEndian.PutUint16(topic[:], uint16(i)) //nolint:gosec // i < 1000 + m.Filter.Topics[3] = append(m.Filter.Topics[3], topic) + } + m.Filter.Topics[0] = append(m.Filter.Topics[0], [32]byte{1}) + got, err := decodeMeta(encodeMeta(m)) + if err != nil { + t.Fatalf("decodeMeta of a meta at every limit: %v", err) + } + if !reflect.DeepEqual(got, m) { + t.Error("meta at every limit does not round-trip") + } + + body := make([]byte, 100) + if b, detail := openFile(seal(body), 100); detail != "" || len(b) != 100 { + t.Errorf("openFile of a body at the limit = %d bytes, %q; want 100, \"\"", len(b), detail) + } + if _, detail := openFile(seal(append(body, 0)), 100); detail != "too large" { + t.Errorf("openFile of a body one byte over the limit = %q, want \"too large\"", detail) + } +} diff --git a/chainlog/wal/norace_test.go b/chainlog/wal/norace_test.go new file mode 100644 index 0000000..e7cb556 --- /dev/null +++ b/chainlog/wal/norace_test.go @@ -0,0 +1,8 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +//go:build !race + +package wal + +const raceEnabled = false diff --git a/chainlog/wal/perf_test.go b/chainlog/wal/perf_test.go new file mode 100644 index 0000000..c26fe41 --- /dev/null +++ b/chainlog/wal/perf_test.go @@ -0,0 +1,52 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "os" + "testing" + "time" +) + +func TestPerf(t *testing.T) { + if os.Getenv("EVMSYNC_PERF") != "1" || raceEnabled { + t.Skip("set EVMSYNC_PERF=1 and run without -race to measure the targets") + } + tests := []struct { + name string + bench func(*testing.B) + check func(r testing.BenchmarkResult) string // "" when the target is met + }{ + {"Append/never", func(b *testing.B) { benchAppend(b, SyncNever) }, func(r testing.BenchmarkResult) string { + if d := time.Duration(r.NsPerOp()); d > 15*time.Microsecond { + return d.String() + "/op, want at most 15µs" + } + return "" + }}, + {"Read", BenchmarkRead, func(r testing.BenchmarkResult) string { + if perSec := 256 * 1e9 / float64(r.NsPerOp()); perSec < 200_000 { + return "fewer than 200,000 entries/s" + } + return "" + }}, + {"Open", BenchmarkOpen, func(r testing.BenchmarkResult) string { + if mibPerSec := float64(256<<20) / float64(r.NsPerOp()) * 1e9 / (1 << 20); mibPerSec < 150 { + return "slower than 150 MiB/s" + } + return "" + }}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := testing.Benchmark(tt.bench) + if r.N == 0 { + t.Fatal("benchmark did not run") + } + t.Logf("%d ns/op, %d allocs/op", r.NsPerOp(), r.AllocsPerOp()) + if msg := tt.check(r); msg != "" { + t.Error(msg) + } + }) + } +} diff --git a/chainlog/wal/race_test.go b/chainlog/wal/race_test.go new file mode 100644 index 0000000..13a0899 --- /dev/null +++ b/chainlog/wal/race_test.go @@ -0,0 +1,8 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +//go:build race + +package wal + +const raceEnabled = true diff --git a/chainlog/wal/recover.go b/chainlog/wal/recover.go new file mode 100644 index 0000000..0b54024 --- /dev/null +++ b/chainlog/wal/recover.go @@ -0,0 +1,351 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "bufio" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" + "io/fs" + "log/slog" + "os" + "path/filepath" + + "github.com/hmdsefi/evmsync/chainlog" +) + +const scanBuffer = 1 << 20 + +// segScan is what Open learns from reading one segment from its start. +type segScan struct { + first, last chainlog.Offset // last == first-1 for an empty segment + size int64 // file size + index []int64 // file position of every 256th record + end int64 // end of the last good record + bad *badRecord // first record that fails Decode or has the wrong offset; nil if none +} + +// badRecord is the first record of a segment that fails. +type badRecord struct { + pos, declaredEnd int64 // declaredEnd = pos + 8 + length, or -1 when fewer than 8 bytes remain + err error // the Decode error or "offset %d, want %d" +} + +// scanSegment reads every record of f, a segment whose name gives first, and stops at the first +// bad record. It returns an error for I/O failures only. +func scanSegment(f *os.File, first chainlog.Offset) (segScan, error) { + fi, err := f.Stat() + if err != nil { + return segScan{}, ioErr("read", f.Name(), err) + } + sc := segScan{first: first, last: first - 1, size: fi.Size()} + r := bufio.NewReaderSize(io.NewSectionReader(f, 0, sc.size), scanBuffer) + var buf []byte + for sc.end < sc.size { + rec, err := readRecord(r, &buf, sc.size-sc.end) + if err != nil { + return segScan{}, ioErr("read", f.Name(), err) + } + want := sc.last + 1 + n, err := decodeAt(rec, want, nil) + if err != nil { + declaredEnd := int64(-1) + if len(rec) >= recHeader { + declaredEnd = sc.end + recHeader + int64(binary.LittleEndian.Uint32(rec)) + } + sc.bad = &badRecord{pos: sc.end, declaredEnd: declaredEnd, err: err} + break + } + if (want-first)%indexEvery == 0 { + sc.index = append(sc.index, sc.end) + } + sc.last = want + sc.end += int64(n) + } + return sc, nil +} + +// dirListing is what Open finds in the directory. +type dirListing struct { + meta, checkpoint bool + segments []string // segment names in name order, which is offset order +} + +// list removes the temporary files of interrupted writes and lists the names the WAL owns. +func (l *Log) list() (dirListing, error) { + for _, tmp := range []string{metaName + ".tmp", checkpointName + ".tmp"} { + path := filepath.Join(l.dir, tmp) + if err := os.Remove(path); err != nil && !errors.Is(err, fs.ErrNotExist) { + return dirListing{}, ioErr("remove", path, err) + } + } + entries, err := os.ReadDir(l.dir) + if err != nil { + return dirListing{}, ioErr("read", l.dir, err) + } + var d dirListing + for _, e := range entries { + switch name := e.Name(); name { + case metaName: + d.meta = true + case checkpointName: + d.checkpoint = true + default: + if _, ok := parseSegmentName(name); ok { + d.segments = append(d.segments, name) + } + } + } + return d, nil +} + +// load reads or creates the log in l.dir and leaves l ready for use. It reports whether the log +// was created. On error the caller closes l's files. +func (l *Log) load(opts Options) (created bool, err error) { + d, err := l.list() + if err != nil { + return false, err + } + l.state = chainlog.NewState(opts.WindowCap) + l.head, l.oldest = 0, 1 + + if !d.meta && !d.checkpoint && len(d.segments) == 0 { + return true, l.create(opts.Meta) + } + if !d.meta { + return false, fmt.Errorf("%w: meta: missing", ErrCorrupt) + } + b, err := l.readFile(metaName, maxMetaSize) + if err != nil { + return false, err + } + if l.meta, err = decodeMeta(b); err != nil { + return false, err + } + if len(d.segments) == 0 { + if d.checkpoint { + return false, fmt.Errorf("%w: no segment", ErrCorrupt) + } + // meta was written but the first segment was not: finish the creation. + return true, l.createSegment(1) + } + + cp := checkpoint{state: chainlog.NewState(opts.WindowCap)} + if d.checkpoint { + b, err := l.readFile(checkpointName, maxCheckpointSize) + if err != nil { + return false, err + } + if cp, err = decodeCheckpoint(b, opts.WindowCap); err != nil { + return false, err + } + l.checkpointAt = cp.x + } + cut, err := l.openSegments(d.segments) + if err != nil { + return false, err + } + + f0, h := l.segs[0].first, l.segs[len(l.segs)-1].last + if d.checkpoint { + if cp.x < f0-1 || cp.x > h { + return false, fmt.Errorf("%w: checkpoint at %d outside [%d, %d]", ErrCorrupt, cp.x, f0-1, h) + } + } else if f0 != 1 { + return false, fmt.Errorf("%w: first segment starts at %d without a checkpoint", ErrCorrupt, f0) + } + if cut != nil { + if err := cut(); err != nil { + return false, err + } + } + l.state, l.lastFin = cp.state, cp.l + if err := l.replay(cp.x+1, h); err != nil { + return false, err + } + l.head, l.oldest = h, f0 + return false, nil +} + +// create writes a new log: meta, then the first segment. +func (l *Log) create(m chainlog.Meta) error { + if err := m.Filter.Validate(); err != nil { + return fmt.Errorf("wal: invalid options: meta: %w", err) + } + if m.ID.IsZero() { + _, _ = rand.Read(m.ID[:]) // never fails since Go 1.24 + } + b := encodeMeta(m) + l.meta, _ = decodeMeta(b) // a normalized deep copy; b is valid + if err := writeFileAtomic(l.dir, metaName, b, l.fsync); err != nil { + return err + } + return l.createSegment(1) +} + +// createSegment creates the empty segment for offset first and makes it the active one. +func (l *Log) createSegment(first chainlog.Offset) error { + name := segmentName(first) + path := filepath.Join(l.dir, name) + f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE|os.O_EXCL|os.O_APPEND, fileMode) //nolint:gosec // a segment name the WAL makes + if err != nil { + return ioErr("open", path, err) + } + if err := syncDir(l.dir, l.fsync); err != nil { + _ = f.Close() + return err + } + l.mu.Lock() + l.segs = append(l.segs, &segment{name: name, first: first, last: first - 1, f: f}) + l.mu.Unlock() + return nil +} + +// openSegments opens and scans the segments, in order, and checks that they fit together. A torn +// tail of the last segment is not cut yet: it returns the cut, which load runs only once the +// checkpoint agrees with the segments, so a failed Open changes no file. +func (l *Log) openSegments(names []string) (cut func() error, err error) { + for i, name := range names { + first, _ := parseSegmentName(name) + if i > 0 { + if prev := l.segs[i-1].last; first != prev+1 { + return nil, fmt.Errorf("%w: segment %s does not follow offset %d", ErrCorrupt, name, prev) + } + } else if first == 0 { + return nil, fmt.Errorf("%w: segment %s does not follow offset 0", ErrCorrupt, name) + } + active := i == len(names)-1 + flag := os.O_RDONLY + if active { + flag = os.O_RDWR | os.O_APPEND + } + path := filepath.Join(l.dir, name) + f, err := os.OpenFile(path, flag, 0) //nolint:gosec // a segment name the WAL owns + if err != nil { + return nil, ioErr("open", path, err) + } + s := &segment{name: name, first: first, f: f} + l.segs = append(l.segs, s) + sc, err := scanSegment(f, first) + if err != nil { + return nil, err + } + if sc.bad != nil { + torn := false + if active { + if torn, err = tornTail(f, sc); err != nil { + return nil, err + } + } + if !torn { + return nil, fmt.Errorf("%w: segment %s position %d: %w", ErrCorrupt, name, sc.bad.pos, sc.bad.err) + } + cut = func() error { return l.cutTail(f, name, sc) } + } + if !active && sc.last < sc.first { + return nil, fmt.Errorf("%w: segment %s is empty", ErrCorrupt, name) + } + s.last, s.size, s.index = sc.last, sc.end, sc.index + } + return cut, nil +} + +// replay applies the entries from..to to l.state, as they follow the checkpoint, and records +// the offset of each Finalize. +func (l *Log) replay(from, to chainlog.Offset) error { + for _, s := range l.segs { + if from > to { + return nil + } + if s.last < from { + continue + } + end := min(to, s.last) + err := s.readRange(from, end, func(e chainlog.Entry) { + l.state.Apply(e) + if e.Kind == chainlog.KindFinalize { + l.lastFin = e.Offset + } + if l.hooks.replay != nil { + l.hooks.replay(e.Offset) + } + }) + if err != nil { + return err + } + from = end + 1 + } + return nil +} + +// readFile reads the meta or checkpoint file name, at most the 8-byte header, a body of maxSize +// bytes and one byte more: enough for the decoder to report a file that is too long. +func (l *Log) readFile(name string, maxSize uint32) ([]byte, error) { + path := filepath.Join(l.dir, name) + f, err := os.Open(path) //nolint:gosec // a file name the WAL owns + if err != nil { + return nil, ioErr("open", path, err) + } + defer f.Close() + fi, err := f.Stat() + if err != nil { + return nil, ioErr("read", path, err) + } + b := make([]byte, min(fi.Size(), fileHeader+int64(maxSize)+1)) + n, err := io.ReadFull(f, b) + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) { + return nil, ioErr("read", path, err) + } + return b[:n], nil +} + +// tornTail reports whether the bad record that ends sc's scan of the last segment f is the tail +// of an interrupted write rather than corruption: fewer than 8 bytes remain, the record's +// declared end is at or past the end of the file, or every byte from it on is zero. A bad record +// whose declared end lies inside the file and is followed by more data is corruption, so the WAL +// does not drop the records after it. (A damaged length field that points past the end of the +// file looks like a torn write; the checkpoint check that runs before the cut catches the case +// where that would drop records the checkpoint covers.) +func tornTail(f *os.File, sc segScan) (bool, error) { + p := sc.bad.pos + if sc.bad.declaredEnd < 0 || sc.bad.declaredEnd >= sc.size { + return true, nil + } + buf := make([]byte, readBuffer) + for p < sc.size { + n, err := f.ReadAt(buf[:min(int64(len(buf)), sc.size-p)], p) + if err != nil { + return false, ioErr("read", f.Name(), err) + } + for _, b := range buf[:n] { + if b != 0 { + return false, nil + } + } + p += int64(n) + } + return true, nil +} + +// cutTail truncates the last segment f to the end of its last whole record. +func (l *Log) cutTail(f *os.File, name string, sc segScan) error { + if err := f.Truncate(sc.bad.pos); err != nil { + return ioErr("truncate", f.Name(), err) + } + // Logged before the fsync: the bytes are gone even if the fsync fails. + l.logger.Warn("wal torn tail cut", + slog.String("segment", name), + slog.Int64("position", sc.bad.pos), + slog.Int64("bytes", sc.size-sc.bad.pos), + ) + if l.fsync != nil { + if err := l.fsync(f); err != nil { + return ioErr("sync", f.Name(), err) + } + } + return nil +} diff --git a/chainlog/wal/recover_test.go b/chainlog/wal/recover_test.go new file mode 100644 index 0000000..c540800 --- /dev/null +++ b/chainlog/wal/recover_test.go @@ -0,0 +1,347 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "bytes" + "errors" + "log/slog" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + + "github.com/hmdsefi/evmsync/chainlog" +) + +// recoveryOpts are testOpts with the default window cap, so a test can append many blocks without +// finalizing. +func recoveryOpts(segmentSize int64) Options { + o := testOpts(segmentSize) + o.WindowCap = 0 + return o +} + +// recordEnds returns the end of each record of es, numbered from first, in a segment that starts +// with first. +func recordEnds(es []chainlog.Entry, first chainlog.Offset) []int64 { + var ends []int64 + var pos int64 + for i, e := range es { + e.Offset = first + chainlog.Offset(i) //nolint:gosec // i < len(es) + pos += int64(len(chainlog.Encode(nil, e))) + ends = append(ends, pos) + } + return ends +} + +// encoded returns the records of es numbered from 1, concatenated. +func encoded(es []chainlog.Entry) []byte { + var b []byte + for i, e := range es { + e.Offset = chainlog.Offset(i + 1) //nolint:gosec // i < len(es) + b = chainlog.Encode(b, e) + } + return b +} + +// asRead returns the records of es with their own offsets, concatenated: what Read returned. +func asRead(es []chainlog.Entry) []byte { + var b []byte + for _, e := range es { + b = chainlog.Encode(b, e) + } + return b +} + +// thirtyBlocks writes 30 linked blocks with one log each, one per Append, in 1 KiB segments and +// returns the directory and the entries. +func thirtyBlocks(t *testing.T) (string, []chainlog.Entry) { + t.Helper() + dir := t.TempDir() + es := linkedBlocks(1, 30, 40) + l, err := Open(dir, recoveryOpts(1024)) + if err != nil { + t.Fatalf("Open: %v", err) + } + for _, e := range es { + mustAppend(t, l, e) + } + mustClose(t, l) + return dir, es +} + +// reopen opens dir with a recording logger and returns the log, its records and the open error. +func reopen(t *testing.T, dir string) (*Log, *recordHandler, error) { + t.Helper() + logger, rec := newRecorder() + opts := recoveryOpts(1024) + opts.Logger = logger + l, err := Open(dir, opts) + return l, rec, err +} + +// wantEntries checks that l holds exactly the first n of es, byte for byte. +func wantEntries(t *testing.T, l *Log, es []chainlog.Entry, n int) { + t.Helper() + if h, _ := l.Head(t.Context()); h != chainlog.Offset(n) { //nolint:gosec // n <= 30 + t.Fatalf("Head = %d, want %d", h, n) + } + got, err := l.Read(t.Context(), 1, 100) + if err != nil { + t.Fatalf("Read: %v", err) + } + if !bytes.Equal(asRead(got), encoded(es[:n])) { + t.Errorf("entries 1..%d differ from the originals", n) + } +} + +func TestRecoverEveryCut(t *testing.T) { + base, es := thirtyBlocks(t) + names := segmentNames(t, base) + last := names[len(names)-1] + lastFirst, _ := parseSegmentName(last) + ends := recordEnds(es[lastFirst-1:], lastFirst) + size := listing(t, base)[last] + if size != ends[len(ends)-1] { + t.Fatalf("test setup: last segment holds %d bytes, want %d", size, ends[len(ends)-1]) + } + + for c := int64(0); c <= size; c++ { + dir := copyDir(t, base) + path := filepath.Join(dir, last) + if err := os.Truncate(path, c); err != nil { + t.Fatal(err) + } + l, rec, err := reopen(t, dir) + if err != nil { + t.Fatalf("cut at %d: Open: %v", c, err) + } + kept, keep := 0, int64(0) + for kept < len(ends) && ends[kept] <= c { + keep = ends[kept] + kept++ + } + wantEntries(t, l, es, index(lastFirst)-1+kept) + + attrs, warned := rec.find(slog.LevelWarn, "wal torn tail cut") + if c == keep { + if warned { + t.Errorf("cut at %d, a record boundary: warning %v, want none", c, attrs) + } + } else { + wantSame(t, "cut at "+strconv.FormatInt(c, 10)+": warning", attrs, + map[string]any{"segment": last, "position": keep, "bytes": c - keep}) + if got := listing(t, dir)[last]; got != keep { + t.Errorf("cut at %d: segment holds %d bytes after Open, want %d", c, got, keep) + } + } + head, _ := l.Head(t.Context()) + if off := mustAppend(t, l, linkedBlocks(uint64(head)+1, 1, 40)...); off != head+1 { + t.Errorf("cut at %d: Append returned %d, want %d", c, off, head+1) + } + mustClose(t, l) + } +} + +func TestTornVersusCorrupt(t *testing.T) { + base, es := thirtyBlocks(t) + names := segmentNames(t, base) + last := names[len(names)-1] + lastFirst, _ := parseSegmentName(last) + ends := recordEnds(es[lastFirst-1:], lastFirst) + if len(ends) < 2 { + t.Fatalf("test setup: last segment holds %d records, want at least 2", len(ends)) + } + lastStart := ends[len(ends)-2] // start of the last record + prevStart := int64(0) // start of the second-to-last record + if len(ends) > 2 { + prevStart = ends[len(ends)-3] + } + wrongOffset := es[29] + wrongOffset.Offset = 99 + reencoded := chainlog.Encode(nil, wrongOffset) + f1, _ := parseSegmentName(names[1]) + sealedEnds := recordEnds(es[:f1-1], 1) + sealedLast := int64(0) // start of the last record of the first segment + if len(sealedEnds) > 1 { + sealedLast = sealedEnds[len(sealedEnds)-2] + } + + tests := []struct { + name string + damage func(dir string) + head int // the Head Open recovers to, for a torn tail + corrupt string // the error prefix after "wal: corrupt log: ", for corruption + }{ + {"zeros after the last record", func(dir string) { + rewrite(t, dir, last, func(b []byte) []byte { return append(b, make([]byte, 4096)...) }) + }, 30, ""}, + {"flipped body byte in the last record", func(dir string) { + rewrite(t, dir, last, func(b []byte) []byte { b[lastStart+20] ^= 1; return b }) + }, 29, ""}, + {"wrong offset in the last record", func(dir string) { + rewrite(t, dir, last, func(b []byte) []byte { return append(b[:lastStart], reencoded...) }) + }, 29, ""}, + {"flipped body byte in the second-to-last record", func(dir string) { + rewrite(t, dir, last, func(b []byte) []byte { b[prevStart+20] ^= 1; return b }) + }, 0, "segment " + last + " position " + strconv.FormatInt(prevStart, 10) + ": "}, + {"a non-zero first byte in trailing bytes", func(dir string) { + rewrite(t, dir, last, func(b []byte) []byte { + tail := make([]byte, 4096) + tail[0] = 1 + return append(b, tail...) + }) + }, 0, "segment " + last + " position " + strconv.FormatInt(ends[len(ends)-1], 10) + ": "}, + {"flipped body byte in the last record of a sealed segment", func(dir string) { + rewrite(t, dir, names[0], func(b []byte) []byte { b[sealedLast+20] ^= 1; return b }) + }, 0, "segment " + names[0] + " position " + strconv.FormatInt(sealedLast, 10) + ": "}, + {"zeros, then one byte past the first 64 KiB", func(dir string) { + rewrite(t, dir, last, func(b []byte) []byte { + tail := make([]byte, 70<<10) + tail[66_000] = 1 + return append(b, tail...) + }) + }, 0, "segment " + last + " position " + strconv.FormatInt(ends[len(ends)-1], 10) + ": "}, + {"a zero header, then a record", func(dir string) { + rewrite(t, dir, last, func(b []byte) []byte { return append(append(b, make([]byte, 8)...), reencoded...) }) + }, 0, "segment " + last + " position " + strconv.FormatInt(ends[len(ends)-1], 10) + ": "}, + {"a damaged last record, then one byte", func(dir string) { + rewrite(t, dir, last, func(b []byte) []byte { b[lastStart+20] ^= 1; return append(b, 0x01) }) + }, 0, "segment " + last + " position " + strconv.FormatInt(lastStart, 10) + ": "}, + {"a cut below the checkpoint", func(dir string) { + // The checkpoint covers every record; a length flip in the active segment's first + // record would cut them. Open must refuse before it cuts. + b, _ := os.ReadFile(filepath.Join(dir, checkpointName)) + cp, _ := decodeCheckpoint(b, 0) + cp.x = 30 + rewrite(t, dir, checkpointName, func([]byte) []byte { return encodeCheckpoint(cp) }) + rewrite(t, dir, last, func(b []byte) []byte { b[3] ^= 0x80; return b }) + }, 0, "checkpoint at 30 outside ["}, + {"flipped byte in the first segment", func(dir string) { + rewrite(t, dir, names[0], func(b []byte) []byte { b[20] ^= 1; return b }) + }, 0, "segment " + names[0] + " position 0: "}, + {"second segment renamed one higher", func(dir string) { + _ = os.Rename(filepath.Join(dir, names[1]), filepath.Join(dir, segmentName(f1+1))) + }, 0, "segment " + segmentName(f1+1) + " does not follow offset"}, + {"empty sealed segment", func(dir string) { + _ = os.Truncate(filepath.Join(dir, names[1]), 0) + }, 0, "segment " + names[1] + " is empty"}, + {"no segment", func(dir string) { + for _, n := range names { + _ = os.Remove(filepath.Join(dir, n)) + } + }, 0, "no segment"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := copyDir(t, base) + tt.damage(dir) + before := listing(t, dir) + l, rec, err := reopen(t, dir) + if tt.corrupt != "" { + if err == nil { + _ = l.Close() + t.Fatal("Open succeeded, want ErrCorrupt") + } + if !errors.Is(err, ErrCorrupt) || !strings.HasPrefix(err.Error(), "wal: corrupt log: "+tt.corrupt) { + t.Errorf("Open = %v, want ErrCorrupt with the prefix %q", err, tt.corrupt) + } + wantSame(t, "files after a failed Open", listing(t, dir), before) + return + } + if err != nil { + t.Fatalf("Open: %v", err) + } + defer l.Close() + wantEntries(t, l, es, tt.head) + if _, warned := rec.find(slog.LevelWarn, "wal torn tail cut"); !warned { + t.Error("no torn tail warning") + } + if got, want := listing(t, dir)[last], ends[tt.head-index(lastFirst)]; got != want { + t.Errorf("last segment holds %d bytes after Open, want %d", got, want) + } + }) + } +} + +// TestRecoverSync cuts a torn tail with SyncAlways: Open fsyncs the cut segment after the cut, +// and a failed fsync fails Open with the I/O error. +func TestRecoverSync(t *testing.T) { + for _, fail := range []bool{false, true} { + t.Run("fsync fails "+strconv.FormatBool(fail), func(t *testing.T) { + dir, _ := thirtyBlocks(t) + names := segmentNames(t, dir) + last := names[len(names)-1] + path := filepath.Join(dir, last) + fi, _ := os.Stat(path) + cutAt := fi.Size() + rewrite(t, dir, last, func(b []byte) []byte { return append(b, make([]byte, 100)...) }) + synced := false + h := hooks{sync: func(name string) error { + if name != path { + return nil + } + synced = true + if fi, err := os.Stat(path); err != nil || fi.Size() != cutAt { + t.Errorf("fsync of the segment before the cut: %d bytes, want %d", fi.Size(), cutAt) + } + if fail { + return syscall.EIO + } + return nil + }} + opts := recoveryOpts(1024) + opts.Sync = SyncAlways + l, err := openWithHooks(dir, opts, h) + if fail { + if err == nil { + _ = l.Close() + } + if !errors.Is(err, syscall.EIO) || errors.Is(err, ErrCorrupt) { + t.Errorf("Open with a failing fsync = %v, want EIO and not ErrCorrupt", err) + } + return + } + if err != nil { + t.Fatalf("Open: %v", err) + } + defer l.Close() + if !synced { + t.Error("Open did not fsync the cut segment") + } + if hd, _ := l.Head(t.Context()); hd != 30 { + t.Errorf("Head = %d, want 30", hd) + } + }) + } +} + +// TestRecoverOneSegment cuts a torn tail in a log that never rolled. +func TestRecoverOneSegment(t *testing.T) { + dir := t.TempDir() + l, err := Open(dir, recoveryOpts(1024)) + if err != nil { + t.Fatal(err) + } + es := linkedBlocks(1, 2, 40) + for _, e := range es { + mustAppend(t, l, e) + } + mustClose(t, l) + if err := os.Truncate(filepath.Join(dir, firstSegment), 300); err != nil { + t.Fatal(err) + } + l, rec, err := reopen(t, dir) + if err != nil { + t.Fatalf("Open: %v", err) + } + defer l.Close() + wantEntries(t, l, es, 1) + if _, warned := rec.find(slog.LevelWarn, "wal torn tail cut"); !warned { + t.Error("no torn tail warning") + } +} diff --git a/chainlog/wal/segment.go b/chainlog/wal/segment.go new file mode 100644 index 0000000..5cb05b3 --- /dev/null +++ b/chainlog/wal/segment.go @@ -0,0 +1,152 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "bufio" + "encoding/binary" + "errors" + "fmt" + "io" + "os" + "strconv" + "strings" + + "github.com/hmdsefi/evmsync/chainlog" +) + +const ( + segSuffix = ".seg" + nameDigits = 20 + // indexEvery is the gap, in records, between two indexed record positions of a segment. + indexEvery = 256 + readBuffer = 64 << 10 + recHeader = 8 // a record's length:uint32le crc:uint32le +) + +// segment is one segment file: the records of offsets first..last. +type segment struct { + name string + first chainlog.Offset + last chainlog.Offset // first-1 for an empty segment + size int64 // bytes of whole records, the end of the last one + index []int64 // file position of the records first, first+256, first+512, ... + f *os.File // read-only for sealed segments; read-write, appending, for the active one +} + +// segmentName returns the file name of the segment whose first record has offset first. +func segmentName(first chainlog.Offset) string { + s := strconv.FormatUint(uint64(first), 10) + return strings.Repeat("0", nameDigits-len(s)) + s + segSuffix +} + +// parseSegmentName returns the first offset that name gives, or false when name is not a +// segment's: 20 decimal digits and ".seg". +func parseSegmentName(name string) (chainlog.Offset, bool) { + digits, ok := strings.CutSuffix(name, segSuffix) + if !ok || len(digits) != nameDigits { + return 0, false + } + for _, c := range digits { + if c < '0' || c > '9' { + return 0, false + } + } + n, err := strconv.ParseUint(digits, 10, 64) + if err != nil { + return 0, false + } + return chainlog.Offset(n), true +} + +// readRecord reads the next record from r into *buf and returns it. remaining is the number of +// bytes left in the input; it bounds the allocation for a record whose declared length is larger. +// When the input ends inside the record it returns the bytes it has and no error, so +// chainlog.Decode reports the record as incomplete. Only other read errors are returned. +func readRecord(r *bufio.Reader, buf *[]byte, remaining int64) ([]byte, error) { + b := (*buf)[:0] + b = append(b, make([]byte, recHeader)...) + n, err := io.ReadFull(r, b) + if err != nil { + return b[:n], eofIsNil(err) + } + length := int64(binary.LittleEndian.Uint32(b)) + if length > chainlog.MaxRecordSize { + return b, nil // Decode rejects the length before it reads the body + } + b = append(b, make([]byte, min(length, max(remaining-recHeader, 0)))...) + n, err = io.ReadFull(r, b[recHeader:]) + *buf = b + if err != nil { + return b[:recHeader+n], eofIsNil(err) + } + return b, nil +} + +// decodeAt decodes rec, which must hold the entry at offset want, and calls fn with it when fn is +// not nil. It returns the record's length, or the Decode error or "offset %d, want %d". +func decodeAt(rec []byte, want chainlog.Offset, fn func(chainlog.Entry)) (int, error) { + e, n, err := chainlog.Decode(rec) + if err != nil { + return 0, err + } + if e.Offset != want { + return 0, fmt.Errorf("offset %d, want %d", e.Offset, want) + } + if fn != nil { + fn(e) + } + return n, nil +} + +func eofIsNil(err error) error { + if errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) { + return nil + } + return err +} + +// readRange calls fn with each entry of s from offset from to offset to, both within s, in +// order. It starts at the nearest indexed record at or below from and skips the records before +// from by their headers, then decodes each record, which checks its CRC. It reads with ReadAt, so +// it is safe while the active segment grows. A record that fails to decode, or has the wrong +// offset, is ErrCorrupt; other errors are I/O errors. +func (s segment) readRange(from, to chainlog.Offset, fn func(chainlog.Entry)) error { + k := (from - s.first) / indexEvery + pos := s.index[k] + r := bufio.NewReaderSize(io.NewSectionReader(s.f, pos, s.size-pos), int(min(readBuffer, max(s.size-pos, 16)))) + for off := s.first + k*indexEvery; off < from; off++ { + hdr, err := r.Peek(recHeader) + if err != nil { + return s.corrupt(pos, eofIsNil(err), chainlog.ErrIncomplete) + } + n := recHeader + int64(binary.LittleEndian.Uint32(hdr)) + if _, err := r.Discard(int(min(n, s.size-pos))); err != nil { + return s.corrupt(pos, eofIsNil(err), chainlog.ErrIncomplete) + } + pos += n + } + var buf []byte + for off := from; off <= to; off++ { + rec, err := readRecord(r, &buf, s.size-pos) + if err != nil { + return ioErr("read", s.f.Name(), err) + } + n, err := decodeAt(rec, off, fn) + if err != nil { + return fmt.Errorf("%w: segment %s position %d: %w", ErrCorrupt, s.name, pos, err) + } + pos += int64(n) + } + return nil +} + +// corrupt returns the error for a record at pos that ends before its header says: an I/O error +// when ioErr is not nil, and otherwise ErrCorrupt wrapping cause. +func (s segment) corrupt(pos int64, readErr, cause error) error { + if readErr != nil { + return ioErr("read", s.f.Name(), readErr) + } + return fmt.Errorf("%w: segment %s position %d: %w", ErrCorrupt, s.name, pos, cause) +} diff --git a/chainlog/wal/slog_test.go b/chainlog/wal/slog_test.go new file mode 100644 index 0000000..61f60fe --- /dev/null +++ b/chainlog/wal/slog_test.go @@ -0,0 +1,53 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "context" + "log/slog" + "sync" +) + +// recordHandler keeps every record logged through it. +type recordHandler struct { + mu sync.Mutex + records []slog.Record +} + +func newRecorder() (*slog.Logger, *recordHandler) { + h := &recordHandler{} + return slog.New(h), h +} + +func (h *recordHandler) Enabled(context.Context, slog.Level) bool { return true } + +func (h *recordHandler) Handle(_ context.Context, r slog.Record) error { + h.mu.Lock() + defer h.mu.Unlock() + h.records = append(h.records, r.Clone()) + return nil +} + +func (h *recordHandler) WithAttrs([]slog.Attr) slog.Handler { return h } + +func (h *recordHandler) WithGroup(string) slog.Handler { return h } + +// find returns the attributes of the last record with level and msg. +func (h *recordHandler) find(level slog.Level, msg string) (map[string]any, bool) { + h.mu.Lock() + defer h.mu.Unlock() + for i := len(h.records) - 1; i >= 0; i-- { + r := h.records[i] + if r.Level != level || r.Message != msg { + continue + } + attrs := make(map[string]any) + r.Attrs(func(a slog.Attr) bool { + attrs[a.Key] = a.Value.Any() + return true + }) + return attrs, true + } + return nil, false +} diff --git a/chainlog/wal/wal.go b/chainlog/wal/wal.go new file mode 100644 index 0000000..cb49be5 --- /dev/null +++ b/chainlog/wal/wal.go @@ -0,0 +1,460 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "context" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "sort" + "strconv" + "sync" + + "github.com/hmdsefi/evmsync/chainlog" +) + +// SyncPolicy selects when the WAL calls fsync. +type SyncPolicy uint8 + +// The sync policies. +const ( + SyncAlways SyncPolicy = iota // fsync before Append returns (default) + SyncNever // never fsync; for tests only +) + +// Options configures Open. The zero value is valid. +type Options struct { + Meta chainlog.Meta // used only when the directory is new + SegmentSize int64 // roll threshold in bytes; 0 means 64 MiB + Sync SyncPolicy // default SyncAlways + WindowCap int // 0 means chainlog.DefaultWindowCap (100,000); at most MaxWindowCap + Logger *slog.Logger // nil discards +} + +// DefaultSegmentSize is the SegmentSize used when Options.SegmentSize is 0. +const DefaultSegmentSize = 64 << 20 + +// MaxWindowCap is the largest Options.WindowCap. The checkpoint holds the window, and a window of +// this many refs still fits the checkpoint's 1 GiB limit. +const MaxWindowCap = 10_000_000 + +const lockName = "lock" + +// Log is a chainlog.Log backed by files in one directory: a lock file, the metadata, segment +// files of records and a checkpoint of the validator state. It is safe for concurrent use. Only +// one Log may have a directory open at a time, also across processes. +type Log struct { + dir string + fsync syncer // nil with SyncNever + segSize int64 + logger *slog.Logger + hooks hooks + lock *os.File + meta chainlog.Meta // normalized; never changes + + // appendMu serializes Append, Truncate and Close. Their writes to the fields below happen + // with mu held as well, so a holder of appendMu may read them without mu. + appendMu sync.Mutex + buf []byte // Append's encode buffer; guarded by appendMu + undelete []string // dropped segment files a failed deletion left, oldest first; guarded by appendMu + starts []int64 // the position of each record in buf; guarded by appendMu + + mu sync.RWMutex + segs []*segment // in offset order; the last is active; guarded by mu + head chainlog.Offset // guarded by mu + oldest chainlog.Offset // guarded by mu + lastFin chainlog.Offset // offset of the newest Finalize entry, 0 if none; guarded by mu + state chainlog.State // guarded by mu + checkpointAt chainlog.Offset // X of the checkpoint Open loaded, 0 without one + failed error // the first write or fsync error; guarded by mu + closed bool // guarded by mu + wake chan struct{} // closed and replaced by every Append and by Close; guarded by mu +} + +var _ chainlog.Log = (*Log)(nil) + +// Open opens the WAL in dir, creating dir and a new log if dir holds none. It takes the +// directory's lock, recovers a torn tail and rebuilds the State. +// +// A new log gets opts.Meta, with a random ID when its ID is zero; an existing log keeps its own +// metadata and ignores opts.Meta. Open fails with an error wrapping ErrLocked when another Log +// holds dir, and ErrCorrupt when the files do not form a log the WAL wrote. On systems other than +// Unix it fails with an error wrapping errors.ErrUnsupported. +func Open(dir string, opts Options) (*Log, error) { + return openWithHooks(dir, opts, hooks{}) +} + +func openWithHooks(dir string, opts Options, h hooks) (*Log, error) { + switch { + case opts.SegmentSize < 0: + return nil, fmt.Errorf("wal: invalid options: segment size %d is negative", opts.SegmentSize) + case opts.Sync > SyncNever: + return nil, fmt.Errorf("wal: invalid options: unknown sync policy %d", opts.Sync) + case opts.WindowCap < 0: + return nil, fmt.Errorf("wal: invalid options: window cap %d is negative", opts.WindowCap) + case opts.WindowCap > MaxWindowCap: + return nil, fmt.Errorf("wal: invalid options: window cap %d is above %d", opts.WindowCap, MaxWindowCap) + } + l := &Log{ + dir: dir, + segSize: opts.SegmentSize, + logger: opts.Logger, + hooks: h, + wake: make(chan struct{}), + } + if l.segSize == 0 { + l.segSize = DefaultSegmentSize + } + if opts.Sync == SyncAlways { + l.fsync = func(f *os.File) error { + if h.sync != nil { + if err := h.sync(f.Name()); err != nil { + return err + } + } + return f.Sync() + } + } + if l.logger == nil { + l.logger = slog.New(slog.DiscardHandler) + } + if err := os.MkdirAll(dir, dirMode); err != nil { //nolint:gosec // dir is the directory the caller chose for the log + return nil, ioErr("open", dir, err) + } + lock, err := lockFile(filepath.Join(dir, lockName)) + if errors.Is(err, errWouldBlock) { + return nil, fmt.Errorf("%w: %s", ErrLocked, dir) + } + if err != nil { + return nil, err + } + l.lock = lock + created, err := l.load(opts) + if err != nil { + _ = l.closeFiles() + return nil, err + } + l.logger.Info("wal opened", + slog.String("dir", dir), + slog.Int("segments", len(l.segs)), + slog.Uint64("oldest", uint64(l.oldest)), + slog.Uint64("head", uint64(l.head)), + slog.Uint64("checkpoint", uint64(l.checkpointAt)), + slog.Bool("new", created), + ) + return l, nil +} + +// check returns ErrClosed or ctx.Err(), in that order. The caller holds mu. +func (l *Log) check(ctx context.Context) error { + if l.closed { + return chainlog.ErrClosed + } + return ctx.Err() +} + +// Meta returns the log's metadata. +func (l *Log) Meta(ctx context.Context) (chainlog.Meta, error) { + l.mu.RLock() + defer l.mu.RUnlock() + if err := l.check(ctx); err != nil { + return chainlog.Meta{}, err + } + m, _ := decodeMeta(encodeMeta(l.meta)) // a deep copy + return m, nil +} + +// Append validates entries in order against the log's State and appends all of them, or none +// with an error wrapping chainlog.ErrMalformed. It assigns offsets Head+1, Head+2, ... and +// returns the last one; with no entries it returns Head. With SyncAlways the entries are durable +// when it returns. After a write or fsync fails, it and every later Append and Truncate return +// an error wrapping ErrFailed. +func (l *Log) Append(ctx context.Context, entries ...chainlog.Entry) (chainlog.Offset, error) { + l.appendMu.Lock() + defer l.appendMu.Unlock() + + // CheckBatch changes the state while it runs, so it needs the write lock. + l.mu.Lock() + if err := l.check(ctx); err != nil { + l.mu.Unlock() + return 0, err + } + if l.failed != nil { + l.mu.Unlock() + return 0, fmt.Errorf("%w: earlier error: %w", ErrFailed, l.failed) + } + head := l.head + if len(entries) == 0 { + l.mu.Unlock() + return head, nil + } + i, err := l.state.CheckBatch(entries) + l.mu.Unlock() + if err != nil { + return 0, fmt.Errorf("chainlog: append entry %d of %d: %w", i+1, len(entries), err) + } + + l.buf, l.starts = l.buf[:0], l.starts[:0] + for i, e := range entries { + e.Offset = head + 1 + chainlog.Offset(i) //nolint:gosec // i < len(entries) + l.starts = append(l.starts, int64(len(l.buf))) + l.buf = chainlog.Encode(l.buf, e) + } + if err := l.write(head, l.buf); err != nil { + l.mu.Lock() + l.failed = err + l.mu.Unlock() + return 0, fmt.Errorf("%w: append: %w", ErrFailed, err) + } + return l.publish(head, entries), nil +} + +// publish makes the batch written after offset head visible: it applies the entries to the +// state, indexes them, moves Head and wakes every Wait. The caller holds appendMu. +func (l *Log) publish(head chainlog.Offset, entries []chainlog.Entry) chainlog.Offset { + l.mu.Lock() + defer l.mu.Unlock() + s := l.segs[len(l.segs)-1] + for i, e := range entries { + e.Offset = head + 1 + chainlog.Offset(i) //nolint:gosec // i < len(entries) + l.state.Apply(e) + if e.Kind == chainlog.KindFinalize { + l.lastFin = e.Offset + } + if (e.Offset-s.first)%indexEvery == 0 { + s.index = append(s.index, s.size+l.starts[i]) + } + } + s.size += int64(len(l.buf)) + s.last = head + chainlog.Offset(len(entries)) + l.head = s.last + close(l.wake) + l.wake = make(chan struct{}) + return l.head +} + +// write writes the encoded batch b after offset head to the active segment, rolling to a new +// segment first when b does not fit, and fsyncs it with SyncAlways. The caller holds appendMu. +func (l *Log) write(head chainlog.Offset, b []byte) error { + if s := l.segs[len(l.segs)-1]; s.size > 0 && s.size+int64(len(b)) > l.segSize { + if err := l.roll(head); err != nil { + return err + } + } + s := l.segs[len(l.segs)-1] + if _, err := l.hooks.writeTo(s.f, b); err != nil { + return ioErr("write", s.f.Name(), err) + } + if l.fsync != nil { + if err := l.fsync(s.f); err != nil { + return ioErr("sync", s.f.Name(), err) + } + } + return nil +} + +// roll seals the active segment: it writes a checkpoint of the state after head, then creates +// the segment for head+1. The caller holds appendMu. +func (l *Log) roll(head chainlog.Offset) error { + cp := encodeCheckpoint(checkpoint{x: head, l: l.lastFin, state: l.state}) + if err := writeFileAtomic(l.dir, checkpointName, cp, l.fsync); err != nil { + return err + } + if err := l.createSegment(head + 1); err != nil { + return err + } + l.logger.Debug("wal segment rolled", slog.String("segment", segmentName(head+1)), slog.Uint64("offset", uint64(head+1))) + return nil +} + +// Read returns up to max entries starting at offset from, in offset order. It returns nil when +// from is Head+1, an error wrapping chainlog.ErrTruncated when from is below Oldest, and an error +// wrapping chainlog.ErrAhead when from is above Head+1. A from of 0 reads from 1. It panics if +// max < 1. A record that fails its checksum is an error wrapping ErrCorrupt. +func (l *Log) Read(ctx context.Context, from chainlog.Offset, max int) ([]chainlog.Entry, error) { + if from == 0 { + from = 1 + } + segs, to, err := l.span(ctx, from, max) + if len(segs) == 0 { + return nil, err + } + es := make([]chainlog.Entry, 0, to-from+1) + for _, s := range segs { + start := from + if s.first > start { + start = s.first + } + err := s.readRange(start, min(to, s.last), func(e chainlog.Entry) { es = append(es, e) }) + if err != nil { + return nil, l.readError(from, err) + } + } + return es, nil +} + +// span checks a Read from offset from, at least 1, and returns copies of the segments that hold +// from..to, which Read then reads without the lock. It returns no segment when there is nothing to +// read. +func (l *Log) span(ctx context.Context, from chainlog.Offset, max int) ([]segment, chainlog.Offset, error) { + l.mu.RLock() + defer l.mu.RUnlock() + if err := l.check(ctx); err != nil { + return nil, 0, err + } + if max < 1 { + panic("evmsync: chainlog: Read max must be at least 1, got " + strconv.Itoa(max)) + } + switch { + case from < l.oldest: + return nil, 0, fmt.Errorf("%w: read from %d, oldest is %d", chainlog.ErrTruncated, from, l.oldest) + case from == l.head+1: + return nil, 0, nil + case from > l.head+1: + return nil, 0, fmt.Errorf("%w: read from %d, head is %d", chainlog.ErrAhead, from, l.head) + } + to := l.head + if n := uint64(to - from); n >= uint64(max) { + to = from + chainlog.Offset(max) - 1 //nolint:gosec // max <= n < head + } + var segs []segment + for i := sort.Search(len(l.segs), func(i int) bool { return l.segs[i].last >= from }); i < len(l.segs) && l.segs[i].first <= to; i++ { + segs = append(segs, *l.segs[i]) + } + return segs, to, nil +} + +// readError returns the error for a Read from offset from that failed with err. A file closed by +// Close or deleted by Truncate under the reader gives ErrClosed or chainlog.ErrTruncated. +func (l *Log) readError(from chainlog.Offset, err error) error { + if errors.Is(err, ErrCorrupt) { + return err + } + l.mu.RLock() + defer l.mu.RUnlock() + if l.closed { + return chainlog.ErrClosed + } + if from < l.oldest { + return fmt.Errorf("%w: read from %d, oldest is %d", chainlog.ErrTruncated, from, l.oldest) + } + return err +} + +// Wait returns nil once Head > after, ctx.Err() if ctx ends first, chainlog.ErrClosed if the log +// closes first, and an error wrapping chainlog.ErrAhead when after > Head. +func (l *Log) Wait(ctx context.Context, after chainlog.Offset) error { + for { + l.mu.RLock() + if err := l.check(ctx); err != nil { + l.mu.RUnlock() + return err + } + head, wake := l.head, l.wake + l.mu.RUnlock() + switch { + case head > after: + return nil + case after > head: + return fmt.Errorf("%w: wait after %d, head is %d", chainlog.ErrAhead, after, head) + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-wake: + } + } +} + +// Head returns the offset of the newest entry, 0 if none was ever appended. +func (l *Log) Head(ctx context.Context) (chainlog.Offset, error) { + l.mu.RLock() + defer l.mu.RUnlock() + if err := l.check(ctx); err != nil { + return 0, err + } + return l.head, nil +} + +// Oldest returns the offset of the first retained entry, Head+1 when none is retained. +func (l *Log) Oldest(ctx context.Context) (chainlog.Offset, error) { + l.mu.RLock() + defer l.mu.RUnlock() + if err := l.check(ctx); err != nil { + return 0, err + } + return l.oldest, nil +} + +// State returns a copy of the validator state after the newest entry. +func (l *Log) State(ctx context.Context) (chainlog.State, error) { + l.mu.RLock() + defer l.mu.RUnlock() + if err := l.check(ctx); err != nil { + return chainlog.State{}, err + } + return l.state.Clone(), nil +} + +// Truncate drops whole segments whose entries are all below min(before, L+1), where L is the +// offset of the newest Finalize entry (0 if none), and deletes their files. It keeps the active +// segment. A Read of a dropped range returns the entries or an error wrapping +// chainlog.ErrTruncated. +func (l *Log) Truncate(ctx context.Context, before chainlog.Offset) error { + l.appendMu.Lock() + defer l.appendMu.Unlock() + l.mu.RLock() + err := l.check(ctx) + if err == nil && l.failed != nil { + err = fmt.Errorf("%w: earlier error: %w", ErrFailed, l.failed) + } + b := min(before, l.lastFin+1) + oldest := l.oldest + l.mu.RUnlock() + if err != nil { + return err + } + // Files a failed deletion left go first: deleting a later segment before them would leave a + // gap that Open rejects. + if err := l.removeSegments(); err != nil { + return err + } + if b <= oldest { + return nil + } + return l.compact(b) +} + +// Close closes every file, which releases the directory's lock, and wakes every Wait with +// chainlog.ErrClosed. It does not fsync: with SyncAlways everything is already durable. Later +// calls of other methods return chainlog.ErrClosed; later calls of Close return nil. +func (l *Log) Close() error { + l.appendMu.Lock() + defer l.appendMu.Unlock() + l.mu.Lock() + defer l.mu.Unlock() + if l.closed { + return nil + } + l.closed = true + close(l.wake) + return l.closeFiles() +} + +// closeFiles closes the segment files and then the lock file. +func (l *Log) closeFiles() error { + var errs []error + for _, s := range l.segs { + if err := s.f.Close(); err != nil { + errs = append(errs, ioErr("close", s.f.Name(), err)) + } + } + if err := l.lock.Close(); err != nil { + errs = append(errs, ioErr("close", l.lock.Name(), err)) + } + return errors.Join(errs...) +} diff --git a/chainlog/wal/wal_test.go b/chainlog/wal/wal_test.go new file mode 100644 index 0000000..c0f6bee --- /dev/null +++ b/chainlog/wal/wal_test.go @@ -0,0 +1,1028 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package wal + +import ( + "bytes" + "errors" + "fmt" + "io/fs" + "log/slog" + "maps" + "os" + "path/filepath" + "runtime" + "slices" + "strconv" + "strings" + "sync" + "syscall" + "testing" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/chainlog/logtest" +) + +const firstSegment = "00000000000000000001.seg" + +func wantCorrupt(t *testing.T, err error, text string) { + t.Helper() + if !errors.Is(err, ErrCorrupt) { + t.Fatalf("Open error = %v, want one wrapping ErrCorrupt", err) + } + if got := err.Error(); got != text { + t.Errorf("Open error =\n%q\nwant\n%q", got, text) + } +} + +func TestOpenOptions(t *testing.T) { + tests := []struct { + name string + opts Options + want string + }{ + {"negative segment size", Options{SegmentSize: -1}, "wal: invalid options: segment size -1 is negative"}, + {"unknown sync policy", Options{Sync: 2}, "wal: invalid options: unknown sync policy 2"}, + {"negative window cap", Options{WindowCap: -1}, "wal: invalid options: window cap -1 is negative"}, + {"window cap above the maximum", Options{WindowCap: MaxWindowCap + 1}, "wal: invalid options: window cap 10000001 is above 10000000"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := filepath.Join(t.TempDir(), "wal") + l, err := Open(dir, tt.opts) + if err == nil { + _ = l.Close() + } + if err == nil || err.Error() != tt.want { + t.Errorf("Open(%+v) = %v, want %q", tt.opts, err, tt.want) + } + if _, err := os.Stat(dir); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("Open with bad options touched the directory: %v", err) + } + }) + } + + dir := filepath.Join(t.TempDir(), "wal") + var five chainlog.Meta + for range 5 { + five.Filter.Topics = append(five.Filter.Topics, nil) + } + _, err := Open(dir, Options{Meta: five}) + if !errors.Is(err, chain.ErrInvalidFilter) { + t.Fatalf("Open with a 5-position filter = %v, want one wrapping chain.ErrInvalidFilter", err) + } + if got, want := err.Error(), "wal: invalid options: meta: chain: invalid filter: 5 topic positions, at most 4"; got != want { + t.Errorf("Open error = %q, want %q", got, want) + } + if got := slices.Sorted(maps.Keys(listing(t, dir))); !slices.Equal(got, []string{"lock"}) { + t.Errorf("directory holds %v, want only lock", got) + } + // The directory is still new: a valid meta creates the log. + l := openLog(t, dir, testOpts(0), hooks{}) + if h, _ := l.Head(t.Context()); h != 0 { + t.Errorf("Head = %d, want 0", h) + } +} + +func TestNewDirectory(t *testing.T) { + dir := filepath.Join(t.TempDir(), "wal") + opts := Options{Sync: SyncAlways} // zero meta: a random ID + l, err := Open(dir, opts) + if err != nil { + t.Fatalf("Open: %v", err) + } + if got := listing(t, dir); !maps.Equal(got, map[string]int64{"lock": 0, "meta": got["meta"], firstSegment: 0}) { + t.Errorf("new directory holds %v, want lock, meta and an empty %s", got, firstSegment) + } + for _, name := range []string{"lock", "meta", firstSegment} { + fi, err := os.Stat(filepath.Join(dir, name)) + if err != nil { + t.Fatalf("Stat: %v", err) + } + if mode := fi.Mode().Perm(); mode != 0o600 { + t.Errorf("%s has mode %v, want 0600", name, mode) + } + } + if fi, _ := os.Stat(dir); fi.Mode().Perm() != 0o750 { + t.Errorf("directory has mode %v, want 0750", fi.Mode().Perm()) + } + m, err := l.Meta(t.Context()) + if err != nil { + t.Fatalf("Meta: %v", err) + } + if m.ID.IsZero() { + t.Error("new log has a zero ID, want a random one") + } + b, _ := os.ReadFile(filepath.Join(dir, "meta")) + if string(b[8:16]) != "EVMSMETA" || b[16] != 0x01 || !bytes.Equal(b[17:33], m.ID[:]) { + t.Errorf("meta starts %x, want the header, EVMSMETA, 01 and the ID", b[:33]) + } + mustClose(t, l) + + // An existing directory keeps its meta and ignores Options.Meta; Open removes stray + // temporary files and keeps files it does not own. + for _, name := range []string{"meta.tmp", "checkpoint.tmp", "notes.txt"} { + if err := os.WriteFile(filepath.Join(dir, name), []byte("x"), 0o600); err != nil { + t.Fatal(err) + } + } + l = openLog(t, dir, testOpts(0), hooks{}) + again, _ := l.Meta(t.Context()) + wantSame(t, "Meta after reopening with another Options.Meta", again, m) + got := listing(t, dir) + if _, ok := got["meta.tmp"]; ok { + t.Error("Open kept meta.tmp") + } + if _, ok := got["checkpoint.tmp"]; ok { + t.Error("Open kept checkpoint.tmp") + } + if _, ok := got["notes.txt"]; !ok { + t.Error("Open removed notes.txt") + } +} + +func TestOpenAfterInterruptedCreate(t *testing.T) { + dir := t.TempDir() + meta := logtest.Meta() + if err := os.WriteFile(filepath.Join(dir, "lock"), nil, 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "meta"), encodeMeta(meta), 0o600); err != nil { + t.Fatal(err) + } + opts := testOpts(0) + opts.Meta.ID = chainlog.ID{0xee} + l := openLog(t, dir, opts, hooks{}) + if h, _ := l.Head(t.Context()); h != 0 { + t.Errorf("Head = %d, want 0", h) + } + m, _ := l.Meta(t.Context()) + wantSame(t, "Meta", m, meta) + if size, ok := listing(t, dir)[firstSegment]; !ok || size != 0 { + t.Errorf("Open did not create an empty %s", firstSegment) + } + mustAppend(t, l, linkedBlocks(1, 2, 8)...) +} + +// newLog creates a log in a new directory with n linked blocks and closes it. +func newLog(t *testing.T, opts Options, n uint64) string { + t.Helper() + dir := t.TempDir() + l, err := Open(dir, opts) + if err != nil { + t.Fatalf("Open: %v", err) + } + if n > 0 { + mustAppend(t, l, linkedBlocks(1, n, 40)...) + } + mustClose(t, l) + return dir +} + +// rewrite replaces dir/name with f applied to its bytes. +func rewrite(t *testing.T, dir, name string, f func([]byte) []byte) { + t.Helper() + path := filepath.Join(dir, name) + b, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, f(bytes.Clone(b)), 0o600); err != nil { + t.Fatal(err) + } +} + +// openCorrupt opens dir, wants ErrCorrupt with text, and checks that no file changed. +func openCorrupt(t *testing.T, dir, text string) { + t.Helper() + before := listing(t, dir) + l, err := Open(dir, testOpts(0)) + if err == nil { + _ = l.Close() + } + wantCorrupt(t, err, text) + wantSame(t, "files after a failed Open", listing(t, dir), before) +} + +func TestMetaCorruptOpen(t *testing.T) { + base := newLog(t, testOpts(0), 3) + m := logtest.Meta() + head := cat([]byte("EVMSMETA"), []byte{0x01}, m.ID[:], uv(m.ChainID), uv(m.Start)) + tests := []struct { + name string + damage func(dir string) + detail string + }{ + {"missing", func(dir string) { _ = os.Remove(filepath.Join(dir, "meta")) }, "missing"}, + {"cut to 5 bytes", func(dir string) { rewrite(t, dir, "meta", func(b []byte) []byte { return b[:5] }) }, "truncated"}, + {"flipped body byte", func(dir string) { + rewrite(t, dir, "meta", func(b []byte) []byte { b[20] ^= 1; return b }) + }, "checksum mismatch"}, + {"bad magic", func(dir string) { + rewrite(t, dir, "meta", func(b []byte) []byte { return seal(cat([]byte("XXXXXXXX"), b[16:])) }) + }, "bad magic"}, + {"version 2", func(dir string) { + rewrite(t, dir, "meta", func(b []byte) []byte { return seal(cat(b[8:16], []byte{0x02}, b[17:])) }) + }, "version 2"}, + {"npos 5", func(dir string) { + rewrite(t, dir, "meta", func([]byte) []byte { return seal(cat(head, uv(0), uv(5))) }) + }, "filter out of range"}, + {"trailing byte", func(dir string) { + rewrite(t, dir, "meta", func(b []byte) []byte { return seal(cat(b[8:], []byte{0x00})) }) + }, "trailing bytes"}, + {"3 MiB after the meta", func(dir string) { + rewrite(t, dir, "meta", func(b []byte) []byte { return append(b, make([]byte, 3<<20)...) }) + }, "trailing bytes"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := copyDir(t, base) + tt.damage(dir) + openCorrupt(t, dir, "wal: corrupt log: meta: "+tt.detail) + }) + } +} + +func TestPersistence(t *testing.T) { + ctx := t.Context() + in := linkedBlocks(1, 20, 16) + in = append(in, finalizeTo(ref(10)), revertTo(ref(15))) + in = append(in, linkedBlocks(16, 25, 16)...) + in = append(in, finalizeTo(ref(30))) + in = append(in, linkedBlocks(41, 2, 16)...) + if len(in) != 50 { + t.Fatalf("test setup: %d entries, want 50", len(in)) + } + opts := testOpts(512) + opts.WindowCap = 0 + dir := t.TempDir() + l := openLog(t, dir, opts, hooks{}) + for _, e := range in { + mustAppend(t, l, e) + } + before := snap(t, l) + meta, _ := l.Meta(ctx) + if l.lastFin != 48 { + t.Errorf("last finalize offset = %d, want 48", l.lastFin) + } + mustClose(t, l) + + mem := chainlog.NewMemory(opts.Meta, 0) + for _, e := range in { + if _, err := mem.Append(ctx, e); err != nil { + t.Fatalf("Memory.Append: %v", err) + } + } + wantSame(t, "WAL before close vs Memory", before, snap(t, mem)) + + l = openLog(t, dir, opts, hooks{}) + wantSame(t, "WAL after reopen", snap(t, l), before) + if l.lastFin != 48 { + t.Errorf("last finalize offset after reopen = %d, want 48", l.lastFin) + } + m, _ := l.Meta(ctx) + wantSame(t, "Meta after reopen", m, meta) + if off := mustAppend(t, l, linkedBlocks(43, 1, 16)...); off != 51 { + t.Errorf("Append after reopen returned %d, want 51", off) + } +} + +func TestRollAndCheckpoint(t *testing.T) { + dir := t.TempDir() + l := openLog(t, dir, testOpts(512), hooks{}) + next := uint64(1) + for len(segmentNames(t, dir)) < 3 { + mustAppend(t, l, linkedBlocks(next, 1, 100)...) + next++ + } + // A batch larger than the segment size gets a segment of its own. + big := linkedBlocks(next, 1, 600) + if n := len(chainlog.Encode(nil, chainlog.Entry{Offset: 1, Kind: chainlog.KindBlock, Block: big[0].Block})); n <= 512 { + t.Fatalf("test setup: big record is %d bytes, want more than 512", n) + } + bigOff := mustAppend(t, l, big...) + mustAppend(t, l, linkedBlocks(next+1, 1, 100)...) + head, _ := l.Head(t.Context()) + before := snap(t, l) + mustClose(t, l) + + names := segmentNames(t, dir) + sizes := listing(t, dir) + for i, name := range names { + first, _ := parseSegmentName(name) + if i > 0 { + prev, _ := parseSegmentName(names[i-1]) + if first <= prev { + t.Errorf("segment %s does not follow %s", name, names[i-1]) + } + } + if i < len(names)-1 && sizes[name] > 512 && first != bigOff { + t.Errorf("sealed segment %s holds %d bytes, more than 512", name, sizes[name]) + } + } + bigName := segmentName(bigOff) + if sizes[bigName] <= 512 { + t.Errorf("the big batch's segment %s holds %d bytes, want more than 512", bigName, sizes[bigName]) + } + if next, _ := parseSegmentName(names[slices.Index(names, bigName)+1]); next != bigOff+1 { + t.Errorf("the big batch's segment holds more than its batch: next segment starts at %d", next) + } + + b, err := os.ReadFile(filepath.Join(dir, "checkpoint")) + if err != nil { + t.Fatalf("checkpoint: %v", err) + } + cp, err := decodeCheckpoint(b, 0) + if err != nil { + t.Fatalf("decodeCheckpoint: %v", err) + } + lastFirst, _ := parseSegmentName(names[len(names)-1]) + if cp.x != lastFirst-1 { + t.Errorf("checkpoint at %d, want the head at the last roll, %d", cp.x, lastFirst-1) + } + + var replayed []chainlog.Offset + l = openLog(t, dir, testOpts(512), hooks{replay: func(off chainlog.Offset) { replayed = append(replayed, off) }}) + var want []chainlog.Offset + for off := cp.x + 1; off <= head; off++ { + want = append(want, off) + } + wantSame(t, "replayed offsets", replayed, want) + wantSame(t, "log after reopen", snap(t, l), before) +} + +// damageCheckpoint rewrites the checkpoint file of dir. +func damageCheckpoint(t *testing.T, dir string, f func([]byte) []byte) { + t.Helper() + rewrite(t, dir, checkpointName, f) +} + +func TestCheckpointOpen(t *testing.T) { + base := t.TempDir() + l := openLog(t, base, testOpts(512), hooks{}) + for n := uint64(1); len(segmentNames(t, base)) < 3; n++ { + mustAppend(t, l, linkedBlocks(n, 1, 100)...) + } + head, _ := l.Head(t.Context()) + mustClose(t, l) + b, _ := os.ReadFile(filepath.Join(base, checkpointName)) + cp, err := decodeCheckpoint(b, 0) + if err != nil { + t.Fatalf("decodeCheckpoint: %v", err) + } + stateBin, _ := cp.state.MarshalBinary() + reencode := func(x, l uint64, state []byte) func([]byte) []byte { + return func([]byte) []byte { return seal(cat([]byte("EVMSCKPT"), []byte{0x01}, uv(x), uv(l), state)) } + } + badState := bytes.Clone(stateBin) + badState[0] = 2 + names := segmentNames(t, base) + f1, _ := parseSegmentName(names[1]) + f2, _ := parseSegmentName(names[2]) + if cp.x != f2-1 { + t.Fatalf("test setup: checkpoint at %d, want %d", cp.x, f2-1) + } + + tests := []struct { + name string + damage func(dir string) + text string + }{ + {"cut", func(dir string) { damageCheckpoint(t, dir, func(b []byte) []byte { return b[:len(b)-1] }) }, "checkpoint: truncated"}, + {"flipped byte", func(dir string) { + damageCheckpoint(t, dir, func(b []byte) []byte { b[len(b)-1] ^= 1; return b }) + }, "checkpoint: checksum mismatch"}, + {"bad magic", func(dir string) { + damageCheckpoint(t, dir, func(b []byte) []byte { return seal(cat([]byte("XXXXXXXX"), b[16:])) }) + }, "checkpoint: bad magic"}, + {"version 2", func(dir string) { + damageCheckpoint(t, dir, func(b []byte) []byte { return seal(cat(b[8:16], []byte{0x02}, b[17:])) }) + }, "checkpoint: version 2"}, + {"l above x", func(dir string) { damageCheckpoint(t, dir, reencode(uint64(cp.x), uint64(cp.x)+1, stateBin)) }, "checkpoint: bad header"}, + {"bad state", func(dir string) { damageCheckpoint(t, dir, reencode(uint64(cp.x), 0, badState)) }, "checkpoint: chainlog: bad state encoding: version 2"}, + {"x above head", func(dir string) { damageCheckpoint(t, dir, reencode(uint64(head)+5, 0, stateBin)) }, + "checkpoint at " + strconv.FormatUint(uint64(head)+5, 10) + " outside [0, " + strconv.FormatUint(uint64(head), 10) + "]"}, + {"x below the first segment", func(dir string) { + _ = os.Remove(filepath.Join(dir, names[0])) + damageCheckpoint(t, dir, reencode(uint64(f1)-2, 0, stateBin)) + }, "checkpoint at " + strconv.FormatUint(uint64(f1)-2, 10) + " outside [" + strconv.FormatUint(uint64(f1)-1, 10) + ", " + strconv.FormatUint(uint64(head), 10) + "]"}, + {"no checkpoint, first segment not 1", func(dir string) { + _ = os.Remove(filepath.Join(dir, checkpointName)) + _ = os.Remove(filepath.Join(dir, names[0])) + }, "first segment starts at " + strconv.FormatUint(uint64(f1), 10) + " without a checkpoint"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := copyDir(t, base) + tt.damage(dir) + openCorrupt(t, dir, "wal: corrupt log: "+tt.text) + }) + } + + // A checkpoint right before the first segment, as compaction leaves it, opens. + dir := copyDir(t, base) + _ = os.Remove(filepath.Join(dir, names[0])) + _ = os.Remove(filepath.Join(dir, names[1])) + l = openLog(t, dir, testOpts(512), hooks{}) + if o, _ := l.Oldest(t.Context()); o != f2 { + t.Errorf("Oldest = %d, want %d", o, f2) + } + if h, _ := l.Head(t.Context()); h != head { + t.Errorf("Head = %d, want %d", h, head) + } + + // A bad checkpoint state also wraps chainlog.ErrBadState. + dir = copyDir(t, base) + damageCheckpoint(t, dir, reencode(uint64(cp.x), 0, badState)) + if l, err := Open(dir, testOpts(512)); !errors.Is(err, chainlog.ErrBadState) { + if err == nil { + _ = l.Close() + } + t.Errorf("Open with a bad checkpoint state = %v, want one wrapping chainlog.ErrBadState", err) + } +} + +func TestOpenScanErrors(t *testing.T) { + base := t.TempDir() + l := openLog(t, base, testOpts(512), hooks{}) + for n := uint64(1); len(segmentNames(t, base)) < 3; n++ { + mustAppend(t, l, linkedBlocks(n, 1, 100)...) + } + mustClose(t, l) + names := segmentNames(t, base) + first1, _ := parseSegmentName(names[1]) + last0 := first1 - 1 + + tests := []struct { + name string + damage func(dir string) + text string + }{ + {"no segment", func(dir string) { + for _, n := range names { + _ = os.Remove(filepath.Join(dir, n)) + } + }, "no segment"}, + {"gap", func(dir string) { + _ = os.Rename(filepath.Join(dir, names[1]), filepath.Join(dir, segmentName(first1+1))) + }, "segment " + segmentName(first1+1) + " does not follow offset " + strconv.FormatUint(uint64(last0), 10)}, + {"empty sealed segment", func(dir string) { + _ = os.Truncate(filepath.Join(dir, names[1]), 0) + }, "segment " + names[1] + " is empty"}, + {"sealed segment cut inside its record", func(dir string) { + rewrite(t, dir, names[0], func(b []byte) []byte { return b[:len(b)-5] }) + }, "segment " + names[0] + " position 0: chainlog: incomplete record"}, + {"sealed segment with a length above the limit", func(dir string) { + rewrite(t, dir, names[0], func(b []byte) []byte { b[0], b[1], b[2], b[3] = 0xff, 0xff, 0xff, 0xff; return b }) + }, "segment " + names[0] + " position 0: chainlog: record too large"}, + {"flipped byte in the first segment", func(dir string) { + rewrite(t, dir, names[0], func(b []byte) []byte { b[20] ^= 1; return b }) + }, "segment " + names[0] + " position 0: chainlog: record checksum mismatch"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir := copyDir(t, base) + tt.damage(dir) + before := listing(t, dir) + l, err := Open(dir, testOpts(512)) + if err == nil { + _ = l.Close() + } + if !errors.Is(err, ErrCorrupt) { + t.Fatalf("Open = %v, want one wrapping ErrCorrupt", err) + } + if want := "wal: corrupt log: " + tt.text; !strings.HasPrefix(err.Error(), want) { + t.Errorf("Open error =\n%q\nwant the prefix\n%q", err, want) + } + wantSame(t, "files after a failed Open", listing(t, dir), before) + }) + } +} + +// failingWrite returns a write hook that fails with syscall.EIO while *fail is set. +func failingWrite(fail *bool) func(f *os.File, p []byte) (int, error) { + return func(f *os.File, p []byte) (int, error) { + if *fail { + return 0, syscall.EIO + } + return f.Write(p) + } +} + +func TestWriteFailure(t *testing.T) { + ctx := t.Context() + dir := t.TempDir() + var fail bool + l := openLog(t, dir, testOpts(0), hooks{write: failingWrite(&fail)}) + mustAppend(t, l, linkedBlocks(1, 3, 8)...) + fail = true + _, err := l.Append(ctx, linkedBlocks(4, 1, 8)...) + if !errors.Is(err, ErrFailed) || !errors.Is(err, syscall.EIO) { + t.Fatalf("Append with a failing write = %v, want one wrapping ErrFailed and EIO", err) + } + ioText := "wal: write " + filepath.Join(dir, firstSegment) + ": " + syscall.EIO.Error() + if got, want := err.Error(), "wal: log failed: append: "+ioText; got != want { + t.Errorf("Append error = %q, want %q", got, want) + } + if h, _ := l.Head(ctx); h != 3 { + t.Errorf("Head = %d after the failed Append, want 3", h) + } + fail = false // the failure is sticky even when writes would work again + earlier := "wal: log failed: earlier error: " + ioText + if _, err := l.Append(ctx, linkedBlocks(4, 1, 8)...); err == nil || err.Error() != earlier || !errors.Is(err, ErrFailed) { + t.Errorf("next Append = %v, want %q", err, earlier) + } + if err := l.Truncate(ctx, 1); err == nil || err.Error() != earlier { + t.Errorf("Truncate = %v, want %q", err, earlier) + } + if es, err := l.Read(ctx, 1, 10); err != nil || len(es) != 3 { + t.Errorf("Read after the failure = %d entries, %v; want 3, nil", len(es), err) + } + mustClose(t, l) + + l = openLog(t, dir, testOpts(0), hooks{}) + if h, _ := l.Head(ctx); h != 3 { + t.Errorf("Head after reopen = %d, want 3", h) + } + if off := mustAppend(t, l, linkedBlocks(4, 1, 8)...); off != 4 { + t.Errorf("Append after reopen returned %d, want 4", off) + } +} + +func TestReadCorrupt(t *testing.T) { + dir := t.TempDir() + l := openLog(t, dir, testOpts(0), hooks{}) + es := linkedBlocks(1, 3, 40) + mustAppend(t, l, es...) + rec1 := len(chainlog.Encode(nil, chainlog.Entry{Offset: 1, Kind: es[0].Kind, Block: es[0].Block})) + f, err := os.OpenFile(filepath.Join(dir, firstSegment), os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt([]byte{0xff}, int64(rec1+30)); err != nil { + t.Fatal(err) + } + _ = f.Close() + _, err = l.Read(t.Context(), 2, 1) + if !errors.Is(err, ErrCorrupt) || !errors.Is(err, chainlog.ErrChecksum) { + t.Fatalf("Read of a damaged record = %v, want one wrapping ErrCorrupt and chainlog.ErrChecksum", err) + } + if want := "wal: corrupt log: segment " + firstSegment + " position " + strconv.Itoa(rec1) + ": chainlog: record checksum mismatch: "; !strings.HasPrefix(err.Error(), want) { + t.Errorf("Read error = %q, want the prefix %q", err, want) + } + if got, err := l.Read(t.Context(), 1, 1); err != nil || len(got) != 1 { + t.Errorf("Read of the record before = %v, %v; want it", got, err) + } +} + +func TestObservability(t *testing.T) { + logger, rec := newRecorder() + dir := filepath.Join(t.TempDir(), "wal") + opts := testOpts(512) + opts.Logger = logger + l := openLog(t, dir, opts, hooks{}) + attrs, ok := rec.find(slog.LevelInfo, "wal opened") + wantSame(t, "wal opened", attrs, map[string]any{ + "dir": dir, "segments": int64(1), "oldest": uint64(1), "head": uint64(0), "checkpoint": uint64(0), "new": true, + }) + if !ok { + t.Fatal("no wal opened line") + } + for n := uint64(1); len(segmentNames(t, dir)) < 2; n++ { + mustAppend(t, l, linkedBlocks(n, 1, 100)...) + } + names := segmentNames(t, dir) + first, _ := parseSegmentName(names[1]) + attrs, _ = rec.find(slog.LevelDebug, "wal segment rolled") + wantSame(t, "wal segment rolled", attrs, map[string]any{"segment": names[1], "offset": uint64(first)}) + head, _ := l.Head(t.Context()) + mustClose(t, l) + + openLog(t, dir, opts, hooks{}) + attrs, _ = rec.find(slog.LevelInfo, "wal opened") + wantSame(t, "wal opened on reopen", attrs, map[string]any{ + "dir": dir, "segments": int64(2), "oldest": uint64(1), "head": uint64(head), "checkpoint": uint64(first - 1), "new": false, + }) +} + +func TestOpenAfterInterruptedRoll(t *testing.T) { + ctx := t.Context() + dir := t.TempDir() + fail := false + // Fail the first write to any segment but the first: the batch that rolls. + h := hooks{write: func(f *os.File, p []byte) (int, error) { + if fail && filepath.Base(f.Name()) != firstSegment { + return 0, syscall.EIO + } + return f.Write(p) + }} + l := openLog(t, dir, testOpts(512), h) + fail = true + var before snapshotOf + for n := uint64(1); ; n++ { + before = snap(t, l) + if _, err := l.Append(ctx, linkedBlocks(n, 1, 100)...); err != nil { + if !errors.Is(err, ErrFailed) { + t.Fatalf("Append: %v", err) + } + break + } + } + mustClose(t, l) + names := segmentNames(t, dir) + if len(names) != 2 || listing(t, dir)[names[1]] != 0 { + t.Fatalf("test setup: segments %v, want the first and an empty second", listing(t, dir)) + } + noSegment := copyDir(t, dir) + _ = os.Remove(filepath.Join(noSegment, names[1])) // the checkpoint was written, the segment never created + + for _, d := range []string{dir, noSegment} { + l := openLog(t, d, testOpts(512), hooks{}) + wantSame(t, "log after the interrupted roll", snap(t, l), before) + if off := mustAppend(t, l, linkedBlocks(uint64(before.head)+1, 1, 100)...); off != before.head+1 { + t.Errorf("Append after reopen returned %d, want %d", off, before.head+1) + } + mustClose(t, l) + } +} + +func TestSegmentName(t *testing.T) { + if got, want := segmentName(1), firstSegment; got != want { + t.Errorf("segmentName(1) = %q, want %q", got, want) + } + if got, want := segmentName(18446744073709551615), "18446744073709551615.seg"; got != want { + t.Errorf("segmentName(max) = %q, want %q", got, want) + } + tests := []struct { + name string + first chainlog.Offset + ok bool + }{ + {firstSegment, 1, true}, + {"00000000000000000300.seg", 300, true}, + {"18446744073709551615.seg", 18446744073709551615, true}, + {"18446744073709551616.seg", 0, false}, // above uint64 + {"0000000000000000001.seg", 0, false}, // 19 digits + {"000000000000000000001.seg", 0, false}, + {"0000000000000000000a.seg", 0, false}, + {"00000000000000000001.txt", 0, false}, + {"meta", 0, false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + first, ok := parseSegmentName(tt.name) + if first != tt.first || ok != tt.ok { + t.Errorf("parseSegmentName(%q) = %d, %v; want %d, %v", tt.name, first, ok, tt.first, tt.ok) + } + }) + } +} + +// TestReadError covers the errors Read reports when reading a segment fails: a file closed by +// Close or deleted by Truncate under the reader gives ErrClosed or ErrTruncated. +func TestReadError(t *testing.T) { + ioErr := errors.New("read failed") + corrupt := fmt.Errorf("%w: segment x position 0: bad", ErrCorrupt) + l := &Log{oldest: 5} + if err := l.readError(3, corrupt); err != corrupt { // the same error is returned + t.Errorf("readError of a corrupt record = %v, want it unchanged", err) + } + err := l.readError(3, ioErr) + if !errors.Is(err, chainlog.ErrTruncated) || err.Error() != "chainlog: offset truncated: read from 3, oldest is 5" { + t.Errorf("readError below Oldest = %v, want ErrTruncated", err) + } + if err := l.readError(5, ioErr); err != ioErr { // the same error is returned + t.Errorf("readError of a retained offset = %v, want the I/O error", err) + } + l.closed = true + if err := l.readError(5, ioErr); err != chainlog.ErrClosed { // unwrapped by contract + t.Errorf("readError of a closed log = %v, want ErrClosed", err) + } +} + +// TestLastFinalizeOffset follows the offset of the newest Finalize, the bound Truncate never +// crosses, through Append, the checkpoint and the replay on Open. +func TestLastFinalizeOffset(t *testing.T) { + ctx := t.Context() + opts := testOpts(512) + opts.WindowCap = 0 + dir := t.TempDir() + l := openLog(t, dir, opts, hooks{}) + mustAppend(t, l, linkedBlocks(1, 3, 100)...) + f1 := mustAppend(t, l, finalizeTo(ref(3))) + for n := uint64(4); len(segmentNames(t, dir)) < 3; n++ { + mustAppend(t, l, linkedBlocks(n, 1, 100)...) + } + head, _ := l.Head(ctx) + if l.lastFin != f1 { + t.Errorf("last finalize offset after Append = %d, want %d", l.lastFin, f1) + } + mustClose(t, l) + b, _ := os.ReadFile(filepath.Join(dir, checkpointName)) + cp, err := decodeCheckpoint(b, 0) + if err != nil || cp.l != f1 || cp.x <= f1 { + t.Fatalf("checkpoint = x %d, l %d, %v; want l %d below x", cp.x, cp.l, err, f1) + } + + // No Finalize after the checkpoint: L comes from the checkpoint. + l = openLog(t, dir, opts, hooks{}) + if l.lastFin != f1 { + t.Errorf("last finalize offset after reopen = %d, want %d from the checkpoint", l.lastFin, f1) + } + // A Finalize after the checkpoint: L comes from the replay. + f2 := mustAppend(t, l, finalizeTo(ref(uint64(head)))) + mustClose(t, l) + l = openLog(t, dir, opts, hooks{}) + if l.lastFin != f2 { + t.Errorf("last finalize offset after reopen = %d, want %d from the replay", l.lastFin, f2) + } +} + +// TestReopenIndex reads past the first 256 records of a segment through the index Open builds. +func TestReopenIndex(t *testing.T) { + ctx := t.Context() + opts := testOpts(0) + opts.WindowCap = 0 + dir := t.TempDir() + es := linkedBlocks(1, 600, 4) + l := openLog(t, dir, opts, hooks{}) + mustAppend(t, l, es...) + mustClose(t, l) + + l = openLog(t, dir, opts, hooks{}) + var pos int64 + var want []int64 + for i, e := range es { + if i%256 == 0 { + want = append(want, pos) + } + e.Offset = chainlog.Offset(i + 1) //nolint:gosec // i < 600 + pos += int64(len(chainlog.Encode(nil, e))) + } + wantSame(t, "index positions of records 1, 257 and 513", l.segs[0].index, want) + for _, r := range []struct { + from chainlog.Offset + max int + }{{256, 1}, {257, 1}, {513, 1}, {250, 400}} { + got, err := l.Read(ctx, r.from, r.max) + if err != nil { + t.Fatalf("Read(%d, %d): %v", r.from, r.max, err) + } + for i, e := range got { + if e.Offset != r.from+chainlog.Offset(i) || e.Block.Ref != es[e.Offset-1].Block.Ref { //nolint:gosec // i < 400 + t.Fatalf("Read(%d, %d) entry %d is offset %d, block %d", r.from, r.max, i, e.Offset, e.Block.Ref.Number) + } + } + } +} + +// TestRollBoundary fills a segment to exactly SegmentSize: that batch does not roll, the next does. +func TestRollBoundary(t *testing.T) { + es := linkedBlocks(1, 3, 40) + var size int64 + for i, e := range es[:2] { + e.Offset = chainlog.Offset(i + 1) //nolint:gosec // i < 2 + size += int64(len(chainlog.Encode(nil, e))) + } + dir := t.TempDir() + l := openLog(t, dir, testOpts(size), hooks{}) + mustAppend(t, l, es[0]) + mustAppend(t, l, es[1]) + if n := len(segmentNames(t, dir)); n != 1 { + t.Errorf("a segment filled to exactly SegmentSize rolled: %d segments", n) + } + mustAppend(t, l, es[2]) + if n := len(segmentNames(t, dir)); n != 2 { + t.Errorf("one byte over SegmentSize: %d segments, want 2", n) + } +} + +func TestMaxWindowCap(t *testing.T) { + opts := testOpts(0) + opts.WindowCap = MaxWindowCap + openLog(t, t.TempDir(), opts, hooks{}) +} + +// TestBoundedFileRead opens a log whose meta is followed by 64 MiB: Open reads at most the largest +// meta and one byte more. +func TestBoundedFileRead(t *testing.T) { + dir := newLog(t, testOpts(0), 1) + if err := os.Truncate(filepath.Join(dir, metaName), 64<<20); err != nil { + t.Fatal(err) + } + var before, after runtime.MemStats + runtime.ReadMemStats(&before) + _, err := Open(dir, testOpts(0)) + runtime.ReadMemStats(&after) + wantCorrupt(t, err, "wal: corrupt log: meta: trailing bytes") + if alloc := after.TotalAlloc - before.TotalAlloc; alloc > 16<<20 { + t.Errorf("Open allocated %d bytes for a 64 MiB meta, want far less", alloc) + } +} + +// TestReadWrongOffset re-seals a record with another offset and a valid CRC under an open log. +func TestReadWrongOffset(t *testing.T) { + dir := t.TempDir() + l := openLog(t, dir, testOpts(0), hooks{}) + es := linkedBlocks(1, 3, 40) + mustAppend(t, l, es...) + first := es[0] + first.Offset = 1 + rec1 := len(chainlog.Encode(nil, first)) + second := es[1] + second.Offset = 9 // a one-byte varint, like 2 + f, err := os.OpenFile(filepath.Join(dir, firstSegment), os.O_RDWR, 0) + if err != nil { + t.Fatal(err) + } + if _, err := f.WriteAt(chainlog.Encode(nil, second), int64(rec1)); err != nil { + t.Fatal(err) + } + _ = f.Close() + _, err = l.Read(t.Context(), 2, 1) + if want := "wal: corrupt log: segment " + firstSegment + " position " + strconv.Itoa(rec1) + ": offset 9, want 2"; err == nil || err.Error() != want || !errors.Is(err, ErrCorrupt) { + t.Errorf("Read of a record with the wrong offset = %v, want %q", err, want) + } +} + +func TestOpenSegmentZero(t *testing.T) { + dir := newLog(t, testOpts(0), 1) + if err := os.Rename(filepath.Join(dir, firstSegment), filepath.Join(dir, segmentName(0))); err != nil { + t.Fatal(err) + } + openCorrupt(t, dir, "wal: corrupt log: segment "+segmentName(0)+" does not follow offset 0") +} + +// syncRecorder is a sync hook that records the name of every file fsynced, and fails the fsync of +// the first file whose base name is failOn. +type syncRecorder struct { + mu sync.Mutex + names []string + failOn string +} + +func (r *syncRecorder) sync(name string) error { + r.mu.Lock() + defer r.mu.Unlock() + r.names = append(r.names, filepath.Base(name)) + if r.failOn != "" && filepath.Base(name) == r.failOn { + r.failOn = "" + return syscall.EIO + } + return nil +} + +func (r *syncRecorder) take() []string { + r.mu.Lock() + defer r.mu.Unlock() + names := r.names + r.names = nil + return names +} + +// TestSyncOrder checks every fsync of SyncAlways, in order: creation, append and roll. +func TestSyncOrder(t *testing.T) { + dir := filepath.Join(t.TempDir(), "wal") + rec := &syncRecorder{} + opts := testOpts(512) + opts.Sync = SyncAlways + l := openLog(t, dir, opts, hooks{sync: rec.sync}) + base := filepath.Base(dir) + wantSame(t, "fsyncs of a new log", rec.take(), []string{"meta.tmp", base, base}) + + mustAppend(t, l, linkedBlocks(1, 1, 100)...) + wantSame(t, "fsyncs of an append", rec.take(), []string{firstSegment}) + + n := uint64(2) + for ; len(segmentNames(t, dir)) < 2; n++ { + mustAppend(t, l, linkedBlocks(n, 1, 100)...) + } + second := segmentNames(t, dir)[1] + got := rec.take() + wantSame(t, "fsyncs of a roll", got[len(got)-4:], []string{"checkpoint.tmp", base, base, second}) + + opts.Sync = SyncNever + never := openLog(t, t.TempDir(), opts, hooks{sync: rec.sync}) + mustAppend(t, never, linkedBlocks(1, 1, 100)...) + wantSame(t, "fsyncs with SyncNever", rec.take(), []string(nil)) +} + +// TestSyncFailure fails one fsync of an append or of a roll: the Append fails with that error, the +// log is unchanged, and the failure is sticky. +func TestSyncFailure(t *testing.T) { + tests := []struct { + name string + file func(dir string) string // the base name of the file whose fsync fails + nth int // fail its nth fsync after the first block + blocks uint64 // blocks appended before the failing Append + }{ + {"segment", func(string) string { return firstSegment }, 1, 0}, + {"checkpoint of a roll", func(string) string { return "checkpoint.tmp" }, 1, 1}, + {"directory after the roll's new segment", filepath.Base, 2, 1}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctx := t.Context() + dir := t.TempDir() + armed, seen := false, 0 + h := hooks{sync: func(name string) error { + if armed && filepath.Base(name) == tt.file(dir) { + if seen++; seen == tt.nth { + return syscall.EIO + } + } + return nil + }} + opts := testOpts(512) + opts.Sync = SyncAlways + l := openLog(t, dir, opts, h) + if tt.blocks > 0 { + mustAppend(t, l, linkedBlocks(1, tt.blocks, 100)...) // a full segment: the next Append rolls + } + before := snap(t, l) + armed = true + next := linkedBlocks(tt.blocks+1, 1, 100) + _, err := l.Append(ctx, next...) + if !errors.Is(err, ErrFailed) || !errors.Is(err, syscall.EIO) { + t.Fatalf("Append with a failing fsync = %v, want ErrFailed and EIO", err) + } + path := filepath.Join(dir, tt.file(dir)) + if tt.file(dir) == filepath.Base(dir) { + path = dir + } + if got, want := err.Error(), "wal: log failed: append: wal: sync "+path+": "+syscall.EIO.Error(); got != want { + t.Errorf("Append error =\n%q\nwant\n%q", got, want) + } + wantSame(t, "log after the failed Append", snap(t, l), before) + if _, err := l.Append(ctx, next...); !errors.Is(err, ErrFailed) { + t.Errorf("next Append = %v, want the sticky ErrFailed", err) + } + }) + } +} + +// TestOpenCorruptActive flips a byte of the first of three records in the active segment: more +// records follow it, so it is corruption, not a torn write. +func TestOpenCorruptActive(t *testing.T) { + dir := newLog(t, testOpts(0), 3) + rewrite(t, dir, firstSegment, func(b []byte) []byte { b[20] ^= 1; return b }) + before := listing(t, dir) + l, err := Open(dir, testOpts(0)) + if err == nil { + _ = l.Close() + } + if !errors.Is(err, ErrCorrupt) || !strings.HasPrefix(err.Error(), "wal: corrupt log: segment "+firstSegment+" position 0: ") { + t.Errorf("Open = %v, want ErrCorrupt at position 0", err) + } + wantSame(t, "files after a failed Open", listing(t, dir), before) +} + +// TestReadClose closes the log under readers: every Read returns entries or ErrClosed. +func TestReadClose(t *testing.T) { + ctx := t.Context() + l, err := Open(t.TempDir(), testOpts(4096)) + if err != nil { + t.Fatal(err) + } + es := linkedBlocks(1, 300, 40) + for i := 0; i < len(es); i += 8 { + if _, err := l.Append(ctx, es[i:min(i+8, len(es))]...); err != nil { + t.Fatal(err) + } + if _, err := l.Append(ctx, finalizeTo(es[min(i+8, len(es))-1].Block.Ref)); err != nil { + t.Fatal(err) + } + } + const readers = 8 + var wg sync.WaitGroup + started := make(chan struct{}, readers) + errs := make([]error, readers) + for r := range readers { + wg.Go(func() { + for i := 0; ; i++ { + _, err := l.Read(ctx, 1, 400) + if i == 0 { + started <- struct{}{} + } + if err != nil { + errs[r] = err + return + } + } + }) + } + for range readers { + <-started + } + if err := l.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + wg.Wait() + for r, err := range errs { + if err != chainlog.ErrClosed { //nolint:errorlint // the contract returns ErrClosed unwrapped + t.Errorf("reader %d stopped with %v, want ErrClosed", r, err) + } + } +} diff --git a/consumer/bench_test.go b/consumer/bench_test.go new file mode 100644 index 0000000..45e6bad --- /dev/null +++ b/consumer/bench_test.go @@ -0,0 +1,77 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package consumer_test + +import ( + "context" + "encoding/binary" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/consumer" +) + +// nop is a consumer whose Apply only commits the offset. +type nop struct{ off chainlog.Offset } + +func (n *nop) Offset(context.Context) (chainlog.Offset, error) { return n.off, nil } + +func (n *nop) Apply(_ context.Context, e chainlog.Entry) error { + n.off = e.Offset + return nil +} + +// BenchmarkRun runs a no-op consumer over a log of 100,000 block entries per iteration and +// reports the time per entry. +func BenchmarkRun(b *testing.B) { + const entries = 100_000 + c, rs := refs(b, entries) + es := chainEntries(c, rs, entries+1)[1:] // blocks only: blk(A1) … blk(A100000) + log := newLog(b, es...) + b.ReportAllocs() + for b.Loop() { + ctx, cancel := context.WithCancel(b.Context()) + if err := consumer.Run(ctx, log, &canceler{Consumer: &nop{}, at: entries, cancel: cancel}); err != context.Canceled { + b.Fatalf("Run = %v, want context.Canceled", err) + } + cancel() + } + b.ReportMetric(float64(b.Elapsed().Nanoseconds())/float64(b.N)/entries, "ns/entry") +} + +// BenchmarkMemoryApply applies one block on top of pending blocks per iteration, with apply and +// clone functions that do nothing. Every 1,024 blocks a Finalize leaves only the newest block +// pending, so the pending blocks stay bounded. +func BenchmarkMemoryApply(b *testing.B) { + ctx := b.Context() + m := consumer.NewMemory(struct{}{}, + func(s struct{}, _ chain.Block) struct{} { return s }, + func(s struct{}) struct{} { return s }) + ref := func(n uint64) chain.BlockRef { + var h common.Hash + binary.BigEndian.PutUint64(h[24:], n+1) + return chain.BlockRef{Number: n, Hash: h} + } + off := chainlog.Offset(0) + next := func(e chainlog.Entry) { + off++ + e.Offset = off + if err := m.Apply(ctx, e); err != nil { + b.Fatal(err) + } + } + next(chainlog.Entry{Kind: chainlog.KindFinalize, To: ref(0)}) + n := uint64(0) + b.ReportAllocs() + for b.Loop() { + n++ + next(chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: ref(n)}}) + if n%1024 == 0 { + next(chainlog.Entry{Kind: chainlog.KindFinalize, To: ref(n - 1)}) + } + } +} diff --git a/consumer/consumer.go b/consumer/consumer.go new file mode 100644 index 0000000..06e292a --- /dev/null +++ b/consumer/consumer.go @@ -0,0 +1,131 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package consumer + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/hmdsefi/evmsync/chainlog" +) + +// Consumer applies canonical-log entries in offset order and commits each entry's offset +// atomically with the state change it causes. +type Consumer interface { + // Offset returns the committed offset: the offset of the last entry whose effects are + // committed, or 0 if none. + Offset(ctx context.Context) (chainlog.Offset, error) + + // Apply applies e, whose offset is the committed offset plus one, and commits e.Offset in + // the same atomic step as the state change. On error nothing of e is committed and Offset is + // unchanged. Apply must not modify e; it may retain it. + Apply(ctx context.Context, e chainlog.Entry) error +} + +var ( + // ErrBehindLog reports that the next offset was truncated from the log. + ErrBehindLog = errors.New("consumer: offset is behind the oldest log entry") + // ErrAheadOfLog reports a committed offset above the log's head. + ErrAheadOfLog = errors.New("consumer: offset is ahead of the log head") + // ErrOffsetGap reports that the log returned an entry other than the next offset. + ErrOffsetGap = errors.New("consumer: log offsets are not contiguous") + // ErrBadEntry reports an entry that does not fit Memory's state. + ErrBadEntry = errors.New("consumer: entry does not fit the state") +) + +// ApplyError is the error Run returns when Apply fails. +type ApplyError struct { + Offset chainlog.Offset // the entry's offset + Kind chainlog.Kind // the entry's kind + Err error // what Apply returned +} + +// Error returns "consumer: apply offset (): ", where is "block", +// "revert", "finalize" or "kind " for any other value. +func (e *ApplyError) Error() string { + var kind string + switch e.Kind { + case chainlog.KindBlock, chainlog.KindRevert, chainlog.KindFinalize: + kind = e.Kind.String() + default: + kind = "kind " + strconv.Itoa(int(e.Kind)) + } + return fmt.Sprintf("consumer: apply offset %d (%s): %v", e.Offset, kind, e.Err) +} + +// Unwrap returns e.Err. +func (e *ApplyError) Unwrap() error { return e.Err } + +// readBatch is the most entries Run reads at once. +const readBatch = 256 + +// Run feeds c the entries of log from c's committed offset plus one, in order, one Apply per +// entry. It reads in batches, waits at the tail with log.Wait and returns only on error: +// ctx.Err() when ctx ends, an *ApplyError when Apply fails, or a wrapped error from the log or +// from c.Offset. Run retries nothing and starts no goroutine. +// +// Run returns an error wrapping ErrAheadOfLog when c's offset is above the log's head, one +// wrapping both ErrBehindLog and chainlog.ErrTruncated when the log no longer holds the next +// offset, and one wrapping ErrOffsetGap when the log returns an offset other than the next one. +func Run(ctx context.Context, log chainlog.Log, c Consumer) error { + if err := ctx.Err(); err != nil { + return err + } + off, err := c.Offset(ctx) + if err != nil { + return orDone(ctx, fmt.Errorf("consumer: offset: %w", err)) + } + head, err := log.Head(ctx) + if err != nil { + return orDone(ctx, fmt.Errorf("consumer: log head: %w", err)) + } + if off > head { + return fmt.Errorf("%w: offset %d, head %d", ErrAheadOfLog, off, head) + } + next := off + 1 + for { + if err := ctx.Err(); err != nil { + return err + } + es, err := log.Read(ctx, next, readBatch) + if err != nil { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + if errors.Is(err, chainlog.ErrTruncated) { + return fmt.Errorf("%w: next offset %d: %w", ErrBehindLog, next, err) + } + return fmt.Errorf("consumer: read from offset %d: %w", next, err) + } + if len(es) == 0 { + if err := log.Wait(ctx, next-1); err != nil { + return orDone(ctx, fmt.Errorf("consumer: wait after offset %d: %w", next-1, err)) + } + continue + } + for i := range es { + e := &es[i] + if e.Offset != next { + return fmt.Errorf("%w: got offset %d, want %d", ErrOffsetGap, e.Offset, next) + } + if err := ctx.Err(); err != nil { + return err + } + if err := c.Apply(ctx, *e); err != nil { + return &ApplyError{Offset: e.Offset, Kind: e.Kind, Err: err} + } + next++ + } + } +} + +// orDone returns ctx.Err() if ctx has ended, and err otherwise. +func orDone(ctx context.Context, err error) error { + if ctxErr := ctx.Err(); ctxErr != nil { + return ctxErr + } + return err +} diff --git a/consumer/doc.go b/consumer/doc.go new file mode 100644 index 0000000..26d138e --- /dev/null +++ b/consumer/doc.go @@ -0,0 +1,7 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +// Package consumer applies a canonical log to state: the Consumer contract, Run, which feeds a +// consumer the log from its committed offset, and Memory, a consumer for in-memory state machines +// that cannot undo their writes. +package consumer diff --git a/consumer/helpers_test.go b/consumer/helpers_test.go new file mode 100644 index 0000000..7c085cd --- /dev/null +++ b/consumer/helpers_test.go @@ -0,0 +1,206 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package consumer_test + +import ( + "context" + "errors" + "slices" + "testing" + "time" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/consumer" + "github.com/hmdsefi/evmsync/internal/fakechain" +) + +// recorder is a durable test consumer: its fields survive across Run calls, as a database's +// would. Apply records every call in calls; unless e.Offset is failAt, it then commits: it appends +// e.Offset to applied and sets off. At failAt it returns failErr and commits nothing. It is not +// safe for concurrent use: read its fields only after Run returns. +type recorder struct { + off chainlog.Offset + applied []chainlog.Offset + calls []chainlog.Offset + failAt chainlog.Offset + failErr error +} + +func (r *recorder) Offset(context.Context) (chainlog.Offset, error) { return r.off, nil } + +func (r *recorder) Apply(_ context.Context, e chainlog.Entry) error { + r.calls = append(r.calls, e.Offset) + if r.failAt != 0 && e.Offset == r.failAt { + return r.failErr + } + r.applied = append(r.applied, e.Offset) + r.off = e.Offset + return nil +} + +// canceler wraps a consumer and calls cancel right after its Apply of offset at succeeds. +type canceler struct { + consumer.Consumer + at chainlog.Offset + cancel context.CancelFunc +} + +func (c *canceler) Apply(ctx context.Context, e chainlog.Entry) error { + err := c.Consumer.Apply(ctx, e) + if err == nil && e.Offset == c.at { + c.cancel() + } + return err +} + +// countingLog wraps a log and records, for each Read, its max in reads and the number of entries +// it returned in got, and the after of each Wait in waits. When waiting is not nil, each Wait +// sends its after on it before waiting, so a test can act once Run is at the tail. It is not safe +// for concurrent use: read its fields only after Run returns, and use waiting to act while Run +// is at the tail. +type countingLog struct { + chainlog.Log + reads []int + got []int + waits []chainlog.Offset + waiting chan chainlog.Offset +} + +func (l *countingLog) Read(ctx context.Context, from chainlog.Offset, max int) ([]chainlog.Entry, error) { + es, err := l.Log.Read(ctx, from, max) + l.reads = append(l.reads, max) + l.got = append(l.got, len(es)) + return es, err +} + +func (l *countingLog) Wait(ctx context.Context, after chainlog.Offset) error { + l.waits = append(l.waits, after) + if ctx.Done() == nil { + return errors.New("countingLog: Wait got a context that never ends") + } + if l.waiting != nil { + select { + case l.waiting <- after: + case <-ctx.Done(): + return ctx.Err() + } + } + return l.Log.Wait(ctx, after) +} + +// refs returns a fake chain with seed 1 and default parameters, mined to block n, and the refs of +// its blocks 0 to n: refs[0] is genesis G and refs[i] is block Ai. +func refs(tb testing.TB, n int) (*fakechain.Chain, []chain.BlockRef) { + tb.Helper() + c := fakechain.New(1, fakechain.Params{}) + return c, append([]chain.BlockRef{c.Genesis().Ref}, c.Mine(n)...) +} + +// blk returns a non-final block entry for x with x's logs that match fakechain.DefaultFilter. +func blk(c *fakechain.Chain, x chain.BlockRef) chainlog.Entry { + b, ok := c.Block(x.Hash, fakechain.DefaultFilter) + if !ok { + panic("consumer_test: unknown block " + x.String()) + } + return chainlog.Entry{Kind: chainlog.KindBlock, Block: b} +} + +// fblk returns blk(c, x) marked final. +func fblk(c *fakechain.Chain, x chain.BlockRef) chainlog.Entry { + e := blk(c, x) + e.Final = true + return e +} + +// rev returns a Revert entry to x. +func rev(x chain.BlockRef) chainlog.Entry { + return chainlog.Entry{Kind: chainlog.KindRevert, To: x} +} + +// fin returns a Finalize entry to x. +func fin(x chain.BlockRef) chainlog.Entry { + return chainlog.Entry{Kind: chainlog.KindFinalize, To: x} +} + +// chainEntries returns fin(G), blk(A1), …, blk(A): n entries over rs from refs. +func chainEntries(c *fakechain.Chain, rs []chain.BlockRef, n int) []chainlog.Entry { + es := []chainlog.Entry{fin(rs[0])} + for _, r := range rs[1:n] { + es = append(es, blk(c, r)) + } + return es +} + +// newLog returns a chainlog.Memory with chain ID 31337 and window cap 100,000 holding entries. +func newLog(tb testing.TB, entries ...chainlog.Entry) *chainlog.Memory { + tb.Helper() + l := chainlog.NewMemory(chainlog.Meta{ChainID: 31337}, 100_000) + if _, err := l.Append(tb.Context(), entries...); err != nil { + tb.Fatalf("Append: %v", err) + } + return l +} + +// offsets returns the offsets from to to, inclusive. +func offsets(from, to chainlog.Offset) []chainlog.Offset { + var out []chainlog.Offset + for o := from; o <= to; o++ { + out = append(out, o) + } + return out +} + +// trace counts the calls of a trace state's apply and clone functions. +type trace struct{ applies, clones int } + +// newTrace returns a trace and the trace state's functions: apply appends the block's number, +// clone copies the slice, and both count their calls in the trace. +func newTrace() (*trace, func([]uint64, chain.Block) []uint64, func([]uint64) []uint64) { + tr := &trace{} + apply := func(s []uint64, b chain.Block) []uint64 { + tr.applies++ + return append(s, b.Ref.Number) + } + clone := func(s []uint64) []uint64 { + tr.clones++ + return slices.Clone(s) + } + return tr, apply, clone +} + +// applyAll applies es to m with consecutive offsets after m's offset and fails the test on an +// error. +func applyAll[S any](tb testing.TB, m *consumer.Memory[S], es ...chainlog.Entry) { + tb.Helper() + for _, e := range es { + off, _ := m.Offset(tb.Context()) + e.Offset = off + 1 + if err := m.Apply(tb.Context(), e); err != nil { + tb.Fatalf("Apply offset %d (%s): %v", e.Offset, e.Kind, err) + } + } +} + +// bounded returns a context that ends 30 seconds after the test starts it, so a Run that should +// return but waits at the tail fails the test with context.DeadlineExceeded instead of hanging. +func bounded(t *testing.T) context.Context { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + t.Cleanup(cancel) + return ctx +} + +// waitFor returns the after of the next Wait call on log, and fails the test if Run, which sends +// its result on done, returns first. +func waitFor(t *testing.T, log *countingLog, done <-chan error) chainlog.Offset { + t.Helper() + select { + case after := <-log.waiting: + return after + case err := <-done: + t.Fatalf("Run returned %v before it waited at the tail", err) + return 0 + } +} diff --git a/consumer/memory.go b/consumer/memory.go new file mode 100644 index 0000000..a406f9b --- /dev/null +++ b/consumer/memory.go @@ -0,0 +1,189 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package consumer + +import ( + "context" + "fmt" + "sort" + "sync" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +// Memory is a Consumer for in-memory state machines that cannot undo their writes. It keeps a +// snapshot of the state at the latest Finalize and the pending blocks since; Revert restores the +// snapshot and re-applies the pending blocks up to the target. Its committed offset lives in +// memory: a new Memory starts at offset 0 and replays the log from offset 1. +// +// Memory is safe for concurrent use: State and Offset may be called while Run drives Apply. +type Memory[S any] struct { + apply func(s S, b chain.Block) S + clone func(s S) S + + mu sync.Mutex + cur S // the current state; guarded by mu + // snap is the state at base, valid only while blocks are pending; guarded by mu. + snap S + // pending[first:] are the non-final blocks applied since base, ascending by number; the + // dead prefix pending[:first] is zeroed. Guarded by mu. + pending []chain.Block + first int // the index of the oldest live pending block; guarded by mu + base chain.BlockRef // the ref below the pending blocks; guarded by mu + off chainlog.Offset // the committed offset; guarded by mu +} + +var _ Consumer = (*Memory[struct{}])(nil) + +// NewMemory returns a Memory with state initial and committed offset 0. +// +// apply returns the state after applying block b to s. It must be deterministic and may modify +// s and return it: Memory owns every S it passes. clone returns a deep copy that shares no +// mutable memory with its argument. NewMemory panics if apply or clone is nil. Memory never +// recovers a panic from apply or clone; after one, the Memory must not be used. +func NewMemory[S any](initial S, apply func(s S, b chain.Block) S, clone func(s S) S) *Memory[S] { + if apply == nil { + panic("evmsync: consumer.NewMemory: nil apply function") + } + if clone == nil { + panic("evmsync: consumer.NewMemory: nil clone function") + } + return &Memory[S]{apply: apply, clone: clone, cur: initial} +} + +// Offset returns the committed offset. It never fails. +func (m *Memory[S]) Offset(context.Context) (chainlog.Offset, error) { + m.mu.Lock() + defer m.mu.Unlock() + return m.off, nil +} + +// Apply implements Consumer. It returns an error wrapping ErrBadEntry, and changes nothing, when +// e does not fit the state: its offset is not the committed offset plus one, a Revert targets +// neither the base nor a pending block, a Finalize conflicts with a pending block, or its kind +// is unknown. +func (m *Memory[S]) Apply(_ context.Context, e chainlog.Entry) error { + m.mu.Lock() + defer m.mu.Unlock() + if e.Offset != m.off+1 { + return fmt.Errorf("%w: offset %d, want %d", ErrBadEntry, e.Offset, m.off+1) + } + switch e.Kind { + case chainlog.KindBlock: + m.block(e.Block, e.Final) + case chainlog.KindRevert: + i, ok := m.revertTarget(e.To) + if !ok { + return fmt.Errorf("%w: revert target %d %s is not an unfinalized applied block", ErrBadEntry, e.To.Number, e.To.Hash.Hex()) + } + m.revert(i) + case chainlog.KindFinalize: + live := m.live() + k := sort.Search(len(live), func(i int) bool { return live[i].Ref.Number > e.To.Number }) + if k > 0 && live[k-1].Ref.Number == e.To.Number && live[k-1].Ref.Hash != e.To.Hash { + return fmt.Errorf("%w: finalize %d %s conflicts with applied block %s", ErrBadEntry, e.To.Number, e.To.Hash.Hex(), live[k-1].Ref.Hash.Hex()) + } + m.finalize(k, e.To) + default: + return fmt.Errorf("%w: unknown kind %d", ErrBadEntry, e.Kind) + } + m.off = e.Offset + return nil +} + +// State returns a clone of the current state. The caller owns the result. +func (m *Memory[S]) State() S { + m.mu.Lock() + defer m.mu.Unlock() + return m.clone(m.cur) +} + +// block applies b. A final block with nothing pending becomes the base without a snapshot; with +// blocks pending, which a log never sends, it is applied as a non-final block. +func (m *Memory[S]) block(b chain.Block, final bool) { + switch { + case len(m.live()) == 0 && final: + m.cur = m.apply(m.cur, b) + m.base = b.Ref + case len(m.live()) == 0: + m.snap = m.clone(m.cur) + m.cur = m.apply(m.cur, b) + m.pending = append(m.pending, b) + default: + m.cur = m.apply(m.cur, b) + m.pending = append(m.pending, b) + } +} + +// revertTarget returns the index in the pending blocks of the block with to's number and hash, +// or -1 when to is the base, and false when it is neither. +func (m *Memory[S]) revertTarget(to chain.BlockRef) (int, bool) { + if to.Number == m.base.Number && to.Hash == m.base.Hash { + return -1, true + } + live := m.live() + for i := len(live) - 1; i >= 0 && live[i].Ref.Number >= to.Number; i-- { + if live[i].Ref.Number == to.Number { + return i, live[i].Ref.Hash == to.Hash + } + } + return -1, false +} + +// revert makes the state the one at pending block i, or at the base when i is -1. +func (m *Memory[S]) revert(i int) { + live := m.live() + switch { + case i == len(live)-1: + // The target is the newest block, or the base with nothing pending: nothing changes. + case i < 0: + m.cur = m.snap + m.clearPending() + default: + m.cur = m.clone(m.snap) + for _, b := range live[:i+1] { + m.cur = m.apply(m.cur, b) + } + end := m.first + i + 1 + clear(m.pending[end:]) + m.pending = m.pending[:end] + } +} + +// finalize makes the first k pending blocks final and moves the base to to when it is not below +// the base. +func (m *Memory[S]) finalize(k int, to chain.BlockRef) { + if live := m.live(); k == len(live) { + m.clearPending() + } else { + for _, b := range live[:k] { + m.snap = m.apply(m.snap, b) + } + clear(live[:k]) + m.first += k + // Copy the live blocks down once the dead prefix fills half the array, so appends reuse it. + if m.first > cap(m.pending)/2 { + n := copy(m.pending, m.live()) + clear(m.pending[n:]) + m.pending = m.pending[:n] + m.first = 0 + } + } + if to.Number >= m.base.Number { + m.base = to + } +} + +// live returns the pending blocks. +func (m *Memory[S]) live() []chain.Block { return m.pending[m.first:] } + +// clearPending drops every pending block and the snapshot. +func (m *Memory[S]) clearPending() { + var zero S + m.snap = zero + clear(m.pending) + m.pending = m.pending[:0] + m.first = 0 +} diff --git a/consumer/memory_internal_test.go b/consumer/memory_internal_test.go new file mode 100644 index 0000000..7ab2392 --- /dev/null +++ b/consumer/memory_internal_test.go @@ -0,0 +1,85 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package consumer + +import ( + "encoding/binary" + "slices" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +// TestMemoryPendingBounded runs 10,000 blocks with finality 5 blocks behind the tip and a +// depth-2 revert every 100 blocks. The pending array stays small and every slot outside the live +// blocks is zero, so retired blocks and their logs can be collected. A Finalize of every pending +// block drops the snapshot. +func TestMemoryPendingBounded(t *testing.T) { + m := NewMemory([]uint64{}, + func(s []uint64, b chain.Block) []uint64 { return append(s, b.Ref.Number) }, + slices.Clone[[]uint64]) + ref := func(n uint64) chain.BlockRef { + var h common.Hash + binary.BigEndian.PutUint64(h[24:], n+1) + return chain.BlockRef{Number: n, Hash: h} + } + block := func(n uint64) chainlog.Entry { + r := ref(n) + logs := []chain.Log{{BlockNumber: n, BlockHash: r.Hash, Data: []byte{1}}} + return chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: r, Logs: logs}} + } + apply := func(e chainlog.Entry) { + t.Helper() + e.Offset = m.off + 1 + if err := m.Apply(t.Context(), e); err != nil { + t.Fatalf("Apply offset %d (%s): %v", e.Offset, e.Kind, err) + } + } + checkSlots := func(n uint64) { + t.Helper() + if c := cap(m.pending); c > 32 { + t.Fatalf("block %d: pending capacity %d, want at most 32", n, c) + } + full := m.pending[:cap(m.pending)] + for i := range full { + if i >= m.first && i < len(m.pending) { + continue + } + if full[i].Ref != (chain.BlockRef{}) || full[i].Logs != nil { + t.Fatalf("block %d: slot %d outside the live blocks [%d, %d) holds block %d", n, i, m.first, len(m.pending), full[i].Ref.Number) + } + } + } + + apply(chainlog.Entry{Kind: chainlog.KindFinalize, To: ref(0)}) + for n := uint64(1); n <= 10_000; n++ { + apply(block(n)) + if n%100 == 0 { + apply(chainlog.Entry{Kind: chainlog.KindRevert, To: ref(n - 2)}) + checkSlots(n) + apply(block(n - 1)) + apply(block(n)) + } + if n > 5 { + apply(chainlog.Entry{Kind: chainlog.KindFinalize, To: ref(n - 5)}) + } + checkSlots(n) + } + want := make([]uint64, 10_000) + for i := range want { + want[i] = uint64(i) + 1 + } + if got := m.State(); !slices.Equal(got, want) { + t.Fatalf("State() has %d numbers, want 1 to 10000", len(got)) + } + + apply(chainlog.Entry{Kind: chainlog.KindFinalize, To: ref(10_000)}) + checkSlots(10_000) + if len(m.pending) != 0 || m.snap != nil { + t.Errorf("after finalizing every block: %d pending, snapshot %v; want none and nil", len(m.pending), m.snap) + } +} diff --git a/consumer/memory_test.go b/consumer/memory_test.go new file mode 100644 index 0000000..81ece5d --- /dev/null +++ b/consumer/memory_test.go @@ -0,0 +1,385 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package consumer_test + +import ( + "context" + "errors" + "fmt" + "maps" + "slices" + "sync" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/consumer" +) + +// newTraceMemory returns a Memory over the trace state, starting empty, and its trace. +func newTraceMemory() (*consumer.Memory[[]uint64], *trace) { + tr, apply, clone := newTrace() + return consumer.NewMemory([]uint64{}, apply, clone), tr +} + +// check fails the test unless the trace counts applies and clones, and State returns want with +// exactly one clone and no apply. It then takes that clone off the count, so counts exclude +// State calls. +func check(t *testing.T, m *consumer.Memory[[]uint64], tr *trace, want []uint64, applies, clones int) { + t.Helper() + if tr.applies != applies || tr.clones != clones { + t.Errorf("applies %d, clones %d; want %d, %d", tr.applies, tr.clones, applies, clones) + } + got := m.State() + if !slices.Equal(got, want) { + t.Errorf("State() = %v, want %v", got, want) + } + if tr.applies != applies || tr.clones != clones+1 { + t.Errorf("State made %d applies and %d clones, want 0 and 1", tr.applies-applies, tr.clones-clones) + } + tr.clones-- +} + +// upTo returns the numbers 1 to n. +func upTo(n uint64) []uint64 { + var out []uint64 + for i := uint64(1); i <= n; i++ { + out = append(out, i) + } + return out +} + +func TestMemoryBlocks(t *testing.T) { + c, rs := refs(t, 3) + m, tr := newTraceMemory() + if tr.applies != 0 || tr.clones != 0 { + t.Fatalf("NewMemory made %d applies and %d clones, want none", tr.applies, tr.clones) + } + applyAll(t, m, fin(rs[0]), blk(c, rs[1]), blk(c, rs[2]), blk(c, rs[3])) + check(t, m, tr, []uint64{1, 2, 3}, 3, 1) + if off, err := m.Offset(t.Context()); off != 4 || err != nil { + t.Errorf("Offset() = %d, %v; want 4, nil", off, err) + } +} + +func TestMemoryRevert(t *testing.T) { + c, a := refs(t, 5) + m, tr := newTraceMemory() + applyAll(t, m, fin(a[0]), blk(c, a[1]), blk(c, a[2]), blk(c, a[3]), blk(c, a[4]), blk(c, a[5])) + check(t, m, tr, []uint64{1, 2, 3, 4, 5}, 5, 1) + applyAll(t, m, rev(a[2])) + check(t, m, tr, []uint64{1, 2}, 5+2, 2) + + b := c.Reorg(3, 2) // B3, B4 on A2 + applyAll(t, m, blk(c, b[0]), blk(c, b[1])) + check(t, m, tr, []uint64{1, 2, 3, 4}, 9, 2) + + applyAll(t, m, rev(a[0])) + check(t, m, tr, nil, 9, 2) + + c1 := c.Reorg(4, 1)[0] + applyAll(t, m, blk(c, c1)) + check(t, m, tr, []uint64{1}, 10, 3) + applyAll(t, m, rev(c1)) + check(t, m, tr, []uint64{1}, 10, 3) + if off, _ := m.Offset(t.Context()); off != 12 { + t.Errorf("Offset() = %d, want 12", off) + } +} + +// TestMemoryRevertDropsPending checks that a partial revert drops the blocks above its target: +// a later revert re-applies only the blocks of the new branch. +func TestMemoryRevertDropsPending(t *testing.T) { + c, a := refs(t, 5) + m, tr := newTraceMemory() + applyAll(t, m, fin(a[0]), blk(c, a[1]), blk(c, a[2]), blk(c, a[3]), blk(c, a[4]), blk(c, a[5]), rev(a[2])) + b := c.Reorg(3, 2) // B3, B4 on A2 + applyAll(t, m, blk(c, b[0]), blk(c, b[1]), rev(b[0])) + check(t, m, tr, []uint64{1, 2, 3}, 5+2+2+3, 3) +} + +// TestMemoryRevertToZeroBase reverts to the zero base while a block numbered 0 is pending. +func TestMemoryRevertToZeroBase(t *testing.T) { + c, a := refs(t, 0) + m, tr := newTraceMemory() + applyAll(t, m, blk(c, a[0]), rev(chain.BlockRef{})) + check(t, m, tr, nil, 1, 1) +} + +// TestMemoryFinalBlockWhilePending applies a final block while blocks are pending, which a log +// never sends: Memory treats it as non-final. +func TestMemoryFinalBlockWhilePending(t *testing.T) { + c, a := refs(t, 2) + m, tr := newTraceMemory() + applyAll(t, m, fin(a[0]), blk(c, a[1]), fblk(c, a[2])) + check(t, m, tr, []uint64{1, 2}, 2, 1) + applyAll(t, m, rev(a[2])) // the newest pending block, not the base + check(t, m, tr, []uint64{1, 2}, 2, 1) + applyAll(t, m, rev(a[1])) + check(t, m, tr, []uint64{1}, 3, 2) + applyAll(t, m, rev(a[0])) // G is still the base + check(t, m, tr, nil, 3, 2) +} + +// TestMemoryRevertInPlaceState reverts a state that apply changes in place, so a snapshot that +// shared memory with the current state would show the reverted blocks. +func TestMemoryRevertInPlaceState(t *testing.T) { + c, a := refs(t, 3) + apply := func(s map[uint64]bool, b chain.Block) map[uint64]bool { + s[b.Ref.Number] = true + return s + } + m := consumer.NewMemory(map[uint64]bool{}, apply, maps.Clone) + applyAll(t, m, fin(a[0]), blk(c, a[1]), blk(c, a[2]), blk(c, a[3]), rev(a[1])) + if got := slices.Sorted(maps.Keys(m.State())); !slices.Equal(got, []uint64{1}) { + t.Errorf("after rev(A1): State() keys %v, want [1]", got) + } + applyAll(t, m, rev(a[0])) + if got := m.State(); len(got) != 0 { + t.Errorf("after rev(G): State() = %v, want no keys", got) + } +} + +func TestMemoryRevertToBaseWithoutPending(t *testing.T) { + _, rs := refs(t, 0) + m, tr := newTraceMemory() + applyAll(t, m, fin(rs[0]), rev(rs[0])) + check(t, m, tr, nil, 0, 0) +} + +func TestMemoryFinalize(t *testing.T) { + c, a := refs(t, 5) + m, tr := newTraceMemory() + applyAll(t, m, fin(a[0]), blk(c, a[1]), blk(c, a[2]), blk(c, a[3]), blk(c, a[4]), blk(c, a[5])) + applyAll(t, m, fin(a[3])) + check(t, m, tr, []uint64{1, 2, 3, 4, 5}, 5+3, 1) + + applyAll(t, m, rev(a[3])) // the base: the snapshot is moved + check(t, m, tr, []uint64{1, 2, 3}, 8, 1) + + applyAll(t, m, blk(c, a[4])) + check(t, m, tr, []uint64{1, 2, 3, 4}, 9, 2) + applyAll(t, m, fin(a[4])) + check(t, m, tr, []uint64{1, 2, 3, 4}, 9, 2) + + // fin(A4) left no pending block, so A4 is the base and a revert to it changes nothing. + applyAll(t, m, rev(a[4])) + check(t, m, tr, []uint64{1, 2, 3, 4}, 9, 2) +} + +func TestMemoryFinalizeThenPartialRevert(t *testing.T) { + c, a := refs(t, 5) + m, tr := newTraceMemory() + applyAll(t, m, fin(a[0]), blk(c, a[1]), blk(c, a[2]), blk(c, a[3]), blk(c, a[4]), blk(c, a[5]), fin(a[2])) + check(t, m, tr, []uint64{1, 2, 3, 4, 5}, 5+2, 1) + applyAll(t, m, rev(a[3])) // the snapshot holds A1 and A2; A3 is re-applied + check(t, m, tr, []uint64{1, 2, 3}, 8, 2) + applyAll(t, m, rev(a[2])) + check(t, m, tr, []uint64{1, 2}, 8, 2) +} + +// TestMemoryMatchesNumberAndHash checks that a Revert or Finalize target is matched by its +// number and hash only. +func TestMemoryMatchesNumberAndHash(t *testing.T) { + c, a := refs(t, 3) + m, tr := newTraceMemory() + applyAll(t, m, fin(a[0]), blk(c, a[1]), blk(c, a[2]), blk(c, a[3])) + applyAll(t, m, fin(chain.BlockRef{Number: 1, Hash: a[1].Hash}), rev(chain.BlockRef{Number: 2, Hash: a[2].Hash})) + check(t, m, tr, []uint64{1, 2}, 3+1+1, 2) + applyAll(t, m, rev(a[1])) // the base, which holds only number and hash + check(t, m, tr, []uint64{1}, 5, 2) +} + +func TestMemoryRevertCost(t *testing.T) { + c, a := refs(t, 64) + m, tr := newTraceMemory() + applyAll(t, m, chainEntries(c, a, 65)...) + check(t, m, tr, upTo(64), 64, 1) + applyAll(t, m, rev(a[61])) + check(t, m, tr, upTo(61), 64+61, 2) +} + +func TestMemoryFinalBlocks(t *testing.T) { + c, a := refs(t, 10) + m, tr := newTraceMemory() + applyAll(t, m, fin(a[0]), fblk(c, a[2]), fblk(c, a[5])) + check(t, m, tr, []uint64{2, 5}, 2, 0) + applyAll(t, m, fin(a[9]), blk(c, a[10])) + check(t, m, tr, []uint64{2, 5, 10}, 3, 1) + applyAll(t, m, rev(a[9])) + check(t, m, tr, []uint64{2, 5}, 3, 1) +} + +func TestMemoryFinalBlockIsBase(t *testing.T) { + c, a := refs(t, 6) + m, tr := newTraceMemory() + applyAll(t, m, fin(a[0]), fblk(c, a[5]), blk(c, a[6]), rev(a[5])) + check(t, m, tr, []uint64{5}, 2, 1) + // A Finalize below the base leaves it: A5 stays the revert target. + applyAll(t, m, fin(a[0]), blk(c, a[6]), rev(a[5])) + check(t, m, tr, []uint64{5}, 3, 2) +} + +// TestMemoryLongRun keeps the finalized block 5 below the tip over 300 blocks, so the pending +// blocks' backing array is reused many times, then reverts into the window. +func TestMemoryLongRun(t *testing.T) { + c, a := refs(t, 300) + m, tr := newTraceMemory() + applyAll(t, m, fin(a[0])) + for n := 1; n <= 300; n++ { + applyAll(t, m, blk(c, a[n])) + if n > 5 && n%3 == 0 { + applyAll(t, m, fin(a[n-5])) + } + } + check(t, m, tr, upTo(300), 300+295, 1) + applyAll(t, m, rev(a[296])) // A296 is the first pending block: the snapshot holds A1 to A295 + check(t, m, tr, upTo(296), 596, 2) + applyAll(t, m, rev(a[295])) // the base + check(t, m, tr, upTo(295), 596, 2) +} + +func TestMemoryRejects(t *testing.T) { + c, a := refs(t, 3) + other := common.Hash{0xaa} + for _, tt := range []struct { + name string + e chainlog.Entry + want string + }{ + {"skipped offset", entryAt(blk(c, a[3]), 5), "offset 5, want 4"}, + {"repeated offset", entryAt(blk(c, a[3]), 3), "offset 3, want 4"}, + {"revert to another hash", entryAt(rev(chain.BlockRef{Number: 1, Hash: other}), 4), + fmt.Sprintf("revert target 1 %s is not an unfinalized applied block", other.Hex())}, + {"revert to a block never applied", entryAt(rev(a[3]), 4), + fmt.Sprintf("revert target 3 %s is not an unfinalized applied block", a[3].Hash.Hex())}, + {"revert below the base", entryAt(rev(chain.BlockRef{Number: 0, Hash: other}), 4), + fmt.Sprintf("revert target 0 %s is not an unfinalized applied block", other.Hex())}, + {"finalize a conflicting block", entryAt(fin(chain.BlockRef{Number: 2, Hash: other}), 4), + fmt.Sprintf("finalize 2 %s conflicts with applied block %s", other.Hex(), a[2].Hash.Hex())}, + {"unknown kind", chainlog.Entry{Offset: 4, Kind: 7}, "unknown kind 7"}, + } { + t.Run(tt.name, func(t *testing.T) { + m, tr := newTraceMemory() + applyAll(t, m, fin(a[0]), blk(c, a[1]), blk(c, a[2])) + err := m.Apply(t.Context(), tt.e) + if !errors.Is(err, consumer.ErrBadEntry) { + t.Fatalf("Apply = %v, want an error that Is ErrBadEntry", err) + } + if got, want := err.Error(), "consumer: entry does not fit the state: "+tt.want; got != want { + t.Errorf("Apply = %q, want %q", got, want) + } + check(t, m, tr, []uint64{1, 2}, 2, 1) + if off, _ := m.Offset(t.Context()); off != 3 { + t.Errorf("Offset() = %d, want 3", off) + } + // The rejection left the pending blocks: a revert into them still works. + applyAll(t, m, rev(a[1])) + check(t, m, tr, []uint64{1}, 3, 2) + }) + } +} + +// entryAt returns e with offset off. +func entryAt(e chainlog.Entry, off chainlog.Offset) chainlog.Entry { + e.Offset = off + return e +} + +func TestMemoryConcurrentState(t *testing.T) { + const n = 10_000 + c, a := refs(t, n) + log := newLog(t, chainEntries(c, a, n+1)[1:]...) // blk(A1) … blk(A10000) + apply := func(s map[uint64]uint64, b chain.Block) map[uint64]uint64 { + s[b.Ref.Number] = b.Ref.Number + return s + } + // A range loop, unlike maps.Clone, makes the race detector see clone's reads. + clone := func(s map[uint64]uint64) map[uint64]uint64 { + out := make(map[uint64]uint64, len(s)) + for k, v := range s { + out[k] = v + } + return out + } + m := consumer.NewMemory(map[uint64]uint64{}, apply, clone) + + ctx, cancel := context.WithCancel(bounded(t)) + defer cancel() + started := make(chan struct{}) + watched := &firstApply{Consumer: &canceler{Consumer: m, at: n, cancel: cancel}, started: started} + done := make(chan error, 1) + go func() { done <- consumer.Run(ctx, log, watched) }() + select { // call State while Run applies, not before + case <-started: + case err := <-done: + t.Fatalf("Run = %v before its first Apply", err) + } + for i := range 1_000 { + s := m.State() + s[uint64(n+1+i)] = 0 // the caller owns the result + if _, err := m.Offset(ctx); err != nil { + t.Errorf("Offset: %v", err) + } + } + if err := <-done; err != context.Canceled { + t.Fatalf("Run = %v, want context.Canceled", err) + } + + got := m.State() + if len(got) != n { + t.Errorf("State() has %d keys, want %d", len(got), n) + } + for k := uint64(1); k <= n; k++ { + if v, ok := got[k]; !ok || v != k { + t.Fatalf("State()[%d] = %d, %t; want %d, true", k, v, ok, k) + } + } +} + +// firstApply wraps a consumer and closes started after its first successful Apply. +type firstApply struct { + consumer.Consumer + started chan struct{} + once sync.Once +} + +func (s *firstApply) Apply(ctx context.Context, e chainlog.Entry) error { + err := s.Consumer.Apply(ctx, e) + if err == nil { + s.once.Do(func() { close(s.started) }) + } + return err +} + +func TestNewMemoryPanics(t *testing.T) { + _, apply, clone := newTrace() + for _, tt := range []struct { + name string + f func() + want string + }{ + {"nil apply", func() { consumer.NewMemory([]uint64{}, nil, clone) }, "evmsync: consumer.NewMemory: nil apply function"}, + {"nil clone", func() { consumer.NewMemory([]uint64{}, apply, nil) }, "evmsync: consumer.NewMemory: nil clone function"}, + } { + t.Run(tt.name, func(t *testing.T) { + if got := panicText(tt.f); got != tt.want { + t.Errorf("panic %q, want %q", got, tt.want) + } + }) + } +} + +// panicText returns the value f panics with, formatted with %v, or "" if it returns. +func panicText(f func()) (text string) { + defer func() { + if r := recover(); r != nil { + text = fmt.Sprint(r) + } + }() + f() + return "" +} diff --git a/consumer/property_test.go b/consumer/property_test.go new file mode 100644 index 0000000..cbae10e --- /dev/null +++ b/consumer/property_test.go @@ -0,0 +1,239 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package consumer_test + +import ( + "context" + "errors" + "slices" + "strconv" + "sync/atomic" + "testing" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/consumer" + "github.com/hmdsefi/evmsync/internal/fakechain" + "github.com/hmdsefi/evmsync/tracker" +) + +// trackerRun drives a tracker over a faulty fake chain for seed, as the tracker's mutual +// consistency property does: 300 iterations of Step, Head of Latest and Finalize of Finalized, +// tolerating only not-found answers. It appends every result to a log and calls onEntries with +// each non-empty result, offsets set, right after its Append. It returns the log. +func trackerRun(t *testing.T, seed uint64, onEntries func([]chainlog.Entry)) *chainlog.Memory { + t.Helper() + ctx := t.Context() + c := fakechain.New(seed, fakechain.Params{}) + src := fakechain.NewSource(c, seed, fakechain.Faults{ + StaleHeadPPM: 100_000, + StaleOrphanPPM: 100_000, + DropOrphans: seed%2 == 0, + InterleavePPM: 100_000, + }) + lg := chainlog.NewMemory(c.Meta(fakechain.DefaultFilter), tracker.DefaultWindowCap) + tr := tracker.New(src, tracker.Config{Filter: fakechain.DefaultFilter}, + chainlog.NewState(tracker.DefaultWindowCap)) + add := func(es []chainlog.Entry, err error) { + t.Helper() + if err != nil { + if !errors.Is(err, chain.ErrNotFound) { + t.Fatalf("seed %d: %v", seed, err) + } + return + } + if len(es) == 0 { + return + } + last, err := lg.Append(ctx, es...) + if err != nil { + t.Fatalf("seed %d: Append: %v", seed, err) + } + for i := range es { + es[i].Offset = last - chainlog.Offset(len(es)-1-i) //nolint:gosec // i < len(es) + } + if onEntries != nil { + onEntries(es) + } + } + add(tr.Finalize(ctx, c.Genesis().Ref)) + for range 300 { + src.Do(func(c *fakechain.Chain) { c.Step() }) + h, err := src.Latest(ctx) + if err != nil { + t.Fatal(err) + } + add(tr.Head(ctx, h)) + f, err := src.Finalized(ctx) + if err != nil { + t.Fatal(err) + } + add(tr.Finalize(ctx, f.Ref)) + } + return lg +} + +// pendingModel tracks what Memory keeps unfinalized, to count the clones Memory's contract +// allows: one for a non-final block with nothing pending, one for a revert that keeps at least one +// pending block and removes at least one, none otherwise. +type pendingModel struct { + pending []chain.BlockRef + base chain.BlockRef + clones int +} + +func (m *pendingModel) apply(t *testing.T, e chainlog.Entry) { + t.Helper() + switch e.Kind { + case chainlog.KindBlock: + switch { + case len(m.pending) == 0 && e.Final: + m.base = e.Block.Ref + case len(m.pending) == 0: + m.clones++ + m.pending = append(m.pending, e.Block.Ref) + default: + m.pending = append(m.pending, e.Block.Ref) + } + case chainlog.KindRevert: + if e.To == m.base { + m.pending = m.pending[:0] + return + } + i := slices.Index(m.pending, e.To) + if i < 0 { + t.Fatalf("offset %d: revert to %s, which is neither pending nor the base", e.Offset, e.To) + } + if i < len(m.pending)-1 { + m.clones++ + } + m.pending = m.pending[:i+1] + case chainlog.KindFinalize: + k := 0 + for k < len(m.pending) && m.pending[k].Number <= e.To.Number { + k++ + } + m.pending = m.pending[k:] + if e.To.Number >= m.base.Number { + m.base = e.To + } + } +} + +// liveNumbers replays entries structurally: a Block pushes its ref, a Revert pops down to its +// target, which must then be the top. It returns the numbers of the live blocks above g. +func liveNumbers(t *testing.T, stack *[]chain.BlockRef, es []chainlog.Entry) []uint64 { + t.Helper() + for _, e := range es { + switch e.Kind { + case chainlog.KindBlock: + *stack = append(*stack, e.Block.Ref) + case chainlog.KindRevert: + s := *stack + for s[len(s)-1].Number > e.To.Number { + s = s[:len(s)-1] + } + if s[len(s)-1] != e.To { + t.Fatalf("offset %d: revert target %s is not on the stack", e.Offset, e.To) + } + *stack = s + } + } + nums := []uint64{} + for _, r := range (*stack)[1:] { + nums = append(nums, r.Number) + } + return nums +} + +func TestMemoryEquivalence(t *testing.T) { + seeds := uint64(200) + if testing.Short() { + seeds = 30 + } + var reverts, partial atomic.Int64 + t.Run("seeds", func(t *testing.T) { + for seed := uint64(1); seed <= seeds; seed++ { + t.Run("seed="+strconv.FormatUint(seed, 10), func(t *testing.T) { + t.Parallel() + r, p := equivalence(t, seed) + reverts.Add(r) + partial.Add(p) + }) + } + }) + if reverts.Load() == 0 || partial.Load() == 0 { + t.Errorf("%d reverts, %d of them partial; the property covers no revert case", reverts.Load(), partial.Load()) + } +} + +// equivalence applies seed's tracker entries to a Memory one by one and checks its state and +// clone count after each. It returns the number of reverts and of partial reverts. +func equivalence(t *testing.T, seed uint64) (reverts, partial int64) { + t.Helper() + tr, apply, clone := newTrace() + m := consumer.NewMemory([]uint64{}, apply, clone) + model := &pendingModel{} + var stack []chain.BlockRef + states := 0 + trackerRun(t, seed, func(es []chainlog.Entry) { + if stack == nil { + stack = []chain.BlockRef{es[0].To} // the first entry finalizes genesis + } + for _, e := range es { + before := model.clones + if err := m.Apply(t.Context(), e); err != nil { + t.Fatalf("seed %d: Apply offset %d (%s): %v", seed, e.Offset, e.Kind, err) + } + model.apply(t, e) + if e.Kind == chainlog.KindRevert { + reverts++ + if model.clones > before { + partial++ + } + } + want := liveNumbers(t, &stack, []chainlog.Entry{e}) + got := m.State() + states++ + if !slices.Equal(got, want) { + t.Fatalf("seed %d offset %d (%s): State() = %v, want %v", seed, e.Offset, e.Kind, got, want) + } + if tr.clones-states != model.clones { + t.Fatalf("seed %d offset %d (%s): %d clones besides State, want %d", + seed, e.Offset, e.Kind, tr.clones-states, model.clones) + } + if off, _ := m.Offset(t.Context()); off != e.Offset { + t.Fatalf("seed %d: Offset() = %d after applying %d", seed, off, e.Offset) + } + } + }) + return reverts, partial +} + +func TestMemoryRestartReplays(t *testing.T) { + lg := trackerRun(t, 7, nil) + head, err := lg.Head(t.Context()) + if err != nil || head < 100 { + t.Fatalf("seed 7 log head %d, %v; want a long log", head, err) + } + runToHead := func() *consumer.Memory[[]uint64] { + _, apply, clone := newTrace() + m := consumer.NewMemory([]uint64{}, apply, clone) + ctx, cancel := context.WithCancel(bounded(t)) + defer cancel() + err := consumer.Run(ctx, lg, &canceler{Consumer: m, at: head, cancel: cancel}) + if !errors.Is(err, context.Canceled) { + t.Fatalf("Run = %v, want context.Canceled at the head", err) + } + if off, _ := m.Offset(t.Context()); off != head { + t.Fatalf("Offset() = %d after Run, want %d", off, head) + } + return m + } + first := runToHead() + second := runToHead() + if a, b := first.State(), second.State(); !slices.Equal(a, b) || len(a) == 0 { + t.Errorf("restarted Memory state %v, first run %v", b, a) + } +} diff --git a/consumer/purity_test.go b/consumer/purity_test.go new file mode 100644 index 0000000..857329a --- /dev/null +++ b/consumer/purity_test.go @@ -0,0 +1,101 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package consumer + +import ( + "go/ast" + "go/parser" + "go/token" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" +) + +// impure returns one finding per go statement in f and per import of time, math/rand, +// math/rand/v2, net or a net/ package, os/exec or testing. +func impure(fset *token.FileSet, f *ast.File) []string { + var out []string + for _, imp := range f.Imports { + path, _ := strconv.Unquote(imp.Path.Value) + switch { + case path == "time", path == "math/rand", path == "math/rand/v2", path == "os/exec", path == "testing", + path == "net", strings.HasPrefix(path, "net/"): + out = append(out, fset.Position(imp.Pos()).String()+": imports "+path) + } + } + ast.Inspect(f, func(n ast.Node) bool { + if g, ok := n.(*ast.GoStmt); ok { + out = append(out, fset.Position(g.Pos()).String()+": go statement") + } + return true + }) + return out +} + +func TestPurity(t *testing.T) { + names, err := filepath.Glob("*.go") + if err != nil { + t.Fatal(err) + } + fset := token.NewFileSet() + for _, name := range names { + if strings.HasSuffix(name, "_test.go") { + continue + } + src, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + f, err := parser.ParseFile(fset, name, src, parser.SkipObjectResolution) + if err != nil { + t.Fatal(err) + } + for _, finding := range impure(fset, f) { + t.Error(finding) + } + } + if !slices.Contains(names, "consumer.go") { + t.Fatalf("checked %v, want consumer.go among them", names) + } +} + +func TestImpureFindings(t *testing.T) { + const src = `package p + +import ( + "fmt" + "math/rand" + _ "math/rand/v2" + _ "net" + "net/http" + _ "netpkg" + _ "os/exec" + _ "testing" + "time" +) + +func f() { go time.Sleep(0); _ = rand.Int; _ = http.Get } +` + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "p.go", src, parser.SkipObjectResolution) + if err != nil { + t.Fatal(err) + } + want := []string{ + "p.go:5:2: imports math/rand", + "p.go:6:2: imports math/rand/v2", + "p.go:7:2: imports net", + "p.go:8:2: imports net/http", + "p.go:10:2: imports os/exec", + "p.go:11:2: imports testing", + "p.go:12:2: imports time", + "p.go:15:12: go statement", + } + if got := impure(fset, f); !slices.Equal(got, want) { + t.Errorf("impure = %q, want %q", got, want) + } +} diff --git a/consumer/run_test.go b/consumer/run_test.go new file mode 100644 index 0000000..55aaace --- /dev/null +++ b/consumer/run_test.go @@ -0,0 +1,556 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package consumer_test + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/consumer" +) + +// tenEntryLog returns the log fin(G), blk(A1) … blk(A9). +func tenEntryLog(t *testing.T) *chainlog.Memory { + t.Helper() + c, rs := refs(t, 9) + return newLog(t, chainEntries(c, rs, 10)...) +} + +// runTo runs Run over log with c wrapped in a canceler at offset at, and returns Run's error. +func runTo(t *testing.T, log chainlog.Log, c consumer.Consumer, at chainlog.Offset) error { + t.Helper() + ctx, cancel := context.WithCancel(bounded(t)) + defer cancel() + return consumer.Run(ctx, log, &canceler{Consumer: c, at: at, cancel: cancel}) +} + +func TestRunFromOffset(t *testing.T) { + r := &recorder{off: 3} + log := &countingLog{Log: tenEntryLog(t)} + err := runTo(t, log, r, 10) + if err != context.Canceled { + t.Fatalf("Run = %v, want context.Canceled", err) + } + if want := offsets(4, 10); !slices.Equal(r.applied, want) { + t.Errorf("applied = %v, want %v", r.applied, want) + } + if len(log.reads) != 1 { + t.Errorf("Run called Read %d times, want 1: it stops once it sees the context end", len(log.reads)) + } +} + +func TestRunBatchesAndWait(t *testing.T) { + c, rs := refs(t, 649) + all := chainEntries(c, rs, 650) + inner := newLog(t, all[:600]...) + log := &countingLog{Log: inner, waiting: make(chan chainlog.Offset)} + r := &recorder{} + + ctx, cancel := context.WithCancel(bounded(t)) + defer cancel() + done := make(chan error, 1) + go func() { done <- consumer.Run(ctx, log, &canceler{Consumer: r, at: 650, cancel: cancel}) }() + + if after := waitFor(t, log, done); after != 600 { + t.Fatalf("first Wait after %d, want 600", after) + } + if _, err := inner.Append(t.Context(), all[600:]...); err != nil { + t.Fatalf("Append: %v", err) + } + if err := <-done; err != context.Canceled { + t.Fatalf("Run = %v, want context.Canceled", err) + } + + for i, max := range log.reads { + if max != 256 { + t.Errorf("Read %d had max %d, want 256", i+1, max) + } + } + if len(log.got) < 4 || !slices.Equal(log.got[:4], []int{256, 256, 88, 0}) { + t.Errorf("Read returned %v entries, want 256, 256, 88, 0 before the first Wait", log.got) + } + if want := []chainlog.Offset{600}; !slices.Equal(log.waits, want) { + t.Errorf("Wait calls after %v, want %v", log.waits, want) + } + if want := offsets(1, 650); !slices.Equal(r.applied, want) { + t.Errorf("applied %d offsets %v…, want 1 … 650", len(r.applied), r.applied[:min(len(r.applied), 5)]) + } +} + +// offsetFunc is a consumer whose Offset calls offset and whose Apply fails the test. +type offsetFunc struct { + t *testing.T + offset func(context.Context) (chainlog.Offset, error) +} + +func (c offsetFunc) Offset(ctx context.Context) (chainlog.Offset, error) { return c.offset(ctx) } + +func (c offsetFunc) Apply(context.Context, chainlog.Entry) error { + c.t.Error("Apply called") + return nil +} + +func TestRunContext(t *testing.T) { + t.Run("canceled", func(t *testing.T) { + ctx, cancel := context.WithCancel(bounded(t)) + cancel() + log := &countingLog{Log: tenEntryLog(t)} + c := offsetFunc{t: t, offset: func(context.Context) (chainlog.Offset, error) { + t.Error("Offset called") + return 0, nil + }} + if err := consumer.Run(ctx, log, c); err != context.Canceled { + t.Fatalf("Run = %v, want context.Canceled", err) + } + if len(log.reads) != 0 { + t.Errorf("Run called Read %d times, want 0", len(log.reads)) + } + }) + t.Run("offset cancels", func(t *testing.T) { + ctx, cancel := context.WithCancel(bounded(t)) + defer cancel() + c := offsetFunc{t: t, offset: func(context.Context) (chainlog.Offset, error) { + cancel() + return 0, errors.New("connection reset") + }} + if err := consumer.Run(ctx, tenEntryLog(t), c); err != context.Canceled { + t.Fatalf("Run = %v, want context.Canceled", err) + } + }) + t.Run("deadline", func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Millisecond) + defer cancel() + err := consumer.Run(ctx, &countingLog{Log: newLog(t)}, &recorder{}) + if err != context.DeadlineExceeded { + t.Fatalf("Run = %v, want context.DeadlineExceeded", err) + } + }) +} + +func TestRunOffsetError(t *testing.T) { + c := offsetFunc{t: t, offset: func(context.Context) (chainlog.Offset, error) { + return 0, errors.New("connection reset") + }} + err := consumer.Run(bounded(t), tenEntryLog(t), c) + if got, want := errText(err), "consumer: offset: connection reset"; got != want { + t.Errorf("Run = %q, want %q", got, want) + } +} + +func TestRunHeadError(t *testing.T) { + log := tenEntryLog(t) + if err := log.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + err := consumer.Run(bounded(t), log, &recorder{}) + if !errors.Is(err, chainlog.ErrClosed) { + t.Errorf("Run = %v, want an error that Is chainlog.ErrClosed", err) + } + if got, want := errText(err), "consumer: log head: chainlog: log closed"; got != want { + t.Errorf("Run = %q, want %q", got, want) + } + + ctx, cancel := context.WithCancel(bounded(t)) + defer cancel() + if err := consumer.Run(ctx, headCancelLog{Log: tenEntryLog(t), cancel: cancel}, &recorder{}); err != context.Canceled { + t.Errorf("Run with a Head that cancels and fails = %v, want context.Canceled", err) + } +} + +// headCancelLog wraps a log whose Head calls cancel and fails. +type headCancelLog struct { + chainlog.Log + cancel context.CancelFunc +} + +func (l headCancelLog) Head(context.Context) (chainlog.Offset, error) { + l.cancel() + return 0, errors.New("connection reset") +} + +func TestRunAheadOfLog(t *testing.T) { + log := &countingLog{Log: tenEntryLog(t)} + r := &recorder{off: 11} + err := consumer.Run(bounded(t), log, r) + if !errors.Is(err, consumer.ErrAheadOfLog) { + t.Fatalf("Run = %v, want an error that Is ErrAheadOfLog", err) + } + if got, want := err.Error(), "consumer: offset is ahead of the log head: offset 11, head 10"; got != want { + t.Errorf("Run = %q, want %q", got, want) + } + if len(log.reads) != 0 || len(r.calls) != 0 { + t.Errorf("Run called Read %d times and Apply %d times, want 0 and 0", len(log.reads), len(r.calls)) + } +} + +func TestRunBehindLog(t *testing.T) { + c, rs := refs(t, 3) + log := newLog(t, fin(rs[0]), blk(c, rs[1]), blk(c, rs[2]), fin(rs[2]), blk(c, rs[3])) + if err := log.Truncate(t.Context(), 4); err != nil { + t.Fatalf("Truncate: %v", err) + } + r := &recorder{off: 1} + err := consumer.Run(bounded(t), log, r) + if !errors.Is(err, consumer.ErrBehindLog) || !errors.Is(err, chainlog.ErrTruncated) { + t.Fatalf("Run = %v, want an error that Is ErrBehindLog and chainlog.ErrTruncated", err) + } + if prefix := "consumer: offset is behind the oldest log entry: next offset 2: "; !strings.HasPrefix(err.Error(), prefix) { + t.Errorf("Run = %q, want prefix %q", err, prefix) + } + if len(r.calls) != 0 { + t.Errorf("Apply called for %v, want no call", r.calls) + } +} + +// errLog is a log whose Read fails with err. +type errLog struct { + chainlog.Log + err error +} + +func (l errLog) Read(context.Context, chainlog.Offset, int) ([]chainlog.Entry, error) { + return nil, l.err +} + +func TestRunReadError(t *testing.T) { + log := errLog{Log: tenEntryLog(t), err: errors.New("disk read")} + err := consumer.Run(bounded(t), log, &recorder{off: 2}) + if got, want := errText(err), "consumer: read from offset 3: disk read"; got != want { + t.Errorf("Run = %q, want %q", got, want) + } + + for _, tt := range []struct { + name string + err error + }{ + {"other error", errors.New("disk read")}, + {"truncated", fmt.Errorf("%w: read from 1, oldest is 2", chainlog.ErrTruncated)}, + } { + t.Run("canceled with "+tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(bounded(t)) + defer cancel() + canceling := cancelLog{Log: errLog{Log: tenEntryLog(t), err: tt.err}, cancel: cancel} + if err := consumer.Run(ctx, canceling, &recorder{}); err != context.Canceled { + t.Errorf("Run with a Read that cancels and fails = %v, want context.Canceled", err) + } + }) + } +} + +// cancelLog wraps a log and calls cancel before each Read. +type cancelLog struct { + chainlog.Log + cancel context.CancelFunc +} + +func (l cancelLog) Read(ctx context.Context, from chainlog.Offset, max int) ([]chainlog.Entry, error) { + l.cancel() + return l.Log.Read(ctx, from, max) +} + +func TestRunApplyError(t *testing.T) { + log := tenEntryLog(t) + diskFull := errors.New("disk full") + r := &recorder{failAt: 6, failErr: diskFull} + err := runTo(t, log, r, 10) + + var ae *consumer.ApplyError + if !errors.As(err, &ae) { + t.Fatalf("Run = %v, want an *ApplyError", err) + } + if ae.Offset != 6 || ae.Kind != chainlog.KindBlock || ae.Err != diskFull { + t.Errorf("ApplyError = %+v, want offset 6, kind block, Err disk full", *ae) + } + if got, want := err.Error(), "consumer: apply offset 6 (block): disk full"; got != want { + t.Errorf("Run = %q, want %q", got, want) + } + if want := offsets(1, 6); !slices.Equal(r.calls, want) { + t.Errorf("Apply calls %v, want %v", r.calls, want) + } + if off, _ := r.Offset(t.Context()); off != 5 { + t.Errorf("Offset = %d, want 5", off) + } + + r.failAt = 0 + if err := runTo(t, log, r, 10); err != context.Canceled { + t.Fatalf("second Run = %v, want context.Canceled", err) + } + if want := offsets(1, 10); !slices.Equal(r.applied, want) { + t.Errorf("applied = %v, want %v", r.applied, want) + } +} + +func TestRunApplyErrorKinds(t *testing.T) { + c, rs := refs(t, 2) + log := newLog(t, fin(rs[0]), blk(c, rs[1]), blk(c, rs[2]), rev(rs[1]), fin(rs[1]), fblk(c, rs[2])) + for _, tt := range []struct { + name string + at chainlog.Offset + want string + }{ + {"revert", 4, "consumer: apply offset 4 (revert): boom"}, + {"finalize", 5, "consumer: apply offset 5 (finalize): boom"}, + {"final block", 6, "consumer: apply offset 6 (block): boom"}, + } { + t.Run(tt.name, func(t *testing.T) { + r := &recorder{failAt: tt.at, failErr: errors.New("boom")} + err := consumer.Run(bounded(t), log, r) + if got := errText(err); got != tt.want { + t.Errorf("Run = %q, want %q", got, tt.want) + } + if want := offsets(1, tt.at-1); !slices.Equal(r.applied, want) { + t.Errorf("applied %v, want %v", r.applied, want) + } + }) + } +} + +// cancelApply is a consumer at offset 0 whose Apply calls cancel and returns ctx.Err(). +type cancelApply struct{ cancel context.CancelFunc } + +func (cancelApply) Offset(context.Context) (chainlog.Offset, error) { return 0, nil } + +func (c cancelApply) Apply(ctx context.Context, _ chainlog.Entry) error { + c.cancel() + return ctx.Err() +} + +func TestRunApplyErrorWrapsContext(t *testing.T) { + ctx, cancel := context.WithCancel(bounded(t)) + defer cancel() + err := consumer.Run(ctx, tenEntryLog(t), cancelApply{cancel: cancel}) + var ae *consumer.ApplyError + if err == context.Canceled || !errors.As(err, &ae) || !errors.Is(err, context.Canceled) { + t.Fatalf("Run = %v, want an *ApplyError wrapping context.Canceled", err) + } + if got, want := err.Error(), "consumer: apply offset 1 (finalize): context canceled"; got != want { + t.Errorf("Run = %q, want %q", got, want) + } +} + +func TestApplyErrorText(t *testing.T) { + inner := errors.New("boom") + for _, tt := range []struct { + name string + kind chainlog.Kind + want string + }{ + {"block", chainlog.KindBlock, "consumer: apply offset 7 (block): boom"}, + {"revert", chainlog.KindRevert, "consumer: apply offset 7 (revert): boom"}, + {"finalize", chainlog.KindFinalize, "consumer: apply offset 7 (finalize): boom"}, + {"kind 9", 9, "consumer: apply offset 7 (kind 9): boom"}, + {"kind 0", 0, "consumer: apply offset 7 (kind 0): boom"}, + } { + t.Run(tt.name, func(t *testing.T) { + e := &consumer.ApplyError{Offset: 7, Kind: tt.kind, Err: inner} + if got := e.Error(); got != tt.want { + t.Errorf("Error() = %q, want %q", got, tt.want) + } + if got := errors.Unwrap(e); got != inner { + t.Errorf("Unwrap() = %v, want %v", got, inner) + } + }) + } +} + +// gapLog wraps a log whose Read from offset 4 returns the entry with offset 4 and then the one +// with offset second. +type gapLog struct { + chainlog.Log + second chainlog.Offset +} + +func (l gapLog) Read(ctx context.Context, from chainlog.Offset, max int) ([]chainlog.Entry, error) { + es, err := l.Log.Read(ctx, from, max) + if from == 4 && err == nil && len(es) >= 3 { + es = []chainlog.Entry{es[0], es[l.second-4]} + } + return es, err +} + +func TestRunOffsetGap(t *testing.T) { + for _, tt := range []struct { + name string + second chainlog.Offset + cancel bool // cancel after applying offset 4: the gap is still reported first + }{ + {"skipped", 6, false}, + {"repeated", 4, false}, + {"skipped after cancel", 6, true}, + } { + t.Run(tt.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(bounded(t)) + defer cancel() + r := &recorder{off: 3} + var c consumer.Consumer = r + if tt.cancel { + c = &canceler{Consumer: r, at: 4, cancel: cancel} + } + err := consumer.Run(ctx, gapLog{Log: tenEntryLog(t), second: tt.second}, c) + if !errors.Is(err, consumer.ErrOffsetGap) { + t.Fatalf("Run = %v, want an error that Is ErrOffsetGap", err) + } + if got, want := err.Error(), fmt.Sprintf("consumer: log offsets are not contiguous: got offset %d, want 5", tt.second); got != want { + t.Errorf("Run = %q, want %q", got, want) + } + if want := []chainlog.Offset{4}; !slices.Equal(r.calls, want) { + t.Errorf("Apply calls %v, want %v", r.calls, want) + } + }) + } +} + +func TestRunRestart(t *testing.T) { + c, rs := refs(t, 11) + log := newLog(t, chainEntries(c, rs, 12)...) + r := &recorder{} + + if err := runTo(t, log, r, 3); err != context.Canceled { + t.Fatalf("Run to 3 = %v, want context.Canceled", err) + } + ctx, cancel := context.WithCancel(bounded(t)) + cancel() + if err := consumer.Run(ctx, log, r); err != context.Canceled { + t.Fatalf("Run with a canceled context = %v, want context.Canceled", err) + } + for _, at := range []chainlog.Offset{7, 12} { + if err := runTo(t, log, r, at); err != context.Canceled { + t.Fatalf("Run to %d = %v, want context.Canceled", at, err) + } + } + if want := offsets(1, 12); !slices.Equal(r.applied, want) { + t.Errorf("applied = %v, want %v", r.applied, want) + } + + restored := &recorder{off: 7} + if err := runTo(t, log, restored, 12); err != context.Canceled { + t.Fatalf("Run from 7 = %v, want context.Canceled", err) + } + if want := offsets(8, 12); !slices.Equal(restored.applied, want) { + t.Errorf("restored applied = %v, want %v", restored.applied, want) + } +} + +func TestRunLogClosed(t *testing.T) { + inner := tenEntryLog(t) + log := &countingLog{Log: inner, waiting: make(chan chainlog.Offset)} + r := &recorder{} + + ctx := bounded(t) + done := make(chan error, 1) + go func() { done <- consumer.Run(ctx, log, r) }() + + if after := waitFor(t, log, done); after != 10 { + t.Fatalf("Wait after %d, want 10", after) + } + if err := inner.Close(); err != nil { + t.Fatalf("Close: %v", err) + } + err := <-done + if !errors.Is(err, chainlog.ErrClosed) { + t.Fatalf("Run = %v, want an error that Is chainlog.ErrClosed", err) + } + if got, want := err.Error(), "consumer: wait after offset 10: chainlog: log closed"; got != want { + t.Errorf("Run = %q, want %q", got, want) + } + if want := offsets(1, 10); !slices.Equal(r.applied, want) { + t.Errorf("applied = %v, want %v", r.applied, want) + } +} + +func TestRunWaitCanceled(t *testing.T) { + ctx, cancel := context.WithCancel(bounded(t)) + log := &countingLog{Log: tenEntryLog(t), waiting: make(chan chainlog.Offset)} + done := make(chan error, 1) + go func() { done <- consumer.Run(ctx, log, &recorder{}) }() + waitFor(t, log, done) + cancel() + if err := <-done; err != context.Canceled { + t.Fatalf("Run = %v, want context.Canceled", err) + } +} + +// runKey is the context key TestRunPassesContext marks Run's context with. +type runKey struct{} + +// ctxLog wraps a log and records, per call of Head, Read and Wait, the call's name and whether +// its context carries runKey. Wait cancels via cancel and records whether its context ended, +// then waits. +type ctxLog struct { + chainlog.Log + cancel context.CancelFunc + calls []string +} + +func (l *ctxLog) record(ctx context.Context, name string) { + if ctx.Value(runKey{}) == nil { + name += " without Run's context" + } + l.calls = append(l.calls, name) +} + +func (l *ctxLog) Head(ctx context.Context) (chainlog.Offset, error) { + l.record(ctx, "Head") + return l.Log.Head(ctx) +} + +func (l *ctxLog) Read(ctx context.Context, from chainlog.Offset, max int) ([]chainlog.Entry, error) { + l.record(ctx, "Read") + return l.Log.Read(ctx, from, max) +} + +func (l *ctxLog) Wait(ctx context.Context, after chainlog.Offset) error { + l.record(ctx, "Wait") + l.cancel() + if ctx.Err() == nil { + l.calls = append(l.calls, "Wait context not canceled") + return errors.New("wait context not canceled") + } + return l.Log.Wait(ctx, after) +} + +// ctxRecorder is a recorder that records, per call of Offset and Apply, the call's name and +// whether its context carries runKey. +type ctxRecorder struct { + *recorder + log *ctxLog +} + +func (c ctxRecorder) Offset(ctx context.Context) (chainlog.Offset, error) { + c.log.record(ctx, "Offset") + return c.recorder.Offset(ctx) +} + +func (c ctxRecorder) Apply(ctx context.Context, e chainlog.Entry) error { + c.log.record(ctx, "Apply") + return c.recorder.Apply(ctx, e) +} + +func TestRunPassesContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.WithValue(bounded(t), runKey{}, "run")) + defer cancel() + log := &ctxLog{Log: tenEntryLog(t), cancel: cancel} + err := consumer.Run(ctx, log, ctxRecorder{recorder: &recorder{off: 8}, log: log}) + if err != context.Canceled { + t.Errorf("Run = %v, want context.Canceled", err) + } + want := []string{"Offset", "Head", "Read", "Apply", "Apply", "Read", "Wait"} + if !slices.Equal(log.calls, want) { + t.Errorf("calls %q, want %q", log.calls, want) + } +} + +// errText returns err's text, or "". +func errText(err error) string { + if err == nil { + return "" + } + return err.Error() +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..da3b9b9 --- /dev/null +++ b/go.mod @@ -0,0 +1,13 @@ +module github.com/hmdsefi/evmsync + +go 1.26.0 + +require ( + github.com/ethereum/go-ethereum v1.17.8 + golang.org/x/crypto v0.58.0 +) + +require ( + github.com/holiman/uint256 v1.3.2 // indirect + golang.org/x/sys v0.49.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..c87b73c --- /dev/null +++ b/go.sum @@ -0,0 +1,8 @@ +github.com/ethereum/go-ethereum v1.17.8 h1:iKuXrT38t1PaU6aBw6MjBJhGoCl/+Ir0vf5aqQhYT38= +github.com/ethereum/go-ethereum v1.17.8/go.mod h1:Iui379BD7ryQSFeIDn3sh9Od2I1uheqJD2/6N0m4xm0= +github.com/holiman/uint256 v1.3.2 h1:a9EgMPSC1AAaj1SZL5zIQD3WbwTuHrMGOerLjGmM/TA= +github.com/holiman/uint256 v1.3.2/go.mod h1:EOMSn4q6Nyt9P6efbI3bueV4e1b3dGlUCXeiRV4ng7E= +golang.org/x/crypto v0.58.0 h1:COkYLr4k7nDI4r6QtMML2AfzeIswWHAJthLS6K/M/54= +golang.org/x/crypto v0.58.0/go.mod h1:Xh+kl5A+M0gMiWDytH2Hhr5Z7WbNCpv/eB/EFgfSk+w= +golang.org/x/sys v0.49.0 h1:XbzkgYJHdqh/8m2Uu0W/dQv8nktxx4BFHp1M0gROTXA= +golang.org/x/sys v0.49.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og= diff --git a/internal/archtest/applypath_test.go b/internal/archtest/applypath_test.go new file mode 100644 index 0000000..2e92865 --- /dev/null +++ b/internal/archtest/applypath_test.go @@ -0,0 +1,492 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package archtest + +import ( + "errors" + "fmt" + "go/ast" + "go/build" + "go/importer" + "go/parser" + "go/token" + "go/types" + "io/fs" + "os" + "path/filepath" + "regexp" + "slices" + "strings" + "testing" +) + +// applyPath lists the apply-path packages, as relative paths; the ones that exist are checked. +var applyPath = []string{"chain", "chainlog", "tracker", "consumer", "memstore"} + +// wallClock lists the functions of package time that read or wait on the wall clock. +var wallClock = []string{"Now", "Since", "Until", "Sleep", "After", "AfterFunc", "Tick", "NewTimer", "NewTicker"} + +// sortedFuncs are the functions of package slices whose first argument may be a maps iterator. +var sortedFuncs = []string{"Sorted", "SortedFunc", "SortedStableFunc"} + +// maporderRE matches the text after "//" of a maporder directive. +var maporderRE = regexp.MustCompile(`^evmsync:maporder(?:[ \t]+(.*\S))?[ \t]*$`) + +// pkgDir is a package directory and the import path to type-check it as. +type pkgDir struct{ dir, path string } + +// checkApplyPath type-checks each package and reports wall-clock reads (AP1), global randomness +// (AP2), go statements (AP3), map ranges (AP4) and bad maporder directives (AP0). Finding files are +// relative to root. +func checkApplyPath(root string, pkgs []pkgDir) ([]finding, error) { + fset := token.NewFileSet() + conf := types.Config{Importer: importer.ForCompiler(fset, "source", nil)} + ctx := build.Default + ctx.CgoEnabled = false + var fs []finding + for _, p := range pkgs { + pf, err := checkPackage(fset, &conf, &ctx, root, p) + if err != nil { + return nil, err + } + fs = append(fs, pf...) + } + sortFindings(fs) + return fs, nil +} + +// source is one parsed file with its bytes and its path relative to the checked root. +type source struct { + file *ast.File + src []byte + rel string +} + +// checker collects the findings of one package. +type checker struct { + fset *token.FileSet + info *types.Info + pkg *types.Package + rel map[string]string // absolute file name to relative path + found []finding +} + +func checkPackage(fset *token.FileSet, conf *types.Config, ctx *build.Context, root string, p pkgDir) ([]finding, error) { + entries, err := os.ReadDir(p.dir) + if err != nil { + return nil, fmt.Errorf("archtest: %w", err) + } + var srcs []source + for _, e := range entries { + name := e.Name() + if !isGoFile(e) || strings.HasSuffix(name, "_test.go") { + continue + } + ok, err := ctx.MatchFile(p.dir, name) + if err != nil { + return nil, fmt.Errorf("archtest: %w", err) + } + if !ok { + continue + } + path := filepath.Join(p.dir, name) + src, err := os.ReadFile(path) + if err != nil { + return nil, fmt.Errorf("archtest: %w", err) + } + f, err := parser.ParseFile(fset, path, src, parser.ParseComments) + if err != nil { + return nil, fmt.Errorf("archtest: parse %s: %w", path, err) + } + rel, err := filepath.Rel(root, path) + if err != nil { + return nil, fmt.Errorf("archtest: %w", err) + } + srcs = append(srcs, source{file: f, src: src, rel: filepath.ToSlash(rel)}) + } + if len(srcs) == 0 { + return nil, nil + } + files := make([]*ast.File, len(srcs)) + c := &checker{ + fset: fset, + info: &types.Info{Uses: map[*ast.Ident]types.Object{}, Types: map[ast.Expr]types.TypeAndValue{}}, + rel: map[string]string{}, + } + for i, s := range srcs { + files[i] = s.file + c.rel[fset.PositionFor(s.file.Pos(), false).Filename] = s.rel + } + c.pkg, err = conf.Check(p.path, fset, files, c.info) + if err != nil { + return nil, fmt.Errorf("archtest: type-check %s: %w", p.path, err) + } + c.uses(sortedArgs(c.info, files)) + for _, f := range files { + c.goStmts(f) + c.mapRanges(f) + } + var out []finding + for _, s := range srcs { + out = append(out, applyDirectives(c.fileFindings(s.rel), directives(fset, s))...) + } + return out, nil +} + +// add records a finding at pos. +func (c *checker) add(pos token.Pos, rule, msg string) { + p := c.fset.PositionFor(pos, false) // ignore //line comments: findings point at the real file + c.found = append(c.found, finding{file: c.rel[p.Filename], line: p.Line, col: p.Column, rule: rule, msg: msg}) +} + +// fileFindings returns the findings in the file rel. +func (c *checker) fileFindings(rel string) []finding { + var out []finding + for _, f := range c.found { + if f.file == rel { + out = append(out, f) + } + } + return out +} + +// uses reports uses of wall-clock functions (AP1), of package-level functions of math/rand and +// math/rand/v2 (AP2), and of maps.All, maps.Keys and maps.Values (AP4) unless the identifier is +// in sorted. +func (c *checker) uses(sorted map[*ast.Ident]bool) { + for id, obj := range c.info.Uses { + fn, ok := obj.(*types.Func) + if !ok || fn.Pkg() == nil || fn.Signature().Recv() != nil { + continue + } + name := fn.Name() + switch path := fn.Pkg().Path(); { + case path == "time" && slices.Contains(wallClock, name): + c.add(id.Pos(), "AP1", fmt.Sprintf("wall-clock time: time.%s is forbidden in apply-path code; read time from the block", name)) + case path == "math/rand" || path == "math/rand/v2": + c.add(id.Pos(), "AP2", fmt.Sprintf("global randomness: %s.%s is forbidden in apply-path code", path, name)) + case path == "maps" && (name == "All" || name == "Keys" || name == "Values") && !sorted[id]: + c.add(id.Pos(), "AP4", fmt.Sprintf("maps.%s yields in random order; wrap it in slices.Sorted or annotate //evmsync:maporder ", name)) + } + } +} + +// sortedArgs returns the callee identifiers of calls that are the first argument of a call to +// slices.Sorted, SortedFunc or SortedStableFunc. +func sortedArgs(info *types.Info, files []*ast.File) map[*ast.Ident]bool { + out := map[*ast.Ident]bool{} + for _, f := range files { + ast.Inspect(f, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok || len(call.Args) == 0 { + return true + } + fn, ok := info.Uses[callee(call)].(*types.Func) + if !ok || fn.Pkg() == nil || fn.Pkg().Path() != "slices" || !slices.Contains(sortedFuncs, fn.Name()) { + return true + } + if arg, ok := ast.Unparen(call.Args[0]).(*ast.CallExpr); ok { + if id := callee(arg); id != nil { + out[id] = true + } + } + return true + }) + } + return out +} + +// callee returns the identifier a call's function expression names: f, pkg.f, f[T] or pkg.f[T]. +func callee(call *ast.CallExpr) *ast.Ident { + fun := ast.Unparen(call.Fun) + switch x := fun.(type) { + case *ast.IndexExpr: + fun = x.X + case *ast.IndexListExpr: + fun = x.X + } + switch x := ast.Unparen(fun).(type) { + case *ast.Ident: + return x + case *ast.SelectorExpr: + return x.Sel + } + return nil +} + +// goStmts reports every go statement (AP3) except those inside the body of the function Run +// without receiver in package consumer. +func (c *checker) goStmts(f *ast.File) { + for _, d := range f.Decls { + fd, ok := d.(*ast.FuncDecl) + if ok && fd.Recv == nil && fd.Name.Name == "Run" && c.pkg.Name() == "consumer" { + continue + } + ast.Inspect(d, func(n ast.Node) bool { + if g, ok := n.(*ast.GoStmt); ok { + c.add(g.Go, "AP3", "go statement is forbidden in apply-path code; only consumer.Run may start goroutines") + } + return true + }) + } +} + +// mapRanges reports every range over a map or over a type parameter whose constraint allows a +// map (AP4). +func (c *checker) mapRanges(f *ast.File) { + qual := func(p *types.Package) string { + if p == c.pkg { + return "" + } + return p.Name() + } + ast.Inspect(f, func(n ast.Node) bool { + rs, ok := n.(*ast.RangeStmt) + if !ok { + return true + } + if t := c.info.TypeOf(rs.X); t != nil && allowsMap(t) { + c.add(rs.For, "AP4", fmt.Sprintf("range over map %s has random order; range over sorted keys or annotate //evmsync:maporder ", types.TypeString(t, qual))) + } + return true + }) +} + +// allowsMap reports whether t is a map, or a type parameter or interface whose embedded types or +// union terms allow a map, at any depth. +func allowsMap(t types.Type) bool { + if tp, ok := t.(*types.TypeParam); ok { + return allowsMap(tp.Constraint()) + } + switch u := t.Underlying().(type) { + case *types.Map: + return true + case *types.Interface: + for i := range u.NumEmbeddeds() { + e := u.EmbeddedType(i) + if un, ok := e.(*types.Union); ok { + for j := range un.Len() { + if allowsMap(un.Term(j).Type()) { + return true + } + } + } else if allowsMap(e) { + return true + } + } + } + return false +} + +// directive is one //evmsync:maporder comment. +type directive struct { + pos token.Position + file string // relative path + line int // the line it applies to + reason string +} + +// directives returns the maporder directives of a file. A directive applies to its own line when +// code precedes it on that line, otherwise to the next line. +func directives(fset *token.FileSet, s source) []directive { + var out []directive + for _, cg := range s.file.Comments { + for _, cm := range cg.List { + if !strings.HasPrefix(cm.Text, "//") { + continue + } + m := maporderRE.FindStringSubmatch(cm.Text[2:]) + if m == nil { + continue + } + pos := fset.PositionFor(cm.Slash, false) + before := s.src[pos.Offset-(pos.Column-1) : pos.Offset] + line := pos.Line + 1 + if strings.Trim(string(before), " \t") != "" { + line = pos.Line + } + out = append(out, directive{pos: pos, file: s.rel, line: line, reason: m[1]}) + } + } + return out +} + +// applyDirectives drops the AP4 findings on lines a directive with a reason applies to, and adds +// an AP0 finding for each directive without a reason or that suppresses nothing. +func applyDirectives(fs []finding, ds []directive) []finding { + used := make([]bool, len(ds)) + var out []finding + for _, f := range fs { + suppressed := false + for i, d := range ds { + if d.reason != "" && f.rule == "AP4" && f.file == d.file && f.line == d.line { + used[i] = true + suppressed = true + } + } + if !suppressed { + out = append(out, f) + } + } + for i, d := range ds { + ap0 := finding{file: d.file, line: d.pos.Line, col: d.pos.Column, rule: "AP0"} + switch { + case d.reason == "": + ap0.msg = "//evmsync:maporder needs a reason" + case !used[i]: + ap0.msg = "//evmsync:maporder suppresses no finding" + default: + continue + } + out = append(out, ap0) + } + return out +} + +func TestApplyPathRules(t *testing.T) { + root := testRoot(t) + var pkgs []pkgDir + for _, rel := range applyPath { + dir := filepath.Join(root, filepath.FromSlash(rel)) + _, err := os.Stat(dir) + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + t.Fatal(err) + } + pkgs = append(pkgs, pkgDir{dir: dir, path: importPath(rel)}) + } + found, err := checkApplyPath(root, pkgs) + if err != nil { + t.Fatal(err) + } + report(t, "apply-path rules", found) +} + +func TestApplyPathFixtures(t *testing.T) { + names := []string{"alias", "clock", "consumer", "directive", "gostmt", "mapsiter", "maprange", "rand"} + for _, name := range names { + t.Run(name, func(t *testing.T) { + dir, err := filepath.Abs(filepath.Join("testdata", "applypath", name)) + if err != nil { + t.Fatal(err) + } + want, err := os.ReadFile(filepath.Join(dir, "want.txt")) + if err != nil { + t.Fatal(err) + } + fs, err := checkApplyPath(dir, []pkgDir{{dir, "fixture/" + name}}) + if err != nil { + t.Fatal(err) + } + equalLines(t, strs(fs), slices.DeleteFunc(strings.Split(string(want), "\n"), func(s string) bool { return s == "" })) + }) + } +} + +// writeFiles writes each name: content pair under dir, creating directories. +func writeFiles(t *testing.T, dir string, files map[string]string) { + t.Helper() + for name, content := range files { + path := filepath.Join(dir, filepath.FromSlash(name)) + if err := os.MkdirAll(filepath.Dir(path), 0o750); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + } +} + +// TestApplyPathExtra covers what the fixtures under testdata/applypath leave out: a map reached +// through an embedded constraint, maps.All and maps.Values, the Sorted exception through +// parentheses and SortedFunc, a go statement in a Run outside consumer and in another function of +// consumer, a directive next to a non-AP4 finding, a block comment that is not a directive, and +// //line comments, which must not move findings. +func TestApplyPathExtra(t *testing.T) { + dir := t.TempDir() + writeFiles(t, dir, map[string]string{ + "extra/extra.go": extraSrc, + "consumer/c.go": "package consumer\n\nfunc f() {}\n\nfunc g() {\n\tgo f()\n}\n", + }) + fs, err := checkApplyPath(dir, []pkgDir{ + {filepath.Join(dir, "consumer"), "fixture/consumer"}, + {filepath.Join(dir, "extra"), "fixture/extra"}, + }) + if err != nil { + t.Fatal(err) + } + const ( + goStmt = "AP3: go statement is forbidden in apply-path code; only consumer.Run may start goroutines" + mapType = "AP4: range over map map[int]int has random order; range over sorted keys or annotate //evmsync:maporder " + ) + equalLines(t, strs(fs), []string{ + "consumer/c.go:6:2: " + goStmt, + "extra/extra.go:14:2: AP4: range over map T has random order; range over sorted keys or annotate //evmsync:maporder ", + "extra/extra.go:21:2: " + goStmt, + "extra/extra.go:25:17: AP4: maps.All yields in random order; wrap it in slices.Sorted or annotate //evmsync:maporder ", + "extra/extra.go:27:17: AP4: maps.Values yields in random order; wrap it in slices.Sorted or annotate //evmsync:maporder ", + "extra/extra.go:29:11: AP1: wall-clock time: time.Now is forbidden in apply-path code; read time from the block", + "extra/extra.go:29:17: AP0: //evmsync:maporder suppresses no finding", + "extra/extra.go:33:2: " + mapType, + "extra/extra.go:40:2: " + mapType, + }) +} + +const extraSrc = `package extra + +import ( + "maps" + "slices" + "time" +) + +type A interface{ ~map[int]int } + +type B interface{ A } + +func F[T B](m T) { + for range m { + } +} + +func f() {} + +func Run() { + go f() +} + +func G(m map[int]int) []int { + for range maps.All(m) { + } + for range maps.Values(m) { + } + _ = time.Now() //evmsync:maporder r + _ = slices.Sorted((maps.Keys(m))) + _ = slices.SortedFunc(maps.Keys(m), func(a, b int) int { return a - b }) + /*evmsync:maporder r*/ + for range m { + } + return slices.SortedStableFunc(maps.Values(m), func(a, b int) int { return a - b }) +} + +//line other.go:100:200 +func H(m map[int]int) { + for range m { + } + for range m { //evmsync:maporder r + } +} +` + +func TestApplyPathParseError(t *testing.T) { + dir := t.TempDir() + bad := filepath.Join(dir, "bad.go") + writeFiles(t, dir, map[string]string{"bad.go": "package bad\n\nfunc\n"}) + _, err := checkApplyPath(dir, []pkgDir{{dir, "fixture/bad"}}) + if want := "archtest: parse " + bad + ": "; err == nil || !strings.HasPrefix(err.Error(), want) { + t.Errorf("error = %v, want prefix %q", err, want) + } +} diff --git a/internal/archtest/doc.go b/internal/archtest/doc.go new file mode 100644 index 0000000..d48d2ce --- /dev/null +++ b/internal/archtest/doc.go @@ -0,0 +1,7 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +// Package archtest holds the tests that enforce evmsync's architecture rules: the import table of +// architecture §5.1 (TestImportBoundaries), the license header of §15.6 (TestLicenseHeaders) and +// the apply-path rules of principle 5 (TestApplyPathRules). It has no non-test code. +package archtest diff --git a/internal/archtest/imports_test.go b/internal/archtest/imports_test.go new file mode 100644 index 0000000..e58984f --- /dev/null +++ b/internal/archtest/imports_test.go @@ -0,0 +1,241 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package archtest + +import ( + "fmt" + "go/build" + "go/parser" + "go/token" + "maps" + "os" + "path/filepath" + "slices" + "strconv" + "strings" + "testing" +) + +// checkContexts are the build contexts a file must match one of to be checked: cgo off, and +// linux/amd64, darwin/arm64 and windows/amd64. +func checkContexts() []build.Context { + var cs []build.Context + for _, p := range [][2]string{{"linux", "amd64"}, {"darwin", "arm64"}, {"windows", "amd64"}} { + c := build.Default + c.CgoEnabled = false + c.GOOS, c.GOARCH = p[0], p[1] + cs = append(cs, c) + } + return cs +} + +// inPlay reports whether at least one context matches the file. +func inPlay(cs []build.Context, dir, name string) (bool, error) { + for _, c := range cs { + ok, err := c.MatchFile(dir, name) + if err != nil { + return false, fmt.Errorf("archtest: %w", err) + } + if ok { + return true, nil + } + } + return false, nil +} + +// importPath returns the import path of the root-module package at rel. +func importPath(rel string) string { + if rel == "" { + return modulePath + } + return modulePath + "/" + rel +} + +// allowed reports whether a package with rule r may import p. +func allowed(p string, r rule) bool { + switch { + case p == "C": + return false + case p == modulePath || strings.HasPrefix(p, modulePath+"/"): + rel := strings.TrimPrefix(strings.TrimPrefix(p, modulePath), "/") + return slices.Contains(r.module, rel) + case !strings.Contains(strings.Split(p, "/")[0], "."): + switch { + case p == "testing" || strings.HasPrefix(p, "testing/"): + return r.testing && p == "testing" + case r.noNet && (p == "net" || strings.HasPrefix(p, "net/") || p == "os/exec" || + p == "math/rand" || p == "math/rand/v2"): + return false + case r.noTime && p == "time": + return false + } + return true + default: + return slices.Contains(r.external, p) + } +} + +// checkImports checks the imports of every package directory below root against rs, and the set +// of package directories against the inventory inv. +func checkImports(root string, inv []string, rs map[string]rule) ([]finding, error) { + var fs []finding + fset := token.NewFileSet() + cs := checkContexts() + pkgs := map[string]bool{} + err := walk(root, true, func(dir, rel string, entries []os.DirEntry) error { + if !slices.ContainsFunc(entries, isGoFile) { + return nil + } + pkgs[rel] = true + r, ok := rs[rel] + if !ok { + return nil + } + for _, e := range entries { + name := e.Name() + if !isGoFile(e) || strings.HasSuffix(name, "_test.go") { + continue + } + ok, err := inPlay(cs, dir, name) + if err != nil { + return err + } + if !ok { + continue + } + path := filepath.Join(dir, name) + f, err := parser.ParseFile(fset, path, nil, parser.ImportsOnly) + if err != nil { + return fmt.Errorf("archtest: parse %s: %w", path, err) + } + for _, imp := range f.Imports { + p, err := strconv.Unquote(imp.Path.Value) + if err != nil { + return fmt.Errorf("archtest: %s: %w", fset.Position(imp.Path.Pos()), err) + } + if allowed(p, r) { + continue + } + pos := fset.Position(imp.Path.Pos()) + fs = append(fs, finding{ + file: relFile(rel, name), line: pos.Line, col: pos.Column, rule: "IB1", + msg: fmt.Sprintf("import %q is not allowed in package %s", p, importPath(rel)), + }) + } + } + return nil + }) + if err != nil { + return nil, err + } + + ib0 := func(rel, msg string) { + fs = append(fs, finding{file: rel, rule: "IB0", msg: fmt.Sprintf("package %s %s", rel, msg)}) + } + seen := map[string]int{} + for _, rel := range inv { + seen[rel]++ + } + for _, rel := range slices.Sorted(maps.Keys(pkgs)) { + if seen[rel] == 0 { + ib0(rel, "exists but is not in the inventory") + } + if _, ok := rs[rel]; !ok { + ib0(rel, "has no import rule; add it to architecture §5.1 and internal/archtest") + } + } + for _, rel := range slices.Sorted(maps.Keys(seen)) { + if !pkgs[rel] { + ib0(rel, "is in the inventory but does not exist") + } + if seen[rel] > 1 { + ib0(rel, "is listed twice in the inventory") + } + } + sortFindings(fs) + return fs, nil +} + +func TestImportBoundaries(t *testing.T) { + fs, err := checkImports(testRoot(t), inventory, rules) + if err != nil { + t.Fatal(err) + } + report(t, "import boundaries", fs) +} + +func TestImportFixtures(t *testing.T) { + root := copyFixture(t, "imports") + inv := []string{"chain", "chainlog", "chainlog/logtest", "chainlog/wal", "consumer", "ingest", "rpc", "tracker"} + fs, err := checkImports(root, inv, rules) + if err != nil { + t.Fatal(err) + } + const pkg = "is not allowed in package " + modulePath + equalLines(t, strs(fs), []string{ + `chain/bad.go:5:2: IB1: import "net/http" ` + pkg + `/chain`, + `chain/cgo.go:3:8: IB1: import "C" ` + pkg + `/chain`, + `chain/win_windows.go:3:8: IB1: import "os/exec" ` + pkg + `/chain`, + `chainlog/l.go:3:8: IB1: import "github.com/hmdsefi/evmsync/tracker" ` + pkg + `/chainlog`, + `chainlog/wal/w.go:3:8: IB1: import "github.com/jackc/pgx/v5" ` + pkg + `/chainlog/wal`, + `consumer/c.go:3:8: IB1: import "math/rand/v2" ` + pkg + `/consumer`, + `ingest:0:0: IB0: package ingest is in the inventory but does not exist`, + `newpkg:0:0: IB0: package newpkg exists but is not in the inventory`, + `newpkg:0:0: IB0: package newpkg has no import rule; add it to architecture §5.1 and internal/archtest`, + `rpc/r.go:3:8: IB1: import "testing" ` + pkg + `/rpc`, + `tracker/t.go:3:8: IB1: import "time" ` + pkg + `/tracker`, + }) +} + +func TestInventoryDuplicate(t *testing.T) { + root := copyFixture(t, "imports") + inv := []string{"chain", "chain", "chainlog", "chainlog/logtest", "chainlog/wal", "consumer", "rpc", "tracker", "newpkg"} + fs, err := checkImports(root, inv, rules) + if err != nil { + t.Fatal(err) + } + if !slices.Contains(strs(fs), "chain:0:0: IB0: package chain is listed twice in the inventory") { + t.Errorf("no duplicate finding in:\n%s", strings.Join(strs(fs), "\n")) + } +} + +func TestImportParseError(t *testing.T) { + root := copyFixture(t, "imports") + bad := filepath.Join(root, "chain", "broken.go") + if err := os.WriteFile(bad, []byte("package chain\n\nimport (\n"), 0o600); err != nil { + t.Fatal(err) + } + _, err := checkImports(root, nil, rules) + if want := "archtest: parse " + bad + ": "; err == nil || !strings.HasPrefix(err.Error(), want) { + t.Errorf("error = %v, want prefix %q", err, want) + } +} + +// TestAllowed covers the standard-library branches the fixtures do not reach. +func TestAllowed(t *testing.T) { + tests := []struct { + p, pkg string + want bool + }{ + {"net", "chain", false}, + {"net", "rpc", true}, + {"math/rand", "consumer", false}, + {"math/rand/v2", "chainlog", false}, + {"os/exec", "tracker", false}, + {"time", "tracker", false}, + {"time", "chain", true}, + {"testing", "chainlog/logtest", true}, + {"testing/quick", "chainlog/logtest", false}, + {"testing/quick", "chain", false}, + {"C", "internal/archtest", false}, + {"github.com/hmdsefi/evmsync", "memstore", true}, + {"github.com/hmdsefi/evmsync/pgstore", "", false}, + {"golang.org/x/crypto/sha3", "chainlog", false}, + } + for _, tt := range tests { + if got := allowed(tt.p, rules[tt.pkg]); got != tt.want { + t.Errorf("allowed(%q) in package %q = %v, want %v", tt.p, tt.pkg, got, tt.want) + } + } +} diff --git a/internal/archtest/inventory_test.go b/internal/archtest/inventory_test.go new file mode 100644 index 0000000..1e6e3f6 --- /dev/null +++ b/internal/archtest/inventory_test.go @@ -0,0 +1,19 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package archtest + +// inventory lists the root module's package directories, as relative paths. v0.2 adds ingest and +// rpc; v0.3 adds "" (package evmsync), cmd/evmsync, internal/gen and memstore. +var inventory = []string{ + // v0.1 + "chain", + "chainlog", + "chainlog/logtest", + "chainlog/wal", + "consumer", + "internal/archtest", + "internal/fakechain", + "internal/invariant", + "tracker", +} diff --git a/internal/archtest/license_test.go b/internal/archtest/license_test.go new file mode 100644 index 0000000..d32a84e --- /dev/null +++ b/internal/archtest/license_test.go @@ -0,0 +1,90 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package archtest + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "testing" +) + +// header is the first bytes of every .go file: the two lines of architecture §15.6 and a blank line. +const header = "// Copyright 2026 Hamed Yousefi\n// SPDX-License-Identifier: Apache-2.0\n\n" + +// checkLicense reports every .go file below root, nested modules included, that does not start +// with header. +func checkLicense(root string) ([]finding, error) { + var fs []finding + err := walk(root, false, func(dir, rel string, entries []os.DirEntry) error { + for _, e := range entries { + if !isGoFile(e) { + continue + } + ok, err := hasHeader(filepath.Join(dir, e.Name())) + if err != nil { + return err + } + if !ok { + fs = append(fs, finding{ + file: relFile(rel, e.Name()), line: 1, col: 1, rule: "LH1", + msg: "missing the license header of architecture §15.6 (two comment lines, then a blank line)", + }) + } + } + return nil + }) + if err != nil { + return nil, err + } + sortFindings(fs) + return fs, nil +} + +// hasHeader reports whether the file starts with header. +func hasHeader(path string) (bool, error) { + f, err := os.Open(path) + if err != nil { + return false, fmt.Errorf("archtest: %w", err) + } + defer f.Close() + buf := make([]byte, len(header)) + n, err := io.ReadFull(f, buf) + if err != nil && !errors.Is(err, io.ErrUnexpectedEOF) && !errors.Is(err, io.EOF) { + return false, fmt.Errorf("archtest: %w", err) + } + return bytes.Equal(buf[:n], []byte(header)), nil +} + +func TestLicenseHeaders(t *testing.T) { + fs, err := checkLicense(testRoot(t)) + if err != nil { + t.Fatal(err) + } + report(t, "license headers", fs) +} + +func TestLicenseFixtures(t *testing.T) { + root := copyFixture(t, "license") + // A file with \r\n line endings is never gofmt-clean, so it is written here, not committed. + crlf := "// Copyright 2026 Hamed Yousefi\r\n// SPDX-License-Identifier: Apache-2.0\r\n\r\npackage license\r\n" + if err := os.WriteFile(filepath.Join(root, "crlf.go"), []byte(crlf), 0o600); err != nil { + t.Fatal(err) + } + fs, err := checkLicense(root) + if err != nil { + t.Fatal(err) + } + const msg = ":1:1: LH1: missing the license header of architecture §15.6 (two comment lines, then a blank line)" + equalLines(t, strs(fs), []string{ + "crlf.go" + msg, + "line2.go" + msg, + "mod/x.go" + msg, + "noblank.go" + msg, + "year.go" + msg, + }) +} diff --git a/internal/archtest/root_test.go b/internal/archtest/root_test.go new file mode 100644 index 0000000..a389897 --- /dev/null +++ b/internal/archtest/root_test.go @@ -0,0 +1,188 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package archtest + +import ( + "cmp" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "slices" + "strings" + "testing" +) + +// modulePath is the root module's path. +const modulePath = "github.com/hmdsefi/evmsync" + +// finding is one rule violation. +type finding struct { + file string // relative to the checked root, "/" separators + line, col int + rule, msg string +} + +func (f finding) String() string { + return fmt.Sprintf("%s:%d:%d: %s: %s", f.file, f.line, f.col, f.rule, f.msg) +} + +// sortFindings sorts by file, line, column, rule and message. +func sortFindings(fs []finding) { + slices.SortFunc(fs, func(a, b finding) int { + return cmp.Or( + cmp.Compare(a.file, b.file), + cmp.Compare(a.line, b.line), + cmp.Compare(a.col, b.col), + cmp.Compare(a.rule, b.rule), + cmp.Compare(a.msg, b.msg), + ) + }) +} + +// strs returns the findings as strings, in order. +func strs(fs []finding) []string { + out := make([]string, len(fs)) + for i, f := range fs { + out[i] = f.String() + } + return out +} + +// report sorts the findings and fails the test once with all of them. +func report(t *testing.T, topic string, fs []finding) { + t.Helper() + if len(fs) == 0 { + return + } + sortFindings(fs) + t.Errorf("%s: %d finding(s):\n%s", topic, len(fs), strings.Join(strs(fs), "\n")) +} + +// moduleRoot walks up from dir to the first directory whose go.mod declares modulePath. +func moduleRoot(dir string) (string, error) { + for d := dir; ; { + data, err := os.ReadFile(filepath.Join(d, "go.mod")) + if err != nil && !errors.Is(err, fs.ErrNotExist) { + return "", fmt.Errorf("archtest: %w", err) + } + if err == nil && declaresModule(string(data)) { + return d, nil + } + parent := filepath.Dir(d) + if parent == d { + return "", fmt.Errorf("archtest: no go.mod for %s above %s", modulePath, dir) + } + d = parent + } +} + +// declaresModule reports whether a go.mod's text has the line "module github.com/hmdsefi/evmsync". +func declaresModule(gomod string) bool { + for line := range strings.Lines(gomod) { + f := strings.Fields(line) + if len(f) == 2 && f[0] == "module" && strings.Trim(f[1], `"`) == modulePath { + return true + } + } + return false +} + +// testRoot returns the module root above the test's working directory. +func testRoot(t *testing.T) string { + t.Helper() + wd, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + root, err := moduleRoot(wd) + if err != nil { + t.Fatal(err) + } + return root +} + +// walk calls fn for root and every directory below it, in lexical order, with the directory's +// relative path ("" for root, "/" separators) and its entries. It skips directories named testdata +// or vendor, directories whose name starts with "." or "_", and, when skipNested is set, every +// directory below root that holds a go.mod. +func walk(root string, skipNested bool, fn func(dir, rel string, entries []os.DirEntry) error) error { + var visit func(dir, rel string) error + visit = func(dir, rel string) error { + entries, err := os.ReadDir(dir) + if err != nil { + return fmt.Errorf("archtest: %w", err) + } + if rel != "" && skipNested && slices.ContainsFunc(entries, isGoMod) { + return nil + } + if err := fn(dir, rel, entries); err != nil { + return err + } + for _, e := range entries { + name := e.Name() + if !e.IsDir() || name == "testdata" || name == "vendor" || + strings.HasPrefix(name, ".") || strings.HasPrefix(name, "_") { + continue + } + if err := visit(filepath.Join(dir, name), relFile(rel, name)); err != nil { + return err + } + } + return nil + } + return visit(root, "") +} + +// relFile joins a relative directory and a file name with "/". +func relFile(rel, name string) string { + if rel == "" { + return name + } + return rel + "/" + name +} + +func isGoMod(e os.DirEntry) bool { + return e.Name() == "go.mod" && !e.IsDir() +} + +// isGoFile reports whether e is a file named *.go. +func isGoFile(e os.DirEntry) bool { + return !e.IsDir() && strings.HasSuffix(e.Name(), ".go") +} + +// copyFixture copies testdata/ to a temporary directory, renames every go.mod.tmpl to +// go.mod, and returns the copy's path. +func copyFixture(t *testing.T, name string) string { + t.Helper() + dst := t.TempDir() + if err := os.CopyFS(dst, os.DirFS(filepath.Join("testdata", name))); err != nil { + t.Fatal(err) + } + r, err := os.OpenRoot(dst) + if err != nil { + t.Fatal(err) + } + defer r.Close() + err = fs.WalkDir(r.FS(), ".", func(name string, d fs.DirEntry, err error) error { + if err != nil || d.IsDir() || d.Name() != "go.mod.tmpl" { + return err + } + return r.Rename(name, path.Join(path.Dir(name), "go.mod")) + }) + if err != nil { + t.Fatal(err) + } + return dst +} + +// equalLines fails the test when got and want differ, showing both. +func equalLines(t *testing.T, got, want []string) { + t.Helper() + if !slices.Equal(got, want) { + t.Errorf("findings differ\ngot:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} diff --git a/internal/archtest/rules_test.go b/internal/archtest/rules_test.go new file mode 100644 index 0000000..d5df555 --- /dev/null +++ b/internal/archtest/rules_test.go @@ -0,0 +1,40 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package archtest + +// rule says what one package may import. +type rule struct { + module []string // module packages it may import, as relative paths ("" is the root) + external []string // exact non-standard import paths it may import + testing bool // it may import "testing" + noNet bool // it may not import net, net/..., os/exec, math/rand or math/rand/v2 + noTime bool // it may not import time +} + +const ( + gethCommon = "github.com/ethereum/go-ethereum/common" + hexutil = "github.com/ethereum/go-ethereum/common/hexutil" + abi = "github.com/ethereum/go-ethereum/accounts/abi" + keccak = "golang.org/x/crypto/sha3" + fakechain = "internal/fakechain" +) + +// rules is the import table of architecture §5.1, keyed by relative package path. +var rules = map[string]rule{ + "chain": {external: []string{gethCommon, keccak}, noNet: true}, + "chainlog": {module: []string{"chain"}, external: []string{gethCommon}, noNet: true}, + "chainlog/wal": {module: []string{"chain", "chainlog"}}, + "chainlog/logtest": {module: []string{"chain", "chainlog"}, testing: true}, + "tracker": {module: []string{"chain", "chainlog"}, external: []string{gethCommon}, noNet: true, noTime: true}, + "consumer": {module: []string{"chain", "chainlog"}, noNet: true}, + "rpc": {module: []string{"chain"}, external: []string{gethCommon, hexutil}}, + "ingest": {module: []string{"chain", "chainlog", "tracker"}, external: []string{gethCommon}}, + "": {module: []string{"chain", "chainlog", "chainlog/wal", "consumer", "ingest", "rpc", "tracker"}, external: []string{gethCommon}}, + "memstore": {module: []string{"", "chain", "chainlog"}, noNet: true}, + "internal/gen": {external: []string{abi, gethCommon}}, + "cmd/evmsync": {module: []string{"internal/gen"}}, + fakechain: {module: []string{"chain", "chainlog"}, external: []string{gethCommon, keccak}}, + "internal/invariant": {module: []string{"chain", "chainlog", "consumer", "tracker", fakechain}, external: []string{gethCommon, keccak}, testing: true}, + "internal/archtest": {}, +} diff --git a/internal/archtest/testdata/applypath/alias/alias.go b/internal/archtest/testdata/applypath/alias/alias.go new file mode 100644 index 0000000..5fac5f7 --- /dev/null +++ b/internal/archtest/testdata/applypath/alias/alias.go @@ -0,0 +1,7 @@ +package alias + +import tm "time" + +func F() tm.Time { + return tm.Now() +} diff --git a/internal/archtest/testdata/applypath/alias/want.txt b/internal/archtest/testdata/applypath/alias/want.txt new file mode 100644 index 0000000..c6dce36 --- /dev/null +++ b/internal/archtest/testdata/applypath/alias/want.txt @@ -0,0 +1 @@ +alias.go:6:12: AP1: wall-clock time: time.Now is forbidden in apply-path code; read time from the block diff --git a/internal/archtest/testdata/applypath/clock/clock.go b/internal/archtest/testdata/applypath/clock/clock.go new file mode 100644 index 0000000..4c48b73 --- /dev/null +++ b/internal/archtest/testdata/applypath/clock/clock.go @@ -0,0 +1,12 @@ +package clock + +import "time" + +func F(t time.Time) time.Time { + _ = time.Now() + _ = time.Since(t) + f := time.Sleep + _ = f + _ = time.NewTimer(1) + return t.Add(time.Second) +} diff --git a/internal/archtest/testdata/applypath/clock/want.txt b/internal/archtest/testdata/applypath/clock/want.txt new file mode 100644 index 0000000..c54b6b0 --- /dev/null +++ b/internal/archtest/testdata/applypath/clock/want.txt @@ -0,0 +1,4 @@ +clock.go:6:11: AP1: wall-clock time: time.Now is forbidden in apply-path code; read time from the block +clock.go:7:11: AP1: wall-clock time: time.Since is forbidden in apply-path code; read time from the block +clock.go:8:12: AP1: wall-clock time: time.Sleep is forbidden in apply-path code; read time from the block +clock.go:10:11: AP1: wall-clock time: time.NewTimer is forbidden in apply-path code; read time from the block diff --git a/internal/archtest/testdata/applypath/consumer/consumer.go b/internal/archtest/testdata/applypath/consumer/consumer.go new file mode 100644 index 0000000..bbef928 --- /dev/null +++ b/internal/archtest/testdata/applypath/consumer/consumer.go @@ -0,0 +1,16 @@ +package consumer + +func f() {} + +func Run() { + go f() + func() { + go f() + }() +} + +type R struct{} + +func (r *R) Run() { + go f() +} diff --git a/internal/archtest/testdata/applypath/consumer/want.txt b/internal/archtest/testdata/applypath/consumer/want.txt new file mode 100644 index 0000000..ee579d2 --- /dev/null +++ b/internal/archtest/testdata/applypath/consumer/want.txt @@ -0,0 +1 @@ +consumer.go:15:2: AP3: go statement is forbidden in apply-path code; only consumer.Run may start goroutines diff --git a/internal/archtest/testdata/applypath/directive/directive.go b/internal/archtest/testdata/applypath/directive/directive.go new file mode 100644 index 0000000..2e1fc58 --- /dev/null +++ b/internal/archtest/testdata/applypath/directive/directive.go @@ -0,0 +1,25 @@ +package directive + +func F(m map[int]int, s []int) int { + n := 0 + //evmsync:maporder sum is order-independent + for _, v := range m { + n += v + } + for _, v := range m { //evmsync:maporder sum is order-independent + n += v + } + //evmsync:maporder + for _, v := range m { + n += v + } + //evmsync:maporder slices keep their order + for _, v := range s { + n += v + } + // evmsync:maporder x + for _, v := range m { + n += v + } + return n +} diff --git a/internal/archtest/testdata/applypath/directive/want.txt b/internal/archtest/testdata/applypath/directive/want.txt new file mode 100644 index 0000000..b3aab47 --- /dev/null +++ b/internal/archtest/testdata/applypath/directive/want.txt @@ -0,0 +1,4 @@ +directive.go:12:2: AP0: //evmsync:maporder needs a reason +directive.go:13:2: AP4: range over map map[int]int has random order; range over sorted keys or annotate //evmsync:maporder +directive.go:16:2: AP0: //evmsync:maporder suppresses no finding +directive.go:21:2: AP4: range over map map[int]int has random order; range over sorted keys or annotate //evmsync:maporder diff --git a/internal/archtest/testdata/applypath/gostmt/gostmt.go b/internal/archtest/testdata/applypath/gostmt/gostmt.go new file mode 100644 index 0000000..0b79428 --- /dev/null +++ b/internal/archtest/testdata/applypath/gostmt/gostmt.go @@ -0,0 +1,7 @@ +package gostmt + +func f() {} + +func G() { + go f() +} diff --git a/internal/archtest/testdata/applypath/gostmt/want.txt b/internal/archtest/testdata/applypath/gostmt/want.txt new file mode 100644 index 0000000..0a6d20e --- /dev/null +++ b/internal/archtest/testdata/applypath/gostmt/want.txt @@ -0,0 +1 @@ +gostmt.go:6:2: AP3: go statement is forbidden in apply-path code; only consumer.Run may start goroutines diff --git a/internal/archtest/testdata/applypath/maprange/maprange.go b/internal/archtest/testdata/applypath/maprange/maprange.go new file mode 100644 index 0000000..c6c72e4 --- /dev/null +++ b/internal/archtest/testdata/applypath/maprange/maprange.go @@ -0,0 +1,24 @@ +package maprange + +type C interface{ ~map[string]int } + +func F(m map[int]int, s []int) int { + n := 0 + for k := range m { + n += k + } + for range s { + n++ + } + return n +} + +func G[M ~map[int]int](m M) { + for range m { + } +} + +func H[T C](m T) { + for range m { + } +} diff --git a/internal/archtest/testdata/applypath/maprange/want.txt b/internal/archtest/testdata/applypath/maprange/want.txt new file mode 100644 index 0000000..66e1f95 --- /dev/null +++ b/internal/archtest/testdata/applypath/maprange/want.txt @@ -0,0 +1,3 @@ +maprange.go:7:2: AP4: range over map map[int]int has random order; range over sorted keys or annotate //evmsync:maporder +maprange.go:17:2: AP4: range over map M has random order; range over sorted keys or annotate //evmsync:maporder +maprange.go:22:2: AP4: range over map T has random order; range over sorted keys or annotate //evmsync:maporder diff --git a/internal/archtest/testdata/applypath/mapsiter/mapsiter.go b/internal/archtest/testdata/applypath/mapsiter/mapsiter.go new file mode 100644 index 0000000..3e0d90a --- /dev/null +++ b/internal/archtest/testdata/applypath/mapsiter/mapsiter.go @@ -0,0 +1,17 @@ +package mapsiter + +import ( + "maps" + "slices" +) + +func F(m map[int]int) int { + n := 0 + for k := range maps.Keys(m) { + n += k + } + for _, k := range slices.Sorted(maps.Keys(m)) { + n += k + } + return n +} diff --git a/internal/archtest/testdata/applypath/mapsiter/want.txt b/internal/archtest/testdata/applypath/mapsiter/want.txt new file mode 100644 index 0000000..68e722d --- /dev/null +++ b/internal/archtest/testdata/applypath/mapsiter/want.txt @@ -0,0 +1 @@ +mapsiter.go:10:22: AP4: maps.Keys yields in random order; wrap it in slices.Sorted or annotate //evmsync:maporder diff --git a/internal/archtest/testdata/applypath/rand/rand.go b/internal/archtest/testdata/applypath/rand/rand.go new file mode 100644 index 0000000..ea5d28a --- /dev/null +++ b/internal/archtest/testdata/applypath/rand/rand.go @@ -0,0 +1,12 @@ +package rand + +import ( + mrand "math/rand" + "math/rand/v2" +) + +func F() int { + a := rand.IntN(3) + b := mrand.Int() + return a + b + rand.New(rand.NewPCG(1, 2)).IntN(3) +} diff --git a/internal/archtest/testdata/applypath/rand/want.txt b/internal/archtest/testdata/applypath/rand/want.txt new file mode 100644 index 0000000..a71af8d --- /dev/null +++ b/internal/archtest/testdata/applypath/rand/want.txt @@ -0,0 +1,4 @@ +rand.go:9:12: AP2: global randomness: math/rand/v2.IntN is forbidden in apply-path code +rand.go:10:13: AP2: global randomness: math/rand.Int is forbidden in apply-path code +rand.go:11:22: AP2: global randomness: math/rand/v2.New is forbidden in apply-path code +rand.go:11:31: AP2: global randomness: math/rand/v2.NewPCG is forbidden in apply-path code diff --git a/internal/archtest/testdata/imports/chain/bad.go b/internal/archtest/testdata/imports/chain/bad.go new file mode 100644 index 0000000..b043546 --- /dev/null +++ b/internal/archtest/testdata/imports/chain/bad.go @@ -0,0 +1,6 @@ +package chain + +import ( + "fmt" + "net/http" +) diff --git a/internal/archtest/testdata/imports/chain/cgo.go b/internal/archtest/testdata/imports/chain/cgo.go new file mode 100644 index 0000000..de4b03e --- /dev/null +++ b/internal/archtest/testdata/imports/chain/cgo.go @@ -0,0 +1,3 @@ +package chain + +import "C" diff --git a/internal/archtest/testdata/imports/chain/never.go b/internal/archtest/testdata/imports/chain/never.go new file mode 100644 index 0000000..4364491 --- /dev/null +++ b/internal/archtest/testdata/imports/chain/never.go @@ -0,0 +1,5 @@ +//go:build ignore + +package chain + +import "net" diff --git a/internal/archtest/testdata/imports/chain/ok_test.go b/internal/archtest/testdata/imports/chain/ok_test.go new file mode 100644 index 0000000..2ea5765 --- /dev/null +++ b/internal/archtest/testdata/imports/chain/ok_test.go @@ -0,0 +1,3 @@ +package chain + +import "net/http" diff --git a/internal/archtest/testdata/imports/chain/win_windows.go b/internal/archtest/testdata/imports/chain/win_windows.go new file mode 100644 index 0000000..57552e9 --- /dev/null +++ b/internal/archtest/testdata/imports/chain/win_windows.go @@ -0,0 +1,3 @@ +package chain + +import "os/exec" diff --git a/internal/archtest/testdata/imports/chainlog/l.go b/internal/archtest/testdata/imports/chainlog/l.go new file mode 100644 index 0000000..228c9cd --- /dev/null +++ b/internal/archtest/testdata/imports/chainlog/l.go @@ -0,0 +1,3 @@ +package chainlog + +import "github.com/hmdsefi/evmsync/tracker" diff --git a/internal/archtest/testdata/imports/chainlog/logtest/lt.go b/internal/archtest/testdata/imports/chainlog/logtest/lt.go new file mode 100644 index 0000000..c207ab8 --- /dev/null +++ b/internal/archtest/testdata/imports/chainlog/logtest/lt.go @@ -0,0 +1,3 @@ +package logtest + +import "testing" diff --git a/internal/archtest/testdata/imports/chainlog/wal/w.go b/internal/archtest/testdata/imports/chainlog/wal/w.go new file mode 100644 index 0000000..ad97e58 --- /dev/null +++ b/internal/archtest/testdata/imports/chainlog/wal/w.go @@ -0,0 +1,3 @@ +package wal + +import "github.com/jackc/pgx/v5" diff --git a/internal/archtest/testdata/imports/consumer/c.go b/internal/archtest/testdata/imports/consumer/c.go new file mode 100644 index 0000000..5b477ae --- /dev/null +++ b/internal/archtest/testdata/imports/consumer/c.go @@ -0,0 +1,3 @@ +package consumer + +import "math/rand/v2" diff --git a/internal/archtest/testdata/imports/go.mod.tmpl b/internal/archtest/testdata/imports/go.mod.tmpl new file mode 100644 index 0000000..7b46470 --- /dev/null +++ b/internal/archtest/testdata/imports/go.mod.tmpl @@ -0,0 +1,3 @@ +module github.com/hmdsefi/evmsync + +go 1.26.0 diff --git a/internal/archtest/testdata/imports/newpkg/n.go b/internal/archtest/testdata/imports/newpkg/n.go new file mode 100644 index 0000000..11d25a3 --- /dev/null +++ b/internal/archtest/testdata/imports/newpkg/n.go @@ -0,0 +1 @@ +package newpkg diff --git a/internal/archtest/testdata/imports/pgstore/go.mod.tmpl b/internal/archtest/testdata/imports/pgstore/go.mod.tmpl new file mode 100644 index 0000000..57270f2 --- /dev/null +++ b/internal/archtest/testdata/imports/pgstore/go.mod.tmpl @@ -0,0 +1,3 @@ +module github.com/hmdsefi/evmsync/pgstore + +go 1.26.0 diff --git a/internal/archtest/testdata/imports/pgstore/p.go b/internal/archtest/testdata/imports/pgstore/p.go new file mode 100644 index 0000000..81ffa72 --- /dev/null +++ b/internal/archtest/testdata/imports/pgstore/p.go @@ -0,0 +1,3 @@ +package pgstore + +import "github.com/jackc/pgx/v5" diff --git a/internal/archtest/testdata/imports/rpc/r.go b/internal/archtest/testdata/imports/rpc/r.go new file mode 100644 index 0000000..181b64b --- /dev/null +++ b/internal/archtest/testdata/imports/rpc/r.go @@ -0,0 +1,3 @@ +package rpc + +import "testing" diff --git a/internal/archtest/testdata/imports/testdata/x/x.go b/internal/archtest/testdata/imports/testdata/x/x.go new file mode 100644 index 0000000..e05a04a --- /dev/null +++ b/internal/archtest/testdata/imports/testdata/x/x.go @@ -0,0 +1,3 @@ +package x + +import "net/http" diff --git a/internal/archtest/testdata/imports/tracker/t.go b/internal/archtest/testdata/imports/tracker/t.go new file mode 100644 index 0000000..e20de77 --- /dev/null +++ b/internal/archtest/testdata/imports/tracker/t.go @@ -0,0 +1,3 @@ +package tracker + +import "time" diff --git a/internal/archtest/testdata/license/.hidden/h.go b/internal/archtest/testdata/license/.hidden/h.go new file mode 100644 index 0000000..ed4ec5b --- /dev/null +++ b/internal/archtest/testdata/license/.hidden/h.go @@ -0,0 +1 @@ +package hidden diff --git a/internal/archtest/testdata/license/_x/x.go b/internal/archtest/testdata/license/_x/x.go new file mode 100644 index 0000000..823aafd --- /dev/null +++ b/internal/archtest/testdata/license/_x/x.go @@ -0,0 +1 @@ +package x diff --git a/internal/archtest/testdata/license/gen.go b/internal/archtest/testdata/license/gen.go new file mode 100644 index 0000000..e3cf972 --- /dev/null +++ b/internal/archtest/testdata/license/gen.go @@ -0,0 +1,6 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +// Code generated by hand for a test. DO NOT EDIT. + +package license diff --git a/internal/archtest/testdata/license/line2.go b/internal/archtest/testdata/license/line2.go new file mode 100644 index 0000000..de6055a --- /dev/null +++ b/internal/archtest/testdata/license/line2.go @@ -0,0 +1,5 @@ +// note +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package license diff --git a/internal/archtest/testdata/license/mod/go.mod.tmpl b/internal/archtest/testdata/license/mod/go.mod.tmpl new file mode 100644 index 0000000..be2822f --- /dev/null +++ b/internal/archtest/testdata/license/mod/go.mod.tmpl @@ -0,0 +1,3 @@ +module example.com/mod + +go 1.26.0 diff --git a/internal/archtest/testdata/license/mod/x.go b/internal/archtest/testdata/license/mod/x.go new file mode 100644 index 0000000..0c751fa --- /dev/null +++ b/internal/archtest/testdata/license/mod/x.go @@ -0,0 +1 @@ +package mod diff --git a/internal/archtest/testdata/license/noblank.go b/internal/archtest/testdata/license/noblank.go new file mode 100644 index 0000000..323e571 --- /dev/null +++ b/internal/archtest/testdata/license/noblank.go @@ -0,0 +1,3 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 +package license diff --git a/internal/archtest/testdata/license/ok.go b/internal/archtest/testdata/license/ok.go new file mode 100644 index 0000000..222031f --- /dev/null +++ b/internal/archtest/testdata/license/ok.go @@ -0,0 +1,4 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package license diff --git a/internal/archtest/testdata/license/testdata/t.go b/internal/archtest/testdata/license/testdata/t.go new file mode 100644 index 0000000..69d29d3 --- /dev/null +++ b/internal/archtest/testdata/license/testdata/t.go @@ -0,0 +1 @@ +package testdata diff --git a/internal/archtest/testdata/license/vendor/v/v.go b/internal/archtest/testdata/license/vendor/v/v.go new file mode 100644 index 0000000..79eb93d --- /dev/null +++ b/internal/archtest/testdata/license/vendor/v/v.go @@ -0,0 +1 @@ +package v diff --git a/internal/archtest/testdata/license/year.go b/internal/archtest/testdata/license/year.go new file mode 100644 index 0000000..763d9e8 --- /dev/null +++ b/internal/archtest/testdata/license/year.go @@ -0,0 +1,4 @@ +// Copyright 2025 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package license diff --git a/internal/fakechain/bench_test.go b/internal/fakechain/bench_test.go new file mode 100644 index 0000000..4c8f99c --- /dev/null +++ b/internal/fakechain/bench_test.go @@ -0,0 +1,17 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package fakechain + +import ( + "testing" +) + +// BenchmarkStep measures one Chain.Step with default parameters; the target is 20 µs/op. +func BenchmarkStep(b *testing.B) { + c := New(1, Params{}) + b.ReportAllocs() + for b.Loop() { + c.Step() + } +} diff --git a/internal/fakechain/chain.go b/internal/fakechain/chain.go new file mode 100644 index 0000000..82c1115 --- /dev/null +++ b/internal/fakechain/chain.go @@ -0,0 +1,324 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package fakechain + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +// Params shape a generated chain. A zero field takes its default. +type Params struct { + ChainID uint64 // default 31337 + ForkPPM uint32 // chance that a Step is a reorg; default 150_000 + MaxForkDepth uint64 // deepest Step reorg; default 8 + MaxMine uint64 // a mining Step adds 1..MaxMine blocks; default 3 + FinalityPPM uint32 // chance that a Step advances finality; default 500_000 + MinLag uint64 // finality lag range, inclusive; default 2 + MaxLag uint64 // default 32 + MoveLogPPM uint32 // chance that an orphaned log is re-included by the reorg; default 300_000 +} + +func (p Params) withDefaults() Params { + def := func(v *uint64, d uint64) { + if *v == 0 { + *v = d + } + } + defPPM := func(v *uint32, d uint32) { + if *v == 0 { + *v = d + } + } + def(&p.ChainID, 31337) + defPPM(&p.ForkPPM, 150_000) + def(&p.MaxForkDepth, 8) + def(&p.MaxMine, 3) + defPPM(&p.FinalityPPM, 500_000) + def(&p.MinLag, 2) + def(&p.MaxLag, 32) + defPPM(&p.MoveLogPPM, 300_000) + return p +} + +// node is one block of the tree with all of its logs, unfiltered. +type node struct { + header chain.Header + logs []chain.Log +} + +// Chain is an in-memory block tree with a canonical head and a finalized number. The same seed, +// Params and method calls always give the same chain. It is not safe for concurrent use; while a +// Source serves it, change it only through Source.Do. +type Chain struct { + seed uint64 + p Params + nodes map[common.Hash]*node // every tree block; for lookup only, never ranged over + canonical []common.Hash // canonical[n] is the canonical block at height n + finalized uint64 + serial uint64 // serial of the last created block; genesis is 0 + txSerial uint64 // serial of the last fresh log's transaction + + // lastOrphan is the head the latest reorg of depth >= 1 orphaned (zero before any), and + // lastFork the number of that reorg's fork point. + lastOrphan common.Hash + lastFork uint64 + + chainRand *Rand // Step decisions + logRand *Rand // log content and log moves +} + +// New returns a chain holding only genesis: number 0, time GenesisTime, no logs, finalized. +func New(seed uint64, p Params) *Chain { + c := &Chain{ + seed: seed, + p: p.withDefaults(), + nodes: map[common.Hash]*node{}, + chainRand: NewRand(seed, "chain"), + logRand: NewRand(seed, "logs"), + } + ref := chain.BlockRef{Hash: blockHash(0, common.Hash{}, 0), Time: GenesisTime} + c.nodes[ref.Hash] = &node{header: chain.Header{Ref: ref}} + c.canonical = []common.Hash{ref.Hash} + return c +} + +// Params returns the parameters with defaults filled in. +func (c *Chain) Params() Params { + return c.p +} + +// Meta returns the chainlog metadata for a log of this chain with filter f: an ID derived from +// the seed, the chain ID, start block 0 and f. +func (c *Chain) Meta(f chain.Filter) chainlog.Meta { + h := keccak([]byte("evmsync/fakechain/log"), u64be(c.seed)) + var id chainlog.ID + copy(id[:], h[:len(id)]) + return chainlog.Meta{ID: id, ChainID: c.p.ChainID, Start: 0, Filter: f} +} + +// EventKind names a Step's event. +type EventKind uint8 + +const ( + EventMine EventKind = iota + 1 // blocks mined on the head + EventReorg // top Depth blocks replaced with Added blocks +) + +// Event describes one Step. +type Event struct { + Kind EventKind + Depth uint64 // EventReorg: the number of canonical blocks replaced + Added int // blocks created + Head chain.BlockRef // head after the event + Finalized uint64 // finalized number after the event +} + +// String returns "mine 2 -> head 41 0x1234abcd fin 30" or +// "reorg depth 3 +4 -> head 42 0x9876fedc fin 30", where the hash is the head hash's first 4 +// bytes in hex. +func (e Event) String() string { + if e.Kind == EventReorg { + return fmt.Sprintf("reorg depth %d +%d -> head %d 0x%x fin %d", + e.Depth, e.Added, e.Head.Number, e.Head.Hash[:4], e.Finalized) + } + return fmt.Sprintf("mine %d -> head %d 0x%x fin %d", e.Added, e.Head.Number, e.Head.Hash[:4], e.Finalized) +} + +// Step performs one seeded event and returns it. The event is a reorg above the finalized block +// with probability ForkPPM, else mining 1 to MaxMine blocks. Then, with probability FinalityPPM, +// finality moves up to a lag of MinLag to MaxLag blocks below the head. +func (c *Chain) Step() Event { + r := c.chainRand + ev := Event{Kind: EventMine} + fork := r.Chance(c.p.ForkPPM) + if head := c.head(); fork && head > c.finalized { + maxD := min(c.p.MaxForkDepth, head-c.finalized) + d := uint64(1) + for d < maxD && r.Chance(500_000) { + d++ + } + n := int(d - 1 + r.Intn(3)) //nolint:gosec // at most MaxForkDepth + 1 blocks + c.Reorg(d, n) + ev = Event{Kind: EventReorg, Depth: d, Added: n} + } else { + n := int(1 + r.Intn(c.p.MaxMine)) //nolint:gosec // at most MaxMine blocks + c.Mine(n) + ev.Added = n + } + adv := r.Chance(c.p.FinalityPPM) + lag := c.p.MinLag + r.Intn(c.p.MaxLag-c.p.MinLag+1) + if head := c.head(); adv && head >= lag && head-lag > c.finalized { + c.SetFinalized(head - lag) + } + ev.Head = c.Head().Ref + ev.Finalized = c.finalized + return ev +} + +// Mine appends n blocks to the head and returns their refs, ascending. Each block draws fresh +// logs. It panics if n is negative. +func (c *Chain) Mine(n int) []chain.BlockRef { + if n < 0 { + panic("evmsync: fakechain: negative block count") + } + refs := make([]chain.BlockRef, 0, n) + parent := c.Head().Ref + for range n { + parent = c.create(parent, nil) + refs = append(refs, parent) + } + return refs +} + +// Reorg replaces the top depth canonical blocks with n new blocks on the block depth below the +// head, and returns the new refs, ascending. n may be 0 (a rollback). Each orphaned log moves to a +// random new block with probability MoveLogPPM, keeping its address, topics, data and transaction +// hash; the orphans stay in the tree. Unlike Step, Reorg ignores finality, so tests can reorg below +// it; finality then drops to the new head if it was above it. It panics if depth is above the head +// number or n is negative. +func (c *Chain) Reorg(depth uint64, n int) []chain.BlockRef { + if n < 0 { + panic("evmsync: fakechain: negative block count") + } + head := c.head() + if depth > head { + panic(fmt.Sprintf("evmsync: fakechain: reorg depth %d above head %d", depth, head)) + } + fork := head - depth + + var pool []chain.Log + for num := fork + 1; num <= head; num++ { + for _, l := range c.nodes[c.canonical[num]].logs { + if c.logRand.Chance(c.p.MoveLogPPM) { + pool = append(pool, l) + } + } + } + moved := make([][]chain.Log, n) + if n > 0 { + for _, l := range pool { + k := c.logRand.Intn(uint64(n)) + moved[k] = append(moved[k], l) + } + } + + if depth >= 1 { + c.lastOrphan, c.lastFork = c.canonical[head], fork + } + c.canonical = c.canonical[:fork+1] + refs := make([]chain.BlockRef, 0, n) + parent := c.nodes[c.canonical[fork]].header.Ref + for i := range n { + parent = c.create(parent, moved[i]) + refs = append(refs, parent) + } + c.finalized = min(c.finalized, c.head()) + return refs +} + +// create makes a canonical block on parent with fresh logs followed by moved, and returns its ref. +func (c *Chain) create(parent chain.BlockRef, moved []chain.Log) chain.BlockRef { + c.serial++ + num := parent.Number + 1 + ref := chain.BlockRef{ + Number: num, + Hash: blockHash(num, parent.Hash, c.serial), + Parent: parent.Hash, + Time: GenesisTime + 12*num, + } + logs := append(c.freshLogs(), moved...) + for i := range logs { + l := &logs[i] + l.BlockNumber, l.BlockHash = num, ref.Hash + l.Index, l.TxIndex = uint32(i), uint32(i) //nolint:gosec // a block holds a few logs + } + if len(logs) == 0 { + logs = nil + } + c.nodes[ref.Hash] = &node{header: chain.Header{Ref: ref, Bloom: chain.BloomOf(logs)}, logs: logs} + c.canonical = append(c.canonical, ref.Hash) + return ref +} + +// SetFinalized sets the finalized number. It panics if n is below the current finalized number or +// above the head. +func (c *Chain) SetFinalized(n uint64) { + if n < c.finalized || n > c.head() { + panic(fmt.Sprintf("evmsync: fakechain: finalized %d outside [%d, %d]", n, c.finalized, c.head())) + } + c.finalized = n +} + +// head returns the head's number. +func (c *Chain) head() uint64 { + return uint64(len(c.canonical) - 1) //nolint:gosec // canonical always holds genesis +} + +// Genesis returns the header of block 0. +func (c *Chain) Genesis() chain.Header { + return c.nodes[c.canonical[0]].header +} + +// Head returns the canonical head's header. +func (c *Chain) Head() chain.Header { + return c.nodes[c.canonical[c.head()]].header +} + +// Finalized returns the canonical header at the finalized number. +func (c *Chain) Finalized() chain.Header { + return c.nodes[c.canonical[c.finalized]].header +} + +// Canonical returns the canonical header at height n, and false when n is above the head. +func (c *Chain) Canonical(n uint64) (chain.Header, bool) { + if n > c.head() { + return chain.Header{}, false + } + return c.nodes[c.canonical[n]].header, true +} + +// Header returns any tree block's header, canonical or orphan, and false for an unknown hash. +func (c *Chain) Header(h common.Hash) (chain.Header, bool) { + nd, ok := c.nodes[h] + if !ok { + return chain.Header{}, false + } + return nd.header, true +} + +// Block returns a tree block with only its logs that match f, in Index order, and false for an +// unknown hash. Logs is nil when no log matches. The result is a copy. +func (c *Chain) Block(h common.Hash, f chain.Filter) (chain.Block, bool) { + nd, ok := c.nodes[h] + if !ok { + return chain.Block{}, false + } + return chain.Block{Ref: nd.header.Ref, Logs: matching(nd.logs, f)}, true +} + +// CanonicalBlocks returns canonical blocks 1 to the head with their logs that match f, ascending, +// including blocks with no matching log. The result is a copy. +func (c *Chain) CanonicalBlocks(f chain.Filter) []chain.Block { + blocks := make([]chain.Block, 0, c.head()) + for _, h := range c.canonical[1:] { + nd := c.nodes[h] + blocks = append(blocks, chain.Block{Ref: nd.header.Ref, Logs: matching(nd.logs, f)}) + } + return blocks +} + +// Orphaned reports whether h is a tree block that is not canonical. +func (c *Chain) Orphaned(h common.Hash) bool { + nd, ok := c.nodes[h] + return ok && !c.isCanonical(nd.header.Ref) +} + +func (c *Chain) isCanonical(r chain.BlockRef) bool { + return r.Number <= c.head() && c.canonical[r.Number] == r.Hash +} diff --git a/internal/fakechain/chain_test.go b/internal/fakechain/chain_test.go new file mode 100644 index 0000000..20c50b9 --- /dev/null +++ b/internal/fakechain/chain_test.go @@ -0,0 +1,626 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package fakechain + +import ( + "encoding/binary" + "slices" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +func TestFixedHashes(t *testing.T) { + genesis := common.HexToHash("0xe8f558fba9e9da77d722ed1ebf6ba8cfab9cbcad9c02b7956519c654125cbee2") + for _, seed := range []uint64{0, 1, 99} { + g := New(seed, Params{}).Genesis() + want := chain.Header{Ref: chain.BlockRef{Number: 0, Hash: genesis, Time: GenesisTime}} + if g != want { + t.Errorf("seed %d: Genesis() = %+v, want %+v", seed, g, want) + } + } + + c := New(1, Params{}) + refs := c.Mine(1) + want1 := common.HexToHash("0x29a0a0981921f5e87dc1acc7b3daac8046cda355a5dc98a4f726c71b70741428") + if len(refs) != 1 || refs[0].Hash != want1 { + t.Fatalf("Mine(1) = %v, want block 1 %s", refs, want1.Hex()) + } + + wantTx := common.HexToHash("0x599a841fa5fe37f98b3d243880780724ce325688e4f206abf98500b6ed31a865") + if got := txHash(1); got != wantTx { + t.Errorf("txHash(1) = %s, want %s", got.Hex(), wantTx.Hex()) + } + // The first fresh log the chain creates is tx serial 1. + for c.Head().Ref.Number < 50 { + blocks := c.CanonicalBlocks(chain.Filter{}) + if i := slices.IndexFunc(blocks, func(b chain.Block) bool { return len(b.Logs) > 0 }); i >= 0 { + if got := blocks[i].Logs[0].TxHash; got != wantTx { + t.Errorf("first log TxHash = %s, want %s", got.Hex(), wantTx.Hex()) + } + break + } + c.Mine(1) + } + + topics := []struct { + sig string + want common.Hash + }{ + {"Transfer(address,address,uint256)", TransferTopic}, + {"Approval(address,address,uint256)", ApprovalTopic}, + {"Other(uint256)", OtherTopic}, + } + for _, tt := range topics { + if got := keccak([]byte(tt.sig)); got != tt.want { + t.Errorf("keccak(%q) = %s, want %s", tt.sig, got.Hex(), tt.want.Hex()) + } + } + + meta := New(1, Params{}).Meta(DefaultFilter) + wantID := chainlog.ID(common.FromHex("0x1624a058136b4172d71fe4203c66094f")) + if meta.ID != wantID { + t.Errorf("Meta.ID = %x, want %x", meta.ID, wantID) + } + if meta.ChainID != 31337 || meta.Start != 0 { + t.Errorf("Meta = %+v, want ChainID 31337, Start 0", meta) + } + if !slices.Equal(meta.Filter.Addresses, DefaultFilter.Addresses) || len(meta.Filter.Topics) != 1 || + !slices.Equal(meta.Filter.Topics[0], DefaultFilter.Topics[0]) { + t.Errorf("Meta.Filter = %+v, want DefaultFilter", meta.Filter) + } + if id := New(2, Params{}).Meta(DefaultFilter).ID; id == wantID { + t.Error("Meta.ID does not depend on the seed") + } + if m := New(1, Params{ChainID: 5}).Meta(chain.Filter{}); m.ChainID != 5 || m.Filter.Addresses != nil || + m.Filter.Topics != nil { + t.Errorf("Meta(Filter{}) = %+v, want ChainID 5 and the empty filter", m) + } +} + +func TestFixtures(t *testing.T) { + for i := range 4 { + a := Address(i) + var want common.Address + want[0], want[19] = 0x10, byte(i) + if a != want { + t.Errorf("Address(%d) = %s, want %s", i, a.Hex(), want.Hex()) + } + } + for j := range 8 { + a := Account(j) + var want common.Address + want[0], want[19] = 0x20, byte(j) + if a != want { + t.Errorf("Account(%d) = %s, want %s", j, a.Hex(), want.Hex()) + } + } + wantPanic(t, "evmsync: fakechain: fixture index 4 out of range", func() { Address(4) }) + wantPanic(t, "evmsync: fakechain: fixture index -1 out of range", func() { Address(-1) }) + wantPanic(t, "evmsync: fakechain: fixture index 8 out of range", func() { Account(8) }) + wantPanic(t, "evmsync: fakechain: fixture index -1 out of range", func() { Account(-1) }) + + for i := range 4 { + l := chain.Log{Address: Address(i), Topics: []common.Hash{TransferTopic, {}, {}}} + if got, want := DefaultFilter.Match(l), i < 2; got != want { + t.Errorf("DefaultFilter.Match(Transfer of contract %d) = %v, want %v", i, got, want) + } + } + for _, topic := range []common.Hash{ApprovalTopic, OtherTopic} { + l := chain.Log{Address: Address(0), Topics: []common.Hash{topic}} + if got, want := DefaultFilter.Match(l), topic == ApprovalTopic; got != want { + t.Errorf("DefaultFilter.Match(%s) = %v, want %v", topic.Hex(), got, want) + } + } +} + +func TestParams(t *testing.T) { + want := Params{ + ChainID: 31337, ForkPPM: 150_000, MaxForkDepth: 8, MaxMine: 3, FinalityPPM: 500_000, + MinLag: 2, MaxLag: 32, MoveLogPPM: 300_000, + } + if got := New(1, Params{}).Params(); got != want { + t.Errorf("Params() = %+v, want %+v", got, want) + } + set := Params{ + ChainID: 1, ForkPPM: 2, MaxForkDepth: 3, MaxMine: 4, FinalityPPM: 5, MinLag: 6, MaxLag: 7, MoveLogPPM: 8, + } + if got := New(1, set).Params(); got != set { + t.Errorf("Params() = %+v, want %+v", got, set) + } +} + +// TestFreshLogs derives the logs of blocks 1 to 3 from the logs stream exactly as the generator must draw +// them, and compares. +func TestFreshLogs(t *testing.T) { + counts := []int{0, 0, 0, 1, 1, 1, 2, 2, 3, 4} + for seed := uint64(1); seed <= 30; seed++ { + r := NewRand(seed, "logs") + var want []chain.Log + tx := uint64(0) + for range 3 { // blocks 1..3 + n := counts[r.Intn(10)] + var logs []chain.Log + for i := range n { + a := int(r.Intn(4)) //nolint:gosec // below 4 + k := r.Intn(10) + l := chain.Log{Address: Address(a), Index: uint32(i), TxIndex: uint32(i)} + data := make([]byte, 32) + if k <= 7 { + topic := TransferTopic + if k >= 6 { + topic = ApprovalTopic + } + from, to := int(r.Intn(8)), int(r.Intn(8)) //nolint:gosec // below 8 + amount := 1 + r.Intn(1000) + l.Topics = []common.Hash{topic, pad(Account(from)), pad(Account(to))} + binary.BigEndian.PutUint64(data[24:], amount) + } else { + l.Topics = []common.Hash{OtherTopic} + binary.BigEndian.PutUint64(data[24:], r.Uint64()) + } + l.Data = data + tx++ + l.TxHash = txHash(tx) + logs = append(logs, l) + } + want = append(want, logs...) + } + c := New(seed, Params{}) + c.Mine(3) + var got []chain.Log + for _, b := range c.CanonicalBlocks(chain.Filter{}) { + for i := range b.Logs { + l := b.Logs[i] + if l.BlockNumber != b.Ref.Number || l.BlockHash != b.Ref.Hash { + t.Errorf("seed %d: log %d of block %d names block %d %s", seed, i, b.Ref.Number, + l.BlockNumber, l.BlockHash.Hex()) + } + l.BlockNumber, l.BlockHash = 0, common.Hash{} + got = append(got, l) + } + } + if !logsEqual(got, want) { + t.Fatalf("seed %d: logs of blocks 1..3 =\n%+v\nwant\n%+v", seed, got, want) + } + } +} + +func pad(a common.Address) common.Hash { + var h common.Hash + copy(h[12:], a[:]) + return h +} + +// cloneLogs deep-copies logs, so an expected value never shares memory with a result under test. +func cloneLogs(logs []chain.Log) []chain.Log { + out := slices.Clone(logs) + for i := range out { + out[i].Topics = slices.Clone(out[i].Topics) + out[i].Data = slices.Clone(out[i].Data) + } + return out +} + +func logsEqual(a, b []chain.Log) bool { + return slices.EqualFunc(a, b, func(x, y chain.Log) bool { + return x.Address == y.Address && slices.Equal(x.Topics, y.Topics) && slices.Equal(x.Data, y.Data) && + x.BlockNumber == y.BlockNumber && x.BlockHash == y.BlockHash && x.TxHash == y.TxHash && + x.TxIndex == y.TxIndex && x.Index == y.Index + }) +} + +func TestBloom(t *testing.T) { + b := chain.BloomOf([]chain.Log{{Address: Address(0), Topics: []common.Hash{TransferTopic}}}) + want := map[int]byte{53: 0x02, 73: 0x08, 75: 0x08, 123: 0x10, 176: 0x40, 195: 0x02} + for i, v := range b { + if v != want[i] { + t.Errorf("bloom byte %d = %#x, want %#x", i, v, want[i]) + } + } + + c := New(3, Params{}) + c.Mine(100) + var withLogs, matching int + for n := uint64(1); n <= 100; n++ { + h, ok := c.Canonical(n) + if !ok { + t.Fatalf("Canonical(%d) missing", n) + } + all, _ := c.Block(h.Ref.Hash, chain.Filter{}) + if h.Bloom != chain.BloomOf(all.Logs) { + t.Errorf("block %d: bloom is not the bloom of its logs", n) + } + if got, want := h.Bloom.IsZero(), len(all.Logs) == 0; got != want { + t.Errorf("block %d: Bloom.IsZero() = %v, want %v", n, got, want) + } + sel, _ := c.Block(h.Ref.Hash, DefaultFilter) + if len(sel.Logs) > 0 && !h.Bloom.MayContain(DefaultFilter) { + t.Errorf("block %d: MayContain(DefaultFilter) = false with %d matching logs", n, len(sel.Logs)) + } + if len(all.Logs) > 0 { + withLogs++ + } + if len(sel.Logs) > 0 { + matching++ + } + } + if withLogs == 0 || withLogs == 100 || matching == 0 { + t.Errorf("seed 3: %d blocks with logs, %d with matching logs; the check covers no case", withLogs, matching) + } +} + +func TestStepInvariants(t *testing.T) { + var shorter, rollback, moved bool + for seed := uint64(1); seed <= 200; seed++ { + c := New(seed, Params{MaxForkDepth: 8, ForkPPM: 400_000}) + for i := range 500 { + head, fin := c.Head().Ref.Number, c.Finalized().Ref.Number + ev := c.Step() + if ev.Kind == EventReorg { + if ev.Depth > head || head-ev.Depth < fin { + t.Fatalf("seed %d step %d: reorg depth %d from head %d forks below finalized %d", + seed, i, ev.Depth, head, fin) + } + shorter = shorter || (ev.Added == int(ev.Depth)-1 && ev.Added > 0) //nolint:gosec // depth <= 8 + rollback = rollback || ev.Added == 0 + } + if got := c.Finalized().Ref.Number; got < fin { + t.Fatalf("seed %d step %d: finalized moved from %d down to %d", seed, i, fin, got) + } + if ev.Head != c.Head().Ref || ev.Finalized != c.Finalized().Ref.Number { + t.Fatalf("seed %d step %d: event %v does not match head %v fin %d", seed, i, ev, + c.Head().Ref, c.Finalized().Ref.Number) + } + } + prev := c.Genesis().Ref + for n := uint64(1); n <= c.Head().Ref.Number; n++ { + h, _ := c.Canonical(n) + if h.Ref.Parent != prev.Hash || h.Ref.Number != n || h.Ref.Time != GenesisTime+12*n { + t.Fatalf("seed %d: canonical block %d is %+v after %s", seed, n, h.Ref, prev) + } + prev = h.Ref + } + if !moved { + moved = movedLog(c) + } + } + if !shorter || !rollback || !moved { + t.Errorf("coverage: shorter branch %v, rollback %v, moved log %v; want all true", shorter, rollback, moved) + } +} + +// movedLog reports whether a TxHash appears in both an orphan and a canonical block. +func movedLog(c *Chain) bool { + canon := map[common.Hash]bool{} + for _, b := range c.CanonicalBlocks(chain.Filter{}) { + for _, l := range b.Logs { + canon[l.TxHash] = true + } + } + for h, nd := range c.nodes { + if !c.Orphaned(h) { + continue + } + for _, l := range nd.logs { + if canon[l.TxHash] { + return true + } + } + } + return false +} + +func TestExplicitAPI(t *testing.T) { + c := New(1, Params{}) + g := c.Genesis().Ref + refs := c.Mine(5) + if len(refs) != 5 { + t.Fatalf("Mine(5) returned %d refs", len(refs)) + } + prev := g + for i, r := range refs { + if r.Number != uint64(i+1) || r.Parent != prev.Hash || r.Time != GenesisTime+12*r.Number { + t.Errorf("Mine(5) ref %d = %+v after %+v", i, r, prev) + } + prev = r + } + if c.Head().Ref != refs[4] { + t.Errorf("Head() = %v, want %v", c.Head().Ref, refs[4]) + } + + re := c.Reorg(2, 3) + if len(re) != 3 || re[0].Number != 4 || re[1].Number != 5 || re[2].Number != 6 { + t.Fatalf("Reorg(2, 3) = %v, want blocks 4, 5, 6", re) + } + if re[0].Parent != refs[2].Hash || re[1].Parent != re[0].Hash || re[2].Parent != re[1].Hash { + t.Errorf("Reorg(2, 3) refs %v are not linked to block 3 %s", re, refs[2].Hash.Hex()) + } + for _, r := range refs[3:] { + if !c.Orphaned(r.Hash) { + t.Errorf("first-branch block %d not Orphaned", r.Number) + } + if h, ok := c.Header(r.Hash); !ok || h.Ref != r { + t.Errorf("Header(orphan %d) = %v, %v", r.Number, h.Ref, ok) + } + if _, ok := c.Block(r.Hash, chain.Filter{}); !ok { + t.Errorf("Block(orphan %d) not found", r.Number) + } + } + for _, r := range append(refs[:3:3], re...) { + if c.Orphaned(r.Hash) { + t.Errorf("canonical block %d reported Orphaned", r.Number) + } + if h, ok := c.Canonical(r.Number); !ok || h.Ref != r { + t.Errorf("Canonical(%d) = %v, %v, want %v", r.Number, h.Ref, ok, r) + } + } + if c.Orphaned(common.Hash{1}) { + t.Error("Orphaned(unknown) = true") + } + if _, ok := c.Canonical(7); ok { + t.Error("Canonical(7) above head found") + } + if _, ok := c.Header(common.Hash{1}); ok { + t.Error("Header(unknown) found") + } + if _, ok := c.Block(common.Hash{1}, chain.Filter{}); ok { + t.Error("Block(unknown) found") + } + + wantPanic(t, "evmsync: fakechain: reorg depth 7 above head 6", func() { c.Reorg(7, 0) }) + wantPanic(t, "evmsync: fakechain: negative block count", func() { c.Reorg(1, -1) }) + wantPanic(t, "evmsync: fakechain: negative block count", func() { c.Mine(-1) }) + wantPanic(t, "evmsync: fakechain: finalized 7 outside [0, 6]", func() { c.SetFinalized(7) }) + c.SetFinalized(2) + c.SetFinalized(2) // the current number is allowed + if c.Finalized().Ref != refs[1] { + t.Errorf("Finalized() = %v, want block 2", c.Finalized().Ref) + } + wantPanic(t, "evmsync: fakechain: finalized 1 outside [2, 6]", func() { c.SetFinalized(1) }) + if got := c.Reorg(0, 0); len(got) != 0 || c.Head().Ref != re[2] { + t.Errorf("Reorg(0, 0) = %v, head %v; want no change", got, c.Head().Ref) + } + + low := c.Reorg(6, 1) + if len(low) != 1 || low[0].Number != 1 || low[0].Parent != g.Hash { + t.Fatalf("Reorg(6, 1) = %v, want one block 1 on genesis", low) + } + if got := c.Finalized().Ref; got != low[0] { + t.Errorf("after Reorg(6, 1) Finalized() = %v, want new block 1", got) + } + + top := New(1, Params{}) + top.Mine(3) + top.SetFinalized(3) + if top.Finalized() != top.Head() { + t.Errorf("SetFinalized(head): Finalized() = %v, want head %v", top.Finalized().Ref, top.Head().Ref) + } + + // Returned values are copies. + c = New(4, Params{}) + c.Mine(30) + var h common.Hash + for _, b := range c.CanonicalBlocks(chain.Filter{}) { + if len(b.Logs) > 0 { + h = b.Ref.Hash + break + } + } + first, _ := c.Block(h, chain.Filter{}) + kb, _ := c.Block(h, chain.Filter{}) + keep := cloneLogs(kb.Logs) + first.Logs[0].Topics[0] = common.Hash{9} + first.Logs[0].Data[31] ^= 0xff + first.Logs[0].Address = common.Address{9} + first.Logs = first.Logs[:0] + second, _ := c.Block(h, chain.Filter{}) + if !logsEqual(second.Logs, keep) { + t.Error("mutating a returned Block changed a later Block result") + } + blocks := c.CanonicalBlocks(chain.Filter{}) + for i := range blocks { + for j := range blocks[i].Logs { + blocks[i].Logs[j].Topics[0] = common.Hash{9} + } + blocks[i].Logs = nil + } + again, _ := c.Block(h, chain.Filter{}) + if !logsEqual(again.Logs, keep) { + t.Error("mutating CanonicalBlocks changed a later Block result") + } +} + +func TestBlockFilter(t *testing.T) { + c := New(6, Params{}) + c.Mine(40) + var none, some int + for _, b := range c.CanonicalBlocks(DefaultFilter) { + all, _ := c.Block(b.Ref.Hash, chain.Filter{}) + var want []chain.Log + for _, l := range all.Logs { + if DefaultFilter.Match(l) { + want = append(want, l) + } + } + if !logsEqual(b.Logs, want) { + t.Errorf("block %d: CanonicalBlocks(DefaultFilter) logs %v, want %v", b.Ref.Number, b.Logs, want) + } + if len(want) == 0 { + none++ + if b.Logs != nil { + t.Errorf("block %d: Logs = %v, want nil", b.Ref.Number, b.Logs) + } + } else { + some++ + } + got, _ := c.Block(b.Ref.Hash, DefaultFilter) + if got.Ref != b.Ref || !logsEqual(got.Logs, want) { + t.Errorf("block %d: Block(DefaultFilter) = %+v, want %+v", b.Ref.Number, got, b) + } + } + if none == 0 || some == 0 { + t.Errorf("seed 6: %d blocks without and %d with matching logs; the check covers no case", none, some) + } + if n := len(c.CanonicalBlocks(DefaultFilter)); n != 40 { + t.Errorf("CanonicalBlocks returned %d blocks, want 40", n) + } +} + +func TestReorgMovesLogs(t *testing.T) { + for seed := uint64(1); seed <= 20; seed++ { + c := New(seed, Params{MoveLogPPM: 1_000_000}) + c.Mine(6) + var pool []chain.Log + for n := uint64(4); n <= 6; n++ { + h, _ := c.Canonical(n) + b, _ := c.Block(h.Ref.Hash, chain.Filter{}) + pool = append(pool, b.Logs...) + } + // Every orphaned log moves (one Chance each); then Intn(2) per pool log picks its block. + model := *c.logRand + for range pool { + model.Uint64() + } + target := map[common.Hash]uint64{} + for _, l := range pool { + target[l.TxHash] = 4 + model.Intn(2) + } + c.Reorg(3, 2) + var gotMoved []chain.Log + for n := uint64(4); n <= 5; n++ { + h, _ := c.Canonical(n) + b, _ := c.Block(h.Ref.Hash, chain.Filter{}) + if h.Bloom != chain.BloomOf(b.Logs) { + t.Fatalf("seed %d block %d: bloom is not the bloom of its fresh and moved logs", seed, n) + } + fresh, lastPool := true, -1 + for i, l := range b.Logs { + if l.BlockNumber != n || l.BlockHash != h.Ref.Hash { + t.Fatalf("seed %d block %d: log %d names block %d %s", seed, n, i, l.BlockNumber, l.BlockHash.Hex()) + } + if l.Index != uint32(i) || l.TxIndex != uint32(i) { + t.Fatalf("seed %d block %d: log %d has Index %d TxIndex %d", seed, n, i, l.Index, l.TxIndex) + } + pi := slices.IndexFunc(pool, func(p chain.Log) bool { return p.TxHash == l.TxHash }) + if pi < 0 && !fresh { + t.Fatalf("seed %d block %d: fresh log %d after a moved log", seed, n, i) + } + if pi >= 0 { + if pi < lastPool { + t.Fatalf("seed %d block %d: moved log %d is pool log %d after pool log %d", seed, n, i, pi, lastPool) + } + if target[l.TxHash] != n { + t.Fatalf("seed %d: pool log %d moved to block %d, want %d", seed, pi, n, target[l.TxHash]) + } + fresh, lastPool = false, pi + gotMoved = append(gotMoved, l) + } + } + } + if len(gotMoved) != len(pool) { + t.Fatalf("seed %d: %d logs moved, want all %d (MoveLogPPM 1,000,000)", seed, len(gotMoved), len(pool)) + } + // Moved logs keep their content. + for _, l := range gotMoved { + i := slices.IndexFunc(pool, func(p chain.Log) bool { return p.TxHash == l.TxHash }) + p := pool[i] + if p.Address != l.Address || !slices.Equal(p.Topics, l.Topics) || !slices.Equal(p.Data, l.Data) { + t.Errorf("seed %d: moved log %s changed content", seed, l.TxHash.Hex()) + } + } + } + + // A rollback drops the pool with no target draw: the logs stream advances by one Chance per + // orphaned log only. + c := New(2, Params{}) + c.Mine(8) + orphaned := 0 + for n := uint64(5); n <= 8; n++ { + h, _ := c.Canonical(n) + b, _ := c.Block(h.Ref.Hash, chain.Filter{}) + orphaned += len(b.Logs) + } + twin := *c.logRand + c.Reorg(4, 0) + for range orphaned { + twin.Uint64() + } + if c.logRand.Uint64() != twin.Uint64() { + t.Errorf("Reorg(4, 0) over %d orphaned logs drew a different number of values", orphaned) + } + if orphaned == 0 { + t.Error("seed 2: no orphaned logs; the rollback check covers no case") + } +} + +func TestDeterminism(t *testing.T) { + run := func() string { + c := New(11, Params{ForkPPM: 400_000}) + var evs []Event + for range 300 { + evs = append(evs, c.Step()) + } + c.Reorg(2, 1) + c.Mine(2) + return chainDigest(c, evs) + } + if a, b := run(), run(); a != b { + t.Errorf("same seed and calls gave digests %s and %s", a, b) + } +} + +func TestEventString(t *testing.T) { + h := common.HexToHash("0x1234abcd00000000000000000000000000000000000000000000000000000000") + e := Event{Kind: EventMine, Added: 2, Head: chain.BlockRef{Number: 41, Hash: h}, Finalized: 30} + if got, want := e.String(), "mine 2 -> head 41 0x1234abcd fin 30"; got != want { + t.Errorf("String() = %q, want %q", got, want) + } + h2 := common.HexToHash("0x9876fedc00000000000000000000000000000000000000000000000000000000") + e = Event{Kind: EventReorg, Depth: 3, Added: 4, Head: chain.BlockRef{Number: 42, Hash: h2}, Finalized: 30} + if got, want := e.String(), "reorg depth 3 +4 -> head 42 0x9876fedc fin 30"; got != want { + t.Errorf("String() = %q, want %q", got, want) + } +} + +// TestStepDraws checks Step's draw order against a model on the chain stream. +func TestStepDraws(t *testing.T) { + for seed := uint64(1); seed <= 50; seed++ { + p := Params{ForkPPM: 400_000, MaxForkDepth: 5, MaxMine: 4, FinalityPPM: 600_000, MinLag: 1, MaxLag: 6} + c := New(seed, p) + r := NewRand(seed, "chain") + for i := range 200 { + head, fin := c.Head().Ref.Number, c.Finalized().Ref.Number + want := Event{} + if r.Chance(p.ForkPPM) && head > fin { + maxD := min(p.MaxForkDepth, head-fin) + d := uint64(1) + for d < maxD && r.Chance(500_000) { + d++ + } + n := int(d - 1 + r.Intn(3)) //nolint:gosec // at most MaxForkDepth + 1 + want = Event{Kind: EventReorg, Depth: d, Added: n} + head = head - d + uint64(n) + } else { + n := int(1 + r.Intn(p.MaxMine)) //nolint:gosec // at most MaxMine + want = Event{Kind: EventMine, Added: n} + head += uint64(n) //nolint:gosec // n >= 1 + } + adv := r.Chance(p.FinalityPPM) + lag := p.MinLag + r.Intn(p.MaxLag-p.MinLag+1) + if adv && head >= lag && head-lag > fin { + fin = head - lag + } + got := c.Step() + if got.Kind != want.Kind || got.Depth != want.Depth || got.Added != want.Added || + got.Head.Number != head || got.Finalized != fin { + t.Fatalf("seed %d step %d: Step() = %v, want kind %d depth %d added %d head %d fin %d", + seed, i, got, want.Kind, want.Depth, want.Added, head, fin) + } + } + } +} diff --git a/internal/fakechain/contract_test.go b/internal/fakechain/contract_test.go new file mode 100644 index 0000000..6299395 --- /dev/null +++ b/internal/fakechain/contract_test.go @@ -0,0 +1,240 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package fakechain + +import ( + "context" + "errors" + "slices" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" +) + +// contractChain builds the chain.Source contract fixture: blocks 0..10 canonical, a side block s7 +// known by hash only, finalized block 5. +func contractChain(seed uint64) (c *Chain, s7 chain.Header) { + c = New(seed, Params{}) + c.Mine(7) + s7 = c.Head() + c.Reorg(1, 1) + c.Mine(3) + c.SetFinalized(5) + return c, s7 +} + +// contractSeed returns the first seed from 1 whose canonical block 7 has at least two logs with +// pairwise different (Address, Topics) and whose block 8 has none. +func contractSeed(t *testing.T) uint64 { + t.Helper() + for seed := uint64(1); seed <= 1000; seed++ { + c, _ := contractChain(seed) + h7, _ := c.Canonical(7) + h8, _ := c.Canonical(8) + b7, _ := c.Block(h7.Ref.Hash, chain.Filter{}) + b8, _ := c.Block(h8.Ref.Hash, chain.Filter{}) + if len(b7.Logs) >= 2 && len(b8.Logs) == 0 && distinctLogs(b7.Logs) { + return seed + } + } + t.Fatal("no seed in 1..1000 gives the contract fixture") + return 0 +} + +func distinctLogs(logs []chain.Log) bool { + for i := range logs { + for j := i + 1; j < len(logs); j++ { + if logs[i].Address == logs[j].Address && slices.Equal(logs[i].Topics, logs[j].Topics) { + return false + } + } + } + return true +} + +func TestSourceContract(t *testing.T) { + seed := contractSeed(t) + c, s7 := contractChain(seed) + src := NewSource(c, seed, Faults{}) + ctx := t.Context() + h := func(n uint64) chain.Header { + hd, _ := c.Canonical(n) + return hd + } + random := common.HexToHash("0xdeadbeef") + b7, _ := c.Block(h(7).Ref.Hash, chain.Filter{}) + b7.Logs = cloneLogs(b7.Logs) + last := b7.Logs[len(b7.Logs)-1] + lastFilter := chain.Filter{Addresses: []common.Address{last.Address}} + for _, tp := range last.Topics { + lastFilter.Topics = append(lastFilter.Topics, []common.Hash{tp}) + } + + header := func(want chain.Header, call func() (chain.Header, error)) func(t *testing.T) { + return func(t *testing.T) { + got, err := call() + if err != nil || got != want { + t.Errorf("got %v, %v, want %v", got.Ref, err, want.Ref) + } + } + } + fails := func(target error, call func() error) func(t *testing.T) { + return func(t *testing.T) { + if err := call(); !errors.Is(err, target) { + t.Errorf("error %v, want one wrapping %v", err, target) + } + } + } + logs := func(want []chain.Log, call func() ([]chain.Log, error)) func(t *testing.T) { + return func(t *testing.T) { + got, err := call() + if err != nil || !logsEqual(got, want) || (got == nil) != (want == nil) { + t.Errorf("got %v, %v, want %v", got, err, want) + } + checkLogValues(t, c, got) + } + } + + var all []chain.Log + for _, b := range c.CanonicalBlocks(chain.Filter{}) { + all = append(all, cloneLogs(b.Logs)...) + } + unsupported := NewSource(c, seed, Faults{FinalizedUnsupported: true}) + + cases := []struct { + name string + check func(t *testing.T) + }{ + {"Latest", header(h(10), func() (chain.Header, error) { return src.Latest(ctx) })}, + {"Finalized", header(h(5), func() (chain.Header, error) { return src.Finalized(ctx) })}, + {"Finalized unsupported", fails(chain.ErrUnsupported, func() error { + _, err := unsupported.Finalized(ctx) + return err + })}, + {"HeaderByNumber(0)", header(c.Genesis(), func() (chain.Header, error) { return src.HeaderByNumber(ctx, 0) })}, + {"HeaderByNumber(10)", header(h(10), func() (chain.Header, error) { return src.HeaderByNumber(ctx, 10) })}, + {"HeaderByNumber(11)", fails(chain.ErrNotFound, func() error { + _, err := src.HeaderByNumber(ctx, 11) + return err + })}, + {"HeaderByHash(7)", header(h(7), func() (chain.Header, error) { return src.HeaderByHash(ctx, h(7).Ref.Hash) })}, + {"HeaderByHash(s7)", header(s7, func() (chain.Header, error) { return src.HeaderByHash(ctx, s7.Ref.Hash) })}, + {"HeaderByHash(random)", fails(chain.ErrNotFound, func() error { + _, err := src.HeaderByHash(ctx, random) + return err + })}, + {"HeaderByHash(zero)", fails(chain.ErrNotFound, func() error { + _, err := src.HeaderByHash(ctx, common.Hash{}) + return err + })}, + {"Logs(7)", logs(b7.Logs, func() ([]chain.Log, error) { return src.Logs(ctx, h(7).Ref.Hash, chain.Filter{}) })}, + {"Logs(8)", logs(nil, func() ([]chain.Log, error) { return src.Logs(ctx, h(8).Ref.Hash, chain.Filter{}) })}, + {"Logs(random)", fails(chain.ErrNotFound, func() error { + _, err := src.Logs(ctx, random, chain.Filter{}) + return err + })}, + {"Logs(7, last log's filter)", logs([]chain.Log{last}, func() ([]chain.Log, error) { + return src.Logs(ctx, h(7).Ref.Hash, lastFilter) + })}, + {"LogsInRange(0, 10)", logs(all, func() ([]chain.Log, error) { return src.LogsInRange(ctx, 0, 10, chain.Filter{}) })}, + {"LogsInRange(8, 8)", logs(nil, func() ([]chain.Log, error) { return src.LogsInRange(ctx, 8, 8, chain.Filter{}) })}, + {"LogsInRange(5, 4)", fails(chain.ErrInvalidRange, func() error { + _, err := src.LogsInRange(ctx, 5, 4, chain.Filter{}) + return err + })}, + {"LogsInRange(5, 11)", fails(chain.ErrNotFound, func() error { + _, err := src.LogsInRange(ctx, 5, 11, chain.Filter{}) + return err + })}, + {"cancelled context", func(t *testing.T) { checkCancelled(t, src, h(7).Ref.Hash) }}, + {"results belong to the caller", func(t *testing.T) { + first, _ := src.Logs(ctx, h(7).Ref.Hash, chain.Filter{}) + first[0].Topics[0] = common.Hash{1} + again, _ := src.Logs(ctx, h(7).Ref.Hash, chain.Filter{}) + if !logsEqual(again, b7.Logs) { + t.Error("mutating Topics[0] of a Logs result changed a later result") + } + rng, _ := src.LogsInRange(ctx, 0, 10, chain.Filter{}) + rng[0].Topics[0] = common.Hash{1} + rng2, _ := src.LogsInRange(ctx, 0, 10, chain.Filter{}) + if !logsEqual(rng2, all) { + t.Error("mutating Topics[0] of a LogsInRange result changed a later result") + } + }}, + {"filter is read-only", func(t *testing.T) { + f := chain.Filter{Addresses: []common.Address{Address(0), Address(1)}, + Topics: [][]common.Hash{{TransferTopic, ApprovalTopic}, nil}} + keep := chain.Filter{Addresses: slices.Clone(f.Addresses), + Topics: [][]common.Hash{slices.Clone(f.Topics[0]), nil}} + _, _ = src.Logs(ctx, h(7).Ref.Hash, f) + _, _ = src.LogsInRange(ctx, 0, 10, f) + if !slices.Equal(f.Addresses, keep.Addresses) || !slices.Equal(f.Topics[0], keep.Topics[0]) || + len(f.Topics) != 2 || f.Topics[1] != nil { + t.Error("a Source method changed its filter") + } + }}, + } + for _, tt := range cases { + t.Run(tt.name, tt.check) + } +} + +// checkLogValues checks the chain.Log rules for a result: each log names its block, the empty +// fields are nil, and each block's logs have strictly increasing Index and non-decreasing TxIndex. +func checkLogValues(t *testing.T, c *Chain, logs []chain.Log) { + t.Helper() + for i, l := range logs { + hd, ok := c.Header(l.BlockHash) + if !ok || hd.Ref.Number != l.BlockNumber { + t.Errorf("log %d names block %d %s, which is not a tree block", i, l.BlockNumber, l.BlockHash.Hex()) + } + if len(l.Topics) > 4 || (l.Topics != nil && len(l.Topics) == 0) || (l.Data != nil && len(l.Data) == 0) { + t.Errorf("log %d breaks the empty-is-nil or topic-count rule: %+v", i, l) + } + if i == 0 { + continue + } + p := logs[i-1] + switch { + case p.BlockNumber > l.BlockNumber: + t.Errorf("log %d: block %d after block %d", i, l.BlockNumber, p.BlockNumber) + case p.BlockNumber == l.BlockNumber && (p.Index >= l.Index || p.TxIndex > l.TxIndex): + t.Errorf("log %d of block %d: Index %d TxIndex %d after Index %d TxIndex %d", + i, l.BlockNumber, l.Index, l.TxIndex, p.Index, p.TxIndex) + } + } +} + +// checkCancelled calls every Source method with a cancelled context: each returns +// context.Canceled unchanged, which wraps none of the chain sentinels, and records no call. The +// arguments would otherwise fail (unknown hash, number above the head, bad range) or succeed (h). +func checkCancelled(t *testing.T, src *Source, h common.Hash) { + t.Helper() + ctx, cancel := context.WithCancel(t.Context()) + cancel() + before := len(src.Calls()) + calls := []struct { + name string + call func() error + }{ + {"Latest", func() error { _, err := src.Latest(ctx); return err }}, + {"Finalized", func() error { _, err := src.Finalized(ctx); return err }}, + {"HeaderByNumber", func() error { _, err := src.HeaderByNumber(ctx, 1<<40); return err }}, + {"HeaderByHash", func() error { _, err := src.HeaderByHash(ctx, h); return err }}, + {"Logs", func() error { _, err := src.Logs(ctx, common.Hash{}, chain.Filter{}); return err }}, + {"LogsInRange", func() error { _, err := src.LogsInRange(ctx, 5, 4, chain.Filter{}); return err }}, + } + for _, tt := range calls { + err := tt.call() + if err != context.Canceled || errors.Is(err, chain.ErrNotFound) || //nolint:errorlint // returned unchanged + errors.Is(err, chain.ErrUnsupported) || errors.Is(err, chain.ErrInvalidRange) { + t.Errorf("%s with a cancelled context: error %v, want context.Canceled", tt.name, err) + } + } + if n := len(src.Calls()) - before; n != 0 { + t.Errorf("cancelled calls recorded %d calls", n) + } +} diff --git a/internal/fakechain/doc.go b/internal/fakechain/doc.go new file mode 100644 index 0000000..6cbba14 --- /dev/null +++ b/internal/fakechain/doc.go @@ -0,0 +1,6 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +// Package fakechain is a deterministic, seeded fake EVM chain with fork injection and moving +// finality, and a chain.Source over it with fault hooks. It is for tests only. +package fakechain diff --git a/internal/fakechain/fixtures.go b/internal/fakechain/fixtures.go new file mode 100644 index 0000000..cfc1c12 --- /dev/null +++ b/internal/fakechain/fixtures.go @@ -0,0 +1,52 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package fakechain + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" +) + +// Address returns contract address i (0 <= i < 4): byte 0 is 0x10, byte 19 is i, the rest zero. +// It panics outside that range. +func Address(i int) common.Address { + return fixture(0x10, i, 4) +} + +// Account returns account address j (0 <= j < 8): byte 0 is 0x20, byte 19 is j, the rest zero. It +// panics outside that range. +func Account(j int) common.Address { + return fixture(0x20, j, 8) +} + +func fixture(prefix byte, i, n int) common.Address { + if i < 0 || i >= n { + panic(fmt.Sprintf("evmsync: fakechain: fixture index %d out of range", i)) + } + var a common.Address + a[0], a[19] = prefix, byte(i) //nolint:gosec // i < n <= 8, checked above + return a +} + +var ( + // TransferTopic is keccak256("Transfer(address,address,uint256)"). + TransferTopic = common.HexToHash("0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef") + // ApprovalTopic is keccak256("Approval(address,address,uint256)"). + ApprovalTopic = common.HexToHash("0x8c5be1e5ebec7d5bd14f71427d1e84f3dd0314c0f7b2291e5b200ac8c7c3b925") + // OtherTopic is keccak256("Other(uint256)"). + OtherTopic = common.HexToHash("0x42d43c8775dfae639a33729abbd197262a51e5f0a31afc5914fbf109e1fd527f") +) + +// DefaultFilter selects Transfer and Approval logs of contracts 0 and 1, so contracts 2 and 3 and +// Other logs are filtered out. +var DefaultFilter = chain.Filter{ + Addresses: []common.Address{Address(0), Address(1)}, + Topics: [][]common.Hash{{TransferTopic, ApprovalTopic}}, +} + +// GenesisTime is the Time of block 0, in unix seconds; block n has Time GenesisTime + 12*n. +const GenesisTime = 1_700_000_000 diff --git a/internal/fakechain/golden_test.go b/internal/fakechain/golden_test.go new file mode 100644 index 0000000..dd66231 --- /dev/null +++ b/internal/fakechain/golden_test.go @@ -0,0 +1,90 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package fakechain + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "flag" + "fmt" + "os" + "path/filepath" + "testing" + + "golang.org/x/crypto/sha3" + + "github.com/hmdsefi/evmsync/chain" +) + +var update = flag.Bool("update", false, "rewrite golden files") + +// TestGolden pins the generator: a change to any draw, hash or log layout changes the digests. +// Rewrite testdata/golden.txt with -update only in a commit that says why the generator changed. +func TestGolden(t *testing.T) { + var buf bytes.Buffer + for seed := uint64(1); seed <= 5; seed++ { + c := New(seed, Params{}) + evs := make([]Event, 0, 300) + for range 300 { + evs = append(evs, c.Step()) + } + fmt.Fprintf(&buf, "%d %s\n", seed, chainDigest(c, evs)) + } + path := filepath.Join("testdata", "golden.txt") + if *update { + if err := os.WriteFile(path, buf.Bytes(), 0o600); err != nil { + t.Fatal(err) + } + } + want, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf.Bytes(), want) { + t.Errorf("generator digests:\n%s\nwant (testdata/golden.txt):\n%s", buf.Bytes(), want) + } +} + +// chainDigest hashes the event lines, then every canonical block's ref and bloom and all of its +// logs. +func chainDigest(c *Chain, evs []Event) string { + h := sha3.NewLegacyKeccak256() + for _, e := range evs { + h.Write([]byte(e.String() + "\n")) + } + var u [8]byte + put := func(v uint64) { + binary.BigEndian.PutUint64(u[:], v) + h.Write(u[:]) + } + g := c.Genesis().Ref + put(g.Number) + h.Write(g.Hash[:]) + for _, b := range c.CanonicalBlocks(chain.Filter{}) { + put(b.Ref.Number) + h.Write(b.Ref.Hash[:]) + h.Write(b.Ref.Parent[:]) + put(b.Ref.Time) + hd, _ := c.Canonical(b.Ref.Number) + h.Write(hd.Bloom[:]) + put(uint64(len(b.Logs))) + for _, l := range b.Logs { + h.Write(l.Address[:]) + put(uint64(len(l.Topics))) + for _, tp := range l.Topics { + h.Write(tp[:]) + } + put(uint64(len(l.Data))) + h.Write(l.Data) + put(l.BlockNumber) + h.Write(l.BlockHash[:]) + h.Write(l.TxHash[:]) + put(uint64(l.TxIndex)) + put(uint64(l.Index)) + } + } + put(c.Finalized().Ref.Number) + return hex.EncodeToString(h.Sum(nil)) +} diff --git a/internal/fakechain/logs.go b/internal/fakechain/logs.go new file mode 100644 index 0000000..ae8dc8f --- /dev/null +++ b/internal/fakechain/logs.go @@ -0,0 +1,95 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package fakechain + +import ( + "bytes" + "encoding/binary" + "slices" + + "github.com/ethereum/go-ethereum/common" + "golang.org/x/crypto/sha3" + + "github.com/hmdsefi/evmsync/chain" +) + +// logCounts maps a draw of Intn(10) to the number of fresh logs in a block. +var logCounts = [10]int{0, 0, 0, 1, 1, 1, 2, 2, 3, 4} + +// freshLogs draws a new block's fresh logs from the logs stream. Each one is a new transaction. +// The caller sets the block fields and the indexes. +func (c *Chain) freshLogs() []chain.Log { + r := c.logRand + n := logCounts[r.Intn(10)] + logs := make([]chain.Log, 0, n) + for range n { + l := chain.Log{Address: Address(int(r.Intn(4)))} //nolint:gosec // below 4 + data := make([]byte, 32) + if k := r.Intn(10); k <= 7 { + topic := TransferTopic + if k >= 6 { + topic = ApprovalTopic + } + from, to := Account(int(r.Intn(8))), Account(int(r.Intn(8))) //nolint:gosec // below 8 + amount := 1 + r.Intn(1000) + l.Topics = []common.Hash{topic, addressTopic(from), addressTopic(to)} + binary.BigEndian.PutUint64(data[24:], amount) + } else { + l.Topics = []common.Hash{OtherTopic} + binary.BigEndian.PutUint64(data[24:], r.Uint64()) + } + l.Data = data + c.txSerial++ + l.TxHash = txHash(c.txSerial) + logs = append(logs, l) + } + return logs +} + +// addressTopic is a as an indexed event argument: 12 zero bytes, then the address. +func addressTopic(a common.Address) common.Hash { + var h common.Hash + copy(h[12:], a[:]) + return h +} + +// matching returns copies of the logs that pass f, or nil when none does. +func matching(logs []chain.Log, f chain.Filter) []chain.Log { + return appendMatching(nil, logs, f) +} + +// appendMatching appends copies of the logs that pass f to dst and returns the result. +func appendMatching(dst, logs []chain.Log, f chain.Filter) []chain.Log { + for _, l := range logs { + if f.Match(l) { + l.Topics = slices.Clone(l.Topics) + l.Data = bytes.Clone(l.Data) + dst = append(dst, l) + } + } + return dst +} + +func blockHash(n uint64, parent common.Hash, serial uint64) common.Hash { + return keccak([]byte("evmsync/fakechain/block"), u64be(n), parent[:], u64be(serial)) +} + +func txHash(serial uint64) common.Hash { + return keccak([]byte("evmsync/fakechain/tx"), u64be(serial)) +} + +// keccak returns the Keccak-256 hash of the concatenated parts. +func keccak(parts ...[]byte) common.Hash { + h := sha3.NewLegacyKeccak256() + for _, p := range parts { + h.Write(p) + } + var out common.Hash + h.Sum(out[:0]) + return out +} + +func u64be(v uint64) []byte { + return binary.BigEndian.AppendUint64(nil, v) +} diff --git a/internal/fakechain/rand.go b/internal/fakechain/rand.go new file mode 100644 index 0000000..9805951 --- /dev/null +++ b/internal/fakechain/rand.go @@ -0,0 +1,65 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package fakechain + +import ( + "math/bits" +) + +// Rand is a SplitMix64 stream. Its outputs depend only on the seed and the label it was created +// with, never on the Go version or the platform. It is not safe for concurrent use. +type Rand struct { + state uint64 +} + +// NewRand returns the stream for label under seed. Each user of randomness has its own label, so +// adding a draw to one stream never shifts another. +func NewRand(seed uint64, label string) *Rand { + return &Rand{state: streamState(seed, label)} +} + +// Uint64 returns the next output. +func (r *Rand) Uint64() uint64 { + return splitMix64(&r.state) +} + +// Intn returns a value in [0, n) from exactly one Uint64 draw, by multiply-shift. It panics if n +// is 0. +func (r *Rand) Intn(n uint64) uint64 { + if n == 0 { + panic("evmsync: fakechain: Intn(0)") + } + hi, _ := bits.Mul64(r.Uint64(), n) + return hi +} + +// Chance reports true with probability ppm/1,000,000 from exactly one Intn draw, also when ppm +// is 0 or at least 1,000,000. +func (r *Rand) Chance(ppm uint32) bool { + return r.Intn(1_000_000) < uint64(ppm) +} + +func fnv1a64(s string) uint64 { + h := uint64(14695981039346656037) + for i := 0; i < len(s); i++ { + h ^= uint64(s[i]) + h *= 1099511628211 + } + return h +} + +func splitMix64(state *uint64) uint64 { + *state += 0x9e3779b97f4a7c15 + z := *state + z = (z ^ (z >> 30)) * 0xbf58476d1ce4e5b9 + z = (z ^ (z >> 27)) * 0x94d049bb133111eb + return z ^ (z >> 31) +} + +// streamState is the initial state of stream label under seed. +func streamState(seed uint64, label string) uint64 { + st := seed + k := splitMix64(&st) + return k ^ fnv1a64(label) +} diff --git a/internal/fakechain/rand_test.go b/internal/fakechain/rand_test.go new file mode 100644 index 0000000..dc8daae --- /dev/null +++ b/internal/fakechain/rand_test.go @@ -0,0 +1,121 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package fakechain + +import ( + "testing" +) + +func TestStreams(t *testing.T) { + fnv := []struct { + in string + want uint64 + }{ + {"", 0xcbf29ce484222325}, + {"chain", 0xa74390764c1a3056}, + {"logs", 0xce07d5ad70ec2c26}, + {"faults/stale", 0x69438d2139abd11a}, + {"invariant/params", 0xba8fbd86898fcb56}, + } + for _, tt := range fnv { + if got := fnv1a64(tt.in); got != tt.want { + t.Errorf("fnv1a64(%q) = %#x, want %#x", tt.in, got, tt.want) + } + } + + streams := []struct { + seed uint64 + label string + state uint64 + first [3]uint64 + }{ + {1, "chain", 0x3649bd9ac5186c97, [3]uint64{0xb47524943eba2e00, 0x16df773fd3f078b4, 0xeb395c59d379f2f6}}, + {1, "logs", 0x5f0df841f9ee70e7, [3]uint64{0x40f075f5f37e5d36, 0xc5c44323ac661942, 0xc9b55d5ceb77c672}}, + {0, "chain", 0x4563384f3707fdf9, [3]uint64{0xde2a3de9127ff783, 0x4f9ff908143d0f5f, 0xa6a22e8c8f1d2d63}}, + {0xffffffffffffffff, "invariant/params", 0x5e56ccf192eae776, + [3]uint64{0xb3e78406c979e937, 0x0f23dcf4c65aca0b, 0xa85b690a296136ae}}, + } + for _, tt := range streams { + if got := streamState(tt.seed, tt.label); got != tt.state { + t.Errorf("streamState(%#x, %q) = %#x, want %#x", tt.seed, tt.label, got, tt.state) + } + r := NewRand(tt.seed, tt.label) + for i, want := range tt.first { + if got := r.Uint64(); got != want { + t.Errorf("NewRand(%#x, %q) output %d = %#x, want %#x", tt.seed, tt.label, i, got, want) + } + } + } + + r := NewRand(1, "chain") + if got := r.Intn(10); got != 7 { + t.Errorf("Intn(10) = %d, want 7", got) + } + if got := r.Intn(1_000_000); got != 89347 { + t.Errorf("Intn(1_000_000) = %d, want 89347", got) + } + if r.Chance(500_000) { + t.Error("Chance(500_000) = true, want false") + } + + r = NewRand(7, "logs") + want := []uint64{2, 2, 1, 1, 1} + for i, w := range want { + if got := r.Intn(4); got != w { + t.Errorf("NewRand(7, \"logs\") Intn(4) #%d = %d, want %d", i, got, w) + } + } + + wantPanic(t, "evmsync: fakechain: Intn(0)", func() { NewRand(1, "chain").Intn(0) }) +} + +func TestChanceDrawsOnce(t *testing.T) { + for _, ppm := range []uint32{0, 1, 500_000, 1_000_000, 2_000_000} { + r, twin := NewRand(5, "chain"), NewRand(5, "chain") + got := r.Chance(ppm) + want := twin.Intn(1_000_000) < uint64(ppm) + if got != want { + t.Errorf("Chance(%d) = %v, want %v", ppm, got, want) + } + if r.Uint64() != twin.Uint64() { + t.Errorf("Chance(%d) did not draw exactly once", ppm) + } + } + v := uint32(NewRand(5, "chain").Intn(1_000_000)) //nolint:gosec // below 1,000,000 + if NewRand(5, "chain").Chance(v) || !NewRand(5, "chain").Chance(v+1) { + t.Errorf("Chance(ppm) for a draw of %d: want false at ppm %d and true at %d", v, v, v+1) + } + if !NewRand(5, "chain").Chance(1_000_000) { + t.Error("Chance(1_000_000) = false, want true") + } + if NewRand(5, "chain").Chance(0) { + t.Error("Chance(0) = true, want false") + } +} + +// wantPanic fails t unless fn panics with exactly msg. +func wantPanic(t *testing.T, msg string, fn func()) { + t.Helper() + defer func() { + t.Helper() + r := recover() + if r == nil { + t.Errorf("no panic, want %q", msg) + return + } + got, ok := r.(string) + if !ok { + err, isErr := r.(error) + if !isErr { + t.Errorf("panic value %v (%T), want string %q", r, r, msg) + return + } + got = err.Error() + } + if got != msg { + t.Errorf("panic %q, want %q", got, msg) + } + }() + fn() +} diff --git a/internal/fakechain/source.go b/internal/fakechain/source.go new file mode 100644 index 0000000..b9b7679 --- /dev/null +++ b/internal/fakechain/source.go @@ -0,0 +1,236 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package fakechain + +import ( + "context" + "fmt" + "slices" + "sync" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" +) + +// Faults configures provider misbehavior. The zero value is an honest provider. +type Faults struct { + StaleHeadPPM uint32 // Latest answers a canonical block 1..StaleDepth below the head + StaleDepth uint64 // default 3 + StaleOrphanPPM uint32 // Latest answers the head the latest reorg orphaned + DropOrphans bool // HeaderByHash and Logs answer ErrNotFound for orphans + FinalizedUnsupported bool // Finalized answers chain.ErrUnsupported + DroppedLogsPPM uint32 // Logs answers no logs for a block that has matching ones + InterleavePPM uint32 // a lookup runs Chain.Step before answering +} + +// Call records one Source call. +type Call struct { + Method string // "Latest", "Finalized", "HeaderByNumber", "HeaderByHash", "Logs", "LogsInRange" + Hash common.Hash // HeaderByHash, Logs + Number uint64 // HeaderByNumber; LogsInRange: from + To uint64 // LogsInRange +} + +// Source implements chain.Source over a Chain, with the provider faults of Faults. It is safe for +// concurrent use: every method holds one mutex, which also guards the Chain. A method called with +// a context that is already done returns ctx.Err() unchanged and records nothing. +type Source struct { + mu sync.Mutex + // mu guards every field below and the chain c. + c *Chain + faults Faults + calls []Call + + staleRand *Rand // faults/stale + orphanRand *Rand // faults/orphan + interleaveRand *Rand // faults/interleave + logsRand *Rand // faults/logs +} + +var _ chain.Source = (*Source)(nil) + +// NewSource returns a Source over c whose fault streams derive from seed. +func NewSource(c *Chain, seed uint64, f Faults) *Source { + return &Source{ + c: c, + faults: f, + staleRand: NewRand(seed, "faults/stale"), + orphanRand: NewRand(seed, "faults/orphan"), + interleaveRand: NewRand(seed, "faults/interleave"), + logsRand: NewRand(seed, "faults/logs"), + } +} + +// Latest returns the head header. With StaleOrphanPPM it may answer the head the latest reorg +// orphaned, as long as that reorg's fork point is not below the finalized number; with +// StaleHeadPPM, a canonical block 1 to StaleDepth below the head. +func (s *Source) Latest(ctx context.Context) (chain.Header, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := ctx.Err(); err != nil { + return chain.Header{}, err + } + s.calls = append(s.calls, Call{Method: "Latest"}) + depth := s.faults.StaleDepth + if depth == 0 { + depth = 3 + } + orphan := s.orphanRand.Chance(s.faults.StaleOrphanPPM) + stale := s.staleRand.Chance(s.faults.StaleHeadPPM) + k := 1 + s.staleRand.Intn(depth) + + c := s.c + if orphan && c.lastOrphan != (common.Hash{}) && c.lastFork >= c.finalized { + return c.nodes[c.lastOrphan].header, nil + } + if stale { + h, _ := c.Canonical(c.head() - min(k, c.head())) + return h, nil + } + return c.Head(), nil +} + +// Finalized returns the canonical header at the chain's finalized number, or an error wrapping +// chain.ErrUnsupported with FinalizedUnsupported. +func (s *Source) Finalized(ctx context.Context) (chain.Header, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := ctx.Err(); err != nil { + return chain.Header{}, err + } + s.calls = append(s.calls, Call{Method: "Finalized"}) + if s.faults.FinalizedUnsupported { + return chain.Header{}, fmt.Errorf("fakechain: finalized: %w", chain.ErrUnsupported) + } + return s.c.Finalized(), nil +} + +// HeaderByNumber returns the canonical header at n, or an error wrapping chain.ErrNotFound when n +// is above the head. +func (s *Source) HeaderByNumber(ctx context.Context, n uint64) (chain.Header, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := ctx.Err(); err != nil { + return chain.Header{}, err + } + s.lookup(Call{Method: "HeaderByNumber", Number: n}) + h, ok := s.c.Canonical(n) + if !ok { + return chain.Header{}, numberNotFound(n) + } + return h, nil +} + +// HeaderByHash returns any tree block's header, or an error wrapping chain.ErrNotFound for an +// unknown hash or, with DropOrphans, an orphan. +func (s *Source) HeaderByHash(ctx context.Context, h common.Hash) (chain.Header, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := ctx.Err(); err != nil { + return chain.Header{}, err + } + s.lookup(Call{Method: "HeaderByHash", Hash: h}) + nd, err := s.node(h) + if err != nil { + return chain.Header{}, err + } + return nd.header, nil +} + +// Logs returns the block's logs that pass f, in Index order, or nil when none passes. It returns +// an error wrapping chain.ErrNotFound for an unknown hash or, with DropOrphans, an orphan. With +// DroppedLogsPPM it may answer nil for a block that has matching logs. +func (s *Source) Logs(ctx context.Context, block common.Hash, f chain.Filter) ([]chain.Log, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := ctx.Err(); err != nil { + return nil, err + } + s.lookup(Call{Method: "Logs", Hash: block}) + nd, err := s.node(block) + if err != nil { + return nil, err + } + if s.logsRand.Chance(s.faults.DroppedLogsPPM) { + return nil, nil + } + return matching(nd.logs, f), nil +} + +// LogsInRange returns the logs of canonical blocks from to to (inclusive) that pass f, by block +// then Index, or nil when none passes. It returns an error wrapping chain.ErrInvalidRange when +// from > to, and chain.ErrNotFound when to is above the head. +func (s *Source) LogsInRange(ctx context.Context, from, to uint64, f chain.Filter) ([]chain.Log, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err := ctx.Err(); err != nil { + return nil, err + } + s.lookup(Call{Method: "LogsInRange", Number: from, To: to}) + if from > to { + return nil, fmt.Errorf("fakechain: logs in range: from %d > to %d: %w", from, to, chain.ErrInvalidRange) + } + c := s.c + if to > c.head() { + return nil, numberNotFound(to) + } + var logs []chain.Log + for n := from; n <= to; n++ { + logs = appendMatching(logs, c.nodes[c.canonical[n]].logs, f) + } + return logs, nil +} + +// Do runs fn with the Source's mutex held, so tests can change the chain while the Source is in +// use. fn must not call the Source's methods: the mutex is not reentrant. +func (s *Source) Do(fn func(c *Chain)) { + s.mu.Lock() + defer s.mu.Unlock() + fn(s.c) +} + +// SetFaults replaces the faults. The fault streams keep their positions. +func (s *Source) SetFaults(f Faults) { + s.mu.Lock() + defer s.mu.Unlock() + s.faults = f +} + +// Calls returns a copy of the calls recorded since creation or the last ResetCalls, in order. +// Calls with a context that was already done are not recorded. +func (s *Source) Calls() []Call { + s.mu.Lock() + defer s.mu.Unlock() + return slices.Clone(s.calls) +} + +// ResetCalls clears the call record. +func (s *Source) ResetCalls() { + s.mu.Lock() + defer s.mu.Unlock() + s.calls = nil +} + +// lookup records a lookup call and, with probability InterleavePPM, moves the chain by one Step +// before the call answers. The caller holds s.mu. +func (s *Source) lookup(call Call) { + s.calls = append(s.calls, call) + if s.interleaveRand.Chance(s.faults.InterleavePPM) { + s.c.Step() + } +} + +// node returns the tree block h as the source serves it. The caller holds s.mu. +func (s *Source) node(h common.Hash) (*node, error) { + nd, ok := s.c.nodes[h] + if !ok || (s.faults.DropOrphans && !s.c.isCanonical(nd.header.Ref)) { + return nil, fmt.Errorf("fakechain: block %s: %w", h.Hex(), chain.ErrNotFound) + } + return nd, nil +} + +func numberNotFound(n uint64) error { + return fmt.Errorf("fakechain: block %d: %w", n, chain.ErrNotFound) +} diff --git a/internal/fakechain/source_test.go b/internal/fakechain/source_test.go new file mode 100644 index 0000000..fc5f063 --- /dev/null +++ b/internal/fakechain/source_test.go @@ -0,0 +1,449 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package fakechain + +import ( + "errors" + "slices" + "sync" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" +) + +func TestHonestSource(t *testing.T) { + ctx := t.Context() + c := New(2, Params{}) + c.Mine(20) + var orphans []chain.BlockRef + for n := uint64(18); n <= 20; n++ { + h, _ := c.Canonical(n) + orphans = append(orphans, h.Ref) + } + c.Reorg(3, 3) + src := NewSource(c, 2, Faults{}) + + var want []Call + head, err := src.Latest(ctx) + if err != nil || head != c.Head() { + t.Errorf("Latest() = %v, %v, want %v", head.Ref, err, c.Head().Ref) + } + want = append(want, Call{Method: "Latest"}) + fin, err := src.Finalized(ctx) + if err != nil || fin != c.Genesis() { + t.Errorf("Finalized() = %v, %v, want genesis", fin.Ref, err) + } + want = append(want, Call{Method: "Finalized"}) + + for n := uint64(0); n <= 20; n++ { + h, err := src.HeaderByNumber(ctx, n) + wantH, _ := c.Canonical(n) + if err != nil || h != wantH { + t.Errorf("HeaderByNumber(%d) = %v, %v, want %v", n, h.Ref, err, wantH.Ref) + } + want = append(want, Call{Method: "HeaderByNumber", Number: n}) + } + _, err = src.HeaderByNumber(ctx, 21) + wantErr(t, err, chain.ErrNotFound, "fakechain: block 21: chain: not found") + want = append(want, Call{Method: "HeaderByNumber", Number: 21}) + + random := common.HexToHash("0x0102") + var hashes []common.Hash + for n := uint64(0); n <= 20; n++ { + h, _ := c.Canonical(n) + hashes = append(hashes, h.Ref.Hash) + } + for _, r := range orphans { + hashes = append(hashes, r.Hash) + } + var none, some int + for _, h := range hashes { + got, err := src.HeaderByHash(ctx, h) + wantH, _ := c.Header(h) + if err != nil || got != wantH { + t.Errorf("HeaderByHash(%s) = %v, %v, want %v", h.Hex(), got.Ref, err, wantH.Ref) + } + logs, err := src.Logs(ctx, h, DefaultFilter) + b, _ := c.Block(h, DefaultFilter) + if err != nil || !logsEqual(logs, b.Logs) { + t.Errorf("Logs(%s) = %v, %v, want %v", h.Hex(), logs, err, b.Logs) + } + if len(b.Logs) == 0 { + none++ + if logs != nil { + t.Errorf("Logs(%s) = %v, want nil", h.Hex(), logs) + } + } else { + some++ + } + want = append(want, Call{Method: "HeaderByHash", Hash: h}, Call{Method: "Logs", Hash: h}) + } + if none == 0 || some == 0 { + t.Errorf("seed 2: %d blocks without and %d with matching logs; the check covers no case", none, some) + } + _, err = src.HeaderByHash(ctx, random) + wantErr(t, err, chain.ErrNotFound, "fakechain: block "+random.Hex()+": chain: not found") + _, err = src.HeaderByHash(ctx, common.Hash{}) + wantErr(t, err, chain.ErrNotFound, "fakechain: block "+common.Hash{}.Hex()+": chain: not found") + _, err = src.Logs(ctx, random, chain.Filter{}) + wantErr(t, err, chain.ErrNotFound, "fakechain: block "+random.Hex()+": chain: not found") + want = append(want, Call{Method: "HeaderByHash", Hash: random}, Call{Method: "HeaderByHash"}, + Call{Method: "Logs", Hash: random}) + + _, err = src.LogsInRange(ctx, 5, 4, chain.Filter{}) + wantErr(t, err, chain.ErrInvalidRange, "fakechain: logs in range: from 5 > to 4: chain: invalid block range") + _, err = src.LogsInRange(ctx, 25, 21, chain.Filter{}) + wantErr(t, err, chain.ErrInvalidRange, "fakechain: logs in range: from 25 > to 21: chain: invalid block range") + _, err = src.LogsInRange(ctx, 1, 21, chain.Filter{}) + wantErr(t, err, chain.ErrNotFound, "fakechain: block 21: chain: not found") + want = append(want, Call{Method: "LogsInRange", Number: 5, To: 4}, Call{Method: "LogsInRange", Number: 25, To: 21}, + Call{Method: "LogsInRange", Number: 1, To: 21}) + + var concat []chain.Log + for _, b := range c.CanonicalBlocks(DefaultFilter) { + concat = append(concat, cloneLogs(b.Logs)...) + } + got, err := src.LogsInRange(ctx, 1, 20, DefaultFilter) + if err != nil || !logsEqual(got, concat) || len(got) == 0 { + t.Errorf("LogsInRange(1, 20) = %d logs, %v, want %d", len(got), err, len(concat)) + } + want = append(want, Call{Method: "LogsInRange", Number: 1, To: 20}) + for n := uint64(0); n <= 20; n++ { + h, _ := c.Canonical(n) + b, _ := c.Block(h.Ref.Hash, DefaultFilter) + one, err := src.LogsInRange(ctx, n, n, DefaultFilter) + if err != nil || !logsEqual(one, b.Logs) || (b.Logs == nil) != (one == nil) { + t.Errorf("LogsInRange(%d, %d) = %v, %v, want %v", n, n, one, err, b.Logs) + } + want = append(want, Call{Method: "LogsInRange", Number: n, To: n}) + } + + if calls := src.Calls(); !slices.Equal(calls, want) { + t.Errorf("Calls() = %d calls %v,\nwant %d calls %v", len(calls), calls, len(want), want) + } + calls := src.Calls() + calls[0].Method = "changed" + if src.Calls()[0].Method != "Latest" { + t.Error("mutating Calls() changed the record") + } + + src.ResetCalls() + if n := len(src.Calls()); n != 0 { + t.Errorf("after ResetCalls, %d calls", n) + } + checkCancelled(t, src, hashes[1]) + + // Results belong to the caller. + first, _ := src.LogsInRange(ctx, 1, 20, DefaultFilter) + first[0].Topics[0] = common.Hash{7} + first[0].Data[0] = 7 + second, _ := src.LogsInRange(ctx, 1, 20, DefaultFilter) + if !logsEqual(second, concat) { + t.Error("mutating a LogsInRange result changed a later result") + } +} + +func TestFaults(t *testing.T) { + ctx := t.Context() + + t.Run("stale head", func(t *testing.T) { + c := New(3, Params{}) + c.Mine(20) + src := NewSource(c, 3, Faults{StaleHeadPPM: 1_000_000, StaleDepth: 3}) + seen := map[uint64]int{} + for range 100 { + h, err := src.Latest(ctx) + if err != nil { + t.Fatal(err) + } + want, _ := c.Canonical(h.Ref.Number) + if h != want || h.Ref.Number < 17 || h.Ref.Number > 19 { + t.Fatalf("Latest() = %v, want a canonical block 1 to 3 below head 20", h.Ref) + } + seen[20-h.Ref.Number]++ + } + if len(seen) != 3 { + t.Errorf("depths seen %v, want 1, 2 and 3", seen) + } + + src.SetFaults(Faults{StaleHeadPPM: 1_000_000}) // StaleDepth 0 means 3 + for range 100 { + h, _ := src.Latest(ctx) + if h.Ref.Number < 17 || h.Ref.Number > 19 { + t.Fatalf("StaleDepth 0: Latest() = %v, want 17 to 19", h.Ref) + } + } + + c2 := New(3, Params{}) + c2.Mine(2) + src2 := NewSource(c2, 3, Faults{StaleHeadPPM: 1_000_000, StaleDepth: 8}) + for range 50 { + h, _ := src2.Latest(ctx) + if h.Ref.Number > 1 { + t.Fatalf("head 2, depth up to 8: Latest() = %v, want 0 or 1", h.Ref) + } + } + }) + + t.Run("stale orphan", func(t *testing.T) { + c := New(4, Params{}) + c.Mine(10) + src := NewSource(c, 4, Faults{StaleOrphanPPM: 1_000_000}) + if h, _ := src.Latest(ctx); h != c.Head() { + t.Errorf("no reorg yet: Latest() = %v, want head", h.Ref) + } + old := c.Head() + src.Do(func(c *Chain) { c.Reorg(2, 3) }) + if h, _ := src.Latest(ctx); h != old { + t.Errorf("after Reorg(2, 3): Latest() = %v, want the orphaned head %v", h.Ref, old.Ref) + } + src.Do(func(c *Chain) { c.SetFinalized(8) }) + if h, _ := src.Latest(ctx); h != old { + t.Errorf("finalized at the fork point: Latest() = %v, want the orphaned head %v", h.Ref, old.Ref) + } + src.Do(func(c *Chain) { c.SetFinalized(9) }) + if h, _ := src.Latest(ctx); h != c.Head() { + t.Errorf("finalized above the fork point: Latest() = %v, want head %v", h.Ref, c.Head().Ref) + } + // A later reorg replaces the pair; a rollback with depth 0 does not record one. + src.Do(func(c *Chain) { c.Reorg(1, 2) }) + orphan, _ := c.Header(c.lastOrphan) + if orphan.Ref.Number != 11 { + t.Fatalf("last orphan = %v, want the former head 11", orphan.Ref) + } + if h, _ := src.Latest(ctx); h != orphan { + t.Errorf("after Reorg(1, 2): Latest() = %v, want %v", h.Ref, orphan.Ref) + } + src.Do(func(c *Chain) { c.Reorg(0, 1) }) + if h, _ := src.Latest(ctx); h != orphan { + t.Errorf("after Reorg(0, 1): Latest() = %v, want unchanged pair %v", h.Ref, orphan.Ref) + } + src.SetFaults(Faults{StaleOrphanPPM: 1_000_000, DropOrphans: true}) + if h, _ := src.Latest(ctx); h != orphan { + t.Errorf("with DropOrphans: Latest() = %v, want the orphaned head %v", h.Ref, orphan.Ref) + } + }) + + t.Run("drop orphans", func(t *testing.T) { + c := New(5, Params{}) + c.Mine(10) + orphan := c.Head() + c.Reorg(2, 2) + src := NewSource(c, 5, Faults{DropOrphans: true}) + _, err := src.HeaderByHash(ctx, orphan.Ref.Hash) + wantErr(t, err, chain.ErrNotFound, "fakechain: block "+orphan.Ref.Hash.Hex()+": chain: not found") + _, err = src.Logs(ctx, orphan.Ref.Hash, chain.Filter{}) + wantErr(t, err, chain.ErrNotFound, "fakechain: block "+orphan.Ref.Hash.Hex()+": chain: not found") + head := c.Head() + if h, err := src.HeaderByHash(ctx, head.Ref.Hash); err != nil || h != head { + t.Errorf("HeaderByHash(head) = %v, %v", h.Ref, err) + } + if _, err := src.Logs(ctx, head.Ref.Hash, chain.Filter{}); err != nil { + t.Errorf("Logs(head) error %v", err) + } + src.SetFaults(Faults{}) + if h, err := src.HeaderByHash(ctx, orphan.Ref.Hash); err != nil || h != orphan { + t.Errorf("honest HeaderByHash(orphan) = %v, %v", h.Ref, err) + } + }) + + t.Run("finalized unsupported", func(t *testing.T) { + c := New(6, Params{}) + c.Mine(5) + c.SetFinalized(3) + src := NewSource(c, 6, Faults{FinalizedUnsupported: true}) + _, err := src.Finalized(ctx) + wantErr(t, err, chain.ErrUnsupported, "fakechain: finalized: chain: unsupported") + if calls := src.Calls(); !slices.Equal(calls, []Call{{Method: "Finalized"}}) { + t.Errorf("Calls() = %v, want the one Finalized call", calls) + } + src.SetFaults(Faults{}) + if h, err := src.Finalized(ctx); err != nil || h.Ref.Number != 3 { + t.Errorf("Finalized() = %v, %v, want block 3", h.Ref, err) + } + }) + + t.Run("dropped logs", func(t *testing.T) { + c := New(7, Params{}) + c.Mine(30) + src := NewSource(c, 7, Faults{DroppedLogsPPM: 1_000_000}) + var checked int + for _, b := range c.CanonicalBlocks(DefaultFilter) { + logs, err := src.Logs(ctx, b.Ref.Hash, DefaultFilter) + if err != nil || logs != nil { + t.Errorf("block %d: Logs = %v, %v, want nil", b.Ref.Number, logs, err) + } + if len(b.Logs) > 0 { + checked++ + } + } + if checked == 0 { + t.Error("seed 7: no block with matching logs; the check covers no case") + } + if logs, _ := src.LogsInRange(ctx, 1, 30, DefaultFilter); len(logs) == 0 { + t.Error("DroppedLogsPPM changed LogsInRange") + } + }) + + t.Run("interleave", func(t *testing.T) { + c, twin := New(8, Params{}), New(8, Params{}) + c.Mine(5) + twin.Mine(5) + src := NewSource(c, 8, Faults{InterleavePPM: 1_000_000}) + g := c.Genesis().Ref.Hash + lookups := []func(){ + func() { _, _ = src.HeaderByHash(ctx, g) }, + func() { _, _ = src.HeaderByNumber(ctx, 0) }, + func() { _, _ = src.Logs(ctx, g, chain.Filter{}) }, + func() { _, _ = src.LogsInRange(ctx, 0, 0, chain.Filter{}) }, + } + for i := range 40 { + lookups[i%len(lookups)]() + twin.Step() + if c.Head() != twin.Head() { + t.Fatalf("lookup %d: head %v, want %v after one Step", i, c.Head().Ref, twin.Head().Ref) + } + } + _, _ = src.Latest(ctx) + _, _ = src.Finalized(ctx) + if c.Head() != twin.Head() { + t.Error("Latest or Finalized ran a Step") + } + }) + + t.Run("draw counts", func(t *testing.T) { + c := New(9, Params{}) + c.Mine(10) + src := NewSource(c, 9, Faults{}) + for range 10 { + if h, _ := src.Latest(ctx); h != c.Head() { + t.Fatalf("honest Latest() = %v, want head", h.Ref) + } + } + h, _ := c.Canonical(4) + _, _ = src.HeaderByHash(ctx, h.Ref.Hash) + _, _ = src.HeaderByNumber(ctx, 99) + _, _ = src.Logs(ctx, h.Ref.Hash, DefaultFilter) + _, _ = src.Logs(ctx, common.Hash{1}, DefaultFilter) // not found: no faults/logs draw + _, _ = src.Logs(ctx, c.Genesis().Ref.Hash, DefaultFilter) // found, no matching log: one draw + _, _ = src.LogsInRange(ctx, 3, 1, DefaultFilter) + _, _ = src.Finalized(ctx) + src.SetFaults(Faults{StaleDepth: 5}) // streams keep their positions + _, _ = src.Latest(ctx) + + draws := []struct { + r *Rand + label string + n int + }{ + {src.staleRand, "faults/stale", 22}, + {src.orphanRand, "faults/orphan", 11}, + {src.interleaveRand, "faults/interleave", 6}, + {src.logsRand, "faults/logs", 2}, + } + for _, d := range draws { + fresh := NewRand(9, d.label) + for range d.n { + fresh.Uint64() + } + if d.r.Uint64() != fresh.Uint64() { + t.Errorf("stream %s did not advance by exactly %d draws", d.label, d.n) + } + } + }) +} + +// TestLatestDraws checks Latest's draws and answer against a model on the two fault streams. +func TestLatestDraws(t *testing.T) { + ctx := t.Context() + var orphans, stales, heads int + for seed := uint64(1); seed <= 30; seed++ { + c := New(seed, Params{ForkPPM: 400_000}) + src := NewSource(c, seed, Faults{StaleHeadPPM: 500_000, StaleOrphanPPM: 500_000, StaleDepth: 4}) + orphanR, staleR := NewRand(seed, "faults/orphan"), NewRand(seed, "faults/stale") + for i := range 200 { + src.Do(func(c *Chain) { c.Step() }) + o := orphanR.Chance(500_000) + s := staleR.Chance(500_000) + k := 1 + staleR.Intn(4) + head := c.Head().Ref.Number + want := c.Head() + switch { + case o && c.lastOrphan != (common.Hash{}) && c.lastFork >= c.finalized: + want, _ = c.Header(c.lastOrphan) + orphans++ + case s: + want, _ = c.Canonical(head - min(k, head)) + stales++ + default: + heads++ + } + if got, _ := src.Latest(ctx); got != want { + t.Fatalf("seed %d call %d: Latest() = %v, want %v", seed, i, got.Ref, want.Ref) + } + } + } + if orphans == 0 || stales == 0 || heads == 0 { + t.Errorf("answers: %d orphan, %d stale, %d head; want each at least once", orphans, stales, heads) + } +} + +func TestConcurrency(t *testing.T) { + ctx := t.Context() + c := New(10, Params{}) + c.Mine(10) + faults := Faults{StaleHeadPPM: 100_000, StaleOrphanPPM: 100_000, InterleavePPM: 10_000} + src := NewSource(c, 10, faults) + g := c.Genesis().Ref.Hash + var wg sync.WaitGroup + for w := range 8 { + wg.Go(func() { + for i := range 1000 { + switch (w + i) % 6 { + case 0: + _, _ = src.Latest(ctx) + case 1: + _, _ = src.Finalized(ctx) + case 2: + _, _ = src.HeaderByNumber(ctx, uint64(i%5)) + case 3: + _, _ = src.HeaderByHash(ctx, g) + case 4: + _, _ = src.Logs(ctx, g, DefaultFilter) + default: + _, _ = src.LogsInRange(ctx, 0, 2, DefaultFilter) + } + } + }) + } + wg.Go(func() { + for range 200 { + src.Do(func(c *Chain) { c.Mine(1) }) + } + }) + wg.Go(func() { + for range 200 { + _ = src.Calls() + src.SetFaults(faults) + } + }) + wg.Wait() + if n := len(src.Calls()); n != 8000 { + t.Errorf("%d calls recorded, want 8000", n) + } +} + +// wantErr fails t unless err wraps target and reads exactly msg. +func wantErr(t *testing.T, err, target error, msg string) { + t.Helper() + if !errors.Is(err, target) { + t.Errorf("error %v, want one wrapping %v", err, target) + return + } + if err.Error() != msg { + t.Errorf("error %q, want %q", err.Error(), msg) + } +} diff --git a/internal/fakechain/testdata/golden.txt b/internal/fakechain/testdata/golden.txt new file mode 100644 index 0000000..526ff22 --- /dev/null +++ b/internal/fakechain/testdata/golden.txt @@ -0,0 +1,5 @@ +1 0746a4dac40c50ad4e5e45ffc52a1675c804b54f87bafda7f855f26669f9b7d8 +2 13a855bb58622e85054a2790fd6dd7892f2253d461629f96e08e9a94f230d0a6 +3 99ef64ac4fca68801427f55945b4a0ad2b55da3dac9d78187da52b563f48ee11 +4 b62e40eee7fec03abaaf3ca8d5c97ddcb95f8162b226a3de26227e508fc26b0d +5 79524669339582396fa6bb57ef2c25d202c85e506b7e2f364b2ae5a41d2e2af3 diff --git a/internal/invariant/balances.go b/internal/invariant/balances.go new file mode 100644 index 0000000..9f2664d --- /dev/null +++ b/internal/invariant/balances.go @@ -0,0 +1,173 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package invariant + +import ( + "bytes" + "context" + "encoding/binary" + "hash" + "maps" + "math" + "slices" + "sync" + + "github.com/ethereum/go-ethereum/common" + "golang.org/x/crypto/sha3" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/consumer" + "github.com/hmdsefi/evmsync/internal/fakechain" +) + +// Key identifies a balance: a token contract and an account. +type Key struct{ Token, Account common.Address } + +// Balances is the reference state: per-token balances from Transfer logs plus a running hash of +// every applied log. +type Balances struct { + Bal map[Key]int64 // zero balances are absent + Hash common.Hash // running hash of every applied log, with its block + Logs uint64 // logs applied +} + +// NewBalances returns the empty state. +func NewBalances() Balances { + return Balances{Bal: map[Key]int64{}} +} + +// hasher is a reusable Keccak-256 state with a scratch buffer. +type hasher struct { + h hash.Hash + buf []byte +} + +// hasherPool keeps ApplyBalances from allocating a Keccak state per block. +var hasherPool = sync.Pool{ + New: func() any { return &hasher{h: sha3.NewLegacyKeccak256()} }, +} + +// ApplyBalances applies b's logs to s in order, modifying and returning s. Every log extends the +// running hash with its block number and hash, index, address, topics and data, so a log applied +// from an orphan differs from the same log in its canonical block. A Transfer log with three +// topics and a 32-byte value of at most math.MaxInt64 moves the value from the first indexed +// account to the second; balances may go negative. +func ApplyBalances(s Balances, b chain.Block) Balances { + if s.Bal == nil { + s.Bal = map[Key]int64{} + } + hs := hasherPool.Get().(*hasher) //nolint:errcheck // the pool only holds *hasher + defer hasherPool.Put(hs) + h, buf := hs.h, hs.buf + for i := range b.Logs { + l := &b.Logs[i] + buf = append(buf[:0], s.Hash[:]...) + buf = binary.BigEndian.AppendUint64(buf, b.Ref.Number) + buf = append(buf, b.Ref.Hash[:]...) + buf = binary.BigEndian.AppendUint32(buf, l.Index) + buf = append(buf, l.Address[:]...) + buf = append(buf, byte(len(l.Topics))) //nolint:gosec // a log has at most 4 topics + for _, tp := range l.Topics { + buf = append(buf, tp[:]...) + } + buf = binary.BigEndian.AppendUint32(buf, uint32(len(l.Data))) //nolint:gosec // block data is far below 4 GiB + buf = append(buf, l.Data...) + h.Reset() + h.Write(buf) + h.Sum(s.Hash[:0]) + s.Logs++ + hs.buf = buf + + v, ok := transferValue(l) + if !ok { + continue + } + from := Key{l.Address, common.BytesToAddress(l.Topics[1][12:])} + to := Key{l.Address, common.BytesToAddress(l.Topics[2][12:])} + add(s.Bal, from, -v) + add(s.Bal, to, v) + } + return s +} + +// transferValue returns the value a Transfer log moves, and false for any other log. +func transferValue(l *chain.Log) (int64, bool) { + if len(l.Topics) != 3 || l.Topics[0] != fakechain.TransferTopic || len(l.Data) != 32 { + return 0, false + } + if !bytes.Equal(l.Data[:24], make([]byte, 24)) { + return 0, false + } + v := binary.BigEndian.Uint64(l.Data[24:]) + if v > math.MaxInt64 { + return 0, false + } + return int64(v), true +} + +// add adds d to bal[k] and deletes the entry when it becomes 0. +func add(bal map[Key]int64, k Key, d int64) { + if v := bal[k] + d; v != 0 { + bal[k] = v + } else { + delete(bal, k) + } +} + +// CloneBalances returns a copy of s that shares no memory with it. +func CloneBalances(s Balances) Balances { + s.Bal = maps.Clone(s.Bal) + if s.Bal == nil { + s.Bal = map[Key]int64{} + } + return s +} + +// Digest returns the 32-byte Keccak-256 of the running hash, the log count, and every balance in +// ascending order of token, then account. Equal states give equal digests. +func (s Balances) Digest() []byte { + keys := slices.SortedFunc(maps.Keys(s.Bal), func(a, b Key) int { + if c := bytes.Compare(a.Token[:], b.Token[:]); c != 0 { + return c + } + return bytes.Compare(a.Account[:], b.Account[:]) + }) + buf := make([]byte, 0, 40+len(keys)*48) + buf = append(buf, s.Hash[:]...) + buf = binary.BigEndian.AppendUint64(buf, s.Logs) + for _, k := range keys { + buf = append(buf, k.Token[:]...) + buf = append(buf, k.Account[:]...) + buf = binary.BigEndian.AppendUint64(buf, uint64(s.Bal[k])) //nolint:gosec // two's complement on purpose + } + h := sha3.NewLegacyKeccak256() + h.Write(buf) + return h.Sum(nil) +} + +// ExpectedBalances folds ApplyBalances over blocks from NewBalances and returns the digest. It is +// the oracle for a subject's state after the final canonical chain. +func ExpectedBalances(blocks []chain.Block) []byte { + s := NewBalances() + for _, b := range blocks { + s = ApplyBalances(s, b) + } + return s.Digest() +} + +// memorySubject is a consumer.Memory over Balances. +type memorySubject struct { + *consumer.Memory[Balances] +} + +// Digest returns the digest of the current state. +func (s memorySubject) Digest(context.Context) ([]byte, error) { + return s.State().Digest(), nil +} + +// NewMemorySubject returns a consumer.Memory[Balances] subject, with ApplyBalances and +// CloneBalances, whose Digest is State().Digest(). +func NewMemorySubject() Subject { + return memorySubject{consumer.NewMemory(NewBalances(), ApplyBalances, CloneBalances)} +} diff --git a/internal/invariant/balances_test.go b/internal/invariant/balances_test.go new file mode 100644 index 0000000..fb804f6 --- /dev/null +++ b/internal/invariant/balances_test.go @@ -0,0 +1,215 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package invariant + +import ( + "bytes" + "encoding/binary" + "maps" + "testing" + + "github.com/ethereum/go-ethereum/common" + "golang.org/x/crypto/sha3" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/internal/fakechain" +) + +// word returns v as a 32-byte big-endian value. +func word(v uint64) []byte { + w := make([]byte, 32) + binary.BigEndian.PutUint64(w[24:], v) + return w +} + +// topic returns a as an indexed event argument. +func topic(a common.Address) common.Hash { + var h common.Hash + copy(h[12:], a[:]) + return h +} + +func transfer(token common.Address, from, to int, data []byte) chain.Log { + return chain.Log{ + Address: token, + Topics: []common.Hash{fakechain.TransferTopic, topic(fakechain.Account(from)), topic(fakechain.Account(to))}, + Data: data, + } +} + +// keccakOf hashes the concatenation of parts. +func keccakOf(parts ...[]byte) common.Hash { + h := sha3.NewLegacyKeccak256() + for _, p := range parts { + h.Write(p) + } + return common.BytesToHash(h.Sum(nil)) +} + +// logLink computes one step of the running hash from the byte layout. +func logLink(prev common.Hash, ref chain.BlockRef, l chain.Log) common.Hash { + var b bytes.Buffer + b.Write(prev[:]) + b.Write(binary.BigEndian.AppendUint64(nil, ref.Number)) + b.Write(ref.Hash[:]) + b.Write(binary.BigEndian.AppendUint32(nil, l.Index)) + b.Write(l.Address[:]) + b.WriteByte(byte(len(l.Topics))) //nolint:gosec // at most 4 topics + for _, tp := range l.Topics { + b.Write(tp[:]) + } + b.Write(binary.BigEndian.AppendUint32(nil, uint32(len(l.Data)))) //nolint:gosec // test data is small + b.Write(l.Data) + return keccakOf(b.Bytes()) +} + +func TestBalances(t *testing.T) { + want := common.HexToHash("0xdaa77426c30c02a43d9fba4e841a6556c524d47030762eb14dc4af897e605d9b") + if got := NewBalances().Digest(); !bytes.Equal(got, want[:]) { + t.Errorf("NewBalances().Digest() = %x, want %x", got, want) + } + + a0 := fakechain.Address(0) + high := word(7) + high[0] = 1 + ref := chain.BlockRef{Number: 9, Hash: common.HexToHash("0x99")} + logs := []chain.Log{ + transfer(a0, 1, 2, word(5)), + {Address: a0, Topics: []common.Hash{fakechain.ApprovalTopic, topic(fakechain.Account(1)), topic(fakechain.Account(2))}, + Data: word(3)}, + {Address: a0, Topics: []common.Hash{fakechain.OtherTopic}, Data: word(11)}, + transfer(a0, 3, 4, high), + } + for i := range logs { + logs[i].Index, logs[i].BlockNumber, logs[i].BlockHash = uint32(i), ref.Number, ref.Hash //nolint:gosec // i < 4 + } + s := ApplyBalances(NewBalances(), chain.Block{Ref: ref, Logs: logs}) + wantBal := map[Key]int64{{a0, fakechain.Account(1)}: -5, {a0, fakechain.Account(2)}: 5} + if !maps.Equal(s.Bal, wantBal) || s.Logs != 4 { + t.Errorf("after the four-log block: Bal %v, Logs %d; want %v, 4", s.Bal, s.Logs, wantBal) + } + var h common.Hash + for _, l := range logs { + h = logLink(h, ref, l) + } + if s.Hash != h { + t.Errorf("Hash = %s, want %s", s.Hash.Hex(), h.Hex()) + } + + // Digest: the running hash, the log count, then the balances sorted by token, then account. + var d bytes.Buffer + d.Write(s.Hash[:]) + d.Write(binary.BigEndian.AppendUint64(nil, 4)) + for _, kv := range []struct { + k Key + v int64 + }{{Key{a0, fakechain.Account(1)}, -5}, {Key{a0, fakechain.Account(2)}, 5}} { + d.Write(kv.k.Token[:]) + d.Write(kv.k.Account[:]) + d.Write(binary.BigEndian.AppendUint64(nil, uint64(kv.v))) //nolint:gosec // two's complement on purpose + } + if got, want := s.Digest(), keccakOf(d.Bytes()); !bytes.Equal(got, want[:]) { + t.Errorf("Digest() = %x, want %x", got, want) + } + + clone := CloneBalances(s) + back := chain.Block{Ref: chain.BlockRef{Number: 10, Hash: common.HexToHash("0xaa")}, + Logs: []chain.Log{transfer(a0, 2, 1, word(5))}} + s = ApplyBalances(s, back) + if len(s.Bal) != 0 || s.Logs != 5 { + t.Errorf("after the reverse transfer: Bal %v, Logs %d; want empty, 5", s.Bal, s.Logs) + } + if !maps.Equal(clone.Bal, wantBal) || clone.Logs != 4 || clone.Hash != h { + t.Errorf("clone changed with the original: %+v", clone) + } +} + +func TestBalancesIgnores(t *testing.T) { + a1 := fakechain.Address(1) + ref := chain.BlockRef{Number: 1, Hash: common.HexToHash("0x01")} + bad := []struct { + name string + l chain.Log + }{ + {"value above MaxInt64", transfer(a1, 1, 2, word(1<<63))}, + {"short data", transfer(a1, 1, 2, word(5)[:31])}, + {"long data", transfer(a1, 1, 2, append(word(5), 0))}, + {"two topics", chain.Log{Address: a1, Topics: []common.Hash{fakechain.TransferTopic, topic(fakechain.Account(1))}, + Data: word(5)}}, + {"four topics", chain.Log{Address: a1, Topics: []common.Hash{fakechain.TransferTopic, + topic(fakechain.Account(1)), topic(fakechain.Account(2)), {}}, Data: word(5)}}, + {"approval", chain.Log{Address: a1, Topics: []common.Hash{fakechain.ApprovalTopic, + topic(fakechain.Account(1)), topic(fakechain.Account(2))}, Data: word(5)}}, + {"high byte 23", func() chain.Log { w := word(5); w[23] = 1; return transfer(a1, 1, 2, w) }()}, + } + for _, tt := range bad { + s := ApplyBalances(NewBalances(), chain.Block{Ref: ref, Logs: []chain.Log{tt.l}}) + if len(s.Bal) != 0 || s.Logs != 1 || s.Hash != logLink(common.Hash{}, ref, tt.l) { + t.Errorf("%s: Bal %v, Logs %d, want no balance change and one hashed log", tt.name, s.Bal, s.Logs) + } + } + top := ApplyBalances(NewBalances(), chain.Block{Ref: ref, Logs: []chain.Log{transfer(a1, 1, 2, word(1<<63-1))}}) + if top.Bal[Key{a1, fakechain.Account(2)}] != 1<<63-1 { + t.Errorf("MaxInt64 transfer: Bal %v", top.Bal) + } + // The zero Balances, as a store subject might rebuild it, is usable. + zero := ApplyBalances(Balances{}, chain.Block{Ref: ref, Logs: []chain.Log{transfer(a1, 1, 2, word(4))}}) + if zero.Bal[Key{a1, fakechain.Account(2)}] != 4 { + t.Errorf("transfer onto the zero Balances: Bal %v", zero.Bal) + } + if c := CloneBalances(Balances{}); c.Bal == nil { + t.Error("CloneBalances(Balances{}) has a nil map") + } + self := ApplyBalances(NewBalances(), chain.Block{Ref: ref, Logs: []chain.Log{transfer(a1, 3, 3, word(9))}}) + if len(self.Bal) != 0 { + t.Errorf("self transfer: Bal %v, want empty", self.Bal) + } +} + +func TestBalancesDigestOrder(t *testing.T) { + ref := chain.BlockRef{Number: 1, Hash: common.HexToHash("0x01")} + var logs []chain.Log + for _, x := range [][3]int{{1, 5, 6}, {0, 7, 2}, {1, 0, 3}, {0, 4, 1}} { + logs = append(logs, transfer(fakechain.Address(x[0]), x[1], x[2], word(uint64(10+x[1])))) //nolint:gosec // small + } + s := ApplyBalances(NewBalances(), chain.Block{Ref: ref, Logs: logs}) + var d bytes.Buffer + d.Write(s.Hash[:]) + d.Write(binary.BigEndian.AppendUint64(nil, s.Logs)) + for _, tok := range []int{0, 1} { + for acct := range 8 { + k := Key{fakechain.Address(tok), fakechain.Account(acct)} + if v, ok := s.Bal[k]; ok { + d.Write(k.Token[:]) + d.Write(k.Account[:]) + d.Write(binary.BigEndian.AppendUint64(nil, uint64(v))) //nolint:gosec // two's complement on purpose + } + } + } + if got, want := s.Digest(), keccakOf(d.Bytes()); !bytes.Equal(got, want[:]) || len(s.Bal) != 8 { + t.Errorf("Digest() = %x over %d balances, want %x over 8", got, len(s.Bal), want) + } +} + +func TestExpectedBalances(t *testing.T) { + c := fakechain.New(3, fakechain.Params{}) + c.Mine(40) + blocks := c.CanonicalBlocks(Filter) + s := NewBalances() + for _, b := range blocks { + s = ApplyBalances(s, b) + } + if got := ExpectedBalances(blocks); !bytes.Equal(got, s.Digest()) || s.Logs == 0 { + t.Errorf("ExpectedBalances = %x, want the fold's digest %x (%d logs)", got, s.Digest(), s.Logs) + } + if got := ExpectedBalances(nil); !bytes.Equal(got, NewBalances().Digest()) { + t.Errorf("ExpectedBalances(nil) = %x, want the empty digest", got) + } + + sub := NewMemorySubject() + ctx := t.Context() + if d, err := sub.Digest(ctx); err != nil || !bytes.Equal(d, NewBalances().Digest()) { + t.Errorf("new subject Digest = %x, %v", d, err) + } +} diff --git a/internal/invariant/bench_test.go b/internal/invariant/bench_test.go new file mode 100644 index 0000000..17d3914 --- /dev/null +++ b/internal/invariant/bench_test.go @@ -0,0 +1,22 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package invariant + +import ( + "testing" +) + +// BenchmarkScenario runs one 200-step scenario with the memory subject per iteration, over seeds +// 1 to 100 in turn; the target is 5 ms per scenario on CI amd64. +func BenchmarkScenario(b *testing.B) { + cfg := memoryConfig() + seed := uint64(1) + b.ReportAllocs() + for b.Loop() { + if _, err := Check(b, cfg, seed); err != nil { + b.Fatalf("seed %d: %v", seed, err) + } + seed = seed%100 + 1 + } +} diff --git a/internal/invariant/doc.go b/internal/invariant/doc.go new file mode 100644 index 0000000..b2c4ae6 --- /dev/null +++ b/internal/invariant/doc.go @@ -0,0 +1,6 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +// Package invariant runs the reorg invariant as a seeded property test: the state a consumer builds +// from the canonical log equals the state built from the final canonical chain alone. +package invariant diff --git a/internal/invariant/harness.go b/internal/invariant/harness.go new file mode 100644 index 0000000..416b95c --- /dev/null +++ b/internal/invariant/harness.go @@ -0,0 +1,633 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package invariant + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "os" + "strconv" + "strings" + "sync" + "testing" + + "golang.org/x/crypto/sha3" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/consumer" + "github.com/hmdsefi/evmsync/internal/fakechain" + "github.com/hmdsefi/evmsync/tracker" +) + +// Subject is the system under test. +type Subject interface { + consumer.Consumer + // Digest returns a digest of the subject's state. Equal states give equal digests; the digest + // never depends on map order, the clock or the offset. + Digest(ctx context.Context) ([]byte, error) +} + +// Config parameterizes the harness for one kind of subject. +type Config struct { + // Package is the `go test` package argument printed in the reproduce line, for example + // "./internal/invariant" or "./memstore". Required. + Package string + // New returns a fresh subject: empty state, offset 0. Required. + New func(t testing.TB) Subject + // Reopen, if set, simulates a process restart: it returns a subject that resumes from old's + // durable state (a memory subject returns a fresh one, which replays from offset 1). Nil + // disables restarts. + Reopen func(t testing.TB, old Subject) Subject + // Expected, if set, returns the expected digest for the final canonical blocks + // (CanonicalBlocks(Filter)), as an oracle independent of the subject. + Expected func(blocks []chain.Block) []byte + // Seeds and ShortSeeds are the seed counts without and with -short. 0 means 1,000 and 100. + Seeds, ShortSeeds int + // Steps is the number of chain steps per scenario. 0 means 200. + Steps int + // Parallel runs seeds as parallel subtests. Set it only when subjects of different seeds share + // nothing. + Parallel bool + // Variant, if set, replaces VariantFor (tests of the harness itself). + Variant func(seed uint64) Variant + // Trace logs every step with t.Logf. Run sets it when one seed is selected. + Trace bool +} + +// withDefaults returns cfg with its zero fields defaulted. It panics if Package or New is missing. +func (cfg Config) withDefaults() Config { + if cfg.Package == "" { + panic("evmsync: invariant: Config.Package is empty") + } + if cfg.New == nil { + panic("evmsync: invariant: Config.New is nil") + } + if cfg.Seeds < 0 || cfg.ShortSeeds < 0 { + panic("evmsync: invariant: Config.Seeds or Config.ShortSeeds is negative") + } + if cfg.Seeds == 0 { + cfg.Seeds = 1000 + } + if cfg.ShortSeeds == 0 { + cfg.ShortSeeds = 100 + } + if cfg.Steps == 0 { + cfg.Steps = 200 + } + if cfg.Variant == nil { + cfg.Variant = VariantFor + } + return cfg +} + +// Report summarizes one scenario. +type Report struct { + Seed uint64 + Entries int // entries in the log + Blocks int // Block entries + Reverts int // Revert entries + MaxRevertDepth uint64 // largest (tip number - target number) over the Reverts + Tolerated int // ErrNotFound answers the poll loop skipped + Restarts int // Reopen calls +} + +// The environment variables that select seeds. +const ( + envSeed = "EVMSYNC_INVARIANT_SEED" + envSeeds = "EVMSYNC_INVARIANT_SEEDS" + envBase = "EVMSYNC_INVARIANT_BASE" +) + +// Run runs the selected seeds' scenarios as subtests named "seed=" and fails each subtest on +// its first divergence, with a line that reproduces it. +// +// EVMSYNC_INVARIANT_SEED= runs only seed n, with Trace on. Otherwise +// EVMSYNC_INVARIANT_SEEDS= runs count seeds, and without it Run runs cfg.ShortSeeds seeds +// under -short and cfg.Seeds seeds otherwise. Seeds start at EVMSYNC_INVARIANT_BASE, or 1. Values +// are decimal or 0x hex. When at least 100 seeds ran, Run also fails unless at least half of them +// reverted and one reverted at least 3 blocks, so a generator that stops forking cannot pass. +func Run(t *testing.T, cfg Config) { + t.Helper() + cfg = cfg.withDefaults() + seeds, one, err := selectSeeds(cfg) + if err != nil { + t.Fatal(err) + } + cfg.Trace = cfg.Trace || one + root := strings.Split(t.Name(), "/")[0] + + var mu sync.Mutex + var reports []Report // guarded by mu + t.Cleanup(func() { + mu.Lock() + defer mu.Unlock() + if reason := tameness(reports); reason != "" { + t.Errorf("invariant: scenarios too tame: %s", reason) + } + }) + for _, seed := range seeds { + t.Run("seed="+strconv.FormatUint(seed, 10), func(t *testing.T) { + if cfg.Parallel { + t.Parallel() + } + rep, err := Check(t, cfg, seed) + mu.Lock() + reports = append(reports, rep) + mu.Unlock() + if err != nil { + t.Errorf("invariant: seed %d: %v\nreproduce: %s=%d go test %s -run '^%s$' -count=1 -v", + seed, err, envSeed, seed, cfg.Package, root) + } + }) + } +} + +// selectSeeds returns the seeds Run runs and whether one seed was selected. +func selectSeeds(cfg Config) ([]uint64, bool, error) { + if s := os.Getenv(envSeed); s != "" { + n, err := parseEnv(envSeed, s) + return []uint64{n}, true, err + } + base := uint64(1) + if s := os.Getenv(envBase); s != "" { + var err error + if base, err = parseEnv(envBase, s); err != nil { + return nil, false, err + } + } + count := uint64(cfg.Seeds) //nolint:gosec // withDefaults rejects a negative count + if testing.Short() { + count = uint64(cfg.ShortSeeds) //nolint:gosec // withDefaults rejects a negative count + } + if s := os.Getenv(envSeeds); s != "" { + var err error + if count, err = parseEnv(envSeeds, s); err != nil { + return nil, false, err + } + } + if count > 0 && base+(count-1) < base { + return nil, false, fmt.Errorf("invariant: bad %s: %d seeds from %d pass the largest seed", envSeeds, count, base) + } + seeds := make([]uint64, 0, min(count, 1<<20)) + for i := range count { + seeds = append(seeds, base+i) + } + return seeds, false, nil +} + +func parseEnv(name, value string) (uint64, error) { + n, err := strconv.ParseUint(value, 0, 64) + if err != nil { + return 0, fmt.Errorf("invariant: bad %s: %s", name, value) + } + return n, nil +} + +// tameness returns why reports are too tame to test reorgs, or "" when they are not or when fewer +// than 100 seeds ran. +func tameness(reports []Report) string { + if len(reports) < 100 { + return "" + } + reverted, deep := 0, false + for _, r := range reports { + if r.Reverts >= 1 { + reverted++ + } + deep = deep || r.MaxRevertDepth >= 3 + } + switch { + case 2*reverted < len(reports): + return fmt.Sprintf("%d of %d seeds reverted, want at least half", reverted, len(reports)) + case !deep: + return fmt.Sprintf("no seed of %d reverted 3 or more blocks", len(reports)) + } + return "" +} + +// Check runs seed's scenario and returns its first divergence as an error. It fails t only for +// harness errors, such as New or Reopen returning nil. Check panics if cfg.Package is empty or +// cfg.New is nil. +// +// A scenario drives a fake chain with the seed's variant through a tracker into an in-memory log, +// and drains the subject from the log at random steps, with occasional restarts. It then settles +// the chain, drains the subject once more and requires: the log's structural replay leaves exactly +// the final canonical blocks; the subject's offset is the log head; the subject's digest equals +// the digest of a fresh subject drained over a log built from the canonical blocks alone; and +// cfg.Expected, if set, agrees. +func Check(t testing.TB, cfg Config, seed uint64) (Report, error) { + t.Helper() + out, err := check(t, cfg, seed) + return out.report, err +} + +// outcome is everything a scenario produced, for the harness's own tests. +type outcome struct { + report Report + log []chainlog.Entry + digest []byte // the subject's final digest; nil after an earlier divergence +} + +// scenario is one seed's run. +type scenario struct { + t testing.TB + cfg Config + seed uint64 + ctx context.Context + c *fakechain.Chain // changed only through src.Do + src *fakechain.Source + lg *chainlog.Memory + tr *tracker.Tracker + sub Subject + rep Report +} + +func check(t testing.TB, cfg Config, seed uint64) (outcome, error) { + t.Helper() + cfg = cfg.withDefaults() + v := cfg.Variant(seed) + c := fakechain.New(seed, v.Params) + src := fakechain.NewSource(c, seed, v.Faults) + // A scenario runs under Background, not t.Context(), so a test's cancellation never becomes a + // divergence; ctx never leaves the scenario. + s := &scenario{ + t: t, + cfg: cfg, + seed: seed, + ctx: context.Background(), + c: c, + src: src, + lg: chainlog.NewMemory(c.Meta(Filter), tracker.DefaultWindowCap), + tr: tracker.New(src, tracker.Config{Filter: Filter}, chainlog.NewState(tracker.DefaultWindowCap)), + rep: Report{Seed: seed}, + } + g := c.Genesis().Ref + es, err := s.tr.Finalize(s.ctx, g) + if err := s.add("setup", "setup", g, "finalize", es, err, false); err != nil { + return s.outcome(), err + } + s.sub = s.newSubject() + r := fakechain.NewRand(seed, "invariant/run") + + for i := range cfg.Steps { + var ev fakechain.Event + src.Do(func(c *fakechain.Chain) { ev = c.Step() }) + s.tracef("step %d: %s", i, ev) + if err := s.poll(fmt.Sprintf("poll at step %d", i), fmt.Sprintf("append at step %d", i), true); err != nil { + return s.outcome(), err + } + drain := r.Chance(500_000) + restart := r.Chance(20_000) + if drain { + if err := s.drain(s.lg, s.sub); err != nil { + return s.outcome(), fmt.Errorf("drain: %w", err) + } + } + if restart && cfg.Reopen != nil { + s.sub = cfg.Reopen(t, s.sub) + if s.sub == nil { + t.Fatalf("invariant: seed %d: Config.Reopen returned nil", seed) + } + s.rep.Restarts++ + s.tracef("restart") + } + } + + src.SetFaults(fakechain.Faults{}) + var head chain.BlockRef + src.Do(func(c *fakechain.Chain) { + c.Mine(1) + c.SetFinalized(c.Head().Ref.Number) + head = c.Head().Ref + }) + if err := s.poll("settle", "settle", false); err != nil { + return s.outcome(), err + } + st, err := s.lg.State(s.ctx) + if err != nil { + return s.outcome(), fmt.Errorf("settle: %w", err) + } + if st.Tip() != head || st.Finalized() != head { + return s.outcome(), fmt.Errorf("settle: log tip %s, finalized %s, want head %s", st.Tip(), st.Finalized(), head) + } + if err := s.drain(s.lg, s.sub); err != nil { + return s.outcome(), fmt.Errorf("drain: %w", err) + } + return s.verify(g) +} + +// verify runs the final checks, in order, on a settled and drained scenario. +func (s *scenario) verify(g chain.BlockRef) (outcome, error) { + var canon []chain.Block + s.src.Do(func(c *fakechain.Chain) { canon = c.CanonicalBlocks(Filter) }) + out := s.outcome() + + live, _, err := replay(out.log, g) + if err != nil { + return out, err + } + for k := range max(len(live), len(canon)) { + got, want := "none", "none" + if k < len(live) { + got = live[k].String() + } + if k < len(canon) { + want = canon[k].Ref.String() + } + if got != want { + return out, fmt.Errorf("structure: live block %d is %s, want %s", k, got, want) + } + } + + head, err := s.lg.Head(s.ctx) + if err != nil { + return out, fmt.Errorf("offset: %w", err) + } + off, err := s.sub.Offset(s.ctx) + if err != nil { + return out, fmt.Errorf("offset: %w", err) + } + if off != head { + return out, fmt.Errorf("offset: %d, want %d", off, head) + } + + digest, err := s.sub.Digest(s.ctx) + if err != nil { + return out, fmt.Errorf("digest: subject: %w", err) + } + out.digest = digest + ref, err := s.reference(g, canon) + if err != nil { + return out, fmt.Errorf("digest: reference: %w", err) + } + if !bytes.Equal(digest, ref) { + return out, fmt.Errorf("digest: subject %x, reference %x", digest, ref) + } + if s.cfg.Expected != nil { + if want := s.cfg.Expected(canon); !bytes.Equal(digest, want) { + return out, fmt.Errorf("expected: subject %x, oracle %x", digest, want) + } + } + return out, nil +} + +// poll asks the source for its head and finality, as an ingest loop would, and appends what the +// tracker returns. With tolerate, a not-found answer is counted and skipped; any other error is a +// divergence that starts with pollPrefix, or appendPrefix when Append refuses the entries. Without +// the finalized tag, finality is the head minus FallbackDepth, and the chain is told never to +// reorg below it. +func (s *scenario) poll(pollPrefix, appendPrefix string, tolerate bool) error { + h, err := s.src.Latest(s.ctx) + if err != nil { + return fmt.Errorf("%s: latest: %w", pollPrefix, err) + } + es, err := s.tr.Head(s.ctx, h) + if err := s.add(pollPrefix, appendPrefix, h.Ref, "head", es, err, tolerate); err != nil { + return err + } + fh, err := s.src.Finalized(s.ctx) + if errors.Is(err, chain.ErrUnsupported) { + if h.Ref.Number < FallbackDepth { + return nil + } + fh, err = s.src.HeaderByNumber(s.ctx, h.Ref.Number-FallbackDepth) + if err == nil { + s.src.Do(func(c *fakechain.Chain) { + if fh.Ref.Number > c.Finalized().Ref.Number { + c.SetFinalized(fh.Ref.Number) + } + }) + } + } + if err != nil { + return s.add(pollPrefix, appendPrefix, h.Ref, "finalized", nil, err, tolerate) + } + es, err = s.tr.Finalize(s.ctx, fh.Ref) + return s.add(pollPrefix, appendPrefix, h.Ref, "finalize", es, err, tolerate) +} + +// add appends the result of one tracker or source call to the log. head is the polled head, for +// the trace. +func (s *scenario) add(pollPrefix, appendPrefix string, head chain.BlockRef, what string, + es []chainlog.Entry, err error, tolerate bool, +) error { + if err != nil { + if tolerate && errors.Is(err, chain.ErrNotFound) { + s.rep.Tolerated++ + s.tracef("tolerated: %v", err) + return nil + } + return fmt.Errorf("%s: %s: %w", pollPrefix, what, err) + } + if len(es) == 0 { + return nil + } + last, err := s.lg.Head(s.ctx) + if err != nil { + return fmt.Errorf("%s: %w", appendPrefix, err) + } + if _, err := s.lg.Append(s.ctx, es...); err != nil { + return fmt.Errorf("%s: offset %d: %s: %w", appendPrefix, last+1, describe(es), err) + } + s.tracef("head %d 0x%x: %s", head.Number, head.Hash[:4], describe(es)) + return nil +} + +// describe returns "+block 41 +revert 38 +finalize 30" for entries. +func describe(es []chainlog.Entry) string { + parts := make([]string, len(es)) + for i, e := range es { + n := e.To.Number + if e.Kind == chainlog.KindBlock { + n = e.Block.Ref.Number + } + parts[i] = "+" + e.Kind.String() + " " + strconv.FormatUint(n, 10) + } + return strings.Join(parts, " ") +} + +// drain applies lg's entries to sub up to the log head with consumer.Run, in the calling +// goroutine. Run must stop with context.Canceled right after the head's Apply, at offset head. +func (s *scenario) drain(lg chainlog.Log, sub Subject) error { + head, err := lg.Head(s.ctx) + if err != nil { + return err + } + off, err := sub.Offset(s.ctx) + if err != nil { + return err + } + if off >= head { + return nil + } + s.tracef("drain to %d", head) + ctx, cancel := context.WithCancel(s.ctx) + defer cancel() + if err := consumer.Run(ctx, lg, &cancelAt{Subject: sub, at: head, cancel: cancel}); err != context.Canceled { //nolint:errorlint // Run returns ctx.Err() unchanged + return fmt.Errorf("run to offset %d: %w", head, err) + } + if off, err = sub.Offset(s.ctx); err != nil { + return fmt.Errorf("offset after the run to %d: %w", head, err) + } + if off != head { + return fmt.Errorf("offset %d after the run to %d", off, head) + } + return nil +} + +// cancelAt wraps a subject and cancels the run right after its Apply of offset at succeeds. +type cancelAt struct { + Subject + at chainlog.Offset + cancel context.CancelFunc +} + +func (c *cancelAt) Apply(ctx context.Context, e chainlog.Entry) error { + err := c.Subject.Apply(ctx, e) + if err == nil && e.Offset == c.at { + c.cancel() + } + return err +} + +// reference returns the digest of a fresh subject drained over a log of the final canonical +// blocks alone: the backfill path, sparse final blocks, against the realtime path of the scenario. +func (s *scenario) reference(g chain.BlockRef, canon []chain.Block) ([]byte, error) { + lg, err := s.referenceLog(g, canon) + if err != nil { + return nil, err + } + sub := s.newSubject() + if err := s.drain(lg, sub); err != nil { + return nil, err + } + return sub.Digest(s.ctx) +} + +// referenceLog returns the reference log: the scenario log's metadata with the reference ID, and +// referenceEntries over canon, whose last block is the head. +func (s *scenario) referenceLog(g chain.BlockRef, canon []chain.Block) (*chainlog.Memory, error) { + meta := s.c.Meta(Filter) + meta.ID = referenceID(s.seed) + head := g + if len(canon) > 0 { + head = canon[len(canon)-1].Ref + } + lg := chainlog.NewMemory(meta, tracker.DefaultWindowCap) + if _, err := lg.Append(s.ctx, referenceEntries(g, head, canon)...); err != nil { + return nil, err + } + return lg, nil +} + +// referenceID returns the reference log's ID for seed: the first 16 bytes of +// Keccak-256("evmsync/invariant/reference" || seed as 8 big-endian bytes). +func referenceID(seed uint64) chainlog.ID { + h := sha3.NewLegacyKeccak256() + h.Write([]byte("evmsync/invariant/reference")) + h.Write(binary.BigEndian.AppendUint64(nil, seed)) + var id chainlog.ID + copy(id[:], h.Sum(nil)) + return id +} + +// referenceEntries returns Finalize g; then the blocks with at least one log, as final blocks, in +// chunks of up to 1,000, each chunk followed by a Finalize of its last block; then a Finalize of +// head when head is above the last block. +func referenceEntries(g, head chain.BlockRef, blocks []chain.Block) []chainlog.Entry { + es := []chainlog.Entry{{Kind: chainlog.KindFinalize, To: g}} + last, n := g, 0 + for _, b := range blocks { + if len(b.Logs) == 0 { + continue + } + es = append(es, chainlog.Entry{Kind: chainlog.KindBlock, Block: b, Final: true}) + last = b.Ref + if n++; n == 1000 { + es = append(es, chainlog.Entry{Kind: chainlog.KindFinalize, To: last}) + n = 0 + } + } + if n > 0 { + es = append(es, chainlog.Entry{Kind: chainlog.KindFinalize, To: last}) + } + if head.Number > last.Number { + es = append(es, chainlog.Entry{Kind: chainlog.KindFinalize, To: head}) + } + return es +} + +// newSubject returns cfg.New's subject and fails the test if it is nil. +func (s *scenario) newSubject() Subject { + sub := s.cfg.New(s.t) + if sub == nil { + s.t.Fatalf("invariant: seed %d: Config.New returned nil", s.seed) + } + return sub +} + +// outcome returns the scenario's report, with the log statistics counted from the log, and the +// log itself. +func (s *scenario) outcome() outcome { + out := outcome{report: s.rep} + head, err := s.lg.Head(s.ctx) + if err != nil || head == 0 { + return out + } + out.log, err = s.lg.Read(s.ctx, 1, int(head)) //nolint:gosec // a scenario's log holds thousands of entries + if err != nil { + return out + } + _, st, _ := replay(out.log, s.c.Genesis().Ref) + out.report.Entries = len(out.log) + out.report.Blocks, out.report.Reverts, out.report.MaxRevertDepth = st.blocks, st.reverts, st.maxDepth + return out +} + +// replayStats counts a log's Block and Revert entries and its deepest Revert. +type replayStats struct { + blocks, reverts int + maxDepth uint64 +} + +// replay replays entries onto a stack that starts at genesis g: a Block pushes its ref, a Revert +// pops every block above its target, which must then be the top (same number and hash), and a +// Finalize changes nothing. It returns the live blocks, the stack above g. +func replay(entries []chainlog.Entry, g chain.BlockRef) ([]chain.BlockRef, replayStats, error) { + stack := []chain.BlockRef{g} + var st replayStats + for _, e := range entries { + switch e.Kind { + case chainlog.KindBlock: + st.blocks++ + stack = append(stack, e.Block.Ref) + case chainlog.KindRevert: + st.reverts++ + top := stack[len(stack)-1] + if e.To.Number > top.Number { + return nil, st, fmt.Errorf("structure: offset %d: revert to %s above the top %s", e.Offset, e.To, top) + } + st.maxDepth = max(st.maxDepth, top.Number-e.To.Number) + for len(stack) > 1 && stack[len(stack)-1].Number > e.To.Number { + stack = stack[:len(stack)-1] + } + if top = stack[len(stack)-1]; top.Number != e.To.Number || top.Hash != e.To.Hash { + return nil, st, fmt.Errorf("structure: offset %d: revert to %s, but the stack holds %s there", + e.Offset, e.To, top) + } + } + } + return stack[1:], st, nil +} + +// tracef logs one trace line when the configuration asks for it. +func (s *scenario) tracef(format string, args ...any) { + if s.cfg.Trace { + s.t.Logf(format, args...) + } +} diff --git a/internal/invariant/invariant_test.go b/internal/invariant/invariant_test.go new file mode 100644 index 0000000..5c6c3e6 --- /dev/null +++ b/internal/invariant/invariant_test.go @@ -0,0 +1,644 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package invariant + +import ( + "bytes" + "context" + "errors" + "fmt" + "reflect" + "slices" + "strings" + "testing" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/internal/fakechain" + "github.com/hmdsefi/evmsync/tracker" +) + +// memoryConfig is the configuration of TestReorgInvariant. +func memoryConfig() Config { + return Config{ + Package: "./internal/invariant", + New: func(testing.TB) Subject { return NewMemorySubject() }, + Reopen: func(testing.TB, Subject) Subject { return NewMemorySubject() }, + Expected: ExpectedBalances, + Parallel: true, + } +} + +func TestReorgInvariant(t *testing.T) { + Run(t, memoryConfig()) +} + +func TestHarnessReplays(t *testing.T) { + cfg := memoryConfig() + var reverts, restarts, tolerated int + for seed := uint64(1); seed <= 20; seed++ { + first, err := check(t, cfg, seed) + if err != nil { + t.Fatalf("seed %d: %v", seed, err) + } + second, err := check(t, cfg, seed) + if err != nil { + t.Fatalf("seed %d, second run: %v", seed, err) + } + if first.report != second.report { + t.Errorf("seed %d: reports %+v and %+v differ", seed, first.report, second.report) + } + if !reflect.DeepEqual(first.log, second.log) || len(first.log) != first.report.Entries { + t.Errorf("seed %d: the two runs' logs differ, or the report counts %d of %d entries", + seed, first.report.Entries, len(first.log)) + } + if !bytes.Equal(first.digest, second.digest) || len(first.digest) != 32 { + t.Errorf("seed %d: digests %x and %x", seed, first.digest, second.digest) + } + if first.report.Seed != seed { + t.Errorf("seed %d: Report.Seed = %d", seed, first.report.Seed) + } + checkReport(t, seed, first.report, first.log) + if first.report.Reverts > 0 { + reverts++ + } + restarts += first.report.Restarts + tolerated += first.report.Tolerated + } + if reverts < 10 || restarts == 0 || tolerated == 0 { + t.Errorf("seeds 1..20: %d with reverts, %d restarts, %d tolerated errors; the runs cover too little", + reverts, restarts, tolerated) + } + rep, err := Check(t, cfg, 3) + if err != nil || rep.Seed != 3 || rep.Entries == 0 { + t.Errorf("Check(3) = %+v, %v", rep, err) + } +} + +// checkReport recounts the report's log statistics from the log. +func checkReport(t *testing.T, seed uint64, r Report, log []chainlog.Entry) { + t.Helper() + var blocks, reverts int + var depth uint64 + stack := []chain.BlockRef{fakechain.New(seed, fakechain.Params{}).Genesis().Ref} + for _, e := range log { + switch e.Kind { + case chainlog.KindBlock: + blocks++ + stack = append(stack, e.Block.Ref) + case chainlog.KindRevert: + reverts++ + depth = max(depth, stack[len(stack)-1].Number-e.To.Number) + for stack[len(stack)-1].Number > e.To.Number { + stack = stack[:len(stack)-1] + } + } + } + if r.Entries != len(log) || r.Blocks != blocks || r.Reverts != reverts || r.MaxRevertDepth != depth { + t.Errorf("seed %d: report %+v, want Entries %d Blocks %d Reverts %d MaxRevertDepth %d", + seed, r, len(log), blocks, reverts, depth) + } +} + +// ignoreReverts wraps the memory subject and drops every Revert entry: the orphaned blocks stay +// applied. It renumbers the entries it passes on so the inner subject sees dense offsets. +type ignoreReverts struct { + Subject + off chainlog.Offset + skipped chainlog.Offset +} + +func (s *ignoreReverts) Offset(context.Context) (chainlog.Offset, error) { return s.off, nil } + +func (s *ignoreReverts) Apply(ctx context.Context, e chainlog.Entry) error { + if e.Kind == chainlog.KindRevert { + s.skipped++ + s.off = e.Offset + return nil + } + off := e.Offset + e.Offset -= s.skipped + if err := s.Subject.Apply(ctx, e); err != nil { + return err + } + s.off = off + return nil +} + +func newIgnoreReverts(testing.TB) Subject { return &ignoreReverts{Subject: NewMemorySubject()} } + +// dropFinalLogs wraps the memory subject and applies final blocks without their logs. +type dropFinalLogs struct{ Subject } + +func (s dropFinalLogs) Apply(ctx context.Context, e chainlog.Entry) error { + if e.Kind == chainlog.KindBlock && e.Final { + e.Block.Logs = nil + } + return s.Subject.Apply(ctx, e) +} + +// claimsOffset is a fresh memory subject that reports the offset of the subject it replaced, as +// if its state had survived a restart. +type claimsOffset struct { + Subject + off chainlog.Offset +} + +func (s claimsOffset) Offset(context.Context) (chainlog.Offset, error) { return s.off, nil } + +// constantDigest wraps the memory subject and reports the same digest whatever its state. +type constantDigest struct{ Subject } + +func (constantDigest) Digest(context.Context) ([]byte, error) { return []byte{1}, nil } + +func TestHarnessDetects(t *testing.T) { + ctx := t.Context() + base := memoryConfig() + tests := []struct { + name string + cfg func(Config) Config + prefix []string + also string // text the error must also hold, when set + }{ + {"dropped logs", func(c Config) Config { + c.Variant = func(seed uint64) Variant { + v := VariantFor(seed) + v.Faults.DroppedLogsPPM = 1_000_000 + return v + } + return c + }, []string{"digest: ", "expected: "}, ""}, + {"reverts ignored", func(c Config) Config { + c.New = newIgnoreReverts + c.Reopen = func(tb testing.TB, _ Subject) Subject { return newIgnoreReverts(tb) } + return c + }, []string{"digest: "}, ""}, + {"final logs dropped", func(c Config) Config { + c.New = func(testing.TB) Subject { return dropFinalLogs{NewMemorySubject()} } + c.Reopen = func(testing.TB, Subject) Subject { return dropFinalLogs{NewMemorySubject()} } + return c + }, []string{"digest: "}, ""}, + {"state lost on restart", func(c Config) Config { + c.Reopen = func(tb testing.TB, old Subject) Subject { + off, err := old.Offset(ctx) + if err != nil { + tb.Fatal(err) + } + return claimsOffset{NewMemorySubject(), off} + } + return c + }, []string{"drain: "}, "consumer: apply offset"}, + {"constant digest", func(c Config) Config { + c.New = func(testing.TB) Subject { return constantDigest{NewMemorySubject()} } + c.Reopen = func(testing.TB, Subject) Subject { return constantDigest{NewMemorySubject()} } + return c + }, []string{"expected: "}, ""}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := tt.cfg(base) + var errs []string + for seed := uint64(1); seed <= 50; seed++ { + if _, err := Check(t, cfg, seed); err != nil { + errs = append(errs, err.Error()) + for _, p := range tt.prefix { + if strings.HasPrefix(err.Error(), p) && strings.Contains(err.Error(), tt.also) { + return + } + } + } + } + t.Errorf("no seed in 1..50 failed with %q and %q; errors: %q", tt.prefix, tt.also, errs) + }) + } +} + +func TestDivergenceTexts(t *testing.T) { + cfg := memoryConfig() + cfg.New = func(testing.TB) Subject { return constantDigest{NewMemorySubject()} } + cfg.Reopen = func(testing.TB, Subject) Subject { return constantDigest{NewMemorySubject()} } + cfg.Expected = nil + if _, err := Check(t, cfg, 1); err != nil { + t.Errorf("constant digest without Expected: %v, want no divergence", err) + } + + cfg = memoryConfig() + cfg.New = newIgnoreReverts + cfg.Reopen = nil + for seed := uint64(1); seed <= 50; seed++ { + rep, err := Check(t, cfg, seed) + if rep.Restarts != 0 { + t.Fatalf("seed %d: Restarts %d without Reopen", seed, rep.Restarts) + } + if err == nil { + continue + } + if !strings.HasPrefix(err.Error(), "digest: subject ") || !strings.Contains(err.Error(), ", reference ") { + t.Fatalf("seed %d: %q, want \"digest: subject , reference \"", seed, err) + } + return + } + t.Error("no seed in 1..50 diverged") +} + +func TestStructuralReplay(t *testing.T) { + c := fakechain.New(1, fakechain.Params{}) + g := c.Genesis().Ref + a := c.Mine(4) + b := c.Reorg(2, 3) + blk := func(r chain.BlockRef) chainlog.Entry { + return chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: r}} + } + es := []chainlog.Entry{ + {Kind: chainlog.KindFinalize, To: g}, + blk(a[0]), blk(a[1]), blk(a[2]), blk(a[3]), + {Kind: chainlog.KindRevert, To: a[1]}, + blk(b[0]), blk(b[1]), blk(b[2]), + {Kind: chainlog.KindFinalize, To: b[0]}, + {Kind: chainlog.KindRevert, To: b[1]}, + } + live, st, err := replay(es, g) + want := []chain.BlockRef{a[0], a[1], b[0], b[1]} + if err != nil || !reflect.DeepEqual(live, want) { + t.Fatalf("replay = %v, %v; want %v", live, err, want) + } + if st.blocks != 7 || st.reverts != 2 || st.maxDepth != 2 { + t.Errorf("replay stats %+v, want 7 blocks, 2 reverts, depth 2", st) + } + + // A Revert above the top, or whose target is not on the stack, is an error. + above := slices.Concat(es[:3], []chainlog.Entry{{Kind: chainlog.KindRevert, To: a[3]}}) + if _, _, err := replay(above, g); err == nil || !strings.Contains(err.Error(), "above the top") { + t.Errorf("replay with a revert above the top: %v", err) + } + bad := slices.Concat(es[:5], []chainlog.Entry{{Kind: chainlog.KindRevert, To: b[0]}}) + if _, _, err := replay(bad, g); err == nil || !strings.HasPrefix(err.Error(), "structure: ") { + t.Errorf("replay with a foreign revert target: %v, want a structure error", err) + } + if live, _, err := replay(nil, g); err != nil || len(live) != 0 { + t.Errorf("replay(nil) = %v, %v", live, err) + } +} + +func TestReferenceLog(t *testing.T) { + c := fakechain.New(5, fakechain.Params{}) + c.Mine(6000) + blocks := c.CanonicalBlocks(Filter) + es := referenceEntries(c.Genesis().Ref, c.Head().Ref, blocks) + if !reflect.DeepEqual(es[0], chainlog.Entry{Kind: chainlog.KindFinalize, To: c.Genesis().Ref}) { + t.Fatalf("first entry %+v, want Finalize genesis", es[0]) + } + var want []chain.Block + for _, b := range blocks { + if len(b.Logs) > 0 { + want = append(want, b) + } + } + rest := es[1:] + if want[len(want)-1].Ref != c.Head().Ref { + if last := rest[len(rest)-1]; !reflect.DeepEqual(last, chainlog.Entry{Kind: chainlog.KindFinalize, To: c.Head().Ref}) { + t.Fatalf("last entry %+v, want Finalize of the head", last) + } + rest = rest[:len(rest)-1] + } + var got []chain.Block + var chunks []int + n := 0 + for i, e := range rest { + switch e.Kind { + case chainlog.KindBlock: + if !e.Final { + t.Fatalf("entry %d: block %d is not final", i+1, e.Block.Ref.Number) + } + got = append(got, e.Block) + n++ + case chainlog.KindFinalize: + if n == 0 || e.To != got[len(got)-1].Ref { + t.Fatalf("entry %d: Finalize %v, want the chunk's last block", i+1, e.To) + } + chunks = append(chunks, n) + n = 0 + default: + t.Fatalf("entry %d: kind %s", i+1, e.Kind) + } + } + if !reflect.DeepEqual(got, want) || len(want) <= 2000 { + t.Errorf("reference blocks: %d, want the %d canonical blocks with a matching log", len(got), len(want)) + } + for i, k := range chunks { + if n != 0 || k > 1000 || (i < len(chunks)-1 && k != 1000) { + t.Fatalf("chunks %v (open %d), want 1000 each but the last", chunks, n) + } + } + lg := chainlog.NewMemory(chainlog.Meta{ChainID: 31337, Filter: Filter}, 0) + if _, err := lg.Append(t.Context(), es...); err != nil { + t.Errorf("the reference entries do not form a valid log: %v", err) + } + + // A head without matching logs above the last block gets its own Finalize. + g := c.Genesis().Ref + head := c.Head().Ref + short := referenceEntries(g, head, want[:1]) + wantShort := []chainlog.Entry{ + {Kind: chainlog.KindFinalize, To: g}, + {Kind: chainlog.KindBlock, Block: want[0], Final: true}, + {Kind: chainlog.KindFinalize, To: want[0].Ref}, + {Kind: chainlog.KindFinalize, To: head}, + } + if !reflect.DeepEqual(short, wantShort) { + t.Errorf("referenceEntries(one block) = %+v, want %+v", short, wantShort) + } + if es := referenceEntries(g, g, nil); !reflect.DeepEqual(es, []chainlog.Entry{{Kind: chainlog.KindFinalize, To: g}}) { + t.Errorf("referenceEntries(no blocks, head G) = %+v", es) + } +} + +// logTB records Logf lines. +type logTB struct { + testing.TB + lines []string +} + +func (l *logTB) Logf(format string, args ...any) { + l.lines = append(l.lines, fmt.Sprintf(format, args...)) +} + +func TestTrace(t *testing.T) { + cfg := memoryConfig() + quiet := &logTB{TB: t} + if _, err := Check(quiet, cfg, 2); err != nil { + t.Fatal(err) + } + if len(quiet.lines) != 0 { + t.Errorf("Trace off logged %d lines: %q", len(quiet.lines), quiet.lines[0]) + } + cfg.Trace = true + loud := &logTB{TB: t} + if _, err := Check(loud, cfg, 2); err != nil { + t.Fatal(err) + } + prefixes := []string{"step 0: ", "step 199: ", "head ", "drain to ", "tolerated: "} + for _, p := range prefixes { + found := false + for _, l := range loud.lines { + found = found || strings.HasPrefix(l, p) + } + if !found { + t.Errorf("no trace line starts with %q", p) + } + } + for _, l := range loud.lines { + if strings.HasPrefix(l, "head ") && !strings.Contains(l, ": +") { + t.Errorf("append line %q has no + part", l) + } + } +} + +func TestConfigMisuse(t *testing.T) { + cfg := memoryConfig() + cfg.Package = "" + wantPanic(t, "evmsync: invariant: Config.Package is empty", func() { _, _ = Check(t, cfg, 1) }) + cfg = memoryConfig() + cfg.New = nil + wantPanic(t, "evmsync: invariant: Config.New is nil", func() { _, _ = Check(t, cfg, 1) }) + cfg = memoryConfig() + cfg.ShortSeeds = -1 + wantPanic(t, "evmsync: invariant: Config.Seeds or Config.ShortSeeds is negative", func() { Run(t, cfg) }) +} + +func wantPanic(t *testing.T, msg string, fn func()) { + t.Helper() + defer func() { + t.Helper() + if r := recover(); r != msg { + t.Errorf("panic %v, want %q", r, msg) + } + }() + fn() +} + +// scenarioFor builds a scenario over chain c without running it, for tests of one stage. +func scenarioFor(t *testing.T, c *fakechain.Chain, lg *chainlog.Memory, sub Subject) *scenario { + return &scenario{ + t: t, cfg: memoryConfig().withDefaults(), seed: 1, ctx: t.Context(), c: c, + src: fakechain.NewSource(c, 1, fakechain.Faults{}), lg: lg, sub: sub, rep: Report{Seed: 1}, + } +} + +func TestPollTolerance(t *testing.T) { + c := fakechain.New(1, fakechain.Params{}) + s := scenarioFor(t, c, chainlog.NewMemory(c.Meta(Filter), 0), NewMemorySubject()) + g := c.Genesis().Ref + notFound := fmt.Errorf("tracker: logs: %w", chain.ErrNotFound) + if err := s.add("poll at step 3", "append at step 3", g, "head", nil, notFound, true); err != nil || + s.rep.Tolerated != 1 { + t.Errorf("tolerated not-found: %v, Tolerated %d; want nil, 1", err, s.rep.Tolerated) + } + other := fmt.Errorf("tracker: %w", tracker.ErrInconsistentSource) + err := s.add("poll at step 3", "append at step 3", g, "head", nil, other, true) + if want := "poll at step 3: head: tracker: tracker: inconsistent source"; err == nil || err.Error() != want { + t.Errorf("inconsistent source: %v, want %q", err, want) + } + err = s.add("settle", "settle", g, "finalize", nil, notFound, false) + if err == nil || !strings.HasPrefix(err.Error(), "settle: finalize: ") || s.rep.Tolerated != 1 { + t.Errorf("not-found without tolerance: %v, Tolerated %d; want a settle error, 1", err, s.rep.Tolerated) + } + bad := []chainlog.Entry{{Kind: chainlog.KindRevert, To: chain.BlockRef{Number: 5, Hash: g.Hash}}} + err = s.add("poll at step 4", "append at step 4", g, "head", bad, nil, true) + if err == nil || !strings.HasPrefix(err.Error(), "append at step 4: offset 1: +revert 5: ") || + !errors.Is(err, chainlog.ErrMalformed) { + t.Errorf("malformed append: %v, want an append error wrapping ErrMalformed", err) + } +} + +func TestVerifyTexts(t *testing.T) { + ctx := t.Context() + // The same seed and calls give the same blocks, so twin's blocks 1 to 3 are c's canonical + // blocks and twin's block 4 extends them: a log over twin can be one block short or one long. + twin := fakechain.New(1, fakechain.Params{}) + long := twin.Mine(4) + c := fakechain.New(1, fakechain.Params{}) + c.Mine(3) + g := c.Genesis().Ref + logOf := func(t *testing.T, refs []chain.BlockRef) *chainlog.Memory { + t.Helper() + lg := chainlog.NewMemory(c.Meta(Filter), 0) + es := []chainlog.Entry{{Kind: chainlog.KindFinalize, To: g}} + for _, r := range refs { + b, ok := twin.Block(r.Hash, Filter) + if !ok { + t.Fatalf("block %s missing", r) + } + es = append(es, chainlog.Entry{Kind: chainlog.KindBlock, Block: b}) + } + if _, err := lg.Append(ctx, es...); err != nil { + t.Fatal(err) + } + return lg + } + tests := []struct { + name string + refs []chain.BlockRef + sub func(head chainlog.Offset) Subject + want string + }{ + {"one block short", long[:2], nil, "structure: live block 2 is none, want " + long[2].String()}, + {"one block long", long, nil, "structure: live block 3 is " + long[3].String() + ", want none"}, + {"offset ahead", long[:3], func(head chainlog.Offset) Subject { + return claimsOffset{NewMemorySubject(), head + 1} + }, "offset: 5, want 4"}, + {"offset behind", long[:3], func(chainlog.Offset) Subject { return NewMemorySubject() }, "offset: 0, want 4"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + lg := logOf(t, tt.refs) + head, _ := lg.Head(ctx) + sub := NewMemorySubject() + if tt.sub != nil { + sub = tt.sub(head) + } + _, err := scenarioFor(t, c, lg, sub).verify(g) + if err == nil || err.Error() != tt.want { + t.Errorf("%v, want %q", err, tt.want) + } + }) + } +} + +func TestReferenceID(t *testing.T) { + var b bytes.Buffer + b.WriteString("evmsync/invariant/reference") + b.Write([]byte{0, 0, 0, 0, 0, 0, 0x01, 0x02}) + h := keccakOf(b.Bytes()) + if got := referenceID(0x0102); !bytes.Equal(got[:], h[:16]) { + t.Errorf("referenceID(0x0102) = %x, want %x", got, h[:16]) + } + c := fakechain.New(1, fakechain.Params{}) + if referenceID(1) == c.Meta(Filter).ID { + t.Error("the reference log shares the scenario log's ID") + } + c.Mine(5) + s := scenarioFor(t, c, nil, nil) + lg, err := s.referenceLog(c.Genesis().Ref, c.CanonicalBlocks(Filter)) + if err != nil { + t.Fatal(err) + } + if meta, _ := lg.Meta(t.Context()); meta.ID != referenceID(1) || meta.ChainID != 31337 { + t.Errorf("reference log meta %+v, want ID %x", meta, referenceID(1)) + } +} + +// TestScenarioGolden pins what three seeds produce. A seed's scenario depends on the harness's +// draws and polls, on fakechain's draws, and on the order of the tracker's Source calls and +// appends, so a change to any of them changes these values. Such a change is expected when +// TestReorgInvariant still passes: re-pin with the got values of the failure messages, and say why +// in the commit. Seed 2's source has no finalized tag. +func TestScenarioGolden(t *testing.T) { + tests := []struct { + want Report + log string + }{ + {Report{Seed: 1, Entries: 766, Blocks: 678, Reverts: 29, MaxRevertDepth: 2, Restarts: 4}, "7f6b4da3a74d4735"}, + {Report{Seed: 2, Entries: 682, Blocks: 521, Reverts: 53, MaxRevertDepth: 6, Tolerated: 5, Restarts: 2}, + "90446bcee516147f"}, + {Report{Seed: 3, Entries: 314, Blocks: 235, Reverts: 55, MaxRevertDepth: 5, Restarts: 4}, "4889f8cea595adc1"}, + } + for _, tt := range tests { + out, err := check(t, memoryConfig(), tt.want.Seed) + if err != nil || out.report != tt.want { + t.Errorf("seed %d: Check = %#v, %v; want %#v", tt.want.Seed, out.report, err, tt.want) + } + if got := logFingerprint(out.log); got != tt.log { + t.Errorf("seed %d: log fingerprint %s, want %s", tt.want.Seed, got, tt.log) + } + } +} + +// logFingerprint hashes every entry's kind and block or target ref, so a change to which blocks +// are appended, reverted or finalized changes it. +func logFingerprint(es []chainlog.Entry) string { + var b bytes.Buffer + for _, e := range es { + r := e.To + if e.Kind == chainlog.KindBlock { + r = e.Block.Ref + } + fmt.Fprintf(&b, "%d %s %v\n", e.Kind, r, e.Final) + } + h := keccakOf(b.Bytes()) + return fmt.Sprintf("%x", h[:8]) +} + +// scriptedSubject is a memory subject whose Apply always fails and whose Offset reports offs in +// turn, then the last one. +type scriptedSubject struct { + Subject + offs []chainlog.Offset +} + +func (s *scriptedSubject) Offset(context.Context) (chainlog.Offset, error) { + off := s.offs[0] + if len(s.offs) > 1 { + s.offs = s.offs[1:] + } + return off, nil +} + +func (s *scriptedSubject) Apply(context.Context, chainlog.Entry) error { return errors.New("boom") } + +func TestDrainErrors(t *testing.T) { + c := fakechain.New(1, fakechain.Params{}) + lg := chainlog.NewMemory(c.Meta(Filter), 0) + if _, err := lg.Append(t.Context(), chainlog.Entry{Kind: chainlog.KindFinalize, To: c.Genesis().Ref}); err != nil { + t.Fatal(err) + } + s := scenarioFor(t, c, lg, nil) + // The offset reads 1 after the run, as if Apply had succeeded: only Run's error shows the failure. + err := s.drain(lg, &scriptedSubject{NewMemorySubject(), []chainlog.Offset{0, 0, 1}}) + if want := "run to offset 1: consumer: apply offset 1 (finalize): boom"; err == nil || err.Error() != want { + t.Errorf("failing Apply: %v, want %q", err, want) + } + if err := s.drain(lg, &scriptedSubject{NewMemorySubject(), []chainlog.Offset{1}}); err != nil { + t.Errorf("subject at the head: %v, want no run", err) + } +} + +// fatalTB records the first Fatalf and stops the scenario with a panic the test recovers. +type fatalTB struct { + testing.TB + msg string +} + +type fatalStop struct{} + +func (f *fatalTB) Fatalf(format string, args ...any) { + f.msg = fmt.Sprintf(format, args...) + panic(fatalStop{}) +} + +func TestNilSubjects(t *testing.T) { + run := func(cfg Config) (msg string) { + f := &fatalTB{TB: t} + defer func() { + if r := recover(); r != nil { + if _, ok := r.(fatalStop); !ok { + panic(r) + } + msg = f.msg + } + }() + _, _ = Check(f, cfg, 1) + return "" + } + cfg := memoryConfig() + cfg.New = func(testing.TB) Subject { return nil } + if got, want := run(cfg), "invariant: seed 1: Config.New returned nil"; got != want { + t.Errorf("New returning nil: Fatalf %q, want %q", got, want) + } + cfg = memoryConfig() + cfg.Reopen = func(testing.TB, Subject) Subject { return nil } + if got, want := run(cfg), "invariant: seed 1: Config.Reopen returned nil"; got != want { + t.Errorf("Reopen returning nil: Fatalf %q, want %q", got, want) + } +} diff --git a/internal/invariant/selection_test.go b/internal/invariant/selection_test.go new file mode 100644 index 0000000..c9e5954 --- /dev/null +++ b/internal/invariant/selection_test.go @@ -0,0 +1,259 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package invariant + +import ( + "fmt" + "os" + "os/exec" + "regexp" + "slices" + "strconv" + "strings" + "testing" + + "github.com/hmdsefi/evmsync/internal/fakechain" +) + +// childEnv marks a re-executed test binary, so the child-only tests run and the parent tests do +// not recurse. +const childEnv = "EVMSYNC_INVARIANT_CHILD" + +func isChild() bool { return os.Getenv(childEnv) == "1" } + +// mutantConfig is TestReorgInvariant's configuration with a subject that ignores reverts. +func mutantConfig() Config { + cfg := memoryConfig() + cfg.New = newIgnoreReverts + cfg.Reopen = func(tb testing.TB, _ Subject) Subject { return newIgnoreReverts(tb) } + return cfg +} + +// TestMutantInvariant runs only in a child process: its subject is broken on purpose. +func TestMutantInvariant(t *testing.T) { + if !isChild() { + t.Skip("runs only when re-executed by TestRunSelection") + } + Run(t, mutantConfig()) +} + +// TestTameInvariant runs only in a child process: its chains almost never reorg. +func TestTameInvariant(t *testing.T) { + if !isChild() { + t.Skip("runs only when re-executed by TestTameness") + } + cfg := memoryConfig() + cfg.Variant = func(uint64) Variant { + return Variant{Params: fakechain.Params{ForkPPM: 1, MaxForkDepth: 1}} + } + Run(t, cfg) +} + +// child runs this test binary with -test.run run, -test.v and the extra arguments, with env +// added to an environment that has no EVMSYNC_INVARIANT_ variable. It returns the combined +// output and whether the run passed. +func child(t *testing.T, run string, env []string, args ...string) (string, bool) { + t.Helper() + if isChild() { + t.Skip("a child process does not re-execute itself") + } + argv := append([]string{"-test.run", run, "-test.v", "-test.count=1"}, args...) + cmd := exec.CommandContext(t.Context(), os.Args[0], argv...) //nolint:gosec // this test binary + for _, kv := range os.Environ() { + if !strings.HasPrefix(kv, "EVMSYNC_INVARIANT_") { + cmd.Env = append(cmd.Env, kv) + } + } + cmd.Env = append(cmd.Env, env...) + cmd.Env = append(cmd.Env, childEnv+"=1") + out, err := cmd.CombinedOutput() + if err != nil { + if _, ok := err.(*exec.ExitError); !ok { //nolint:errorlint // only the exit status is expected + t.Fatalf("running %s: %v", run, err) + } + } + return string(out), err == nil +} + +var seedRun = regexp.MustCompile(`(?m)^=== RUN\s+\S+/seed=(\d+)$`) + +// seedsRun returns the seeds of the subtests the output shows starting, in order. +func seedsRun(t *testing.T, out string) []uint64 { + t.Helper() + var seeds []uint64 + for _, m := range seedRun.FindAllStringSubmatch(out, -1) { + n, err := strconv.ParseUint(m[1], 10, 64) + if err != nil { + t.Fatal(err) + } + seeds = append(seeds, n) + } + return seeds +} + +func span(from, n uint64) []uint64 { + var s []uint64 + for i := range n { + s = append(s, from+i) + } + return s +} + +func TestRunSelection(t *testing.T) { + const run = "^TestReorgInvariant$" + tests := []struct { + name string + env []string + args []string + seeds []uint64 + trace bool + }{ + {"one seed", []string{"EVMSYNC_INVARIANT_SEED=7"}, nil, []uint64{7}, true}, + {"hex seed", []string{"EVMSYNC_INVARIANT_SEED=0x10"}, nil, []uint64{16}, true}, + {"seed wins over count", []string{"EVMSYNC_INVARIANT_SEED=7", "EVMSYNC_INVARIANT_SEEDS=3"}, nil, + []uint64{7}, true}, + {"count from base", []string{"EVMSYNC_INVARIANT_SEEDS=3", "EVMSYNC_INVARIANT_BASE=10"}, nil, + []uint64{10, 11, 12}, false}, + {"count from 1", []string{"EVMSYNC_INVARIANT_SEEDS=2"}, nil, []uint64{1, 2}, false}, + {"short", nil, []string{"-test.short"}, span(1, 100), false}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + out, ok := child(t, run, tt.env, tt.args...) + got := seedsRun(t, out) + slices.Sort(got) + if !ok || !slices.Equal(got, tt.seeds) { + t.Fatalf("passed %v, seeds %v, want %v; output:\n%s", ok, got, tt.seeds, tail(out)) + } + if traced := strings.Contains(out, "step 0: "); traced != tt.trace { + t.Errorf("trace lines present %v, want %v", traced, tt.trace) + } + }) + } + + bad := []struct{ env, want string }{ + {"EVMSYNC_INVARIANT_SEED=x", "invariant: bad EVMSYNC_INVARIANT_SEED: x"}, + {"EVMSYNC_INVARIANT_SEEDS=-1", "invariant: bad EVMSYNC_INVARIANT_SEEDS: -1"}, + {"EVMSYNC_INVARIANT_BASE=1e3", "invariant: bad EVMSYNC_INVARIANT_BASE: 1e3"}, + } + for _, tt := range bad { + t.Run(tt.env, func(t *testing.T) { + out, ok := child(t, run, []string{tt.env}) + if ok || !strings.Contains(out, tt.want) || len(seedsRun(t, out)) != 0 { + t.Errorf("passed %v, want a failure with %q and no seed run; output:\n%s", ok, tt.want, tail(out)) + } + }) + } + + t.Run("reproduce line", func(t *testing.T) { + var seed uint64 + for s := uint64(1); s <= 50 && seed == 0; s++ { + if _, err := Check(t, mutantConfig(), s); err != nil { + seed = s + } + } + if seed == 0 { + t.Fatal("the mutant subject passes seeds 1..50") + } + out, ok := child(t, "^TestMutantInvariant$", []string{fmt.Sprintf("EVMSYNC_INVARIANT_SEED=%d", seed)}) + want := fmt.Sprintf("reproduce: EVMSYNC_INVARIANT_SEED=%d go test ./internal/invariant "+ + "-run '^TestMutantInvariant$' -count=1 -v", seed) + lines := strings.Split(out, "\n") + at := slices.IndexFunc(lines, func(l string) bool { return strings.TrimSpace(l) == want }) + if ok || at < 1 || !strings.Contains(lines[at-1], fmt.Sprintf("invariant: seed %d: ", seed)) { + t.Errorf("passed %v, want a failure line then %q; output:\n%s", ok, want, tail(out)) + } + }) +} + +func TestTameness(t *testing.T) { + out, ok := child(t, "^TestTameInvariant$", []string{"EVMSYNC_INVARIANT_SEEDS=100"}) + if ok || !strings.Contains(out, "invariant: scenarios too tame: ") || len(seedsRun(t, out)) != 100 { + t.Errorf("100 tame seeds: passed %v, want a failure with the tameness message; output:\n%s", ok, tail(out)) + } + out, ok = child(t, "^TestTameInvariant$", []string{"EVMSYNC_INVARIANT_SEEDS=99"}) + if !ok || strings.Contains(out, "too tame") { + t.Errorf("99 tame seeds: passed %v, want a pass without the guard; output:\n%s", ok, tail(out)) + } +} + +// tail returns the last 40 lines of out. +func tail(out string) string { + lines := strings.Split(out, "\n") + return strings.Join(lines[max(0, len(lines)-40):], "\n") +} + +func TestSelectSeeds(t *testing.T) { + cfg := memoryConfig().withDefaults() + cfg.Seeds, cfg.ShortSeeds = 7, 3 + tests := []struct { + name string + env map[string]string + want []uint64 + one bool + error string + }{ + {"default", nil, span(1, map[bool]uint64{false: 7, true: 3}[testing.Short()]), false, ""}, + {"base", map[string]string{envBase: "40"}, span(40, map[bool]uint64{false: 7, true: 3}[testing.Short()]), false, ""}, + {"count", map[string]string{envSeeds: "0x2", envBase: "9"}, []uint64{9, 10}, false, ""}, + {"zero count", map[string]string{envSeeds: "0"}, []uint64{}, false, ""}, + {"one", map[string]string{envSeed: "5", envSeeds: "9", envBase: "x"}, []uint64{5}, true, ""}, + {"max seed", map[string]string{envSeed: "18446744073709551615"}, []uint64{1<<64 - 1}, true, ""}, + {"bad seed", map[string]string{envSeed: "-3"}, nil, false, "invariant: bad EVMSYNC_INVARIANT_SEED: -3"}, + {"bad count", map[string]string{envSeeds: "two"}, nil, false, "invariant: bad EVMSYNC_INVARIANT_SEEDS: two"}, + {"bad base", map[string]string{envBase: "0x"}, nil, false, "invariant: bad EVMSYNC_INVARIANT_BASE: 0x"}, + {"last seed", map[string]string{envSeeds: "2", envBase: "18446744073709551614"}, []uint64{1<<64 - 2, 1<<64 - 1}, + false, ""}, + {"past the last seed", map[string]string{envSeeds: "3", envBase: "18446744073709551614"}, nil, false, + "invariant: bad EVMSYNC_INVARIANT_SEEDS: 3 seeds from 18446744073709551614 pass the largest seed"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for _, name := range []string{envSeed, envSeeds, envBase} { + t.Setenv(name, tt.env[name]) + } + got, one, err := selectSeeds(cfg) + if tt.error != "" { + if err == nil || err.Error() != tt.error { + t.Errorf("error %v, want %q", err, tt.error) + } + return + } + if err != nil || one != tt.one || !slices.Equal(got, tt.want) { + t.Errorf("selectSeeds = %v, %v, %v; want %v, %v", got, one, err, tt.want, tt.one) + } + }) + } +} + +func TestTamenessGuard(t *testing.T) { + reports := func(n, reverted int, depth uint64) []Report { + rs := make([]Report, n) + for i := range reverted { + rs[i].Reverts = 1 + rs[i].MaxRevertDepth = 1 + } + if reverted > 0 { + rs[0].MaxRevertDepth = depth + } + return rs + } + tests := []struct { + name string + rs []Report + want string + }{ + {"fewer than 100 seeds", reports(99, 0, 0), ""}, + {"lively", reports(100, 50, 3), ""}, + {"too few reverts", reports(100, 49, 8), "49 of 100 seeds reverted, want at least half"}, + {"too shallow", reports(101, 101, 2), "no seed of 101 reverted 3 or more blocks"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := tameness(tt.rs); got != tt.want { + t.Errorf("tameness = %q, want %q", got, tt.want) + } + }) + } +} diff --git a/internal/invariant/variant.go b/internal/invariant/variant.go new file mode 100644 index 0000000..e49809a --- /dev/null +++ b/internal/invariant/variant.go @@ -0,0 +1,38 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package invariant + +import ( + "github.com/hmdsefi/evmsync/internal/fakechain" +) + +// Filter is the ingest filter of every scenario: fakechain.DefaultFilter. +var Filter = fakechain.DefaultFilter + +// FallbackDepth is the finality depth the harness uses when the source has no finalized tag. +const FallbackDepth = 64 + +// Variant is a scenario's chain parameters and provider faults. +type Variant struct { + Params fakechain.Params + Faults fakechain.Faults +} + +// VariantFor draws seed's variant from the seed's "invariant/params" stream: the fork chance and +// depth, the blocks per mining step, and the provider faults the tracker must survive. The other +// fields are zero, so they take their defaults. Logs are never dropped: the tracker does not +// validate logs, ingest does. +func VariantFor(seed uint64) Variant { + r := fakechain.NewRand(seed, "invariant/params") + var v Variant + v.Params.ForkPPM = [3]uint32{50_000, 150_000, 400_000}[r.Intn(3)] + v.Params.MaxForkDepth = [3]uint64{1, 3, 8}[r.Intn(3)] + v.Params.MaxMine = [3]uint64{1, 3, 6}[r.Intn(3)] + v.Faults.FinalizedUnsupported = r.Chance(250_000) + v.Faults.StaleHeadPPM = [2]uint32{0, 100_000}[r.Intn(2)] + v.Faults.StaleOrphanPPM = [2]uint32{0, 100_000}[r.Intn(2)] + v.Faults.DropOrphans = r.Chance(500_000) + v.Faults.InterleavePPM = [3]uint32{0, 50_000, 100_000}[r.Intn(3)] + return v +} diff --git a/internal/invariant/variant_test.go b/internal/invariant/variant_test.go new file mode 100644 index 0000000..0bc4726 --- /dev/null +++ b/internal/invariant/variant_test.go @@ -0,0 +1,113 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package invariant + +import ( + "fmt" + "testing" + + "github.com/hmdsefi/evmsync/internal/fakechain" +) + +func TestVariantFor(t *testing.T) { + tests := []struct { + seed uint64 + want Variant + }{ + {1, Variant{ + Params: fakechain.Params{ForkPPM: 150_000, MaxForkDepth: 1, MaxMine: 6}, + Faults: fakechain.Faults{DropOrphans: true}, + }}, + {2, Variant{ + Params: fakechain.Params{ForkPPM: 400_000, MaxForkDepth: 3, MaxMine: 6}, + Faults: fakechain.Faults{FinalizedUnsupported: true, StaleHeadPPM: 100_000, StaleOrphanPPM: 100_000, + DropOrphans: true}, + }}, + } + for _, tt := range tests { + t.Run(fmt.Sprintf("seed=%d", tt.seed), func(t *testing.T) { + if got := VariantFor(tt.seed); got != tt.want { + t.Errorf("VariantFor = %+v, want %+v", got, tt.want) + } + }) + } + + // Every choice of every field appears over the first 200 seeds, and nothing else does. + seen := map[string]map[any]bool{} + note := func(field string, v any) { + if seen[field] == nil { + seen[field] = map[any]bool{} + } + seen[field][v] = true + } + for seed := uint64(1); seed <= 200; seed++ { + v := VariantFor(seed) + note("ForkPPM", v.Params.ForkPPM) + note("MaxForkDepth", v.Params.MaxForkDepth) + note("MaxMine", v.Params.MaxMine) + note("FinalizedUnsupported", v.Faults.FinalizedUnsupported) + note("StaleHeadPPM", v.Faults.StaleHeadPPM) + note("StaleOrphanPPM", v.Faults.StaleOrphanPPM) + note("DropOrphans", v.Faults.DropOrphans) + note("InterleavePPM", v.Faults.InterleavePPM) + rest := v + rest.Params.ForkPPM, rest.Params.MaxForkDepth, rest.Params.MaxMine = 0, 0, 0 + rest.Faults.FinalizedUnsupported, rest.Faults.StaleHeadPPM, rest.Faults.StaleOrphanPPM = false, 0, 0 + rest.Faults.DropOrphans, rest.Faults.InterleavePPM = false, 0 + if rest != (Variant{}) { + t.Fatalf("VariantFor(%d) sets other fields: %+v", seed, v) + } + } + want := map[string]int{ + "ForkPPM": 3, "MaxForkDepth": 3, "MaxMine": 3, "FinalizedUnsupported": 2, + "StaleHeadPPM": 2, "StaleOrphanPPM": 2, "DropOrphans": 2, "InterleavePPM": 3, + } + for field, n := range want { + if len(seen[field]) != n { + t.Errorf("%s took values %v over 200 seeds, want %d distinct", field, seen[field], n) + } + } + for _, v := range []any{uint32(50_000), uint32(150_000), uint32(400_000)} { + if !seen["ForkPPM"][v] { + t.Errorf("ForkPPM never %v", v) + } + } + for _, v := range []any{uint64(1), uint64(3), uint64(8)} { + if !seen["MaxForkDepth"][v] { + t.Errorf("MaxForkDepth never %v", v) + } + } + for _, v := range []any{uint64(1), uint64(3), uint64(6)} { + if !seen["MaxMine"][v] { + t.Errorf("MaxMine never %v", v) + } + } + for _, v := range []any{uint32(0), uint32(50_000), uint32(100_000)} { + if !seen["InterleavePPM"][v] { + t.Errorf("InterleavePPM never %v", v) + } + } + if !seen["StaleHeadPPM"][uint32(100_000)] || !seen["StaleOrphanPPM"][uint32(100_000)] { + t.Error("a stale fault never took 100,000") + } +} + +// TestVariantDraws checks VariantFor's draw order on the "invariant/params" stream against a model. +func TestVariantDraws(t *testing.T) { + for seed := uint64(1); seed <= 100; seed++ { + r := fakechain.NewRand(seed, "invariant/params") + var want Variant + want.Params.ForkPPM = []uint32{50_000, 150_000, 400_000}[r.Intn(3)] + want.Params.MaxForkDepth = []uint64{1, 3, 8}[r.Intn(3)] + want.Params.MaxMine = []uint64{1, 3, 6}[r.Intn(3)] + want.Faults.FinalizedUnsupported = r.Intn(1_000_000) < 250_000 + want.Faults.StaleHeadPPM = []uint32{0, 100_000}[r.Intn(2)] + want.Faults.StaleOrphanPPM = []uint32{0, 100_000}[r.Intn(2)] + want.Faults.DropOrphans = r.Intn(1_000_000) < 500_000 + want.Faults.InterleavePPM = []uint32{0, 50_000, 100_000}[r.Intn(3)] + if got := VariantFor(seed); got != want { + t.Fatalf("VariantFor(%d) = %+v, want %+v", seed, got, want) + } + } +} diff --git a/scripts/gate.sh b/scripts/gate.sh new file mode 100755 index 0000000..d5bf137 --- /dev/null +++ b/scripts/gate.sh @@ -0,0 +1,162 @@ +#!/usr/bin/env bash +# Copyright 2026 Hamed Yousefi +# SPDX-License-Identifier: Apache-2.0 + +# The local mirror of CI. Run it before every commit; it checks the working tree as it is. +# +# usage: scripts/gate.sh no arguments; exit 0 green, 1 red, 2 usage error +# GATE_TCS space-separated GOTOOLCHAIN values to run, default "local" (e.g. "local go1.26.0") +# GATE_LOGS log directory, default "/tmp/gate" +# GATE_LINT golangci-lint binary, default "golangci-lint" +# +# It prints one PASS, FAIL or SKIP line per step and toolchain, then RESULT GREEN or RESULT RED. +# Each step's output goes to $GATE_LOGS/-.log. + +set -uo pipefail + +if [ $# -ne 0 ]; then + echo "usage: scripts/gate.sh (no arguments; set GATE_TCS, GATE_LOGS, GATE_LINT)" >&2 + exit 2 +fi + +unset GOROOT CDPATH +export GOWORK=off + +root=$(git -C "$(dirname "$0")" rev-parse --show-toplevel) || exit 2 +GATE_TCS=${GATE_TCS:-local} +if [ -z "${GATE_TCS// /}" ]; then + GATE_TCS=local +fi +GATE_LOGS=${GATE_LOGS:-$root/tmp/gate} +GATE_LINT=${GATE_LINT:-golangci-lint} + +mkdir -p "$GATE_LOGS" || exit 2 +GATE_LOGS=$(cd "$GATE_LOGS" && pwd) || exit 2 +# A relative GATE_LINT path must keep working after the cd below and in nested modules. +case $GATE_LINT in +*/*) GATE_LINT=$(cd "$(dirname "$GATE_LINT")" && pwd)/$(basename "$GATE_LINT") || exit 2 ;; +esac +rm -f "$GATE_LOGS"/*.log +cd "$root" || exit 2 + +failures=0 +v= + +# step NAME CMD...: runs CMD with its output in the step's log and prints the step's result line. +# The first three failures of a run also print the interesting end of their log. +step() { + local name=$1 log out + shift + log="$GATE_LOGS/$v-$name.log" + if "$@" >"$log" 2>&1; then + echo "PASS $name [$v]" + return + fi + echo "FAIL $name [$v]" + failures=$((failures + 1)) + if [ "$failures" -gt 3 ]; then + return + fi + out=$(grep -E '^(--- FAIL|FAIL|panic:|# |.*\.go:[0-9]+|.*[Ee]rror|.*undefined|.*cannot )' "$log" | tail -n 25) + if [ -z "$out" ]; then + out=$(tail -n 25 "$log") + fi + if [ -n "$out" ]; then + printf '%s\n' "$out" | sed 's/^/ /' + fi +} + +# run CMD...: writes "+ CMD" to the log, then runs CMD. +run() { + echo "+ $*" + "$@" +} + +# check_gofmt lists the tracked and untracked Go files gofmt would change, using the gofmt of the +# toolchain under test. git ls-files keeps gitignored trees such as .helper/ out of the walk. +check_gofmt() { + local files bad + files=$(git ls-files --cached --others --exclude-standard '*.go') || return 1 + if [ -z "$files" ]; then + return 0 + fi + # Word splitting on purpose, as in CI. + # shellcheck disable=SC2086 + bad=$("$(go env GOROOT)/bin/gofmt" -l $files) || { + printf '%s\n' "$bad" + return 1 + } + if [ -n "$bad" ]; then + printf '%s\n' "$bad" + return 1 + fi +} + +# check_nested tidies and vets each nested module and runs its tests, except e2e's, which need +# Anvil and run in their own CI job. +check_nested() { + local dir rc=0 + for dir in $mods; do + run go -C "$dir" mod tidy -diff || rc=1 + run go -C "$dir" vet ./... || rc=1 + if [ "$dir" != e2e ]; then + run go -C "$dir" test -race -count=1 ./... || rc=1 + fi + done + return $rc +} + +# check_lint runs golangci-lint at the root and in each nested module, after checking its version. +check_lint() { + local version dir rc=0 + version=$("$GATE_LINT" version 2>&1) + case "$version" in + *"version 2.14.0 "* | *"version 2.14.0") ;; + *) + echo "golangci-lint v2.14.0 required, found: $(printf '%s\n' "$version" | head -n 1)" + return 1 + ;; + esac + run "$GATE_LINT" run ./... || rc=1 + for dir in $mods; do + (cd "$dir" && run "$GATE_LINT" run ./...) || rc=1 + done + return $rc +} + +mods=$(git ls-files '*/go.mod') +if [ -n "$mods" ]; then + mods=$(printf '%s\n' "$mods" | xargs -n 1 dirname | sort) +fi + +first=1 +for tc in $GATE_TCS; do + export GOTOOLCHAIN=$tc + v=$(go env GOVERSION 2>/dev/null) || v= + if [ -z "$v" ]; then + v=$tc + fi + step gofmt check_gofmt + step tidy go mod tidy -diff + step vet go vet ./... + step build env CGO_ENABLED=0 go build ./... + step test env CGO_ENABLED=0 go test -count=1 ./... + step race go test -race -count=1 ./... + if [ -z "$mods" ]; then + echo "no nested module" >"$GATE_LOGS/$v-nested.log" + echo "SKIP nested [$v] (no nested module)" + else + step nested check_nested + fi + if [ "$first" = 1 ]; then + step lint check_lint + first=0 + fi +done + +if [ "$failures" -eq 0 ]; then + echo "RESULT GREEN" + exit 0 +fi +echo "RESULT RED" +exit 1 diff --git a/tracker/bench_test.go b/tracker/bench_test.go new file mode 100644 index 0000000..32e7e22 --- /dev/null +++ b/tracker/bench_test.go @@ -0,0 +1,170 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package tracker + +import ( + "context" + "encoding/binary" + "fmt" + "runtime" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +// benchSource answers from a precomputed chain at zero cost: no block has a matching log. +type benchSource struct { + headers map[common.Hash]chain.Header +} + +func (s *benchSource) Latest(context.Context) (chain.Header, error) { return chain.Header{}, nil } +func (s *benchSource) Finalized(context.Context) (chain.Header, error) { return chain.Header{}, nil } + +func (s *benchSource) HeaderByNumber(context.Context, uint64) (chain.Header, error) { + return chain.Header{}, chain.ErrNotFound +} + +func (s *benchSource) HeaderByHash(_ context.Context, h common.Hash) (chain.Header, error) { + if hd, ok := s.headers[h]; ok { + return hd, nil + } + return chain.Header{}, chain.ErrNotFound +} + +func (s *benchSource) Logs(context.Context, common.Hash, chain.Filter) ([]chain.Log, error) { + return nil, nil +} + +func (s *benchSource) LogsInRange(context.Context, uint64, uint64, chain.Filter) ([]chain.Log, error) { + return nil, nil +} + +// benchChain returns n parent-linked refs numbered 0 to n-1 and a source that knows them. +func benchChain(n int) ([]chain.BlockRef, *benchSource) { + hash := func(i int) common.Hash { + var h common.Hash + h[0] = 0xbe + binary.BigEndian.PutUint64(h[24:], uint64(i)+1) //nolint:gosec // i >= 0 + return h + } + refs := make([]chain.BlockRef, n) + src := &benchSource{headers: make(map[common.Hash]chain.Header, n)} + for i := range refs { + refs[i] = chain.BlockRef{Number: uint64(i), Hash: hash(i)} + if i > 0 { + refs[i].Parent = refs[i-1].Hash + } + src.headers[refs[i].Hash] = chain.Header{Ref: refs[i]} + } + return refs, src +} + +// benchTracker returns a tracker finalized at refs[0] with refs[1:window+1] in its window. +func benchTracker(tb testing.TB, refs []chain.BlockRef, src chain.Source, window int) *Tracker { + tb.Helper() + st := chainlog.NewState(0) + st.Apply(chainlog.Entry{Kind: chainlog.KindFinalize, To: refs[0]}) + for _, r := range refs[1 : window+1] { + st.Apply(chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: r}}) + } + return New(src, Config{}, st) +} + +// BenchmarkHeadExtend measures Head of the tip's child over a window of 1,000 to 2,000 blocks: +// the tracker is rebuilt with 1,000 blocks, outside the timer, every 1,000 heads. +func BenchmarkHeadExtend(b *testing.B) { + const window = 1000 + refs, src := benchChain(2*window + 1) + ctx := b.Context() + tr := benchTracker(b, refs, src, window) + next := window + 1 + b.ReportAllocs() + for b.Loop() { + if next == len(refs) { + b.StopTimer() + tr = benchTracker(b, refs, src, window) + next = window + 1 + b.StartTimer() + } + if _, err := tr.Head(ctx, chain.Header{Ref: refs[next]}); err != nil { + b.Fatal(err) + } + next++ + } +} + +// BenchmarkHeadDuplicate measures Head of the tip again. +func BenchmarkHeadDuplicate(b *testing.B) { + const window = 1000 + refs, src := benchChain(window + 1) + ctx := b.Context() + tr := benchTracker(b, refs, src, window) + h := chain.Header{Ref: refs[window]} + b.ReportAllocs() + for b.Loop() { + if got, err := tr.Head(ctx, h); err != nil || got != nil { + b.Fatalf("Head(tip) = %v, %v", got, err) + } + } +} + +// BenchmarkFinalizeSteady measures Finalize of the oldest window block, one block per call. The +// tracker is rebuilt with a full window, outside the timer, once the window is empty, so the +// window shrinks from its full size to zero between rebuilds and compacts along the way. +func BenchmarkFinalizeSteady(b *testing.B) { + for _, window := range []int{1000, 100_000} { + b.Run(fmt.Sprintf("window=%d", window), func(b *testing.B) { + refs, src := benchChain(window + 1) + ctx := b.Context() + tr := benchTracker(b, refs, src, window) + next := 1 + b.ReportAllocs() + for b.Loop() { + if next == len(refs) { + b.StopTimer() + tr = benchTracker(b, refs, src, window) + next = 1 + b.StartTimer() + } + if _, err := tr.Finalize(ctx, refs[next]); err != nil { + b.Fatal(err) + } + next++ + } + }) + } +} + +// BenchmarkFillWindow fills an empty window with DefaultWindowCap heads, one block each, on one +// tracker. B/op is everything the fill allocates, returned entries included; retained-B is the +// heap the full tracker still holds after a garbage collection. +func BenchmarkFillWindow(b *testing.B) { + refs, src := benchChain(DefaultWindowCap + 1) + ctx := b.Context() + var retained float64 + b.ReportAllocs() + for b.Loop() { + var before, after runtime.MemStats + b.StopTimer() + runtime.GC() + runtime.ReadMemStats(&before) + b.StartTimer() + tr := benchTracker(b, refs, src, 0) + for _, r := range refs[1:] { + if _, err := tr.Head(ctx, chain.Header{Ref: r}); err != nil { + b.Fatal(err) + } + } + b.StopTimer() + runtime.GC() + runtime.ReadMemStats(&after) + runtime.KeepAlive(tr) + retained = float64(after.HeapAlloc) - float64(before.HeapAlloc) + b.StartTimer() + } + b.ReportMetric(retained, "retained-B") +} diff --git a/tracker/cap_test.go b/tracker/cap_test.go new file mode 100644 index 0000000..8de653f --- /dev/null +++ b/tracker/cap_test.go @@ -0,0 +1,143 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package tracker_test + +import ( + "errors" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/internal/fakechain" + "github.com/hmdsefi/evmsync/tracker" +) + +func TestWindowCap(t *testing.T) { + t.Run("cap 4", func(t *testing.T) { + e := newEnv(t, tracker.Config{WindowCap: 4}, fakechain.Faults{}, nil) + a := e.mine("A", 8) + e.want(t, e.mustHead(t, a[3]), e.blks(a[:4]...)...) + // State carries the tracker's cap: its window is full too. + st := e.tr.State() + if err := st.Check(e.blk(a[4])); !errors.Is(err, chainlog.ErrMalformed) { + t.Errorf("State().Check(blk(A5)) = %v, want an error wrapping chainlog.ErrMalformed", err) + } + + got, err := e.head(t, a[4]) + wantErr(t, err, tracker.ErrWindowFull, + "tracker: unfinalized window full: block 5 is 5 blocks above finalized block 0 (cap 4)") + e.want(t, got) + e.wantCalls(t, "") + + e.want(t, e.mustFinalize(t, a[1]), e.fin(a[1])) + e.wantCalls(t, "") + e.want(t, e.mustHead(t, a[5]), e.blk(a[4]), e.blk(a[5])) + + before := viewOf(e.tr.State()) + got, err = e.head(t, a[7]) + wantErr(t, err, tracker.ErrWindowFull, + "tracker: unfinalized window full: block 8 is 6 blocks above finalized block 2 (cap 4)") + e.want(t, got) + e.wantCalls(t, "") + + got, err = e.finalize(t, a[7]) + wantErr(t, err, tracker.ErrWindowFull, + "tracker: unfinalized window full: block 8 is 6 blocks above finalized block 2 (cap 4)") + e.want(t, got) + e.wantCalls(t, "") + e.unchanged(t, before) + }) + t.Run("unanchored", func(t *testing.T) { + e := newBareEnv(t, tracker.Config{WindowCap: 4}, fakechain.Faults{}, nil) + a := e.mine("A", 8) + got, err := e.head(t, a[6]) + wantErr(t, err, tracker.ErrUnanchored, "tracker: no finalized block: head 7 "+a[6].Hash.Hex()) + e.want(t, got) + e.wantCalls(t, "") + e.want(t, e.mustFinalize(t, a[6]), e.fin(a[6])) + e.wantCalls(t, "") + }) + t.Run("default cap", func(t *testing.T) { + e := newEnv(t, tracker.Config{WindowCap: 0}, fakechain.Faults{}, nil) + a := e.mine("A", tracker.DefaultWindowCap+1) + got := e.mustHead(t, a[tracker.DefaultWindowCap-1]) + if len(got) != tracker.DefaultWindowCap { + t.Fatalf("Head(block %d) returned %d entries, want %d", tracker.DefaultWindowCap, len(got), tracker.DefaultWindowCap) + } + got, err := e.head(t, a[tracker.DefaultWindowCap]) + wantErr(t, err, tracker.ErrWindowFull, + "tracker: unfinalized window full: block 100001 is 100001 blocks above finalized block 0 (cap 100000)") + e.want(t, got) + e.wantCalls(t, "") + }) +} + +// TestNewOverCap resumes a tracker with cap 4 over a log whose window holds 6 blocks. +func TestNewOverCap(t *testing.T) { + e := newBareEnv(t, tracker.Config{}, fakechain.Faults{}, nil) + a := e.mine("A", 7) + if _, err := e.lg.Append(t.Context(), append([]chainlog.Entry{e.fin(e.g)}, e.blks(a[:6]...)...)...); err != nil { + t.Fatal(err) + } + st, err := e.lg.State(t.Context()) + if err != nil { + t.Fatal(err) + } + e.tr = tracker.New(e.src, tracker.Config{Filter: flt, WindowCap: 4}, st) + e.sameState(t) + + got, err := e.head(t, a[6]) + wantErr(t, err, tracker.ErrWindowFull, + "tracker: unfinalized window full: block 7 is 7 blocks above finalized block 0 (cap 4)") + e.want(t, got) + e.wantCalls(t, "") + + // A head at the tip's height and a Finalize below the tip, both off the window, would add + // blocks too. + before := viewOf(e.tr.State()) + refused := []struct { + name string + call func(t *testing.T) ([]chainlog.Entry, error) + want string + }{ + { + "Head at the tip's height", + func(t *testing.T) ([]chainlog.Entry, error) { + return e.head(t, chain.BlockRef{Number: 6, Hash: common.HexToHash("0x06"), Parent: a[4].Hash}) + }, + "tracker: unfinalized window full: block 6 is 6 blocks above finalized block 0 (cap 4)", + }, + { + "Finalize below the tip", + func(t *testing.T) ([]chainlog.Entry, error) { + return e.finalize(t, chain.BlockRef{Number: 5, Hash: common.HexToHash("0x05"), Parent: a[3].Hash}) + }, + "tracker: unfinalized window full: block 5 is 5 blocks above finalized block 0 (cap 4)", + }, + } + for _, r := range refused { + t.Run(r.name, func(t *testing.T) { + got, err := r.call(t) + wantErr(t, err, tracker.ErrWindowFull, r.want) + e.want(t, got) + e.wantCalls(t, "") + }) + } + e.unchanged(t, before) + + // A duplicate head and a head behind the tip are ignored before the cap is checked. + sibling := chain.BlockRef{Number: 5, Hash: common.HexToHash("0x05"), Parent: a[3].Hash} + for _, h := range []chain.BlockRef{a[5], sibling} { + e.want(t, e.mustHead(t, h)) + e.wantCalls(t, "") + } + + // A Finalize inside the window never fails the cap check, even 5 blocks above F with cap 4. + e.want(t, e.mustFinalize(t, a[4]), e.fin(a[4])) + e.wantCalls(t, "") + e.want(t, e.mustHead(t, a[6]), e.blk(a[6])) + e.wantCalls(t, "L(A7)") +} diff --git a/tracker/doc.go b/tracker/doc.go new file mode 100644 index 0000000..36d0f11 --- /dev/null +++ b/tracker/doc.go @@ -0,0 +1,6 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +// Package tracker turns new heads and finalized blocks of a chain into canonical-log entries. It +// detects reorgs by following parent hashes and keeps the window of unfinalized blocks. +package tracker diff --git a/tracker/errors.go b/tracker/errors.go new file mode 100644 index 0000000..f8b21bc --- /dev/null +++ b/tracker/errors.go @@ -0,0 +1,19 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package tracker + +import "errors" + +var ( + // ErrReorgBelowFinalized reports a branch that does not descend from the finalized + // block: the provider claims a different finalized history. + ErrReorgBelowFinalized = errors.New("tracker: reorg below the finalized block") + // ErrWindowFull reports that the call would grow the window past its cap. + ErrWindowFull = errors.New("tracker: unfinalized window full") + // ErrUnanchored reports a Head call before the first Finalize. + ErrUnanchored = errors.New("tracker: no finalized block") + // ErrInconsistentSource reports a Source answer that contradicts the request or + // an earlier answer. + ErrInconsistentSource = errors.New("tracker: inconsistent source") +) diff --git a/tracker/finalize.go b/tracker/finalize.go new file mode 100644 index 0000000..a10cc27 --- /dev/null +++ b/tracker/finalize.go @@ -0,0 +1,65 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package tracker + +import ( + "context" + "fmt" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +// Finalize processes f, the chain's new finalized block, and returns the entries that +// make the log treat f as final: nothing when f is at or below the current finalized +// number; a single Finalize when f is in the window; and, when f is not in the window, +// the entries of the walk to f followed by Finalize. +// +// Finalize returns ErrWindowFull, without calling the Source, when walking to f would +// grow the window past the cap. On error Finalize returns nil entries and the Tracker's +// state is unchanged. +func (t *Tracker) Finalize(ctx context.Context, f chain.BlockRef) ([]chainlog.Entry, error) { + if f.Hash == (common.Hash{}) { + return nil, fmt.Errorf("%w: zero block hash", ErrInconsistentSource) + } + if !t.anchored() { + t.reset(f) + return []chainlog.Entry{finalizeEntry(f)}, nil + } + if f.Number < t.fin.Number { + return nil, nil + } + if f.Number == t.fin.Number { + if f.Hash == t.fin.Hash { + return nil, nil + } + return nil, fmt.Errorf("%w: finalized block %d is %s, got %s", + ErrReorgBelowFinalized, t.fin.Number, t.fin.Hash.Hex(), f.Hash.Hex()) + } + // Heights at or below F returned above, so a known ref here is a window ref, or F at a + // height it does not have. + if stored, ok := t.known(f.Hash); ok { + if stored.Number != f.Number { + return nil, fmt.Errorf("%w: block %s is at height %d, got %d", + ErrInconsistentSource, f.Hash.Hex(), stored.Number, f.Number) + } + t.finalizeTo(stored) + return []chainlog.Entry{finalizeEntry(stored)}, nil + } + if err := t.checkCap(f); err != nil { + return nil, err + } + _, entries, err := t.walk(ctx, f, 1) + if err != nil { + return nil, err + } + t.reset(f) + return append(entries, finalizeEntry(f)), nil +} + +func finalizeEntry(f chain.BlockRef) chainlog.Entry { + return chainlog.Entry{Kind: chainlog.KindFinalize, To: f} +} diff --git a/tracker/head.go b/tracker/head.go new file mode 100644 index 0000000..7ab555e --- /dev/null +++ b/tracker/head.go @@ -0,0 +1,165 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package tracker + +import ( + "context" + "fmt" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +// Head processes the new head h and returns the entries that make the log follow it: +// nothing for a duplicate, lagging or behind-the-tip head; otherwise an optional +// Revert followed by one Block per new block in increasing number order. +// +// Head returns ErrWindowFull, without calling the Source, when following h would grow +// the window past the cap. On error Head returns nil entries and the Tracker's state +// is unchanged. The caller appends the returned entries with one chainlog.Log.Append +// call. +func (t *Tracker) Head(ctx context.Context, h chain.Header) ([]chainlog.Entry, error) { + r := h.Ref + if r.Hash == (common.Hash{}) { + return nil, fmt.Errorf("%w: zero block hash", ErrInconsistentSource) + } + if !t.anchored() { + return nil, fmt.Errorf("%w: head %d %s", ErrUnanchored, r.Number, r.Hash.Hex()) + } + if _, ok := t.known(r.Hash); ok { + return nil, nil + } + if r.Number <= t.fin.Number || r.Number < t.tip().Number { + return nil, nil + } + if err := t.checkCap(r); err != nil { + return nil, err + } + anc, entries, err := t.walk(ctx, r, 0) + if err != nil { + return nil, err + } + t.follow(anc, entries) + return entries, nil +} + +// walk follows the branch of x down to K and fetches the logs of its new blocks. It returns the +// ancestor and the entries: a Revert to the ancestor when it is below the tip, then one Block per +// new block in increasing number order, with room for spare more entries. walk changes nothing in +// t. +func (t *Tracker) walk(ctx context.Context, x chain.BlockRef, spare int) (chain.BlockRef, []chainlog.Entry, error) { + // Most walks add one or two blocks, which fit in buf. + var buf [4]chain.BlockRef + anc, refs, err := t.branch(ctx, x, buf[:0]) + if err != nil { + return chain.BlockRef{}, nil, err + } + entries, err := t.withLogs(ctx, anc, refs, spare) + if err != nil { + return chain.BlockRef{}, nil, err + } + return anc, entries, nil +} + +// branch follows parent hashes from x until a parent is in K, appending x and every fetched ref to +// refs, highest first. It returns that K ref, the ancestor, and refs. +func (t *Tracker) branch(ctx context.Context, x chain.BlockRef, refs []chain.BlockRef) (chain.BlockRef, []chain.BlockRef, error) { + refs = append(refs, x) + // seen maps the hashes in refs to their numbers. Extending the tip never fetches, so it is + // built only once a fetch is needed. + var seen map[common.Hash]uint64 + cur := x + for { + if p, ok := t.known(cur.Parent); ok { + if p.Number+1 != cur.Number { + return chain.BlockRef{}, nil, fmt.Errorf("%w: block %d %s has parent %s at height %d", + ErrInconsistentSource, cur.Number, cur.Hash.Hex(), p.Hash.Hex(), p.Number) + } + return p, refs, nil + } + if cur.Number-1 <= t.fin.Number { + return chain.BlockRef{}, nil, fmt.Errorf("%w: block %d %s does not descend from finalized block %d %s", + ErrReorgBelowFinalized, cur.Number, cur.Hash.Hex(), t.fin.Number, t.fin.Hash.Hex()) + } + if cur.Parent == (common.Hash{}) { + return chain.BlockRef{}, nil, fmt.Errorf("%w: block %d %s has a zero parent hash", + ErrInconsistentSource, cur.Number, cur.Hash.Hex()) + } + if seen == nil { + seen = map[common.Hash]uint64{x.Hash: x.Number} + } + if n, ok := seen[cur.Parent]; ok { + return chain.BlockRef{}, nil, fmt.Errorf("%w: block %d %s has parent %s, which is block %d of the same branch", + ErrInconsistentSource, cur.Number, cur.Hash.Hex(), cur.Parent.Hex(), n) + } + if err := ctx.Err(); err != nil { + return chain.BlockRef{}, nil, fmt.Errorf("tracker: %w", err) + } + hd, err := t.src.HeaderByHash(ctx, cur.Parent) + if err != nil { + return chain.BlockRef{}, nil, fmt.Errorf("tracker: header %s (block %d): %w", cur.Parent.Hex(), cur.Number-1, err) + } + if hd.Ref.Hash != cur.Parent { + return chain.BlockRef{}, nil, fmt.Errorf("%w: header %s: got hash %s", + ErrInconsistentSource, cur.Parent.Hex(), hd.Ref.Hash.Hex()) + } + if hd.Ref.Number != cur.Number-1 { + return chain.BlockRef{}, nil, fmt.Errorf("%w: header %s: got number %d, want %d", + ErrInconsistentSource, cur.Parent.Hex(), hd.Ref.Number, cur.Number-1) + } + refs = append(refs, hd.Ref) + seen[hd.Ref.Hash] = hd.Ref.Number + cur = hd.Ref + } +} + +// withLogs fetches the logs of refs, lowest first, and returns the entries of a walk that ends at +// anc: a Revert to anc when it is below the tip, then one Block per ref in increasing number +// order. The slice holds exactly those entries plus room for spare more. +func (t *Tracker) withLogs(ctx context.Context, anc chain.BlockRef, refs []chain.BlockRef, spare int) ([]chainlog.Entry, error) { + revert := anc.Number < t.tip().Number + n := len(refs) + spare + if revert { + n++ + } + entries := make([]chainlog.Entry, 0, n) + if revert { + entries = append(entries, chainlog.Entry{Kind: chainlog.KindRevert, To: anc}) + } + for i := len(refs) - 1; i >= 0; i-- { + b := refs[i] + if err := ctx.Err(); err != nil { + return nil, fmt.Errorf("tracker: %w", err) + } + logs, err := t.src.Logs(ctx, b.Hash, t.filter) + if err != nil { + return nil, fmt.Errorf("tracker: logs of block %d %s: %w", b.Number, b.Hash.Hex(), err) + } + entries = append(entries, chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: b, Logs: logs}}) + } + return entries, nil +} + +// checkCap returns ErrWindowFull when adding blocks up to x would grow the window past the cap. +// The window is contiguous above F, so it would hold x.Number - F.Number blocks. x must be above +// F. +func (t *Tracker) checkCap(x chain.BlockRef) error { + if n := x.Number - t.fin.Number; n > uint64(t.cap) { //nolint:gosec // New rejects a negative cap + return fmt.Errorf("%w: block %d is %d blocks above finalized block %d (cap %d)", + ErrWindowFull, x.Number, n, t.fin.Number, t.cap) + } + return nil +} + +// follow applies the result of a successful walk: the window up to anc, then the new blocks. +func (t *Tracker) follow(anc chain.BlockRef, entries []chainlog.Entry) { + t.cut(anc.Number) + for i := range entries { + if entries[i].Kind == chainlog.KindBlock { + t.push(entries[i].Block.Ref) + } + } +} diff --git a/tracker/helpers_test.go b/tracker/helpers_test.go new file mode 100644 index 0000000..6a3675b --- /dev/null +++ b/tracker/helpers_test.go @@ -0,0 +1,346 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package tracker_test + +import ( + "context" + "errors" + "fmt" + "reflect" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/internal/fakechain" + "github.com/hmdsefi/evmsync/tracker" +) + +var flt = fakechain.DefaultFilter + +// env is one tracker over a fake chain, with the log its results are appended to. +type env struct { + c *fakechain.Chain + src *fakechain.Source // records the calls; the tracker may see it through a wrapper + tr *tracker.Tracker // reads src, or the wrapper around it + lg *chainlog.Memory // every returned batch is appended here and must succeed + names map[common.Hash]string // "G", "A5", "B7"… for call traces and entry lists + cfg tracker.Config // the tracker's configuration; a zero Filter becomes flt + g chain.BlockRef // genesis +} + +// newBareEnv returns an env over fakechain.New(1, Params{}) with an unanchored tracker. A nil wrap +// gives the tracker src itself. +func newBareEnv(t *testing.T, cfg tracker.Config, f fakechain.Faults, wrap func(chain.Source) chain.Source) *env { + t.Helper() + c := fakechain.New(1, fakechain.Params{}) + src := fakechain.NewSource(c, 1, f) + var s chain.Source = src + if wrap != nil { + s = wrap(src) + } + if cfg.Filter.Addresses == nil && cfg.Filter.Topics == nil { + cfg.Filter = flt + } + g := c.Genesis().Ref + return &env{ + c: c, + src: src, + tr: tracker.New(s, cfg, chainlog.NewState(tracker.DefaultWindowCap)), + lg: chainlog.NewMemory(c.Meta(flt), tracker.DefaultWindowCap), + names: map[common.Hash]string{g.Hash: "G"}, + cfg: cfg, + g: g, + } +} + +// newEnv returns an env whose tracker is anchored at genesis. +func newEnv(t *testing.T, cfg tracker.Config, f fakechain.Faults, wrap func(chain.Source) chain.Source) *env { + t.Helper() + e := newBareEnv(t, cfg, f, wrap) + got, err := e.finalize(t, e.g) + if err != nil { + t.Fatalf("Finalize(G): %v", err) + } + e.want(t, got, e.fin(e.g)) + return e +} + +// tracked returns an env anchored at genesis, with the chain mined to block n of branch A and +// Head of that block done. +func tracked(t *testing.T, n int) *env { + t.Helper() + e := newEnv(t, tracker.Config{}, fakechain.Faults{}, nil) + refs := e.mine("A", n) + e.mustHead(t, refs[n-1]) + return e +} + +// label names refs "". +func (e *env) label(branch string, refs []chain.BlockRef) { + for _, r := range refs { + e.names[r.Hash] = fmt.Sprintf("%s%d", branch, r.Number) + } +} + +// mine mines n blocks through the Source's lock and names them on branch. +func (e *env) mine(branch string, n int) []chain.BlockRef { + var refs []chain.BlockRef + e.src.Do(func(c *fakechain.Chain) { refs = c.Mine(n) }) + e.label(branch, refs) + return refs +} + +// reorg runs Chain.Reorg through the Source's lock and names the new blocks on branch. +func (e *env) reorg(branch string, depth uint64, n int) []chain.BlockRef { + var refs []chain.BlockRef + e.src.Do(func(c *fakechain.Chain) { refs = c.Reorg(depth, n) }) + e.label(branch, refs) + return refs +} + +// head resets the call record, calls Head with h and appends a non-empty result to the log. +func (e *env) head(t *testing.T, h chain.BlockRef) ([]chainlog.Entry, error) { + t.Helper() + return e.headHeader(t, chain.Header{Ref: h}) +} + +func (e *env) headHeader(t *testing.T, h chain.Header) ([]chainlog.Entry, error) { + t.Helper() + e.src.ResetCalls() + got, err := e.tr.Head(t.Context(), h) + e.appendAll(t, got, err) + return got, err +} + +// finalize resets the call record, calls Finalize with f and appends a non-empty result to the +// log. +func (e *env) finalize(t *testing.T, f chain.BlockRef) ([]chainlog.Entry, error) { + t.Helper() + e.src.ResetCalls() + got, err := e.tr.Finalize(t.Context(), f) + e.appendAll(t, got, err) + return got, err +} + +func (e *env) appendAll(t *testing.T, got []chainlog.Entry, err error) { + t.Helper() + if err != nil { + if got != nil { + t.Fatalf("entries %s returned with error %v", e.show(got), err) + } + return + } + if len(got) == 0 { + return + } + if _, aerr := e.lg.Append(t.Context(), got...); aerr != nil { + t.Fatalf("Append(%s): %v", e.show(got), aerr) + } + e.sameState(t) +} + +// mustHead calls head and fails the test on an error. +func (e *env) mustHead(t *testing.T, h chain.BlockRef) []chainlog.Entry { + t.Helper() + got, err := e.head(t, h) + if err != nil { + t.Fatalf("Head(%s): %v", e.name(h.Hash), err) + } + return got +} + +// mustFinalize calls finalize and fails the test on an error. +func (e *env) mustFinalize(t *testing.T, f chain.BlockRef) []chainlog.Entry { + t.Helper() + got, err := e.finalize(t, f) + if err != nil { + t.Fatalf("Finalize(%s): %v", e.name(f.Hash), err) + } + return got +} + +// calls returns the recorded calls, for example []string{"H(A9)", "H(A8)", "L(A8)"}. +func (e *env) calls() []string { + var out []string + for _, c := range e.src.Calls() { + switch c.Method { + case "HeaderByHash": + out = append(out, "H("+e.name(c.Hash)+")") + case "Logs": + out = append(out, "L("+e.name(c.Hash)+")") + default: + out = append(out, c.Method) + } + } + return out +} + +// wantCalls fails the test unless the recorded calls are want, written "H(A5) L(A5)". +func (e *env) wantCalls(t *testing.T, want string) { + t.Helper() + if got := strings.Join(e.calls(), " "); got != want { + t.Errorf("calls = %q, want %q", got, want) + } +} + +// name returns the label of h, or its first four bytes. +func (e *env) name(h common.Hash) string { + if n, ok := e.names[h]; ok { + return n + } + return h.Hex()[:10] +} + +// blk returns the Block entry of x with its logs that match flt. +func (e *env) blk(x chain.BlockRef) chainlog.Entry { + b, ok := e.c.Block(x.Hash, flt) + if !ok { + panic("blk of an unknown block " + x.String()) + } + return chainlog.Entry{Kind: chainlog.KindBlock, Block: b} +} + +func (e *env) rev(x chain.BlockRef) chainlog.Entry { + return chainlog.Entry{Kind: chainlog.KindRevert, To: x} +} + +func (e *env) fin(x chain.BlockRef) chainlog.Entry { + return chainlog.Entry{Kind: chainlog.KindFinalize, To: x} +} + +// show formats entries as "rev(A4) blk(B5)". +func (e *env) show(entries []chainlog.Entry) string { + parts := make([]string, len(entries)) + for i, en := range entries { + switch en.Kind { + case chainlog.KindBlock: + parts[i] = "blk(" + e.name(en.Block.Ref.Hash) + ")" + case chainlog.KindRevert: + parts[i] = "rev(" + e.name(en.To.Hash) + ")" + case chainlog.KindFinalize: + parts[i] = "fin(" + e.name(en.To.Hash) + ")" + default: + parts[i] = en.Kind.String() + } + } + return "[" + strings.Join(parts, " ") + "]" +} + +// want fails the test unless got deep-equals want. +func (e *env) want(t *testing.T, got []chainlog.Entry, want ...chainlog.Entry) { + t.Helper() + if len(want) == 0 { + want = nil + } + if !reflect.DeepEqual(got, want) { + t.Errorf("entries = %s, want %s\n got %+v\n want %+v", e.show(got), e.show(want), got, want) + } +} + +// view is the part of a chainlog.State the tracker must keep equal to the log's. +type view struct { + Tip, Finalized chain.BlockRef + Window []chain.BlockRef +} + +func viewOf(s chainlog.State) view { + return view{Tip: s.Tip(), Finalized: s.Finalized(), Window: s.Window()} +} + +// sameState fails the test unless the tracker's State equals the log's. +func (e *env) sameState(t *testing.T) { + t.Helper() + ls, err := e.lg.State(t.Context()) + if err != nil { + t.Fatalf("log State: %v", err) + } + if got, want := viewOf(e.tr.State()), viewOf(ls); !reflect.DeepEqual(got, want) { + t.Errorf("tracker state = %+v, log state %+v", got, want) + } +} + +// unchanged fails the test unless the tracker's view equals before. +func (e *env) unchanged(t *testing.T, before view) { + t.Helper() + if after := viewOf(e.tr.State()); !reflect.DeepEqual(after, before) { + t.Errorf("state changed: %+v, want %+v", after, before) + } +} + +// ref returns the canonical ref of number n. +func (e *env) ref(n uint64) chain.BlockRef { + return e.refs(n, n)[0] +} + +// refs returns the canonical refs of numbers from..to. +func (e *env) refs(from, to uint64) []chain.BlockRef { + var out []chain.BlockRef + for n := from; n <= to; n++ { + h, ok := e.c.Canonical(n) + if !ok { + panic(fmt.Sprintf("no canonical block %d", n)) + } + out = append(out, h.Ref) + } + return out +} + +// blks returns the Block entries of refs. +func (e *env) blks(refs ...chain.BlockRef) []chainlog.Entry { + out := make([]chainlog.Entry, len(refs)) + for i, r := range refs { + out[i] = e.blk(r) + } + return out +} + +// wantErr fails the test unless err wraps target and has the text want. +func wantErr(t *testing.T, err, target error, want string) { + t.Helper() + if err == nil { + t.Fatalf("err = nil, want %q", want) + } + if !errors.Is(err, target) { + t.Errorf("err %q does not wrap %q", err, target) + } + if err.Error() != want { + t.Errorf("err = %q\nwant %q", err.Error(), want) + } +} + +// wrapSource overrides some methods of the Source it wraps. +type wrapSource struct { + chain.Source + header func(ctx context.Context, h common.Hash) (chain.Header, error) + logs func(ctx context.Context, h common.Hash, f chain.Filter) ([]chain.Log, error) +} + +func (w *wrapSource) HeaderByHash(ctx context.Context, h common.Hash) (chain.Header, error) { + if w.header != nil { + return w.header(ctx, h) + } + return w.Source.HeaderByHash(ctx, h) +} + +func (w *wrapSource) Logs(ctx context.Context, h common.Hash, f chain.Filter) ([]chain.Log, error) { + if w.logs != nil { + return w.logs(ctx, h, f) + } + return w.Source.Logs(ctx, h, f) +} + +// mustPanic fails the test unless fn panics with the value want. +func mustPanic(t *testing.T, want string, fn func()) { + t.Helper() + defer func() { + t.Helper() + if got := fmt.Sprint(recover()); got != want { + t.Errorf("panic = %q, want %q", got, want) + } + }() + fn() +} diff --git a/tracker/property_test.go b/tracker/property_test.go new file mode 100644 index 0000000..d77215a --- /dev/null +++ b/tracker/property_test.go @@ -0,0 +1,164 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package tracker_test + +import ( + "errors" + "fmt" + "reflect" + "slices" + "testing" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/internal/fakechain" + "github.com/hmdsefi/evmsync/tracker" +) + +func TestStateSnapshots(t *testing.T) { + e := tracked(t, 4) + a := e.refs(1, 4) + s := e.tr.State() + + a5 := e.mine("A", 1)[0] + e.mustHead(t, a5) + e.mustFinalize(t, a[1]) + + if got, want := viewOf(s), (view{Tip: a[3], Finalized: e.g, Window: a}); !reflect.DeepEqual(got, want) { + t.Errorf("earlier snapshot = %+v, want %+v", got, want) + } + want := view{Tip: a5, Finalized: a[1], Window: []chain.BlockRef{a[2], a[3], a5}} + if got := viewOf(e.tr.State()); !reflect.DeepEqual(got, want) { + t.Errorf("State() = %+v, want %+v", got, want) + } + e.sameState(t) +} + +// propertyRun is the record of one seed's run: every Source call, and every call's entries or +// error text, in order. +type propertyRun struct { + calls []fakechain.Call + results []result +} + +// result is what one Head or Finalize call returned. +type result struct { + entries []chainlog.Entry + err string +} + +// runSeed drives a tracker over a faulty source for seed: 300 iterations of Step, Head of Latest +// and Finalize of Finalized. Every non-empty result is appended to a log, which must accept it, +// and after every call, whatever it returned, the tracker's State must equal the log's. +func runSeed(t *testing.T, seed uint64) propertyRun { + t.Helper() + ctx := t.Context() + c := fakechain.New(seed, fakechain.Params{}) + src := fakechain.NewSource(c, seed, fakechain.Faults{ + StaleHeadPPM: 100_000, + StaleOrphanPPM: 100_000, + DropOrphans: seed%2 == 0, + InterleavePPM: 100_000, + }) + lg := chainlog.NewMemory(c.Meta(flt), tracker.DefaultWindowCap) + tr := tracker.New(src, tracker.Config{Filter: flt}, chainlog.NewState(tracker.DefaultWindowCap)) + logView := viewOf(chainlog.NewState(tracker.DefaultWindowCap)) + var run propertyRun + + apply := func(step int, what string, got []chainlog.Entry, err error) { + t.Helper() + if err != nil { + if !errors.Is(err, chain.ErrNotFound) { + t.Fatalf("seed %d step %d: %s: %v", seed, step, what, err) + } + run.results = append(run.results, result{err: err.Error()}) + } else { + run.results = append(run.results, result{entries: got}) + } + if len(got) > 0 { + if _, aerr := lg.Append(ctx, got...); aerr != nil { + t.Fatalf("seed %d step %d: %s: Append: %v", seed, step, what, aerr) + } + ls, serr := lg.State(ctx) + if serr != nil { + t.Fatal(serr) + } + logView = viewOf(ls) + } + if tv := viewOf(tr.State()); !sameView(tv, logView) { + t.Fatalf("seed %d step %d: %s: tracker state %+v, log state %+v", seed, step, what, tv, logView) + } + } + + got, err := tr.Finalize(ctx, c.Genesis().Ref) + apply(0, "Finalize(genesis)", got, err) + for step := 1; step <= 300; step++ { + src.Do(func(c *fakechain.Chain) { c.Step() }) + h, err := src.Latest(ctx) + if err != nil { + t.Fatalf("seed %d step %d: Latest: %v", seed, step, err) + } + got, err := tr.Head(ctx, h) + apply(step, "Head", got, err) + f, err := src.Finalized(ctx) + if err != nil { + t.Fatalf("seed %d step %d: Finalized: %v", seed, step, err) + } + got, err = tr.Finalize(ctx, f.Ref) + apply(step, "Finalize", got, err) + } + run.calls = src.Calls() + return run +} + +// sameView compares two views without reflection, which dominated the property test's run time. +func sameView(a, b view) bool { + return a.Tip == b.Tip && a.Finalized == b.Finalized && slices.Equal(a.Window, b.Window) +} + +// firstDiff returns the first index at which two sequences of lengths na and nb differ, where +// same(i) compares element i of both, or -1 when they are equal. +func firstDiff(na, nb int, same func(i int) bool) int { + for i := range min(na, nb) { + if !same(i) { + return i + } + } + if na != nb { + return min(na, nb) + } + return -1 +} + +// at returns element i of s formatted with %+v, or "nothing" when s is shorter. +func at[T any](s []T, i int) string { + if i < len(s) { + return fmt.Sprintf("%+v", s[i]) + } + return "nothing" +} + +func TestMutualConsistency(t *testing.T) { + seeds := uint64(300) + if testing.Short() { + seeds = 50 + } + for seed := uint64(1); seed <= seeds; seed++ { + t.Run(fmt.Sprintf("seed=%d", seed), func(t *testing.T) { + t.Parallel() + first := runSeed(t, seed) + second := runSeed(t, seed) + if i := firstDiff(len(first.calls), len(second.calls), func(i int) bool { + return first.calls[i] == second.calls[i] + }); i >= 0 { + t.Fatalf("Source call %d differs: first run %s, repeated run %s", i, at(first.calls, i), at(second.calls, i)) + } + if i := firstDiff(len(first.results), len(second.results), func(i int) bool { + return reflect.DeepEqual(first.results[i], second.results[i]) + }); i >= 0 { + t.Fatalf("result %d differs: first run %s, repeated run %s", i, at(first.results, i), at(second.results, i)) + } + }) + } +} diff --git a/tracker/purity_test.go b/tracker/purity_test.go new file mode 100644 index 0000000..2ba0799 --- /dev/null +++ b/tracker/purity_test.go @@ -0,0 +1,153 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package tracker + +import ( + "go/ast" + "go/importer" + "go/parser" + "go/token" + "go/types" + "os" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// allowedImports are the only packages the tracker's non-test files may import. +var allowedImports = map[string]bool{ + "context": true, + "errors": true, + "fmt": true, + "github.com/ethereum/go-ethereum/common": true, + "github.com/hmdsefi/evmsync/chain": true, + "github.com/hmdsefi/evmsync/chainlog": true, +} + +// TestPurity checks that the package's non-test files import only allowedImports and contain no +// go statement, no select statement and no range over a map. +func TestPurity(t *testing.T) { + fset := token.NewFileSet() + files := parseNonTest(t, fset, ".") + if len(files) == 0 { + t.Fatal("no non-test files found") + } + info := &types.Info{Types: map[ast.Expr]types.TypeAndValue{}} + conf := types.Config{Importer: importer.ForCompiler(fset, "source", nil)} + if _, err := conf.Check("github.com/hmdsefi/evmsync/tracker", fset, files, info); err != nil { + t.Fatalf("type-check: %v", err) + } + for _, f := range files { + for _, msg := range purityFindings(fset, f, info) { + t.Error(msg) + } + } +} + +// TestPurityFindings checks that purityFindings reports each forbidden construct. +func TestPurityFindings(t *testing.T) { + const src = `package p + +import ( + "sync" + "time" +) + +var m = map[int]int{} + +type named map[string]bool + +var n named + +func f(c chan int) { + go func() {}() + select { + case <-c: + } + for range m { + } + for range []int{} { + } + for range n { + } + _ = sync.Mutex{} + _ = time.Second +} +` + fset := token.NewFileSet() + f, err := parser.ParseFile(fset, "p.go", src, 0) + if err != nil { + t.Fatal(err) + } + info := &types.Info{Types: map[ast.Expr]types.TypeAndValue{}} + conf := types.Config{Importer: importer.ForCompiler(fset, "source", nil)} + if _, err := conf.Check("p", fset, []*ast.File{f}, info); err != nil { + t.Fatalf("type-check: %v", err) + } + want := []string{ + `p.go:4:2: import "sync" is not allowed`, + `p.go:5:2: import "time" is not allowed`, + "p.go:15:2: go statement", + "p.go:16:2: select statement", + "p.go:19:2: range over a map", + "p.go:23:2: range over a map", + } + if got := purityFindings(fset, f, info); strings.Join(got, "\n") != strings.Join(want, "\n") { + t.Errorf("findings:\n%s\nwant:\n%s", strings.Join(got, "\n"), strings.Join(want, "\n")) + } +} + +// parseNonTest parses every non-test .go file in dir. +func parseNonTest(t *testing.T, fset *token.FileSet, dir string) []*ast.File { + t.Helper() + entries, err := os.ReadDir(dir) + if err != nil { + t.Fatal(err) + } + var files []*ast.File + for _, de := range entries { + name := de.Name() + if de.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + f, err := parser.ParseFile(fset, filepath.Join(dir, name), nil, 0) + if err != nil { + t.Fatal(err) + } + files = append(files, f) + } + return files +} + +// purityFindings returns one message per forbidden import, go statement, select statement and +// range over a map in f, in source order. +func purityFindings(fset *token.FileSet, f *ast.File, info *types.Info) []string { + var out []string + report := func(n ast.Node, msg string) { + out = append(out, fset.Position(n.Pos()).String()+": "+msg) + } + for _, imp := range f.Imports { + path, err := strconv.Unquote(imp.Path.Value) + if err != nil || !allowedImports[path] { + report(imp, "import "+imp.Path.Value+" is not allowed") + } + } + ast.Inspect(f, func(n ast.Node) bool { + switch n := n.(type) { + case *ast.GoStmt: + report(n, "go statement") + case *ast.SelectStmt: + report(n, "select statement") + case *ast.RangeStmt: + if tv, ok := info.Types[n.X]; ok { + if _, isMap := tv.Type.Underlying().(*types.Map); isMap { + report(n, "range over a map") + } + } + } + return true + }) + return out +} diff --git a/tracker/tracker.go b/tracker/tracker.go new file mode 100644 index 0000000..67b9e18 --- /dev/null +++ b/tracker/tracker.go @@ -0,0 +1,173 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package tracker + +import ( + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" +) + +// DefaultWindowCap is the window cap used when Config.WindowCap is 0. +const DefaultWindowCap = 100_000 + +// Config configures a Tracker. +type Config struct { + // Filter is passed unchanged to Source.Logs for every new block. + Filter chain.Filter + // WindowCap is the most unfinalized blocks the window may hold. 0 means + // DefaultWindowCap. A negative value makes New panic. + WindowCap int +} + +// Tracker turns new heads and finalized blocks into canonical-log entries. +// +// A Tracker is not safe for concurrent use. It starts no goroutines, never sleeps +// and never reads the clock: given the same Source answers it makes the same Source +// calls and returns the same entries. +type Tracker struct { + src chain.Source + filter chain.Filter + cap int + + // fin is the finalized ref F; its zero hash means the tracker is unanchored. + fin chain.BlockRef + // win[start:] is the window W: the refs numbered fin.Number+1, fin.Number+2, … in order. + // Finalize advances start instead of copying. + win []chain.BlockRef + start int + // byHash maps the hash of each window ref to its number. It is only looked up, never + // ranged over. + byHash map[common.Hash]uint64 +} + +// New returns a Tracker whose view of the chain is st: the finalized ref, the tip +// and the window of a log's validator state (chainlog.Log.State). For an empty log, +// pass chainlog.NewState(cap); the Tracker is then unanchored until Finalize. +// +// New panics if src is nil, if cfg.WindowCap is negative, if st has blocks but no +// finalized ref, or if st's window is not a parent-linked chain above its finalized +// ref. +func New(src chain.Source, cfg Config, st chainlog.State) *Tracker { + if src == nil { + panic("evmsync: tracker.New: nil Source") + } + if cfg.WindowCap < 0 { + panic("evmsync: tracker.New: negative WindowCap") + } + fin, tip, win := st.Finalized(), st.Tip(), st.Window() + if fin.Hash == (common.Hash{}) && tip != (chain.BlockRef{}) { + panic("evmsync: tracker.New: state has blocks but no finalized block") + } + if !linked(fin, tip, win) { + panic("evmsync: tracker.New: window is not a parent-linked chain above the finalized block") + } + t := &Tracker{ + src: src, + filter: cfg.Filter, + cap: cfg.WindowCap, + fin: fin, + win: win, + byHash: make(map[common.Hash]uint64, len(win)), + } + if t.cap == 0 { + t.cap = DefaultWindowCap + } + for _, r := range win { + t.byHash[r.Hash] = r.Number + } + return t +} + +// linked reports whether win is the contiguous, parent-linked chain above fin that ends at tip, +// or empty with tip equal to fin. +func linked(fin, tip chain.BlockRef, win []chain.BlockRef) bool { + prev := fin + for _, r := range win { + if r.Parent != prev.Hash || r.Number != prev.Number+1 { + return false + } + prev = r + } + return prev == tip +} + +// State returns a validator state whose Tip, Finalized and Window equal the Tracker's +// view. Later calls on the Tracker never change a State already returned. +func (t *Tracker) State() chainlog.State { + s := chainlog.NewState(t.cap) + if !t.anchored() { + return s + } + s.Apply(chainlog.Entry{Kind: chainlog.KindFinalize, To: t.fin}) + for _, r := range t.window() { + s.Apply(chainlog.Entry{Kind: chainlog.KindBlock, Block: chain.Block{Ref: r}}) + } + return s +} + +func (t *Tracker) anchored() bool { return t.fin.Hash != (common.Hash{}) } + +// window returns the live window. The caller must not keep it across a change. +func (t *Tracker) window() []chain.BlockRef { return t.win[t.start:] } + +// tip returns T: the last window ref, or F when the window is empty. +func (t *Tracker) tip() chain.BlockRef { + if w := t.window(); len(w) > 0 { + return w[len(w)-1] + } + return t.fin +} + +// known returns the ref of K (F or a window ref) with hash h. +func (t *Tracker) known(h common.Hash) (chain.BlockRef, bool) { + if h == t.fin.Hash && t.anchored() { + return t.fin, true + } + n, ok := t.byHash[h] + if !ok { + return chain.BlockRef{}, false + } + return t.window()[n-t.fin.Number-1], true +} + +// cut drops the window refs above number n. +func (t *Tracker) cut(n uint64) { + dropped := t.window()[n-t.fin.Number:] + for _, r := range dropped { + delete(t.byHash, r.Hash) + } + t.win = t.win[:len(t.win)-len(dropped)] +} + +// push appends r to the window. +func (t *Tracker) push(r chain.BlockRef) { + t.win = append(t.win, r) + t.byHash[r.Hash] = r.Number +} + +// finalizeTo makes f, a window ref, the finalized ref and drops the window refs at or below it. +func (t *Tracker) finalizeTo(f chain.BlockRef) { + dead := t.window()[:f.Number-t.fin.Number] + for _, r := range dead { + delete(t.byHash, r.Hash) + } + t.start += len(dead) + t.fin = f + // Compact once the dead prefix is longer than the window, so pruning stays amortized O(1). + if t.start > len(t.win)-t.start { + n := copy(t.win, t.win[t.start:]) + t.win = t.win[:n] + t.start = 0 + } +} + +// reset makes f the finalized ref and the tip, with an empty window. +func (t *Tracker) reset(f chain.BlockRef) { + clear(t.byHash) + t.win = t.win[:0] + t.start = 0 + t.fin = f +} diff --git a/tracker/tracker_test.go b/tracker/tracker_test.go new file mode 100644 index 0000000..76df097 --- /dev/null +++ b/tracker/tracker_test.go @@ -0,0 +1,775 @@ +// Copyright 2026 Hamed Yousefi +// SPDX-License-Identifier: Apache-2.0 + +package tracker_test + +import ( + "context" + "errors" + "reflect" + "strings" + "testing" + + "github.com/ethereum/go-ethereum/common" + + "github.com/hmdsefi/evmsync/chain" + "github.com/hmdsefi/evmsync/chainlog" + "github.com/hmdsefi/evmsync/internal/fakechain" + "github.com/hmdsefi/evmsync/tracker" +) + +func TestAnchoring(t *testing.T) { + e := newBareEnv(t, tracker.Config{}, fakechain.Faults{}, nil) + a := e.mine("A", 10) + if got, want := viewOf(e.tr.State()), viewOf(chainlog.NewState(tracker.DefaultWindowCap)); !reflect.DeepEqual(got, want) { + t.Errorf("unanchored state = %+v, want %+v", got, want) + } + + got, err := e.head(t, a[2]) + wantErr(t, err, tracker.ErrUnanchored, "tracker: no finalized block: head 3 "+a[2].Hash.Hex()) + e.want(t, got) + e.wantCalls(t, "") + got, err = e.head(t, chain.BlockRef{Number: 3}) + wantErr(t, err, tracker.ErrInconsistentSource, "tracker: inconsistent source: zero block hash") + e.want(t, got) + got, err = e.finalize(t, chain.BlockRef{Number: 7}) + wantErr(t, err, tracker.ErrInconsistentSource, "tracker: inconsistent source: zero block hash") + e.want(t, got) + + got = e.mustFinalize(t, a[6]) + e.want(t, got, e.fin(a[6])) + e.wantCalls(t, "") + st := e.tr.State() + if st.Tip() != a[6] || st.Finalized() != a[6] || st.Window() != nil { + t.Errorf("state = %+v, want tip and finalized A7, empty window", viewOf(st)) + } + + got = e.mustHead(t, a[9]) + e.want(t, got, e.blks(a[7], a[8], a[9])...) + e.wantCalls(t, "H(A9) H(A8) L(A8) L(A9) L(A10)") +} + +func TestNewPanics(t *testing.T) { + e := newBareEnv(t, tracker.Config{}, fakechain.Faults{}, nil) + a := e.mine("A", 4) + stateOf := func(entries ...chainlog.Entry) chainlog.State { + s := chainlog.NewState(100) + for _, en := range entries { + s.Apply(en) + } + return s + } + final := func(x chain.BlockRef) chainlog.Entry { + en := e.blk(x) + en.Final = true + return en + } + tests := []struct { + name string + src chain.Source + cfg tracker.Config + st chainlog.State + want string + }{ + {"nil source", nil, tracker.Config{}, stateOf(), "evmsync: tracker.New: nil Source"}, + {"negative cap", e.src, tracker.Config{WindowCap: -1}, stateOf(), "evmsync: tracker.New: negative WindowCap"}, + { + "blocks without finalized", + e.src, tracker.Config{}, stateOf(e.blk(a[1])), + "evmsync: tracker.New: state has blocks but no finalized block", + }, + { + "tip not the last window block", + e.src, tracker.Config{}, stateOf(e.fin(e.g), e.blk(a[0]), e.blk(a[1]), + e.rev(chain.BlockRef{Number: 1, Hash: common.HexToHash("0x01")})), + "evmsync: tracker.New: window is not a parent-linked chain above the finalized block", + }, + { + "tip without finalized and an empty window", + e.src, tracker.Config{}, stateOf(e.blk(a[0]), e.rev(e.g)), + "evmsync: tracker.New: state has blocks but no finalized block", + }, + { + "window block at the wrong height", + e.src, tracker.Config{}, stateOf(e.fin(e.g), chainlog.Entry{ + Kind: chainlog.KindBlock, + Block: chain.Block{Ref: chain.BlockRef{Number: 2, Hash: common.HexToHash("0x02"), Parent: e.g.Hash}}, + }), + "evmsync: tracker.New: window is not a parent-linked chain above the finalized block", + }, + { + "window with a gap", + e.src, tracker.Config{}, stateOf(e.fin(e.g), e.blk(a[0]), e.blk(a[2])), + "evmsync: tracker.New: window is not a parent-linked chain above the finalized block", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mustPanic(t, tt.want, func() { tracker.New(tt.src, tt.cfg, tt.st) }) + }) + } + + t.Run("backfill prefix", func(t *testing.T) { + prefix := []chainlog.Entry{e.fin(e.g), final(a[0]), final(a[2])} + if _, err := e.lg.Append(t.Context(), prefix...); err != nil { + t.Fatal(err) + } + e.tr = tracker.New(e.src, tracker.Config{Filter: flt}, stateOf(prefix...)) + e.sameState(t) + e.want(t, e.mustHead(t, a[3]), e.blk(a[3])) + e.wantCalls(t, "L(A4)") + }) +} + +func TestResume(t *testing.T) { + e := newBareEnv(t, tracker.Config{}, fakechain.Faults{}, nil) + a := e.mine("A", 4) + if _, err := e.lg.Append(t.Context(), e.fin(e.g), e.blk(a[0]), e.blk(a[1]), e.blk(a[2])); err != nil { + t.Fatal(err) + } + st, err := e.lg.State(t.Context()) + if err != nil { + t.Fatal(err) + } + e.tr = tracker.New(e.src, e.cfg, st) + e.sameState(t) + + got := e.mustHead(t, a[3]) + e.want(t, got, e.blk(a[3])) + e.wantCalls(t, "L(A4)") +} + +func TestExtendByOne(t *testing.T) { + t.Run("honest", func(t *testing.T) { + e := newEnv(t, tracker.Config{}, fakechain.Faults{}, nil) + a := e.mine("A", 1) + got := e.mustHead(t, a[0]) + e.want(t, got, e.blk(a[0])) + e.wantCalls(t, "L(A1)") + }) + t.Run("logs kept as returned", func(t *testing.T) { + var ls []chain.Log + e := newEnv(t, tracker.Config{}, fakechain.Faults{}, func(s chain.Source) chain.Source { + return &wrapSource{Source: s, logs: func(context.Context, common.Hash, chain.Filter) ([]chain.Log, error) { + return ls, nil + }} + }) + a := e.mine("A", 1) + ls = []chain.Log{{ + Address: fakechain.Address(0), + Topics: []common.Hash{fakechain.TransferTopic}, + BlockNumber: a[0].Number, + BlockHash: a[0].Hash, + }} + got := e.mustHead(t, a[0]) + if len(got) != 1 || len(got[0].Block.Logs) != 1 || &got[0].Block.Logs[0] != &ls[0] { + t.Fatalf("entries = %+v, want one block whose Logs is the returned slice", got) + } + }) + t.Run("filter passed unchanged", func(t *testing.T) { + cfgFilter := chain.Filter{Addresses: []common.Address{fakechain.Address(2)}} + var seen []chain.Filter + e := newEnv(t, tracker.Config{Filter: cfgFilter}, fakechain.Faults{}, func(s chain.Source) chain.Source { + return &wrapSource{Source: s, logs: func(ctx context.Context, h common.Hash, f chain.Filter) ([]chain.Log, error) { + seen = append(seen, f) + return s.Logs(ctx, h, f) + }} + }) + a := e.mine("A", 2) + e.mustHead(t, a[1]) + if want := []chain.Filter{cfgFilter, cfgFilter}; !reflect.DeepEqual(seen, want) { + t.Errorf("Logs filters = %v, want %v", seen, want) + } + }) + t.Run("nil logs stay nil", func(t *testing.T) { + e := newEnv(t, tracker.Config{}, fakechain.Faults{}, func(s chain.Source) chain.Source { + return &wrapSource{Source: s, logs: func(context.Context, common.Hash, chain.Filter) ([]chain.Log, error) { + return nil, nil + }} + }) + a := e.mine("A", 1) + got := e.mustHead(t, a[0]) + if len(got) != 1 || got[0].Block.Logs != nil { + t.Fatalf("entries = %+v, want one block with nil Logs", got) + } + }) +} + +func TestDuplicates(t *testing.T) { + e := tracked(t, 6) + before := viewOf(e.tr.State()) + for _, h := range []chain.BlockRef{e.ref(6), e.ref(3), e.g} { + got := e.mustHead(t, h) + e.want(t, got) + e.wantCalls(t, "") + } + e.unchanged(t, before) +} + +func TestLaggingProvider(t *testing.T) { + e := tracked(t, 8) + e.mustFinalize(t, e.ref(5)) + for _, h := range e.refs(4, 5) { + e.want(t, e.mustHead(t, h)) + e.wantCalls(t, "") + } + b := e.reorg("B", 6, 6) + for _, h := range b[1:3] { // B4, B5 + e.want(t, e.mustHead(t, h)) + e.wantCalls(t, "") + } +} + +func TestGapFill(t *testing.T) { + e := tracked(t, 2) + e.mine("A", 4) + a := e.refs(3, 6) + got := e.mustHead(t, a[3]) + e.want(t, got, e.blks(a...)...) + e.wantCalls(t, "H(A5) H(A4) H(A3) L(A3) L(A4) L(A5) L(A6)") +} + +func TestDepthOneReorg(t *testing.T) { + e := tracked(t, 5) + b := e.reorg("B", 1, 1) + got := e.mustHead(t, b[0]) + e.want(t, got, e.rev(e.ref(4)), e.blk(b[0])) + e.wantCalls(t, "L(B5)") +} + +func TestDepthNReorg(t *testing.T) { + e := tracked(t, 10) + a6 := e.ref(6) + b := e.reorg("B", 4, 4) + got := e.mustHead(t, b[3]) + e.want(t, got, append([]chainlog.Entry{e.rev(a6)}, e.blks(b...)...)...) + e.wantCalls(t, "H(B9) H(B8) H(B7) L(B7) L(B8) L(B9) L(B10)") +} + +func TestReorgDuringGapFill(t *testing.T) { + e := tracked(t, 5) + a3 := e.ref(3) + b := e.reorg("B", 2, 5) + got := e.mustHead(t, b[4]) + e.want(t, got, append([]chainlog.Entry{e.rev(a3)}, e.blks(b...)...)...) + e.wantCalls(t, "H(B7) H(B6) H(B5) H(B4) L(B4) L(B5) L(B6) L(B7) L(B8)") +} + +func TestShorterBranch(t *testing.T) { + e := tracked(t, 6) + a4 := e.ref(4) + b5 := e.reorg("B", 2, 1)[0] + e.want(t, e.mustHead(t, b5)) + e.wantCalls(t, "") + b6 := e.mine("B", 1)[0] + got := e.mustHead(t, b6) + e.want(t, got, e.rev(a4), e.blk(b5), e.blk(b6)) + e.wantCalls(t, "H(B5) L(B5) L(B6)") +} + +func TestSameHeightSibling(t *testing.T) { + e := tracked(t, 6) + a := e.refs(5, 6) + b6 := e.reorg("B", 1, 1)[0] + e.want(t, e.mustHead(t, b6), e.rev(a[0]), e.blk(b6)) + got := e.mustHead(t, a[1]) + e.want(t, got, e.rev(a[0]), e.blk(a[1])) + e.wantCalls(t, "L(A6)") +} + +func TestRollback(t *testing.T) { + e := tracked(t, 6) + a3 := e.ref(3) + e.reorg("A", 3, 0) + e.want(t, e.mustHead(t, a3)) + e.wantCalls(t, "") + c := e.mine("C", 2) + for _, h := range c { + e.want(t, e.mustHead(t, h)) + e.wantCalls(t, "") + } + c6 := e.mine("C", 1)[0] + got := e.mustHead(t, c6) + e.want(t, got, e.rev(a3), e.blk(c[0]), e.blk(c[1]), e.blk(c6)) + e.wantCalls(t, "H(C5) H(C4) L(C4) L(C5) L(C6)") +} + +func TestWalkReachesFinalized(t *testing.T) { + e := tracked(t, 8) + a5 := e.ref(5) + e.mustFinalize(t, a5) + before := viewOf(e.tr.State()) + b := e.reorg("B", 5, 6) // B4…B9 + got, err := e.head(t, b[5]) + wantErr(t, err, tracker.ErrReorgBelowFinalized, "tracker: reorg below the finalized block: block 6 "+ + b[2].Hash.Hex()+" does not descend from finalized block 5 "+a5.Hash.Hex()) + e.want(t, got) + e.wantCalls(t, "H(B8) H(B7) H(B6)") + e.unchanged(t, before) +} + +func TestDroppedFork(t *testing.T) { + e := newEnv(t, tracker.Config{}, fakechain.Faults{DropOrphans: true}, nil) + a := e.mine("A", 5) + e.mustHead(t, a[4]) + b := e.reorg("B", 2, 2) + hB := b[1] + c := e.reorg("C", 3, 4) // C3…C6 + before := viewOf(e.tr.State()) + + got, err := e.head(t, hB) + wantErr(t, err, chain.ErrNotFound, "tracker: header "+b[0].Hash.Hex()+" (block 4): fakechain: block "+ + b[0].Hash.Hex()+": "+chain.ErrNotFound.Error()) + e.want(t, got) + e.wantCalls(t, "H(B4)") + e.unchanged(t, before) + + got = e.mustHead(t, c[3]) + e.want(t, got, append([]chainlog.Entry{e.rev(a[1])}, e.blks(c...)...)...) +} + +// failingSource passes every call to the Source it wraps, and replaces the result of the +// failAt-th HeaderByHash or Logs call with errors.New("boom"). failAt 0 fails nothing. +type failingSource struct { + chain.Source + failAt, n int +} + +var errBoom = errors.New("boom") + +func (f *failingSource) fail() bool { + f.n++ + return f.n == f.failAt +} + +func (f *failingSource) HeaderByHash(ctx context.Context, h common.Hash) (chain.Header, error) { + hd, err := f.Source.HeaderByHash(ctx, h) + if f.fail() { + return chain.Header{}, errBoom + } + return hd, err +} + +func (f *failingSource) Logs(ctx context.Context, h common.Hash, flt chain.Filter) ([]chain.Log, error) { + ls, err := f.Source.Logs(ctx, h, flt) + if f.fail() { + return nil, errBoom + } + return ls, err +} + +func TestErrorsKeepState(t *testing.T) { + healthyCalls := []string{"H(B7)", "H(B6)", "H(B5)", "H(B4)", "L(B4)", "L(B5)", "L(B6)", "L(B7)", "L(B8)"} + setup := func(t *testing.T) (*env, *failingSource, []chain.BlockRef) { + t.Helper() + var fs *failingSource + e := newEnv(t, tracker.Config{}, fakechain.Faults{}, func(s chain.Source) chain.Source { + fs = &failingSource{Source: s} + return fs + }) + a := e.mine("A", 5) + e.mustHead(t, a[4]) + return e, fs, e.reorg("B", 2, 5) + } + for k := 1; k <= len(healthyCalls); k++ { + t.Run(healthyCalls[k-1], func(t *testing.T) { + e, fs, b := setup(t) + before := viewOf(e.tr.State()) + fs.failAt, fs.n = k, 0 + got, err := e.head(t, b[4]) + if err == nil || !errors.Is(err, errBoom) || !strings.HasSuffix(err.Error(), ": boom") { + t.Fatalf("err = %v, want one ending in \": boom\"", err) + } + switch k { + case 1: + wantErr(t, err, errBoom, "tracker: header "+b[3].Hash.Hex()+" (block 7): boom") + case 5: + wantErr(t, err, errBoom, "tracker: logs of block 4 "+b[0].Hash.Hex()+": boom") + } + e.want(t, got) + if calls := strings.Join(e.calls(), " "); calls != strings.Join(healthyCalls[:k], " ") { + t.Errorf("calls = %q, want %q", calls, strings.Join(healthyCalls[:k], " ")) + } + e.unchanged(t, before) + fs.failAt = 0 + got = e.mustHead(t, b[4]) + e.want(t, got, append([]chainlog.Entry{e.rev(e.ref(3))}, e.blks(b...)...)...) + }) + } + t.Run("context canceled during the walk", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + var b4 common.Hash + e := newEnv(t, tracker.Config{}, fakechain.Faults{}, func(s chain.Source) chain.Source { + return &wrapSource{Source: s, header: func(ctx context.Context, h common.Hash) (chain.Header, error) { + hd, err := s.HeaderByHash(ctx, h) + if h == b4 { + cancel() + } + return hd, err + }} + }) + a := e.mine("A", 5) + e.mustHead(t, a[4]) + b := e.reorg("B", 2, 5) + b4 = b[0].Hash + before := viewOf(e.tr.State()) + e.src.ResetCalls() + got, err := e.tr.Head(ctx, chain.Header{Ref: b[4]}) + wantErr(t, err, context.Canceled, "tracker: context canceled") + e.want(t, got) + e.wantCalls(t, "H(B7) H(B6) H(B5) H(B4)") + e.unchanged(t, before) + }) + t.Run("canceled context", func(t *testing.T) { + e, _, b := setup(t) + before := viewOf(e.tr.State()) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + e.src.ResetCalls() + got, err := e.tr.Head(ctx, chain.Header{Ref: b[4]}) + wantErr(t, err, context.Canceled, "tracker: context canceled") + e.want(t, got) + e.wantCalls(t, "") + e.unchanged(t, before) + }) +} + +func TestInconsistentSource(t *testing.T) { + // answer replaces the Source's HeaderByHash answer for h; a zero header and nil error pass the + // call through to the fake chain. + type answer func(e *env, h common.Hash) (chain.Header, error) + setup := func(t *testing.T, ans answer) *env { + t.Helper() + var e *env + e = newEnv(t, tracker.Config{}, fakechain.Faults{}, func(s chain.Source) chain.Source { + return &wrapSource{Source: s, header: func(ctx context.Context, h common.Hash) (chain.Header, error) { + if ans != nil { + if hd, err := ans(e, h); err != nil || hd != (chain.Header{}) { + return hd, err + } + } + return s.HeaderByHash(ctx, h) + }} + }) + a := e.mine("A", 5) + e.mustHead(t, a[4]) + e.mine("A", 3) + return e + } + headerOf := func(e *env, n uint64) chain.Header { + h, _ := e.c.Canonical(n) + return h + } + tests := []struct { + name string + ans answer + head func(e *env) chain.Header + want func(e *env) string + }{ + { + name: "wrong hash", + ans: func(e *env, h common.Hash) (chain.Header, error) { + if h == headerOf(e, 7).Ref.Hash { + return headerOf(e, 6), nil + } + return chain.Header{}, nil + }, + head: func(e *env) chain.Header { return headerOf(e, 8) }, + want: func(e *env) string { + return "tracker: inconsistent source: header " + headerOf(e, 7).Ref.Hash.Hex() + + ": got hash " + headerOf(e, 6).Ref.Hash.Hex() + }, + }, + { + name: "wrong number", + ans: func(e *env, h common.Hash) (chain.Header, error) { + if hd := headerOf(e, 7); h == hd.Ref.Hash { + hd.Ref.Number = 9 + return hd, nil + } + return chain.Header{}, nil + }, + head: func(e *env) chain.Header { return headerOf(e, 8) }, + want: func(e *env) string { + return "tracker: inconsistent source: header " + headerOf(e, 7).Ref.Hash.Hex() + ": got number 9, want 7" + }, + }, + { + name: "known parent at the wrong height", + head: func(e *env) chain.Header { + return chain.Header{Ref: chain.BlockRef{Number: 8, Parent: headerOf(e, 3).Ref.Hash, Hash: common.HexToHash("0xf8")}} + }, + want: func(e *env) string { + return "tracker: inconsistent source: block 8 " + common.HexToHash("0xf8").Hex() + " has parent " + + headerOf(e, 3).Ref.Hash.Hex() + " at height 3" + }, + }, + { + name: "zero hash", + head: func(*env) chain.Header { return chain.Header{} }, + want: func(*env) string { return "tracker: inconsistent source: zero block hash" }, + }, + { + name: "zero parent hash", + head: func(*env) chain.Header { + return chain.Header{Ref: chain.BlockRef{Number: 8, Hash: common.HexToHash("0xf8")}} + }, + want: func(*env) string { + return "tracker: inconsistent source: block 8 " + common.HexToHash("0xf8").Hex() + " has a zero parent hash" + }, + }, + { + name: "own hash as parent", + head: func(*env) chain.Header { + x := common.HexToHash("0xf8") + return chain.Header{Ref: chain.BlockRef{Number: 8, Hash: x, Parent: x}} + }, + want: func(*env) string { + x := common.HexToHash("0xf8").Hex() + return "tracker: inconsistent source: block 8 " + x + " has parent " + x + ", which is block 8 of the same branch" + }, + }, + { + name: "cycle of three", + ans: func(e *env, h common.Hash) (chain.Header, error) { + switch h { + case common.HexToHash("0xf7"): + return chain.Header{Ref: chain.BlockRef{Number: 7, Hash: h, Parent: common.HexToHash("0xf6")}}, nil + case common.HexToHash("0xf6"): + return chain.Header{Ref: chain.BlockRef{Number: 6, Hash: h, Parent: common.HexToHash("0xf7")}}, nil + } + return chain.Header{}, nil + }, + head: func(*env) chain.Header { + return chain.Header{Ref: chain.BlockRef{Number: 8, Hash: common.HexToHash("0xf8"), Parent: common.HexToHash("0xf7")}} + }, + want: func(*env) string { + return "tracker: inconsistent source: block 6 " + common.HexToHash("0xf6").Hex() + " has parent " + + common.HexToHash("0xf7").Hex() + ", which is block 7 of the same branch" + }, + }, + { + name: "cycle of two", + ans: func(e *env, h common.Hash) (chain.Header, error) { + if h == common.HexToHash("0xf7") { + return chain.Header{Ref: chain.BlockRef{Number: 7, Hash: h, Parent: common.HexToHash("0xf8")}}, nil + } + return chain.Header{}, nil + }, + head: func(*env) chain.Header { + return chain.Header{Ref: chain.BlockRef{Number: 8, Hash: common.HexToHash("0xf8"), Parent: common.HexToHash("0xf7")}} + }, + want: func(*env) string { + return "tracker: inconsistent source: block 7 " + common.HexToHash("0xf7").Hex() + " has parent " + + common.HexToHash("0xf8").Hex() + ", which is block 8 of the same branch" + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + e := setup(t, tt.ans) + before := viewOf(e.tr.State()) + got, err := e.headHeader(t, tt.head(e)) + wantErr(t, err, tracker.ErrInconsistentSource, tt.want(e)) + e.want(t, got) + e.unchanged(t, before) + }) + } +} + +func TestFinalizeInWindow(t *testing.T) { + e := tracked(t, 6) + a := e.refs(1, 6) + // The caller's ref lacks Parent and Time; the tracker finalizes the window's own ref. + got := e.mustFinalize(t, chain.BlockRef{Number: 3, Hash: a[2].Hash}) + e.want(t, got, e.fin(a[2])) + e.wantCalls(t, "") + if w := e.tr.State().Window(); !reflect.DeepEqual(w, a[3:]) { + t.Errorf("window = %v, want A4…A6", w) + } + e.want(t, e.mustFinalize(t, a[1])) + e.want(t, e.mustFinalize(t, a[2])) + + before := viewOf(e.tr.State()) + other := chain.BlockRef{Number: 3, Hash: common.HexToHash("0x03")} + got, err := e.finalize(t, other) + wantErr(t, err, tracker.ErrReorgBelowFinalized, "tracker: reorg below the finalized block: finalized block 3 is "+ + a[2].Hash.Hex()+", got "+other.Hash.Hex()) + e.want(t, got) + + wrong := a[4] + wrong.Number = 4 + got, err = e.finalize(t, wrong) + wantErr(t, err, tracker.ErrInconsistentSource, "tracker: inconsistent source: block "+a[4].Hash.Hex()+ + " is at height 5, got 4") + e.want(t, got) + + e.unchanged(t, before) + + higherF := a[2] + higherF.Number = 5 + got, err = e.finalize(t, higherF) + wantErr(t, err, tracker.ErrInconsistentSource, "tracker: inconsistent source: block "+a[2].Hash.Hex()+ + " is at height 3, got 5") + e.want(t, got) + e.wantCalls(t, "") + + got, err = e.finalize(t, chain.BlockRef{Number: 7}) + wantErr(t, err, tracker.ErrInconsistentSource, "tracker: inconsistent source: zero block hash") + e.want(t, got) + e.unchanged(t, before) +} + +func TestFinalizeErrorsKeepState(t *testing.T) { + healthyCalls := []string{"H(A5)", "H(A4)", "L(A4)", "L(A5)", "L(A6)"} + for k := 1; k <= len(healthyCalls); k++ { + t.Run(healthyCalls[k-1], func(t *testing.T) { + var fs *failingSource + e := newEnv(t, tracker.Config{}, fakechain.Faults{}, func(s chain.Source) chain.Source { + fs = &failingSource{Source: s} + return fs + }) + e.mustHead(t, e.mine("A", 3)[2]) + a := e.mine("A", 5) // A4…A8 + before := viewOf(e.tr.State()) + fs.failAt, fs.n = k, 0 + got, err := e.finalize(t, a[2]) + if err == nil || !errors.Is(err, errBoom) { + t.Fatalf("err = %v, want boom", err) + } + e.want(t, got) + if calls := strings.Join(e.calls(), " "); calls != strings.Join(healthyCalls[:k], " ") { + t.Errorf("calls = %q, want %q", calls, strings.Join(healthyCalls[:k], " ")) + } + e.unchanged(t, before) + fs.failAt = 0 + e.want(t, e.mustFinalize(t, a[2]), e.blk(a[0]), e.blk(a[1]), e.blk(a[2]), e.fin(a[2])) + }) + } +} + +// TestWindowStartIndex finalizes inside the window without compacting it, then reverts, extends +// and walks over it. +func TestWindowStartIndex(t *testing.T) { + e := tracked(t, 6) + a := e.refs(1, 6) + e.want(t, e.mustFinalize(t, a[1]), e.fin(a[1])) + e.want(t, e.mustFinalize(t, a[2]), e.fin(a[2])) + b6 := e.reorg("B", 1, 1)[0] + e.want(t, e.mustHead(t, b6), e.rev(a[4]), e.blk(b6)) + b7 := e.mine("B", 1)[0] + e.want(t, e.mustHead(t, b7), e.blk(b7)) + b := e.mine("B", 2) // B8, B9 + e.want(t, e.mustFinalize(t, b[1]), e.blk(b[0]), e.blk(b[1]), e.fin(b[1])) + b10 := e.mine("B", 1)[0] + e.want(t, e.mustHead(t, b10), e.blk(b10)) +} + +// TestFinalizeInWindowRepeatedly finalizes three window blocks in a row, then checks that every +// block at or below F is ignored as a lagging head. +func TestFinalizeInWindowRepeatedly(t *testing.T) { + e := tracked(t, 6) + a := e.refs(1, 6) + for _, f := range a[1:4] { + e.want(t, e.mustFinalize(t, f), e.fin(f)) + } + for _, h := range a[:4] { + e.want(t, e.mustHead(t, h)) + e.wantCalls(t, "") + } +} + +// TestWalkCheckOrder pins the order of the walk's checks for one block: the finalized height, then +// the parent hash, then the context. +func TestWalkCheckOrder(t *testing.T) { + x := common.HexToHash("0xf4") + t.Run("finalized height first", func(t *testing.T) { + e := newEnv(t, tracker.Config{}, fakechain.Faults{}, nil) + a3 := e.mine("A", 3)[2] + e.mustFinalize(t, a3) + for _, parent := range []common.Hash{{}, x} { + got, err := e.head(t, chain.BlockRef{Number: 4, Hash: x, Parent: parent}) + wantErr(t, err, tracker.ErrReorgBelowFinalized, "tracker: reorg below the finalized block: block 4 "+ + x.Hex()+" does not descend from finalized block 3 "+a3.Hash.Hex()) + e.want(t, got) + e.wantCalls(t, "") + } + }) + t.Run("parent hash before the context", func(t *testing.T) { + e := tracked(t, 6) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + tests := []struct { + parent common.Hash + want string + }{ + {common.Hash{}, "tracker: inconsistent source: block 8 " + x.Hex() + " has a zero parent hash"}, + {x, "tracker: inconsistent source: block 8 " + x.Hex() + " has parent " + x.Hex() + + ", which is block 8 of the same branch"}, + } + for _, tt := range tests { + got, err := e.tr.Head(ctx, chain.Header{Ref: chain.BlockRef{Number: 8, Hash: x, Parent: tt.parent}}) + wantErr(t, err, tracker.ErrInconsistentSource, tt.want) + e.want(t, got) + } + }) +} + +func TestFinalizeAboveTip(t *testing.T) { + e := tracked(t, 3) + oldWindow := e.refs(1, 3) + e.mine("A", 5) + a := e.refs(4, 8) + got := e.mustFinalize(t, a[2]) + e.want(t, got, e.blk(a[0]), e.blk(a[1]), e.blk(a[2]), e.fin(a[2])) + e.wantCalls(t, "H(A5) H(A4) L(A4) L(A5) L(A6)") + if st := e.tr.State(); st.Tip() != a[2] || st.Finalized() != a[2] || st.Window() != nil { + t.Errorf("state = %+v, want tip and finalized A6, empty window", viewOf(st)) + } + + // The window is empty and the tip is F, so only the lagging-provider rule ignores these heads: + // the blocks of the window before the Finalize, and a sibling of F. + sibling := chain.BlockRef{Number: 6, Hash: common.HexToHash("0x66"), Parent: a[1].Hash} + for _, h := range append(oldWindow, sibling) { + e.want(t, e.mustHead(t, h)) + e.wantCalls(t, "") + } + + e.want(t, e.mustHead(t, a[4]), e.blk(a[3]), e.blk(a[4])) +} + +func TestFinalizeDisagrees(t *testing.T) { + e := tracked(t, 6) + a3 := e.ref(3) + oldWindow := e.refs(1, 5) + b := e.reorg("B", 3, 3) // B4…B6 + got := e.mustFinalize(t, b[1]) + e.want(t, got, e.rev(a3), e.blk(b[0]), e.blk(b[1]), e.fin(b[1])) + e.wantCalls(t, "H(B4) L(B4) L(B5)") + if st := e.tr.State(); st.Tip() != b[1] || st.Finalized() != b[1] { + t.Errorf("state = %+v, want tip and finalized B5", viewOf(st)) + } + for _, h := range oldWindow { + e.want(t, e.mustHead(t, h)) + e.wantCalls(t, "") + } + e.want(t, e.mustHead(t, b[2]), e.blk(b[2])) + + c := e.reorg("C", 4, 5) // C3…C7 + e.src.Do(func(ch *fakechain.Chain) { ch.SetFinalized(4) }) + e.want(t, e.mustFinalize(t, c[1])) + + before := viewOf(e.tr.State()) + got, err := e.finalize(t, c[2]) + wantErr(t, err, tracker.ErrReorgBelowFinalized, "tracker: reorg below the finalized block: finalized block 5 is "+ + b[1].Hash.Hex()+", got "+c[2].Hash.Hex()) + e.want(t, got) + + e.unchanged(t, before) + got, err = e.finalize(t, c[4]) + wantErr(t, err, tracker.ErrReorgBelowFinalized, "tracker: reorg below the finalized block: block 6 "+ + c[3].Hash.Hex()+" does not descend from finalized block 5 "+b[1].Hash.Hex()) + e.want(t, got) + e.wantCalls(t, "H(C6)") + e.unchanged(t, before) +}