From fe91a463c054f470915c6505358548a27bada52d Mon Sep 17 00:00:00 2001 From: Raphael Panic Date: Mon, 13 Apr 2026 16:41:39 +0200 Subject: [PATCH] Revert "Clear atomicity units" --- .../persistance/src/PrismaRedisDatabase.ts | 18 +++++++---- .../services/prisma/PrismaLinkedLeafStore.ts | 8 +++-- packages/sequencer/src/index.ts | 1 - .../production/BatchProducerModule.ts | 7 +---- .../sequencing/BlockProducerModule.ts | 19 +++++------- .../src/state/async/AsyncLinkedLeafStore.ts | 4 ++- .../src/state/lmt/CachedLinkedLeafStore.ts | 30 +++++-------------- .../inmemory/InMemoryAsyncLinkedLeafStore.ts | 6 +++- .../src/storage/inmemory/InMemoryDatabase.ts | 3 -- .../merkle/CachedLinkedMerkleStore.test.ts | 17 ++++------- .../StateTransitionTracingService.test.ts | 2 +- 11 files changed, 50 insertions(+), 65 deletions(-) diff --git a/packages/persistance/src/PrismaRedisDatabase.ts b/packages/persistance/src/PrismaRedisDatabase.ts index 5d0dfe6e8..8f81c7a1c 100644 --- a/packages/persistance/src/PrismaRedisDatabase.ts +++ b/packages/persistance/src/PrismaRedisDatabase.ts @@ -13,6 +13,7 @@ import { log, } from "@proto-kit/common"; import { PrismaClient } from "@prisma/client"; +import { RedisClientType } from "redis"; import { inject } from "tsyringe"; import { @@ -21,8 +22,10 @@ import { PrismaDatabaseConnection, } from "./PrismaDatabaseConnection"; import { + RedisConnection, RedisConnectionConfig, RedisConnectionModule, + RedisTransaction, } from "./RedisConnection"; export interface PrismaRedisCombinedConfig { @@ -36,7 +39,7 @@ export interface PrismaRedisCombinedConfig { @dependencyFactory() export class PrismaRedisDatabase extends SequencerModule - implements PrismaConnection, Database, Prunable + implements PrismaConnection, RedisConnection, Database, Prunable { public prisma: PrismaDatabaseConnection; @@ -52,6 +55,14 @@ export class PrismaRedisDatabase return this.prisma.prismaClient; } + public get redisClient(): RedisClientType { + return this.redis.redisClient; + } + + public get currentMulti(): RedisTransaction { + return this.redis.currentMulti; + } + public create(childContainerProvider: ChildContainerProvider) { super.create(childContainerProvider); this.prisma.create(childContainerProvider); @@ -62,9 +73,6 @@ export class PrismaRedisDatabase return { ...PrismaDatabaseConnection.dependencies(), ...RedisConnectionModule.dependencies(), - TreeDatabase: { - useGenerated: (dbModule) => dbModule.redis, - }, }; } @@ -95,7 +103,7 @@ export class PrismaRedisDatabase // TODO Long-term we want to somehow make sure we can rollback one data source // if commiting the other one's transaction fails await this.prisma.executeInTransaction(async () => { - await f(); + await this.redis.executeInTransaction(f); }); } } diff --git a/packages/persistance/src/services/prisma/PrismaLinkedLeafStore.ts b/packages/persistance/src/services/prisma/PrismaLinkedLeafStore.ts index a0e37af6a..bd2c7cba3 100644 --- a/packages/persistance/src/services/prisma/PrismaLinkedLeafStore.ts +++ b/packages/persistance/src/services/prisma/PrismaLinkedLeafStore.ts @@ -1,4 +1,4 @@ -import { StoredLeaf } from "@proto-kit/common"; +import { noop, StoredLeaf } from "@proto-kit/common"; import { AsyncLinkedLeafStore, trace, Tracer } from "@proto-kit/sequencer"; import { injectable } from "tsyringe"; import { Prisma } from "@prisma/client"; @@ -23,8 +23,12 @@ export class PrismaLinkedLeafStore implements AsyncLinkedLeafStore { } } + public async openTransaction(): Promise { + noop(); + } + @trace("LinkedLeafStore.commit") - public async flush(): Promise { + public async commit(): Promise { if (this.cache.length > 0) { const data = this.cache.map((entry) => ({ path: entry.leaf.path.toString(), diff --git a/packages/sequencer/src/index.ts b/packages/sequencer/src/index.ts index cd54bba67..4946ec580 100644 --- a/packages/sequencer/src/index.ts +++ b/packages/sequencer/src/index.ts @@ -86,7 +86,6 @@ export * from "./storage/repositories/MessageStorage"; export * from "./storage/repositories/TransactionStorage"; export * from "./storage/inmemory/InMemoryDatabase"; export * from "./storage/inmemory/InMemoryAsyncMerkleTreeStore"; -export * from "./storage/inmemory/InMemoryAsyncLinkedLeafStore"; export * from "./storage/inmemory/InMemoryBlockStorage"; export * from "./storage/inmemory/InMemoryBatchStorage"; export * from "./storage/inmemory/InMemorySettlementStorage"; diff --git a/packages/sequencer/src/protocol/production/BatchProducerModule.ts b/packages/sequencer/src/protocol/production/BatchProducerModule.ts index a1e4eec98..1dad75eaa 100644 --- a/packages/sequencer/src/protocol/production/BatchProducerModule.ts +++ b/packages/sequencer/src/protocol/production/BatchProducerModule.ts @@ -54,8 +54,6 @@ export class BatchProducerModule extends SequencerModule { @inject("BatchStorage") private readonly batchStorage: BatchStorage, @inject("Database") private readonly database: Database, - @inject("TreeDatabase") - private readonly treeDatabase: Database, private readonly batchFlow: BatchFlow, private readonly blockProofSerializer: BlockProofSerializer, private readonly batchTraceService: BatchTracingService @@ -90,11 +88,8 @@ export class BatchProducerModule extends SequencerModule { // Apply state changes to current MerkleTreeStore await this.database.executeInTransaction(async () => { await this.batchStorage.pushBatch(batchWithStateDiff.batch); + await batchWithStateDiff.changes.mergeIntoParent(); }); - await batchWithStateDiff.changes.mergeIntoParent( - this.database, - this.treeDatabase - ); // TODO Add transition from unproven to proven state for stateservice // This needs proper DB-level masking diff --git a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts index 70a0292d3..0c0f19f3d 100644 --- a/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts +++ b/packages/sequencer/src/protocol/production/sequencing/BlockProducerModule.ts @@ -57,7 +57,6 @@ export class BlockProducerModule extends SequencerModule { private readonly methodIdResolver: MethodIdResolver, @inject("Runtime") private readonly runtime: Runtime, @inject("Database") private readonly database: Database, - @inject("TreeDatabase") private readonly treeDatabase: Database, @inject("Tracer") public readonly tracer: Tracer ) { super(); @@ -123,17 +122,14 @@ export class BlockProducerModule extends SequencerModule { await this.tracer.trace( "block.result.commit", - async () => { + async () => await this.database.executeInTransaction(async () => { - await this.blockQueue.pushResult(result); - await stateService.mergeIntoParent(); - await treeStore.mergeLeavesIntoParent(); - }); - await this.treeDatabase.executeInTransaction(async () => { await blockHashTreeStore.mergeIntoParent(); - await treeStore.mergeTreeIntoParent(); - }); - }, + await treeStore.mergeIntoParent(); + await stateService.mergeIntoParent(); + + await this.blockQueue.pushResult(result); + }), traceMetadata ); @@ -213,6 +209,7 @@ export class BlockProducerModule extends SequencerModule { async () => { // Push changes to the database atomically await this.database.executeInTransaction(async () => { + await stateChanges.mergeIntoParent(); await this.blockQueue.pushBlock(block); // Remove included or dropped txs, leave skipped ones alone @@ -231,8 +228,6 @@ export class BlockProducerModule extends SequencerModule { await this.transactionStorage.reportSkippedTransactions( orderingMetadata.skippedPaths ); - - await stateChanges.mergeIntoParent(); }); }, { diff --git a/packages/sequencer/src/state/async/AsyncLinkedLeafStore.ts b/packages/sequencer/src/state/async/AsyncLinkedLeafStore.ts index 3725a9986..b3ac3c6f4 100644 --- a/packages/sequencer/src/state/async/AsyncLinkedLeafStore.ts +++ b/packages/sequencer/src/state/async/AsyncLinkedLeafStore.ts @@ -1,7 +1,9 @@ import { StoredLeaf } from "@proto-kit/common"; export interface AsyncLinkedLeafStore { - flush: () => Promise; + openTransaction: () => Promise; + + commit: () => Promise; writeLeaves: (leaves: StoredLeaf[]) => void; diff --git a/packages/sequencer/src/state/lmt/CachedLinkedLeafStore.ts b/packages/sequencer/src/state/lmt/CachedLinkedLeafStore.ts index 53c83203e..0ee781cc9 100644 --- a/packages/sequencer/src/state/lmt/CachedLinkedLeafStore.ts +++ b/packages/sequencer/src/state/lmt/CachedLinkedLeafStore.ts @@ -14,7 +14,6 @@ import groupBy from "lodash/groupBy"; import { AsyncLinkedLeafStore } from "../async/AsyncLinkedLeafStore"; import { CachedMerkleTreeStore } from "../merkle/CachedMerkleTreeStore"; import { AsyncMerkleTreeStore } from "../async/AsyncMerkleTreeStore"; -import { Database } from "../../storage/Database"; export class CachedLinkedLeafStore implements LinkedLeafStore { private writeCache: { @@ -27,7 +26,7 @@ export class CachedLinkedLeafStore implements LinkedLeafStore { private constructor( private readonly parent: AsyncLinkedLeafStore, - parentTreeStore: AsyncMerkleTreeStore + private readonly parentTreeStore: AsyncMerkleTreeStore ) { this.treeCache = new CachedMerkleTreeStore(parentTreeStore); } @@ -216,37 +215,24 @@ export class CachedLinkedLeafStore implements LinkedLeafStore { await this.preloadKeysInternal(paths); } - public async mergeLeavesIntoParent() { + // This merges the cache into the parent tree and resets the cache, but not the + // in-memory merkle tree. + public async mergeIntoParent(): Promise { const leaves = this.getWrittenLeaves(); // In case no state got set we can skip this step if (leaves.length === 0) { return; } - this.parent.writeLeaves(Object.values(leaves)); + await this.parent.openTransaction(); - await this.parent.flush(); + this.parent.writeLeaves(Object.values(leaves)); - this.resetWrittenLeaves(); - } + await this.parent.commit(); - public async mergeTreeIntoParent() { await this.treeCache.mergeIntoParent(); - } - // This merges the cache into the parent tree and resets the cache, but not the - // in-memory merkle tree. - public async mergeIntoParent( - stateDb: Database, - treeDb: Database - ): Promise { - await stateDb.executeInTransaction(async () => { - await this.mergeLeavesIntoParent(); - }); - - await treeDb.executeInTransaction(async () => { - await this.mergeTreeIntoParent(); - }); + this.resetWrittenLeaves(); } public getPreviousLeaf(path: bigint) { diff --git a/packages/sequencer/src/storage/inmemory/InMemoryAsyncLinkedLeafStore.ts b/packages/sequencer/src/storage/inmemory/InMemoryAsyncLinkedLeafStore.ts index 964d38a05..c74f4ffdb 100644 --- a/packages/sequencer/src/storage/inmemory/InMemoryAsyncLinkedLeafStore.ts +++ b/packages/sequencer/src/storage/inmemory/InMemoryAsyncLinkedLeafStore.ts @@ -5,7 +5,11 @@ import { AsyncLinkedLeafStore } from "../../state/async/AsyncLinkedLeafStore"; export class InMemoryAsyncLinkedLeafStore implements AsyncLinkedLeafStore { private readonly leafStore = new InMemoryLinkedLeafStore(); - public async flush(): Promise { + public async openTransaction(): Promise { + noop(); + } + + public async commit(): Promise { noop(); } diff --git a/packages/sequencer/src/storage/inmemory/InMemoryDatabase.ts b/packages/sequencer/src/storage/inmemory/InMemoryDatabase.ts index e1082b955..c29f32f8b 100644 --- a/packages/sequencer/src/storage/inmemory/InMemoryDatabase.ts +++ b/packages/sequencer/src/storage/inmemory/InMemoryDatabase.ts @@ -62,9 +62,6 @@ export class InMemoryDatabase extends SequencerModule implements Database { asyncTreeStore: { useClass: InMemoryAsyncMerkleTreeStore, }, - treeDatabase: { - useToken: "Database", - }, }; } diff --git a/packages/sequencer/test/merkle/CachedLinkedMerkleStore.test.ts b/packages/sequencer/test/merkle/CachedLinkedMerkleStore.test.ts index c95eaeab3..0df9e4d38 100644 --- a/packages/sequencer/test/merkle/CachedLinkedMerkleStore.test.ts +++ b/packages/sequencer/test/merkle/CachedLinkedMerkleStore.test.ts @@ -30,8 +30,7 @@ describe("cached linked merkle store", () => { const tmpTree = new LinkedMerkleTree(cachedStore.treeStore, cachedStore); tmpTree.setLeaf(5n, 10n); - await cachedStore.mergeLeavesIntoParent(); - await cachedStore.mergeTreeIntoParent(); + await cachedStore.mergeIntoParent(); cache1 = await CachedLinkedLeafStore.new(mainStore, mainTreeStore); tree1 = new LinkedMerkleTree(cache1.treeStore, cache1); @@ -85,8 +84,7 @@ describe("cached linked merkle store", () => { expectDefined(cache1.treeStore.getNode(1n, 0)); tree1.setLeaf(10n, 10n); - await cache1.mergeLeavesIntoParent(); - await cache1.mergeTreeIntoParent(); + await cache1.mergeIntoParent(); const leaf5 = tree1.getLeaf(5n); const leaf10 = tree1.getLeaf(10n); @@ -121,8 +119,7 @@ describe("cached linked merkle store", () => { tree1.setLeaf(11n, 11n); tree1.setLeaf(12n, 12n); tree1.setLeaf(13n, 13n); - await cache1.mergeLeavesIntoParent(); - await cache1.mergeTreeIntoParent(); + await cache1.mergeIntoParent(); const cache2 = new SyncCachedLinkedLeafStore(cache1); await cache2.preloadKeys([14n]); @@ -221,7 +218,7 @@ describe("cached linked merkle store", () => { leaf2.hash().toBigInt() ); expect(tree1.getRoot()).not.toEqual(tree2.getRoot()); - cache2.mergeIntoParent(); + await cache2.mergeIntoParent(); expect(tree1.getRoot()).toEqual(tree2.getRoot()); }); @@ -232,8 +229,7 @@ describe("cached linked merkle store", () => { await cache1.preloadKeys([10n, 20n]); treeCache1.setLeaf(10n, 10n); treeCache1.setLeaf(20n, 20n); - await cache1.mergeLeavesIntoParent(); - await cache1.mergeTreeIntoParent(); + await cache1.mergeIntoParent(); const cache2 = new SyncCachedLinkedLeafStore(cache1); const treeCache2 = new LinkedMerkleTree(cache2.treeStore, cache2); @@ -389,8 +385,7 @@ describe("cached linked merkle store", () => { ); // Now the mainstore has the new 15n root. - await cache1.mergeLeavesIntoParent(); - await cache1.mergeTreeIntoParent(); + await cache1.mergeIntoParent(); const cachedStore = await CachedLinkedLeafStore.new( mainStore, diff --git a/packages/sequencer/test/production/tracing/StateTransitionTracingService.test.ts b/packages/sequencer/test/production/tracing/StateTransitionTracingService.test.ts index 40a2b579c..426dba1cd 100644 --- a/packages/sequencer/test/production/tracing/StateTransitionTracingService.test.ts +++ b/packages/sequencer/test/production/tracing/StateTransitionTracingService.test.ts @@ -20,8 +20,8 @@ import { ConsoleTracer, CachedLinkedLeafStore, InMemoryAsyncMerkleTreeStore, - InMemoryAsyncLinkedLeafStore, } from "../../../src"; +import { InMemoryAsyncLinkedLeafStore } from "../../../src/storage/inmemory/InMemoryAsyncLinkedLeafStore"; function createST(obj: { path: string;