From 3a454fee453f044240a8daabd998b1c578692001 Mon Sep 17 00:00:00 2001 From: MJ Date: Wed, 26 Feb 2025 10:48:27 -0600 Subject: [PATCH 01/32] Upgrade abstract-level to 2.0.2 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 1366a37..7f78b37 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,7 @@ ], "dependencies": { "@vweevers/length-prefixed-stream": "^1.0.0", - "abstract-level": "^1.0.3", + "abstract-level": "^2.0.2", "module-error": "^1.0.2", "protocol-buffers-encodings": "^1.1.0", "readable-stream": "^4.0.0" From f42faf5bfa4687400da7d50b4bb34bd19b1f270d Mon Sep 17 00:00:00 2001 From: MJ Date: Mon, 10 Mar 2025 13:17:24 -0600 Subject: [PATCH 02/32] Refactor type declarations to remove callbacks --- index.d.ts | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/index.d.ts b/index.d.ts index 203fc95..8b02870 100644 --- a/index.d.ts +++ b/index.d.ts @@ -2,7 +2,6 @@ import { AbstractLevel, AbstractDatabaseOptions, AbstractOpenOptions, - NodeCallback } from 'abstract-level' // Requires `npm install @types/readable-stream`. @@ -25,8 +24,6 @@ export class ManyLevelGuest open (): Promise open (options: GuestOpenOptions): Promise - open (callback: NodeCallback): void - open (options: GuestOpenOptions, callback: NodeCallback): void /** * Create a duplex guest stream to be piped into a host stream. Until that's done, @@ -158,17 +155,17 @@ declare interface HostOptions { /** * A function to be called before `db.put()` operations. */ - preput?: (key: Buffer, value: Buffer, callback: NodeCallback) => void + preput?: (key: Buffer, value: Buffer) => Promise /** * A function to be called before `db.del()` operations. */ - predel?: (key: Buffer, callback: NodeCallback) => void + predel?: (key: Buffer) => Promise /** * A function to be called before `db.batch()` operations. */ - prebatch?: (operations: HostBatchOperation[], callback: NodeCallback) => void + prebatch?: (operations: HostBatchOperation[]) => Promise } declare type HostBatchOperation = HostBatchPutOperation | HostBatchDelOperation From df34e1cd4abeadaa3d6c7071b3cdf1ae9807ff49 Mon Sep 17 00:00:00 2001 From: MJ Date: Mon, 10 Mar 2025 15:14:58 -0600 Subject: [PATCH 03/32] Initial async refactor of guest - Still need to refactor handling of outstanding reqs - Still need to refactor rpc stream event handlers to not use req/iterator callbacks - Determine if we need a new mechanism for passing errors up to request initiator --- guest.js | 101 +++++++++++++++++++++++++------------------------------ 1 file changed, 45 insertions(+), 56 deletions(-) diff --git a/guest.js b/guest.js index d3abf63..383b7ea 100644 --- a/guest.js +++ b/guest.js @@ -208,102 +208,96 @@ class ManyLevelGuest extends AbstractLevel { } } - _get (key, opts, cb) { + async _get (key, opts) { // TODO: this and other methods assume db state matches our state - if (this[kDb]) return this[kDb]._get(key, opts, cb) + if (this[kDb]) return this[kDb]._get(key, opts) const req = { tag: input.get, id: 0, - key: key, - callback: cb + key: key } req.id = this[kRequests].add(req) - this[kWrite](req) + req.promise = this[kWrite](req) } - _getMany (keys, opts, cb) { - if (this[kDb]) return this[kDb]._getMany(keys, opts, cb) + async _getMany (keys, opts) { + if (this[kDb]) return this[kDb]._getMany(keys, opts) const req = { tag: input.getMany, id: 0, - keys: keys, - callback: cb + keys: keys } req.id = this[kRequests].add(req) - this[kWrite](req) + req.promise = this[kWrite](req) } - _put (key, value, opts, cb) { - if (this[kDb]) return this[kDb]._put(key, value, opts, cb) + async _put (key, value, opts) { + if (this[kDb]) return this[kDb]._put(key, value, opts) const req = { tag: input.put, id: 0, key: key, - value: value, - callback: cb + value: value } req.id = this[kRequests].add(req) - this[kWrite](req) + req.promise = this[kWrite](req) } - _del (key, opts, cb) { - if (this[kDb]) return this[kDb]._del(key, opts, cb) + async _del (key, opts) { + if (this[kDb]) return this[kDb]._del(key, opts) const req = { tag: input.del, id: 0, - key: key, - callback: cb + key: key } req.id = this[kRequests].add(req) - this[kWrite](req) + req.promise = this[kWrite](req) } - _batch (batch, opts, cb) { - if (this[kDb]) return this[kDb]._batch(batch, opts, cb) + async _batch (batch, opts) { + if (this[kDb]) return this[kDb]._batch(batch, opts) const req = { tag: input.batch, id: 0, - ops: batch, - callback: cb + ops: batch } req.id = this[kRequests].add(req) - this[kWrite](req) + req.promise = this[kWrite](req) } - _clear (opts, cb) { - if (this[kDb]) return this[kDb]._clear(opts, cb) + async _clear (opts) { + if (this[kDb]) return this[kDb]._clear(opts) const req = { tag: input.clear, id: 0, - options: opts, - callback: cb + options: opts } req.id = this[kRequests].add(req) - this[kWrite](req) + req.promise = this[kWrite](req) } - [kWrite] (req) { + async [kWrite] (req) { if (this[kRequests].size + this[kIterators].size === 1) ref(this[kRef]) const enc = input.encoding(req.tag) const buf = Buffer.allocUnsafe(enc.encodingLength(req) + 1) buf[0] = req.tag enc.encode(req, buf, 1) - this[kEncode].write(buf) + return this[kEncode].write(buf) } - _close (cb) { + async _close () { // Even if forward() was used, still need to abort requests made before forward(). this[kExplicitClose] = true this[kAbortRequests]('Aborted on database close()', 'LEVEL_DATABASE_NOT_OPEN') @@ -311,18 +305,16 @@ class ManyLevelGuest extends AbstractLevel { if (this[kRpcStream]) { finished(this[kRpcStream], () => { this[kRpcStream] = null - this._close(cb) + this._close() }) this[kRpcStream].destroy() } else if (this[kDb]) { // To be safe, use close() not _close(). - this[kDb].close(cb) - } else { - this.nextTick(cb) + this[kDb].close() } } - _open (options, cb) { + async _open (options) { if (this[kRemote]) { // For tests only so does not need error handling this[kExplicitClose] = false @@ -338,8 +330,6 @@ class ManyLevelGuest extends AbstractLevel { code: 'LEVEL_NOT_SUPPORTED' }) } - - this.nextTick(cb) } iterator (options) { @@ -369,7 +359,6 @@ class Iterator extends AbstractIterator { this[kEnded] = false this[kErrored] = false this[kPending] = [] - this[kCallback] = null this[kSeq] = 0 const req = this[kRequest] = { @@ -419,12 +408,16 @@ class Iterator extends AbstractIterator { } // TODO: implement optimized `nextv()` - _next (callback) { - this[kCallback] = null - + async _next () { if (this[kRequest].consumed >= this.limit || this[kErrored]) { - this.nextTick(callback) - } else if (this[kPending].length !== 0) { + return + } else if (this[kEnded]) { + return + // TODO: no more callbacks, verify what to do with this[kCallback] + // } else { + // this[kCallback] = callback + } + if (this[kPending].length !== 0) { const next = this[kPending][0] const req = this[kRequest] @@ -433,9 +426,9 @@ class Iterator extends AbstractIterator { this[kErrored] = true this[kPending] = [] - return this.nextTick(callback, new ModuleError('Could not read entry', { + throw new ModuleError('Could not read entry', { code: next.error - })) + }) } const consumed = ++req.consumed @@ -459,19 +452,15 @@ class Iterator extends AbstractIterator { req.bookmark = key } - this.nextTick(callback, undefined, key, val) - } else if (this[kEnded]) { - this.nextTick(callback) - } else { - this[kCallback] = callback + // TODO: is this the right return type?? + return [key, val] } } - _close (cb) { - this.db[kWrite]({ tag: input.iteratorClose, id: this[kRequest].id }) + async _close () { + await this.db[kWrite]({ tag: input.iteratorClose, id: this[kRequest].id }) this.db[kIterators].remove(this[kRequest].id) this.db[kFlushed]() - this.nextTick(cb) } } From e092f559ef358506a0b91ba48f76984a04e7c1b8 Mon Sep 17 00:00:00 2001 From: MJ Date: Thu, 13 Mar 2025 18:02:02 -0600 Subject: [PATCH 04/32] Fix method returns so they resolve correctly --- guest.js | 25 ++++++++++++++++--------- index.d.ts | 2 +- 2 files changed, 17 insertions(+), 10 deletions(-) diff --git a/guest.js b/guest.js index 383b7ea..b3f389b 100644 --- a/guest.js +++ b/guest.js @@ -149,15 +149,15 @@ class ManyLevelGuest extends AbstractLevel { function oncallback (res) { const req = self[kRequests].remove(res.id) if (!req) return - if (res.error) req.callback(new ModuleError('Could not get value', { code: res.error })) - else req.callback(null, normalizeValue(res.value)) + if (res.error) req.promise.then(new ModuleError('Could not get value', { code: res.error })) + else req.promise.then(null, normalizeValue(res.value)) } function ongetmanycallback (res) { const req = self[kRequests].remove(res.id) if (!req) return - if (res.error) req.callback(new ModuleError('Could not get values', { code: res.error })) - else req.callback(null, res.values.map(v => normalizeValue(v.value))) + if (res.error) req.promise.then(new ModuleError('Could not get values', { code: res.error })) + else req.promise.then(null, res.values.map(v => normalizeValue(v.value))) } } @@ -191,7 +191,8 @@ class ManyLevelGuest extends AbstractLevel { [kAbortRequests] (msg, code) { for (const req of this[kRequests].clear()) { - req.callback(new ModuleError(msg, { code })) + // TODO: this doesn't actually abort the request, but neither did the old way + req.promise.then(new ModuleError(msg, { code })) } for (const req of this[kIterators].clear()) { @@ -220,6 +221,7 @@ class ManyLevelGuest extends AbstractLevel { req.id = this[kRequests].add(req) req.promise = this[kWrite](req) + return req.promise } async _getMany (keys, opts) { @@ -233,6 +235,7 @@ class ManyLevelGuest extends AbstractLevel { req.id = this[kRequests].add(req) req.promise = this[kWrite](req) + return req.promise } async _put (key, value, opts) { @@ -247,6 +250,7 @@ class ManyLevelGuest extends AbstractLevel { req.id = this[kRequests].add(req) req.promise = this[kWrite](req) + return req.promise } async _del (key, opts) { @@ -260,6 +264,7 @@ class ManyLevelGuest extends AbstractLevel { req.id = this[kRequests].add(req) req.promise = this[kWrite](req) + return req.promise } async _batch (batch, opts) { @@ -273,6 +278,7 @@ class ManyLevelGuest extends AbstractLevel { req.id = this[kRequests].add(req) req.promise = this[kWrite](req) + return req.promise } async _clear (opts) { @@ -286,6 +292,7 @@ class ManyLevelGuest extends AbstractLevel { req.id = this[kRequests].add(req) req.promise = this[kWrite](req) + return req.promise } async [kWrite] (req) { @@ -294,7 +301,7 @@ class ManyLevelGuest extends AbstractLevel { const buf = Buffer.allocUnsafe(enc.encodingLength(req) + 1) buf[0] = req.tag enc.encode(req, buf, 1) - return this[kEncode].write(buf) + return await this[kEncode].write(buf) } async _close () { @@ -310,7 +317,7 @@ class ManyLevelGuest extends AbstractLevel { this[kRpcStream].destroy() } else if (this[kDb]) { // To be safe, use close() not _close(). - this[kDb].close() + return this[kDb].close() } } @@ -441,7 +448,7 @@ class Iterator extends AbstractIterator { // Acknowledge receipt. Not needed if we don't want more data. if (consumed < this.limit) { this[kAckMessage].consumed = consumed - this.db[kWrite](this[kAckMessage]) + await this.db[kWrite](this[kAckMessage]) } } @@ -453,7 +460,7 @@ class Iterator extends AbstractIterator { } // TODO: is this the right return type?? - return [key, val] + return val } } diff --git a/index.d.ts b/index.d.ts index 8b02870..e821809 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,7 +1,7 @@ import { AbstractLevel, AbstractDatabaseOptions, - AbstractOpenOptions, + AbstractOpenOptions } from 'abstract-level' // Requires `npm install @types/readable-stream`. From a64c78e421bc2b0d5daf58c56d3cc5463943753e Mon Sep 17 00:00:00 2001 From: MJ Date: Thu, 13 Mar 2025 18:02:14 -0600 Subject: [PATCH 05/32] WIP TODOs --- guest.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/guest.js b/guest.js index b3f389b..a4ef834 100644 --- a/guest.js +++ b/guest.js @@ -135,6 +135,7 @@ class ManyLevelGuest extends AbstractLevel { const req = self[kIterators].get(res.id) if (!req || req.iterator[kSeq] !== res.seq) return req.iterator[kPending].push(res) + // TODO: deprecate callbacks if (req.iterator[kCallback]) req.iterator._next(req.iterator[kCallback]) } @@ -143,9 +144,11 @@ class ManyLevelGuest extends AbstractLevel { if (!req || req.iterator[kSeq] !== res.seq) return // https://github.com/Level/abstract-level/issues/19 req.iterator[kEnded] = true + // TODO: deprecate callbacks if (req.iterator[kCallback]) req.iterator._next(req.iterator[kCallback]) } + // TODO: no more callbacks function oncallback (res) { const req = self[kRequests].remove(res.id) if (!req) return @@ -153,6 +156,7 @@ class ManyLevelGuest extends AbstractLevel { else req.promise.then(null, normalizeValue(res.value)) } + // TODO: no more callbacks function ongetmanycallback (res) { const req = self[kRequests].remove(res.id) if (!req) return @@ -197,6 +201,7 @@ class ManyLevelGuest extends AbstractLevel { for (const req of this[kIterators].clear()) { // Cancel in-flight operation if any + // TODO: do we need a new mechanism to pass the error back up to the request initiator? const callback = req.iterator[kCallback] req.iterator[kCallback] = null @@ -310,6 +315,7 @@ class ManyLevelGuest extends AbstractLevel { this[kAbortRequests]('Aborted on database close()', 'LEVEL_DATABASE_NOT_OPEN') if (this[kRpcStream]) { + // TODO: need to do something with finished. Can we use readable-stream.promises or whatever?? That would make life much easier... finished(this[kRpcStream], () => { this[kRpcStream] = null this._close() @@ -326,6 +332,7 @@ class ManyLevelGuest extends AbstractLevel { // For tests only so does not need error handling this[kExplicitClose] = false const remote = this[kRemote]() + // TODO: Need to promisify pipeline pipeline( remote, this.connect(), From 0dc88e702f9c5babd40c2fa77517d9a1c56fb157 Mon Sep 17 00:00:00 2001 From: MJ Date: Mon, 17 Mar 2025 07:31:44 -0600 Subject: [PATCH 06/32] Convert host-response callbacks to promises We still need an internal callback since the host response occurs in a separate call stack, so it cannot be directly awaited. --- guest.js | 71 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 43 insertions(+), 28 deletions(-) diff --git a/guest.js b/guest.js index a4ef834..2818415 100644 --- a/guest.js +++ b/guest.js @@ -148,20 +148,16 @@ class ManyLevelGuest extends AbstractLevel { if (req.iterator[kCallback]) req.iterator._next(req.iterator[kCallback]) } - // TODO: no more callbacks function oncallback (res) { const req = self[kRequests].remove(res.id) - if (!req) return - if (res.error) req.promise.then(new ModuleError('Could not get value', { code: res.error })) - else req.promise.then(null, normalizeValue(res.value)) + if (!req || !req.callback) return + req.callback(new ModuleError('Could not get value', { code: res.error }), normalizeValue(res.value)) } - // TODO: no more callbacks function ongetmanycallback (res) { const req = self[kRequests].remove(res.id) - if (!req) return - if (res.error) req.promise.then(new ModuleError('Could not get values', { code: res.error })) - else req.promise.then(null, res.values.map(v => normalizeValue(v.value))) + if (!req || !req.callback) return + req.callback(new ModuleError('Could not get value', { code: res.error }), normalizeValue(res.value)) } } @@ -196,9 +192,10 @@ class ManyLevelGuest extends AbstractLevel { [kAbortRequests] (msg, code) { for (const req of this[kRequests].clear()) { // TODO: this doesn't actually abort the request, but neither did the old way - req.promise.then(new ModuleError(msg, { code })) + req.callback(new ModuleError(msg, { code })) } + // TODO: iterators for (const req of this[kIterators].clear()) { // Cancel in-flight operation if any // TODO: do we need a new mechanism to pass the error back up to the request initiator? @@ -218,86 +215,104 @@ class ManyLevelGuest extends AbstractLevel { // TODO: this and other methods assume db state matches our state if (this[kDb]) return this[kDb]._get(key, opts) + const promise = new Promise() const req = { tag: input.get, id: 0, - key: key + key: key, + callback: (err, value) => err ? promise.reject(err) : promise.resolve(value) } req.id = this[kRequests].add(req) - req.promise = this[kWrite](req) - return req.promise + this[kWrite](req) + // This should resolve or reject based on the Host's response + return promise } async _getMany (keys, opts) { if (this[kDb]) return this[kDb]._getMany(keys, opts) + const promise = new Promise() const req = { tag: input.getMany, id: 0, - keys: keys + keys: keys, + callback: (err, value) => err ? promise.reject(err) : promise.resolve(value) } req.id = this[kRequests].add(req) - req.promise = this[kWrite](req) - return req.promise + this[kWrite](req) + // This should resolve or reject based on the Host's response + return promise } async _put (key, value, opts) { if (this[kDb]) return this[kDb]._put(key, value, opts) + const promise = new Promise() const req = { tag: input.put, id: 0, key: key, - value: value + value: value, + callback: (err, value) => err ? promise.reject(err) : promise.resolve(value) } req.id = this[kRequests].add(req) - req.promise = this[kWrite](req) - return req.promise + this[kWrite](req) + // This should resolve or reject based on the Host's response + return promise } async _del (key, opts) { if (this[kDb]) return this[kDb]._del(key, opts) + const promise = new Promise() const req = { tag: input.del, id: 0, - key: key + key: key, + callback: (err, value) => err ? promise.reject(err) : promise.resolve(value) } req.id = this[kRequests].add(req) - req.promise = this[kWrite](req) - return req.promise + this[kWrite](req) + // This should resolve or reject based on the Host's response + return promise } async _batch (batch, opts) { if (this[kDb]) return this[kDb]._batch(batch, opts) + const promise = new Promise() const req = { tag: input.batch, id: 0, - ops: batch + ops: batch, + callback: (err, value) => err ? promise.reject(err) : promise.resolve(value) } req.id = this[kRequests].add(req) - req.promise = this[kWrite](req) - return req.promise + this[kWrite](req) + // This should resolve or reject based on the Host's response + return promise } async _clear (opts) { if (this[kDb]) return this[kDb]._clear(opts) + const promise = new Promise() const req = { tag: input.clear, id: 0, - options: opts + options: opts, + callback: (err, value) => err ? promise.reject(err) : promise.resolve(value) } req.id = this[kRequests].add(req) - req.promise = this[kWrite](req) - return req.promise + this[kWrite](req) + // This should resolve or reject based on the Host's response + return promise } async [kWrite] (req) { @@ -306,7 +321,7 @@ class ManyLevelGuest extends AbstractLevel { const buf = Buffer.allocUnsafe(enc.encodingLength(req) + 1) buf[0] = req.tag enc.encode(req, buf, 1) - return await this[kEncode].write(buf) + return this[kEncode].write(buf) } async _close () { From 10c2f00a90b5c1e65cdaab7d6029dda6263191fb Mon Sep 17 00:00:00 2001 From: MJ Date: Tue, 25 Mar 2025 13:36:13 -0600 Subject: [PATCH 07/32] Fix promise resolvers/rejectors --- guest.js | 41 +++++++++++++++++++++++++++-------------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/guest.js b/guest.js index 2818415..2676e80 100644 --- a/guest.js +++ b/guest.js @@ -151,13 +151,13 @@ class ManyLevelGuest extends AbstractLevel { function oncallback (res) { const req = self[kRequests].remove(res.id) if (!req || !req.callback) return - req.callback(new ModuleError('Could not get value', { code: res.error }), normalizeValue(res.value)) + req.callback(res.error ? new ModuleError('Could not get value', { code: res.error }) : null, normalizeValue(res.value)) } function ongetmanycallback (res) { const req = self[kRequests].remove(res.id) if (!req || !req.callback) return - req.callback(new ModuleError('Could not get value', { code: res.error }), normalizeValue(res.value)) + req.callback(res.error ? new ModuleError('Could not get value', { code: res.error }) : null, normalizeValue(res.value)) } } @@ -211,16 +211,29 @@ class ManyLevelGuest extends AbstractLevel { } } + static _promiseFactory () { + let promiseResolve, promiseReject + const promise = new Promise((resolve, reject) => { + promiseResolve = resolve + promiseReject = reject + }) + return { + resolve: promiseResolve, + reject: promiseReject, + promise + } + } + async _get (key, opts) { // TODO: this and other methods assume db state matches our state if (this[kDb]) return this[kDb]._get(key, opts) - const promise = new Promise() + const { promise, resolve, reject } = ManyLevelGuest._promiseFactory() const req = { tag: input.get, id: 0, key: key, - callback: (err, value) => err ? promise.reject(err) : promise.resolve(value) + callback: (err, value) => err ? reject(err) : resolve(value) } req.id = this[kRequests].add(req) @@ -232,12 +245,12 @@ class ManyLevelGuest extends AbstractLevel { async _getMany (keys, opts) { if (this[kDb]) return this[kDb]._getMany(keys, opts) - const promise = new Promise() + const { promise, resolve, reject } = ManyLevelGuest._promiseFactory() const req = { tag: input.getMany, id: 0, keys: keys, - callback: (err, value) => err ? promise.reject(err) : promise.resolve(value) + callback: (err, value) => err ? reject(err) : resolve(value) } req.id = this[kRequests].add(req) @@ -249,13 +262,13 @@ class ManyLevelGuest extends AbstractLevel { async _put (key, value, opts) { if (this[kDb]) return this[kDb]._put(key, value, opts) - const promise = new Promise() + const { promise, resolve, reject } = ManyLevelGuest._promiseFactory() const req = { tag: input.put, id: 0, key: key, value: value, - callback: (err, value) => err ? promise.reject(err) : promise.resolve(value) + callback: (err, value) => err ? reject(err) : resolve(value) } req.id = this[kRequests].add(req) @@ -267,12 +280,12 @@ class ManyLevelGuest extends AbstractLevel { async _del (key, opts) { if (this[kDb]) return this[kDb]._del(key, opts) - const promise = new Promise() + const { promise, resolve, reject } = ManyLevelGuest._promiseFactory() const req = { tag: input.del, id: 0, key: key, - callback: (err, value) => err ? promise.reject(err) : promise.resolve(value) + callback: (err, value) => err ? reject(err) : resolve(value) } req.id = this[kRequests].add(req) @@ -284,12 +297,12 @@ class ManyLevelGuest extends AbstractLevel { async _batch (batch, opts) { if (this[kDb]) return this[kDb]._batch(batch, opts) - const promise = new Promise() + const { promise, resolve, reject } = ManyLevelGuest._promiseFactory() const req = { tag: input.batch, id: 0, ops: batch, - callback: (err, value) => err ? promise.reject(err) : promise.resolve(value) + callback: (err, value) => err ? reject(err) : resolve(value) } req.id = this[kRequests].add(req) @@ -301,12 +314,12 @@ class ManyLevelGuest extends AbstractLevel { async _clear (opts) { if (this[kDb]) return this[kDb]._clear(opts) - const promise = new Promise() + const { promise, resolve, reject } = ManyLevelGuest._promiseFactory() const req = { tag: input.clear, id: 0, options: opts, - callback: (err, value) => err ? promise.reject(err) : promise.resolve(value) + callback: (err, value) => err ? reject(err) : resolve(value) } req.id = this[kRequests].add(req) From d8f3405c0019030259a00a48208d303b768f476e Mon Sep 17 00:00:00 2001 From: MJ Date: Tue, 25 Mar 2025 15:35:40 -0600 Subject: [PATCH 08/32] Refactor data decode callback to be closer to old implementation --- guest.js | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/guest.js b/guest.js index 2676e80..d7951f3 100644 --- a/guest.js +++ b/guest.js @@ -151,13 +151,15 @@ class ManyLevelGuest extends AbstractLevel { function oncallback (res) { const req = self[kRequests].remove(res.id) if (!req || !req.callback) return - req.callback(res.error ? new ModuleError('Could not get value', { code: res.error }) : null, normalizeValue(res.value)) + if (res.error) req.callback(new ModuleError('Could not get value', { code: res.error })) + else req.callback(null, normalizeValue(res.value)) } function ongetmanycallback (res) { const req = self[kRequests].remove(res.id) if (!req || !req.callback) return - req.callback(res.error ? new ModuleError('Could not get value', { code: res.error }) : null, normalizeValue(res.value)) + if (res.error) req.callback(new ModuleError('Could not get values', { code: res.error })) + else req.callback(null, res.values.map(v => normalizeValue(v.value))) } } From 51078702a0d122f56b0719492fca0c7247f8e2fd Mon Sep 17 00:00:00 2001 From: MJ Date: Tue, 25 Mar 2025 15:36:27 -0600 Subject: [PATCH 09/32] Simplify promise factory usage --- guest.js | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/guest.js b/guest.js index d7951f3..803e404 100644 --- a/guest.js +++ b/guest.js @@ -220,9 +220,11 @@ class ManyLevelGuest extends AbstractLevel { promiseReject = reject }) return { - resolve: promiseResolve, - reject: promiseReject, - promise + promise, + callback: (err, value) => { + if (err) promiseReject(err) + else promiseResolve(value) + } } } @@ -230,12 +232,12 @@ class ManyLevelGuest extends AbstractLevel { // TODO: this and other methods assume db state matches our state if (this[kDb]) return this[kDb]._get(key, opts) - const { promise, resolve, reject } = ManyLevelGuest._promiseFactory() + const { promise, callback } = ManyLevelGuest._promiseFactory() const req = { tag: input.get, id: 0, key: key, - callback: (err, value) => err ? reject(err) : resolve(value) + callback } req.id = this[kRequests].add(req) @@ -247,12 +249,12 @@ class ManyLevelGuest extends AbstractLevel { async _getMany (keys, opts) { if (this[kDb]) return this[kDb]._getMany(keys, opts) - const { promise, resolve, reject } = ManyLevelGuest._promiseFactory() + const { promise, callback } = ManyLevelGuest._promiseFactory() const req = { tag: input.getMany, id: 0, keys: keys, - callback: (err, value) => err ? reject(err) : resolve(value) + callback } req.id = this[kRequests].add(req) @@ -264,13 +266,13 @@ class ManyLevelGuest extends AbstractLevel { async _put (key, value, opts) { if (this[kDb]) return this[kDb]._put(key, value, opts) - const { promise, resolve, reject } = ManyLevelGuest._promiseFactory() + const { promise, callback } = ManyLevelGuest._promiseFactory() const req = { tag: input.put, id: 0, key: key, value: value, - callback: (err, value) => err ? reject(err) : resolve(value) + callback } req.id = this[kRequests].add(req) @@ -282,12 +284,12 @@ class ManyLevelGuest extends AbstractLevel { async _del (key, opts) { if (this[kDb]) return this[kDb]._del(key, opts) - const { promise, resolve, reject } = ManyLevelGuest._promiseFactory() + const { promise, callback } = ManyLevelGuest._promiseFactory() const req = { tag: input.del, id: 0, key: key, - callback: (err, value) => err ? reject(err) : resolve(value) + callback } req.id = this[kRequests].add(req) @@ -299,12 +301,12 @@ class ManyLevelGuest extends AbstractLevel { async _batch (batch, opts) { if (this[kDb]) return this[kDb]._batch(batch, opts) - const { promise, resolve, reject } = ManyLevelGuest._promiseFactory() + const { promise, callback } = ManyLevelGuest._promiseFactory() const req = { tag: input.batch, id: 0, ops: batch, - callback: (err, value) => err ? reject(err) : resolve(value) + callback } req.id = this[kRequests].add(req) @@ -316,12 +318,12 @@ class ManyLevelGuest extends AbstractLevel { async _clear (opts) { if (this[kDb]) return this[kDb]._clear(opts) - const { promise, resolve, reject } = ManyLevelGuest._promiseFactory() + const { promise, callback } = ManyLevelGuest._promiseFactory() const req = { tag: input.clear, id: 0, options: opts, - callback: (err, value) => err ? reject(err) : resolve(value) + callback } req.id = this[kRequests].add(req) From 6b1779b7ac82451ea70366d4abd13b70a05774c0 Mon Sep 17 00:00:00 2001 From: MJ Date: Tue, 25 Mar 2025 15:38:35 -0600 Subject: [PATCH 10/32] Convert host decode handlers to async --- host.js | 67 +++++++++++++++++++++++++++++++-------------------------- 1 file changed, 37 insertions(+), 30 deletions(-) diff --git a/host.js b/host.js index cb0bd78..73b783a 100644 --- a/host.js +++ b/host.js @@ -131,48 +131,55 @@ function createRpcStream (db, options, streamOptions) { encode.write(encodeMessage(msg, output.getManyCallback)) } - function onput (req) { - preput(req.key, req.value, function (err) { - if (err) return callback(req.id, err) - db.put(req.key, req.value, encodingOptions, function (err) { - callback(req.id, err, null) - }) - }) + async function onput (req) { + preput(req.key, req.value, function (err) { return callback(req.id, err) }) + try { + await db.put(req.key, req.value, encodingOptions) + } catch (err) { + return callback(req.id, err, null) + } } - function onget (req) { - db.get(req.key, encodingOptions, function (err, value) { - callback(req.id, err, value) - }) + async function onget (req) { + try { + const value = await db.get(req.key, encodingOptions) + return callback(req.id, null, value) + } catch (err) { + return callback(req.id, err, null) + } } - function ongetmany (req) { - db.getMany(req.keys, encodingOptions, function (err, values) { - getManyCallback(req.id, err, values.map(value => ({ value }))) - }) + async function ongetmany (req) { + try { + const values = await db.getMany(req.keys, encodingOptions) + return getManyCallback(req.id, null, values.map(value => ({ value }))) + } catch (err) { + return getManyCallback(req.id, err, []) + } } - function ondel (req) { - predel(req.key, function (err) { - if (err) return callback(req.id, err) - db.del(req.key, encodingOptions, function (err) { - callback(req.id, err) - }) - }) + async function ondel (req) { + predel(req.key, function (err) { return callback(req.id, err) }) + try { + await db.del(req.key, encodingOptions) + return callback(req.id, null) + } catch (err) { + return callback(req.id, err) + } } function onreadonly (req) { callback(req.id, new ModuleError('Database is readonly', { code: 'LEVEL_READONLY' })) } - function onbatch (req) { - prebatch(req.ops, function (err) { - if (err) return callback(req.id, err) - - db.batch(req.ops, encodingOptions, function (err) { - callback(req.id, err) - }) - }) + async function onbatch (req) { + prebatch(req.key, function (err) { return callback(req.id, err) }) + try { + await db.batch(req.ops, encodingOptions) + return callback(req.id, null) + } catch (err) { + return callback(req.id, err) + } } function oniterator ({ id, seq, options, consumed, bookmark, seek }) { From b4457444dd14e94136668f2814ed29d3e9bced7c Mon Sep 17 00:00:00 2001 From: MJ Date: Tue, 25 Mar 2025 15:39:16 -0600 Subject: [PATCH 11/32] Upgrade memory-level to 2.0.0 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 7f78b37..18ed612 100644 --- a/package.json +++ b/package.json @@ -39,7 +39,7 @@ "faucet": "^0.0.3", "hallmark": "^4.0.0", "level-read-stream": "^1.1.0", - "memory-level": "^1.0.0", + "memory-level": "^2.0.0", "nyc": "^15.1.0", "protocol-buffers": "^5.0.0", "standard": "^16.0.3", From 9ea7323b1a217fd3b3bc09d9914187de943f40d1 Mon Sep 17 00:00:00 2001 From: MJ Date: Tue, 25 Mar 2025 16:00:32 -0600 Subject: [PATCH 12/32] Fix host createRpcStream for async db.open --- host.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/host.js b/host.js index 73b783a..eb8e24c 100644 --- a/host.js +++ b/host.js @@ -65,7 +65,7 @@ function createRpcStream (db, options, streamOptions) { const predel = options.predel const prebatch = options.prebatch - db.open({ passive: true }, ready) + db.open({ passive: true }).then(() => ready()).catch(ready) // TODO: send events to guest. Challenges: // - Need to know encodings or emit encoded data; current abstract-level events don't suffice From ff43b0a8c45c63b1df98befe45c9c53b1804e6ee Mon Sep 17 00:00:00 2001 From: MJ Date: Tue, 25 Mar 2025 16:15:10 -0600 Subject: [PATCH 13/32] Add test script which includes stack trace in output --- package.json | 1 + 1 file changed, 1 insertion(+) diff --git a/package.json b/package.json index 18ed612..764a753 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "types": "./index.d.ts", "scripts": { "test": "standard && ts-standard *.ts && hallmark && (nyc -s tape test/*.js | faucet) && nyc report", + "test:stacktrace": "nyc -s tape test/*.js && nyc report", "coverage": "nyc report -r lcovonly", "hallmark": "hallmark --fix", "protobuf": "protocol-buffers schema.proto -o messages.js", From fd5aec3f71884307e62d3813b39b4559f6b7d82a Mon Sep 17 00:00:00 2001 From: MJ Date: Wed, 26 Mar 2025 11:51:28 -0600 Subject: [PATCH 14/32] Move promise factory out of ManyLevelGuest so iterator can also use it --- guest.js | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/guest.js b/guest.js index 803e404..1184da3 100644 --- a/guest.js +++ b/guest.js @@ -213,26 +213,11 @@ class ManyLevelGuest extends AbstractLevel { } } - static _promiseFactory () { - let promiseResolve, promiseReject - const promise = new Promise((resolve, reject) => { - promiseResolve = resolve - promiseReject = reject - }) - return { - promise, - callback: (err, value) => { - if (err) promiseReject(err) - else promiseResolve(value) - } - } - } - async _get (key, opts) { // TODO: this and other methods assume db state matches our state if (this[kDb]) return this[kDb]._get(key, opts) - const { promise, callback } = ManyLevelGuest._promiseFactory() + const { promise, callback } = promiseFactory() const req = { tag: input.get, id: 0, @@ -249,7 +234,7 @@ class ManyLevelGuest extends AbstractLevel { async _getMany (keys, opts) { if (this[kDb]) return this[kDb]._getMany(keys, opts) - const { promise, callback } = ManyLevelGuest._promiseFactory() + const { promise, callback } = promiseFactory() const req = { tag: input.getMany, id: 0, @@ -266,7 +251,7 @@ class ManyLevelGuest extends AbstractLevel { async _put (key, value, opts) { if (this[kDb]) return this[kDb]._put(key, value, opts) - const { promise, callback } = ManyLevelGuest._promiseFactory() + const { promise, callback } = promiseFactory() const req = { tag: input.put, id: 0, @@ -284,7 +269,7 @@ class ManyLevelGuest extends AbstractLevel { async _del (key, opts) { if (this[kDb]) return this[kDb]._del(key, opts) - const { promise, callback } = ManyLevelGuest._promiseFactory() + const { promise, callback } = promiseFactory() const req = { tag: input.del, id: 0, @@ -301,7 +286,7 @@ class ManyLevelGuest extends AbstractLevel { async _batch (batch, opts) { if (this[kDb]) return this[kDb]._batch(batch, opts) - const { promise, callback } = ManyLevelGuest._promiseFactory() + const { promise, callback } = promiseFactory() const req = { tag: input.batch, id: 0, @@ -318,7 +303,7 @@ class ManyLevelGuest extends AbstractLevel { async _clear (opts) { if (this[kDb]) return this[kDb]._clear(opts) - const { promise, callback } = ManyLevelGuest._promiseFactory() + const { promise, callback } = promiseFactory() const req = { tag: input.clear, id: 0, @@ -510,6 +495,21 @@ class Iterator extends AbstractIterator { } } +function promiseFactory () { + let promiseResolve, promiseReject + const promise = new Promise((resolve, reject) => { + promiseResolve = resolve + promiseReject = reject + }) + return { + promise, + callback: (err, value) => { + if (err) promiseReject(err) + else promiseResolve(value) + } + } +} + function normalizeValue (value) { return value === null ? undefined : value } From d31fe1ee2e81fd53b1a3b2e9cb94b73cb29f8114 Mon Sep 17 00:00:00 2001 From: MJ Date: Wed, 2 Apr 2025 22:35:12 -0600 Subject: [PATCH 15/32] Async clear --- host.js | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/host.js b/host.js index eb8e24c..12c0b60 100644 --- a/host.js +++ b/host.js @@ -218,10 +218,13 @@ function createRpcStream (db, options, streamOptions) { if (it !== undefined) it.close() } - function onclear (req) { - db.clear(cleanRangeOptions(req.options), function (err) { - callback(req.id, err) - }) + async function onclear (req) { + try { + await db.clear(cleanRangeOptions(req.options)) + return callback(req.id, null) + } catch (err) { + return callback(req.id, err) + } } } } From 84324f961363df3999c4d007329b6e212832a104 Mon Sep 17 00:00:00 2001 From: MJ Date: Tue, 1 Apr 2025 14:17:13 -0600 Subject: [PATCH 16/32] Rename iterator classes for clarity --- guest.js | 4 ++-- host.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/guest.js b/guest.js index 1184da3..22b661a 100644 --- a/guest.js +++ b/guest.js @@ -373,13 +373,13 @@ class ManyLevelGuest extends AbstractLevel { } _iterator (options) { - return new Iterator(this, options) + return new ManyLevelGuestIterator(this, options) } } exports.ManyLevelGuest = ManyLevelGuest -class Iterator extends AbstractIterator { +class ManyLevelGuestIterator extends AbstractIterator { constructor (db, options) { // Need keys to know where to restart if (db[kRetry]) options.keys = true diff --git a/host.js b/host.js index 12c0b60..5796549 100644 --- a/host.js +++ b/host.js @@ -185,7 +185,7 @@ function createRpcStream (db, options, streamOptions) { function oniterator ({ id, seq, options, consumed, bookmark, seek }) { if (iterators.has(id)) return - const it = new Iterator(db, id, seq, options, consumed, encode) + const it = new ManyLevelHostIterator(db, id, seq, options, consumed, encode) iterators.set(id, it) if (seek) { @@ -229,7 +229,7 @@ function createRpcStream (db, options, streamOptions) { } } -class Iterator { +class ManyLevelHostIterator { constructor (db, id, seq, options, consumed, encode) { options = cleanRangeOptions(options) From fb092f1f46d125e3f8f64d8f22bdb2ecafa7a28f Mon Sep 17 00:00:00 2001 From: MJ Date: Wed, 2 Apr 2025 22:12:37 -0600 Subject: [PATCH 17/32] Initial async refactor for iterators --- guest.js | 78 +++++++++++++++++++++++++++++--------------------------- host.js | 14 ++++++---- 2 files changed, 49 insertions(+), 43 deletions(-) diff --git a/guest.js b/guest.js index 22b661a..2c570b0 100644 --- a/guest.js +++ b/guest.js @@ -135,8 +135,7 @@ class ManyLevelGuest extends AbstractLevel { const req = self[kIterators].get(res.id) if (!req || req.iterator[kSeq] !== res.seq) return req.iterator[kPending].push(res) - // TODO: deprecate callbacks - if (req.iterator[kCallback]) req.iterator._next(req.iterator[kCallback]) + if (req.iterator[kCallback]) req.iterator[kCallback](null, res) } function oniteratorend (res) { @@ -144,8 +143,7 @@ class ManyLevelGuest extends AbstractLevel { if (!req || req.iterator[kSeq] !== res.seq) return // https://github.com/Level/abstract-level/issues/19 req.iterator[kEnded] = true - // TODO: deprecate callbacks - if (req.iterator[kCallback]) req.iterator._next(req.iterator[kCallback]) + if (req.iterator[kCallback]) req.iterator[kCallback](null, res) } function oncallback (res) { @@ -390,6 +388,7 @@ class ManyLevelGuestIterator extends AbstractIterator { this[kEnded] = false this[kErrored] = false this[kPending] = [] + this[kCallback] = null this[kSeq] = 0 const req = this[kRequest] = { @@ -442,50 +441,53 @@ class ManyLevelGuestIterator extends AbstractIterator { async _next () { if (this[kRequest].consumed >= this.limit || this[kErrored]) { return - } else if (this[kEnded]) { - return - // TODO: no more callbacks, verify what to do with this[kCallback] - // } else { - // this[kCallback] = callback } - if (this[kPending].length !== 0) { - const next = this[kPending][0] - const req = this[kRequest] + // If nothing is pending, wait for the host to send more data + if (!this[kPending].length) { + const { promise, callback } = promiseFactory() + this[kCallback] = callback + // oniteratordata (in ManyLevelGuest) will use the callback to resolve + // this promise to the data received from the host. + await promise + // This could have ended the iterator + if (this[kEnded] && !this[kPending].length) return + } + const next = this[kPending][0] + const req = this[kRequest] - // TODO: document that error ends the iterator - if (next.error) { - this[kErrored] = true - this[kPending] = [] + // TODO: document that error ends the iterator + if (next.error) { + this[kErrored] = true + this[kPending] = [] - throw new ModuleError('Could not read entry', { - code: next.error - }) - } + throw new ModuleError('Could not read entry', { + code: next.error + }) + } - const consumed = ++req.consumed - const key = req.options.keys ? next.data.shift() : undefined - const val = req.options.values ? next.data.shift() : undefined + const consumed = ++req.consumed + const key = req.options.keys ? next.data.shift() : undefined + const val = req.options.values ? next.data.shift() : undefined - if (next.data.length === 0) { - this[kPending].shift() + if (next.data.length === 0) { + this[kPending].shift() - // Acknowledge receipt. Not needed if we don't want more data. - if (consumed < this.limit) { - this[kAckMessage].consumed = consumed - await this.db[kWrite](this[kAckMessage]) - } + // Acknowledge receipt. Not needed if we don't want more data. + if (consumed < this.limit) { + this[kAckMessage].consumed = consumed + this.db[kWrite](this[kAckMessage]) } + } - // Once we've consumed the result of a seek() it must not get retried - req.seek = null - - if (this.db[kRetry]) { - req.bookmark = key - } + // Once we've consumed the result of a seek() it must not get retried + req.seek = null - // TODO: is this the right return type?? - return val + if (this.db[kRetry]) { + req.bookmark = key } + if (req.options.keys && req.options.values) return [key, val] + if (req.options.keys) return key + if (req.options.values) return val } async _close () { diff --git a/host.js b/host.js index 5796549..62aa771 100644 --- a/host.js +++ b/host.js @@ -23,7 +23,6 @@ const kBusy = Symbol('busy') const kPendingSeek = Symbol('pendingSeek') const kLimit = Symbol('limit') const kReadAhead = Symbol('readAhead') -const noop = () => {} const limbo = Symbol('limbo') // TODO: make use of db.supports manifest @@ -255,7 +254,7 @@ class ManyLevelHostIterator { this.pendingAcks = 0 } - next (first) { + async next (first) { if (this[kBusy] || this[kClosed]) return if (this[kEnded] || this.pendingAcks > 1) return @@ -279,7 +278,12 @@ class ManyLevelHostIterator { if (size <= 0) { process.nextTick(this[kHandleMany], null, []) } else { - this[kIterator].nextv(size, this[kHandleMany]) + try { + const nextVal = await this[kIterator].nextv(size) + this[kHandleMany](null, nextVal) + } catch (err) { + this[kHandleMany](err, []) + } } } @@ -351,10 +355,10 @@ class ManyLevelHostIterator { } } - close () { + async close () { if (this[kClosed]) return this[kClosed] = true - this[kIterator].close(noop) + await this[kIterator].close() } } From 64655213cf6bd8e36e970993d9af2d07a6791d92 Mon Sep 17 00:00:00 2001 From: MJ Date: Wed, 2 Apr 2025 23:31:10 -0600 Subject: [PATCH 18/32] WIP TODOs --- guest.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/guest.js b/guest.js index 2c570b0..2c49573 100644 --- a/guest.js +++ b/guest.js @@ -443,6 +443,8 @@ class ManyLevelGuestIterator extends AbstractIterator { return } // If nothing is pending, wait for the host to send more data + // TODO: except if this[kEnded] is true and nothing is pending, then + // don't wait! Return undefined. if (!this[kPending].length) { const { promise, callback } = promiseFactory() this[kCallback] = callback @@ -466,6 +468,8 @@ class ManyLevelGuestIterator extends AbstractIterator { } const consumed = ++req.consumed + // TODO: might need to add `.toString()` to the key and value to convert from Buffer + // Depends on whether abstract level does that itself or not. const key = req.options.keys ? next.data.shift() : undefined const val = req.options.values ? next.data.shift() : undefined From 469c7db98c0aba08673ea10421df319b782bdfef Mon Sep 17 00:00:00 2001 From: MJ Date: Thu, 3 Apr 2025 13:59:19 -0600 Subject: [PATCH 19/32] Fix infinite wait for more data after ended --- guest.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/guest.js b/guest.js index 2c49573..6cfc27a 100644 --- a/guest.js +++ b/guest.js @@ -445,21 +445,21 @@ class ManyLevelGuestIterator extends AbstractIterator { // If nothing is pending, wait for the host to send more data // TODO: except if this[kEnded] is true and nothing is pending, then // don't wait! Return undefined. - if (!this[kPending].length) { + if (!this[kEnded] && !this[kPending].length) { const { promise, callback } = promiseFactory() this[kCallback] = callback // oniteratordata (in ManyLevelGuest) will use the callback to resolve // this promise to the data received from the host. await promise - // This could have ended the iterator - if (this[kEnded] && !this[kPending].length) return } const next = this[kPending][0] const req = this[kRequest] - // TODO: document that error ends the iterator + // If the host iterator has ended and we have no pending data, we are done. + if (!next && this[kEnded]) return if (next.error) { this[kErrored] = true + this[kEnded] = true this[kPending] = [] throw new ModuleError('Could not read entry', { @@ -468,8 +468,6 @@ class ManyLevelGuestIterator extends AbstractIterator { } const consumed = ++req.consumed - // TODO: might need to add `.toString()` to the key and value to convert from Buffer - // Depends on whether abstract level does that itself or not. const key = req.options.keys ? next.data.shift() : undefined const val = req.options.values ? next.data.shift() : undefined @@ -489,9 +487,7 @@ class ManyLevelGuestIterator extends AbstractIterator { if (this.db[kRetry]) { req.bookmark = key } - if (req.options.keys && req.options.values) return [key, val] - if (req.options.keys) return key - if (req.options.values) return val + return [key, val] } async _close () { From 319df001c18ce3fbb816ef81a07421434ca9f04c Mon Sep 17 00:00:00 2001 From: MJ Date: Wed, 9 Apr 2025 09:32:28 -0600 Subject: [PATCH 20/32] Replace promiseFactory with a more conventional use of promises --- guest.js | 187 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 98 insertions(+), 89 deletions(-) diff --git a/guest.js b/guest.js index 6cfc27a..abfc8c8 100644 --- a/guest.js +++ b/guest.js @@ -215,104 +215,122 @@ class ManyLevelGuest extends AbstractLevel { // TODO: this and other methods assume db state matches our state if (this[kDb]) return this[kDb]._get(key, opts) - const { promise, callback } = promiseFactory() - const req = { - tag: input.get, - id: 0, - key: key, - callback - } + return new Promise((resolve, reject) => { + const req = { + tag: input.get, + id: 0, + key: key, + // This will resolve or reject based on the Host's response + callback: (err, value) => { + if (err) reject(err) + else resolve(value) + } + } - req.id = this[kRequests].add(req) - this[kWrite](req) - // This should resolve or reject based on the Host's response - return promise + req.id = this[kRequests].add(req) + this[kWrite](req) + }) } async _getMany (keys, opts) { if (this[kDb]) return this[kDb]._getMany(keys, opts) - const { promise, callback } = promiseFactory() - const req = { - tag: input.getMany, - id: 0, - keys: keys, - callback - } + return new Promise((resolve, reject) => { + const req = { + tag: input.getMany, + id: 0, + keys: keys, + // This will resolve or reject based on the Host's response + callback: (err, values) => { + if (err) reject(err) + else resolve(values) + } + } - req.id = this[kRequests].add(req) - this[kWrite](req) - // This should resolve or reject based on the Host's response - return promise + req.id = this[kRequests].add(req) + this[kWrite](req) + }) } async _put (key, value, opts) { if (this[kDb]) return this[kDb]._put(key, value, opts) - const { promise, callback } = promiseFactory() - const req = { - tag: input.put, - id: 0, - key: key, - value: value, - callback - } + return new Promise((resolve, reject) => { + const req = { + tag: input.put, + id: 0, + key: key, + value: value, + // This will resolve or reject based on the Host's response + callback: (err) => { + if (err) reject(err) + else resolve() + } + } - req.id = this[kRequests].add(req) - this[kWrite](req) - // This should resolve or reject based on the Host's response - return promise + req.id = this[kRequests].add(req) + this[kWrite](req) + }) } async _del (key, opts) { if (this[kDb]) return this[kDb]._del(key, opts) - const { promise, callback } = promiseFactory() - const req = { - tag: input.del, - id: 0, - key: key, - callback - } + return new Promise((resolve, reject) => { + const req = { + tag: input.del, + id: 0, + key: key, + // This will resolve or reject based on the Host's response + callback: (err) => { + if (err) reject(err) + else resolve() + } + } - req.id = this[kRequests].add(req) - this[kWrite](req) - // This should resolve or reject based on the Host's response - return promise + req.id = this[kRequests].add(req) + this[kWrite](req) + }) } async _batch (batch, opts) { if (this[kDb]) return this[kDb]._batch(batch, opts) - const { promise, callback } = promiseFactory() - const req = { - tag: input.batch, - id: 0, - ops: batch, - callback - } + return new Promise((resolve, reject) => { + const req = { + tag: input.batch, + id: 0, + ops: batch, + // This will resolve or reject based on the Host's response + callback: (err) => { + if (err) reject(err) + else resolve() + } + } - req.id = this[kRequests].add(req) - this[kWrite](req) - // This should resolve or reject based on the Host's response - return promise + req.id = this[kRequests].add(req) + this[kWrite](req) + }) } async _clear (opts) { if (this[kDb]) return this[kDb]._clear(opts) - const { promise, callback } = promiseFactory() - const req = { - tag: input.clear, - id: 0, - options: opts, - callback - } + return new Promise((resolve, reject) => { + const req = { + tag: input.clear, + id: 0, + options: opts, + // This will resolve or reject based on the Host's response + callback: (err) => { + if (err) reject(err) + else resolve() + } + } - req.id = this[kRequests].add(req) - this[kWrite](req) - // This should resolve or reject based on the Host's response - return promise + req.id = this[kRequests].add(req) + this[kWrite](req) + }) } async [kWrite] (req) { @@ -445,12 +463,18 @@ class ManyLevelGuestIterator extends AbstractIterator { // If nothing is pending, wait for the host to send more data // TODO: except if this[kEnded] is true and nothing is pending, then // don't wait! Return undefined. - if (!this[kEnded] && !this[kPending].length) { - const { promise, callback } = promiseFactory() - this[kCallback] = callback - // oniteratordata (in ManyLevelGuest) will use the callback to resolve - // this promise to the data received from the host. - await promise + if (this[kEnded] && !this[kPending].length) { + return undefined + } + // oniteratordata (in ManyLevelGuest) will use the callback to resolve + // this promise to the data received from the host. + if (!this[kPending].length) { + await new Promise((resolve, reject) => { + this[kCallback] = (err, data) => { + if (err) reject(err) + else resolve(data) + } + }) } const next = this[kPending][0] const req = this[kRequest] @@ -477,7 +501,7 @@ class ManyLevelGuestIterator extends AbstractIterator { // Acknowledge receipt. Not needed if we don't want more data. if (consumed < this.limit) { this[kAckMessage].consumed = consumed - this.db[kWrite](this[kAckMessage]) + await this.db[kWrite](this[kAckMessage]) } } @@ -497,21 +521,6 @@ class ManyLevelGuestIterator extends AbstractIterator { } } -function promiseFactory () { - let promiseResolve, promiseReject - const promise = new Promise((resolve, reject) => { - promiseResolve = resolve - promiseReject = reject - }) - return { - promise, - callback: (err, value) => { - if (err) promiseReject(err) - else promiseResolve(value) - } - } -} - function normalizeValue (value) { return value === null ? undefined : value } From ce6f245012231a8dc95312d0876ec5329f2a0ff5 Mon Sep 17 00:00:00 2001 From: MJ Date: Wed, 9 Apr 2025 09:32:42 -0600 Subject: [PATCH 21/32] Abstract Level 2.0 removes support for Node <16 --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 764a753..f465642 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ }, "homepage": "https://github.com/Level/many-level", "engines": { - "node": ">=12" + "node": ">=16" }, "standard": { "ignore": [ From eb9076d7d5acfdb379c8c386cdd878ca0809799f Mon Sep 17 00:00:00 2001 From: MJ Date: Thu, 10 Apr 2025 09:52:53 -0600 Subject: [PATCH 22/32] Promisify pipeline and finished in guest --- guest.js | 61 ++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 42 insertions(+), 19 deletions(-) diff --git a/guest.js b/guest.js index abfc8c8..f8d815a 100644 --- a/guest.js +++ b/guest.js @@ -4,12 +4,14 @@ const { AbstractLevel, AbstractIterator } = require('abstract-level') const lpstream = require('@vweevers/length-prefixed-stream') const ModuleError = require('module-error') const { input, output } = require('./tags') -const { Duplex, pipeline, finished } = require('readable-stream') +const { promises: readablePromises, Duplex } = require('readable-stream') +const { pipeline, finished } = readablePromises const kExplicitClose = Symbol('explicitClose') const kAbortRequests = Symbol('abortRequests') const kEnded = Symbol('kEnded') const kRemote = Symbol('remote') +const kCleanup = Symbol('cleanup') const kAckMessage = Symbol('ackMessage') const kEncode = Symbol('encode') const kRef = Symbol('ref') @@ -45,6 +47,7 @@ class ManyLevelGuest extends AbstractLevel { this[kRetry] = !!retry this[kEncode] = lpstream.encode() this[kRemote] = _remote || null + this[kCleanup] = null this[kRpcStream] = null this[kRef] = null this[kDb] = null @@ -108,12 +111,15 @@ class ManyLevelGuest extends AbstractLevel { }) const proxy = Duplex.from({ writable: decode, readable: encode }) - finished(proxy, cleanup) - this[kRpcStream] = proxy - return proxy - - function cleanup () { + self[kCleanup] = (async () => { + await finished(proxy).catch(err => { + // Abort error is expected on close, which is what triggers finished + if (err.code !== 'ABORT_ERR') { + throw err + } + }) self[kRpcStream] = null + // Create a dummy stream to flush pending requests to self[kEncode] = lpstream.encode() if (!self[kRetry]) { @@ -123,13 +129,15 @@ class ManyLevelGuest extends AbstractLevel { } for (const req of self[kRequests].values()) { - self[kWrite](req) + await self[kWrite](req) } for (const req of self[kIterators].values()) { - self[kWrite](req) + await self[kWrite](req) } - } + })() + self[kRpcStream] = proxy + return proxy function oniteratordata (res) { const req = self[kIterators].get(res.id) @@ -348,13 +356,25 @@ class ManyLevelGuest extends AbstractLevel { this[kAbortRequests]('Aborted on database close()', 'LEVEL_DATABASE_NOT_OPEN') if (this[kRpcStream]) { - // TODO: need to do something with finished. Can we use readable-stream.promises or whatever?? That would make life much easier... - finished(this[kRpcStream], () => { - this[kRpcStream] = null - this._close() + try { + this[kRpcStream].destroy() + } catch (err) { + // Abort error is expected on close + if (err.code !== 'ABORT_ERR') { + throw err + } + } + await finished(this[kRpcStream]).catch(err => { + // Abort error is expected on close + if (err.code !== 'ABORT_ERR') { + throw err + } }) - this[kRpcStream].destroy() - } else if (this[kDb]) { + if (this[kCleanup]) await this[kCleanup] + this[kRpcStream] = null + this[kCleanup] = null + } + if (this[kDb]) { // To be safe, use close() not _close(). return this[kDb].close() } @@ -365,13 +385,16 @@ class ManyLevelGuest extends AbstractLevel { // For tests only so does not need error handling this[kExplicitClose] = false const remote = this[kRemote]() - // TODO: Need to promisify pipeline pipeline( remote, this.connect(), - remote, - () => {} - ) + remote + ).catch(err => { + // TODO: proper abort handling + if (err.code === 'ABORT_ERR') { + return this.close() + } + }) } else if (this[kExplicitClose]) { throw new ModuleError('Cannot reopen many-level database after close()', { code: 'LEVEL_NOT_SUPPORTED' From 89aaa1d71f9eaf0fd7cdb9350131e9bce280091c Mon Sep 17 00:00:00 2001 From: MJ Date: Thu, 10 Apr 2025 13:52:48 -0600 Subject: [PATCH 23/32] Promisify stream finished in host --- host.js | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/host.js b/host.js index 62aa771..8b446ca 100644 --- a/host.js +++ b/host.js @@ -2,7 +2,8 @@ const lpstream = require('@vweevers/length-prefixed-stream') const ModuleError = require('module-error') -const { Duplex, finished } = require('readable-stream') +const { Duplex, promises: readablePromises } = require('readable-stream') +const { finished } = readablePromises const { input, output } = require('./tags') const rangeOptions = new Set(['gt', 'gte', 'lt', 'lte']) @@ -83,13 +84,21 @@ function createRpcStream (db, options, streamOptions) { const iterators = new Map() - finished(stream, function () { + const cleanup = async () => { + await finished(stream).catch(err => { + // Abort error is expected on close, which is what triggers finished + if (err.code !== 'ABORT_ERR') { + throw err + } + }) for (const iterator of iterators.values()) { iterator.close() } iterators.clear() - }) + } + // Don't await + cleanup() decode.on('data', function (data) { if (!data.length) return From fc36e0e4719426f4eab590d8cfbb58d9d89ffd1d Mon Sep 17 00:00:00 2001 From: MJ Date: Thu, 17 Apr 2025 19:44:35 -0600 Subject: [PATCH 24/32] Refactor to avoid uncaught exceptions --- guest.js | 21 +++++---------------- host.js | 7 +------ 2 files changed, 6 insertions(+), 22 deletions(-) diff --git a/guest.js b/guest.js index f8d815a..87ec1de 100644 --- a/guest.js +++ b/guest.js @@ -114,8 +114,8 @@ class ManyLevelGuest extends AbstractLevel { self[kCleanup] = (async () => { await finished(proxy).catch(err => { // Abort error is expected on close, which is what triggers finished - if (err.code !== 'ABORT_ERR') { - throw err + if (err.code === 'ABORT_ERR') { + // TODO: abort in-flight ops } }) self[kRpcStream] = null @@ -356,20 +356,9 @@ class ManyLevelGuest extends AbstractLevel { this[kAbortRequests]('Aborted on database close()', 'LEVEL_DATABASE_NOT_OPEN') if (this[kRpcStream]) { - try { - this[kRpcStream].destroy() - } catch (err) { - // Abort error is expected on close - if (err.code !== 'ABORT_ERR') { - throw err - } - } - await finished(this[kRpcStream]).catch(err => { - // Abort error is expected on close - if (err.code !== 'ABORT_ERR') { - throw err - } - }) + const finishedPromise = finished(this[kRpcStream]).catch(() => null) + this[kRpcStream].destroy().catch(() => null) + await finishedPromise if (this[kCleanup]) await this[kCleanup] this[kRpcStream] = null this[kCleanup] = null diff --git a/host.js b/host.js index 8b446ca..74f10fd 100644 --- a/host.js +++ b/host.js @@ -85,12 +85,7 @@ function createRpcStream (db, options, streamOptions) { const iterators = new Map() const cleanup = async () => { - await finished(stream).catch(err => { - // Abort error is expected on close, which is what triggers finished - if (err.code !== 'ABORT_ERR') { - throw err - } - }) + await finished(stream).catch(() => null) for (const iterator of iterators.values()) { iterator.close() } From a19ef44eb936cc7541445d8ee29d85afd284f8a2 Mon Sep 17 00:00:00 2001 From: MJ Date: Wed, 16 Apr 2025 19:04:20 -0600 Subject: [PATCH 25/32] Use createRpcStream directly instead of the back-compat method --- guest.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/guest.js b/guest.js index 87ec1de..466d660 100644 --- a/guest.js +++ b/guest.js @@ -376,7 +376,7 @@ class ManyLevelGuest extends AbstractLevel { const remote = this[kRemote]() pipeline( remote, - this.connect(), + this.createRpcStream(), remote ).catch(err => { // TODO: proper abort handling From 2fa5dcf1438ad0c416da8881bcbe25905c909346 Mon Sep 17 00:00:00 2001 From: MJ Date: Wed, 7 May 2025 12:27:15 -0600 Subject: [PATCH 26/32] Clean up done TODOs --- guest.js | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/guest.js b/guest.js index 466d660..11bac0f 100644 --- a/guest.js +++ b/guest.js @@ -203,10 +203,9 @@ class ManyLevelGuest extends AbstractLevel { req.callback(new ModuleError(msg, { code })) } - // TODO: iterators for (const req of this[kIterators].clear()) { // Cancel in-flight operation if any - // TODO: do we need a new mechanism to pass the error back up to the request initiator? + // TODO: does this need to be refactored to use AbortError to pass back up to the request initiator? const callback = req.iterator[kCallback] req.iterator[kCallback] = null @@ -379,7 +378,6 @@ class ManyLevelGuest extends AbstractLevel { this.createRpcStream(), remote ).catch(err => { - // TODO: proper abort handling if (err.code === 'ABORT_ERR') { return this.close() } @@ -473,7 +471,7 @@ class ManyLevelGuestIterator extends AbstractIterator { return } // If nothing is pending, wait for the host to send more data - // TODO: except if this[kEnded] is true and nothing is pending, then + // except if this[kEnded] is true and nothing is pending, then // don't wait! Return undefined. if (this[kEnded] && !this[kPending].length) { return undefined From 7adbb795f0114ca3933fe24bee2145ea3b0aeb51 Mon Sep 17 00:00:00 2001 From: MJ Date: Tue, 11 Nov 2025 12:57:40 -0600 Subject: [PATCH 27/32] kRpcStream.destroy() is not a promise, needs conventional try..catch --- guest.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/guest.js b/guest.js index 11bac0f..4558018 100644 --- a/guest.js +++ b/guest.js @@ -356,7 +356,11 @@ class ManyLevelGuest extends AbstractLevel { if (this[kRpcStream]) { const finishedPromise = finished(this[kRpcStream]).catch(() => null) - this[kRpcStream].destroy().catch(() => null) + try { + this[kRpcStream].destroy() + } catch { + // nothing goes here + } await finishedPromise if (this[kCleanup]) await this[kCleanup] this[kRpcStream] = null From d7729a00af1107eb916daac6bffda12980253970 Mon Sep 17 00:00:00 2001 From: Germano Eichenberg Date: Wed, 17 Jun 2026 19:09:02 -0300 Subject: [PATCH 28/32] Support abstract-level 3 test semantics --- guest.js | 33 ++-- package.json | 4 +- test/basic.js | 275 ++++++++++++-------------------- test/concurrent.js | 47 +++--- test/iterator-error.js | 5 +- test/retry.js | 345 +++++++++++++++++++++-------------------- test/streams.js | 28 ++-- test/sublevel.js | 171 ++++++++++---------- 8 files changed, 411 insertions(+), 497 deletions(-) diff --git a/guest.js b/guest.js index 4558018..434dd46 100644 --- a/guest.js +++ b/guest.js @@ -27,7 +27,8 @@ const kPending = Symbol('pending') const kCallback = Symbol('callback') const kSeq = Symbol('seq') const kErrored = Symbol('errored') -const noop = function () {} +const kAbortIterator = Symbol('abortIterator') +const kOnDbClosing = Symbol('onDbClosing') class ManyLevelGuest extends AbstractLevel { constructor (options) { @@ -204,17 +205,7 @@ class ManyLevelGuest extends AbstractLevel { } for (const req of this[kIterators].clear()) { - // Cancel in-flight operation if any - // TODO: does this need to be refactored to use AbortError to pass back up to the request initiator? - const callback = req.iterator[kCallback] - req.iterator[kCallback] = null - - if (callback) { - callback(new ModuleError(msg, { code })) - } - - // Note: an in-flight operation would block close() - req.iterator.close(noop) + req.iterator[kAbortIterator](code) } } @@ -377,13 +368,14 @@ class ManyLevelGuest extends AbstractLevel { // For tests only so does not need error handling this[kExplicitClose] = false const remote = this[kRemote]() + const local = this.createRpcStream() pipeline( remote, - this.createRpcStream(), + local, remote ).catch(err => { if (err.code === 'ABORT_ERR') { - return this.close() + if (!this[kExplicitClose] && this[kRpcStream] === local) return this.close() } }) } else if (this[kExplicitClose]) { @@ -422,6 +414,8 @@ class ManyLevelGuestIterator extends AbstractIterator { this[kPending] = [] this[kCallback] = null this[kSeq] = 0 + this[kOnDbClosing] = () => this[kAbortIterator]('LEVEL_ITERATOR_NOT_OPEN') + db.once('closing', this[kOnDbClosing]) const req = this[kRequest] = { tag: input.iterator, @@ -529,10 +523,21 @@ class ManyLevelGuestIterator extends AbstractIterator { } async _close () { + this.db.removeListener('closing', this[kOnDbClosing]) await this.db[kWrite]({ tag: input.iteratorClose, id: this[kRequest].id }) this.db[kIterators].remove(this[kRequest].id) this.db[kFlushed]() } + + [kAbortIterator] (code) { + this[kPending].push({ error: code }) + this[kEnded] = true + + const callback = this[kCallback] + this[kCallback] = null + + if (callback) callback(null) + } } function normalizeValue (value) { diff --git a/package.json b/package.json index f465642..42120e3 100644 --- a/package.json +++ b/package.json @@ -27,7 +27,7 @@ ], "dependencies": { "@vweevers/length-prefixed-stream": "^1.0.0", - "abstract-level": "^2.0.2", + "abstract-level": "^3.0.0", "module-error": "^1.0.2", "protocol-buffers-encodings": "^1.1.0", "readable-stream": "^4.0.0" @@ -40,7 +40,7 @@ "faucet": "^0.0.3", "hallmark": "^4.0.0", "level-read-stream": "^1.1.0", - "memory-level": "^2.0.0", + "memory-level": "^3.0.0", "nyc": "^15.1.0", "protocol-buffers": "^5.0.0", "standard": "^16.0.3", diff --git a/test/basic.js b/test/basic.js index 6b18457..fadd31a 100644 --- a/test/basic.js +++ b/test/basic.js @@ -7,8 +7,15 @@ const { pipeline } = require('readable-stream') const concat = require('concat-stream') const { ManyLevelHost, ManyLevelGuest } = require('..') -tape('get', function (t) { - t.plan(7) +function collect (stream) { + return new Promise((resolve, reject) => { + stream.on('error', reject) + stream.pipe(concat(resolve)) + }) +} + +tape('get', async function (t) { + t.plan(3) const db = new MemoryLevel() const host = new ManyLevelHost(db) @@ -17,27 +24,14 @@ tape('get', function (t) { stream.pipe(guest.createRpcStream()).pipe(stream) - db.put('hello', 'world', function (err) { - t.error(err, 'no err') + await db.put('hello', 'world') - guest.get('hello', function (err, value) { - t.error(err, 'no err') - t.same(value, 'world') - }) - - guest.get(Buffer.from('hello'), function (err, value) { - t.error(err, 'no err') - t.same(value, 'world') - }) - - guest.get('hello', { valueEncoding: 'buffer' }, function (err, value) { - t.error(err, 'no err') - t.same(value, Buffer.from('world')) - }) - }) + t.same(await guest.get('hello'), 'world') + t.same(await guest.get(Buffer.from('hello')), 'world') + t.same(await guest.get('hello', { valueEncoding: 'buffer' }), Buffer.from('world')) }) -tape('get with valueEncoding: json in constructor', function (t) { +tape('get with valueEncoding: json in constructor', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream() @@ -45,18 +39,11 @@ tape('get with valueEncoding: json in constructor', function (t) { stream.pipe(guest.createRpcStream()).pipe(stream) - db.put('hello', '{"foo":"world"}', function () { - guest.get('hello', function (err, value) { - t.error(err, 'no err') - t.same(value, { foo: 'world' }) - t.end() - }) - }) + await db.put('hello', '{"foo":"world"}') + t.same(await guest.get('hello'), { foo: 'world' }) }) -tape('get with valueEncoding: json in get options', function (t) { - t.plan(5) - +tape('get with valueEncoding: json in get options', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream() @@ -64,22 +51,13 @@ tape('get with valueEncoding: json in get options', function (t) { stream.pipe(guest.createRpcStream()).pipe(stream) - db.put('hello', '{"foo":"world"}', function (err) { - t.error(err, 'no err') - - guest.get('hello', { valueEncoding: 'json' }, function (err, value) { - t.error(err, 'no err') - t.same(value, { foo: 'world' }) - }) + await db.put('hello', '{"foo":"world"}') - guest.get('hello', function (err, value) { - t.error(err, 'no err') - t.same(value, '{"foo":"world"}') - }) - }) + t.same(await guest.get('hello', { valueEncoding: 'json' }), { foo: 'world' }) + t.same(await guest.get('hello'), '{"foo":"world"}') }) -tape('put', function (t) { +tape('put', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream() @@ -87,19 +65,11 @@ tape('put', function (t) { stream.pipe(guest.createRpcStream()).pipe(stream) - guest.put('hello', 'world', function (err) { - t.error(err, 'no err') - guest.get('hello', function (err, value) { - t.error(err, 'no err') - t.same(value, 'world') - t.end() - }) - }) + await guest.put('hello', 'world') + t.same(await guest.get('hello'), 'world') }) -tape('put with valueEncoding: json in constructor', function (t) { - t.plan(5) - +tape('put with valueEncoding: json in constructor', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream() @@ -107,24 +77,13 @@ tape('put with valueEncoding: json in constructor', function (t) { stream.pipe(guest.createRpcStream()).pipe(stream) - guest.put('hello', { foo: 'world' }, function (err) { - t.error(err, 'no err') - - db.get('hello', function (err, value) { - t.error(err, 'no err') - t.same(value, '{"foo":"world"}') - }) + await guest.put('hello', { foo: 'world' }) - guest.get('hello', function (err, value) { - t.error(err, 'no err') - t.same(value, { foo: 'world' }) - }) - }) + t.same(await db.get('hello'), '{"foo":"world"}') + t.same(await guest.get('hello'), { foo: 'world' }) }) -tape('put with valueEncoding: json in put options', function (t) { - t.plan(5) - +tape('put with valueEncoding: json in put options', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream() @@ -132,19 +91,10 @@ tape('put with valueEncoding: json in put options', function (t) { stream.pipe(guest.createRpcStream()).pipe(stream) - guest.put('hello', { foo: 'world' }, { valueEncoding: 'json' }, function (err) { - t.error(err, 'no err') - - db.get('hello', function (err, value) { - t.error(err, 'no err') - t.same(value, '{"foo":"world"}') - }) + await guest.put('hello', { foo: 'world' }, { valueEncoding: 'json' }) - guest.get('hello', function (err, value) { - t.error(err, 'no err') - t.same(value, '{"foo":"world"}') - }) - }) + t.same(await db.get('hello'), '{"foo":"world"}') + t.same(await guest.get('hello'), '{"foo":"world"}') }) tape('readonly', async function (t) { @@ -168,7 +118,7 @@ tape('readonly', async function (t) { t.is(await guest.get('hello'), 'verden', 'old value') }) -tape('del', function (t) { +tape('del', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream() @@ -176,19 +126,17 @@ tape('del', function (t) { stream.pipe(guest.createRpcStream()).pipe(stream) - guest.put('hello', 'world', function (err) { - t.error(err, 'no err') - guest.del('hello', function (err) { - t.error(err, 'no err') - guest.get('hello', function (err) { - t.is(err && err.code, 'LEVEL_NOT_FOUND') - t.end() - }) - }) - }) + await guest.put('hello', 'world') + await guest.del('hello') + + try { + await guest.get('hello') + } catch (err) { + t.is(err && err.code, 'LEVEL_NOT_FOUND') + } }) -tape('batch', function (t) { +tape('batch', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream() @@ -196,21 +144,13 @@ tape('batch', function (t) { stream.pipe(guest.createRpcStream()).pipe(stream) - guest.batch([{ type: 'put', key: 'hello', value: 'world' }, { type: 'put', key: 'hej', value: 'verden' }], function (err) { - t.error(err, 'no err') - guest.get('hello', function (err, value) { - t.error(err, 'no err') - t.same(value, 'world') - guest.get('hej', function (err, value) { - t.error(err, 'no err') - t.same(value, 'verden') - t.end() - }) - }) - }) + await guest.batch([{ type: 'put', key: 'hello', value: 'world' }, { type: 'put', key: 'hej', value: 'verden' }]) + + t.same(await guest.get('hello'), 'world') + t.same(await guest.get('hej'), 'verden') }) -tape('read stream', function (t) { +tape('read stream', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream() @@ -218,19 +158,15 @@ tape('read stream', function (t) { stream.pipe(guest.createRpcStream()).pipe(stream) - guest.batch([{ type: 'put', key: 'hello', value: 'world' }, { type: 'put', key: 'hej', value: 'verden' }], function (err) { - t.error(err, 'no err') - const rs = new EntryStream(guest) - rs.pipe(concat(function (entries) { - t.same(entries.length, 2) - t.same(entries[0], { key: 'hej', value: 'verden' }) - t.same(entries[1], { key: 'hello', value: 'world' }) - t.end() - })) - }) + await guest.batch([{ type: 'put', key: 'hello', value: 'world' }, { type: 'put', key: 'hej', value: 'verden' }]) + + const entries = await collect(new EntryStream(guest)) + t.same(entries.length, 2) + t.same(entries[0], { key: 'hej', value: 'verden' }) + t.same(entries[1], { key: 'hello', value: 'world' }) }) -tape('read stream (gt)', function (t) { +tape('read stream (gt)', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream() @@ -238,15 +174,11 @@ tape('read stream (gt)', function (t) { stream.pipe(guest.createRpcStream()).pipe(stream) - guest.batch([{ type: 'put', key: 'hello', value: 'world' }, { type: 'put', key: 'hej', value: 'verden' }], function (err) { - t.error(err, 'no err') - const rs = new EntryStream(guest, { gt: 'hej' }) - rs.pipe(concat(function (entries) { - t.same(entries.length, 1) - t.same(entries[0], { key: 'hello', value: 'world' }) - t.end() - })) - }) + await guest.batch([{ type: 'put', key: 'hello', value: 'world' }, { type: 'put', key: 'hej', value: 'verden' }]) + + const entries = await collect(new EntryStream(guest, { gt: 'hej' })) + t.same(entries.length, 1) + t.same(entries[0], { key: 'hello', value: 'world' }) }) tape('for await...of iterator', async function (t) { @@ -267,23 +199,26 @@ tape('for await...of iterator', async function (t) { t.same(entries, [['hej', 'verden'], ['hello', 'world']]) }) -tape('close with pending request', function (t) { +tape('close with pending request', async function (t) { t.plan(2) const guest = new ManyLevelGuest() - guest.put('hello', 'world', function (err) { + const pending = guest.put('hello', 'world').catch(async function (err) { t.is(err && err.code, 'LEVEL_DATABASE_NOT_OPEN') - guest.put('hello', 'world', function (err) { + try { + await guest.put('hello', 'world') + } catch (err) { t.is(err && err.code, 'LEVEL_DATABASE_NOT_OPEN') - }) + } }) - guest.close() + await guest.close() + await pending }) -tape('disconnect with pending request', function (t) { +tape('disconnect with pending request', async function (t) { t.plan(3) const db = new MemoryLevel() @@ -293,49 +228,41 @@ tape('disconnect with pending request', function (t) { pipeline(stream, guest.createRpcStream(), stream, () => {}) - db.open(function (err) { - t.ifError(err) - - guest.open(function (err) { - t.ifError(err) + await db.open() + t.pass('db opened') - guest.put('hello', 'world', function (err) { - t.is(err && err.code, 'LEVEL_CONNECTION_LOST') + await guest.open() + t.pass('guest opened') - // TODO: what are we expecting here? - // guest.put('hello', 'world', function (err) { - // t.is(err && err.code, ?) - // }) - }) - - stream.destroy() - }) + const pending = guest.put('hello', 'world').catch(function (err) { + t.is(err && err.code, 'LEVEL_CONNECTION_LOST') }) + + stream.destroy() + await pending }) -tape('close with pending iterator', function (t) { - t.plan(3) +tape('close with pending iterator', async function (t) { + t.plan(2) const guest = new ManyLevelGuest() - guest.open(function (err) { - t.ifError(err) - - const it = guest.iterator() + await guest.open() + t.pass('guest opened') - it.next(function (err) { - t.is(err && err.code, 'LEVEL_ITERATOR_NOT_OPEN') + const it = guest.iterator() - it.next(function (err) { - t.is(err && err.code, 'LEVEL_ITERATOR_NOT_OPEN') - }) - }) - - guest.close() + const pending = it.next().then(function (value) { + t.is(value, undefined) + }, function (err) { + t.is(err && err.code, 'LEVEL_ITERATOR_NOT_OPEN') }) + + await guest.close() + await pending }) -tape('disconnect with pending iterator', function (t) { +tape('disconnect with pending iterator', async function (t) { t.plan(3) const db = new MemoryLevel() @@ -345,22 +272,16 @@ tape('disconnect with pending iterator', function (t) { pipeline(stream, guest.createRpcStream(), stream, () => {}) - db.open(function (err) { - t.ifError(err) - - guest.open(function (err) { - t.ifError(err) + await db.open() + t.pass('db opened') - guest.iterator().next(function (err) { - t.is(err && err.code, 'LEVEL_CONNECTION_LOST') + await guest.open() + t.pass('guest opened') - // TODO: what are we expecting here? - // guest.iterator().next(function (err) { - // t.is(err && err.code, ?) - // }) - }) - - stream.destroy() - }) + const pending = guest.iterator().next().catch(function (err) { + t.is(err && err.code, 'LEVEL_CONNECTION_LOST') }) + + stream.destroy() + await pending }) diff --git a/test/concurrent.js b/test/concurrent.js index 5375ee1..d6a5b65 100644 --- a/test/concurrent.js +++ b/test/concurrent.js @@ -6,7 +6,14 @@ const { EntryStream } = require('level-read-stream') const concat = require('concat-stream') const { ManyLevelHost, ManyLevelGuest } = require('..') -tape('two concurrent iterators', function (t) { +function collect (stream) { + return new Promise((resolve, reject) => { + stream.on('error', reject) + stream.pipe(concat(resolve)) + }) +} + +tape('two concurrent iterators', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream() @@ -20,23 +27,17 @@ tape('two concurrent iterators', function (t) { batch.push({ type: 'put', key: 'key-' + i, value: 'value-' + i }) } - guest.batch(batch, function (err) { - t.error(err) + await guest.batch(batch) - const rs1 = new EntryStream(guest) - const rs2 = new EntryStream(guest) + const rs1 = new EntryStream(guest) + const rs2 = new EntryStream(guest) + const [list1, list2] = await Promise.all([collect(rs1), collect(rs2)]) - rs1.pipe(concat(function (list1) { - t.same(list1.length, 100) - rs2.pipe(concat(function (list2) { - t.same(list2.length, 100) - t.end() - })) - })) - }) + t.same(list1.length, 100) + t.same(list2.length, 100) }) -tape('two concurrent guests', function (t) { +tape('two concurrent guests', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream1 = host.createRpcStream() @@ -50,18 +51,12 @@ tape('two concurrent guests', function (t) { const batch = [] for (let i = 0; i < 100; i++) batch.push({ type: 'put', key: 'key-' + i, value: 'value-' + i }) - guest1.batch(batch, function (err) { - t.error(err) + await guest1.batch(batch) - const rs1 = new EntryStream(guest1) - const rs2 = new EntryStream(guest2) + const rs1 = new EntryStream(guest1) + const rs2 = new EntryStream(guest2) + const [list1, list2] = await Promise.all([collect(rs1), collect(rs2)]) - rs1.pipe(concat(function (list1) { - t.same(list1.length, 100) - rs2.pipe(concat(function (list2) { - t.same(list2.length, 100) - t.end() - })) - })) - }) + t.same(list1.length, 100) + t.same(list2.length, 100) }) diff --git a/test/iterator-error.js b/test/iterator-error.js index f71f9d1..a1b1797 100644 --- a/test/iterator-error.js +++ b/test/iterator-error.js @@ -25,9 +25,10 @@ test('iterator.next() error', async function (t) { db._iterator = function (options) { const it = original.call(this, options) - it._nextv = function (size, options, cb) { - this.nextTick(cb, first ? new Error('foo') : new ModuleError('bar', { code: 'LEVEL_XYZ' })) + it._nextv = async function () { + const err = first ? new Error('foo') : new ModuleError('bar', { code: 'LEVEL_XYZ' }) first = false + throw err } return it diff --git a/test/retry.js b/test/retry.js index 077d2c3..e5a2088 100644 --- a/test/retry.js +++ b/test/retry.js @@ -6,72 +6,88 @@ const { EntryStream } = require('level-read-stream') const { pipeline } = require('readable-stream') const { ManyLevelHost, ManyLevelGuest } = require('..') -tape('retry get', function (t) { +function reconnect (host, guest, isDone, onconnect) { + if (isDone()) return + + let local + try { + local = guest.createRpcStream() + } catch (err) { + if (err.message === 'Only one rpc stream can be active') { + setImmediate(() => reconnect(host, guest, isDone, onconnect)) + return + } + + throw err + } + + const remote = host.createRpcStream() + onconnect(local) + pipeline(remote, local, remote, () => { + setImmediate(() => reconnect(host, guest, isDone, onconnect)) + }) +} + +tape('retry get', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream() const guest = new ManyLevelGuest({ retry: true }) - db.put('hello', 'world', function () { - guest.get('hello', function (err, value) { - t.error(err, 'no err') - t.same(value, 'world') - t.end() - }) + await db.put('hello', 'world') - stream.pipe(guest.createRpcStream()).pipe(stream) - }) + const pending = guest.get('hello') + stream.pipe(guest.createRpcStream()).pipe(stream) + + t.same(await pending, 'world') }) -tape('no retry get', function (t) { +tape('no retry get', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream() const guest = new ManyLevelGuest({ retry: false }) - guest.open(function () { - db.put('hello', 'world', function () { - guest.get('hello', function (err, value) { - t.ok(err, 'had error') - t.end() - }) - - const rpc = guest.createRpcStream() - stream.pipe(rpc).pipe(stream) - rpc.destroy() - - setTimeout(function () { - const rpc = guest.createRpcStream() - stream.pipe(rpc).pipe(stream) - }, 100) - }) + await guest.open() + await db.put('hello', 'world') + + const pending = guest.get('hello').catch(function (err) { + t.ok(err, 'had error') }) + + const rpc = guest.createRpcStream() + stream.pipe(rpc).pipe(stream) + rpc.destroy() + + setTimeout(function () { + const rpc = guest.createRpcStream() + stream.pipe(rpc).pipe(stream) + }, 100) + + await pending }) -tape('retry get', function (t) { +tape('retry get', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream() const guest = new ManyLevelGuest({ retry: true }) - guest.open(function () { - db.put('hello', 'world', function () { - guest.get('hello', function (err, value) { - t.error(err, 'no err') - t.same(value, 'world') - t.end() - }) - - const rpc = guest.createRpcStream() - stream.pipe(rpc).pipe(stream) - rpc.destroy() - - setTimeout(function () { - const rpc = guest.createRpcStream() - stream.pipe(rpc).pipe(stream) - }, 100) - }) - }) + await guest.open() + await db.put('hello', 'world') + + const pending = guest.get('hello') + + const rpc = guest.createRpcStream() + stream.pipe(rpc).pipe(stream) + rpc.destroy() + + setTimeout(function () { + const rpc = guest.createRpcStream() + stream.pipe(rpc).pipe(stream) + }, 100) + + t.same(await pending, 'world') }) for (const reverse of [false, true]) { @@ -99,31 +115,22 @@ for (const reverse of [false, true]) { const it = original.call(this, options) const nextv = it._nextv - it._nextv = function (size, options, cb) { + it._nextv = async function (size, options) { if (n++ > entryCount * 10) throw new Error('Infinite loop') - nextv.call(this, size, options, (...args) => { - setTimeout(cb.bind(null, ...args), 10) - }) + const entries = await nextv.call(this, size, options) + await new Promise(resolve => setTimeout(resolve, 10)) + return entries } return it } // (Re)connect every 50ms - ;(function connect () { - if (done) return - + reconnect(host, guest, () => done, function (local) { attempts++ - - const remote = host.createRpcStream() - const local = guest.createRpcStream() - - // TODO: calls back too soon if you destroy remote instead of local, - // because duplexify does not satisfy node's willEmitClose() check - pipeline(remote, local, remote, connect) setTimeout(local.destroy.bind(local), 50) - })() + }) const entries = await guest.iterator({ gte: '1'.padStart(padding, '0'), reverse }).all() done = true @@ -169,13 +176,10 @@ for (const reverse of [false, true]) { // Don't test deferredOpen await guest.open() - ;(function connect () { - if (done) return + reconnect(host, guest, () => done, function (nextLocal) { attempts++ - const remote = host.createRpcStream() - local = guest.createRpcStream() - pipeline(remote, local, remote, connect) - })() + local = nextLocal + }) // Wait for first connection await guest.get(Buffer.alloc(0)) @@ -213,13 +217,10 @@ tape('retry value iterator', async function (t) { return { type: 'put', key, value } })) - ;(function connect () { - if (done) return + reconnect(host, guest, () => done, function (nextLocal) { attempts++ - const remote = host.createRpcStream() - local = guest.createRpcStream() - pipeline(remote, local, remote, connect) - })() + local = nextLocal + }) // Wait for first connection await guest.get('a') @@ -259,13 +260,10 @@ for (const reverse of [false, true]) { // Don't test deferredOpen await guest.open() - ;(function connect () { - if (done) return + reconnect(host, guest, () => done, function (nextLocal) { attempts++ - const remote = host.createRpcStream() - local = guest.createRpcStream() - pipeline(remote, local, remote, connect) - })() + local = nextLocal + }) const result = [] const it = guest.keys({ reverse }) @@ -306,13 +304,10 @@ for (const reverse of [false, true]) { // Don't test deferredOpen await guest.open() - ;(function connect () { - if (done) return + reconnect(host, guest, () => done, function (nextLocal) { attempts++ - const remote = host.createRpcStream() - local = guest.createRpcStream() - pipeline(remote, local, remote, connect) - })() + local = nextLocal + }) const result = [] const it = guest.keys({ reverse }) @@ -347,107 +342,115 @@ for (const reverse of [false, true]) { }) } -tape('retry read stream', function (t) { +tape('retry read stream', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const guest = new ManyLevelGuest({ retry: true }) - guest.open(function () { - db.batch([{ - type: 'put', - key: 'hej', - value: 'verden' - }, { - type: 'put', - key: 'hello', - value: 'world' - }, { - type: 'put', - key: 'hola', - value: 'mundo' - }], function () { - const rs = new EntryStream(guest) - const expected = [{ - key: 'hej', - value: 'verden' - }, { - key: 'hello', - value: 'world' - }, { - key: 'hola', - value: 'mundo' - }] - - rs.on('data', function (data) { - t.same(data, expected.shift(), 'stream continues over retry') - }) - - rs.on('end', function () { - t.same(expected.length, 0, 'no more data') - t.end() - }) - - let stream - let guestStream - - const connect = function () { - stream = host.createRpcStream() - guestStream = guest.createRpcStream() - stream.pipe(guestStream).pipe(stream) - } + await guest.open() + await db.batch([{ + type: 'put', + key: 'hej', + value: 'verden' + }, { + type: 'put', + key: 'hello', + value: 'world' + }, { + type: 'put', + key: 'hola', + value: 'mundo' + }]) + + const rs = new EntryStream(guest) + const expected = [{ + key: 'hej', + value: 'verden' + }, { + key: 'hello', + value: 'world' + }, { + key: 'hola', + value: 'mundo' + }] + + const ended = new Promise((resolve, reject) => { + rs.on('data', function (data) { + t.same(data, expected.shift(), 'stream continues over retry') + }) - connect() + rs.on('end', function () { + t.same(expected.length, 0, 'no more data') + resolve() }) + + rs.on('error', reject) }) + + let stream + let guestStream + + const connect = function () { + stream = host.createRpcStream() + guestStream = guest.createRpcStream() + stream.pipe(guestStream).pipe(stream) + } + + connect() + await ended }) -tape('retry read stream and limit', function (t) { +tape('retry read stream and limit', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const guest = new ManyLevelGuest({ retry: true }) - guest.open(function () { - db.batch([{ - type: 'put', - key: 'hej', - value: 'verden' - }, { - type: 'put', - key: 'hello', - value: 'world' - }, { - type: 'put', - key: 'hola', - value: 'mundo' - }], function () { - const rs = new EntryStream(guest, { limit: 2 }) - const expected = [{ - key: 'hej', - value: 'verden' - }, { - key: 'hello', - value: 'world' - }] - - rs.on('data', function (data) { - t.same(data, expected.shift(), 'stream continues over retry') - }) - - rs.on('end', function () { - t.same(expected.length, 0, 'no more data') - t.end() - }) - - let stream - let guestStream - - const connect = function () { - stream = host.createRpcStream() - guestStream = guest.createRpcStream() - stream.pipe(guestStream).pipe(stream) - } + await guest.open() + await db.batch([{ + type: 'put', + key: 'hej', + value: 'verden' + }, { + type: 'put', + key: 'hello', + value: 'world' + }, { + type: 'put', + key: 'hola', + value: 'mundo' + }]) + + const rs = new EntryStream(guest, { limit: 2 }) + const expected = [{ + key: 'hej', + value: 'verden' + }, { + key: 'hello', + value: 'world' + }] + + const ended = new Promise((resolve, reject) => { + rs.on('data', function (data) { + t.same(data, expected.shift(), 'stream continues over retry') + }) - connect() + rs.on('end', function () { + t.same(expected.length, 0, 'no more data') + resolve() }) + + rs.on('error', reject) }) + + let stream + let guestStream + + const connect = function () { + stream = host.createRpcStream() + guestStream = guest.createRpcStream() + stream.pipe(guestStream).pipe(stream) + } + + connect() + await ended }) diff --git a/test/streams.js b/test/streams.js index f89ffe3..f5f3573 100644 --- a/test/streams.js +++ b/test/streams.js @@ -6,7 +6,14 @@ const { EntryStream } = require('level-read-stream') const concat = require('concat-stream') const { ManyLevelHost, ManyLevelGuest } = require('..') -tape('two concurrent iterators', function (t) { +function collect (stream) { + return new Promise((resolve, reject) => { + stream.on('error', reject) + stream.pipe(concat(resolve)) + }) +} + +tape('two concurrent iterators', async function (t) { const db = new MemoryLevel() const host = new ManyLevelHost(db) const stream = host.createRpcStream(db) @@ -17,19 +24,12 @@ tape('two concurrent iterators', function (t) { const batch = [] for (let i = 0; i < 100; i++) batch.push({ type: 'put', key: 'key-' + i, value: 'value-' + i }) - guest.batch(batch, function (err) { - t.error(err) + await guest.batch(batch) - // TODO: use iterator.all() instead - const rs1 = new EntryStream(guest) - const rs2 = new EntryStream(guest) + const rs1 = new EntryStream(guest) + const rs2 = new EntryStream(guest) + const [list1, list2] = await Promise.all([collect(rs1), collect(rs2)]) - rs1.pipe(concat(function (list1) { - t.same(list1.length, 100) - rs2.pipe(concat(function (list2) { - t.same(list2.length, 100) - t.end() - })) - })) - }) + t.same(list1.length, 100) + t.same(list2.length, 100) }) diff --git a/test/sublevel.js b/test/sublevel.js index 9846eb7..c9dbd2a 100644 --- a/test/sublevel.js +++ b/test/sublevel.js @@ -6,7 +6,14 @@ const { EntryStream } = require('level-read-stream') const concat = require('concat-stream') const { ManyLevelHost, ManyLevelGuest } = require('..') -tape('sublevel on deferred many-level guest', function (t) { +function collect (stream) { + return new Promise((resolve, reject) => { + stream.on('error', reject) + stream.pipe(concat(resolve)) + }) +} + +tape('sublevel on deferred many-level guest', async function (t) { t.plan(5) const db = new MemoryLevel() @@ -19,25 +26,21 @@ tape('sublevel on deferred many-level guest', function (t) { t.is(guest.status, 'opening') stream.pipe(guest.createRpcStream()).pipe(stream) - sub1.put('hello', { test: 'world' }, function (err) { - t.error(err, 'no err') + await sub1.put('hello', { test: 'world' }) + t.pass('put succeeded') - // TODO: use iterator.all() instead - new EntryStream(sub1).pipe(concat(function (entries) { - t.same(entries, [{ key: 'hello', value: { test: 'world' } }]) - })) + const [sub1Entries, sub2Entries, dbEntries] = await Promise.all([ + collect(new EntryStream(sub1)), + collect(new EntryStream(sub2)), + collect(new EntryStream(db)) + ]) - new EntryStream(sub2).pipe(concat(function (entries) { - t.same(entries, [{ key: 'hello', value: '{"test":"world"}' }]) - })) - - new EntryStream(db).pipe(concat(function (entries) { - t.same(entries, [{ key: '!test!hello', value: '{"test":"world"}' }]) - })) - }) + t.same(sub1Entries, [{ key: 'hello', value: { test: 'world' } }]) + t.same(sub2Entries, [{ key: 'hello', value: '{"test":"world"}' }]) + t.same(dbEntries, [{ key: '!test!hello', value: '{"test":"world"}' }]) }) -tape('sublevel on non-deferred many-level guest', function (t) { +tape('sublevel on non-deferred many-level guest', async function (t) { t.plan(5) const db = new MemoryLevel() @@ -47,32 +50,27 @@ tape('sublevel on non-deferred many-level guest', function (t) { stream.pipe(guest.createRpcStream()).pipe(stream) - guest.once('open', function () { - t.is(guest.status, 'open') + await guest.open() + t.is(guest.status, 'open') - const sub1 = guest.sublevel('test', { valueEncoding: 'json' }) - const sub2 = guest.sublevel('test') - - sub1.put('hello', { test: 'world' }, function (err) { - t.error(err, 'no err') + const sub1 = guest.sublevel('test', { valueEncoding: 'json' }) + const sub2 = guest.sublevel('test') - // TODO: use iterator.all() instead - new EntryStream(sub1).pipe(concat(function (entries) { - t.same(entries, [{ key: 'hello', value: { test: 'world' } }]) - })) + await sub1.put('hello', { test: 'world' }) + t.pass('put succeeded') - new EntryStream(sub2).pipe(concat(function (entries) { - t.same(entries, [{ key: 'hello', value: '{"test":"world"}' }]) - })) + const [sub1Entries, sub2Entries, dbEntries] = await Promise.all([ + collect(new EntryStream(sub1)), + collect(new EntryStream(sub2)), + collect(new EntryStream(db)) + ]) - new EntryStream(db).pipe(concat(function (entries) { - t.same(entries, [{ key: '!test!hello', value: '{"test":"world"}' }]) - })) - }) - }) + t.same(sub1Entries, [{ key: 'hello', value: { test: 'world' } }]) + t.same(sub2Entries, [{ key: 'hello', value: '{"test":"world"}' }]) + t.same(dbEntries, [{ key: '!test!hello', value: '{"test":"world"}' }]) }) -tape('many-level host on deferred sublevel', function (t) { +tape('many-level host on deferred sublevel', async function (t) { t.plan(4) const db = new MemoryLevel() @@ -83,63 +81,57 @@ tape('many-level host on deferred sublevel', function (t) { stream.pipe(guest.createRpcStream()).pipe(stream) - guest.put('from', 'guest', function (err) { - t.error(err, 'no err') + await guest.put('from', 'guest') + t.pass('guest put succeeded') - sub2.put('from', 'host', function (err) { - t.error(err, 'no err') + await sub2.put('from', 'host') + t.pass('host put succeeded') - // TODO: use iterator.all() instead - new EntryStream(guest).pipe(concat(function (entries) { - t.same(entries, [{ key: 'from', value: 'guest' }]) - })) + const [guestEntries, dbEntries] = await Promise.all([ + collect(new EntryStream(guest)), + collect(new EntryStream(db)) + ]) - new EntryStream(db).pipe(concat(function (entries) { - t.same(entries, [ - { key: '!test1!from', value: 'guest' }, - { key: '!test2!from', value: 'host' } - ]) - })) - }) - }) + t.same(guestEntries, [{ key: 'from', value: 'guest' }]) + t.same(dbEntries, [ + { key: '!test1!from', value: 'guest' }, + { key: '!test2!from', value: 'host' } + ]) }) -tape('many-level host on non-deferred sublevel', function (t) { +tape('many-level host on non-deferred sublevel', async function (t) { t.plan(4) const db = new MemoryLevel() const sub1 = db.sublevel('test1') const sub2 = db.sublevel('test2') - sub1.once('open', function () { - const stream = new ManyLevelHost(sub1).createRpcStream() - const guest = new ManyLevelGuest() + await sub1.open() - stream.pipe(guest.createRpcStream()).pipe(stream) + const stream = new ManyLevelHost(sub1).createRpcStream() + const guest = new ManyLevelGuest() - guest.put('from', 'guest', function (err) { - t.error(err, 'no err') + stream.pipe(guest.createRpcStream()).pipe(stream) - sub2.put('from', 'host', function (err) { - t.error(err, 'no err') + await guest.put('from', 'guest') + t.pass('guest put succeeded') - // TODO: use iterator.all() instead - new EntryStream(guest).pipe(concat(function (entries) { - t.same(entries, [{ key: 'from', value: 'guest' }]) - })) + await sub2.put('from', 'host') + t.pass('host put succeeded') - new EntryStream(db).pipe(concat(function (entries) { - t.same(entries, [ - { key: '!test1!from', value: 'guest' }, - { key: '!test2!from', value: 'host' } - ]) - })) - }) - }) - }) + const [guestEntries, dbEntries] = await Promise.all([ + collect(new EntryStream(guest)), + collect(new EntryStream(db)) + ]) + + t.same(guestEntries, [{ key: 'from', value: 'guest' }]) + t.same(dbEntries, [ + { key: '!test1!from', value: 'guest' }, + { key: '!test2!from', value: 'host' } + ]) }) -tape('many-level host on nested sublevel', function (t) { +tape('many-level host on nested sublevel', async function (t) { t.plan(4) const db = new MemoryLevel() @@ -151,23 +143,20 @@ tape('many-level host on nested sublevel', function (t) { stream.pipe(guest.createRpcStream()).pipe(stream) - guest.put('from', 'guest', function (err) { - t.error(err, 'no err') + await guest.put('from', 'guest') + t.pass('guest put succeeded') - sub3.put('from', 'host', function (err) { - t.error(err, 'no err') + await sub3.put('from', 'host') + t.pass('host put succeeded') - // TODO: use iterator.all() instead - new EntryStream(guest).pipe(concat(function (entries) { - t.same(entries, [{ key: 'from', value: 'guest' }]) - })) + const [guestEntries, dbEntries] = await Promise.all([ + collect(new EntryStream(guest)), + collect(new EntryStream(db)) + ]) - new EntryStream(db).pipe(concat(function (entries) { - t.same(entries, [ - { key: '!test1!!test2!from', value: 'guest' }, - { key: '!test3!from', value: 'host' } - ]) - })) - }) - }) + t.same(guestEntries, [{ key: 'from', value: 'guest' }]) + t.same(dbEntries, [ + { key: '!test1!!test2!from', value: 'guest' }, + { key: '!test3!from', value: 'host' } + ]) }) From 8d4b3716d8a736e7468ff137cb17aff5ac6141a9 Mon Sep 17 00:00:00 2001 From: Germano Eichenberg Date: Wed, 17 Jun 2026 21:10:22 -0300 Subject: [PATCH 29/32] Support v3 has and snapshot APIs --- guest.js | 119 ++++- host.js | 98 +++- messages.js | 1368 +++++++++++++++++++++++++++++++++++--------------- schema.proto | 36 ++ tags.js | 16 +- test/v3.js | 114 +++++ 6 files changed, 1339 insertions(+), 412 deletions(-) create mode 100644 test/v3.js diff --git a/guest.js b/guest.js index 434dd46..4525dd8 100644 --- a/guest.js +++ b/guest.js @@ -1,6 +1,6 @@ 'use strict' -const { AbstractLevel, AbstractIterator } = require('abstract-level') +const { AbstractLevel, AbstractIterator, AbstractSnapshot } = require('abstract-level') const lpstream = require('@vweevers/length-prefixed-stream') const ModuleError = require('module-error') const { input, output } = require('./tags') @@ -18,6 +18,7 @@ const kRef = Symbol('ref') const kDb = Symbol('db') const kRequests = Symbol('requests') const kIterators = Symbol('iterators') +const kSnapshots = Symbol('snapshots') const kRetry = Symbol('retry') const kRpcStream = Symbol('rpcStream') const kFlushed = Symbol('flushed') @@ -29,6 +30,7 @@ const kSeq = Symbol('seq') const kErrored = Symbol('errored') const kAbortIterator = Symbol('abortIterator') const kOnDbClosing = Symbol('onDbClosing') +const kId = Symbol('id') class ManyLevelGuest extends AbstractLevel { constructor (options) { @@ -37,14 +39,18 @@ class ManyLevelGuest extends AbstractLevel { super({ encodings: { buffer: true }, snapshots: !retry, + implicitSnapshots: !retry, + explicitSnapshots: !retry, permanence: true, seek: true, + has: true, createIfMissing: false, errorIfExists: false }, forward) this[kIterators] = new IdMap() this[kRequests] = new IdMap() + this[kSnapshots] = new IdMap() this[kRetry] = !!retry this[kEncode] = lpstream.encode() this[kRemote] = _remote || null @@ -106,6 +112,14 @@ class ManyLevelGuest extends AbstractLevel { case output.getManyCallback: ongetmanycallback(res) break + + case output.hasCallback: + onhascallback(res) + break + + case output.hasManyCallback: + onhasmanycallback(res) + break } self[kFlushed]() @@ -168,6 +182,20 @@ class ManyLevelGuest extends AbstractLevel { if (res.error) req.callback(new ModuleError('Could not get values', { code: res.error })) else req.callback(null, res.values.map(v => normalizeValue(v.value))) } + + function onhascallback (res) { + const req = self[kRequests].remove(res.id) + if (!req || !req.callback) return + if (res.error) req.callback(new ModuleError('Could not check key', { code: res.error })) + else req.callback(null, res.value) + } + + function onhasmanycallback (res) { + const req = self[kRequests].remove(res.id) + if (!req || !req.callback) return + if (res.error) req.callback(new ModuleError('Could not check keys', { code: res.error })) + else req.callback(null, res.values) + } } // Alias for backwards compat with multileveldown @@ -218,6 +246,7 @@ class ManyLevelGuest extends AbstractLevel { tag: input.get, id: 0, key: key, + snapshot: snapshotId(opts.snapshot), // This will resolve or reject based on the Host's response callback: (err, value) => { if (err) reject(err) @@ -238,6 +267,7 @@ class ManyLevelGuest extends AbstractLevel { tag: input.getMany, id: 0, keys: keys, + snapshot: snapshotId(opts.snapshot), // This will resolve or reject based on the Host's response callback: (err, values) => { if (err) reject(err) @@ -250,6 +280,46 @@ class ManyLevelGuest extends AbstractLevel { }) } + async _has (key, opts) { + if (this[kDb]) return this[kDb]._has(key, opts) + + return new Promise((resolve, reject) => { + const req = { + tag: input.has, + id: 0, + key: key, + snapshot: snapshotId(opts.snapshot), + callback: (err, value) => { + if (err) reject(err) + else resolve(value) + } + } + + req.id = this[kRequests].add(req) + this[kWrite](req) + }) + } + + async _hasMany (keys, opts) { + if (this[kDb]) return this[kDb]._hasMany(keys, opts) + + return new Promise((resolve, reject) => { + const req = { + tag: input.hasMany, + id: 0, + keys: keys, + snapshot: snapshotId(opts.snapshot), + callback: (err, values) => { + if (err) reject(err) + else resolve(values) + } + } + + req.id = this[kRequests].add(req) + this[kWrite](req) + }) + } + async _put (key, value, opts) { if (this[kDb]) return this[kDb]._put(key, value, opts) @@ -314,6 +384,8 @@ class ManyLevelGuest extends AbstractLevel { async _clear (opts) { if (this[kDb]) return this[kDb]._clear(opts) + opts = encodeOptionsSnapshot(opts) + return new Promise((resolve, reject) => { const req = { tag: input.clear, @@ -331,6 +403,16 @@ class ManyLevelGuest extends AbstractLevel { }) } + _snapshot (options) { + if (this[kDb]) return this[kDb].snapshot(options) + + const snapshot = new ManyLevelGuestSnapshot(this, options) + const req = { tag: input.snapshot, id: snapshot[kId] } + + this[kWrite](req) + return snapshot + } + async [kWrite] (req) { if (this[kRequests].size + this[kIterators].size === 1) ref(this[kRef]) const enc = input.encoding(req.tag) @@ -414,7 +496,11 @@ class ManyLevelGuestIterator extends AbstractIterator { this[kPending] = [] this[kCallback] = null this[kSeq] = 0 - this[kOnDbClosing] = () => this[kAbortIterator]('LEVEL_ITERATOR_NOT_OPEN') + this[kOnDbClosing] = () => { + if (this.db[kRpcStream] === null && this.db[kDb] === null) { + this[kAbortIterator]('LEVEL_ITERATOR_NOT_OPEN') + } + } db.once('closing', this[kOnDbClosing]) const req = this[kRequest] = { @@ -422,10 +508,11 @@ class ManyLevelGuestIterator extends AbstractIterator { id: 0, seq: 0, iterator: this, - options, + options: encodeOptionsSnapshot(options), consumed: 0, bookmark: null, - seek: null + seek: null, + snapshot: snapshotId(options.snapshot) } const ack = this[kAckMessage] = { @@ -540,6 +627,30 @@ class ManyLevelGuestIterator extends AbstractIterator { } } +class ManyLevelGuestSnapshot extends AbstractSnapshot { + constructor (db, options) { + super(options) + this.db = db + this[kId] = db[kSnapshots].add(this) + } + + async _close () { + this.db[kSnapshots].remove(this[kId]) + await this.db[kWrite]({ tag: input.snapshotClose, id: this[kId] }) + this.db[kFlushed]() + } +} + +function snapshotId (snapshot) { + return snapshot instanceof ManyLevelGuestSnapshot ? snapshot[kId] : 0 +} + +function encodeOptionsSnapshot (options) { + if (!options || !options.snapshot) return options + const snapshot = snapshotId(options.snapshot) + return snapshot ? Object.assign({}, options, { snapshot }) : options +} + function normalizeValue (value) { return value === null ? undefined : value } diff --git a/host.js b/host.js index 74f10fd..d80c750 100644 --- a/host.js +++ b/host.js @@ -83,6 +83,7 @@ function createRpcStream (db, options, streamOptions) { if (err) return stream.destroy(err) const iterators = new Map() + const snapshots = new Map() const cleanup = async () => { await finished(stream).catch(() => null) @@ -90,7 +91,12 @@ function createRpcStream (db, options, streamOptions) { iterator.close() } + for (const snapshot of snapshots.values()) { + snapshot.close() + } + iterators.clear() + snapshots.clear() } // Don't await cleanup() @@ -121,6 +127,10 @@ function createRpcStream (db, options, streamOptions) { case input.iteratorSeek: return oniteratorseek(req) case input.clear: return readonly ? onreadonly(req) : onclear(req) case input.getMany: return ongetmany(req) + case input.has: return onhas(req) + case input.hasMany: return onhasmany(req) + case input.snapshot: return onsnapshot(req) + case input.snapshotClose: return onsnapshotclose(req) } }) @@ -134,6 +144,27 @@ function createRpcStream (db, options, streamOptions) { encode.write(encodeMessage(msg, output.getManyCallback)) } + function hasCallback (id, err, value) { + const msg = { id, error: errorCode(err), value } + encode.write(encodeMessage(msg, output.hasCallback)) + } + + function hasManyCallback (id, err, values) { + const msg = { id, error: errorCode(err), values } + encode.write(encodeMessage(msg, output.hasManyCallback)) + } + + function snapshot (id) { + if (!id) return undefined + + const snapshot = snapshots.get(id) + if (snapshot !== undefined) return snapshot + + throw new ModuleError('Snapshot is not open', { + code: 'LEVEL_SNAPSHOT_NOT_OPEN' + }) + } + async function onput (req) { preput(req.key, req.value, function (err) { return callback(req.id, err) }) try { @@ -145,7 +176,7 @@ function createRpcStream (db, options, streamOptions) { async function onget (req) { try { - const value = await db.get(req.key, encodingOptions) + const value = await db.get(req.key, withSnapshot(encodingOptions, snapshot(req.snapshot))) return callback(req.id, null, value) } catch (err) { return callback(req.id, err, null) @@ -154,13 +185,31 @@ function createRpcStream (db, options, streamOptions) { async function ongetmany (req) { try { - const values = await db.getMany(req.keys, encodingOptions) + const values = await db.getMany(req.keys, withSnapshot(encodingOptions, snapshot(req.snapshot))) return getManyCallback(req.id, null, values.map(value => ({ value }))) } catch (err) { return getManyCallback(req.id, err, []) } } + async function onhas (req) { + try { + const value = await db.has(req.key, withSnapshot(encodingOptions, snapshot(req.snapshot))) + return hasCallback(req.id, null, value) + } catch (err) { + return hasCallback(req.id, err, false) + } + } + + async function onhasmany (req) { + try { + const values = await db.hasMany(req.keys, withSnapshot(encodingOptions, snapshot(req.snapshot))) + return hasManyCallback(req.id, null, values) + } catch (err) { + return hasManyCallback(req.id, err, []) + } + } + async function ondel (req) { predel(req.key, function (err) { return callback(req.id, err) }) try { @@ -185,9 +234,17 @@ function createRpcStream (db, options, streamOptions) { } } - function oniterator ({ id, seq, options, consumed, bookmark, seek }) { + function oniterator ({ id, seq, options, consumed, bookmark, seek, snapshot: snapshotId }) { if (iterators.has(id)) return + try { + options = withSnapshot(options, snapshot(snapshotId)) + } catch (err) { + const data = { id, error: errorCode(err), seq } + encode.write(encodeMessage(data, output.iteratorError)) + return + } + const it = new ManyLevelHostIterator(db, id, seq, options, consumed, encode) iterators.set(id, it) @@ -223,12 +280,32 @@ function createRpcStream (db, options, streamOptions) { async function onclear (req) { try { - await db.clear(cleanRangeOptions(req.options)) + const options = cleanRangeOptions(req.options) + await db.clear(withSnapshot(options, snapshot(options.snapshot))) return callback(req.id, null) } catch (err) { return callback(req.id, err) } } + + function onsnapshot (req) { + if (snapshots.has(req.id)) return + + try { + snapshots.set(req.id, db.snapshot()) + } catch (err) { + snapshots.set(req.id, { + close () {}, + error: err + }) + } + } + + async function onsnapshotclose (req) { + const snapshot = snapshots.get(req.id) + snapshots.delete(req.id) + if (snapshot !== undefined) await snapshot.close() + } } } @@ -417,3 +494,16 @@ function cleanRangeOptions (options) { return result } + +function withSnapshot (options, snapshot) { + if (snapshot === undefined) { + if (!options || !hasOwnProperty.call(options, 'snapshot')) return options + + const result = { ...options } + delete result.snapshot + return result + } + + if (snapshot.error) throw snapshot.error + return { ...options, snapshot } +} diff --git a/messages.js b/messages.js index 61d33c7..9661abe 100644 --- a/messages.js +++ b/messages.js @@ -1,141 +1,189 @@ -// This file is auto generated by the protocol-buffers compiler - -/* eslint-disable quotes */ -/* eslint-disable indent */ -/* eslint-disable no-redeclare */ -/* eslint-disable camelcase */ - -// Remember to `npm install --save protocol-buffers-encodings` -var encodings = require('protocol-buffers-encodings') -var varint = encodings.varint -var skip = encodings.skip - -var Get = exports.Get = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var Put = exports.Put = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var Delete = exports.Delete = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var Batch = exports.Batch = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var Clear = exports.Clear = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var Iterator = exports.Iterator = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var Callback = exports.Callback = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var IteratorData = exports.IteratorData = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var IteratorAck = exports.IteratorAck = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var IteratorSeek = exports.IteratorSeek = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var IteratorEnd = exports.IteratorEnd = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var IteratorError = exports.IteratorError = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var IteratorClose = exports.IteratorClose = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var GetMany = exports.GetMany = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -var GetManyCallback = exports.GetManyCallback = { - buffer: true, - encodingLength: null, - encode: null, - decode: null -} - -defineGet() -definePut() -defineDelete() -defineBatch() -defineClear() -defineIterator() -defineCallback() -defineIteratorData() -defineIteratorAck() -defineIteratorSeek() -defineIteratorEnd() -defineIteratorError() -defineIteratorClose() -defineGetMany() -defineGetManyCallback() - -function defineGet () { - Get.encodingLength = encodingLength - Get.encode = encode - Get.decode = decode - +// This file is auto generated by the protocol-buffers compiler + +/* eslint-disable quotes */ +/* eslint-disable indent */ +/* eslint-disable no-redeclare */ +/* eslint-disable camelcase */ + +// Remember to `npm install --save protocol-buffers-encodings` +var encodings = require('protocol-buffers-encodings') +var varint = encodings.varint +var skip = encodings.skip + +var Get = exports.Get = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Put = exports.Put = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Delete = exports.Delete = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Batch = exports.Batch = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Clear = exports.Clear = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Iterator = exports.Iterator = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Callback = exports.Callback = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var IteratorData = exports.IteratorData = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var IteratorAck = exports.IteratorAck = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var IteratorSeek = exports.IteratorSeek = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var IteratorEnd = exports.IteratorEnd = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var IteratorError = exports.IteratorError = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var IteratorClose = exports.IteratorClose = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var GetMany = exports.GetMany = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var GetManyCallback = exports.GetManyCallback = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Has = exports.Has = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var HasMany = exports.HasMany = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var HasCallback = exports.HasCallback = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var HasManyCallback = exports.HasManyCallback = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var Snapshot = exports.Snapshot = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +var SnapshotClose = exports.SnapshotClose = { + buffer: true, + encodingLength: null, + encode: null, + decode: null +} + +defineGet() +definePut() +defineDelete() +defineBatch() +defineClear() +defineIterator() +defineCallback() +defineIteratorData() +defineIteratorAck() +defineIteratorSeek() +defineIteratorEnd() +defineIteratorError() +defineIteratorClose() +defineGetMany() +defineGetManyCallback() +defineHas() +defineHasMany() +defineHasCallback() +defineHasManyCallback() +defineSnapshot() +defineSnapshotClose() + +function defineGet () { + Get.encodingLength = encodingLength + Get.encode = encode + Get.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -144,9 +192,13 @@ function defineGet () { if (!defined(obj.key)) throw new Error("key is required") var len = encodings.bytes.encodingLength(obj.key) length += 1 + len + if (defined(obj.snapshot)) { + var len = encodings.varint.encodingLength(obj.snapshot) + length += 1 + len + } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -159,10 +211,15 @@ function defineGet () { buf[offset++] = 18 encodings.bytes.encode(obj.key, buf, offset) offset += encodings.bytes.encode.bytes + if (defined(obj.snapshot)) { + buf[offset++] = 24 + encodings.varint.encode(obj.snapshot, buf, offset) + offset += encodings.varint.encode.bytes + } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -170,7 +227,8 @@ function defineGet () { var oldOffset = offset var obj = { id: 0, - key: null + key: null, + snapshot: 0 } var found0 = false var found1 = false @@ -194,18 +252,22 @@ function defineGet () { offset += encodings.bytes.decode.bytes found1 = true break + case 3: + obj.snapshot = encodings.varint.decode(buf, offset) + offset += encodings.varint.decode.bytes + break default: offset = skip(prefix & 7, buf, offset) } } - } -} - -function definePut () { - Put.encodingLength = encodingLength - Put.encode = encode - Put.decode = decode - + } +} + +function definePut () { + Put.encodingLength = encodingLength + Put.encode = encode + Put.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -219,8 +281,8 @@ function definePut () { length += 1 + len } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -240,8 +302,8 @@ function definePut () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -282,14 +344,14 @@ function definePut () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineDelete () { - Delete.encodingLength = encodingLength - Delete.encode = encode - Delete.decode = decode - + } +} + +function defineDelete () { + Delete.encodingLength = encodingLength + Delete.encode = encode + Delete.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -299,8 +361,8 @@ function defineDelete () { var len = encodings.bytes.encodingLength(obj.key) length += 1 + len return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -315,8 +377,8 @@ function defineDelete () { offset += encodings.bytes.encode.bytes encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -352,24 +414,24 @@ function defineDelete () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineBatch () { - var Operation = Batch.Operation = { - buffer: true, - encodingLength: null, - encode: null, - decode: null - } - - defineOperation() - - function defineOperation () { - Operation.encodingLength = encodingLength - Operation.encode = encode - Operation.decode = decode - + } +} + +function defineBatch () { + var Operation = Batch.Operation = { + buffer: true, + encodingLength: null, + encode: null, + decode: null + } + + defineOperation() + + function defineOperation () { + Operation.encodingLength = encodingLength + Operation.encode = encode + Operation.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.type)) throw new Error("type is required") @@ -383,8 +445,8 @@ function defineBatch () { length += 1 + len } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -404,8 +466,8 @@ function defineBatch () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -446,13 +508,13 @@ function defineBatch () { offset = skip(prefix & 7, buf, offset) } } - } - } - - Batch.encodingLength = encodingLength - Batch.encode = encode - Batch.decode = decode - + } + } + + Batch.encodingLength = encodingLength + Batch.encode = encode + Batch.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -467,8 +529,8 @@ function defineBatch () { } } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -489,8 +551,8 @@ function defineBatch () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -526,24 +588,24 @@ function defineBatch () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineClear () { - var ClearOptions = Clear.ClearOptions = { - buffer: true, - encodingLength: null, - encode: null, - decode: null - } - - defineClearOptions() - - function defineClearOptions () { - ClearOptions.encodingLength = encodingLength - ClearOptions.encode = encode - ClearOptions.decode = decode - + } +} + +function defineClear () { + var ClearOptions = Clear.ClearOptions = { + buffer: true, + encodingLength: null, + encode: null, + decode: null + } + + defineClearOptions() + + function defineClearOptions () { + ClearOptions.encodingLength = encodingLength + ClearOptions.encode = encode + ClearOptions.decode = decode + function encodingLength (obj) { var length = 0 if (defined(obj.gt)) { @@ -570,9 +632,13 @@ function defineClear () { var len = encodings.bool.encodingLength(obj.reverse) length += 1 + len } + if (defined(obj.snapshot)) { + var len = encodings.varint.encodingLength(obj.snapshot) + length += 1 + len + } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -607,10 +673,15 @@ function defineClear () { encodings.bool.encode(obj.reverse, buf, offset) offset += encodings.bool.encode.bytes } + if (defined(obj.snapshot)) { + buf[offset++] = 72 + encodings.varint.encode(obj.snapshot, buf, offset) + offset += encodings.varint.encode.bytes + } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -622,7 +693,8 @@ function defineClear () { lt: null, lte: null, limit: 0, - reverse: false + reverse: false, + snapshot: 0 } while (true) { if (end <= offset) { @@ -657,17 +729,21 @@ function defineClear () { obj.reverse = encodings.bool.decode(buf, offset) offset += encodings.bool.decode.bytes break + case 9: + obj.snapshot = encodings.varint.decode(buf, offset) + offset += encodings.varint.decode.bytes + break default: offset = skip(prefix & 7, buf, offset) } } - } - } - - Clear.encodingLength = encodingLength - Clear.encode = encode - Clear.decode = decode - + } + } + + Clear.encodingLength = encodingLength + Clear.encode = encode + Clear.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -679,8 +755,8 @@ function defineClear () { length += 1 + len } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -698,8 +774,8 @@ function defineClear () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -735,24 +811,24 @@ function defineClear () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineIterator () { - var Options = Iterator.Options = { - buffer: true, - encodingLength: null, - encode: null, - decode: null - } - - defineOptions() - - function defineOptions () { - Options.encodingLength = encodingLength - Options.encode = encode - Options.decode = decode - + } +} + +function defineIterator () { + var Options = Iterator.Options = { + buffer: true, + encodingLength: null, + encode: null, + decode: null + } + + defineOptions() + + function defineOptions () { + Options.encodingLength = encodingLength + Options.encode = encode + Options.decode = decode + function encodingLength (obj) { var length = 0 if (defined(obj.keys)) { @@ -788,8 +864,8 @@ function defineIterator () { length += 1 + len } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -836,8 +912,8 @@ function defineIterator () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -898,13 +974,13 @@ function defineIterator () { offset = skip(prefix & 7, buf, offset) } } - } - } - - Iterator.encodingLength = encodingLength - Iterator.encode = encode - Iterator.decode = decode - + } + } + + Iterator.encodingLength = encodingLength + Iterator.encode = encode + Iterator.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -928,9 +1004,13 @@ function defineIterator () { var len = encodings.bytes.encodingLength(obj.seek) length += 1 + len } + if (defined(obj.snapshot)) { + var len = encodings.varint.encodingLength(obj.snapshot) + length += 1 + len + } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -963,10 +1043,15 @@ function defineIterator () { encodings.bytes.encode(obj.seek, buf, offset) offset += encodings.bytes.encode.bytes } + if (defined(obj.snapshot)) { + buf[offset++] = 56 + encodings.varint.encode(obj.snapshot, buf, offset) + offset += encodings.varint.encode.bytes + } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -978,7 +1063,8 @@ function defineIterator () { options: null, consumed: 0, bookmark: null, - seek: null + seek: null, + snapshot: 0 } var found0 = false var found1 = false @@ -1024,18 +1110,22 @@ function defineIterator () { obj.seek = encodings.bytes.decode(buf, offset) offset += encodings.bytes.decode.bytes break + case 7: + obj.snapshot = encodings.varint.decode(buf, offset) + offset += encodings.varint.decode.bytes + break default: offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineCallback () { - Callback.encodingLength = encodingLength - Callback.encode = encode - Callback.decode = decode - + } +} + +function defineCallback () { + Callback.encodingLength = encodingLength + Callback.encode = encode + Callback.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -1050,8 +1140,8 @@ function defineCallback () { length += 1 + len } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -1072,8 +1162,8 @@ function defineCallback () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -1112,14 +1202,14 @@ function defineCallback () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineIteratorData () { - IteratorData.encodingLength = encodingLength - IteratorData.encode = encode - IteratorData.decode = decode - + } +} + +function defineIteratorData () { + IteratorData.encodingLength = encodingLength + IteratorData.encode = encode + IteratorData.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -1136,8 +1226,8 @@ function defineIteratorData () { } } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -1160,8 +1250,8 @@ function defineIteratorData () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -1202,14 +1292,14 @@ function defineIteratorData () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineIteratorAck () { - IteratorAck.encodingLength = encodingLength - IteratorAck.encode = encode - IteratorAck.decode = decode - + } +} + +function defineIteratorAck () { + IteratorAck.encodingLength = encodingLength + IteratorAck.encode = encode + IteratorAck.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -1222,8 +1312,8 @@ function defineIteratorAck () { var len = encodings.varint.encodingLength(obj.consumed) length += 1 + len return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -1242,8 +1332,8 @@ function defineIteratorAck () { offset += encodings.varint.encode.bytes encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -1286,14 +1376,14 @@ function defineIteratorAck () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineIteratorSeek () { - IteratorSeek.encodingLength = encodingLength - IteratorSeek.encode = encode - IteratorSeek.decode = decode - + } +} + +function defineIteratorSeek () { + IteratorSeek.encodingLength = encodingLength + IteratorSeek.encode = encode + IteratorSeek.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -1306,8 +1396,8 @@ function defineIteratorSeek () { var len = encodings.bytes.encodingLength(obj.target) length += 1 + len return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -1326,8 +1416,8 @@ function defineIteratorSeek () { offset += encodings.bytes.encode.bytes encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -1370,14 +1460,14 @@ function defineIteratorSeek () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineIteratorEnd () { - IteratorEnd.encodingLength = encodingLength - IteratorEnd.encode = encode - IteratorEnd.decode = decode - + } +} + +function defineIteratorEnd () { + IteratorEnd.encodingLength = encodingLength + IteratorEnd.encode = encode + IteratorEnd.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -1387,8 +1477,8 @@ function defineIteratorEnd () { var len = encodings.varint.encodingLength(obj.seq) length += 1 + len return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -1403,8 +1493,8 @@ function defineIteratorEnd () { offset += encodings.varint.encode.bytes encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -1440,14 +1530,14 @@ function defineIteratorEnd () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineIteratorError () { - IteratorError.encodingLength = encodingLength - IteratorError.encode = encode - IteratorError.decode = decode - + } +} + +function defineIteratorError () { + IteratorError.encodingLength = encodingLength + IteratorError.encode = encode + IteratorError.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -1460,8 +1550,8 @@ function defineIteratorError () { var len = encodings.string.encodingLength(obj.error) length += 1 + len return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -1480,8 +1570,8 @@ function defineIteratorError () { offset += encodings.string.encode.bytes encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -1524,22 +1614,22 @@ function defineIteratorError () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineIteratorClose () { - IteratorClose.encodingLength = encodingLength - IteratorClose.encode = encode - IteratorClose.decode = decode - + } +} + +function defineIteratorClose () { + IteratorClose.encodingLength = encodingLength + IteratorClose.encode = encode + IteratorClose.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") var len = encodings.varint.encodingLength(obj.id) length += 1 + len return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -1550,8 +1640,8 @@ function defineIteratorClose () { offset += encodings.varint.encode.bytes encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -1580,14 +1670,14 @@ function defineIteratorClose () { offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineGetMany () { - GetMany.encodingLength = encodingLength - GetMany.encode = encode - GetMany.decode = decode - + } +} + +function defineGetMany () { + GetMany.encodingLength = encodingLength + GetMany.encode = encode + GetMany.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -1600,9 +1690,13 @@ function defineGetMany () { length += 1 + len } } + if (defined(obj.snapshot)) { + var len = encodings.varint.encodingLength(obj.snapshot) + length += 1 + len + } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -1619,10 +1713,15 @@ function defineGetMany () { offset += encodings.bytes.encode.bytes } } + if (defined(obj.snapshot)) { + buf[offset++] = 24 + encodings.varint.encode(obj.snapshot, buf, offset) + offset += encodings.varint.encode.bytes + } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -1630,7 +1729,8 @@ function defineGetMany () { var oldOffset = offset var obj = { id: 0, - keys: [] + keys: [], + snapshot: 0 } var found0 = false while (true) { @@ -1652,28 +1752,32 @@ function defineGetMany () { obj.keys.push(encodings.bytes.decode(buf, offset)) offset += encodings.bytes.decode.bytes break + case 3: + obj.snapshot = encodings.varint.decode(buf, offset) + offset += encodings.varint.decode.bytes + break default: offset = skip(prefix & 7, buf, offset) } } - } -} - -function defineGetManyCallback () { - var Value = GetManyCallback.Value = { - buffer: true, - encodingLength: null, - encode: null, - decode: null - } - - defineValue() - - function defineValue () { - Value.encodingLength = encodingLength - Value.encode = encode - Value.decode = decode - + } +} + +function defineGetManyCallback () { + var Value = GetManyCallback.Value = { + buffer: true, + encodingLength: null, + encode: null, + decode: null + } + + defineValue() + + function defineValue () { + Value.encodingLength = encodingLength + Value.encode = encode + Value.decode = decode + function encodingLength (obj) { var length = 0 if (defined(obj.value)) { @@ -1681,8 +1785,8 @@ function defineGetManyCallback () { length += 1 + len } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -1694,8 +1798,8 @@ function defineGetManyCallback () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -1721,13 +1825,13 @@ function defineGetManyCallback () { offset = skip(prefix & 7, buf, offset) } } - } - } - - GetManyCallback.encodingLength = encodingLength - GetManyCallback.encode = encode - GetManyCallback.decode = decode - + } + } + + GetManyCallback.encodingLength = encodingLength + GetManyCallback.encode = encode + GetManyCallback.decode = decode + function encodingLength (obj) { var length = 0 if (!defined(obj.id)) throw new Error("id is required") @@ -1746,8 +1850,8 @@ function defineGetManyCallback () { } } return length - } - + } + function encode (obj, buf, offset) { if (!offset) offset = 0 if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) @@ -1773,8 +1877,8 @@ function defineGetManyCallback () { } encode.bytes = offset - oldOffset return buf - } - + } + function decode (buf, offset, end) { if (!offset) offset = 0 if (!end) end = buf.length @@ -1815,9 +1919,469 @@ function defineGetManyCallback () { offset = skip(prefix & 7, buf, offset) } } - } -} - + } +} + +function defineHas () { + Has.encodingLength = encodingLength + Has.encode = encode + Has.decode = decode + + function encodingLength (obj) { + var length = 0 + if (!defined(obj.id)) throw new Error("id is required") + var len = encodings.varint.encodingLength(obj.id) + length += 1 + len + if (!defined(obj.key)) throw new Error("key is required") + var len = encodings.bytes.encodingLength(obj.key) + length += 1 + len + if (defined(obj.snapshot)) { + var len = encodings.varint.encodingLength(obj.snapshot) + length += 1 + len + } + return length + } + + function encode (obj, buf, offset) { + if (!offset) offset = 0 + if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) + var oldOffset = offset + if (!defined(obj.id)) throw new Error("id is required") + buf[offset++] = 8 + encodings.varint.encode(obj.id, buf, offset) + offset += encodings.varint.encode.bytes + if (!defined(obj.key)) throw new Error("key is required") + buf[offset++] = 18 + encodings.bytes.encode(obj.key, buf, offset) + offset += encodings.bytes.encode.bytes + if (defined(obj.snapshot)) { + buf[offset++] = 24 + encodings.varint.encode(obj.snapshot, buf, offset) + offset += encodings.varint.encode.bytes + } + encode.bytes = offset - oldOffset + return buf + } + + function decode (buf, offset, end) { + if (!offset) offset = 0 + if (!end) end = buf.length + if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") + var oldOffset = offset + var obj = { + id: 0, + key: null, + snapshot: 0 + } + var found0 = false + var found1 = false + while (true) { + if (end <= offset) { + if (!found0 || !found1) throw new Error("Decoded message is not valid") + decode.bytes = offset - oldOffset + return obj + } + var prefix = varint.decode(buf, offset) + offset += varint.decode.bytes + var tag = prefix >> 3 + switch (tag) { + case 1: + obj.id = encodings.varint.decode(buf, offset) + offset += encodings.varint.decode.bytes + found0 = true + break + case 2: + obj.key = encodings.bytes.decode(buf, offset) + offset += encodings.bytes.decode.bytes + found1 = true + break + case 3: + obj.snapshot = encodings.varint.decode(buf, offset) + offset += encodings.varint.decode.bytes + break + default: + offset = skip(prefix & 7, buf, offset) + } + } + } +} + +function defineHasMany () { + HasMany.encodingLength = encodingLength + HasMany.encode = encode + HasMany.decode = decode + + function encodingLength (obj) { + var length = 0 + if (!defined(obj.id)) throw new Error("id is required") + var len = encodings.varint.encodingLength(obj.id) + length += 1 + len + if (defined(obj.keys)) { + for (var i = 0; i < obj.keys.length; i++) { + if (!defined(obj.keys[i])) continue + var len = encodings.bytes.encodingLength(obj.keys[i]) + length += 1 + len + } + } + if (defined(obj.snapshot)) { + var len = encodings.varint.encodingLength(obj.snapshot) + length += 1 + len + } + return length + } + + function encode (obj, buf, offset) { + if (!offset) offset = 0 + if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) + var oldOffset = offset + if (!defined(obj.id)) throw new Error("id is required") + buf[offset++] = 8 + encodings.varint.encode(obj.id, buf, offset) + offset += encodings.varint.encode.bytes + if (defined(obj.keys)) { + for (var i = 0; i < obj.keys.length; i++) { + if (!defined(obj.keys[i])) continue + buf[offset++] = 18 + encodings.bytes.encode(obj.keys[i], buf, offset) + offset += encodings.bytes.encode.bytes + } + } + if (defined(obj.snapshot)) { + buf[offset++] = 24 + encodings.varint.encode(obj.snapshot, buf, offset) + offset += encodings.varint.encode.bytes + } + encode.bytes = offset - oldOffset + return buf + } + + function decode (buf, offset, end) { + if (!offset) offset = 0 + if (!end) end = buf.length + if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") + var oldOffset = offset + var obj = { + id: 0, + keys: [], + snapshot: 0 + } + var found0 = false + while (true) { + if (end <= offset) { + if (!found0) throw new Error("Decoded message is not valid") + decode.bytes = offset - oldOffset + return obj + } + var prefix = varint.decode(buf, offset) + offset += varint.decode.bytes + var tag = prefix >> 3 + switch (tag) { + case 1: + obj.id = encodings.varint.decode(buf, offset) + offset += encodings.varint.decode.bytes + found0 = true + break + case 2: + obj.keys.push(encodings.bytes.decode(buf, offset)) + offset += encodings.bytes.decode.bytes + break + case 3: + obj.snapshot = encodings.varint.decode(buf, offset) + offset += encodings.varint.decode.bytes + break + default: + offset = skip(prefix & 7, buf, offset) + } + } + } +} + +function defineHasCallback () { + HasCallback.encodingLength = encodingLength + HasCallback.encode = encode + HasCallback.decode = decode + + function encodingLength (obj) { + var length = 0 + if (!defined(obj.id)) throw new Error("id is required") + var len = encodings.varint.encodingLength(obj.id) + length += 1 + len + if (defined(obj.error)) { + var len = encodings.string.encodingLength(obj.error) + length += 1 + len + } + if (defined(obj.value)) { + var len = encodings.bool.encodingLength(obj.value) + length += 1 + len + } + return length + } + + function encode (obj, buf, offset) { + if (!offset) offset = 0 + if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) + var oldOffset = offset + if (!defined(obj.id)) throw new Error("id is required") + buf[offset++] = 8 + encodings.varint.encode(obj.id, buf, offset) + offset += encodings.varint.encode.bytes + if (defined(obj.error)) { + buf[offset++] = 18 + encodings.string.encode(obj.error, buf, offset) + offset += encodings.string.encode.bytes + } + if (defined(obj.value)) { + buf[offset++] = 24 + encodings.bool.encode(obj.value, buf, offset) + offset += encodings.bool.encode.bytes + } + encode.bytes = offset - oldOffset + return buf + } + + function decode (buf, offset, end) { + if (!offset) offset = 0 + if (!end) end = buf.length + if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") + var oldOffset = offset + var obj = { + id: 0, + error: "", + value: false + } + var found0 = false + while (true) { + if (end <= offset) { + if (!found0) throw new Error("Decoded message is not valid") + decode.bytes = offset - oldOffset + return obj + } + var prefix = varint.decode(buf, offset) + offset += varint.decode.bytes + var tag = prefix >> 3 + switch (tag) { + case 1: + obj.id = encodings.varint.decode(buf, offset) + offset += encodings.varint.decode.bytes + found0 = true + break + case 2: + obj.error = encodings.string.decode(buf, offset) + offset += encodings.string.decode.bytes + break + case 3: + obj.value = encodings.bool.decode(buf, offset) + offset += encodings.bool.decode.bytes + break + default: + offset = skip(prefix & 7, buf, offset) + } + } + } +} + +function defineHasManyCallback () { + HasManyCallback.encodingLength = encodingLength + HasManyCallback.encode = encode + HasManyCallback.decode = decode + + function encodingLength (obj) { + var length = 0 + if (!defined(obj.id)) throw new Error("id is required") + var len = encodings.varint.encodingLength(obj.id) + length += 1 + len + if (defined(obj.error)) { + var len = encodings.string.encodingLength(obj.error) + length += 1 + len + } + if (defined(obj.values)) { + for (var i = 0; i < obj.values.length; i++) { + if (!defined(obj.values[i])) continue + var len = encodings.bool.encodingLength(obj.values[i]) + length += 1 + len + } + } + return length + } + + function encode (obj, buf, offset) { + if (!offset) offset = 0 + if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) + var oldOffset = offset + if (!defined(obj.id)) throw new Error("id is required") + buf[offset++] = 8 + encodings.varint.encode(obj.id, buf, offset) + offset += encodings.varint.encode.bytes + if (defined(obj.error)) { + buf[offset++] = 18 + encodings.string.encode(obj.error, buf, offset) + offset += encodings.string.encode.bytes + } + if (defined(obj.values)) { + for (var i = 0; i < obj.values.length; i++) { + if (!defined(obj.values[i])) continue + buf[offset++] = 24 + encodings.bool.encode(obj.values[i], buf, offset) + offset += encodings.bool.encode.bytes + } + } + encode.bytes = offset - oldOffset + return buf + } + + function decode (buf, offset, end) { + if (!offset) offset = 0 + if (!end) end = buf.length + if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") + var oldOffset = offset + var obj = { + id: 0, + error: "", + values: [] + } + var found0 = false + while (true) { + if (end <= offset) { + if (!found0) throw new Error("Decoded message is not valid") + decode.bytes = offset - oldOffset + return obj + } + var prefix = varint.decode(buf, offset) + offset += varint.decode.bytes + var tag = prefix >> 3 + switch (tag) { + case 1: + obj.id = encodings.varint.decode(buf, offset) + offset += encodings.varint.decode.bytes + found0 = true + break + case 2: + obj.error = encodings.string.decode(buf, offset) + offset += encodings.string.decode.bytes + break + case 3: + obj.values.push(encodings.bool.decode(buf, offset)) + offset += encodings.bool.decode.bytes + break + default: + offset = skip(prefix & 7, buf, offset) + } + } + } +} + +function defineSnapshot () { + Snapshot.encodingLength = encodingLength + Snapshot.encode = encode + Snapshot.decode = decode + + function encodingLength (obj) { + var length = 0 + if (!defined(obj.id)) throw new Error("id is required") + var len = encodings.varint.encodingLength(obj.id) + length += 1 + len + return length + } + + function encode (obj, buf, offset) { + if (!offset) offset = 0 + if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) + var oldOffset = offset + if (!defined(obj.id)) throw new Error("id is required") + buf[offset++] = 8 + encodings.varint.encode(obj.id, buf, offset) + offset += encodings.varint.encode.bytes + encode.bytes = offset - oldOffset + return buf + } + + function decode (buf, offset, end) { + if (!offset) offset = 0 + if (!end) end = buf.length + if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") + var oldOffset = offset + var obj = { + id: 0 + } + var found0 = false + while (true) { + if (end <= offset) { + if (!found0) throw new Error("Decoded message is not valid") + decode.bytes = offset - oldOffset + return obj + } + var prefix = varint.decode(buf, offset) + offset += varint.decode.bytes + var tag = prefix >> 3 + switch (tag) { + case 1: + obj.id = encodings.varint.decode(buf, offset) + offset += encodings.varint.decode.bytes + found0 = true + break + default: + offset = skip(prefix & 7, buf, offset) + } + } + } +} + +function defineSnapshotClose () { + SnapshotClose.encodingLength = encodingLength + SnapshotClose.encode = encode + SnapshotClose.decode = decode + + function encodingLength (obj) { + var length = 0 + if (!defined(obj.id)) throw new Error("id is required") + var len = encodings.varint.encodingLength(obj.id) + length += 1 + len + return length + } + + function encode (obj, buf, offset) { + if (!offset) offset = 0 + if (!buf) buf = Buffer.allocUnsafe(encodingLength(obj)) + var oldOffset = offset + if (!defined(obj.id)) throw new Error("id is required") + buf[offset++] = 8 + encodings.varint.encode(obj.id, buf, offset) + offset += encodings.varint.encode.bytes + encode.bytes = offset - oldOffset + return buf + } + + function decode (buf, offset, end) { + if (!offset) offset = 0 + if (!end) end = buf.length + if (!(end <= buf.length && offset <= buf.length)) throw new Error("Decoded message is not valid") + var oldOffset = offset + var obj = { + id: 0 + } + var found0 = false + while (true) { + if (end <= offset) { + if (!found0) throw new Error("Decoded message is not valid") + decode.bytes = offset - oldOffset + return obj + } + var prefix = varint.decode(buf, offset) + offset += varint.decode.bytes + var tag = prefix >> 3 + switch (tag) { + case 1: + obj.id = encodings.varint.decode(buf, offset) + offset += encodings.varint.decode.bytes + found0 = true + break + default: + offset = skip(prefix & 7, buf, offset) + } + } + } +} + function defined (val) { return val !== null && val !== undefined && (typeof val !== 'number' || !isNaN(val)) -} +} diff --git a/schema.proto b/schema.proto index c21211d..3acc1ba 100644 --- a/schema.proto +++ b/schema.proto @@ -1,6 +1,7 @@ message Get { required uint32 id = 1; required bytes key = 2; + optional uint32 snapshot = 3; } message Put { @@ -36,6 +37,7 @@ message Clear { optional bytes lte = 6; optional sint32 limit = 7; optional bool reverse = 8; + optional uint32 snapshot = 9; } } @@ -46,6 +48,7 @@ message Iterator { required uint64 consumed = 4; optional bytes bookmark = 5; optional bytes seek = 6; + optional uint32 snapshot = 7; message Options { optional bool keys = 1; @@ -101,6 +104,7 @@ message IteratorClose { message GetMany { required uint32 id = 1; repeated bytes keys = 2; + optional uint32 snapshot = 3; } message GetManyCallback { @@ -113,3 +117,35 @@ message GetManyCallback { optional bytes value = 1; } } + +message Has { + required uint32 id = 1; + required bytes key = 2; + optional uint32 snapshot = 3; +} + +message HasMany { + required uint32 id = 1; + repeated bytes keys = 2; + optional uint32 snapshot = 3; +} + +message HasCallback { + required uint32 id = 1; + optional string error = 2; + optional bool value = 3; +} + +message HasManyCallback { + required uint32 id = 1; + optional string error = 2; + repeated bool values = 3; +} + +message Snapshot { + required uint32 id = 1; +} + +message SnapshotClose { + required uint32 id = 1; +} diff --git a/tags.js b/tags.js index abc7544..d68190b 100644 --- a/tags.js +++ b/tags.js @@ -12,7 +12,11 @@ const INPUT = [ messages.GetMany, messages.IteratorClose, messages.IteratorAck, - messages.IteratorSeek + messages.IteratorSeek, + messages.Has, + messages.HasMany, + messages.Snapshot, + messages.SnapshotClose ] const OUTPUT = [ @@ -20,7 +24,9 @@ const OUTPUT = [ messages.IteratorData, messages.GetManyCallback, messages.IteratorError, - messages.IteratorEnd + messages.IteratorEnd, + messages.HasCallback, + messages.HasManyCallback ] exports.input = { @@ -34,6 +40,10 @@ exports.input = { iteratorClose: 7, iteratorAck: 8, iteratorSeek: 9, + has: 10, + hasMany: 11, + snapshot: 12, + snapshotClose: 13, encoding (tag) { return INPUT[tag] @@ -46,6 +56,8 @@ exports.output = { getManyCallback: 2, iteratorError: 3, iteratorEnd: 4, + hasCallback: 5, + hasManyCallback: 6, encoding (tag) { return OUTPUT[tag] diff --git a/test/v3.js b/test/v3.js new file mode 100644 index 0000000..e40fad5 --- /dev/null +++ b/test/v3.js @@ -0,0 +1,114 @@ +'use strict' + +const test = require('tape') +const { MemoryLevel } = require('memory-level') +const { ManyLevelHost, ManyLevelGuest } = require('..') + +function pair (options) { + const db = new MemoryLevel() + const host = new ManyLevelHost(db) + const guest = new ManyLevelGuest({ + ...options, + _remote: () => host.createRpcStream() + }) + + return { db, guest } +} + +test('has and hasMany over rpc', async function (t) { + const { guest } = pair() + + await guest.open() + await guest.put('hello', 'world') + + t.is(await guest.has('hello'), true) + t.is(await guest.has('missing'), false) + t.same(await guest.hasMany(['hello', 'missing']), [true, false]) + + await guest.close() +}) + +test('explicit snapshot over rpc', async function (t) { + const { guest } = pair() + + await guest.open() + await guest.batch([ + { type: 'put', key: 'a', value: '1' }, + { type: 'put', key: 'b', value: '2' } + ]) + + const snapshot = guest.snapshot() + await guest.batch([ + { type: 'put', key: 'a', value: 'changed' }, + { type: 'put', key: 'c', value: '3' }, + { type: 'del', key: 'b' } + ]) + + t.is(await guest.get('a'), 'changed') + t.is(await guest.get('a', { snapshot }), '1') + t.same(await guest.getMany(['a', 'b', 'c'], { snapshot }), ['1', '2', undefined]) + t.is(await guest.has('b'), false) + t.is(await guest.has('b', { snapshot }), true) + t.same(await guest.hasMany(['a', 'b', 'c'], { snapshot }), [true, true, false]) + t.same(await guest.iterator({ snapshot }).all(), [['a', '1'], ['b', '2']]) + + await snapshot.close() + await guest.close() +}) + +test('clear uses explicit snapshot over rpc', async function (t) { + const { guest } = pair() + + await guest.open() + await guest.put('a', '1') + + const snapshot = guest.snapshot() + await guest.put('b', '2') + await guest.clear({ snapshot }) + + t.same(await guest.keys().all(), ['b']) + t.same(await guest.keys({ snapshot }).all(), ['a']) + + await snapshot.close() + await guest.close() +}) + +test('closed snapshot rejects has and hasMany over rpc', async function (t) { + const { guest } = pair() + + await guest.open() + await guest.put('hello', 'world') + + const snapshot = guest.snapshot() + await snapshot.close() + + try { + await guest.has('hello', { snapshot }) + } catch (err) { + t.is(err.code, 'LEVEL_SNAPSHOT_NOT_OPEN') + } + + try { + await guest.hasMany(['hello'], { snapshot }) + } catch (err) { + t.is(err.code, 'LEVEL_SNAPSHOT_NOT_OPEN') + } + + await guest.close() +}) + +test('retry mode supports has but not snapshots', async function (t) { + const { guest } = pair({ retry: true }) + + await guest.open() + await guest.put('hello', 'world') + + t.is(guest.supports.has, true) + t.is(guest.supports.snapshots, false) + t.is(guest.supports.implicitSnapshots, false) + t.is(guest.supports.explicitSnapshots, false) + t.is(await guest.has('hello'), true) + t.same(await guest.hasMany(['hello', 'missing']), [true, false]) + + await guest.close() +}) From 1295525a43e1e0ad3df7d2c563f17af1ffb222be Mon Sep 17 00:00:00 2001 From: Germano Eichenberg Date: Wed, 17 Jun 2026 21:14:24 -0300 Subject: [PATCH 30/32] Expand v3 API coverage --- guest.js | 6 ++ test/v3.js | 185 ++++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 175 insertions(+), 16 deletions(-) diff --git a/guest.js b/guest.js index 4525dd8..1ef9098 100644 --- a/guest.js +++ b/guest.js @@ -404,6 +404,12 @@ class ManyLevelGuest extends AbstractLevel { } _snapshot (options) { + if (this[kRetry]) { + throw new ModuleError('Database does not support explicit snapshots', { + code: 'LEVEL_NOT_SUPPORTED' + }) + } + if (this[kDb]) return this[kDb].snapshot(options) const snapshot = new ManyLevelGuestSnapshot(this, options) diff --git a/test/v3.js b/test/v3.js index e40fad5..a45d1de 100644 --- a/test/v3.js +++ b/test/v3.js @@ -5,25 +5,56 @@ const { MemoryLevel } = require('memory-level') const { ManyLevelHost, ManyLevelGuest } = require('..') function pair (options) { - const db = new MemoryLevel() - const host = new ManyLevelHost(db) + const db = options?.db || new MemoryLevel() + const hostDb = options?.hostDb || db + const host = new ManyLevelHost(hostDb, options?.hostOptions) const guest = new ManyLevelGuest({ - ...options, + ...options?.guestOptions, _remote: () => host.createRpcStream() }) - return { db, guest } + return { db, guest, hostDb } +} + +async function rejectsCode (t, promise, code) { + try { + await promise + t.fail('operation should reject') + } catch (err) { + t.is(err && err.code, code) + } } test('has and hasMany over rpc', async function (t) { const { guest } = pair() await guest.open() - await guest.put('hello', 'world') + await guest.batch([ + { type: 'put', key: 'a', value: '1' }, + { type: 'put', key: 'b', value: '2' } + ]) - t.is(await guest.has('hello'), true) + t.is(await guest.has('a'), true) t.is(await guest.has('missing'), false) - t.same(await guest.hasMany(['hello', 'missing']), [true, false]) + t.same(await guest.hasMany(['b', 'missing', 'a']), [true, false, true]) + t.same(await guest.hasMany([]), []) + + await guest.close() +}) + +test('has and hasMany honor key encoding over rpc', async function (t) { + const { db, guest } = pair() + + await db.put(JSON.stringify(['a', 1]), 'one') + await db.put(Buffer.from([0, 1]), 'buffer') + await guest.open() + + t.is(await guest.has(['a', 1], { keyEncoding: 'json' }), true) + t.is(await guest.has(['a', 2], { keyEncoding: 'json' }), false) + t.same(await guest.hasMany([ + Buffer.from([0, 1]), + Buffer.from([0, 2]) + ], { keyEncoding: 'buffer' }), [true, false]) await guest.close() }) @@ -51,11 +82,37 @@ test('explicit snapshot over rpc', async function (t) { t.is(await guest.has('b', { snapshot }), true) t.same(await guest.hasMany(['a', 'b', 'c'], { snapshot }), [true, true, false]) t.same(await guest.iterator({ snapshot }).all(), [['a', '1'], ['b', '2']]) + t.same(await guest.keys({ snapshot }).all(), ['a', 'b']) + t.same(await guest.values({ snapshot }).all(), ['1', '2']) await snapshot.close() await guest.close() }) +test('multiple explicit snapshots over rpc are independent', async function (t) { + const { guest } = pair() + + await guest.open() + await guest.put('number', '0') + + const before = guest.snapshot() + await guest.put('number', '1') + + const middle = guest.snapshot() + await guest.put('number', '2') + + t.is(await guest.get('number'), '2') + t.is(await guest.get('number', { snapshot: before }), '0') + t.is(await guest.get('number', { snapshot: middle }), '1') + + await before.close() + + t.is(await guest.get('number', { snapshot: middle }), '1') + + await middle.close() + await guest.close() +}) + test('clear uses explicit snapshot over rpc', async function (t) { const { guest } = pair() @@ -73,7 +130,7 @@ test('clear uses explicit snapshot over rpc', async function (t) { await guest.close() }) -test('closed snapshot rejects has and hasMany over rpc', async function (t) { +test('closed snapshot rejects snapshot-aware operations over rpc', async function (t) { const { guest } = pair() await guest.open() @@ -82,23 +139,112 @@ test('closed snapshot rejects has and hasMany over rpc', async function (t) { const snapshot = guest.snapshot() await snapshot.close() - try { - await guest.has('hello', { snapshot }) - } catch (err) { - t.is(err.code, 'LEVEL_SNAPSHOT_NOT_OPEN') - } + await rejectsCode(t, guest.get('hello', { snapshot }), 'LEVEL_SNAPSHOT_NOT_OPEN') + await rejectsCode(t, guest.getMany(['hello'], { snapshot }), 'LEVEL_SNAPSHOT_NOT_OPEN') + await rejectsCode(t, guest.has('hello', { snapshot }), 'LEVEL_SNAPSHOT_NOT_OPEN') + await rejectsCode(t, guest.hasMany(['hello'], { snapshot }), 'LEVEL_SNAPSHOT_NOT_OPEN') + await rejectsCode(t, guest.iterator({ snapshot }).all(), 'LEVEL_SNAPSHOT_NOT_OPEN') + await rejectsCode(t, guest.keys({ snapshot }).all(), 'LEVEL_SNAPSHOT_NOT_OPEN') + await rejectsCode(t, guest.values({ snapshot }).all(), 'LEVEL_SNAPSHOT_NOT_OPEN') + await rejectsCode(t, guest.clear({ snapshot }), 'LEVEL_SNAPSHOT_NOT_OPEN') + + await guest.close() +}) + +test('database close closes explicit snapshots over rpc', async function (t) { + const { guest } = pair() + + await guest.open() + const snapshot = guest.snapshot() + await guest.close() try { - await guest.hasMany(['hello'], { snapshot }) + snapshot.ref() + t.fail('snapshot should be closed') } catch (err) { - t.is(err.code, 'LEVEL_SNAPSHOT_NOT_OPEN') + t.is(err && err.code, 'LEVEL_SNAPSHOT_NOT_OPEN') } +}) + +test('guest sublevels use explicit snapshots over rpc', async function (t) { + const { guest } = pair() + + await guest.open() + + const sub = guest.sublevel('sub') + const other = guest.sublevel('other') + + await sub.put('a', '1') + await other.put('a', 'other') + + const snapshot = sub.snapshot() + + await sub.put('a', '2') + await sub.put('b', '3') + await other.put('b', 'ignored') + + t.is(await sub.get('a'), '2') + t.is(await sub.get('a', { snapshot }), '1') + t.same(await sub.hasMany(['a', 'b'], { snapshot }), [true, false]) + t.same(await sub.keys({ snapshot }).all(), ['a']) + t.same(await other.keys().all(), ['a', 'b']) + + await snapshot.close() + await guest.close() +}) + +test('host sublevels use explicit snapshots over rpc', async function (t) { + const db = new MemoryLevel() + const hostDb = db.sublevel('hosted') + const other = db.sublevel('other') + const { guest } = pair({ db, hostDb }) + + await guest.open() + await guest.put('a', '1') + await other.put('a', 'other') + + const snapshot = guest.snapshot() + + await guest.put('a', '2') + await guest.put('b', '3') + await other.put('b', 'ignored') + + t.same(await guest.iterator().all(), [['a', '2'], ['b', '3']]) + t.same(await guest.iterator({ snapshot }).all(), [['a', '1']]) + t.same(await guest.hasMany(['a', 'b'], { snapshot }), [true, false]) + t.same(await db.keys().all(), [ + '!hosted!a', + '!hosted!b', + '!other!a', + '!other!b' + ]) + await snapshot.close() + await guest.close() +}) + +test('readonly host allows v3 reads and rejects clear', async function (t) { + const db = new MemoryLevel() + const { guest } = pair({ db, hostOptions: { readonly: true } }) + + await db.put('a', '1') + await db.put('b', '2') + await guest.open() + + const snapshot = guest.snapshot() + + t.is(await guest.has('a'), true) + t.same(await guest.hasMany(['a', 'missing']), [true, false]) + t.same(await guest.keys({ snapshot }).all(), ['a', 'b']) + + await rejectsCode(t, guest.clear({ snapshot }), 'LEVEL_READONLY') + + await snapshot.close() await guest.close() }) test('retry mode supports has but not snapshots', async function (t) { - const { guest } = pair({ retry: true }) + const { guest } = pair({ guestOptions: { retry: true } }) await guest.open() await guest.put('hello', 'world') @@ -110,5 +256,12 @@ test('retry mode supports has but not snapshots', async function (t) { t.is(await guest.has('hello'), true) t.same(await guest.hasMany(['hello', 'missing']), [true, false]) + try { + guest.snapshot() + t.fail('snapshot should not be supported') + } catch (err) { + t.is(err && err.code, 'LEVEL_NOT_SUPPORTED') + } + await guest.close() }) From fca50cd0b22f53d454b329f71dbbcfa73c0b6348 Mon Sep 17 00:00:00 2001 From: Germano Eichenberg Date: Wed, 17 Jun 2026 21:25:13 -0300 Subject: [PATCH 31/32] Add opt-in getSync support --- guest.js | 18 ++++++++++++- index.d.ts | 14 +++++++++- test/v3.js | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 108 insertions(+), 2 deletions(-) diff --git a/guest.js b/guest.js index 1ef9098..4bf24de 100644 --- a/guest.js +++ b/guest.js @@ -23,6 +23,7 @@ const kRetry = Symbol('retry') const kRpcStream = Symbol('rpcStream') const kFlushed = Symbol('flushed') const kWrite = Symbol('write') +const kGetSync = Symbol('getSync') const kRequest = Symbol('request') const kPending = Symbol('pending') const kCallback = Symbol('callback') @@ -34,7 +35,8 @@ const kId = Symbol('id') class ManyLevelGuest extends AbstractLevel { constructor (options) { - const { retry, _remote, ...forward } = options || {} + const { retry, _remote, getSync, ...forward } = options || {} + const hasGetSync = getSync === true || typeof getSync === 'function' super({ encodings: { buffer: true }, @@ -44,6 +46,7 @@ class ManyLevelGuest extends AbstractLevel { permanence: true, seek: true, has: true, + getSync: hasGetSync, createIfMissing: false, errorIfExists: false }, forward) @@ -54,6 +57,7 @@ class ManyLevelGuest extends AbstractLevel { this[kRetry] = !!retry this[kEncode] = lpstream.encode() this[kRemote] = _remote || null + this[kGetSync] = typeof getSync === 'function' ? getSync : null this[kCleanup] = null this[kRpcStream] = null this[kRef] = null @@ -259,6 +263,18 @@ class ManyLevelGuest extends AbstractLevel { }) } + _getSync (key, opts) { + if (this[kDb]) return this[kDb]._getSync(key, opts) + + if (this[kGetSync]) { + return normalizeValue(this[kGetSync](key, encodeOptionsSnapshot(opts))) + } + + throw new ModuleError('Database does not support getSync()', { + code: 'LEVEL_NOT_SUPPORTED' + }) + } + async _getMany (keys, opts) { if (this[kDb]) return this[kDb]._getMany(keys, opts) diff --git a/index.d.ts b/index.d.ts index e821809..6ca742f 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,7 +1,8 @@ import { AbstractLevel, AbstractDatabaseOptions, - AbstractOpenOptions + AbstractOpenOptions, + AbstractGetOptions } from 'abstract-level' // Requires `npm install @types/readable-stream`. @@ -88,6 +89,12 @@ declare interface GuestDatabaseOptions extends AbstractDatabaseOptions void } +declare type GuestGetSync = ( + key: Buffer, + options: AbstractGetOptions +) => Buffer | Uint8Array | undefined + /** * Options for the {@link ManyLevelHost} constructor. */ diff --git a/test/v3.js b/test/v3.js index a45d1de..3b05289 100644 --- a/test/v3.js +++ b/test/v3.js @@ -42,6 +42,84 @@ test('has and hasMany over rpc', async function (t) { await guest.close() }) +test('forwarded database supports getSync', async function (t) { + const hostDb = new MemoryLevel() + const guest = new ManyLevelGuest({ getSync: true }) + + await hostDb.open() + guest.forward(hostDb) + await guest.open() + + await guest.put('a', '1') + await guest.put(['json'], { ok: true }, { + keyEncoding: 'json', + valueEncoding: 'json' + }) + + t.is(guest.supports.getSync, true) + t.is(guest.getSync('a'), '1') + t.same(guest.getSync(['json'], { + keyEncoding: 'json', + valueEncoding: 'json' + }), { ok: true }) + t.is(guest.getSync('missing'), undefined) + + await guest.close() +}) + +test('forwarded database getSync honors snapshots', async function (t) { + const hostDb = new MemoryLevel() + const guest = new ManyLevelGuest({ getSync: true }) + + await hostDb.open() + guest.forward(hostDb) + await guest.open() + + await guest.put('a', '1') + const snapshot = guest.snapshot() + await guest.put('a', '2') + + t.is(guest.getSync('a'), '2') + t.is(guest.getSync('a', { snapshot }), '1') + + await snapshot.close() + await guest.close() +}) + +test('custom getSync handler supports remote reads', async function (t) { + const { guest } = pair({ + guestOptions: { + getSync (key) { + if (key.toString() === 'missing') return undefined + return Buffer.from('sync:' + key.toString()) + } + } + }) + + await guest.open() + + t.is(guest.supports.getSync, true) + t.is(guest.getSync('a'), 'sync:a') + t.is(guest.getSync('missing'), undefined) + + await guest.close() +}) + +test('remote getSync rejects without handler', async function (t) { + const { guest } = pair() + + await guest.open() + + try { + guest.getSync('a') + t.fail('getSync should not be supported') + } catch (err) { + t.is(err && err.code, 'LEVEL_NOT_SUPPORTED') + } + + await guest.close() +}) + test('has and hasMany honor key encoding over rpc', async function (t) { const { db, guest } = pair() From 67f301ebbd6964a6a8782626ba5764c0248a6dd4 Mon Sep 17 00:00:00 2001 From: Germano Eichenberg Date: Wed, 8 Jul 2026 16:47:48 -0300 Subject: [PATCH 32/32] Clarify snapshot encoding --- schema.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/schema.proto b/schema.proto index 3acc1ba..3761f65 100644 --- a/schema.proto +++ b/schema.proto @@ -37,6 +37,10 @@ message Clear { optional bytes lte = 6; optional sint32 limit = 7; optional bool reverse = 8; + + // Clear has no request-level snapshot envelope. Iterators carry the + // snapshot id next to Options because the same Options shape is reused by + // iterator follow-up messages, but clear sends its range options once. optional uint32 snapshot = 9; } }