diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 4dafbbd..783bfe6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1929,8 +1929,8 @@ packages: resolution: {integrity: sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg==} engines: {node: '>= 0.4'} - fs-extra@11.3.5: - resolution: {integrity: sha512-eKpRKAovdpZtR1WopLHxlBWvAgPny3c4gX1G5Jhwmmw4XJj0ifSD5qB5TOo8hmA0wlRKDAOAhEE1yVPgs6Fgcg==} + fs-extra@11.3.6: + resolution: {integrity: sha512-w8ZNZr2mKIc7qeNaQ9AVPT1+iFaI+Avd4xudVOvdDJ8VytREi1Ft5Ih7hd9jjehod8vAM5GMsfQ/TpPf4EyoEA==} engines: {node: '>=14.14'} fs-minipass@3.0.3: @@ -4720,7 +4720,7 @@ snapshots: aggregate-error: 5.0.0 env-ci: 11.2.0 execa: 9.6.1 - fs-extra: 11.3.5 + fs-extra: 11.3.6 lodash-es: 4.18.1 nerf-dart: 1.0.0 normalize-url: 9.0.1 @@ -5997,7 +5997,7 @@ snapshots: dependencies: is-callable: 1.2.7 - fs-extra@11.3.5: + fs-extra@11.3.6: dependencies: graceful-fs: 4.2.11 jsonfile: 6.2.1 diff --git a/src/unique.ts b/src/unique.ts index d4e2d09..3f2478d 100644 --- a/src/unique.ts +++ b/src/unique.ts @@ -1,6 +1,11 @@ import { isAsyncIterable } from './predicate/isAsyncIterable.js'; import { isIterable } from './predicate/isIterable.js'; +export type MembershipStore = { + has(value: T): boolean; + add(value: T): void; +}; + export type UniqueOptions = { /** * Return a custom ID to uniquely identify an item. @@ -9,9 +14,19 @@ export type UniqueOptions = { /** * Return `true` to yield the item. * - * Use https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Iterator/filter when available. + * @todo Refactor to use `filter` on the (async)iterator when available. + * @see https://github.com/tc39/proposal-async-iterator-helpers + * @see https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Iterator/filter */ filter?: (item: Type) => boolean; + /** + * A store to keep track of which items have been seen. + * + * If you have a large dataset, you may want to use a more memory-efficient store than a `Set`, such as a Bloom filter. + * + * @default `Set`. + */ + store?: MembershipStore; }; export function unique(items: AsyncIterable, options?: UniqueOptions): AsyncIterable; @@ -27,8 +42,7 @@ export function unique( items: AsyncIterable | Iterable, options: UniqueOptions = {}, ): AsyncIterable | Iterable { - const ids = new Set(); - const { filter, identity } = options; + const { filter, identity, store = new Set() } = options; if (typeof items === 'string' || isIterable(items)) { return { @@ -37,8 +51,8 @@ export function unique( if (!filter || filter(item) === true) { const id = identity?.(item) ?? item; - if (!ids.has(id)) { - ids.add(id); + if (!store.has(id)) { + store.add(id); yield item; } @@ -55,8 +69,8 @@ export function unique( if (!filter || filter(item) === true) { const id = identity?.(item) ?? item; - if (!ids.has(id)) { - ids.add(id); + if (!store.has(id)) { + store.add(id); yield item; } diff --git a/test/unique.test.ts b/test/unique.test.ts index ea37420..0fbc830 100644 --- a/test/unique.test.ts +++ b/test/unique.test.ts @@ -1,6 +1,6 @@ -import { describe, expect, it } from 'vitest'; +import { describe, expect, it, vi } from 'vitest'; -import { unique } from '../src/unique.js'; +import { unique, type MembershipStore } from '../src/unique.js'; describe('unique()', () => { it('Yields unique items from an Iterable', () => { @@ -70,4 +70,38 @@ describe('unique()', () => { unique(false as unknown as Iterable); }).toThrow(); }); + + it('Can use a custom MembershipStore via the `store` option', () => { + const has = vi.fn((value: unknown) => value === 2); + const add = vi.fn(); + + const store: MembershipStore = { has, add }; + + expect([...unique([1, 2, 3], { store })]).toEqual([1, 3]); + + expect(has).toHaveBeenCalledWith(1); + expect(has).toHaveBeenCalledWith(2); + expect(has).toHaveBeenCalledWith(3); + expect(add).toHaveBeenCalledWith(1); + expect(add).not.toHaveBeenCalledWith(2); + expect(add).toHaveBeenCalledWith(3); + }); + + it('Shares uniqueness state across calls when the same `store` is reused', () => { + const store: MembershipStore = new Set(); + + expect([...unique([1, 2, 3], { store })]).toEqual([1, 2, 3]); + expect([...unique([2, 3, 4], { store })]).toEqual([4]); + }); + + it('Uses the `store` option with an AsyncIterable', async () => { + const demo = async function* (): AsyncGenerator { + yield 1; + yield 2; + yield 2; + yield 3; + }; + + await expect(Array.fromAsync(unique(demo(), { store: new Set([2]) }))).resolves.toEqual([1, 3]); + }); });