initial commit

Author: michaelgoeke

.github/workflows/ci.yml

@@ -0,0 +1,29 @@
+name: CI
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: true
+          clean: true
+
+      - name: Setup Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: '20.x'
+
+      - name: Authenticate with npm
+        run: |
+          if [ -f .npmrc ]; then echo "Using .npmrc for npm authentication"; else echo "No .npmrc found, skipping authentication"; fi
+
+      - name: Run build script
+        run: ./azure-build.sh

.gitignore

@@ -0,0 +1,3 @@
+node_modules
+dist
+sqljs-documentstore-*.tgz

.npmignore

@@ -0,0 +1,2 @@
+tsconfig.json
+src

GitVersion.yml

@@ -0,0 +1,5 @@
+mode: Mainline
+branches: {}
+ignore:
+  sha: []
+merge-message-formats: {}

README.md

@@ -0,0 +1,43 @@
+# @dnvgl-electricgrid/sqljs-documentstore
+
+`@dnvgl-electricgrid/sqljs-documentstore` Minimal, encrypted, sql friendly typed document store, with support for indexed columns. Protects against transactional conflicts.  
+The schema for each table is [id, json, ...indexedColumns]
+
+
+## Install
+`npm install --save @dnvgl-electricgrid/sqljs-documentstore`
+
+## Usage
+
+exported pieces:  
+* `Db` abstract class, extend it and add `TypedDocumentStore`s for each table => class mapping
+* `TypedDocumentStore` is meant to map a class type to a table, and has read and write methods
+* `LockedDatabase` is consumed in TypedDocumentStore. This is a transactional wrapper around sql.js. It handles transactional scope for multiple programmatic writes and ensures locking so separate txns can't collide.
+* `sqlHelpers` has methods to load, save, etc
+
+-----
+### Getting started
+
+first, load your database
+recommend creating a wrapping "DocumentStores" class as shown below:
+```ts
+const database = await sqljsHelpers.load('mydb', await sqlite());
+const lockedDatabase = new LockedDatabase(database, () => sqljsHelpers.save('mydb', database));
+
+public class DocumentStores extends Db {
+  public CustomerData = new TypedDocumentStore(this.db, 'CustomerData', <CustomerData>{}, {
+    name: x => x.name,
+    orderCount: x => x.orders?.length
+  }).asInterface;
+
+  //... more table declarations
+
+  
+  ///////
+  private static instance: DocumentStores;
+  public static get Instance(): DocumentStores { return this.instance ?? (this.instance = new this()); }
+  constructor() { super(lockedDatabase); }
+}
+```
+
+see [examples.md] (https://github.com/dnvgl-opensource/sqljs-documentstore/docs/examples.md) for usage examples.

azure-build.sh

@@ -0,0 +1,11 @@
+set -x
+dotnet tool install --global GitVersion.Tool --version 5.*
+npm install -g yarn
+
+GitVersion_SemVer=`dotnet-gitversion /showvariable SemVer`
+npm version $GitVersion_SemVer --no-git-tag-version
+echo "##vso[build.updatebuildnumber]$GitVersion_SemVer"
+yarn
+yarn bundle
+
+npm publish

docs/examples.md

@@ -0,0 +1,37 @@
+Example sqljs-documentstore usages
+
+``` ts
+  interface CustomerData {
+      id: string;
+      name: string;
+      address: string;
+      orders: any[];//OrderData[];
+  }
+
+  // provide access to a CustomerData documentstore, with columns defined for name and orderCount
+  let db: ILockedDatabase;
+  const CustomerData = new TypedDocumentStore(() => db, 'CustomerData', <CustomerData>{}, {
+    name: x => x.name,
+    orderCount: x => x.orders?.length
+  }).asInterface;
+
+  
+  // usage calling examples
+  async function exampleUsage() {
+    const allCustomers = await CustomerData.getAll();
+
+    const dave_id = 'abc123';
+    const dave = await CustomerData.get(dave_id); //prefer use of 'get' if data is expected to exist, it will throw if it doesn't exist
+
+    const david_id = dave_id + 'id';
+    const david = await CustomerData.tryGet(dave_id); //this will return a T? and you'll have to assert it's returned what you want
+
+    db.txn('update Dave', async txnId => {
+      CustomerData.update(txnId, dave_id, x => {
+        x.name += ' the great';
+        x.address = 'cloud 9';
+        x.orders = [];
+      })
+    });
+  }
+  ```

package.json

@@ -0,0 +1,40 @@
+{
+  "name": "@dnvgl-electricgrid/sqljs-documentstore",
+  "version": "1.0.0",
+  "description": "Minimal, encrypted, sql friendly typed document store, with support for indexed columns. Protects against transactional conflicts",
+  "main": "dist/index.js",
+  "types": "dist/index.d.ts",
+  "files": [
+    "/dist",
+    "/src"
+  ],
+  "scripts": {
+    "bundle": "tsc"
+  },
+  "keywords": [
+    "SQLite",
+    "DocumentStore",
+    "Document",
+    "Store",
+    "ORM",
+    "Micro",
+    "Typescript",
+    "Generic",
+    "ES7"
+  ],
+  "license": "MIT",
+  "dependencies": {
+    "async-lock": "^1.4.1",
+    "idb-keyval": "^6.2.2",
+    "lodash": "^4.17.21",
+    "sql.js": "^1.13.0"
+
+  },
+  "devDependencies": {
+    "@types/async-lock": "^1.4.2",
+    "@types/lodash": "^4.17.17",
+    "@types/sql.js": "^1.4.9",
+    "typescript": "^5.8.3"
+  },
+  "packageManager": "yarn@1.22.22+sha512.a6b2f7906b721bba3d67d4aff083df04dad64c399707841b7acf00f6b133b7ac24255f2652fa22ae3534329dc6180534e98d17432037ff6fd140556e2bb3137e"
+}

src/Db.ts

@@ -0,0 +1,33 @@
+import { isTypedDocumentStore } from "./TypedDocumentStore";
+import { ILockedDatabase } from "./LockedDatabase";
+import { sqljsHelpers } from "./sqljsHelpers";
+
+export abstract class Db {
+    constructor(private database: () => ILockedDatabase) { }
+
+    public txn(description: string, actions: (txnId: string) => void): void { return this.database().txn(description, txnId => actions(txnId)); }
+    public async txnAsync(description: string, actions: (txnId: string) => Promise<void>): Promise<void> { return this.database().txnAsync(description, txnId => actions(txnId)); }
+    
+    public initted = false;
+    public init() { Object.values(this).filter(value => isTypedDocumentStore(value)).forEach(store => store.init()); this.initted = true; }
+
+   /**
+   * Used to query from console, or for advanced querying scenarios in code
+   * @example //from console
+   * console.table(await window.db.query('select * from FieldDefinitionData f join CalculatedFieldDefinitionData c on f.id = c.fieldId;'));
+   * @example //from code - advanced query - join tables and union the data types:
+   * await doc.query<FieldDefinitionData & CalculatedFieldDefinitionData, {fJson: string, cJson: string}>(
+   * 'select f.json fJson, c.json cJson from FieldDefinitionData f join CalculatedFieldDefinitionData c on f.id = c.fieldId',
+   * undefined, row => _.merge(<FieldDefinitionData>JSON.parse(row.fJson), <CalculatedFieldDefinitionData>JSON.parse(row.cJson)));
+   * @example //from code - less type checking:
+   * <(FieldDefinitionData & CalculatedFieldDefinitionData)[]> await doc.query(
+   * 'select f.json fJson, c.json cJson from FieldDefinitionData f join CalculatedFieldDefinitionData c on f.id = c.fieldId',
+   * undefined, row => _.merge(JSON.parse(row.fJson), JSON.parse(row.cJson)));
+   */
+    public query<T = unknown, TRow = T>(sql: string, params?: unknown[], projection?: (row: TRow) => T): T[] {
+        const result = sqljsHelpers.query<TRow>(this.database(), sql, params);
+        if (result.length == 0) return [];
+        if (!projection) return <T[]><any>result;
+        return result.map(row => projection(row));
+    }
+}

src/LockedDatabase.ts

@@ -0,0 +1,78 @@
+import {BindParams, QueryExecResult, type Database} from 'sql.js';
+import AsyncLock from "async-lock";
+
+export class LockedDatabase implements ILockedDatabase {
+  lock = new AsyncLock();
+  config: configType = {};
+  state: {queued: queuedItemType[]} = { queued: [] };
+  txnId?: string;
+
+  constructor(private db: Pick<Database, 'exec'|'run'>, private flush: () => void) {}
+
+  run(txnId: string, sql: string, params: BindParams = []): void {
+    if (txnId !== this.txnId) throw new Error(`run: transaction doesn't match, txn in progress: ${this.txnId}, attempted txn: ${this.txnId}`);
+    this.db.run(sql, params);
+  }
+
+  exec(sql: string, params: BindParams = []): QueryExecResult[] { return this.db.exec(sql, params); }
+
+  /**
+   * all write operations must be wrapped in a transaction
+   * txns are locked using async-lock library to avoid bleeding across session/txn
+   */
+  async txnAsync(description: string, actions: (txnId: string) => Promise<void>): Promise<void> {
+    const txnId = `${description} ${Math.floor(Math.random() * 4294967296).toString(16)}`; //generate a unique but identifiable txnId
+    const queuedItem = { txnId, description, timing: { waitMs: 0, actionMs: 0, flushMs: 0 } };
+    this.state.queued.push(queuedItem);
+    var ms = performance.now();
+    await this.lock.acquire('txn_lock', async () => {
+      try {
+        queuedItem.timing.waitMs = performance.now() - ms; ms = performance.now();
+        this.txnId = txnId;
+        this.run(txnId, 'BEGIN TRANSACTION;');
+        await actions(txnId);
+        queuedItem.timing.actionMs = performance.now() - ms; ms = performance.now();
+        this.run(txnId, 'COMMIT TRANSACTION;');
+        this.flush();
+        queuedItem.timing.flushMs = performance.now() - ms;
+      } catch (error) {
+        this.run(txnId, 'ROLLBACK TRANSACTION;');
+        throw error;
+      } finally {
+        this.txnId = undefined;
+        this.state.queued.pop();
+        this.config.loggingHook?.(queuedItem);
+      }
+    });
+  }
+
+  txn(description: string, actions: (txnId: string) => void): void {
+    const txnId = `${description} ${Math.floor(Math.random() * 4294967296).toString(16)}`; //generate a unique but identifiable txnId
+    const queuedItem = { txnId, description, timing: { waitMs: 0, actionMs: 0, flushMs: 0 } };
+    this.state.queued.push(queuedItem);
+    var ms = performance.now();
+    if (this.lock.isBusy('txn_lock')) throw new Error(`_txnSync: transaction already in progress, txn in progress: ${this.txnId}, attempted txn: ${txnId}`);
+    
+    try {
+      queuedItem.timing.waitMs = performance.now() - ms; ms = performance.now();
+      this.txnId = txnId;
+      this.run(txnId, 'BEGIN TRANSACTION;');
+      actions(txnId);
+      queuedItem.timing.actionMs = performance.now() - ms; ms = performance.now();
+      this.run(txnId, 'COMMIT TRANSACTION;');
+      this.flush();
+      queuedItem.timing.flushMs = performance.now() - ms;
+    } catch (error) {
+      this.run(txnId, 'ROLLBACK TRANSACTION;');
+      throw error;
+    } finally {
+      this.txnId = undefined;
+      this.state.queued.pop();
+      this.config.loggingHook?.(queuedItem);
+    }
+  }
+}
+
+export interface ILockedDatabase extends LockedDatabase{};
+export interface configType { loggingHook?: (q: queuedItemType) => Promise<void> };
+export interface queuedItemType {txnId: string, description: string, timing: { waitMs: number, actionMs: number, flushMs: number}};