refactor multithreading

Author: neverRare

src/dictionary/build.ts

@@ -1,9 +1,8 @@
 // this code is Deno only
 
-import { unreachable } from "@std/assert/unreachable";
 import { extractResultError, ResultError } from "../compound.ts";
 import { PositionedError } from "../parser/parser_lib.ts";
-import { HEADS, parseDictionary } from "./parser.ts";
+import { parseDictionary } from "./parallel_parser.ts";
 import { Dictionary } from "./type.ts";
 
 const SOURCE = new URL("../../dictionary.txt", import.meta.url);
@@ -28,69 +27,27 @@ export const dictionary: Dictionary = new Map(Object.entries(json));
 `;
   await Deno.writeTextFile(DESTINATION, code);
 }
-function buildOffloaded(src: string): Promise<Dictionary> {
-  return new Promise((resolve, reject) => {
-    const worker = new Worker(
-      new URL("./worker.ts", import.meta.url),
-      { type: "module" },
-    );
-    worker.postMessage(src);
-    worker.onmessage = (event) => {
-      resolve(event.data as Dictionary);
-      worker.terminate();
-    };
-    worker.onerror = (event) => {
-      reject(event.error);
-    };
-  });
-}
 export async function build(): Promise<boolean> {
   // deno-lint-ignore no-console
   console.log(
     `Building dictionary with ${navigator.hardwareConcurrency} threads...`,
   );
   const start = performance.now();
   const text = await Deno.readTextFile(SOURCE);
-  const heads = [...text.matchAll(HEADS)].map((match) => match.index);
-  const regionIndices = [...new Array(navigator.hardwareConcurrency).keys()]
-    .map((index) => {
-      const start = index * text.length / navigator.hardwareConcurrency;
-      for (const head of heads) {
-        if (start <= head) {
-          return head;
-        }
-      }
-    });
-  const regions = regionIndices.map((index, i) =>
-    text.slice(index, regionIndices[i + 1] ?? text.length)
-  );
-  const dictionary: Dictionary = new Map();
+  let dictionary: Dictionary;
   try {
-    const entries = await Promise.all(
-      regions.map((region) => buildOffloaded(region)),
-    );
-    for (const entry of entries) {
-      for (const [name, definition] of entry) {
-        if (dictionary.has(name)) {
-          throw new Error();
-        }
-        dictionary.set(name, definition);
-      }
-    }
-  } catch (_) {
-    try {
-      parseDictionary(text);
-    } catch (error) {
-      displayError(`${SOURCE}`, extractResultError(error));
-      return false;
-    }
-    unreachable();
+    dictionary = await parseDictionary(text);
+  } catch (error) {
+    displayError(text, extractResultError(error));
+    return false;
   }
   await buildWithDictionary(dictionary);
   const end = performance.now();
   const total = Math.floor(end - start);
   // deno-lint-ignore no-console
-  console.log(`Building dictionary done in ${total}ms`);
+  console.log(
+    `Building dictionary done in ${total}ms`,
+  );
   return true;
 }
 function displayError(source: string, errors: ReadonlyArray<ResultError>) {

src/dictionary/parallel_parser.ts

@@ -0,0 +1,102 @@
+import { unreachable } from "@std/assert/unreachable";
+import { extractResultError, ResultError } from "../compound.ts";
+import { Position, PositionedError } from "../parser/parser_lib.ts";
+import { HEADS } from "./parser.ts";
+import { Dictionary } from "./type.ts";
+
+type WorkerError =
+  | Readonly<
+    {
+      type: "positioned error";
+      errors: ReadonlyArray<Position & Readonly<{ message: string }>>;
+    }
+  >
+  | Readonly<{ type: "other"; error: unknown }>;
+
+function buildOffloaded(src: string): Promise<Dictionary> {
+  return new Promise((resolve, reject) => {
+    const worker = new Worker(
+      new URL("./worker.ts", import.meta.url),
+      { type: "module" },
+    );
+    worker.postMessage(src);
+    worker.onmessage = (event) => {
+      resolve(event.data as Dictionary);
+      worker.terminate();
+    };
+    worker.onerror = (event) => {
+      const error = event.error as WorkerError;
+      switch (error.type) {
+        case "positioned error":
+          reject(
+            new AggregateError(
+              error.errors.map((error) =>
+                new PositionedError(error.message, error)
+              ),
+            ),
+          );
+          break;
+        case "other":
+          reject(error.error);
+          break;
+      }
+    };
+  });
+}
+export async function parseDictionary(src: string): Promise<Dictionary> {
+  const heads = [
+    ...[...src.matchAll(HEADS)].map((match) => match.index),
+    src.length,
+  ];
+  const regionIndices = [...new Array(navigator.hardwareConcurrency).keys()]
+    .map((index) => {
+      const start = index * src.length / navigator.hardwareConcurrency;
+      for (const head of heads) {
+        if (start <= head) {
+          return head;
+        }
+      }
+      unreachable();
+    });
+  const jobs = regionIndices.map((index, i) => ({
+    index: index,
+    job: buildOffloaded(src.slice(index, regionIndices[i + 1] ?? src.length)),
+  }));
+  const dictionary: Dictionary = new Map();
+  const errors: Array<ResultError> = [];
+  for (const job of jobs) {
+    let entries: Dictionary;
+    try {
+      // deno-lint-ignore no-await-in-loop
+      entries = await job.job;
+    } catch (error) {
+      for (const resultError of extractResultError(error)) {
+        if (
+          resultError instanceof PositionedError && resultError.position != null
+        ) {
+          errors.push(
+            new PositionedError(resultError.message, {
+              position: job.index + resultError.position.position,
+              length: resultError.position.length,
+              cause: resultError,
+            }),
+          );
+        } else {
+          errors.push(resultError);
+        }
+      }
+      continue;
+    }
+    if (errors.length === 0) {
+      for (const [word, definition] of entries.entries()) {
+        if (dictionary.has(word)) {
+          errors.push(new ResultError(`duplicate Toki Pona word "${word}"`));
+          break;
+        } else {
+          dictionary.set(word, definition);
+        }
+      }
+    }
+  }
+  return dictionary;
+}

src/dictionary/worker.ts

@@ -1,5 +1,19 @@
+import { extractResultError, ResultError } from "../compound.ts";
 import { parseDictionary } from "./parser.ts";
 
 onmessage = (message) => {
-  postMessage(parseDictionary(message.data as string));
+  try {
+    postMessage(parseDictionary(message.data as string));
+  } catch (error) {
+    let errors: ReadonlyArray<ResultError>;
+    try {
+      errors = extractResultError(error);
+    } catch (error) {
+      throw { type: "other", error };
+    }
+    throw {
+      type: "positioned error",
+      error: errors.map((error) => ({ ...error })),
+    };
+  }
 };

src/parser/parser_lib.ts

@@ -106,6 +106,7 @@ export class PositionedError extends ResultError {
   position: null | Position;
   constructor(
     message: string,
+    // TODO: it's better to separate these two instead
     option?: Position & ErrorOption,
   ) {
     super(message, option);