fix docs and examples

Author: dem1fo

README.md

@@ -10,6 +10,7 @@ This repository is designed to be used with file-based data programmatically; fo
 📦common-identifier-algorithm-shared
  ┣ 📂src
  ┃ ┣ 📂config     # functions related to the handling of configuration files
+ ┃ ┣ 📂crypto     # functions related to encryption & signing of files
  ┃ ┣ 📂decoding   # reading and decoding files - CSV or XLSX
  ┃ ┣ 📂encoding   # encoding and writing files - CSV or XLSX
  ┃ ┣ 📂hashing    # base hashing logic and supporting utilities
@@ -46,4 +47,8 @@ This repository is designed to be used with file-based data programmatically; fo
 - The `src/processing` processing function identifies if the target is a mapping document based on the current configuration and the data in the file. Using the active configuration it collects data into `static` `to_translate` and `reference` buckets per-row and passes it to the active algorithm for processing
 - The active algorithm takes the `{ static:[...], to_translate:[...], reference: [...] }` per-row data and returns a map with the columns it wants to add -- ex: `{ USCADI: "....", DOCUMENT_HASH: "..." }`
 - The data returned by the algorithm is merged into the source rows so the encoders can package multiple different outputs
-- The `src/encoding` Encoders (CSV and XLSX) write the output based on the relevant `[destination]` / `[destination_map]` section of the active configuration.
+- The `src/encoding` Encoders (CSV and XLSX) write the output based on the relevant `[destination]` / `[destination_map]` section of the active configuration.
+
+### Post-processing
+
+- Any post-processing steps (e.g. encryption, signing, etc.) are handled after the encoding step, using the relevant classes in `src/crypto` (e.g. `GpgWrapper` for GPG encryption/signing)

docs/configuration-files.md

@@ -10,26 +10,24 @@ Look in any of the existing algorithm implementation repositories for examples o
 
 The meta section of the file contains information related to application versioning.
 
-```
+```toml
 [meta]
-region="ABC"    # application installations are region dependent, the value specified here MUST match the built-in region.
+id="ABC"    # application installations are typically region dependent, the value specified here MUST match the built-in region.
 version="1.0.0" # the version information is shown in the top-right of the desktop UI for user visibility.
 signature = "aaabbb"
 ```
-The signature is the calculated `md5` hash value of the configuration file itself, computed using the `src/config/generateConfigHash.ts` utility. This feature exists to reduce the likelihood of accidental changes to the config file causing issues in the deterministic processing of input data.
+The signature is the calculated `md5` hash value of the configuration file itself, computed using the [`generateConfigHash`](../src/config/utils.ts) utility. This feature exists to reduce the likelihood of accidental changes to the config file causing issues in the deterministic processing of input data.
 
 When a change is made to the configuration file, a new signature value must be created to reflect its new content.
 
-> TODO: git pre-commit hook to validate the config file on commit and throw an error if values don't match.
-
 ### Messages
 
 > [!IMPORTANT]
 > If you are intending on using the UI component of this application, the messages section in the configuration file is NOT optional. The values for `terms_and_conditions`, `error_in_config`, and `error_in_salt` MUST be defined.
 
 Messages are an optional field used to set the default error and terms & conditions messages within the UI application. Each of these fields supports `HTML` tag syntax.
 
-```
+```toml
 [messages]
 # terms and conditions are shown to the user on first start and upon configuration file changes.
 terms_and_conditions="""
@@ -46,9 +44,7 @@ error_in_salt=""
 
 The `source` sections defines the expected input columns in the source dataset. The `name` key is the human readable name in the header of the CSV file, `alias` is the more machine-friendly name used by the application internally, and `default` is the default value to use for empty cells where necessary.
 
-> TODO: make `alias` an optional parameter - it is not relevant for programmatic usage.
-
-```
+```toml
 [source]
 # an array of column names, their aliases, and default values where necessary.
 columns = [
@@ -62,7 +58,7 @@ columns = [
 
 This file section details which validation rules to apply to which columns in the input file.
 
-```
+```toml
 [validations]
 # per column name to apply validation rules to, define an array of validation rules
 column_name = [
@@ -74,7 +70,7 @@ column_name = [
 ]
 ```
 
-```
+```toml
 # the structure of a validation rule is as follows:
 {
     # the name of the validation rule, from the supported list.
@@ -107,7 +103,7 @@ This is the list of currently supported validation rules, these are further desc
 
 ### Algorithm
 
-```
+```toml
 [algorithm]
 # the aliased columns to use as part of the algo implementation.
 [algorithm.columns]
@@ -119,7 +115,7 @@ static = [ "col_b" ]
 reference = [ "col_c" ]
 ```
 
-```
+```toml
 [algorithm.hash]
 # the hashing algorithm to use, currently only SHA256 is supported
 strategy = "SHA256"
@@ -146,7 +142,7 @@ darwin = "$APPDATA/<path_to_file>/file.asc"
 
 Define the columns to include in the output file, including the human-readable names to convert to where necessary.
 
-```
+```toml
 [destination]
 # array of column names and aliases to include in the output file
 columns = [
@@ -163,7 +159,7 @@ postfix = "_OUTPUT"
 
 Define the columns to include in the output mapping file, including the human-readable names to convert to where necessary.
 
-```
+```toml
 [destination_map]
 # array of column names and aliases to include in the output mapping file
 columns = [
@@ -183,7 +179,7 @@ Define the columns to include in the error report, including the human-readable
 > [!IMPORTANT]
 > Make sure to include the `Errors | errors` column in the output configuration, otherwise they will not be included in the output file.
 
-```
+```toml
 [destination_errors]
 # array of column names and aliases to include in the errors file
 columns = [
@@ -195,4 +191,15 @@ columns = [
 # suffix that is appended to the errors filename -> <input_file_name><postfix>.csv
 # for XLSX, this is also the name of the sheet containing the final data
 postfix = "_ERRORS"
+```
+
+### Post-processing
+
+#### Encryption / Signing
+
+```toml
+[post_processing]
+[post_processing.encryption]
+recipient = "QWERTY" # the fingerprint or identifier of the recipient to encrypt the file for
+gpgBinaryPath = "" # optional path to gpg binary, overrides system path lookup
 ```

examples/example_algorithm/config.toml

@@ -1,7 +1,7 @@
 [meta]
 id="ANY"
 version="1.0.0"
-signature="a36d008f81060928709a6704da61bbd6"
+signature="f2789d9afdb774d88be5998aa37bdb78"
 
 [source]
 columns = [
@@ -35,18 +35,25 @@ strategy = "SHA256"
 columns = [
     { name = "Common Identifier", alias = "hashed_id" },
 ]
-postfix = "-OUTPUT-{{yyyy-MM-dd--HH-mm-ss}}"
+prefix = "OUTPUT-"
+postfix = "-{{yyyyMMddHHmm}}"
 
 [destination_map]
 columns = [
     { name = "ID",       alias = "id" },
     { name = "Common Identifier", alias = "hashed_id" }
 ]
-postfix = "-MAPPING-{{yyyy-MM-dd--HH-mm-ss}}"
+prefix = "MAPPING-"
+postfix = "-{{yyyyMMddHHmm}}"
 
 [destination_errors]
 columns = [
     { name = "Errors",            alias = "errors" },
     { name = "ID",                alias = "id" },
 ]
-postfix = "-ERRORS-{{yyyy-MM-dd--HH-mm-ss}}"
+prefix = "ERRORS-"
+postfix = "-{{yyyyMMddHHmm}}"
+
+# [post_processing]
+# [post_processing.encryption]
+# recipient = "SOME_PUBLIC_KEY_FINGERPRINT"

examples/file_based_usage.ts

@@ -1,10 +1,9 @@
 // REPLACE ALL REFERENCES TO "_generic_hasher" WITH THE DESIRED ALGORITHM IN THE ALGORITHMS DIRECTORY.
 
 import { existsSync, rmSync } from 'node:fs';
-import { encryptFileWithGpg } from '../src/crypto/gpg';
-import { loadConfig, preprocessFile, processFile } from '../src/index';
+import { loadConfig, postprocessFile, preprocessFile, processFile } from '../src/index';
 import { makeHasher, ALGORITHM_ID } from './example_algorithm/_generic_hasher';
-import { dirname, join, resolve } from 'node:path';
+import { dirname, join, parse, resolve } from 'node:path';
 import { fileURLToPath } from 'node:url';
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -16,45 +15,50 @@ const VALIDATION_ERRORS_PATH = join(__dirname, 'output', 'validation_errors.csv'
 
 if (existsSync(resolve(__dirname, "output"))) rmSync(join(__dirname, "output"), { recursive: true, force: true});
 
-// load configuration from file
+// 1. load configuration from file
 const configLoadResult = loadConfig({
   configPath: CONFIG_PATH,
   algorithmId: ALGORITHM_ID,
   validateConfig: true,
   embeddedSalt: { source: "STRING", value: "SOME_SALT_VALUE" }
 });
-if (!configLoadResult.success) throw new Error(`ERROR: Unable to load configuration file >> ${configLoadResult.error}`);
+if (!configLoadResult.success)
+  throw new Error(`ERROR: Unable to load configuration file >> ${configLoadResult.error}`);
 
-// validate the input file against all configured validation rules.
+// 2. validate the input file against all configured validation rules.
 const preprocessResult = await preprocessFile({
   config: configLoadResult.config,
   inputFilePath: INPUT_PATH,
   errorFileOutputPath: VALIDATION_ERRORS_PATH,
 });
 
 if (!preprocessResult.isValid)
-  throw new Error('Validation errors found in input file, review error file output.');
+  throw new Error('ERROR: Validation errors found in input file, review error file output.');
 
-// validate the input file against all configured validation rules.
+// 3. process the input file according to the configuration.
 const processFileResult = await processFile({
   config: configLoadResult.config,
   inputFilePath: INPUT_PATH,
   outputPath: OUTPUT_PATH,
   hasherFactory: makeHasher,
 });
-// print the result, save the result, etc.
-// console.dir(processFileResult, { depth: 3 });
-
-
-const result = await encryptFileWithGpg({
-  inputPath: processFileResult.outputFilePath!,
-  outputPath: processFileResult.outputFilePath! + '.gpg',
-  gpgPath: "gpg",
-  recipient: '5A42A8421D19427562C5DEBE0CC75DCC440E1B46',
-  // recipient: "9463D5547670C915D23671A8DA57DA502A841FEF",
-  homedir: process.env.HOMEDIR,
-  // signer: "5A42A8421D19427562C5DEBE0CC75DCC440E1B46"
-  signer: "9463D5547670C915D23671A8DA57DA502A841FEF",
-  trustAlways: true,
+if (!processFileResult.outputFilePath)
+  throw new Error(`ERROR: Unable to process input file`);
+
+// 4. print the result, save the result, etc.
+console.dir(processFileResult, { depth: 3 });
+
+
+// 5. postprocess the output file according to the configuration.
+const postprocessResult = await postprocessFile({
+  config: configLoadResult.config,
+  inputPath: processFileResult.outputFilePath,
+  outputPath: join(parse(processFileResult.outputFilePath).dir, "ENCRYPTED.gpg"),
+  options: {
+    signer: "" // OPTIONAL: Signing key 
+  }
 });
-console.log(result)
+if (!postprocessResult.success) {
+  const errors = postprocessResult.steps.filter(step => !step.success);
+  throw new Error(`ERROR: Postprocessing failed, one or more steps failed >> ${JSON.stringify(errors)}`);
+}

examples/programmatic_usage.ts

@@ -1,6 +1,6 @@
 // REPLACE ALL REFERENCES TO "_generic_hasher" WITH THE DESIRED ALGORITHM IN THE ALGORITHMS DIRECTORY.
-import { generateHashesForDocument, validateConfigCore, validateDocument, type CidDocument, type Config } from '../src/index';
-import { SUPPORTED_VALIDATORS } from '../src/validation/Validation';
+import { generateHashesForDocument, validateConfigCore, validateDocument, type CidDocument, type Config } from '@/index';
+import { SUPPORTED_VALIDATORS } from '@/validation/Validation';
 import { makeHasher } from './example_algorithm/_generic_hasher';
 
 /*
@@ -39,9 +39,6 @@ const config: Config.CoreConfiguration = {
       reference: [],
       static: [ "id" ]
     },
-  },
-  post_processing: {
-    
   }
 }
 
@@ -57,7 +54,7 @@ const doc: CidDocument = {
   ]
 }
 
-function main() {
+async function main() {
   const configValidationResult = validateConfigCore(config, "UNKNOWN");
   if (!!configValidationResult) {
     console.log(`ERROR: ${configValidationResult}`);
@@ -78,6 +75,9 @@ function main() {
 
   // print the results, save the results, up to you.
   console.dir(result, { depth: 5 });
+
+  // NOTE: do any postprocessing steps as required, e.g. encryption, signing, etc.
+  //       see the GpgWrapper class in src/crypto/gpg.ts for example usage.
 }
 
 main();

tsconfig.json

@@ -26,6 +26,6 @@
 
     "noEmit": true
   },
-  "include": ["src/**/*.ts", "tests/**/*.ts" ],
+  "include": ["src/**/*.ts", "tests/**/*.ts", "examples/**/*.ts" ],
   "exclude": ["node_modules", "dist", "tmp", "**/tmp"]
 }