feat: ✨ allow json validation caching

Author: Thomas G.

package.json

@@ -185,14 +185,11 @@
         "*.php": [
             "composer lint"
         ],
-        "*.{js,jsx,ts,tsx,md,html,css,json,vue, yaml, yml, sh}": [
+        "*.{js,jsx,ts,tsx,md,html,css,vue,yaml,yml,sh}": [
             "npx --yes prettier --write"
         ],
         "*.json": [
-            "normalize-json -s -a -i -t 4 -l true"
-        ],
-        "package.json": [
-            "npx --yes sort-package-json"
+            "normalize-json -c -w -a -i -t 4 -f local -l true"
         ],
         "composer.lock": [
             "composer validate --ansi --strict"

src/common-utils/_configure-feature.sh

@@ -92,7 +92,7 @@ for package in package composer; do
 
     # Post merge normalize package.json
     zz_log i "Post-merge normalize {U $package.json}"
-    normalize-json -s -a -i -t ${tabSize:-4} $package.json
+    normalize-json -c -w -a -i -t ${tabSize:-4} $package.json
 done
 
 # Call all configure-xxx.sh scripts

src/common-utils/_default.schema.json

@@ -31,13 +31,19 @@
             }
         },
         "dependsOn": {
-            "type": "array",
+            "type": [
+                "array",
+                "object"
+            ],
             "items": {
                 "type": "string"
             }
         },
         "installsAfter": {
-            "type": "array",
+            "type": [
+                "array",
+                "object"
+            ],
             "items": {
                 "type": "string"
             }

src/common-utils/_devcontainer-feature.schema.json

@@ -8,28 +8,32 @@ set -e
 # Function to print help and manage arguments
 eval $(
     zz_args "Normalize JSON according to schema" $0 "$@" <<-help
-        s -         save	    save normalized json to original file
+        w -         write	    write normalized json to original file
         t tabSize   tabSize	    tab size for indentation
+        c -         cache	    allow caching of schema validation map
         a -         allow	    allow additional properties at root level
         d -         debug	    debug output
         f fallback  fallback	fallback schema to use if none found locally
         l local     local	    infer schema in <local> folder from json file name (x.y.json => <local>/y.schema.json)
         i -         import	    infer on schema store if nothing found locally (x.y.json => "y" on schema store)
-        - json	    json		json to normalize
-        + schema	schema		schema to use for normalization
+        s schema	schema		schema to use for normalization
+        + files	    files		jsons to normalize
+        
 help
 )
 
-# Validate JSON
-zz_log i "Normalizing JSON..."
-list=$(validate-json ${allow:+-a} ${debug:+-d} ${fallback:+-f "$fallback"} ${local:+-l "$local"} ${import:+-i} "$json" "$schema")
+for file in $files; do
+
+    # Validate JSON
+    zz_log i "Normalizing {U $file}..."
+    list=$(validate-json ${allow:+-a} ${cache:+-c} ${debug:+-d} ${fallback:+-f "$fallback"} ${local:+-l "$local"} ${import:+-i} ${schema:+-s"$schema"} $file)
 
-if test -z "$list"; then
-    zz_log e "JSON {U $json} not valid, cannot normalize" && exit 1
-fi
+    if test -z "$list"; then
+        zz_log e "JSON {U $file} not valid, cannot normalize" && exit 1
+    fi
 
-# Normalize JSON
-zz_json $json | jq -r --arg list "$list" '
+    # Normalize JSON
+    zz_json $file | jq -r --arg list "$list" '
         def transform($lst):
             $lst | split("\n") 
                 | map(select(length > 0))
@@ -54,17 +58,18 @@ zz_json $json | jq -r --arg list "$list" '
         
         traverse(transform($list))' >/tmp/$$.json
 
-# Handle output
-if test -s /tmp/$$.json; then
-    if test -z "$save"; then
-        jq -C --indent ${tabSize:-2} . /tmp/$$.json
+    # Handle output
+    if test -s /tmp/$$.json; then
+        if test -z "$write"; then
+            jq -C --indent ${tabSize:-2} . /tmp/$$.json
+        else
+            jq -M --indent ${tabSize:-4} . /tmp/$$.json >$file
+        fi
+        zz_log s "File {U $file} normalized"
     else
-        jq -M --indent ${tabSize:-4} . /tmp/$$.json >$json
+        zz_log e "File {U $file} not normalized"
     fi
-    zz_log s "File {U $json} normalized"
-else
-    zz_log e "File {U $json} not normalized"
-fi
 
-# Clean up
-rm -f /tmp/$$.*
+    # Clean up
+    rm -f /tmp/$$.*
+done

src/common-utils/_normalize-json.sh

@@ -8,11 +8,12 @@ eval $(
     zz_args "Validate JSON according to schema" $0 "$@" <<-help
         a -         allow	    allow additional properties at root level
         d -         debug	    debug output
+        c -         cache	    allow caching
         f fallback  fallback	fallback schema to use if none found locally
         l local     local	    infer schema in <local> folder from json file name (x.y.json => <local>/y.schema.json). Use "true" to use script folder
         i -         import	    infer on schema store if nothing found locally (x.y.json => "y" on schema store)
-        - json	    json		json to normalize
-        + schema	schema		schema to use for normalization
+        s schema	schema		schema to use to validate json
+        - json	    json		json to validate
 help
 )
 
@@ -455,43 +456,41 @@ validate() {
 }
 
 # if local flag is set, get schema from json file name
-if [ -n "$local" ]; then
+if [ -n "$local" ] && [ -z "$schema" ]; then
 
     # Identify package type from file name just before json extension
     type=$(basename -s .json $json | sed -E 's/.*\.(.*)/\1/')
 
     if [ "$local" == "true" ]; then
-        local=$(dirname $0)
+        local=$(dirname $(readlink -f $0))
     fi
 
-    # Identify schema file
-    schema=$local/_$type.schema.json
+    # Check if schema file exists
+    if [ -f "$local/_$type.schema.json" ]; then
+        schema=$local/_$type.schema.json
+    fi
 
     # log
     zz_log i "Infering schema from local folder {U $local} for {UYellow $json}"
 fi
 
 # Check if schema file exists, and if not and import allowed, download it from schema store
-if [ -n "$import" ] && [ ! -f "$schema" ]; then
+if [ -n "$import" ] && [ -z "$schema" ]; then
 
     search=$(basename -s .json $json | sed -E 's/.*\.(.*)/\1/').json
 
     # find schema file url from schema store catalog
     schema=$(get_schema_url $search)
 
     # log
-    zz_log i "Infering schema from schema store for {U $search} ${schema:+"found!"}"
+    zz_log i "Infering schema from schema store for {UYellow $search} ${schema:+"found!"}"
 fi
 
 # if schema file does not exist, use fallback schema
 if [ -n "$fallback" ] && [ -z "$schema" ]; then
 
-    if [ "$fallback" == "true" ]; then
-        if [ -n "$local" ]; then
-            schema=$(readlink -f $local)/_default.schema.json
-        else
-            schema=$(dirname $0)/_default.schema.json
-        fi
+    if [ "$fallback" == "local" ] && [ -n "$local" ]; then
+        schema=$(readlink -f $local)/_default.schema.json
     elif [ -f "$fallback" ]; then
         schema=$fallback
     else
@@ -502,14 +501,14 @@ if [ -n "$fallback" ] && [ -z "$schema" ]; then
     zz_log w "Using fallback schema {UYellow $schema}"
 fi
 
+# Download schema file and add id to it if not present
+schema=$(zz_json -s "$schema")
+
 # Check if schema is readable
 if test -z "$schema"; then
     zz_log e "Schema is missing" && exit 1
 fi
 
-# Download schema file and add id to it if not present
-schema=$(zz_json -s "$schema")
-
 # if schema is not a valid JSON, return error
 if ! is_json <<<"$schema"; then
     zz_log e "Invalid schema" && exit 1
@@ -521,9 +520,25 @@ if test -n "$allow"; then
     schema=$(echo "$schema" | jq '. + {"additionalProperties": true}' -)
 fi
 
-# Validate JSON according to schema and display valid json paths
-if validate "$json" "$schema"; then
-    zz_log s "File {U $json} valid"
+# is cache flag is set, cache schema
+hash=$(
+    (
+        jq 'paths | map(tostring) | join(".")' $json
+        echo "$schema" | jq 'paths | map(tostring) | join(".")'
+    ) | sort -u | md5sum | awk '{print $1}'
+)
+map=~/.cache/$hash.schema.map
+zz_log i "Hash is {B $hash}"
+
+if test -n "$cache" && test -s $map; then
+    zz_log i "Using cached validation map"
+    cat $map
 else
-    zz_log e "File {U $json} empty or invalid" && exit 1
-fi | sed -n -e 's/^.//g' -e '/^$/d' -e 'G; s/\n/&&/; /^\([ -~]*\n\).*\n\1/d; s/\n//; h; P'
+
+    # Validate JSON according to schema and display valid json paths
+    if validate "$json" "$schema"; then
+        zz_log s "File {U $json} valid"
+    else
+        zz_log e "File {U $json} empty or invalid" && exit 1
+    fi | sed -n -e 's/^.//g' -e '/^$/d' -e 'G; s/\n/&&/; /^\([ -~]*\n\).*\n\1/d; s/\n//; h; P' | tee $map
+fi

src/common-utils/_validate-json.sh

@@ -6,30 +6,30 @@
 lvl="$1" && shift
 
 case $lvl in
-i*)
-    picto="{BBlue →} "
-    base="White"
-    ;;
-w*)
-    picto="{BYellow !} "
-    base="Yellow"
-    ;;
-e*)
-    picto="{BRed ✕} "
-    base="Red"
-    ;;
-s*)
-    picto="{BGreen ✔} "
-    base="Green"
-    ;;
--)
-    picto="  "
-    base="White"
-    ;;
-*)
-    picto="$lvl "
-    base="White"
-    ;;
+    i*)
+        picto="{BBlue →} "
+        base="White"
+        ;;
+    w*)
+        picto="{BYellow !} "
+        base="Yellow"
+        ;;
+    e*)
+        picto="{BRed ✕} "
+        base="Red"
+        ;;
+    s*)
+        picto="{Green ✔} "
+        base="Green"
+        ;;
+    -)
+        picto="  "
+        base="White"
+        ;;
+    *)
+        picto="$lvl "
+        base="White"
+        ;;
 esac
 
 eval "$(

src/common-utils/_zz_log.sh

@@ -1,24 +1,16 @@
 {
+    "id": "common-utils",
     "name": "Common Utils",
-    "version": "3.4.3",
     "description": "Common utils for tomgrv/devcontainer-features",
+    "version": "3.5.1",
     "dependsOn": {
         "ghcr.io/devcontainers/features/common-utils": {}
     },
-    "id": "common-utils",
     "installsAfter": [
         "ghcr.io/devcontainers/features/common-utils"
     ],
-    "options": {
-        "utils": {
-            "type": "string",
-            "default": "jq dos2unix",
-            "description": "utilities to install"
-        }
-    },
     "customizations": {
         "vscode": {
-            // Set *default* container specific settings.json values on container create.
             "settings": {
                 "editor.formatOnSave": true,
                 "editor.formatOnPaste": true,
@@ -50,5 +42,12 @@
                 "ms-azuretools.vscode-docker"
             ]
         }
+    },
+    "options": {
+        "utils": {
+            "type": "string",
+            "default": "jq dos2unix",
+            "description": "utilities to install"
+        }
     }
-}
+}

src/common-utils/devcontainer-feature.json

@@ -6,11 +6,11 @@
         "*.php": [
             "composer lint"
         ],
-        "*.{js,jsx,ts,tsx,md,html,css,json,vue, yaml, yml, sh}": [
+        "*.{js,jsx,ts,tsx,md,html,css,vue,yaml,yml,sh}": [
             "npx --yes prettier --write"
         ],
         "*.json": [
-            "normalize-json -s -a -i -t 4 -l true"
+            "normalize-json -c -w -a -i -t 4 -f local -l true"
         ]
     }
 }

src/githooks/_lint-staged.package.json

@@ -2,7 +2,7 @@
     "id": "githooks",
     "name": "Git Hooks",
     "description": "A feature to add useful Git hooks to your project",
-    "version": "5.6.0",
+    "version": "5.7.2",
     "dependsOn": {
         "ghcr.io/devcontainers/features/node": {},
         "ghcr.io/tomgrv/devcontainer-features/common-utils:3": {},