Merkle log support (#259)

* Merkle log support

Confirmation status is now an enumerated type.
Confirmation logic tests for both CONFIRMED(simple_hash) or
COMMITTED (merkle_log).

Added suitable unit ahd func tests.

Fixes AB#9103

Author: eccles

Taskfile.yml

@@ -10,8 +10,6 @@ tasks:
     desc: Generate about.py
     cmds:
       - ./scripts/builder.sh ./scripts/version.sh
-    status:
-      - test -s archivist/about.py
 
   audit:
     desc: Audit the code
@@ -54,7 +52,7 @@ tasks:
       - task: check-pyright
 
   check-pyright:
-    desc: Execute pyright conditinally - currently does not work in 3.12 (https://github.com/ekalinin/nodeenv/issues/341)
+    desc: Execute pyright conditionally - currently does not work in 3.12 (https://github.com/ekalinin/nodeenv/issues/341)
     cmds:
       - |
         if [ "{{.PYVERSION}}" != "3.12" ]; then

archivist/confirmation_status.py

@@ -0,0 +1,27 @@
+"""Archivist Confirmation Status
+
+   Enumerated type that allows user to select the confirmation status option when
+   creating an asset.
+
+"""
+
+# pylint: disable=unused-private-member
+
+from enum import Enum
+
+
+class ConfirmationStatus(Enum):
+    """Enumerate confirmation status options"""
+
+    UNSPECIFIED = 0
+    # not yet committed
+    PENDING = 1
+    CONFIRMED = 2
+    # permanent failure
+    FAILED = 3
+    # forestrie, "its in the db"
+    STORED = 4
+    # forestrie, "you can know if its changed"
+    COMMITTED = 5
+    # forestrie, "You easily prove it was publicly available to all"
+    UNEQUIVOCAL = 6

archivist/confirmer.py

@@ -15,12 +15,8 @@
     from .events import Event, _EventsPublic, _EventsRestricted
 
 
-from .constants import (
-    CONFIRMATION_CONFIRMED,
-    CONFIRMATION_FAILED,
-    CONFIRMATION_PENDING,
-    CONFIRMATION_STATUS,
-)
+from .confirmation_status import ConfirmationStatus
+from .constants import CONFIRMATION_STATUS
 from .errors import ArchivistUnconfirmedError
 from .utils import backoff_handler
 
@@ -54,25 +50,29 @@ def __on_giveup_confirmation(details: "Details"):
 
 @overload
 def _wait_for_confirmation(
-    self: "_AssetsRestricted", identity: str
+    self: "_AssetsRestricted",
+    identity: str,
 ) -> "Asset": ...  # pragma: no cover
 
 
 @overload
 def _wait_for_confirmation(
-    self: "_AssetsPublic", identity: str
+    self: "_AssetsPublic",
+    identity: str,
 ) -> "Asset": ...  # pragma: no cover
 
 
 @overload
 def _wait_for_confirmation(
-    self: "_EventsRestricted", identity: str
+    self: "_EventsRestricted",
+    identity: str,
 ) -> "Event": ...  # pragma: no cover
 
 
 @overload
 def _wait_for_confirmation(
-    self: "_EventsPublic", identity: str
+    self: "_EventsPublic",
+    identity: str,
 ) -> "Event": ...  # pragma: no cover
 
 
@@ -94,12 +94,20 @@ def _wait_for_confirmation(self: Managers, identity: str) -> ReturnTypes:
             f"cannot confirm {identity} as confirmation_status is not present"
         )
 
-    if entity[CONFIRMATION_STATUS] == CONFIRMATION_FAILED:
+    if entity[CONFIRMATION_STATUS] == ConfirmationStatus.FAILED.name:
         raise ArchivistUnconfirmedError(
             f"confirmation for {identity} FAILED - this is unusable"
         )
 
-    if entity[CONFIRMATION_STATUS] == CONFIRMATION_CONFIRMED:
+    # Simple hash
+    if entity[CONFIRMATION_STATUS] == ConfirmationStatus.CONFIRMED.name:
+        return entity
+
+    # merkle_log
+    if (
+        ConfirmationStatus[entity[CONFIRMATION_STATUS]].value
+        >= ConfirmationStatus.COMMITTED.value
+    ):
         return entity
 
     return None  # pyright: ignore
@@ -122,21 +130,24 @@ def __on_giveup_confirmed(details: "Details"):
     on_giveup=__on_giveup_confirmed,
 )
 def _wait_for_confirmed(
-    self: PrivateManagers, *, props: "dict[str, Any]|None" = None, **kwargs: Any
+    self: PrivateManagers,
+    *,
+    props: "dict[str, Any]|None" = None,
+    **kwargs: Any,
 ) -> bool:
     """Return False until all entities are confirmed"""
 
     # look for unconfirmed entities
     newprops = deepcopy(props) if props else {}
-    newprops[CONFIRMATION_STATUS] = CONFIRMATION_PENDING
+    newprops[CONFIRMATION_STATUS] = ConfirmationStatus.PENDING.name
 
     LOGGER.debug("Count unconfirmed entities %s", newprops)
     count = self.count(props=newprops, **kwargs)
 
     if count == 0:
         # did any fail
         newprops = deepcopy(props) if props else {}
-        newprops[CONFIRMATION_STATUS] = CONFIRMATION_FAILED
+        newprops[CONFIRMATION_STATUS] = ConfirmationStatus.FAILED.name
         count = self.count(props=newprops, **kwargs)
         if count > 0:
             raise ArchivistUnconfirmedError(f"There are {count} FAILED entities")

archivist/constants.py

@@ -18,10 +18,9 @@
 HEADERS_TOTAL_COUNT = "X-Total-Count"
 HEADERS_RETRY_AFTER = "Archivist-Rate-Limit-Reset"
 
+PROOF_MECHANISM = "proof_mechanism"
+
 CONFIRMATION_STATUS = "confirmation_status"
-CONFIRMATION_PENDING = "PENDING"
-CONFIRMATION_FAILED = "FAILED"
-CONFIRMATION_CONFIRMED = "CONFIRMED"
 
 APPIDP_SUBPATH = "iam/v1"
 APPIDP_LABEL = "appidp"

archivist/subjects_confirmer.py

@@ -9,8 +9,8 @@
 if TYPE_CHECKING:
     from .subjects import Subject, _SubjectsClient
 
+from .confirmation_status import ConfirmationStatus
 from .constants import (
-    CONFIRMATION_CONFIRMED,
     CONFIRMATION_STATUS,
 )
 from .errors import ArchivistUnconfirmedError
@@ -48,7 +48,7 @@ def _wait_for_confirmation(self: "_SubjectsClient", identity: str) -> "Subject":
     if CONFIRMATION_STATUS not in subject:
         return None  # pyright: ignore
 
-    if subject[CONFIRMATION_STATUS] == CONFIRMATION_CONFIRMED:
+    if subject[CONFIRMATION_STATUS] == ConfirmationStatus.CONFIRMED.name:
         return subject
 
     return None  # pyright: ignore

functests/execassets.py

@@ -33,6 +33,13 @@
     "some_custom_attribute": "value",
 }
 
+SIMPLE_HASH = {
+    "proof_mechanism": ProofMechanism.SIMPLE_HASH.name,
+}
+MERKLE_LOG = {
+    "proof_mechanism": ProofMechanism.MERKLE_LOG.name,
+}
+
 ASSET_NAME = "Telephone with 2 attachments - one bad or not scanned 2022-03-01"
 REQUEST_EXISTS_ATTACHMENTS_SIMPLE_HASH = {
     "selector": [
@@ -121,6 +128,10 @@ def setUp(self):
         self.attrs = deepcopy(ATTRS)
         self.traffic_light = deepcopy(ATTRS)
         self.traffic_light["arc_display_type"] = "Traffic light with violation camera"
+        self.traffic_light_merkle_log = deepcopy(ATTRS)
+        self.traffic_light_merkle_log["arc_display_type"] = (
+            "Traffic light with violation camera (merkle_log)"
+        )
 
     def tearDown(self):
         self.arch.close()
@@ -132,6 +143,7 @@ def test_asset_create_simple_hash(self):
         """
         Test asset creation uses simple hash proof mechanism
         """
+        # default is simple hash so it is unspecified
         asset = self.arch.assets.create(
             attrs=self.traffic_light,
             confirm=True,
@@ -145,7 +157,25 @@ def test_asset_create_simple_hash(self):
         tenancy = self.arch.tenancies.publicinfo(asset["tenant_identity"])
         LOGGER.debug("tenancy %s", json_dumps(tenancy, sort_keys=True, indent=4))
 
-    def test_asset_create_with_fixtures(self):
+    def test_asset_create_merkle_log(self):
+        """
+        Test asset creation uses merkle_log proof mechanism
+        """
+        asset = self.arch.assets.create(
+            props=MERKLE_LOG,
+            attrs=self.traffic_light_merkle_log,
+            confirm=True,
+        )
+        LOGGER.debug("asset %s", json_dumps(asset, sort_keys=True, indent=4))
+        self.assertEqual(
+            asset["proof_mechanism"],
+            ProofMechanism.MERKLE_LOG.name,
+            msg="Incorrect asset proof mechanism",
+        )
+        tenancy = self.arch.tenancies.publicinfo(asset["tenant_identity"])
+        LOGGER.debug("tenancy %s", json_dumps(tenancy, sort_keys=True, indent=4))
+
+    def test_asset_create_with_fixtures_simple_hash(self):
         """
         Test creation with fixtures
         """
@@ -196,6 +226,59 @@ def test_asset_create_with_fixtures(self):
             msg="Incorrect number of fancy_traffic_lights",
         )
 
+    def test_asset_create_with_fixtures_merkle_log(self):
+        """
+        Test creation with fixtures
+        """
+        # creates simple_hash endpoint
+        simple_hash = copy(self.arch)
+        simple_hash.fixtures = {
+            "assets": {
+                "proof_mechanism": ProofMechanism.MERKLE_LOG.name,
+            },
+        }
+
+        # create traffic lights endpoint from simple_hash
+        traffic_lights = copy(simple_hash)
+        traffic_lights.fixtures = {
+            "assets": {
+                "attributes": {
+                    "arc_display_type": "Traffic light with violation camera (merkle_log)",
+                    "arc_namespace": f"functests {uuid4()}",
+                },
+            },
+        }
+        traffic_lights.assets.create(
+            props=MERKLE_LOG,
+            attrs=self.attrs,
+            confirm=True,
+        )
+        self.assertEqual(
+            traffic_lights.assets.count(),
+            1,
+            msg="Incorrect number of traffic_lights",
+        )
+
+        # create fancy traffic lights endpoint from traffic lights
+        fancy_traffic_lights = copy(traffic_lights)
+        fancy_traffic_lights.fixtures = {
+            "assets": {
+                "attributes": {
+                    "arc_namespace1": f"functests {uuid4()}",
+                },
+            },
+        }
+        fancy_traffic_lights.assets.create(
+            props=MERKLE_LOG,
+            attrs=self.attrs,
+            confirm=True,
+        )
+        self.assertEqual(
+            fancy_traffic_lights.assets.count(),
+            1,
+            msg="Incorrect number of fancy_traffic_lights",
+        )
+
     def test_asset_create_event_merkle_log(self):
         """
         Test list

scripts/builder.sh

@@ -31,6 +31,7 @@ docker run \
     -e DATATRAILS_AUTHTOKEN_FILENAME_2 \
     -e DATATRAILS_BLOB_IDENTITY \
     -e DATATRAILS_APPREG_CLIENT \
+    -e DATATRAILS_APPREG_CLIENT_FILENAME \
     -e DATATRAILS_APPREG_SECRET \
     -e DATATRAILS_APPREG_SECRET_FILENAME \
     -e DATATRAILS_LOGLEVEL \

scripts/functests.sh

@@ -7,36 +7,35 @@ then
     echo "DATATRAILS_URL is undefined"
     exit 1
 fi
+if [ -n "${DATATRAILS_APPREG_CLIENT_FILENAME}" ]
+then
+    if [ -s "${DATATRAILS_APPREG_CLIENT_FILENAME}" ]
+    then
+	export DATATRAILS_APPREG_CLIENT=$(cat ${DATATRAILS_APPREG_CLIENT_FILENAME})
+    fi
+fi
 if [ -n "${DATATRAILS_APPREG_CLIENT}" ]
 then
     if [ -n "${DATATRAILS_APPREG_SECRET_FILENAME}" ]
     then
-        if [ ! -s "${DATATRAILS_APPREG_SECRET_FILENAME}" ]
+        if [ -s "${DATATRAILS_APPREG_SECRET_FILENAME}" ]
         then
-            echo "${DATATRAILS_APPREG_SECRET_FILENAME} does not exist"
-            exit 1
+	    export DATATRAILS_APPREG_SECRET=$(cat ${DATATRAILS_APPREG_SECRET_FILENAME})
         fi
-    elif [ -z "${DATATRAILS_APPREG_SECRET}" ]
-    then
-        echo "Both DATATRAILS_APPREG_SECRET_FILENAME"
-        echo "and DATATRAILS_APPREG_SECRET are undefined"
-        exit 1
     fi
-else
-    if [ -n "${DATATRAILS_AUTHTOKEN_FILENAME}" ]
-    then
-        if [ ! -s "${DATATRAILS_AUTHTOKEN_FILENAME}" ]
-        then
-            echo "${DATATRAILS_AUTHTOKEN_FILENAME} does not exist"
-            exit 1
-        fi
-    elif [ -z "${DATATRAILS_AUTHTOKEN}" ]
+fi
+if [ -n "${DATATRAILS_AUTHTOKEN_FILENAME}" ]
+then
+    if [ -s "${DATATRAILS_AUTHTOKEN_FILENAME}" ]
     then
-        echo "Both DATATRAILS_AUTHTOKEN_FILENAME"
-        echo "and DATATRAILS_AUTHTOKEN are undefined"
-        exit 1
+	export DATATRAILS_AUTHTOKEN=$(cat ${DATATRAILS_AUTHTOKEN_FILENAME})
     fi
 fi
+if [ -z "${DATATRAILS_AUTHTOKEN}" -a -z "${DATATRAILS_APPREG_CLIENT}" -a -z "${DATATRAILS_APPREG_SECRET}" ]
+then
+    echo "No credentials found, DATATRAILS_AUTHTOKEN, DATATRAILS_APPREG_CLIENT,  DATATRAILS_APPREG_SECRET"
+    exit 1
+fi
 
 python3 --version
 

unittests/testaccess_policies.py

@@ -17,7 +17,7 @@
 from archivist.errors import ArchivistBadRequestError
 
 from .mock_response import MockResponse
-from .testassets import RESPONSE as ASSET
+from .testassets import RESPONSE_SIMPLE_HASH as ASSET
 
 # pylint: disable=missing-docstring
 # pylint: disable=protected-access