Implement a basic __main__.py file able to process a --excavator arg.

This commit adds a basic argparse based package level main file which
given a filename and a named excavator will perform an excavation and
output the usual HTML files.

By default this uses the standard temporary path, but additionally
support specifying -d and allow a directory argument of "." which will
cause outputting output folder in the cwd: ./output/_autoarchaologist

In order to ensure things are working include tests of the new command
line as well as the basic operation of excavating the example file. The
tests check the expected HTML files were written for this known input
and assert some basic properties of the excavated artifacts.

While here repair the example excavation. The excavation it executes is
explicitly declared thus CLI selectable as --excavator examples.showcase

Author: alexjeffburke

.gitignore

@@ -6,5 +6,10 @@ __pycache__/
 # Sphinx documentation
 docs/_build/
 
+# Build files
+venv/
+
 # Temporary files
 _.*
+/output/
+/tests/_scratch/

Makefile

@@ -0,0 +1,7 @@
+default: example
+
+example:
+	./venv/bin/python3 run_example.py -d .
+
+test:
+	@./venv/bin/python3 -m unittest

autoarchaeologist/__main__.py

@@ -0,0 +1,73 @@
+import argparse
+import importlib
+import os
+import sys
+
+from autoarchaeologist import Excavation
+
+EXCAVATORS = [
+    'examples.showcase'
+]
+
+
+def action_for_args(args):
+    if args.excavator is not None:
+        return ("excavator", load_excavator_by_name(args.excavator))
+    else:
+        raise argparse.ArgumentError(None, "no valid excavation was requsted")
+
+
+def load_excavator_by_name(excavator):
+    package_name, excavation_prefix = excavator.split('.')
+    excavation_name = f"{excavation_prefix}_excavation"
+
+    exacations_package = importlib.import_module(package_name)
+    return getattr(exacations_package, excavation_name)
+
+
+def parse_arguments(argv=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument("-d", "--dir", default="/tmp/_autoarchaologist")
+    parser.add_argument('--excavator', choices=EXCAVATORS)
+    parser.add_argument('filename')
+
+    return parser.parse_args(args=argv)
+
+
+def process_arguments(args):
+    if args.dir == ".":
+        args.dir = os.path.join(os.getcwd(), "output", "_autoarchaologist")
+    if args.filename is not None:
+        args.filename = os.path.abspath(args.filename)
+    else:
+        raise ValueError()
+
+    return args
+
+
+def perform_excavation(args):
+    match action_for_args(args):
+        case "excavator", AnExcavation:
+            assert issubclass(AnExcavation, Excavation)
+            ctx = AnExcavation(html_dir=args.dir)
+        case action, _:
+            raise NotImplementedError(f"action: {action}")
+
+    ff = ctx.add_file_artifact(args.filename)
+
+    ctx.start_examination()
+
+    return ctx
+
+
+if __name__ == "__main__":
+    args = process_arguments(parse_arguments())
+
+    try:
+        os.mkdir(args.dir)
+    except FileExistsError:
+        pass
+
+    ctx = perform_excavation(args)
+    ctx.produce_html()
+    print("Now point your browser at", ctx.filename_for(ctx).link)

ddhf/ddhf/decorated_context.py

@@ -110,13 +110,29 @@ def from_argv(self):
     "AUTOARCHAEOLOGIST_BITSTORE_CACHE": "ddhf_bitstore_cache",
 }
 
-def main(job, html_subdir="tmp", **kwargs):
+def parse_arguments(argv=None):
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-o', '--out', default='/tmp/_autoarchaologist')
+
+    args = parser.parse_args(args=argv)
+    if args.out == '.':
+        args.out = os.path.join(os.getcwd(), "_autoarchaologist")
+    return args
+
+def main(job, html_subdir, **kwargs):
+    args = parse_arguments()
+    kwargs["html_dir"] = args.out
+
     ''' A standard main routine to reduce boiler-plate '''
     for key in os.environ:
         i = OK_ENVS.get(key)
         if i:
             kwargs[i] = os.environ[key]
 
+    if 'html_dir' not in kwargs:
+        raise AttributeError("missing: html_dir")
+
+
     kwargs['html_dir'] = os.path.join(kwargs['html_dir'], html_subdir)
     kwargs.setdefault('download_links', True)
     kwargs.setdefault('download_limit', 1 << 20)

examples/__init__.py

@@ -0,0 +1,3 @@
+from .examples_showcase import ShowcaseExcacation
+
+showcase_excavation = ShowcaseExcacation

examples/examples_showcase.py

@@ -0,0 +1,15 @@
+from autoarchaeologist.base.excavation import Excavation
+from autoarchaeologist.generic.bigtext import BigText
+from autoarchaeologist.generic.samesame import SameSame
+from autoarchaeologist.data_general.absbin import AbsBin
+from autoarchaeologist.data_general.papertapechecksum import DGC_PaperTapeCheckSum
+
+
+class ShowcaseExcacation(Excavation):
+    def __init__(self, **kwargs):
+        super().__init__(**kwargs)
+
+        self.add_examiner(BigText)
+        self.add_examiner(AbsBin)
+        self.add_examiner(DGC_PaperTapeCheckSum)
+        self.add_examiner(SameSame)

output/.gitkeep

@@ -1,32 +1,17 @@
-
 import os
+import sys
+from types import SimpleNamespace
 
-import autoarchaeologist
-
-from autoarchaeologist.generic.bigdigits import BigDigits
-from autoarchaeologist.generic.samesame import SameSame
-from autoarchaeologist.data_general.absbin import AbsBin
-from autoarchaeologist.data_general.papertapechecksum import DGC_PaperTapeCheckSum
-
+from autoarchaeologist.__main__ import parse_arguments, process_arguments, perform_excavation
+from examples import ShowcaseExcacation
 
 if __name__ == "__main__":
+    argv = sys.argv[1:]
+    # force the example as the filename
+    argv.append("examples/30001393.bin")
+    args = process_arguments(parse_arguments(argv=argv))
 
-    ctx = autoarchaeologist.Excavation()
-
-    ctx.add_examiner(BigDigits)
-    ctx.add_examiner(AbsBin)
-    ctx.add_examiner(DGC_PaperTapeCheckSum)
-    ctx.add_examiner(SameSame)
-
-    ff = ctx.add_file_artifact("examples/30001393.bin")
-
-    ctx.start_examination()
-
-    try:
-        os.mkdir("/tmp/_autoarchaologist")
-    except FileExistsError:
-        pass
-
-    ctx.produce_html(html_dir="/tmp/_autoarchaologist")
+    ctx = perform_excavation(args, ("excavator", ShowcaseExcacation))
+    ctx.produce_html()
 
     print("Now point your browser at", ctx.filename_for(ctx).link)

run_example.py

@@ -0,0 +1,51 @@
+import os
+import shutil
+from types import SimpleNamespace
+import unittest
+
+TESTS_DIR = os.path.dirname(os.path.abspath(__file__))
+SCRATCH_DIR = os.path.join(TESTS_DIR, "_scratch")
+
+from autoarchaeologist.__main__ import perform_excavation
+
+
+class Test_Example_BasicHtml(unittest.TestCase):
+    """
+    Ensure run_example produces expected HTML files for the example input.
+    """
+
+    DIR_TREE = None
+
+    @classmethod
+    def setUpClass(cls):
+        args = SimpleNamespace(
+            dir=SCRATCH_DIR,
+            filename="examples/30001393.bin",
+            excavator="examples.showcase",
+        )
+        shutil.rmtree(args.dir, ignore_errors=True)
+        os.makedirs(args.dir, exist_ok=True)
+        ctx = perform_excavation(args)
+        ctx.produce_html()
+        cls.DIR_TREE = list(os.walk(args.dir))
+
+    def toplevel(self):
+        return self.__class__.DIR_TREE[0]
+
+    def toplevel_dirnames(self):
+        _, dirs, __ = self.toplevel()
+        dirs.sort()
+        return dirs
+
+    def toplevel_filenames(self):
+        _, __, filenames = self.toplevel()
+        return filenames
+
+    def test_produces_top_level_index(self):
+        toplevel_filenames = self.toplevel_filenames()
+        self.assertTrue("index.html" in toplevel_filenames)
+        self.assertTrue("index.css" in toplevel_filenames)
+
+    def test_produces_digest_directories(self):
+        toplevel_dirnames = self.toplevel_dirnames()
+        self.assertEqual(toplevel_dirnames, ['08', '79', 'fa'])