updates to finish out functionality

Author: Tom Reitz

lightbeam/api.py

@@ -337,6 +337,8 @@ def get_params_for_endpoint(self, endpoint, type='required'):
         definition = util.get_swagger_ref_for_endpoint(self.lightbeam.config["namespace"], swagger, endpoint)
         if type=='required':
             return self.get_required_params_from_swagger(swagger, definition)
+        elif type=='all':
+            return self.get_all_params_from_swagger(swagger, definition)
         else:
             # descriptor endpoints all have the same structure and identity fields:
             if "Descriptor" in endpoint:
@@ -360,6 +362,28 @@ def get_required_params_from_swagger(self, swagger, definition, prefix=""):
                 params[prop] = prefix + prop
         return params
 
+    def get_all_params_from_swagger(self, swagger, definition, prefix=""):
+        params = {}
+        schema = util.resolve_swagger_ref(swagger, definition)
+        if not schema:
+            self.logger.critical(f"Swagger contains neither `definitions` nor `components.schemas` - check that the Swagger is valid.")
+        
+        for prop in schema["properties"].keys():
+            if prop in ["_etag", "id", "link"]: continue
+            if "required" in schema.keys() and prop in schema["required"]: prop_name = "[required]"+prop
+            else: prop_name = "[optional]"+prop
+            if "$ref" in schema["properties"][prop].keys():
+                params[prop_name] = {}
+                sub_definition = schema["properties"][prop]["$ref"]
+                sub_params = self.get_all_params_from_swagger(swagger, sub_definition, prefix=prop+"_")
+                for k,v in sub_params.items():
+                    params[prop_name][k] = v
+            elif schema["properties"][prop]["type"]!="array":
+                params[prop_name] = f"[{schema['properties'][prop]['type']}]" + prefix + prop
+            else:
+                params[prop_name] = [self.get_all_params_from_swagger(swagger, schema["properties"][prop]["items"]["$ref"], prefix=prop+"_")]
+        return params
+
     def get_identity_params_from_swagger(self, swagger, definition, prefix=""):
         params = {}
         schema = util.resolve_swagger_ref(swagger, definition)

lightbeam/create.py

@@ -1,4 +1,5 @@
 import os
+import re
 import json
 import yaml
 from lightbeam import util
@@ -12,6 +13,7 @@ def __init__(self, lightbeam=None):
         self.earthmover_file = "earthmover.yml"
 
     def create(self):
+        self.lightbeam.api.load_swagger_docs()
         os.makedirs(self.template_folder, exist_ok=True)
         earthmover_yaml = {}
         # check if file exists!
@@ -31,6 +33,12 @@ def create(self):
 # This is an earthmover.yml file, generated with `lightbeam create`, for creating Ed-Fi JSON payloads
 # using earthmover. See https://github.com/edanalytics/earthmover for documentation.
 
+config:
+  macros: >
+    {% macro descriptor_namespace() -%}
+      uri://ed-fi.org
+    {%- endmacro %}
+
 # Define your source data here:
 sources:
   # Example:
@@ -57,59 +65,41 @@ def create_em_destination_node(self, endpoint):
     template: {self.template_folder}{endpoint}.jsont
     extension: jsonl
     linearize: True"""
+
+    def upper_repl(self, match):
+        value = match.group(1)
+        return "/" + value[0].upper() + value[1:] + "Descriptor#"
+        
 
     def create_jsont(self, endpoint):
         template_file = f"{self.template_folder}{endpoint}.jsont"
         # check if file exists!
         if os.path.isfile(template_file):
             self.logger.critical(f"The file `{template_file}` already exists in the current directory; to re-create it, please first manually delete it.")
+        # generate base JSON structure:
+        content = self.lightbeam.api.get_params_for_endpoint(endpoint, type='all')
+        # pretty-print it:
+        content = json.dumps(content, indent=2)
+        # annotate required/optional properties:
+        content = content.replace('"[required]', '{# (required) #} "')
+        content = content.replace('"[optional]', '{# (optional) #} "')
+        # appropriate quoting based on property data type:
+        content = re.sub('"\[string\](.*)Descriptor"', r'"{{descriptor_namespace()}}/\1Descriptor#{{\1Descriptor}}"', content)
+        content = re.sub('/(.*)_(.*)Descriptor#', r'/\2Descriptor#', content)
+        content = re.sub(r'/(.*)Descriptor#', self.upper_repl, content)
+        content = re.sub('"\[string\](.*)"', r'"{{\1}}"', content)
+        content = re.sub('"\[(integer|boolean)\](.*)"', r'{{\2}}', content)
+        # for loops over arrays:
+        content = re.sub('"(.*)": \[', r'"\1": [ {% for item in \1 %}', content)
+        content = re.sub('\]', r'{% endfor %} ]', content)
+        content = re.sub('{{(.*)_(.*)}}', r'{{item.\2}}', content)
+        # add info header message:
+        content = """{#
+  This is an earthmover JSON template file, generated with `lightbeam create`, for creating Ed-Fi JSON `"""+endpoint+"""`
+  payloads using earthmover. See https://github.com/edanalytics/earthmover for documentation.
+#}
+""" + content
         # write out json template
         self.logger.info(f"creating file `{template_file}`...")
         with open(template_file, 'w+') as file:
-            # TODO: implement a function in `lightbeam/api.py` that constructs a "sample" payload for the endpoint
-            # Example:
-            # {
-            #   "property_bool": true,
-            #   "property_int": 1,
-            #   "property_float": 1.0,
-            #   "property_string": "string",
-            #   "property_date": "date",
-            #   "property_string_optional": "string",
-            #   "property_descriptor": "uri://ed-fi.org/SomeDescriptor#SomeValue",
-            #   "property_object": {
-            #     "property_object_1": "string",
-            #     "property_object_2": "string"
-            #   },
-            #   "property_array": [
-            #     {
-            #       "property_array_1": "string",
-            #       "property_array_2": "string"
-            #     }
-            #   ]
-            # }
-            # TODO: turn the "sample" payload into a Jinja template
-            # Example:
-            # {
-            #   "property_bool": {{property_bool}},
-            #   "property_int": {{property_int}},
-            #   "property_float": {{property_float}},
-            #   "property_string": "{{property_string}}",
-            #   "property_date": "{{property_date}}",
-            #   {% if property_string_optional %}
-            #   "property_string_optional": "{{property_string_optional}}",
-            #   {% endif %}
-            #   "property_descriptor": "uri://ed-fi.org/SomeDescriptor#{{property_descriptor}}",
-            #   "property_object": {
-            #     "property_object_1": "{{property_object_1}}",
-            #     "property_object_2": "{{property_object_2}}"
-            #   },
-            #   "property_array": [
-            #     {% for item in property_array %}
-            #     {
-            #       "property_array_1": "{{item.property_array_1}}",
-            #       "property_array_2": "{{item.property_array_2}}"
-            #     } {% if not loop.last %},{% endif %}
-            #     {% endfor %}
-            #   ]
-            # }
-            file.write("coming soon...") # (for now)
+            file.write(content)