Merge pull request #33 from mcriscenzo/feature/custom_edfi_urls

Feature - Custom metadata URLs

Author: tomreitz

README.md

@@ -43,6 +43,10 @@ data_dir: ./
 namespace: ed-fi
 edfi_api:
   base_url: https://api.schooldistrict.org/v5.3/api
+  oauth_url: https://api.schooldistrict.org/v5.3/api/oauth/token 
+  dependencies_url: https://api.schooldistrict.org/v5.3/api/metadata/data/v3/2024/dependencies
+  descriptors_swagger_url: https://api.schooldistrict.org/v5.3/api/metadata/data/v3/2024/descriptors/swagger.json
+  resources_swagger_url: https://api.schooldistrict.org/v5.3/api/metadata/data/v3/2024/resources/swagger.json
   version: 3
   mode: year_specific
   year: 2021
@@ -68,6 +72,11 @@ show_stacktrace: True
 * (optional) Specify the `namespace` to use when accessing the Ed-Fi API. The default is `ed-fi` but others include `tpdm` or custom values. To send data to multiple namespaces, you must use a YAML configuration file and `lightbeam send` for each.
 * Specify the details of the `edfi_api` to which to connect including
   * (optional) The `base_url` which serves a JSON object specifying the paths to data endpoints, Swagger, and dependencies. The default is `https://localhost/api` (the address of an Ed-Fi API [running locally in Docker](https://techdocs.ed-fi.org/display/EDFITOOLS/Docker+Deployment)), but the location varies depending on how Ed-Fi is deployed.
+  * If the metadata for a particular API is not located in the "default" location (at the root of the base_url), then ALL the following urls should be explicitly specified.  These can normally be left blank, unless you are encountering errors indicating that the metadata files cannot be found (such as "Could not parse response from [base_url]").
+    * (optional) `oauth_url` (usually [base_url]/oauth/token)
+    * (optional) `dependencies_url` (usually [base_url]/metadata/data/v3/dependencies)
+    * (optional) `descriptors_swagger_url` (usually [base_url]/metadata/data/v3/descriptors/swagger.json)
+    * (optional) `resources_swagger_url` (usually [base_url]/metadata/data/v3/resources/swagger.json)
   * The `version` as one of `3` or `2` (`2` is currently unsupported).
   * (optional) The `mode` as one of `shared_instance`, `sandbox`, `district_specific`, `year_specific`, or `instance_year_specific`.
   * (required if `mode` is `year_specific` or `instance_year_specific`) The `year` used to build the resource URL. The default is the current year.

lightbeam/api.py

@@ -35,15 +35,22 @@ def prepare(self):
         except Exception as e:
             self.logger.critical("could not connect to {0} ({1})".format(self.config["base_url"], str(e)))
 
-        try:
-            api_base = api_base.json()
-        except Exception as e:
-            self.logger.critical("could not parse response from {0} ({1})".format(self.config["base_url"], str(e)))
-
-        self.config["oauth_url"] = api_base["urls"]["oauth"]
-        self.config["dependencies_url"] = api_base["urls"]["dependencies"]
+        # Data URL doesn't rely on metadata connection
         self.config["data_url"] = self.get_data_url()
-        self.config["open_api_metadata_url"] = api_base["urls"]["openApiMetadata"]
+
+        # If ALL urls are set in config (probably from source/destination file),
+        # then they don't need to be pulled from api metadata, so this section can be skipped.
+        # This is most common if the api metadata json files are not in the "default" location.
+        # Otherwise, pull urls from api metadata.
+        if self.config["oauth_url"] == "" or self.config["dependencies_url"]== "":
+            try:
+                api_base = api_base.json()
+            except Exception as e:
+                self.logger.critical("could not parse response from {0} ({1})".format(self.config["base_url"], str(e)))
+
+            self.config["oauth_url"] = api_base["urls"]["oauth"]
+            self.config["dependencies_url"] = api_base["urls"]["dependencies"]
+            self.config["open_api_metadata_url"] = api_base["urls"]["openApiMetadata"]
 
         # load all endpoints in dependency-order
         all_endpoints = self.get_sorted_endpoints()
@@ -160,17 +167,33 @@ def get_sorted_endpoints(self):
 
     # Loads the Swagger JSON from the Ed-Fi API
     def load_swagger_docs(self):
-        # grab Descriptors and Resources swagger URLs
-        try:
-            self.logger.debug("fetching swagger docs...")
-            response = requests.get(self.config["open_api_metadata_url"],
-                                    verify=self.lightbeam.config["connection"]["verify_ssl"])
-            if not response.ok:
-                raise Exception("OpenAPI metadata URL returned status {0} ({1})".format(response.status_code, (response.content[:75] + "...") if len(response.content)>75 else response.content))
-            openapi = response.json()
 
-        except Exception as e:
-            self.logger.critical("Unable to load Swagger docs from API... terminating. Check API connectivity.")
+        # If the metadata URL is set (pulled from root metadata file earlier), then pull endpoint urls from metadata
+        if "open_api_metadata_url" in self.config.keys() and self.config["open_api_metadata_url"] != "":
+            # grab Descriptors and Resources swagger URLs
+            try:
+                self.logger.debug("fetching swagger docs...")
+                response = requests.get(self.config["open_api_metadata_url"],
+                                        verify=self.lightbeam.config["connection"]["verify_ssl"])
+                if not response.ok:
+                    raise Exception("OpenAPI metadata URL returned status {0} ({1})".format(response.status_code, (response.content[:75] + "...") if len(response.content)>75 else response.content))
+                openapi = response.json()
+
+            except Exception as e:
+                self.logger.critical("Unable to load Swagger docs from API... terminating. Check API connectivity.")
+        
+        # If metadata URL is not found, set endpoint URLs from config file
+        else:
+            openapi = [
+                {
+                    "name": "descriptors",
+                    "endpointUri": self.config["descriptors_swagger_url"],
+                }, 
+                {
+                    "name": "resources",
+                    "endpointUri": self.config["resources_swagger_url"],
+                }
+            ]
 
         # load (or re-use cached) Descriptors and Resources swagger
         self.descriptors_swagger = None

lightbeam/lightbeam.py

@@ -24,6 +24,10 @@ class Lightbeam:
         "namespace": "ed-fi",
         "edfi_api": {
             "base_url": "",
+            "oauth_url": "", 
+            "dependencies_url": "",
+            "descriptors_swagger_url": "",
+            "resources_swagger_url": "",
             "version": 3,
             "mode": "year_specific",
             "year": datetime.today().year,