[Vis Tools] Load examples on app start up (#6068)

## Related PR

This is a follow-up to
[6066](#6066).

## Issue

[b/489513837](https://buganizer.corp.google.com/issues/489513837)

## Description

This PR updates how the example content that populate the vis tools are
loaded. Beforehand, both the very old examples and the newly added
examples were loaded directly in the route, and so read from the
relevant files on each request. This functionality is now performed on
app started.

## Testing

All the same testing considerations from the original PR apply here.
This should not change anything visually. However, an important note
about testing is that you will now have to restart the server to effect
changes to the vis tools (due to the fact that the files are now loaded
once on server start).

## Notes

I've also updated the route level loading process to make a copy of the
server cached JSON, rather than loading it directly. This is because now
that this cache is shared, accidental updates of this anywhere
downstream would affect the cache for all subsequent requests.

Author: nick-nlb

server/__init__.py

@@ -41,6 +41,7 @@
 from server.lib.nl.detection import llm_prompt
 from server.lib.nl.detection.agent.agent import create_detection_agent
 import server.lib.util as libutil
+from server.routes.tools import html as tools_html
 import server.services.bigtable as bt
 from server.services.discovery import configure_endpoints_from_ingress
 from server.services.discovery import get_health_check_urls
@@ -450,6 +451,9 @@ def create_app(nl_root=DEFAULT_NL_ROOT):
   custom_dc_template_folder = app.config.get(
       'CUSTOM_DC_TEMPLATE_FOLDER', None) or app.config.get('ENV', None)
 
+  app.config['VIS_TOOL_EXAMPLES'] = tools_html.get_all_tool_examples(
+      app, custom_dc_template_folder)
+
   # Get and save the blocklisted svgs.
   blocklist_svg = []
   if os.path.isfile(BLOCKLIST_SVG_FILE):

server/routes/tools/html.py

@@ -12,6 +12,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import copy
+import glob
 import json
 import os
 
@@ -27,28 +29,46 @@
 bp = flask.Blueprint("tools", __name__, url_prefix='/tools')
 
 
-def get_example_file(tool):
-  example_file = os.path.join(current_app.root_path, 'templates/custom_dc',
-                              g.custom_dc_template_folder,
-                              '{}_examples.json'.format(tool))
-  if os.path.exists(example_file):
-    return example_file
+def get_all_tool_examples(app, custom_dc_template_folder):
+  """Finds and loads all example files into a dictionary."""
+  example_paths = {}
+
+  # Find all example files (with custom DC versions taking precedence)
+  search_dirs = [os.path.join(app.root_path, 'templates/tools')]
+  if custom_dc_template_folder:
+    search_dirs.append(
+        os.path.join(app.root_path, 'templates/custom_dc',
+                     custom_dc_template_folder))
+  for directory in search_dirs:
+    for filepath in glob.glob(os.path.join(directory, '*_examples.json')):
+      filename = os.path.basename(filepath)
+      tool_name = filename.removesuffix('_examples.json')
+      example_paths[tool_name] = filepath
+
+  # Load the examples and return them
+  loaded_examples = {}
+  for tool, filepath in example_paths.items():
+    try:
+      with open(filepath) as f:
+        loaded_examples[tool] = json.load(f)
+    except json.JSONDecodeError:
+      app.logger.error('Malformed JSON for tool %s in %s', tool, filepath)
+    except OSError as e:
+      app.logger.error('Failed to read tool %s at %s: %s', tool, filepath, e)
 
-  return os.path.join(current_app.root_path,
-                      'templates/tools/{}_examples.json'.format(tool))
+  return loaded_examples
 
 
-def load_example_file(tool_or_filename, default=None):
+def _load_example_file(tool_or_filename, default=None):
   """Loads a JSON example file, returning a default if missing."""
   if default is None:
     default = {}
-  filepath = get_example_file(tool_or_filename)
-  if os.path.exists(filepath):
-    try:
-      with open(filepath) as f:
-        return json.load(f)
-    except json.JSONDecodeError:
-      current_app.logger.error('Malformed JSON in %s', filepath)
+
+  examples = current_app.config.get('VIS_TOOL_EXAMPLES', {})
+  data = examples.get(tool_or_filename)
+
+  if data is not None:
+    return copy.deepcopy(data)
   return default
 
 
@@ -57,8 +77,8 @@ def _get_vis_tool_examples(tool_name):
   use_standardized_ui = is_feature_enabled(STANDARDIZED_VIS_TOOL_FEATURE_FLAG,
                                            request=request)
   if use_standardized_ui:
-    return {}, load_example_file(f'{tool_name}_vis_tool', default=[]), True
-  return load_example_file(tool_name, default={}), [], False
+    return {}, _load_example_file(f'{tool_name}_vis_tool', default=[]), True
+  return _load_example_file(tool_name, default={}), [], False
 
 
 @bp.route('/timeline')
@@ -124,7 +144,7 @@ def stat_var():
 def download():
   # List of DCIDs displayed in the info page for download tool
   # NOTE: EXACTLY 2 EXAMPLES REQUIRED.
-  info_places = load_example_file('download', default=[])
+  info_places = _load_example_file('download', default=[])
 
   return flask.render_template('tools/download.html',
                                info_places=json.dumps(info_places),
@@ -136,7 +156,7 @@ def download():
 
 @bp.route('/visualization')
 def visualization():
-  info_json = load_example_file('visualization', default={})
+  info_json = _load_example_file('visualization', default={})
 
   return flask.render_template('tools/visualization.html',
                                manual_ga_pageview=True,

server/templates/tools/map.html

@@ -33,7 +33,7 @@
   <div id="main-pane"></div>
   <script>
     globalThis.infoConfig = {{ info_json | tojson | safe }};
-    globalThis.visToolExamples = {{ vis_tool_examples_json | default([]) | tojson | safe }};
+    globalThis.visToolExamples = {{ vis_tool_examples_json | tojson | safe }};
     globalThis.minStatVarGeoCoverage = {{ config['MIN_STAT_VAR_GEO_COVERAGE'] | int(1) }};
   </script>
 {% endblock %}

server/templates/tools/scatter.html

@@ -32,7 +32,7 @@
 <div id="main-pane"></div>
 <script>
   globalThis.infoConfig = {{ info_json | tojson | safe }};
-  globalThis.visToolExamples = {{ vis_tool_examples_json | default([]) | tojson | safe }};
+  globalThis.visToolExamples = {{ vis_tool_examples_json | tojson | safe }};
   globalThis.minStatVarGeoCoverage = {{ config['MIN_STAT_VAR_GEO_COVERAGE'] | int(1) }};
 </script>
 {% endblock %}

server/templates/tools/timeline.html

@@ -32,7 +32,7 @@
 <div id="main-pane"></div>
 <script>
   globalThis.infoConfig = {{ info_json | tojson | safe }};
-  globalThis.visToolExamples = {{ vis_tool_examples_json | default([]) | tojson | safe }};
+  globalThis.visToolExamples = {{ vis_tool_examples_json | tojson | safe }};
 </script>
 {% endblock %}