Merge pull request #298 from Metaswitch/md/loady

Add `yaml`,`json` and `toml` loading capability to the tera engine

Author: Max Dymond

CHANGELOG.md

@@ -14,6 +14,13 @@ Status: Available for use
 ### Breaking Changes
 
 ### Added
+- Add `yaml` function to tera templating engine so that floki templates
+  can use `yaml(file="<filepath>")` in order to load values.
+- Add `json` function to tera templating engine so that floki templates
+  can use `json(file="<filepath>")` in order to load values.
+- Add `toml` function to tera templating engine so that floki templates
+  can use `toml(file="<filepath>")` in order to load values.
+- Add `floki render` to print out the rendered configuration template.
 
 ### Fixed
 

Cargo.lock

@@ -29,6 +29,8 @@ sha2 = "0.10.7"
 anyhow = "1.0.71"
 thiserror = "1.0.40"
 tera = "1"
+serde_json = "1.0.100"
+toml = "0.5.11"
 
 [dev-dependencies]
 tempfile = "3.6.0"

Cargo.toml

@@ -20,6 +20,11 @@ pub(crate) enum Subcommand {
         #[structopt(name = "SHELL", parse(try_from_str))]
         shell: structopt::clap::Shell,
     },
+
+    /// Render the configuration file to stdout, performing any templating
+    /// operations.
+    #[structopt(name = "render")]
+    Render {},
 }
 
 /// Main CLI interface

src/cli.rs

@@ -3,12 +3,13 @@ use crate::errors;
 use crate::image;
 use anyhow::Error;
 use serde::{Deserialize, Serialize};
+use tera::from_value;
 use tera::Context;
 use tera::Tera;
 
 use std::collections::BTreeMap;
 use std::collections::HashMap;
-use std::path;
+use std::path::{Path, PathBuf};
 
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
 #[serde(untagged)]
@@ -56,7 +57,7 @@ pub(crate) struct Volume {
     pub(crate) shared: bool,
     /// The mount path is the path at which the volume is mounted
     /// inside the floki container.
-    pub(crate) mount: path::PathBuf,
+    pub(crate) mount: PathBuf,
 }
 
 #[derive(Debug, PartialEq, Serialize, Deserialize)]
@@ -83,7 +84,7 @@ pub(crate) struct FlokiConfig {
     #[serde(default = "default_shell")]
     pub(crate) shell: Shell,
     #[serde(default = "default_mount")]
-    pub(crate) mount: path::PathBuf,
+    pub(crate) mount: PathBuf,
     #[serde(default = "Vec::new")]
     pub(crate) docker_switches: Vec<String>,
     #[serde(default = "default_to_false")]
@@ -98,27 +99,80 @@ pub(crate) struct FlokiConfig {
     pub(crate) entrypoint: Entrypoint,
 }
 
-impl FlokiConfig {
-    pub fn from_file(file: &path::Path) -> Result<FlokiConfig, Error> {
-        debug!("Reading configuration file: {:?}", file);
+fn path_from_args(args: &HashMap<String, tera::Value>) -> tera::Result<String> {
+    let file = match args.get("file") {
+        Some(file) => file,
+        None => return Err("file parameter is required".into()),
+    };
+    Ok(from_value::<String>(file.clone())?)
+}
+
+fn yamlloader(args: &HashMap<String, tera::Value>) -> tera::Result<tera::Value> {
+    let path = path_from_args(args)?;
+    let f = std::fs::File::open(path)?;
+    serde_yaml::from_reader(f).map_err(|_| "Failed to read file".into())
+}
+
+fn jsonloader(args: &HashMap<String, tera::Value>) -> tera::Result<tera::Value> {
+    let path = path_from_args(args)?;
+    let f = std::fs::File::open(path)?;
+    serde_json::from_reader(f).map_err(|_| "Failed to read file".into())
+}
+
+fn tomlloader(args: &HashMap<String, tera::Value>) -> tera::Result<tera::Value> {
+    let path = path_from_args(args)?;
+    let contents = std::fs::read_to_string(path)?;
+    toml::from_str(&contents).map_err(|_| "Failed to read file".into())
+}
+
+// Renders a template from a given string.
+pub fn render_template(template: &str, source_filename: &Path) -> Result<String, Error> {
+    let template_path = source_filename.display().to_string();
+
+    debug!("Rendering template: {template_path}");
+
+    // Read the template using tera
+    let mut tera = Tera::default();
+
+    // Allow templates to load variables files as Values.
+    tera.register_function("yaml", yamlloader);
+    tera.register_function("json", jsonloader);
+    tera.register_function("toml", tomlloader);
+
+    tera.add_raw_template(&template_path, template)
+        .map_err(|e| errors::FlokiError::ProblemRenderingTemplate {
+            name: template_path.clone(),
+            error: e,
+        })?;
+
+    // Read the environment variables and store them in a tera context
+    // under the `env` name.
+    let vars: HashMap<String, String> = std::env::vars().collect();
+    let mut context = Context::new();
+    context.insert("env", &vars);
 
-        // Read the template using tera
-        let mut tera = Tera::default();
-        tera.add_template_file(file, Some("floki")).map_err(|e| {
+    // Render the floki file to string using the context.
+    Ok(tera.render(&template_path, &context)?)
+}
+
+impl FlokiConfig {
+    pub fn render(file: &Path) -> Result<String, Error> {
+        let content = std::fs::read_to_string(file).map_err(|e| {
             errors::FlokiError::ProblemOpeningConfigYaml {
                 name: file.display().to_string(),
                 error: e,
             }
         })?;
 
-        // Read the environment variables and store them in a tera context
-        // under the `env` name.
-        let vars: HashMap<String, String> = std::env::vars().collect();
-        let mut context = Context::new();
-        context.insert("env", &vars);
+        // Render the template first before parsing it.
+        render_template(&content, file)
+    }
+
+    pub fn from_file(file: &Path) -> Result<Self, Error> {
+        debug!("Reading configuration file: {:?}", file);
 
-        // Render the floki file to string using the context.
-        let output = tera.render("floki", &context)?;
+        // Render the output from the configuration file before parsing.
+        let output = Self::render(file)?;
 
         // Parse the rendered floki file from the string.
         let mut config: FlokiConfig = serde_yaml::from_str(&output).map_err(|e| {
@@ -161,8 +215,8 @@ fn default_shell() -> Shell {
     Shell::Shell("sh".into())
 }
 
-fn default_mount() -> path::PathBuf {
-    path::Path::new("/src").to_path_buf()
+fn default_mount() -> PathBuf {
+    Path::new("/src").to_path_buf()
 }
 
 fn default_to_false() -> bool {
@@ -176,9 +230,6 @@ fn default_entrypoint() -> Entrypoint {
 #[cfg(test)]
 mod test {
     use super::*;
-    use crate::image::Image;
-    use std::io::Write;
-    use tempfile::NamedTempFile;
 
     #[derive(Debug, PartialEq, Serialize, Deserialize)]
     struct TestShellConfig {
@@ -264,19 +315,37 @@ mod test {
     }
 
     #[test]
-    fn test_tera_file() -> Result<(), Box<dyn std::error::Error>> {
-        let yaml = r#"{% set var = "test" %}image: {{ var }}"#;
+    fn test_tera_render() -> Result<(), Box<dyn std::error::Error>> {
+        let template = r#"{% set var = "test" %}image: {{ var }}"#;
+        let config = render_template(template, Path::new("floki"))?;
+        assert_eq!(config, "image: test");
+        Ok(())
+    }
 
-        // Write to a temporary file.
-        let mut tmp = NamedTempFile::new()?;
-        tmp.write_all(yaml.as_bytes())?;
-        let (_file, path) = tmp.keep()?;
+    #[test]
+    fn test_tera_yamlload() -> Result<(), Box<dyn std::error::Error>> {
+        let template =
+            r#"{% set values = yaml(file="test_resources/values.yaml") %}shell: {{ values.foo }}"#;
+        let config = render_template(template, Path::new("floki"))?;
+        assert_eq!(config, "shell: bar");
+        Ok(())
+    }
 
-        // Let's try and parse the file.
-        let config = FlokiConfig::from_file(&path)?;
+    #[test]
+    fn test_tera_jsonload() -> Result<(), Box<dyn std::error::Error>> {
+        let template =
+            r#"{% set values = json(file="test_resources/values.json") %}shell: {{ values.foo }}"#;
+        let config = render_template(template, Path::new("floki"))?;
+        assert_eq!(config, "shell: bar");
+        Ok(())
+    }
 
-        println!("Config: {:?}", config);
-        assert_eq!(config.image, Image::Name("test".to_string()));
+    #[test]
+    fn test_tera_tomlload() -> Result<(), Box<dyn std::error::Error>> {
+        let template =
+            r#"{% set values = toml(file="Cargo.toml") %}floki: {{ values.package.name }}"#;
+        let config = render_template(template, Path::new("floki"))?;
+        assert_eq!(config, "floki: floki");
         Ok(())
     }
 }

src/config.rs

@@ -19,8 +19,11 @@ pub enum FlokiError {
     #[error("Could not normalize the file path '{name}': {error:?}")]
     ProblemNormalizingFilePath { name: String, error: io::Error },
 
+    #[error("There was a problem rendering the template '{name}': {error:?}")]
+    ProblemRenderingTemplate { name: String, error: tera::Error },
+
     #[error("There was a problem opening the configuration file '{name}': {error:?}")]
-    ProblemOpeningConfigYaml { name: String, error: tera::Error },
+    ProblemOpeningConfigYaml { name: String, error: std::io::Error },
 
     #[error("There was a problem parsing the configuration file '{name}': {error:?}")]
     ProblemParsingConfigYaml {

src/errors.rs

@@ -63,6 +63,13 @@ fn run_floki_from_args(args: &Cli) -> Result<(), Error> {
             Ok(())
         }
 
+        Some(Subcommand::Render {}) => {
+            let env = Environment::gather(&args.config_file)?;
+            let contents = FlokiConfig::render(&env.config_file)?;
+            println!("{contents}");
+            Ok(())
+        }
+
         // Launch an interactive floki shell (the default)
         None => {
             let env = Environment::gather(&args.config_file)?;

src/main.rs

@@ -0,0 +1,3 @@
+{
+    "foo": "bar"
+}

test_resources/values.json

@@ -0,0 +1 @@
+foo: bar