Index each schema's lines once

src/generate_rust.rs

@@ -1,11 +1,11 @@
 use crate::{
     identifier::Identifier,
     schema::{self, relativize_namespace},
+    schema_loader::LoadedSchema,
 };
 use std::{
     collections::BTreeMap,
     fmt::{self, Write},
-    path::PathBuf,
 };
 
 // The string to be used for each indentation level.
@@ -101,7 +101,7 @@ use Direction::{Atlas, In, Out};
 #[allow(clippy::too_many_lines)]
 pub fn generate(
     typical_version: &str,
-    schemas: &BTreeMap<schema::Namespace, (schema::Schema, PathBuf, String)>,
+    schemas: &BTreeMap<schema::Namespace, LoadedSchema>,
 ) -> String {
     // Construct a tree of modules and schemas. We start with an empty tree.
     let mut tree = Module {
@@ -114,7 +114,7 @@ pub fn generate(
     };
 
     // Populate the tree with all the schemas.
-    for (namespace, (schema, _, _)) in schemas {
+    for (namespace, LoadedSchema { schema, .. }) in schemas {
         insert_schema(&mut tree, namespace, schema);
     }
 

src/generate_typescript.rs

@@ -1,4 +1,4 @@
-use crate::{identifier::Identifier, schema};
+use crate::{identifier::Identifier, schema, schema_loader::LoadedSchema};
 use std::{
     collections::BTreeMap,
     fmt::{self, Write},
@@ -142,7 +142,7 @@ impl DeclarationFunctionNames {
 // to the configured output directory.
 pub fn generate(
     typical_version: &str,
-    schemas: &BTreeMap<schema::Namespace, (schema::Schema, PathBuf, String)>,
+    schemas: &BTreeMap<schema::Namespace, LoadedSchema>,
     import_extension: &str,
 ) -> BTreeMap<PathBuf, String> {
     let mut files = BTreeMap::new();
@@ -155,7 +155,7 @@ pub fn generate(
         common_buffer,
     );
 
-    for (namespace, (schema, _, _)) in schemas {
+    for (namespace, LoadedSchema { schema, .. }) in schemas {
         let mut buffer = String::new();
         // The `unwrap` is safe because the `std::fmt::Write` impl for `String` is infallible.
         write_schema_file(
@@ -2625,7 +2625,9 @@ mod tests {
     use crate::{
         error::SourceRange,
         generate_typescript::{COMMON_FILE_STEM, generate},
+        line_index::LineIndex,
         schema::{self, Namespace},
+        schema_loader::LoadedSchema,
         schema_loader::load_schemas,
         validator::validate,
     };
@@ -2684,20 +2686,21 @@ mod tests {
         let mut schemas = BTreeMap::new();
         schemas.insert(
             dependency_namespace,
-            (
-                schema::Schema {
+            LoadedSchema {
+                schema: schema::Schema {
                     comment: vec![],
                     imports: BTreeMap::new(),
                     declarations: vec![],
                 },
-                PathBuf::from("Foo/FirstSchema.t"),
-                String::new(),
-            ),
+                path: PathBuf::from("Foo/FirstSchema.t"),
+                contents: String::new(),
+                line_index: LineIndex::new(""),
+            },
         );
         schemas.insert(
             types_namespace,
-            (
-                schema::Schema {
+            LoadedSchema {
+                schema: schema::Schema {
                     comment: vec![],
                     imports,
                     declarations: vec![schema::Declaration {
@@ -2709,9 +2712,10 @@ mod tests {
                         deleted: BTreeSet::new(),
                     }],
                 },
-                PathBuf::from("Bar/SecondSchema.t"),
-                String::new(),
-            ),
+                path: PathBuf::from("Bar/SecondSchema.t"),
+                contents: String::new(),
+                line_index: LineIndex::new(""),
+            },
         );
 
         let generated = generate("0.0.0", &schemas, ".ts");

src/main.rs

@@ -20,7 +20,7 @@ use crate::{
     count::count,
     error::{Error, format_errors},
     format::{CodePath, CodeStr},
-    schema_loader::load_schemas,
+    schema_loader::{LoadedSchema, load_schemas},
     validator::validate,
 };
 use clap::{ArgAction, Args, CommandFactory, Parser, Subcommand as ClapSubcommand};
@@ -147,7 +147,10 @@ fn generate_code(
         // The `unwrap` is safe since otherwise the schema would've failed to load above.
         let directory = schema_path.parent().unwrap();
 
-        for (_, source_path, _) in schemas.values() {
+        for LoadedSchema {
+            path: source_path, ..
+        } in schemas.values()
+        {
             println!("{}", directory.join(source_path).display());
         }
     }
@@ -251,7 +254,13 @@ fn format_schema(schema_path: &Path, check: bool) -> Result<(), Vec<Error>> {
         "{} schemas\u{2026}",
         if check { "Checking" } else { "Formatting" },
     );
-    for (schema, source_path, source_contents) in schemas.values() {
+    for LoadedSchema {
+        schema,
+        path: source_path,
+        contents: source_contents,
+        ..
+    } in schemas.values()
+    {
         // Compute the full path and new contents of the schema.
         let full_source_path = directory.join(source_path);
         let new_source_contents = schema.to_string();

src/parser.rs

@@ -203,17 +203,16 @@ macro_rules! consume_token_1 {
 pub fn parse(
     source_path: &Path,
     source_contents: &str,
+    line_index: &LineIndex,
     tokens: &[token::Token],
 ) -> Result<schema::Schema, Vec<Error>> {
-    // Try to parse the tokens into a schema, indexing the lines of the source so any errors can
-    // show them.
+    // Try to parse the tokens into a schema.
     let mut position = 0;
     let mut errors = vec![];
-    let line_index = LineIndex::new(source_contents);
     let schema = parse_schema(
         source_path,
         source_contents,
-        &line_index,
+        line_index,
         tokens,
         &mut position,
         &mut errors,
@@ -227,7 +226,7 @@ pub fn parse(
             Some(source_path),
             Some((
                 source_contents,
-                &line_index,
+                line_index,
                 token_source_range(tokens, position),
             )),
             None,
@@ -923,7 +922,8 @@ fn parse_type(
 #[cfg(test)]
 mod tests {
     use crate::{
-        assert_fails, assert_same, error::SourceRange, parser::parse, schema, tokenizer::tokenize,
+        assert_fails, assert_same, error::SourceRange, line_index::LineIndex, parser::parse,
+        schema, tokenizer::tokenize,
     };
     use std::{
         collections::{BTreeMap, BTreeSet},
@@ -935,10 +935,11 @@ mod tests {
     fn parse_empty() {
         let source_path = Path::new("foo.t");
         let source = "";
-        let tokens = tokenize(source_path, source).unwrap();
+        let line_index = LineIndex::new(source);
+        let tokens = tokenize(source_path, source, &line_index).unwrap();
 
         assert_same!(
-            parse(source_path, source, &tokens[..]),
+            parse(source_path, source, &line_index, &tokens[..]),
             Ok(schema::Schema {
                 comment: vec![],
                 imports: BTreeMap::new(),
@@ -979,7 +980,8 @@ mod tests {
                 z = 3
             }
         ";
-        let tokens = tokenize(source_path, source).unwrap();
+        let line_index = LineIndex::new(source);
+        let tokens = tokenize(source_path, source, &line_index).unwrap();
 
         let mut imports = BTreeMap::new();
 
@@ -1183,7 +1185,7 @@ mod tests {
         ];
 
         assert_same!(
-            parse(source_path, source, &tokens[..]),
+            parse(source_path, source, &line_index, &tokens[..]),
             Ok(schema::Schema {
                 comment: vec!["This is an example schema.".to_owned()],
                 imports,
@@ -1199,10 +1201,11 @@ mod tests {
             import 'foo.t' as Qux
             import 'bar.t' as qux
         ";
-        let tokens = tokenize(source_path, source).unwrap();
+        let line_index = LineIndex::new(source);
+        let tokens = tokenize(source_path, source, &line_index).unwrap();
 
         assert_fails!(
-            parse(source_path, source, &tokens[..]),
+            parse(source_path, source, &line_index, &tokens[..]),
             "An import named `qux` already exists in this file.",
         );
     }
@@ -1215,10 +1218,11 @@ mod tests {
                 deleted 1 2 2 3
             }
         ";
-        let tokens = tokenize(source_path, source).unwrap();
+        let line_index = LineIndex::new(source);
+        let tokens = tokenize(source_path, source, &line_index).unwrap();
 
         assert_fails!(
-            parse(source_path, source, &tokens[..]),
+            parse(source_path, source, &line_index, &tokens[..]),
             "Index `2` is already marked as deleted.",
         );
     }

src/schema_loader.rs

@@ -15,6 +15,14 @@ use std::{
     sync::Arc,
 };
 
+// This is a schema along with the file it was loaded from.
+pub struct LoadedSchema {
+    pub schema: schema::Schema,
+    pub path: PathBuf,
+    pub contents: String,
+    pub line_index: LineIndex,
+}
+
 // Convert a UTF-8 path containing only normal components to a namespace.
 fn path_to_namespace(path: &Path) -> Option<schema::Namespace> {
     let mut path = path.to_owned();
@@ -37,10 +45,9 @@ fn path_to_namespace(path: &Path) -> Option<schema::Namespace> {
 // Load a schema and its transitive dependencies. The imports in the returned schemas are guaranteed
 // to resolve.
 #[allow(clippy::too_many_lines)]
-#[allow(clippy::type_complexity)]
 pub fn load_schemas(
     schema_path: &Path,
-) -> Result<BTreeMap<schema::Namespace, (schema::Schema, PathBuf, String)>, Vec<Error>> {
+) -> Result<BTreeMap<schema::Namespace, LoadedSchema>, Vec<Error>> {
     // Reject paths that cannot be represented faithfully in schema namespaces and diagnostics
     // [tag:schema_path_valid_utf8].
     if schema_path.to_str().is_none() {
@@ -129,7 +136,7 @@ pub fn load_schemas(
     let mut schemas_to_load = vec![(
         schema_namespace.clone(),
         based_schema_path.to_owned(),
-        None as Option<(PathBuf, String, SourceRange)>,
+        None as Option<(PathBuf, String, LineIndex, SourceRange)>,
     )];
     let mut visited_namespaces = HashSet::new();
     visited_namespaces.insert(schema_namespace);
@@ -142,15 +149,17 @@ pub fn load_schemas(
             Err(error) => {
                 let message = format!("Unable to load {}.", path.code_path());
 
-                if let Some((origin_path, origin_contents, origin_source_range)) = origin {
+                if let Some((
+                    origin_path,
+                    origin_contents,
+                    origin_line_index,
+                    origin_source_range,
+                )) = origin
+                {
                     errors.push(Error::new(
                         &message,
                         Some(&origin_path),
-                        Some((
-                            &origin_contents,
-                            &LineIndex::new(&origin_contents),
-                            origin_source_range,
-                        )),
+                        Some((&origin_contents, &origin_line_index, origin_source_range)),
                         Some(Arc::new(error)),
                     ));
                 } else {
@@ -161,11 +170,11 @@ pub fn load_schemas(
             }
         };
 
-        // Index the lines of the contents so errors about the imports can show them.
+        // Index the lines of the contents so errors can show them.
         let line_index = LineIndex::new(&contents);
 
         // Tokenize the contents.
-        let tokens = match tokenize(&path, &contents) {
+        let tokens = match tokenize(&path, &contents, &line_index) {
             Ok(tokens) => tokens,
             Err(error) => {
                 errors.extend_from_slice(&error);
@@ -175,7 +184,7 @@ pub fn load_schemas(
         };
 
         // Parse the tokens.
-        let mut schema = match parse(&path, &contents, &tokens) {
+        let mut schema = match parse(&path, &contents, &line_index, &tokens) {
             Ok(schema) => schema,
             Err(error) => {
                 errors.extend_from_slice(&error);
@@ -250,15 +259,29 @@ pub fn load_schemas(
                 schemas_to_load.push((
                     import_namespace,
                     based_import_path,
-                    Some((path.clone(), contents.clone(), import.source_range)),
+                    Some((
+                        path.clone(),
+                        contents.clone(),
+                        line_index.clone(),
+                        import.source_range,
+                    )),
                 ));
             }
         }
 
         // Store the schema.
-        if let Some((_, conflicting_schema_path, _)) =
-            schemas.insert(namespace.clone(), (schema, path.clone(), contents))
-        {
+        if let Some(LoadedSchema {
+            path: conflicting_schema_path,
+            ..
+        }) = schemas.insert(
+            namespace.clone(),
+            LoadedSchema {
+                schema,
+                path: path.clone(),
+                contents,
+                line_index,
+            },
+        ) {
             errors.push(Error::new(
                 &format!(
                     "This file conflicts with {}, since both correspond to the same namespace {}.",