diff --git a/src/generate_rust.rs b/src/generate_rust.rs index bcf7441d..0951c0c2 100644 --- a/src/generate_rust.rs +++ b/src/generate_rust.rs @@ -1,11 +1,11 @@ use crate::{ identifier::Identifier, schema::{self, relativize_namespace}, + schema_loader::LoadedSchema, }; use std::{ collections::BTreeMap, fmt::{self, Write}, - path::PathBuf, }; // The string to be used for each indentation level. @@ -101,7 +101,7 @@ use Direction::{Atlas, In, Out}; #[allow(clippy::too_many_lines)] pub fn generate( typical_version: &str, - schemas: &BTreeMap, + schemas: &BTreeMap, ) -> String { // Construct a tree of modules and schemas. We start with an empty tree. let mut tree = Module { @@ -114,7 +114,7 @@ pub fn generate( }; // Populate the tree with all the schemas. - for (namespace, (schema, _, _)) in schemas { + for (namespace, LoadedSchema { schema, .. }) in schemas { insert_schema(&mut tree, namespace, schema); } diff --git a/src/generate_typescript.rs b/src/generate_typescript.rs index bd625248..9c132e32 100644 --- a/src/generate_typescript.rs +++ b/src/generate_typescript.rs @@ -1,4 +1,4 @@ -use crate::{identifier::Identifier, schema}; +use crate::{identifier::Identifier, schema, schema_loader::LoadedSchema}; use std::{ collections::BTreeMap, fmt::{self, Write}, @@ -142,7 +142,7 @@ impl DeclarationFunctionNames { // to the configured output directory. pub fn generate( typical_version: &str, - schemas: &BTreeMap, + schemas: &BTreeMap, import_extension: &str, ) -> BTreeMap { let mut files = BTreeMap::new(); @@ -155,7 +155,7 @@ pub fn generate( common_buffer, ); - for (namespace, (schema, _, _)) in schemas { + for (namespace, LoadedSchema { schema, .. }) in schemas { let mut buffer = String::new(); // The `unwrap` is safe because the `std::fmt::Write` impl for `String` is infallible. write_schema_file( @@ -2625,7 +2625,9 @@ mod tests { use crate::{ error::SourceRange, generate_typescript::{COMMON_FILE_STEM, generate}, + line_index::LineIndex, schema::{self, Namespace}, + schema_loader::LoadedSchema, schema_loader::load_schemas, validator::validate, }; @@ -2684,20 +2686,21 @@ mod tests { let mut schemas = BTreeMap::new(); schemas.insert( dependency_namespace, - ( - schema::Schema { + LoadedSchema { + schema: schema::Schema { comment: vec![], imports: BTreeMap::new(), declarations: vec![], }, - PathBuf::from("Foo/FirstSchema.t"), - String::new(), - ), + path: PathBuf::from("Foo/FirstSchema.t"), + contents: String::new(), + line_index: LineIndex::new(""), + }, ); schemas.insert( types_namespace, - ( - schema::Schema { + LoadedSchema { + schema: schema::Schema { comment: vec![], imports, declarations: vec![schema::Declaration { @@ -2709,9 +2712,10 @@ mod tests { deleted: BTreeSet::new(), }], }, - PathBuf::from("Bar/SecondSchema.t"), - String::new(), - ), + path: PathBuf::from("Bar/SecondSchema.t"), + contents: String::new(), + line_index: LineIndex::new(""), + }, ); let generated = generate("0.0.0", &schemas, ".ts"); diff --git a/src/main.rs b/src/main.rs index 0645ce26..1c83300a 100644 --- a/src/main.rs +++ b/src/main.rs @@ -20,7 +20,7 @@ use crate::{ count::count, error::{Error, format_errors}, format::{CodePath, CodeStr}, - schema_loader::load_schemas, + schema_loader::{LoadedSchema, load_schemas}, validator::validate, }; use clap::{ArgAction, Args, CommandFactory, Parser, Subcommand as ClapSubcommand}; @@ -147,7 +147,10 @@ fn generate_code( // The `unwrap` is safe since otherwise the schema would've failed to load above. let directory = schema_path.parent().unwrap(); - for (_, source_path, _) in schemas.values() { + for LoadedSchema { + path: source_path, .. + } in schemas.values() + { println!("{}", directory.join(source_path).display()); } } @@ -251,7 +254,13 @@ fn format_schema(schema_path: &Path, check: bool) -> Result<(), Vec> { "{} schemas\u{2026}", if check { "Checking" } else { "Formatting" }, ); - for (schema, source_path, source_contents) in schemas.values() { + for LoadedSchema { + schema, + path: source_path, + contents: source_contents, + .. + } in schemas.values() + { // Compute the full path and new contents of the schema. let full_source_path = directory.join(source_path); let new_source_contents = schema.to_string(); diff --git a/src/parser.rs b/src/parser.rs index 62a30caa..782871a8 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -203,17 +203,16 @@ macro_rules! consume_token_1 { pub fn parse( source_path: &Path, source_contents: &str, + line_index: &LineIndex, tokens: &[token::Token], ) -> Result> { - // Try to parse the tokens into a schema, indexing the lines of the source so any errors can - // show them. + // Try to parse the tokens into a schema. let mut position = 0; let mut errors = vec![]; - let line_index = LineIndex::new(source_contents); let schema = parse_schema( source_path, source_contents, - &line_index, + line_index, tokens, &mut position, &mut errors, @@ -227,7 +226,7 @@ pub fn parse( Some(source_path), Some(( source_contents, - &line_index, + line_index, token_source_range(tokens, position), )), None, @@ -923,7 +922,8 @@ fn parse_type( #[cfg(test)] mod tests { use crate::{ - assert_fails, assert_same, error::SourceRange, parser::parse, schema, tokenizer::tokenize, + assert_fails, assert_same, error::SourceRange, line_index::LineIndex, parser::parse, + schema, tokenizer::tokenize, }; use std::{ collections::{BTreeMap, BTreeSet}, @@ -935,10 +935,11 @@ mod tests { fn parse_empty() { let source_path = Path::new("foo.t"); let source = ""; - let tokens = tokenize(source_path, source).unwrap(); + let line_index = LineIndex::new(source); + let tokens = tokenize(source_path, source, &line_index).unwrap(); assert_same!( - parse(source_path, source, &tokens[..]), + parse(source_path, source, &line_index, &tokens[..]), Ok(schema::Schema { comment: vec![], imports: BTreeMap::new(), @@ -979,7 +980,8 @@ mod tests { z = 3 } "; - let tokens = tokenize(source_path, source).unwrap(); + let line_index = LineIndex::new(source); + let tokens = tokenize(source_path, source, &line_index).unwrap(); let mut imports = BTreeMap::new(); @@ -1183,7 +1185,7 @@ mod tests { ]; assert_same!( - parse(source_path, source, &tokens[..]), + parse(source_path, source, &line_index, &tokens[..]), Ok(schema::Schema { comment: vec!["This is an example schema.".to_owned()], imports, @@ -1199,10 +1201,11 @@ mod tests { import 'foo.t' as Qux import 'bar.t' as qux "; - let tokens = tokenize(source_path, source).unwrap(); + let line_index = LineIndex::new(source); + let tokens = tokenize(source_path, source, &line_index).unwrap(); assert_fails!( - parse(source_path, source, &tokens[..]), + parse(source_path, source, &line_index, &tokens[..]), "An import named `qux` already exists in this file.", ); } @@ -1215,10 +1218,11 @@ mod tests { deleted 1 2 2 3 } "; - let tokens = tokenize(source_path, source).unwrap(); + let line_index = LineIndex::new(source); + let tokens = tokenize(source_path, source, &line_index).unwrap(); assert_fails!( - parse(source_path, source, &tokens[..]), + parse(source_path, source, &line_index, &tokens[..]), "Index `2` is already marked as deleted.", ); } diff --git a/src/schema_loader.rs b/src/schema_loader.rs index 8638f595..fdeb07b2 100644 --- a/src/schema_loader.rs +++ b/src/schema_loader.rs @@ -15,6 +15,14 @@ use std::{ sync::Arc, }; +// This is a schema along with the file it was loaded from. +pub struct LoadedSchema { + pub schema: schema::Schema, + pub path: PathBuf, + pub contents: String, + pub line_index: LineIndex, +} + // Convert a UTF-8 path containing only normal components to a namespace. fn path_to_namespace(path: &Path) -> Option { let mut path = path.to_owned(); @@ -37,10 +45,9 @@ fn path_to_namespace(path: &Path) -> Option { // Load a schema and its transitive dependencies. The imports in the returned schemas are guaranteed // to resolve. #[allow(clippy::too_many_lines)] -#[allow(clippy::type_complexity)] pub fn load_schemas( schema_path: &Path, -) -> Result, Vec> { +) -> Result, Vec> { // Reject paths that cannot be represented faithfully in schema namespaces and diagnostics // [tag:schema_path_valid_utf8]. if schema_path.to_str().is_none() { @@ -129,7 +136,7 @@ pub fn load_schemas( let mut schemas_to_load = vec![( schema_namespace.clone(), based_schema_path.to_owned(), - None as Option<(PathBuf, String, SourceRange)>, + None as Option<(PathBuf, String, LineIndex, SourceRange)>, )]; let mut visited_namespaces = HashSet::new(); visited_namespaces.insert(schema_namespace); @@ -142,15 +149,17 @@ pub fn load_schemas( Err(error) => { let message = format!("Unable to load {}.", path.code_path()); - if let Some((origin_path, origin_contents, origin_source_range)) = origin { + if let Some(( + origin_path, + origin_contents, + origin_line_index, + origin_source_range, + )) = origin + { errors.push(Error::new( &message, Some(&origin_path), - Some(( - &origin_contents, - &LineIndex::new(&origin_contents), - origin_source_range, - )), + Some((&origin_contents, &origin_line_index, origin_source_range)), Some(Arc::new(error)), )); } else { @@ -161,11 +170,11 @@ pub fn load_schemas( } }; - // Index the lines of the contents so errors about the imports can show them. + // Index the lines of the contents so errors can show them. let line_index = LineIndex::new(&contents); // Tokenize the contents. - let tokens = match tokenize(&path, &contents) { + let tokens = match tokenize(&path, &contents, &line_index) { Ok(tokens) => tokens, Err(error) => { errors.extend_from_slice(&error); @@ -175,7 +184,7 @@ pub fn load_schemas( }; // Parse the tokens. - let mut schema = match parse(&path, &contents, &tokens) { + let mut schema = match parse(&path, &contents, &line_index, &tokens) { Ok(schema) => schema, Err(error) => { errors.extend_from_slice(&error); @@ -250,15 +259,29 @@ pub fn load_schemas( schemas_to_load.push(( import_namespace, based_import_path, - Some((path.clone(), contents.clone(), import.source_range)), + Some(( + path.clone(), + contents.clone(), + line_index.clone(), + import.source_range, + )), )); } } // Store the schema. - if let Some((_, conflicting_schema_path, _)) = - schemas.insert(namespace.clone(), (schema, path.clone(), contents)) - { + if let Some(LoadedSchema { + path: conflicting_schema_path, + .. + }) = schemas.insert( + namespace.clone(), + LoadedSchema { + schema, + path: path.clone(), + contents, + line_index, + }, + ) { errors.push(Error::new( &format!( "This file conflicts with {}, since both correspond to the same namespace {}.", diff --git a/src/tokenizer.rs b/src/tokenizer.rs index a1c2cc22..128c1f93 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -17,14 +17,16 @@ const RAW_IDENTIFIER_SIGIL: char = '$'; // Tokenize the contents of a schema file. #[allow(clippy::cognitive_complexity)] #[allow(clippy::too_many_lines)] -pub fn tokenize(schema_path: &Path, schema_contents: &str) -> Result, Vec> { +pub fn tokenize( + schema_path: &Path, + schema_contents: &str, + line_index: &LineIndex, +) -> Result, Vec> { // We'll be building up this vector of tokens. let mut tokens = vec![]; - // Construct a vector to hold any errors that might be detected below, and index the lines of - // the schema so the errors can show them. + // Construct a vector to hold any errors that might be detected below. let mut errors = vec![]; - let line_index = LineIndex::new(schema_contents); // We want to iterate one code point at a time, but we also want the byte indices so we can // capture slices. @@ -191,7 +193,7 @@ pub fn tokenize(schema_path: &Path, schema_contents: &str) -> Result, errors.push(Error::new( "Identifiers cannot be empty.", Some(schema_path), - Some((schema_contents, &line_index, SourceRange { start: i, end })), + Some((schema_contents, line_index, SourceRange { start: i, end })), None, )); } @@ -200,7 +202,7 @@ pub fn tokenize(schema_path: &Path, schema_contents: &str) -> Result, errors.push(Error::new( "Identifiers cannot begin with `_`.", Some(schema_path), - Some((schema_contents, &line_index, SourceRange { start: i, end })), + Some((schema_contents, line_index, SourceRange { start: i, end })), None, )); } @@ -241,7 +243,7 @@ pub fn tokenize(schema_path: &Path, schema_contents: &str) -> Result, schema_contents[i..end].code_str(), ), Some(schema_path), - Some((schema_contents, &line_index, SourceRange { start: i, end })), + Some((schema_contents, line_index, SourceRange { start: i, end })), None, )); } @@ -269,7 +271,7 @@ pub fn tokenize(schema_path: &Path, schema_contents: &str) -> Result, Some(schema_path), Some(( schema_contents, - &line_index, + line_index, SourceRange { start: i, end: i + 1, @@ -376,7 +378,7 @@ pub fn tokenize(schema_path: &Path, schema_contents: &str) -> Result, Some(schema_path), Some(( schema_contents, - &line_index, + line_index, SourceRange { start: i, end: i }, )), None, @@ -399,6 +401,7 @@ mod tests { use crate::{ assert_fails, assert_same, error::SourceRange, + line_index::LineIndex, token::{ AS_KEYWORD, ASYMMETRIC_KEYWORD, BOOL_KEYWORD, BYTES_KEYWORD, CHOICE_KEYWORD, DELETED_KEYWORD, F64_KEYWORD, IMPORT_KEYWORD, OPTIONAL_KEYWORD, S64_KEYWORD, @@ -421,7 +424,7 @@ mod tests { "; assert_same!( - tokenize(Path::new("foo.t"), source).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![ Token { source_range: SourceRange { start: 13, end: 32 }, @@ -484,18 +487,30 @@ mod tests { #[test] fn tokenize_empty() { - assert_same!(tokenize(Path::new("foo.t"), "").unwrap(), vec![]); + let source = ""; + + assert_same!( + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), + vec![], + ); } #[test] fn tokenize_whitespace() { - assert_same!(tokenize(Path::new("foo.t"), " \t\n").unwrap(), vec![]); + let source = " \t\n"; + + assert_same!( + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), + vec![], + ); } #[test] fn tokenize_as() { + let source = AS_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), AS_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -508,8 +523,10 @@ mod tests { #[test] fn tokenize_asymmetric() { + let source = ASYMMETRIC_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), ASYMMETRIC_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -522,8 +539,10 @@ mod tests { #[test] fn tokenize_bool() { + let source = BOOL_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), BOOL_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -536,8 +555,10 @@ mod tests { #[test] fn tokenize_bytes() { + let source = BYTES_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), BYTES_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -550,8 +571,10 @@ mod tests { #[test] fn tokenize_choice() { + let source = CHOICE_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), CHOICE_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -564,8 +587,10 @@ mod tests { #[test] fn tokenize_colon() { + let source = ":"; + assert_same!( - tokenize(Path::new("foo.t"), ":").unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, end: 1 }, variant: Variant::Colon, @@ -575,8 +600,10 @@ mod tests { #[test] fn tokenize_comment_simple() { + let source = "# Hello, World!"; + assert_same!( - tokenize(Path::new("foo.t"), "# Hello, World!").unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, end: 15 }, variant: Variant::Comment(vec!["Hello, World!".to_owned()]), @@ -586,12 +613,10 @@ mod tests { #[test] fn tokenize_comment_complex() { + let source = " # \n # Hello, \n # World! \n # \n # Hello, \n # Earth! \n # "; + assert_same!( - tokenize( - Path::new("foo.t"), - " # \n # Hello, \n # World! \n # \n # Hello, \n # Earth! \n # ", - ) - .unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 1, end: 55 }, variant: Variant::Comment(vec![ @@ -604,8 +629,10 @@ mod tests { #[test] fn tokenize_deleted() { + let source = DELETED_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), DELETED_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -618,8 +645,10 @@ mod tests { #[test] fn tokenize_dot() { + let source = "."; + assert_same!( - tokenize(Path::new("foo.t"), ".").unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, end: 1 }, variant: Variant::Dot, @@ -629,8 +658,10 @@ mod tests { #[test] fn tokenize_equals() { + let source = "="; + assert_same!( - tokenize(Path::new("foo.t"), "=").unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, end: 1 }, variant: Variant::Equals, @@ -640,8 +671,10 @@ mod tests { #[test] fn tokenize_f64() { + let source = F64_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), F64_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -654,8 +687,10 @@ mod tests { #[test] fn tokenize_bare_identifier() { + let source = "\u{5e78}\u{798f}"; + assert_same!( - tokenize(Path::new("foo.t"), "\u{5e78}\u{798f}").unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, end: 6 }, variant: Variant::Identifier("\u{5e78}\u{798f}".into()), @@ -665,12 +700,10 @@ mod tests { #[test] fn tokenize_raw_identifier() { + let source = format!("{RAW_IDENTIFIER_SIGIL}{STRUCT_KEYWORD}"); + assert_same!( - tokenize( - Path::new("foo.t"), - &format!("{RAW_IDENTIFIER_SIGIL}{STRUCT_KEYWORD}"), - ) - .unwrap(), + tokenize(Path::new("foo.t"), &source, &LineIndex::new(&source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, end: 7 }, variant: Variant::Identifier(STRUCT_KEYWORD.into()), @@ -680,27 +713,30 @@ mod tests { #[test] fn tokenize_bare_identifier_underscore_prefix() { + let source = "_foo"; + assert_fails!( - tokenize(Path::new("foo.t"), "_foo"), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)), "Identifiers cannot begin with `_`.", ); } #[test] fn tokenize_raw_identifier_underscore_prefix() { + let source = format!("{}{}", RAW_IDENTIFIER_SIGIL, "_foo"); + assert_fails!( - tokenize( - Path::new("foo.t"), - &format!("{}{}", RAW_IDENTIFIER_SIGIL, "_foo"), - ), + tokenize(Path::new("foo.t"), &source, &LineIndex::new(&source)), "Identifiers cannot begin with `_`.", ); } #[test] fn tokenize_import() { + let source = IMPORT_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), IMPORT_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -713,8 +749,10 @@ mod tests { #[test] fn tokenize_integer_literal_valid() { + let source = "42"; + assert_same!( - tokenize(Path::new("foo.t"), "42").unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, end: 2 }, variant: Variant::Integer(42), @@ -724,16 +762,20 @@ mod tests { #[test] fn tokenize_integer_literal_out_of_range() { + let source = "18446744073709551616"; + assert_fails!( - tokenize(Path::new("foo.t"), "18446744073709551616"), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)), "Integer `18446744073709551616` must be less than 2^64.", ); } #[test] fn tokenize_left_curly() { + let source = "{"; + assert_same!( - tokenize(Path::new("foo.t"), "{").unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, end: 1 }, variant: Variant::LeftCurly, @@ -743,8 +785,10 @@ mod tests { #[test] fn tokenize_left_square() { + let source = "["; + assert_same!( - tokenize(Path::new("foo.t"), "[").unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, end: 1 }, variant: Variant::LeftSquare, @@ -754,8 +798,10 @@ mod tests { #[test] fn tokenize_optional() { + let source = OPTIONAL_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), OPTIONAL_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -768,8 +814,10 @@ mod tests { #[test] fn tokenize_path_non_empty() { + let source = "'bar.t'"; + assert_same!( - tokenize(Path::new("foo.t"), "'bar.t'").unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, end: 7 }, variant: Variant::Path(Path::new("bar.t").to_owned()), @@ -779,8 +827,10 @@ mod tests { #[test] fn tokenize_path_empty() { + let source = "''"; + assert_same!( - tokenize(Path::new("foo.t"), "''").unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, end: 2 }, variant: Variant::Path(Path::new("").to_owned()), @@ -790,16 +840,20 @@ mod tests { #[test] fn tokenize_path_non_terminated() { + let source = "'bar.t"; + assert_fails!( - tokenize(Path::new("foo.t"), "'bar.t"), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)), "Path starting here must be terminated by a `\'`.", ); } #[test] fn tokenize_right_curly() { + let source = "}"; + assert_same!( - tokenize(Path::new("foo.t"), "}").unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, end: 1 }, variant: Variant::RightCurly, @@ -809,8 +863,10 @@ mod tests { #[test] fn tokenize_right_square() { + let source = "]"; + assert_same!( - tokenize(Path::new("foo.t"), "]").unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, end: 1 }, variant: Variant::RightSquare, @@ -820,8 +876,10 @@ mod tests { #[test] fn tokenize_s64() { + let source = S64_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), S64_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -834,8 +892,10 @@ mod tests { #[test] fn tokenize_string() { + let source = STRING_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), STRING_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -848,8 +908,10 @@ mod tests { #[test] fn tokenize_struct() { + let source = STRUCT_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), STRUCT_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -862,8 +924,10 @@ mod tests { #[test] fn tokenize_u64() { + let source = U64_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), U64_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -876,8 +940,10 @@ mod tests { #[test] fn tokenize_unit() { + let source = UNIT_KEYWORD; + assert_same!( - tokenize(Path::new("foo.t"), UNIT_KEYWORD).unwrap(), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)).unwrap(), vec![Token { source_range: SourceRange { start: 0, @@ -890,8 +956,10 @@ mod tests { #[test] fn tokenize_unexpected_symbol() { + let source = "\u{1f610}\u{fe0f}"; + assert_fails!( - tokenize(Path::new("foo.t"), "\u{1f610}\u{fe0f}"), + tokenize(Path::new("foo.t"), source, &LineIndex::new(source)), "Unexpected symbol `\u{1f610}\u{fe0f}`.", ); } diff --git a/src/validator.rs b/src/validator.rs index f99e5acb..2c8b91f9 100644 --- a/src/validator.rs +++ b/src/validator.rs @@ -1,10 +1,10 @@ use crate::{ error::Error, format::CodeStr, generate_typescript::COMMON_FILE_STEM, identifier::Identifier, - line_index::LineIndex, schema, + line_index::LineIndex, schema, schema_loader::LoadedSchema, }; use std::{ collections::{BTreeMap, HashMap, HashSet}, - path::{Path, PathBuf}, + path::Path, }; // The index will be encoded as a 64-bit integer, but two of the bits are used to help encode the @@ -19,9 +19,7 @@ const TYPESCRIPT_RESERVED_MODULE_NAME_ERROR: &str = concat!( // This function validates a schema and its transitive dependencies. #[allow(clippy::too_many_lines)] -pub fn validate( - schemas: &BTreeMap, -) -> Result<(), Vec> { +pub fn validate(schemas: &BTreeMap) -> Result<(), Vec> { // We'll add any errors to this. let mut errors: Vec = vec![]; @@ -30,7 +28,13 @@ pub fn validate( let common_namespace = schema::Namespace { components: vec![COMMON_FILE_STEM.into()], }; - for (namespace, (_, source_path, _)) in schemas { + for ( + namespace, + LoadedSchema { + path: source_path, .. + }, + ) in schemas + { if namespace == &common_namespace { errors.push(Error::new( TYPESCRIPT_RESERVED_MODULE_NAME_ERROR, @@ -44,7 +48,7 @@ pub fn validate( // For the purpose of validating types, construct a map from (namespace, name) to // (schema, declaration). let mut all_types = HashMap::new(); - for (namespace, (schema, _, _)) in schemas { + for (namespace, LoadedSchema { schema, .. }) in schemas { for declaration in &schema.declarations { all_types.insert( (namespace.clone(), declaration.name.clone()), @@ -54,10 +58,16 @@ pub fn validate( } // Validate each file. - for (namespace, (schema, source_path, source_contents)) in schemas { - // Index the lines of the file so errors can show them. - let line_index = LineIndex::new(source_contents); - + for ( + namespace, + LoadedSchema { + schema, + path: source_path, + contents: source_contents, + line_index, + }, + ) in schemas + { // Validate the declarations in the file. let mut declaration_names = HashSet::new(); @@ -70,7 +80,7 @@ pub fn validate( declaration.name.code_str(), ), Some(source_path), - Some((source_contents, &line_index, declaration.source_range)), + Some((source_contents, line_index, declaration.source_range)), None, )); } @@ -88,7 +98,7 @@ pub fn validate( field.name.code_str(), ), Some(source_path), - Some((source_contents, &line_index, field.source_range)), + Some((source_contents, line_index, field.source_range)), None, )); } @@ -101,7 +111,7 @@ pub fn validate( field.index.to_string().code_str(), ), Some(source_path), - Some((source_contents, &line_index, field.source_range)), + Some((source_contents, line_index, field.source_range)), None, )); } @@ -114,7 +124,7 @@ pub fn validate( field.index.to_string().code_str(), ), Some(source_path), - Some((source_contents, &line_index, field.source_range)), + Some((source_contents, line_index, field.source_range)), None, )); } @@ -128,7 +138,7 @@ pub fn validate( MAX_FIELD_INDEX.to_string().code_str(), ), Some(source_path), - Some((source_contents, &line_index, field.source_range)), + Some((source_contents, line_index, field.source_range)), None, )); } @@ -141,7 +151,7 @@ pub fn validate( schema, source_path, source_contents, - &line_index, + line_index, &field.r#type, ); } @@ -155,7 +165,7 @@ pub fn validate( index.to_string().code_str(), ), Some(source_path), - Some((source_contents, &line_index, declaration.source_range)), + Some((source_contents, line_index, declaration.source_range)), None, )); } @@ -170,7 +180,7 @@ pub fn validate( let mut types_visited_set = HashSet::new(); let mut types_visited_vec = vec![]; - for (namespace, (schema, _, _)) in schemas { + for (namespace, LoadedSchema { schema, .. }) in schemas { for declaration in &schema.declarations { check_declaration_for_cycles( &all_types, @@ -409,8 +419,8 @@ fn check_type_for_cycles( mod tests { use super::TYPESCRIPT_RESERVED_MODULE_NAME_ERROR; use crate::{ - assert_fails, assert_same, parser::parse, schema::Namespace, tokenizer::tokenize, - validator::validate, + assert_fails, assert_same, line_index::LineIndex, parser::parse, schema::Namespace, + schema_loader::LoadedSchema, tokenizer::tokenize, validator::validate, }; use std::{collections::BTreeMap, fmt::Write, path::Path}; @@ -422,11 +432,21 @@ mod tests { let path = Path::new("common.t").to_owned(); let contents = String::new(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!(validate(&schemas), TYPESCRIPT_RESERVED_MODULE_NAME_ERROR); } @@ -439,11 +459,21 @@ mod tests { let path = Path::new("Common.t").to_owned(); let contents = String::new(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!(validate(&schemas), TYPESCRIPT_RESERVED_MODULE_NAME_ERROR); } @@ -456,11 +486,21 @@ mod tests { let path = Path::new("foo.t").to_owned(); let contents = String::new(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_same!(validate(&schemas), Ok(())); } @@ -495,16 +535,36 @@ mod tests { " .to_owned(); - let foo_tokens = tokenize(&foo_path, &foo_contents).unwrap(); - let mut foo_schema = parse(&foo_path, &foo_contents, &foo_tokens).unwrap(); + let foo_line_index = LineIndex::new(&foo_contents); + + let foo_tokens = tokenize(&foo_path, &foo_contents, &foo_line_index).unwrap(); + let mut foo_schema = parse(&foo_path, &foo_contents, &foo_line_index, &foo_tokens).unwrap(); foo_schema.imports.get_mut(&"bar".into()).unwrap().namespace = Some(bar_namespace.clone()); - let bar_tokens = tokenize(&bar_path, &bar_contents).unwrap(); - let bar_schema = parse(&bar_path, &bar_contents, &bar_tokens).unwrap(); + let bar_line_index = LineIndex::new(&bar_contents); + + let bar_tokens = tokenize(&bar_path, &bar_contents, &bar_line_index).unwrap(); + let bar_schema = parse(&bar_path, &bar_contents, &bar_line_index, &bar_tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(foo_namespace, (foo_schema, foo_path, foo_contents)); - schemas.insert(bar_namespace, (bar_schema, bar_path, bar_contents)); + schemas.insert( + foo_namespace, + LoadedSchema { + schema: foo_schema, + path: foo_path, + contents: foo_contents, + line_index: foo_line_index, + }, + ); + schemas.insert( + bar_namespace, + LoadedSchema { + schema: bar_schema, + path: bar_path, + contents: bar_contents, + line_index: bar_line_index, + }, + ); assert_same!(validate(&schemas), Ok(())); } @@ -523,11 +583,20 @@ mod tests { } " .to_owned(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!( validate(&schemas), @@ -548,11 +617,20 @@ mod tests { } " .to_owned(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!( validate(&schemas), @@ -573,11 +651,20 @@ mod tests { } " .to_owned(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!( validate(&schemas), @@ -599,11 +686,20 @@ mod tests { } " .to_owned(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!( validate(&schemas), @@ -626,11 +722,20 @@ mod tests { } " .to_owned(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!( validate(&schemas), @@ -651,11 +756,20 @@ mod tests { } " .to_owned(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!( validate(&schemas), @@ -676,11 +790,20 @@ mod tests { } " .to_owned(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!( validate(&schemas), @@ -702,11 +825,20 @@ mod tests { } " .to_owned(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!( validate(&schemas), @@ -729,11 +861,20 @@ mod tests { } " .to_owned(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!( validate(&schemas), @@ -753,11 +894,20 @@ mod tests { } " .to_owned(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!( validate(&schemas), @@ -777,11 +927,20 @@ mod tests { } " .to_owned(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!( validate(&schemas), @@ -801,11 +960,20 @@ mod tests { } " .to_owned(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!( validate(&schemas), @@ -825,11 +993,20 @@ mod tests { } " .to_owned(); - let tokens = tokenize(&path, &contents).unwrap(); - let schema = parse(&path, &contents, &tokens).unwrap(); + let line_index = LineIndex::new(&contents); + let tokens = tokenize(&path, &contents, &line_index).unwrap(); + let schema = parse(&path, &contents, &line_index, &tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(namespace, (schema, path, contents)); + schemas.insert( + namespace, + LoadedSchema { + schema, + path, + contents, + line_index, + }, + ); assert_fails!( validate(&schemas), @@ -862,16 +1039,36 @@ mod tests { " .to_owned(); - let foo_tokens = tokenize(&foo_path, &foo_contents).unwrap(); - let mut foo_schema = parse(&foo_path, &foo_contents, &foo_tokens).unwrap(); + let foo_line_index = LineIndex::new(&foo_contents); + + let foo_tokens = tokenize(&foo_path, &foo_contents, &foo_line_index).unwrap(); + let mut foo_schema = parse(&foo_path, &foo_contents, &foo_line_index, &foo_tokens).unwrap(); foo_schema.imports.get_mut(&"bar".into()).unwrap().namespace = Some(bar_namespace.clone()); - let bar_tokens = tokenize(&bar_path, &bar_contents).unwrap(); - let bar_schema = parse(&bar_path, &bar_contents, &bar_tokens).unwrap(); + let bar_line_index = LineIndex::new(&bar_contents); + + let bar_tokens = tokenize(&bar_path, &bar_contents, &bar_line_index).unwrap(); + let bar_schema = parse(&bar_path, &bar_contents, &bar_line_index, &bar_tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(foo_namespace, (foo_schema, foo_path, foo_contents)); - schemas.insert(bar_namespace, (bar_schema, bar_path, bar_contents)); + schemas.insert( + foo_namespace, + LoadedSchema { + schema: foo_schema, + path: foo_path, + contents: foo_contents, + line_index: foo_line_index, + }, + ); + schemas.insert( + bar_namespace, + LoadedSchema { + schema: bar_schema, + path: bar_path, + contents: bar_contents, + line_index: bar_line_index, + }, + ); assert_fails!( validate(&schemas), @@ -904,16 +1101,36 @@ mod tests { " .to_owned(); - let foo_tokens = tokenize(&foo_path, &foo_contents).unwrap(); - let mut foo_schema = parse(&foo_path, &foo_contents, &foo_tokens).unwrap(); + let foo_line_index = LineIndex::new(&foo_contents); + + let foo_tokens = tokenize(&foo_path, &foo_contents, &foo_line_index).unwrap(); + let mut foo_schema = parse(&foo_path, &foo_contents, &foo_line_index, &foo_tokens).unwrap(); foo_schema.imports.get_mut(&"bar".into()).unwrap().namespace = Some(bar_namespace.clone()); - let bar_tokens = tokenize(&bar_path, &bar_contents).unwrap(); - let bar_schema = parse(&bar_path, &bar_contents, &bar_tokens).unwrap(); + let bar_line_index = LineIndex::new(&bar_contents); + + let bar_tokens = tokenize(&bar_path, &bar_contents, &bar_line_index).unwrap(); + let bar_schema = parse(&bar_path, &bar_contents, &bar_line_index, &bar_tokens).unwrap(); let mut schemas = BTreeMap::new(); - schemas.insert(foo_namespace, (foo_schema, foo_path, foo_contents)); - schemas.insert(bar_namespace, (bar_schema, bar_path, bar_contents)); + schemas.insert( + foo_namespace, + LoadedSchema { + schema: foo_schema, + path: foo_path, + contents: foo_contents, + line_index: foo_line_index, + }, + ); + schemas.insert( + bar_namespace, + LoadedSchema { + schema: bar_schema, + path: bar_path, + contents: bar_contents, + line_index: bar_line_index, + }, + ); assert_fails!( validate(&schemas), @@ -949,17 +1166,37 @@ mod tests { " .to_owned(); - let foo_tokens = tokenize(&foo_path, &foo_contents).unwrap(); - let mut foo_schema = parse(&foo_path, &foo_contents, &foo_tokens).unwrap(); + let foo_line_index = LineIndex::new(&foo_contents); + + let foo_tokens = tokenize(&foo_path, &foo_contents, &foo_line_index).unwrap(); + let mut foo_schema = parse(&foo_path, &foo_contents, &foo_line_index, &foo_tokens).unwrap(); foo_schema.imports.get_mut(&"bar".into()).unwrap().namespace = Some(bar_namespace.clone()); - let bar_tokens = tokenize(&bar_path, &bar_contents).unwrap(); - let mut bar_schema = parse(&bar_path, &bar_contents, &bar_tokens).unwrap(); + let bar_line_index = LineIndex::new(&bar_contents); + + let bar_tokens = tokenize(&bar_path, &bar_contents, &bar_line_index).unwrap(); + let mut bar_schema = parse(&bar_path, &bar_contents, &bar_line_index, &bar_tokens).unwrap(); bar_schema.imports.get_mut(&"foo".into()).unwrap().namespace = Some(foo_namespace.clone()); let mut schemas = BTreeMap::new(); - schemas.insert(foo_namespace, (foo_schema, foo_path, foo_contents)); - schemas.insert(bar_namespace, (bar_schema, bar_path, bar_contents)); + schemas.insert( + foo_namespace, + LoadedSchema { + schema: foo_schema, + path: foo_path, + contents: foo_contents, + line_index: foo_line_index, + }, + ); + schemas.insert( + bar_namespace, + LoadedSchema { + schema: bar_schema, + path: bar_path, + contents: bar_contents, + line_index: bar_line_index, + }, + ); assert_fails!( validate(&schemas), @@ -995,17 +1232,37 @@ mod tests { " .to_owned(); - let foo_tokens = tokenize(&foo_path, &foo_contents).unwrap(); - let mut foo_schema = parse(&foo_path, &foo_contents, &foo_tokens).unwrap(); + let foo_line_index = LineIndex::new(&foo_contents); + + let foo_tokens = tokenize(&foo_path, &foo_contents, &foo_line_index).unwrap(); + let mut foo_schema = parse(&foo_path, &foo_contents, &foo_line_index, &foo_tokens).unwrap(); foo_schema.imports.get_mut(&"bar".into()).unwrap().namespace = Some(bar_namespace.clone()); - let bar_tokens = tokenize(&bar_path, &bar_contents).unwrap(); - let mut bar_schema = parse(&bar_path, &bar_contents, &bar_tokens).unwrap(); + let bar_line_index = LineIndex::new(&bar_contents); + + let bar_tokens = tokenize(&bar_path, &bar_contents, &bar_line_index).unwrap(); + let mut bar_schema = parse(&bar_path, &bar_contents, &bar_line_index, &bar_tokens).unwrap(); bar_schema.imports.get_mut(&"foo".into()).unwrap().namespace = Some(foo_namespace.clone()); let mut schemas = BTreeMap::new(); - schemas.insert(foo_namespace, (foo_schema, foo_path, foo_contents)); - schemas.insert(bar_namespace, (bar_schema, bar_path, bar_contents)); + schemas.insert( + foo_namespace, + LoadedSchema { + schema: foo_schema, + path: foo_path, + contents: foo_contents, + line_index: foo_line_index, + }, + ); + schemas.insert( + bar_namespace, + LoadedSchema { + schema: bar_schema, + path: bar_path, + contents: bar_contents, + line_index: bar_line_index, + }, + ); assert_fails!( validate(&schemas),