From d0e77d4cceac6ca25302a729651046a183b3117b Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Sat, 10 Oct 2026 21:14:08 +0000 Subject: [PATCH 1/7] rt: take the DATA table from the program at startup; a host feature Two changes so the runtime can be linked into the compiler itself, which the JIT (xbasic64 run) needs; compiled programs behave as before. - The runtime named three symbols the program defined: _data_table, _data_count and _data_ptr. Now a program with DATA hands its table and item count to the new _rt_data_init as main starts, and the runtime keeps the READ position itself. The table is a local, read-only object, and a program without DATA has none. rt.o now needs nothing of a program -- only the C library, which its nm test asserts. - A host feature drops the runtime's #[panic_handler], which would clash with std's in the compiler. build.rs's staticlib builds without it. - A debug compiler builds rt optimized, without overflow checks or debug assertions, as build.rs builds the copy programs link. Differential run against 705d7e0: identical output on 648 of 651 programs, the rest RND/TIMER. Co-Authored-By: Claude Opus 5.5 --- Cargo.toml | 8 +++++ rt/Cargo.toml | 5 +++ rt/src/data.rs | 84 +++++++++++++++++++-------------------------- rt/src/lib.rs | 4 ++- src/codegen/mod.rs | 68 +++++++++++++++--------------------- src/codegen/stmt.rs | 8 +++++ src/rtsig.rs | 1 + src/runtime.rs | 12 ++++--- 8 files changed, 94 insertions(+), 96 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index ac3412c..fdc51cc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -34,5 +34,13 @@ target-lexicon = "0.13" [dev-dependencies] tempfile = "3" +# The runtime as programs get it -- build.rs builds it optimized, without +# overflow checks or debug assertions -- even in a debug compiler, which runs +# it in-process for `xbasic64 run`. +[profile.dev.package.xbasic64-rt] +opt-level = 3 +debug-assertions = false +overflow-checks = false + [profile.release] lto = true diff --git a/rt/Cargo.toml b/rt/Cargo.toml index f64725c..e44ffaa 100644 --- a/rt/Cargo.toml +++ b/rt/Cargo.toml @@ -17,5 +17,10 @@ rust-version = "1.96" path = "src/lib.rs" doctest = false +[features] +# Linked into the compiler, which runs programs in its own process +# (`xbasic64 run`): std supplies the panic handler there. +host = [] + [dev-dependencies] tempfile = "3" diff --git a/rt/src/data.rs b/rt/src/data.rs index a34d225..39af9d5 100644 --- a/rt/src/data.rs +++ b/rt/src/data.rs @@ -1,8 +1,9 @@ //! DATA, READ and RESTORE, and creating an array by using it (data.s) //! -//! The compiler collects every DATA item into `_data_table`, which the -//! program defines along with `_data_count` (its length) and `_data_ptr` -//! (the next item READ takes, from 0). An item is kept as written, because +//! The compiler collects every DATA item into a table in the program, which +//! the program hands to `_rt_data_init` as it starts, with its length. The +//! runtime keeps the next item READ takes, from 0. An item is kept as +//! written, because //! GW-BASIC decides what it is when it is READ: `DATA 007` is the string //! "007" to READ A$ and the number 7 to READ A. A quoted item is a string //! only. @@ -24,48 +25,18 @@ pub struct Item { pub text: *const c_char, } -/// The program's DATA, reached by symbol. -#[cfg(not(test))] -mod program { - use super::Item; - use crate::Word; - - unsafe extern "C" { - static _data_table: [Item; 0]; - static _data_count: u64; - static _data_ptr: Word; - } - - pub fn item(i: u64) -> *const Item { - unsafe { (&raw const _data_table).cast::().add(i as usize) } - } - pub fn count() -> u64 { - unsafe { _data_count } - } - pub fn next() -> &'static Word { - unsafe { &_data_ptr } - } -} - -/// A DATA table of the tests' own, in place of the program's. -#[cfg(test)] +/// The program's DATA: where its table is, how long, and what READ takes +/// next. A program without DATA never sets them, and READ finds nothing. mod program { use super::Item; use crate::{State, Word}; - pub static TABLE: State<[Item; 8]> = State::new( - [const { - Item { - quoted: 0, - text: core::ptr::null(), - } - }; 8], - ); + pub static TABLE: State<*const Item> = State::new(core::ptr::null()); pub static COUNT: Word = Word::new(0); pub static NEXT: Word = Word::new(0); pub fn item(i: u64) -> *const Item { - unsafe { &TABLE.get()[i as usize] } + unsafe { TABLE.get().add(i as usize) } } pub fn count() -> u64 { COUNT.get() @@ -86,6 +57,18 @@ fn take() -> Option<&'static Item> { } export! { + /// The program's DATA: `count` items at `table`, which the program + /// keeps for as long as it runs (READ returns views of their text). + /// Called once, as the program starts, by a program with DATA; it + /// passes them rather than the runtime naming them, so that the runtime + /// needs nothing of a program to link -- or to run a program compiled + /// into the compiler's own process. + fn _rt_data_init(table: *const Item, count: i64) { + unsafe { *program::TABLE.get() = table }; + program::COUNT.set(count as u64); + program::NEXT.set(0); + } + /// READ into a numeric variable. The item's text must be a number and /// nothing else but blanks, read as VAL reads one; anything else -- a /// quoted item, or text that is not a number -- is GW-BASIC's Syntax @@ -185,18 +168,21 @@ mod tests { /// The table is one, and tests run in parallel. static LOCK: Mutex<()> = Mutex::new(()); - /// Make `items` the DATA, and READ from the first. + /// Make `items` the DATA, as a program's start does, and READ from the + /// first. The table lives on, as a program's does. fn data(items: &[(bool, &'static [u8])]) { - let t = unsafe { program::TABLE.get() }; - for (slot, &(quoted, text)) in t.iter_mut().zip(items) { - assert_eq!(text.last(), Some(&0)); - *slot = Item { - quoted: quoted as u64, - text: text.as_ptr().cast(), - }; - } - program::COUNT.set(items.len() as u64); - unsafe { _rt_restore(0) }; + let table: Vec = items + .iter() + .map(|&(quoted, text)| { + assert_eq!(text.last(), Some(&0)); + Item { + quoted: quoted as u64, + text: text.as_ptr().cast(), + } + }) + .collect(); + let table = table.leak(); + unsafe { _rt_data_init(table.as_ptr(), table.len() as i64) }; } fn read_s() -> Vec { @@ -229,7 +215,7 @@ mod tests { // A string READ returns a view of the table's text. _rt_restore(1); let r = _rt_read_string(10); - assert_eq!(r.ptr, program::TABLE.get()[1].text.cast()); + assert_eq!(r.ptr, (*program::item(1)).text.cast()); } } diff --git a/rt/src/lib.rs b/rt/src/lib.rs index 2ef177b..bdcf830 100644 --- a/rt/src/lib.rs +++ b/rt/src/lib.rs @@ -155,7 +155,9 @@ pub fn alloc_str(len: usize) -> *mut u8 { p } -#[cfg(not(test))] +/// A program's: in the compiler's own process (`host`, for `xbasic64 run`) +/// std's handler is the one, and the unit tests have std's too. +#[cfg(not(any(test, feature = "host")))] #[panic_handler] fn panic(_: &core::panic::PanicInfo) -> ! { unsafe { sys::abort() } diff --git a/src/codegen/mod.rs b/src/codegen/mod.rs index 822490c..447d9bf 100644 --- a/src/codegen/mod.rs +++ b/src/codegen/mod.rs @@ -688,52 +688,38 @@ impl CodeGen { id } + /// The DATA table, declared by `main`, which passes its address to the + /// runtime, and defined with its contents by `define_data`. + fn data_table(&mut self) -> DataId { + self.module + .declare_data("_data_table", Linkage::Local, false, false) + .expect("the DATA table") + } + /// Define every zero-filled data object, and the DATA table. fn define_data(&mut self) -> Result<(), String> { // The DATA table: (quoted, text) pairs, which the runtime's READ - // helpers walk. Always defined, even empty, since they name it. + // helpers walk. Only a program with DATA has one; `main` hands it to + // the runtime (`_rt_data_init`), which names nothing of a program. let items = std::mem::take(&mut self.data_items); - let texts: Vec = items.iter().map(|i| self.string_literal(&i.text)).collect(); - let table = self - .module - .declare_data("_data_table", Linkage::Export, true, false) - .map_err(|e| e.to_string())?; - let mut desc = DataDescription::new(); - let mut bytes = vec![0u8; 16 * items.len().max(1)]; - for (i, item) in items.iter().enumerate() { - bytes[16 * i..16 * i + 8].copy_from_slice(&u64::from(item.quoted).to_le_bytes()); - } - desc.define(bytes.into_boxed_slice()); - desc.set_align(8); - for (i, id) in texts.iter().enumerate() { - let gv = self.module.declare_data_in_data(*id, &mut desc); - desc.write_data_addr(16 * i as u32 + 8, gv, 0); + if !items.is_empty() { + let texts: Vec = items.iter().map(|i| self.string_literal(&i.text)).collect(); + let table = self.data_table(); + let mut desc = DataDescription::new(); + let mut bytes = vec![0u8; 16 * items.len()]; + for (i, item) in items.iter().enumerate() { + bytes[16 * i..16 * i + 8].copy_from_slice(&u64::from(item.quoted).to_le_bytes()); + } + desc.define(bytes.into_boxed_slice()); + desc.set_align(8); + for (i, id) in texts.iter().enumerate() { + let gv = self.module.declare_data_in_data(*id, &mut desc); + desc.write_data_addr(16 * i as u32 + 8, gv, 0); + } + self.module + .define_data(table, &desc) + .map_err(|e| e.to_string())?; } - self.module - .define_data(table, &desc) - .map_err(|e| e.to_string())?; - - let count = self - .module - .declare_data("_data_count", Linkage::Export, true, false) - .map_err(|e| e.to_string())?; - let mut desc = DataDescription::new(); - desc.define(Box::new((items.len() as u64).to_le_bytes())); - desc.set_align(8); - self.module - .define_data(count, &desc) - .map_err(|e| e.to_string())?; - - let ptr = self - .module - .declare_data("_data_ptr", Linkage::Export, true, false) - .map_err(|e| e.to_string())?; - let mut desc = DataDescription::new(); - desc.define_zeroinit(8); - desc.set_align(8); - self.module - .define_data(ptr, &desc) - .map_err(|e| e.to_string())?; for (id, size) in self.data.values() { if *size == 0 { diff --git a/src/codegen/stmt.rs b/src/codegen/stmt.rs index d55f9ab..6cd63d9 100644 --- a/src/codegen/stmt.rs +++ b/src/codegen/stmt.rs @@ -22,6 +22,14 @@ impl FnGen<'_> { let sp = self.global_addr("_gosub_sp", 8); self.b.ins().store(MemFlagsData::trusted(), top, sp, 0); } + // The program's DATA, which the runtime's READ walks. + let items = self.cg.data_items.len(); + if items > 0 { + let id = self.cg.data_table(); + let table = self.data_addr(id); + let count = self.iconst(types::I64, items as i64); + self.call_rt("_rt_data_init", &[table, count]); + } // The console is file handle 0, and each platform's runtime seeds // that slot its own way. self.call_rt("_rt_platform_init", &[]); diff --git a/src/rtsig.rs b/src/rtsig.rs index 943fad2..a944c90 100644 --- a/src/rtsig.rs +++ b/src/rtsig.rs @@ -163,6 +163,7 @@ pub static HELPERS: &[RtHelper] = &[ fallible(h("_rt_read_number", &[I64], &[F64])), fallible(h("_rt_read_string", &[I64], STR)), h("_rt_restore", &[I64], &[]), + h("_rt_data_init", &[I64, I64], &[]), fallible(h("_rt_array_autodim", &[I64, I64, I64, I64], &[])), // Strings fallible(h("_rt_val", STR, &[F64])), diff --git a/src/runtime.rs b/src/runtime.rs index 51cc188..65153e5 100644 --- a/src/runtime.rs +++ b/src/runtime.rs @@ -17,11 +17,12 @@ mod tests { use super::*; use std::process::Command; - /// The runtime needs nothing but the C library and the program's DATA - /// table. In particular no part of `core` is left undefined -- a panic - /// path the build did not merge in -- since a program could not link. + /// The runtime needs nothing but the C library: no part of `core` -- a + /// panic path the build did not merge in -- since a program could not + /// link, and nothing of a program, so that it can run one compiled into + /// the compiler's own process. #[test] - fn test_runtime_needs_only_libc_and_the_program() { + fn test_runtime_needs_only_the_c_library() { let dir = tempfile::tempdir().unwrap(); let obj = dir.path().join("rt.o"); std::fs::write(&obj, OBJECT).unwrap(); @@ -36,6 +37,7 @@ mod tests { .filter(|s| s.starts_with("_R") || s.starts_with("_ZN") || s.contains("core")) .collect(); assert!(foreign.is_empty(), "rt.o needs Rust symbols: {foreign:?}"); - assert!(text.contains("_data_table"), "{text}"); + // Nothing of a program: DATA is handed over by `_rt_data_init`. + assert!(!text.contains("_data_"), "{text}"); } } From 0296a8da8bbee301cb9b708a153ddae1e276ee63 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Sat, 10 Oct 2026 21:17:51 +0000 Subject: [PATCH 2/7] codegen: lower into any Cranelift Module, not just an ObjectModule A pure refactor, toward the JIT. CodeGen keeps the module as its last field, so CodeGen -- or a JITModule's -- is used as CodeGen, and the rest of codegen is unchanged and written once. lower() builds one around any module and lowers the program into it; generate() is the executable's: an ObjectModule, then its bytes. A colocate flag keeps the two overrides that make every reference PC-relative. An executable sets it; the JIT will not, since the runtime there is the compiler's own, possibly beyond reach of a 32-bit offset. The -S CLIF of all 651 examples and corpus programs is byte-identical to d0e77d4's. Co-Authored-By: Claude Opus 5.5 --- src/codegen/mod.rs | 198 ++++++++++++++++++++++++++++++--------------- 1 file changed, 131 insertions(+), 67 deletions(-) diff --git a/src/codegen/mod.rs b/src/codegen/mod.rs index 447d9bf..eb7ff94 100644 --- a/src/codegen/mod.rs +++ b/src/codegen/mod.rs @@ -6,11 +6,13 @@ //! expression is lowered straight into Cranelift IR with a //! `FunctionBuilder`, and Cranelift does instruction selection, register //! allocation and the object file. The runtime library is a separate object, -//! prebuilt (see `rtbuild.rs`), reached through calls whose signatures come -//! from `rtsig`. +//! prebuilt from `rt/` by `build.rs`, reached through calls whose signatures +//! come from `rtsig`. //! -//! One `CodeGen` holds what is module-wide: the `ObjectModule`, the -//! declared functions and data, and module-level storage. Each function -- +//! One `CodeGen` holds what is module-wide: the Cranelift module -- an +//! `ObjectModule` for an executable (`generate`), or any other `Module`, such +//! as the JIT's (`lower`) -- the declared functions and data, and +//! module-level storage. Each function -- //! `main`, every SUB and FUNCTION, and the error handler -- is lowered by an //! `FnGen`, which owns that function's builder and its own names. //! @@ -246,9 +248,20 @@ impl ArrayInfo { } } -/// Module-wide state: the object being built and what every function shares. -pub struct CodeGen { - module: ObjectModule, +/// Module-wide state: the module being built and what every function shares. +/// +/// The module is any Cranelift `Module`: an `ObjectModule` for an executable, +/// a `JITModule` for `xbasic64 run`. `CodeGen` is built around the +/// concrete one, and lowering works through `CodeGen` -- `CodeGen` -- so the rest of codegen is written once, for both. +pub struct CodeGen { + /// Whether every reference may be PC-relative. True for an executable, + /// where everything -- the runtime and the C library included -- is + /// linked within reach. Not in the JIT, where the runtime is the + /// compiler's own and may be anywhere in its address space: a reference + /// to it is an absolute address, as Cranelift makes an imported one by + /// default. + colocate: bool, /// The same target, with the cheap settings for huge functions. huge_isa: OwnedTargetIsa, call_conv: CallConv, @@ -296,6 +309,9 @@ pub struct CodeGen { /// the address of its limit expression: shared by main and the handler, /// which is main compiled a second time and may GOTO into the loop. for_slots: HashMap, + /// Last, so that `CodeGen` and the rest can be used as + /// `CodeGen`. + module: M, } /// The convention runtime helpers are called with, where it is not the @@ -317,15 +333,49 @@ pub fn generate( opts: Options, want_clif: bool, ) -> Result { - let isa = make_isa(false)?; - let huge_isa = make_isa(true)?; - let call_conv = isa.default_call_conv(); - let builder = ObjectBuilder::new(isa, "basic", cranelift_module::default_libcall_names()) - .map_err(|e| e.to_string())?; - let module = ObjectModule::new(builder); + let builder = ObjectBuilder::new( + make_isa(false)?, + "basic", + cranelift_module::default_libcall_names(), + ) + .map_err(|e| e.to_string())?; + let lowered = lower( + program, + symbols, + opts, + ObjectModule::new(builder), + want_clif, + true, + )?; + let object = lowered.module.finish().emit().map_err(|e| e.to_string())?; + Ok(Output { + object, + clif: lowered.clif, + }) +} +/// A program lowered into a module: every function and data object +/// defined, `main` among them. +pub struct Lowered { + pub module: M, + /// The CLIF, when asked for, with a header naming each `uN:M`. + pub clif: String, +} + +/// Lower `program` into `module`, whose ISA is `make_isa(false)`'s. With +/// `colocate`, every reference is PC-relative (see `CodeGen::colocate`). +pub fn lower( + program: &Program, + symbols: Symbols, + opts: Options, + module: M, + want_clif: bool, + colocate: bool, +) -> Result, String> { + let huge_isa = make_isa(true)?; + let call_conv = module.isa().default_call_conv(); let mut cg = CodeGen { - module, + colocate, huge_isa, call_conv, symbols, @@ -350,60 +400,69 @@ pub fn generate( stmt_count: 0, module_in_memory: HashSet::new(), for_slots: HashMap::new(), + module, }; + let clif = (&mut cg as &mut CodeGen).lower(program)?; + Ok(Lowered { + module: cg.module, + clif, + }) +} - for stmt in &program.statements { - cg.preprocess(stmt, true); - } - cg.module_in_memory = ssa::module_in_memory(program, &cg.symbols); - if cg.traps { - let names = ssa::handler_names(program, &cg.err_handlers); - cg.module_in_memory.extend(names); - } - cg.declare_procedures(program)?; - - // Main first, so every module-level name is known before a procedure - // refers to it. - cg.compile_function(FnKind::Main, None, program)?; - if cg.traps { - cg.compile_function(FnKind::Handler, None, program)?; - } - for stmt in &program.statements { - if matches!(stmt.kind, StmtKind::Sub { .. } | StmtKind::Function { .. }) { - cg.compile_function(FnKind::Proc, Some(stmt), program)?; +impl CodeGen { + /// Lower every function and data object of `program`; the CLIF, if + /// asked for. + fn lower(&mut self, program: &Program) -> Result { + for stmt in &program.statements { + self.preprocess(stmt, true); } - } + self.module_in_memory = ssa::module_in_memory(program, &self.symbols); + if self.traps { + let names = ssa::handler_names(program, &self.err_handlers); + self.module_in_memory.extend(names); + } + self.declare_procedures(program)?; - cg.define_data()?; - - // CLIF names a function `u0:N` and a data object `u1:N`; the header says - // which symbol each is. - let clif = match cg.clif.take() { - Some(body) => { - let decls = cg.module.declarations(); - let mut header = String::from("; symbols\n"); - for (id, f) in decls.get_functions() { - header.push_str(&format!( - "; u0:{} = {}\n", - id.as_u32(), - f.name.as_deref().unwrap_or("?") - )); - } - for (id, d) in decls.get_data_objects() { - header.push_str(&format!( - "; u1:{} = {}\n", - id.as_u32(), - d.name.as_deref().unwrap_or("?") - )); + // Main first, so every module-level name is known before a procedure + // refers to it. + self.compile_function(FnKind::Main, None, program)?; + if self.traps { + self.compile_function(FnKind::Handler, None, program)?; + } + for stmt in &program.statements { + if matches!(stmt.kind, StmtKind::Sub { .. } | StmtKind::Function { .. }) { + self.compile_function(FnKind::Proc, Some(stmt), program)?; } - header.push('\n'); - header + &body } - None => String::new(), - }; - let product = cg.module.finish(); - let object = product.emit().map_err(|e| e.to_string())?; - Ok(Output { object, clif }) + + self.define_data()?; + + // CLIF names a function `u0:N` and a data object `u1:N`; the header + // says which symbol each is. + Ok(match self.clif.take() { + Some(body) => { + let decls = self.module.declarations(); + let mut header = String::from("; symbols\n"); + for (id, f) in decls.get_functions() { + header.push_str(&format!( + "; u0:{} = {}\n", + id.as_u32(), + f.name.as_deref().unwrap_or("?") + )); + } + for (id, d) in decls.get_data_objects() { + header.push_str(&format!( + "; u1:{} = {}\n", + id.as_u32(), + d.name.as_deref().unwrap_or("?") + )); + } + header.push('\n'); + header + &body + } + None => String::new(), + }) + } } /// Blocks past which a function is compiled cheaply: with the single-pass @@ -1054,10 +1113,11 @@ impl<'a> FnGen<'a> { Some(gv) => *gv, None => { let gv = self.cg.module.declare_data_in_func(id, self.b.func); - // Everything is linked into one executable, so a direct, - // PC-relative reference reaches it. - if let GlobalValueData::Symbol { colocated, .. } = - &mut self.b.func.global_values[gv] + // In an executable everything is linked within reach, so a + // direct, PC-relative reference reaches it. + if self.cg.colocate + && let GlobalValueData::Symbol { colocated, .. } = + &mut self.b.func.global_values[gv] { *colocated = true; } @@ -1225,7 +1285,11 @@ impl<'a> FnGen<'a> { return *r; } let r = self.cg.module.declare_func_in_func(id, self.b.func); - self.b.func.dfg.ext_funcs[r].colocated = true; + // A direct call in an executable; in the JIT, a call to the runtime + // goes through its absolute address (see `CodeGen::colocate`). + if self.cg.colocate { + self.b.func.dfg.ext_funcs[r].colocated = true; + } self.func_refs.insert(id, r); r } From b384e2a517baf638f6e3ae650882e16db81ed1c5 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Sat, 10 Oct 2026 21:23:58 +0000 Subject: [PATCH 3/7] xbasic64 run: compile a program into memory and run it (JIT) The half step to a JIT, on the shared codegen: the same lowering as an executable's, into a cranelift-jit JITModule, and the same runtime -- rt, now also linked into the compiler. The whole program is compiled before it starts; nothing is interpreted. src/jit.rs. - xbasic64 run prog.bas [--unsafe] is a subcommand; plain xbasic64 prog.bas still compiles an executable. Compile errors are reported as before; otherwise the program's exit is the process's, and the compiler prints nothing of its own. - The runtime and the C library are the compiler's, anywhere in its address space, so generated code reaches them by absolute address (colocate off) and the JIT is given every one's address: each rtsig helper, the trapping words and messages, and the C functions codegen calls (codegen::LIBC_FUNCTIONS). A unit test holds the table to rtsig. - The program's own code and data stay PC-relative, as in an executable: they come from one arena (ArenaMemoryProvider), 64 MB to start, doubled and the program lowered again if it runs out, up to 1 GB -- within the 2 GB a 32-bit offset reaches. (A fixed large arena would commit that much on Windows, and fail under a ulimit.) - When main returns, the C library's streams are flushed as an executable's C runtime would, and the process exits with main's status. - Symbols is Clone, for the arena's retry. tests/jit: output and nothing else, END in a SUB, STOP, a runtime error's report and exit 1, a compile error, INPUT from stdin, DATA/READ/RESTORE, ON ERROR/RESUME NEXT, GOSUB, --unsafe, files in the working directory, a huge function (define_function_bytes), and the megatest byte-identical to the executable's. Executables are unchanged: the -S CLIF of all 651 examples and corpus programs is byte-identical to before. Co-Authored-By: Claude Opus 5.5 --- Cargo.lock | 166 +++++++++++++++++++++- Cargo.toml | 4 + src/codegen/mod.rs | 12 +- src/jit.rs | 321 +++++++++++++++++++++++++++++++++++++++++++ src/main.rs | 45 +++++- src/sema.rs | 2 +- tests/common/mod.rs | 45 ++++++ tests/integration.rs | 1 + tests/jit/mod.rs | 124 +++++++++++++++++ 9 files changed, 706 insertions(+), 14 deletions(-) create mode 100644 src/jit.rs create mode 100644 tests/jit/mod.rs diff --git a/Cargo.lock b/Cargo.lock index b848322..458875e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -44,7 +44,7 @@ version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -55,7 +55,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", "once_cell_polyfill", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -70,6 +70,12 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.13.1" @@ -257,6 +263,27 @@ version = "0.136.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a2f30ea3fd5a11f7d3473976d3b62b5b3f2b62fa943c85e0331d88bf505082c" +[[package]] +name = "cranelift-jit" +version = "0.136.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3b295275f5b395c764bc8e19f10afe1845cadd8fbcdd933ed40c298427c95ec2" +dependencies = [ + "anyhow", + "cranelift-codegen", + "cranelift-control", + "cranelift-entity", + "cranelift-module", + "cranelift-native", + "libc", + "log", + "memmap2", + "region", + "target-lexicon", + "wasmtime-internal-jit-icache-coherence", + "windows-sys 0.61.2", +] + [[package]] name = "cranelift-module" version = "0.136.2" @@ -269,6 +296,17 @@ dependencies = [ "hashbrown 0.17.1", ] +[[package]] +name = "cranelift-native" +version = "0.136.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "397508a8d817dc74019603ac5a4c4e40b4df627e5aed4f2f8088345d4c01d401" +dependencies = [ + "cranelift-codegen", + "libc", + "target-lexicon", +] + [[package]] name = "cranelift-object" version = "0.136.2" @@ -313,7 +351,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -418,12 +456,30 @@ version = "0.4.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f9f8bd3e56ce4dfc153cf470fffbfa98c7620958b312ca5c3a4b8d5181fd13c6" +[[package]] +name = "mach2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +dependencies = [ + "libc", +] + [[package]] name = "memchr" version = "2.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" +[[package]] +name = "memmap2" +version = "0.9.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d1219ed1b7f229ee7104d281dd01d6802fe28bb6e95d292942c4daacdeb798c0" +dependencies = [ + "libc", +] + [[package]] name = "object" version = "0.40.0" @@ -486,6 +542,18 @@ dependencies = [ "smallvec", ] +[[package]] +name = "region" +version = "3.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6b6ebd13bc009aef9cd476c1310d49ac354d36e240cf1bd753290f3dc7199a7" +dependencies = [ + "bitflags 1.3.2", + "libc", + "mach2", + "windows-sys 0.52.0", +] + [[package]] name = "rustc-hash" version = "2.1.3" @@ -498,11 +566,11 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.13.1", "errno", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -579,7 +647,7 @@ dependencies = [ "getrandom", "once_cell", "rustix", - "windows-sys", + "windows-sys 0.61.2", ] [[package]] @@ -604,12 +672,32 @@ dependencies = [ "libm", ] +[[package]] +name = "wasmtime-internal-jit-icache-coherence" +version = "49.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5dead9a2d7b5d835698a2e1a32006e7d0b5711f4697b24bab1564bb2488e3d7a" +dependencies = [ + "libc", + "wasmtime-internal-core", + "windows-sys 0.61.2", +] + [[package]] name = "windows-link" version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-sys" +version = "0.52.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d" +dependencies = [ + "windows-targets", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -619,6 +707,70 @@ dependencies = [ "windows-link", ] +[[package]] +name = "windows-targets" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973" +dependencies = [ + "windows_aarch64_gnullvm", + "windows_aarch64_msvc", + "windows_i686_gnu", + "windows_i686_gnullvm", + "windows_i686_msvc", + "windows_x86_64_gnu", + "windows_x86_64_gnullvm", + "windows_x86_64_msvc", +] + +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" + +[[package]] +name = "windows_aarch64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" + +[[package]] +name = "windows_i686_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" + +[[package]] +name = "windows_i686_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" + +[[package]] +name = "windows_i686_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" + +[[package]] +name = "windows_x86_64_gnu" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" + +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" + +[[package]] +name = "windows_x86_64_msvc" +version = "0.52.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" + [[package]] name = "xbasic64" version = "1.0.2" @@ -626,10 +778,12 @@ dependencies = [ "clap", "cranelift-codegen", "cranelift-frontend", + "cranelift-jit", "cranelift-module", "cranelift-object", "target-lexicon", "tempfile", + "xbasic64-rt", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fdc51cc..a49aa62 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -27,9 +27,13 @@ clap = { version = "4", features = ["derive"] } # Pinned exactly: Cranelift's API changes between minor releases. cranelift-codegen = { version = "=0.136.2", default-features = false, features = ["std", "x86", "arm64", "unwind"] } cranelift-frontend = "=0.136.2" +cranelift-jit = "=0.136.2" cranelift-module = "=0.136.2" cranelift-object = "=0.136.2" target-lexicon = "0.13" +# The runtime, linked into the compiler too, for `xbasic64 run`; programs +# compiled to executables link the copy build.rs builds. +xbasic64-rt = { path = "rt", features = ["host"] } [dev-dependencies] tempfile = "3" diff --git a/src/codegen/mod.rs b/src/codegen/mod.rs index eb7ff94..403b681 100644 --- a/src/codegen/mod.rs +++ b/src/codegen/mod.rs @@ -358,6 +358,7 @@ pub fn generate( /// defined, `main` among them. pub struct Lowered { pub module: M, + pub main: FuncId, /// The CLIF, when asked for, with a header naming each `uN:M`. pub clif: String, } @@ -403,8 +404,10 @@ pub fn lower( module, }; let clif = (&mut cg as &mut CodeGen).lower(program)?; + let main = cg.funcs["main"]; Ok(Lowered { module: cg.module, + main, clif, }) } @@ -479,7 +482,7 @@ const HUGE_FUNCTION_BLOCKS: usize = 20_000; /// The target: this host, as the compiler is built for the machine it runs on. /// `huge` selects the cheap settings for functions past /// `HUGE_FUNCTION_BLOCKS`. -fn make_isa(huge: bool) -> Result { +pub(crate) fn make_isa(huge: bool) -> Result { let mut flags = settings::builder(); let set = |f: &mut settings::Builder, k: &str, v: &str| { f.set(k, v).map_err(|e| format!("{}={}: {}", k, v, e)) @@ -536,7 +539,14 @@ fn rt_type(t: rtsig::Ty) -> Type { } /// The C library functions generated code calls, with their signatures. +/// The C library functions generated code calls, all of them. +pub(crate) const LIBC_FUNCTIONS: [&str; 12] = [ + "pow", "sin", "cos", "tan", "atan", "exp", "log", "calloc", "realloc", "free", "memset", + "memcpy", +]; + fn libc_sig(name: &str) -> (&'static [Type], &'static [Type]) { + debug_assert!(LIBC_FUNCTIONS.contains(&name), "{name} is not listed"); const F: &[Type] = &[types::F64]; match name { "pow" => (&[types::F64, types::F64], F), diff --git a/src/jit.rs b/src/jit.rs new file mode 100644 index 0000000..840750c --- /dev/null +++ b/src/jit.rs @@ -0,0 +1,321 @@ +//! `xbasic64 run`: compile a program into memory and run it, here +//! +//! The same lowering as an executable's (`codegen::lower`), into a +//! `JITModule` instead of an `ObjectModule`, and the same runtime -- the +//! `rt` crate, linked into the compiler as well as built for programs. The +//! whole program is compiled before it starts; nothing is interpreted. +//! +//! Two things differ from an executable, both in where things are: +//! +//! - The runtime and the C library are this process's, anywhere in its +//! address space, so generated code reaches them by absolute address +//! (`colocate` off), and the JIT is told each one's address here: +//! `symbols` lists every helper, message and trapping word, and every C +//! function generated code calls. An executable does not export its +//! functions, so nothing is found by name otherwise. +//! - The program's own code and data are PC-relative, as in an executable, +//! which needs them within 2 GB of each other: they are allocated from one +//! arena, reserved up front. It starts small and is doubled, and the +//! program lowered again, when it runs out. +//! +//! The program ends the process, as an executable does: END, STOP and a +//! fatal error exit from the runtime, and the end of main exits with what +//! main returns. + +// Copyright (c) 2025-2026 Jeff Garzik +// SPDX-License-Identifier: MIT + +use crate::codegen::{self, Options}; +use crate::parser::Program; +use crate::sema::Symbols; +use cranelift_jit::{ArenaMemoryProvider, JITBuilder, JITModule}; +use std::ffi::c_void; +use std::io::Write; +use xbasic64_rt as rt; + +/// The arena's first size, and the largest it may grow to: below 2 GB, the +/// reach of a PC-relative reference. +const ARENA_START: usize = 64 << 20; +const ARENA_MAX: usize = 1 << 30; + +/// Compile `program` and run it; returns only if it cannot be compiled. +pub fn run(program: &Program, symbols: &Symbols, opts: Options) -> String { + let mut arena = ARENA_START; + let lowered = loop { + let module = match new_module(arena) { + Ok(m) => m, + Err(e) => return e, + }; + match codegen::lower(program, symbols.clone(), opts, module, false, false) { + Ok(l) => break l, + // Lowering defines each function as it goes, so a full arena + // shows as an error part-way; lower again into a larger one. + Err(e) if e.contains("exhausted") && arena < ARENA_MAX => arena *= 2, + Err(e) => return e, + } + }; + let mut module = lowered.module; + if let Err(e) = module.finalize_definitions() { + return e.to_string(); + } + let main = module.get_finalized_function(lowered.main); + // `main` has the platform's C convention: `codegen::lower` declares it + // with the target's default, as an executable's C runtime calls it. + let main: extern "C" fn() -> i32 = unsafe { std::mem::transmute(main) }; + // The program writes through the C library's stdout; nothing of the + // compiler's own may be left waiting in front of it. + let _ = std::io::stdout().flush(); + let status = main(); + // What an executable's C runtime does on return from main. Windows' + // ExitProcess, under std's exit, would not flush the C library. + rt::sys::flush_all(); + drop(module); // after main: its code must outlive the call + std::process::exit(status) +} + +/// A JIT module for this host, the program's memory from an arena of +/// `arena` bytes, every outside symbol it may need known. +fn new_module(arena: usize) -> Result { + let isa = codegen::make_isa(false)?; + let mut builder = JITBuilder::with_isa(isa, cranelift_module::default_libcall_names()); + builder.symbols(symbols()); + let memory = ArenaMemoryProvider::new_with_size(arena) + .map_err(|e| format!("cannot reserve {} MB for the JIT: {e}", arena >> 20))?; + builder.memory_provider(Box::new(memory)); + Ok(JITModule::new(builder)) +} + +// The C library functions generated code calls (`codegen::LIBC_FUNCTIONS`), +// for their addresses. Declared with the C library's own types, which is +// all an address needs. +unsafe extern "C" { + fn pow(x: f64, y: f64) -> f64; + fn sin(x: f64) -> f64; + fn cos(x: f64) -> f64; + fn tan(x: f64) -> f64; + fn atan(x: f64) -> f64; + fn exp(x: f64) -> f64; + fn log(x: f64) -> f64; + fn calloc(n: usize, size: usize) -> *mut c_void; + fn realloc(p: *mut c_void, size: usize) -> *mut c_void; + fn free(p: *mut c_void); + fn memset(p: *mut c_void, c: i32, n: usize) -> *mut c_void; + fn memcpy(d: *mut c_void, s: *const c_void, n: usize) -> *mut c_void; +} + +/// Every symbol generated code may name that is not its own: the runtime's +/// helpers (`rtsig`), its messages and trapping words, and the C library +/// functions codegen calls. A test holds this to `rtsig` and codegen. +fn symbols() -> Vec<(&'static str, *const u8)> { + let word = |w: &'static rt::Word| (w as *const rt::Word).cast::(); + vec![ + ( + "_rt_array_autodim", + rt::data::_rt_array_autodim as *const u8, + ), + ("_rt_beep", rt::math::_rt_beep as *const u8), + ("_rt_chr", rt::string::_rt_chr as *const u8), + ("_rt_cls", rt::math::_rt_cls as *const u8), + ("_rt_color", rt::math::_rt_color as *const u8), + ("_rt_cvd", rt::file::_rt_cvd as *const u8), + ("_rt_cvi", rt::file::_rt_cvi as *const u8), + ("_rt_cvl", rt::file::_rt_cvl as *const u8), + ("_rt_cvs", rt::file::_rt_cvs as *const u8), + ("_rt_data_init", rt::data::_rt_data_init as *const u8), + ("_rt_date", rt::math::_rt_date as *const u8), + ("_rt_end", rt::error::_rt_end as *const u8), + ("_rt_erl", rt::error::_rt_erl as *const u8), + ("_rt_err", rt::error::_rt_err as *const u8), + ("_rt_error", rt::error::_rt_error as *const u8), + ("_rt_error_num", rt::error::_rt_error_num as *const u8), + ("_rt_fatal", rt::error::_rt_fatal as *const u8), + ("_rt_field_next", rt::file::_rt_field_next as *const u8), + ("_rt_field_reset", rt::file::_rt_field_reset as *const u8), + ("_rt_file_close", rt::file::_rt_file_close as *const u8), + ( + "_rt_file_close_all", + rt::file::_rt_file_close_all as *const u8, + ), + ("_rt_file_eof", rt::file::_rt_file_eof as *const u8), + ("_rt_file_get", rt::file::_rt_file_get as *const u8), + ( + "_rt_file_input_number", + rt::file::_rt_file_input_number as *const u8, + ), + ( + "_rt_file_input_string", + rt::file::_rt_file_input_string as *const u8, + ), + ( + "_rt_file_line_input", + rt::file::_rt_file_line_input as *const u8, + ), + ("_rt_file_loc", rt::file::_rt_file_loc as *const u8), + ("_rt_file_lof", rt::file::_rt_file_lof as *const u8), + ("_rt_file_open", rt::file::_rt_file_open as *const u8), + ( + "_rt_file_open_random", + rt::file::_rt_file_open_random as *const u8, + ), + ( + "_rt_file_print_char", + rt::file::_rt_file_print_char as *const u8, + ), + ( + "_rt_file_print_float", + rt::file::_rt_file_print_float as *const u8, + ), + ( + "_rt_file_print_newline", + rt::file::_rt_file_print_newline as *const u8, + ), + ( + "_rt_file_print_single", + rt::file::_rt_file_print_single as *const u8, + ), + ( + "_rt_file_print_spc", + rt::file::_rt_file_print_spc as *const u8, + ), + ( + "_rt_file_print_string", + rt::file::_rt_file_print_string as *const u8, + ), + ( + "_rt_file_print_tab", + rt::file::_rt_file_print_tab as *const u8, + ), + ("_rt_file_put", rt::file::_rt_file_put as *const u8), + ("_rt_fixed", rt::string::_rt_fixed as *const u8), + ("_rt_hex", rt::string::_rt_hex as *const u8), + ( + "_rt_input_field_num", + rt::input::_rt_input_field_num as *const u8, + ), + ( + "_rt_input_field_str", + rt::input::_rt_input_field_str as *const u8, + ), + ("_rt_input_line", rt::input::_rt_input_line as *const u8), + ("_rt_input_string", rt::input::_rt_input_string as *const u8), + ("_rt_instr", rt::string::_rt_instr as *const u8), + ("_rt_lcase", rt::string::_rt_lcase as *const u8), + ("_rt_left", rt::string::_rt_left as *const u8), + ("_rt_locate", rt::math::_rt_locate as *const u8), + ("_rt_lock", rt::file::_rt_lock as *const u8), + ("_rt_lset", rt::file::_rt_lset as *const u8), + ("_rt_ltrim", rt::string::_rt_ltrim as *const u8), + ("_rt_mid", rt::string::_rt_mid as *const u8), + ("_rt_mid_assign", rt::string::_rt_mid_assign as *const u8), + ("_rt_mk", rt::file::_rt_mk as *const u8), + ("_rt_oct", rt::string::_rt_oct as *const u8), + ( + "_rt_platform_init", + rt::print::_rt_platform_init as *const u8, + ), + ("_rt_pos", rt::math::_rt_pos as *const u8), + ("_rt_print_number", rt::file::_rt_print_number as *const u8), + ( + "_rt_print_number_single", + rt::file::_rt_print_number_single as *const u8, + ), + ("_rt_print_zone", rt::file::_rt_print_zone as *const u8), + ("_rt_randomize", rt::math::_rt_randomize as *const u8), + ("_rt_read_number", rt::data::_rt_read_number as *const u8), + ("_rt_read_string", rt::data::_rt_read_string as *const u8), + ("_rt_restore", rt::data::_rt_restore as *const u8), + ("_rt_right", rt::string::_rt_right as *const u8), + ("_rt_rnd", rt::math::_rt_rnd as *const u8), + ("_rt_rset", rt::file::_rt_rset as *const u8), + ("_rt_rtrim", rt::string::_rt_rtrim as *const u8), + ("_rt_space", rt::string::_rt_space as *const u8), + ("_rt_stop", rt::error::_rt_stop as *const u8), + ("_rt_str", rt::string::_rt_str as *const u8), + ("_rt_strcat", rt::string::_rt_strcat as *const u8), + ("_rt_strcmp", rt::string::_rt_strcmp as *const u8), + ("_rt_strdup", rt::string::_rt_strdup as *const u8), + ("_rt_string_n", rt::string::_rt_string_n as *const u8), + ("_rt_str_single", rt::string::_rt_str_single as *const u8), + ("_rt_time", rt::math::_rt_time as *const u8), + ("_rt_timer", rt::math::_rt_timer as *const u8), + ("_rt_ucase", rt::string::_rt_ucase as *const u8), + ("_rt_unlock", rt::file::_rt_unlock as *const u8), + ("_rt_using_end", rt::using::_rt_using_end as *const u8), + ("_rt_using_num", rt::using::_rt_using_num as *const u8), + ("_rt_using_start", rt::using::_rt_using_start as *const u8), + ("_rt_using_str", rt::using::_rt_using_str as *const u8), + ("_rt_val", rt::num::_rt_val as *const u8), + ("_rt_width", rt::file::_rt_width as *const u8), + // The runtime's error-trapping words and messages. + ("_err_handler", word(&rt::error::_err_handler)), + ("_err_pending", word(&rt::error::_err_pending)), + ("_err_active", word(&rt::error::_err_active)), + ("_err_line", word(&rt::error::_err_line)), + ("_err_syntax", rt::error::_err_syntax.as_ptr()), + ("_err_subscript", rt::error::_err_subscript.as_ptr()), + ("_err_typemismatch", rt::error::_err_typemismatch.as_ptr()), + ("_err_div0", rt::error::_err_div0.as_ptr()), + ("_err_domain", rt::error::_err_domain.as_ptr()), + ("_err_overflow", rt::error::_err_overflow.as_ptr()), + ("_err_dupdef", rt::error::_err_dupdef.as_ptr()), + ("_err_memory", rt::error::_err_memory.as_ptr()), + ("_err_gosub", rt::error::_err_gosub.as_ptr()), + ("_err_noreturn", rt::error::_err_noreturn.as_ptr()), + ("_err_outofdata", rt::error::_err_outofdata.as_ptr()), + ("_err_badfile", rt::error::_err_badfile.as_ptr()), + ("_err_badmode", rt::error::_err_badmode.as_ptr()), + ("_err_fieldovf", rt::error::_err_fieldovf.as_ptr()), + ("_err_permission", rt::error::_err_permission.as_ptr()), + ("_err_notfound", rt::error::_err_notfound.as_ptr()), + ("_err_alreadyopen", rt::error::_err_alreadyopen.as_ptr()), + ("_err_pastend", rt::error::_err_pastend.as_ptr()), + ("_err_badname", rt::error::_err_badname.as_ptr()), + ("_err_access", rt::error::_err_access.as_ptr()), + ("_err_pathnotfound", rt::error::_err_pathnotfound.as_ptr()), + ("_err_unprintable", rt::error::_err_unprintable.as_ptr()), + ("_err_resnoerr", rt::error::_err_resnoerr.as_ptr()), + // The C library. + ("pow", pow as *const u8), + ("sin", sin as *const u8), + ("cos", cos as *const u8), + ("tan", tan as *const u8), + ("atan", atan as *const u8), + ("exp", exp as *const u8), + ("log", log as *const u8), + ("calloc", calloc as *const u8), + ("realloc", realloc as *const u8), + ("free", free as *const u8), + ("memset", memset as *const u8), + ("memcpy", memcpy as *const u8), + ] +} + +#[cfg(test)] +mod tests { + use super::*; + use std::collections::BTreeSet; + + /// The JIT knows every outside symbol generated code may name -- each + /// runtime helper in `rtsig`, each message codegen raises, the trapping + /// words and each C function codegen calls -- once, and nothing else: + /// one it did not know would stop `run` with a panic at load time. + #[test] + fn test_the_jit_knows_every_outside_symbol() { + let table: Vec<&str> = symbols().iter().map(|(n, _)| *n).collect(); + let known: BTreeSet<&str> = table.iter().copied().collect(); + assert_eq!(known.len(), table.len(), "a symbol is listed twice"); + let mut wanted: BTreeSet<&str> = crate::rtsig::HELPERS.iter().map(|h| h.name).collect(); + wanted.extend(codegen::LIBC_FUNCTIONS); + wanted.extend(["_err_handler", "_err_pending", "_err_active", "_err_line"]); + for name in &wanted { + assert!(known.contains(name), "the JIT does not know {name}"); + } + for name in &known { + assert!( + wanted.contains(name) || name.starts_with("_err_"), + "{name} is not something generated code names" + ); + } + assert!(symbols().iter().all(|(_, p)| !p.is_null())); + } +} diff --git a/src/main.rs b/src/main.rs index 7f19cbe..236b6bb 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,19 +2,21 @@ //! //! Compiles 1980s-era BASIC programs to native executables (x86-64, aarch64). //! Code generation is Cranelift, which writes the object file; the program is -//! linked with `cc` on Linux and MSVC `link.exe` on Windows. +//! linked with `cc` on Linux and macOS and MSVC `link.exe` on Windows. Or +//! `xbasic64 run` compiles a program into memory and runs it at once (`jit`). // Copyright (c) 2025-2026 Jeff Garzik // SPDX-License-Identifier: MIT mod codegen; +mod jit; mod lexer; mod parser; mod rtsig; mod runtime; mod sema; -use clap::Parser; +use clap::{Parser, Subcommand}; use std::fs; use std::path::Path; use std::process::Command; @@ -23,9 +25,14 @@ use std::process::Command; #[derive(Parser)] #[command(name = "xbasic64")] #[command(about = "Compiles 1980s-era BASIC programs to native executables")] +#[command(args_conflicts_with_subcommands = true, subcommand_negates_reqs = true)] struct Args { + #[command(subcommand)] + mode: Option, + /// Input BASIC source file - input: String, + #[arg(required = true)] + input: Option, /// Output file name #[arg(short, long)] @@ -43,6 +50,19 @@ struct Args { no_checks: bool, } +#[derive(Subcommand)] +enum Mode { + /// Compile a program into memory and run it, without an executable + Run { + /// Input BASIC source file + input: String, + + /// Disable runtime safety checks (array bounds, division by zero, overflow) + #[arg(long = "unsafe")] + no_checks: bool, + }, +} + /// Format a diagnostic position as `file:line`, or just `file` when the line is /// unknown. GCC-style, so editors and IDEs can parse it. fn locate(file: &str, line: u32) -> String { @@ -82,9 +102,16 @@ fn looks_like_coreutils_link(stderr: &str) -> bool { } fn main() { - let args = Args::parse(); - - let input_file = &args.input; + let mut args = Args::parse(); + let run = match args.mode.take() { + Some(Mode::Run { input, no_checks }) => { + args.input = Some(input); + args.no_checks = no_checks; + true + } + None => false, + }; + let input_file = &args.input.clone().expect("clap requires an input file"); // Read source file let source = match fs::read_to_string(input_file) { @@ -141,6 +168,12 @@ fn main() { let opts = codegen::Options { checks: !args.no_checks, }; + if run { + // Runs the program, which ends the process; back only on failure. + let e = jit::run(&program, &symbols, opts); + eprintln!("{}: error: {}", input_file, e); + std::process::exit(1); + } let output = match codegen::generate(&program, symbols, opts, args.asm_only) { Ok(o) => o, Err(e) => { diff --git a/src/sema.rs b/src/sema.rs index d95383f..523627d 100644 --- a/src/sema.rs +++ b/src/sema.rs @@ -320,7 +320,7 @@ pub enum Scope { } /// Everything the analysis learned about a program. -#[derive(Debug, Default)] +#[derive(Clone, Debug, Default)] pub struct Symbols { pub procs: HashMap, pub arrays: HashMap<(Scope, String), ArrayInfo>, diff --git a/tests/common/mod.rs b/tests/common/mod.rs index d31dce8..6d0c22d 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -236,6 +236,51 @@ pub fn compile_and_run_env( }) } +/// Run `source` with `xbasic64 run` -- compiled into memory and run in the +/// compiler's own process -- from a temp dir, with `stdin_input` and extra +/// `flags`. Like [`compile_and_run_raw`], `Ok` whatever the program did; but +/// here a compile error is the same process failing, so it is in the +/// output too (exit 1, with the diagnostic on stderr). +pub fn run_jit(source: &str, stdin_input: &str, flags: &[&str]) -> Result { + let tmp = TempDir::new().map_err(|e| e.to_string())?; + run_jit_in(&tmp, source, stdin_input, flags) +} + +/// [`run_jit`] in the directory `dir`, which files the program opens are in. +pub fn run_jit_in( + dir: &TempDir, + source: &str, + stdin_input: &str, + flags: &[&str], +) -> Result { + let bas_file = dir.path().join("test.bas"); + fs::write(&bas_file, source).map_err(|e| e.to_string())?; + let mut child = Command::new(env!("CARGO_BIN_EXE_xbasic64")) + .arg("run") + .arg(&bas_file) + .args(flags) + .current_dir(dir.path()) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .map_err(|e| format!("Failed to run the compiler: {}", e))?; + child + .stdin + .as_mut() + .expect("stdin was piped") + .write_all(stdin_input.as_bytes()) + .map_err(|e| format!("Failed to write to stdin: {}", e))?; + let out = child + .wait_with_output() + .map_err(|e| format!("Failed to wait for the compiler: {}", e))?; + Ok(RunOutput { + stdout: String::from_utf8_lossy(&out.stdout).to_string(), + stderr: String::from_utf8_lossy(&out.stderr).to_string(), + exit_code: out.status.code(), + }) +} + pub fn compile_and_run(source: &str) -> Result { compile_and_run_with_stdin(source, "") } diff --git a/tests/integration.rs b/tests/integration.rs index f10612c..f5ead29 100644 --- a/tests/integration.rs +++ b/tests/integration.rs @@ -14,6 +14,7 @@ mod docs; mod errors; mod file_io; mod input; +mod jit; mod math; mod megatest; mod print; diff --git a/tests/jit/mod.rs b/tests/jit/mod.rs new file mode 100644 index 0000000..aacac86 --- /dev/null +++ b/tests/jit/mod.rs @@ -0,0 +1,124 @@ +//! `xbasic64 run`: a program compiled into memory and run in the compiler's +//! own process (src/jit.rs) +//! +//! The rest of the suite runs in this mode too, with `XB_JIT=1`; these are +//! the things particular to it -- how the process ends, what reaches stdout +//! and stderr, and that it agrees with an executable. + +// Copyright (c) 2025-2026 Jeff Garzik +// SPDX-License-Identifier: MIT + +use crate::common::{compile_and_run_raw, compile_and_run_with_files, run_jit, run_jit_in}; + +/// Output, and nothing of the compiler's: no "Compiled" line, and the last +/// line's bytes even without a newline -- flushed as the program ends. +#[test] +fn test_run_prints_only_the_program() { + let run = run_jit("PRINT \"hi\"; 1 / 3\nPRINT \"last\";\n", "", &[]).unwrap(); + run.assert_ran_to_completion("run"); + assert_eq!( + run.stdout.replace("\r\n", "\n"), + "hi .3333333333333333 \nlast" + ); + assert_eq!(run.stderr, ""); +} + +/// The program's exit is the process's: END from inside a SUB, STOP, and +/// a runtime error, which reports itself and exits 1 after what was +/// printed before it. +#[test] +fn test_run_ends_as_an_executable_does() { + let run = run_jit( + "SUB S\n PRINT \"in\"\n END\nEND SUB\nS\nPRINT \"no\"\n", + "", + &[], + ) + .unwrap(); + run.assert_ran_to_completion("END in a SUB"); + assert_eq!(run.lines(), vec!["in"]); + + let run = run_jit("10 PRINT \"a\"\n20 STOP\n30 PRINT \"no\"\n", "", &[]).unwrap(); + assert_eq!(run.exit_code, Some(0)); + assert_eq!(run.lines(), vec!["a"]); + assert!(run.stderr.contains("Break in 20"), "{:?}", run.stderr); + + let run = run_jit("PRINT \"before\"\nX = 0\nPRINT 1 / X\n", "", &[]).unwrap(); + assert_eq!(run.exit_code, Some(1)); + assert_eq!(run.lines(), vec!["before"]); + assert!( + run.stderr.contains("?Division by zero in 3"), + "{:?}", + run.stderr + ); +} + +/// A program that does not compile is reported as `xbasic64` reports it, +/// and nothing runs. +#[test] +fn test_run_of_a_bad_program_reports_it() { + let run = run_jit("PRINT \"x\"\nGOTO 999\n", "", &[]).unwrap(); + assert_eq!(run.exit_code, Some(1)); + assert_eq!(run.stdout, ""); + assert!(run.stderr.contains("error"), "{:?}", run.stderr); +} + +/// INPUT reads the process's stdin; DATA, READ and RESTORE; ON ERROR and +/// RESUME NEXT; GOSUB; `--unsafe`. +#[test] +fn test_run_of_the_runtime_parts() { + let run = run_jit("INPUT \"n\"; N, S$\nPRINT N * 2; S$\n", "21, hi\n", &[]).unwrap(); + run.assert_ran_to_completion("INPUT"); + assert!(run.stdout.contains(" 42 hi"), "{:?}", run.stdout); + + let src = "DATA 1, \"two\", 3\nREAD A, B$, C\nRESTORE\nREAD D\nPRINT A; B$; C; D\n\ + ON ERROR GOTO 90\nX = 0: Y = 1 / X: PRINT \"next\"\nGOSUB 80\nEND\n\ + 80 PRINT \"sub\": RETURN\n90 PRINT \"trap\"; ERR: RESUME NEXT\n"; + let run = run_jit(src, "", &[]).unwrap(); + run.assert_ran_to_completion("DATA, ON ERROR and GOSUB"); + assert_eq!(run.lines(), vec!["1 two 3 1", "trap 11", "next", "sub"]); + + let run = run_jit("DIM A(3)\nA(2) = 5\nPRINT A(2)\n", "", &["--unsafe"]).unwrap(); + run.assert_ran_to_completion("--unsafe"); + assert_eq!(run.lines(), vec!["5"]); +} + +/// Files a program opens by name are in its working directory. +#[test] +fn test_run_opens_files_where_it_runs() { + let dir = tempfile::TempDir::new().unwrap(); + let src = "OPEN \"t.txt\" FOR OUTPUT AS #1\nPRINT #1, 42\nCLOSE #1\n\ + OPEN \"t.txt\" FOR INPUT AS #1\nINPUT #1, N\nCLOSE #1\nPRINT N + 1\n"; + let run = run_jit_in(&dir, src, "", &[]).unwrap(); + run.assert_ran_to_completion("files"); + assert_eq!(run.lines(), vec!["43"]); + assert!(dir.path().join("t.txt").exists()); +} + +/// A function too large for the optimizing register allocator goes through +/// the cheap ISA and `define_function_bytes`, which the JIT takes too. +#[test] +fn test_run_of_a_huge_function() { + let mut src = String::new(); + for i in 0..5000 { + let (a, b) = (i % 7, (i + 1) % 7); + src.push_str(&format!( + "A{a} = A{b} + {i}: IF A{a} > 1000 THEN A{a} = A{a} - 1000\n" + )); + } + src.push_str("PRINT A0; A1; A2; A3; A4; A5; A6\n"); + let jit = run_jit(&src, "", &[]).unwrap(); + jit.assert_ran_to_completion("a huge function"); + let aot = compile_and_run_raw(&src, "").unwrap(); + assert_eq!(jit.stdout, aot.stdout); +} + +/// The megatest, run both ways, prints the same. +#[test] +fn test_run_agrees_with_an_executable_on_the_megatest() { + let mega = include_str!("../megatest/mega.bas"); + let (aot, _tmp) = compile_and_run_with_files(mega, |_| Ok(())).unwrap(); + let jit = run_jit(mega, "", &[]).unwrap(); + jit.assert_ran_to_completion("the megatest"); + assert_eq!(jit.stdout, aot); + assert!(!aot.contains("FAIL "), "{aot}"); +} From dbb5d02a37dfae19098899e77ee1144454ee7e67 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Sat, 10 Oct 2026 21:27:50 +0000 Subject: [PATCH 4/7] Run the whole integration suite through the JIT too (XB_JIT=1) With XB_JIT=1 the harness runs every program by xbasic64 run instead of as an executable: compile_and_run_env, compile_and_run_with_files and the new run_with_address_space_limit check that the program compiles (with -S, so a compile error is still an Err apart from the run) and then run xbasic64 run on it, with the same directory, stdin and environment. The two ulimit -v memory tests share that helper now. CI runs the integration suite a second time that way on all four jobs, and runs each example through xbasic64 run as well as compiled. Checked here: - x86-64 Linux: 599 of 599 tests pass both ways; the two memory tests fit the compiler, the JIT's 64 MB arena and the program in 256 MB. - The 651 examples and corpus programs give the same stdout, stderr and exit code by xbasic64 run as compiled (3 differ: RND seeded by TIMER). - Valgrind on 95 of them run through the JIT: no errors. - aarch64 under qemu-user: 599 of 599 compiled, 597 of 599 by run. The two memory tests cannot reserve the arena there because qemu's own guest reservation counts against the ulimit; CI's native ARM job is the check. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/TestingCI.yml | 36 ++++++++++ tests/common/mod.rs | 117 ++++++++++++++++++++++---------- tests/strings/mod.rs | 55 ++------------- 3 files changed, 125 insertions(+), 83 deletions(-) diff --git a/.github/workflows/TestingCI.yml b/.github/workflows/TestingCI.yml index 18ddc62..d490068 100644 --- a/.github/workflows/TestingCI.yml +++ b/.github/workflows/TestingCI.yml @@ -18,6 +18,13 @@ jobs: run: cargo build --release --verbose - name: Run tests run: cargo test --release --verbose + # The integration suite again, every program run by `xbasic64 run` + # (compiled into memory, in the compiler's process) instead of as an + # executable: the two must agree. + - name: Run tests through the JIT + run: cargo test --release --test integration + env: + XB_JIT: "1" - name: Run fmt check run: cargo fmt --all -- --check - name: Run clippy @@ -33,6 +40,7 @@ jobs: echo "== $f" ./target/release/xbasic64 "$f" -o /tmp/example /tmp/example /dev/null + ./target/release/xbasic64 run "$f" /dev/null done linux-aarch64: @@ -48,6 +56,13 @@ jobs: run: cargo build --release --verbose - name: Run tests run: cargo test --release --verbose + # The integration suite again, every program run by `xbasic64 run` + # (compiled into memory, in the compiler's process) instead of as an + # executable: the two must agree. + - name: Run tests through the JIT + run: cargo test --release --test integration + env: + XB_JIT: "1" - name: Run fmt check run: cargo fmt --all -- --check - name: Run clippy @@ -63,6 +78,7 @@ jobs: echo "== $f" ./target/release/xbasic64 "$f" -o /tmp/example /tmp/example /dev/null + ./target/release/xbasic64 run "$f" /dev/null done macos-arm64: @@ -78,6 +94,13 @@ jobs: run: cargo build --release --verbose - name: Run tests run: cargo test --release --verbose + # The integration suite again, every program run by `xbasic64 run` + # (compiled into memory, in the compiler's process) instead of as an + # executable: the two must agree. + - name: Run tests through the JIT + run: cargo test --release --test integration + env: + XB_JIT: "1" - name: Run fmt check run: cargo fmt --all -- --check - name: Run clippy @@ -93,6 +116,7 @@ jobs: echo "== $f" ./target/release/xbasic64 "$f" -o /tmp/example /tmp/example /dev/null + ./target/release/xbasic64 run "$f" /dev/null done windows-native: @@ -113,6 +137,13 @@ jobs: run: cargo build --release --verbose - name: Run tests run: cargo test --release --verbose + # The integration suite again, every program run by `xbasic64 run` + # (compiled into memory, in the compiler's process) instead of as an + # executable: the two must agree. + - name: Run tests through the JIT + run: cargo test --release --test integration + env: + XB_JIT: "1" - name: Run fmt check run: cargo fmt --all -- --check - name: Run clippy @@ -146,6 +177,11 @@ jobs: # [uint32] then threw on the very value it was there to show. throw ("run failed: {0} (exit {1}, 0x{2:X8})" -f $_.Name, $LASTEXITCODE, $LASTEXITCODE) } + # The same program through the JIT: `xbasic64 run`. + & ./target/release/xbasic64.exe run $_.FullName | Out-Null + if ($LASTEXITCODE -ne 0) { + throw ("xbasic64 run failed: {0} (exit {1}, 0x{2:X8})" -f $_.Name, $LASTEXITCODE, $LASTEXITCODE) + } } # An example is a whole program, so a crash in one names the program and diff --git a/tests/common/mod.rs b/tests/common/mod.rs index 6d0c22d..36f85ca 100644 --- a/tests/common/mod.rs +++ b/tests/common/mod.rs @@ -151,6 +151,55 @@ pub fn compile_only_flags(source: &str, flags: &[&str]) -> Result<(), CompileErr } } +/// Whether the suite runs programs with `xbasic64 run` (in the compiler's +/// own process, through the JIT) rather than as executables: `XB_JIT=1`. +/// The whole suite runs both ways in CI; the two must agree. +pub fn jit_mode() -> bool { + std::env::var("XB_JIT").is_ok_and(|v| v == "1") +} + +/// Compile `bas_file` with `flags`, `Err` with the compiler's output if it +/// does not compile: to the executable `exe_file`, or -- in JIT mode -- only +/// as far as `-S`, so that a compile error stays apart from the run. +fn compile_step( + bas_file: &std::path::Path, + exe_file: &std::path::Path, + flags: &[&str], +) -> Result<(), String> { + let compile_output = Command::new(env!("CARGO_BIN_EXE_xbasic64")) + .arg(bas_file) + .args(flags) + .args(if jit_mode() { &["-S"][..] } else { &[] }) + .arg("-o") + .arg(exe_file) + .output() + .map_err(|e| format!("Failed to run compiler: {}", e))?; + if !compile_output.status.success() { + return Err(format!( + "Compilation failed:\nstdout: {}\nstderr: {}", + String::from_utf8_lossy(&compile_output.stdout), + String::from_utf8_lossy(&compile_output.stderr) + )); + } + Ok(()) +} + +/// The command that runs the program: the executable, or in JIT mode +/// `xbasic64 run` on its source, with `flags`. +fn program_command( + bas_file: &std::path::Path, + exe_file: &std::path::Path, + flags: &[&str], +) -> Command { + if jit_mode() { + let mut c = Command::new(env!("CARGO_BIN_EXE_xbasic64")); + c.arg("run").arg(bas_file).args(flags); + c + } else { + Command::new(exe_file) + } +} + /// Compile and run, returning `Ok` **regardless of the program's exit status**. /// /// `Err` is reserved for compilation failure. This is the primitive the other @@ -185,24 +234,9 @@ pub fn compile_and_run_env( let exe_file = tmp.path().join("test"); fs::write(&bas_file, source).map_err(|e| e.to_string())?; + compile_step(&bas_file, &exe_file, flags)?; - let compile_output = Command::new(env!("CARGO_BIN_EXE_xbasic64")) - .arg(&bas_file) - .args(flags) - .arg("-o") - .arg(&exe_file) - .output() - .map_err(|e| format!("Failed to run compiler: {}", e))?; - - if !compile_output.status.success() { - return Err(format!( - "Compilation failed:\nstdout: {}\nstderr: {}", - String::from_utf8_lossy(&compile_output.stdout), - String::from_utf8_lossy(&compile_output.stderr) - )); - } - - let mut child = Command::new(&exe_file) + let mut child = program_command(&bas_file, &exe_file, flags) // Run inside the temp dir, so a program that opens a file by a bare // name writes it there and it dies with the TempDir. Without this the // program inherited the test runner's directory -- the repository root @@ -236,6 +270,36 @@ pub fn compile_and_run_env( }) } +/// Compile `source` and run it under an address-space limit of `kb` KB +/// (`ulimit -v`, which only Linux enforces), as an executable or -- in JIT +/// mode -- with the compiler, the JIT's arena and the program all within it. +#[cfg(target_os = "linux")] +pub fn run_with_address_space_limit(source: &str, kb: u32) -> RunOutput { + let tmp = TempDir::new().unwrap(); + let bas_file = tmp.path().join("test.bas"); + let exe_file = tmp.path().join("test"); + fs::write(&bas_file, source).unwrap(); + compile_step(&bas_file, &exe_file, &[]).unwrap(); + let program = program_command(&bas_file, &exe_file, &[]); + let line: Vec = std::iter::once(program.get_program()) + .chain(program.get_args()) + .map(|a| a.to_string_lossy().into_owned()) + .collect(); + let out = Command::new("sh") + .arg("-c") + .arg(format!("ulimit -v {kb} && exec \"$@\"",)) + .arg("sh") + .args(&line) + .current_dir(tmp.path()) + .output() + .unwrap(); + RunOutput { + stdout: String::from_utf8_lossy(&out.stdout).to_string(), + stderr: String::from_utf8_lossy(&out.stderr).to_string(), + exit_code: out.status.code(), + } +} + /// Run `source` with `xbasic64 run` -- compiled into memory and run in the /// compiler's own process -- from a temp dir, with `stdin_input` and extra /// `flags`. Like [`compile_and_run_raw`], `Ok` whatever the program did; but @@ -361,25 +425,10 @@ where setup(tmp.path())?; fs::write(&bas_file, source).map_err(|e| e.to_string())?; - - // Compile - let compile_output = Command::new(env!("CARGO_BIN_EXE_xbasic64")) - .arg(&bas_file) - .arg("-o") - .arg(&exe_file) - .output() - .map_err(|e| format!("Failed to run compiler: {}", e))?; - - if !compile_output.status.success() { - return Err(format!( - "Compilation failed:\nstdout: {}\nstderr: {}", - String::from_utf8_lossy(&compile_output.stdout), - String::from_utf8_lossy(&compile_output.stderr) - )); - } + compile_step(&bas_file, &exe_file, &[])?; // Run from the temp directory so relative file paths work - let run_output = Command::new(&exe_file) + let run_output = program_command(&bas_file, &exe_file, &[]) .current_dir(tmp.path()) .output() .map_err(|e| format!("Failed to run executable: {}", e))?; diff --git a/tests/strings/mod.rs b/tests/strings/mod.rs index aa5fa4a..b2075c3 100644 --- a/tests/strings/mod.rs +++ b/tests/strings/mod.rs @@ -596,32 +596,9 @@ FOR I& = 1 TO 50000 NEXT I& PRINT LEN(A$); LEN(B$); LEN(C$); LEN(S$(3)) "#; - let tmp = tempfile::TempDir::new().unwrap(); - let bas = tmp.path().join("churn.bas"); - let exe = tmp.path().join("churn"); - std::fs::write(&bas, src).unwrap(); - let c = std::process::Command::new(env!("CARGO_BIN_EXE_xbasic64")) - .arg(&bas) - .arg("-o") - .arg(&exe) - .output() - .unwrap(); - assert!(c.status.success(), "{}", String::from_utf8_lossy(&c.stderr)); - let run = std::process::Command::new("sh") - .arg("-c") - .arg(format!("ulimit -v 262144 && exec {}", exe.display())) - .output() - .unwrap(); - assert!( - run.status.success(), - "stdout {} stderr {}", - String::from_utf8_lossy(&run.stdout), - String::from_utf8_lossy(&run.stderr) - ); - assert_eq!( - String::from_utf8_lossy(&run.stdout).trim(), - "1006 10 1000 100" - ); + let run = crate::common::run_with_address_space_limit(src, 262144); + run.assert_ran_to_completion("under a 256 MB address-space limit"); + assert_eq!(run.stdout.trim(), "1006 10 1000 100"); } /// A statement left by a trapped error frees its temporaries too: the @@ -661,29 +638,9 @@ RESUME TOP DONE: PRINT LEN(F); I& "#; - let tmp = tempfile::TempDir::new().unwrap(); - let bas = tmp.path().join("leak.bas"); - let exe = tmp.path().join("leak"); - std::fs::write(&bas, src).unwrap(); - let c = std::process::Command::new(env!("CARGO_BIN_EXE_xbasic64")) - .arg(&bas) - .arg("-o") - .arg(&exe) - .output() - .unwrap(); - assert!(c.status.success(), "{}", String::from_utf8_lossy(&c.stderr)); - let run = std::process::Command::new("sh") - .arg("-c") - .arg(format!("ulimit -v 262144 && exec {}", exe.display())) - .output() - .unwrap(); - assert!( - run.status.success(), - "stdout {} stderr {}", - String::from_utf8_lossy(&run.stdout), - String::from_utf8_lossy(&run.stderr) - ); - assert_eq!(String::from_utf8_lossy(&run.stdout).trim(), "20000 20001"); + let run = crate::common::run_with_address_space_limit(src, 262144); + run.assert_ran_to_completion("under a 256 MB address-space limit"); + assert_eq!(run.stdout.trim(), "20000 20001"); } /// The places a string changes hands, each with a later reassignment that From 8a2a185214dff6c5fbf68871f9c63b4f1c46ff92 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Sat, 10 Oct 2026 21:28:45 +0000 Subject: [PATCH 5/7] Docs: xbasic64 run, the JIT's code model, XB_JIT README: usage and architecture mention xbasic64 run. CLAUDE.md: jit.rs, the run subcommand, the code model (absolute references to the runtime, the program's own code and data PC-relative within one arena), the rule that a new helper needs a JIT symbol, and the XB_JIT suite mode. Measured, x86-64 Linux (best of 3): source to finished run, compile+link+exec vs run mega.bas 0.05s vs 0.04s, a 2000-line program 0.15s vs 0.15s, a 10000-line program 0.64s vs 0.62s -- compiling dominates; the link is about 10 ms. compiled code's run time, executable vs run (which includes compiling) array loops 0.12s vs 0.13s, fib(32) 0.01s vs 0.01s, strings 0.07s vs 0.07s, PRINT 2M numbers 0.23s vs 0.24s. Co-Authored-By: Claude Opus 5.5 --- CLAUDE.md | 34 ++++++++++++++++++++++++++++------ README.md | 10 ++++++++-- 2 files changed, 36 insertions(+), 8 deletions(-) diff --git a/CLAUDE.md b/CLAUDE.md index 555788e..d6deb79 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -23,6 +23,10 @@ cargo test strings:: cargo run -- program.bas # Output: program, beside the source cargo run -- program.bas -o out # Custom output name cargo run -- -S program.bas # Cranelift IR to .clif instead (no object, no link) +cargo run -- run program.bas # Compile into memory and run it at once (the JIT) + +# The integration suite with every program run by `xbasic64 run` +XB_JIT=1 cargo test --test integration ``` ## Architecture @@ -58,8 +62,14 @@ Source → Lexer → Parser → Sema → CodeGen → Object → Executable Every call goes through `FnGen::call`, which builds the call from it; tests read a call graph from `rt/src` and check that `fallible` is exactly "can reach `raise_with_code`" and `fresh` exactly "can reach `alloc_str`" +- **jit.rs** - `xbasic64 run`: the same lowering (`codegen::lower`) into a + cranelift-jit `JITModule`, then main is called in-process. The runtime is + the `rt` crate linked into the compiler (feature `host`); `symbols()` gives + the JIT the address of every helper, runtime word and message, and C + function generated code names, since an executable exports none by name - **main.rs** - CLI driver: reads source, runs the pipeline, writes the object - and the runtime's, and links them with `cc` (or `link.exe`) + and the runtime's, and links them with `cc` (or `link.exe`); or, for the + `run` subcommand, hands the program to `jit.rs` - **build.rs** - Builds the runtime (`rt/`) with rustc for the target when the compiler is built: a `no_std` staticlib with fat LTO, of which only the crate's one merged object is kept (`$OUT_DIR/rt.o`; the rest of the @@ -87,7 +97,9 @@ covers the workspace). In the runtime: Integration tests organized by feature area: - `integration.rs` - The one test crate: every module directory below is declared there, so a new one must be added -- `common/mod.rs` - Test harness. `compile_and_run()` for the usual case; +- `common/mod.rs` - Test harness. `XB_JIT=1` runs every program through + `xbasic64 run` instead of as an executable (CI runs the suite both ways). + `compile_and_run()` for the usual case; `compile_only()` for "this must be rejected, with this message"; `compile_and_run_raw()` for "printed this, then aborted with this exit code"; and `compile_to_clif()` / `uses_symbol()` for the IR the compiler generates @@ -95,6 +107,8 @@ Integration tests organized by feature area: - `megatest/` - One large program (`mega.bas`) exercising most of the language at once, so that features are tested coexisting rather than in isolation. It is its own assertion harness: a regression names itself +- `jit/` - What is particular to `xbasic64 run`: how the process ends, what + reaches stdout and stderr, and that it agrees with an executable - `codegen/` - Assertions on the Cranelift IR the compiler generates (through `common::compile_to_clif()`), and behavioural tests of code paths with special cases: constant divisors, scalars in SSA values, huge functions @@ -124,10 +138,18 @@ Integration tests organized by feature area: a function gets no optimizer and the single-pass register allocator, since the backtracking one is superlinear with SSA values live across it; that code runs about 2x slower -- **Every reference is colocated**: functions and data, the runtime's and libc's - included, are reached PC-relative (`call rel32` and RIP-relative on x86-64, - `bl` and adrp+add on aarch64); on Linux the link is `-no-pie`, and macOS, - where executables are always PIE, needs nothing more +- **Every reference is colocated in an executable**: functions and data, the + runtime's and libc's included, are reached PC-relative (`call rel32` and + RIP-relative on x86-64, `bl` and adrp+add on aarch64); on Linux the link is + `-no-pie`, and macOS, where executables are always PIE, needs nothing more. + `CodeGen::colocate` says so; the JIT clears it +- **The JIT's code model** (`jit.rs`): the runtime and libc are the + compiler's own, anywhere in its address space, so a reference to them is + an absolute address (Cranelift's default for an import). The program's + own code and data stay PC-relative, which needs them within 2 GB: they + come from one arena, 64 MB, doubled -- and the program lowered again -- + when it runs out. A new runtime helper or C function needs an entry in + `jit::symbols()`, which a test holds to rtsig - **Native only, two ISAs**: Cranelift's `x86` and `arm64` backends are built in, and the compiler targets the host it was built for. aarch64 Linux can be checked on an x86-64 host under qemu-user: cross-build with diff --git a/README.md b/README.md index a2662f6..0c622ca 100644 --- a/README.md +++ b/README.md @@ -48,6 +48,9 @@ cargo build --release # Compile a BASIC program to executable xbasic64 program.bas +# Or compile it into memory and run it at once, with no executable +xbasic64 run program.bas + # Specify output file xbasic64 program.bas -o myprogram @@ -92,9 +95,12 @@ Source → Lexer → Parser → Semantic Analysis → Code Generator → Object 3. **Semantic analysis** - Resolves names, checks types and argument counts, and reports problems with a source line 4. **Code Generator** - Lowers the AST to Cranelift IR; Cranelift does instruction - selection and register allocation and writes the object file + selection and register allocation and writes the object file -- or, for + `xbasic64 run`, puts the code in memory (a JIT) and runs it in the compiler's + own process. The whole program is compiled first either way, from the same + lowering; nothing is interpreted -The runtime library (I/O, strings, number formatting, math) is a `no_std` Rust crate in `rt/` that calls the C library. It is built once, when the compiler is built, and linked beside each program. +The runtime library (I/O, strings, number formatting, math) is a `no_std` Rust crate in `rt/` that calls the C library. It is built once, when the compiler is built, and linked beside each program; `xbasic64 run` uses the copy linked into the compiler. Key design choices: - No IR of its own: the AST is lowered directly to Cranelift's From 3ea783a3418ce65ac6a8aaeeed9fc1a69351fa01 Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Sat, 10 Oct 2026 21:33:06 +0000 Subject: [PATCH 6/7] README: lead with both ways to run a program, and every platform The opening said only "a native code compiler" and promised executables, with no word of xbasic64 run. It now says what the compiler does both ways -- an executable, or compiled into memory and run at once -- and on which platforms; usage leads with run; the pipeline shows both ends; requirements note that run needs no linker; the platforms are a table of OS, architecture and the two modes. Co-Authored-By: Claude Opus 5.5 --- README.md | 56 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 0c622ca..990a09d 100644 --- a/README.md +++ b/README.md @@ -1,10 +1,21 @@ # xbasic64 -A BASIC native code compiler for x86-64 and aarch64. +A compiler for classic BASIC: build a native executable, or compile into +memory and run at once. x86-64 and aarch64, on Linux, Windows and macOS. ## Overview -xbasic64 compiles 1980s-era BASIC dialects (Tandy Color BASIC, GW-BASIC, QuickBASIC) directly to native x86-64 and aarch64 executables. No interpreter, no bytecode—just fast native binaries. +xbasic64 compiles 1980s-era BASIC dialects (Tandy Color BASIC, GW-BASIC, +QuickBASIC) to native machine code with [Cranelift](https://cranelift.dev). +It does so two ways, from one code generator: + +- **Ahead of time**: `xbasic64 prog.bas` writes a standalone executable. +- **Just in time**: `xbasic64 run prog.bas` compiles the program into memory + and runs it at once, with no executable and no linker -- the immediacy of + an interpreter, at native speed. + +Either way the whole program is compiled before it starts. Nothing is +interpreted, and there is no bytecode. **Why xbasic64?** @@ -45,20 +56,21 @@ cargo build --release ### Usage ```bash -# Compile a BASIC program to executable -xbasic64 program.bas - -# Or compile it into memory and run it at once, with no executable +# Run a BASIC program at once (compiled into memory, then run) xbasic64 run program.bas +# Compile a BASIC program to an executable +xbasic64 program.bas + # Specify output file xbasic64 program.bas -o myprogram # Write the program's Cranelift IR to program.clif instead of compiling it xbasic64 -S program.bas -# Omit the runtime safety checks +# Omit the runtime safety checks (for either) xbasic64 --unsafe program.bas +xbasic64 run --unsafe program.bas ``` ### Example @@ -75,7 +87,8 @@ FOR I = 1 TO 10 NEXT I ``` -Save as `fib.bas`, compile with `xbasic64 fib.bas`, and run `./fib`. +Save as `fib.bas` and run it with `xbasic64 run fib.bas` -- or compile it +with `xbasic64 fib.bas` and run `./fib`. ## Documentation @@ -83,11 +96,14 @@ Save as `fib.bas`, compile with `xbasic64 fib.bas`, and run `./fib`. ## Architecture -The compiler is a four-stage pipeline: +The compiler is a four-stage pipeline, ending in one of two places: ``` -Source → Lexer → Parser → Semantic Analysis → Code Generator → Object → Executable - (tokens) (AST) (symbols) (Cranelift IR) +Source → Lexer → Parser → Semantic Analysis → Code Generator ─┬→ Object → Executable + (tokens) (AST) (symbols) │ (xbasic64 prog.bas) + (Cranelift IR) + └→ Memory → Run now + (xbasic64 run prog.bas) ``` 1. **Lexer** - Tokenizes BASIC source (case-insensitive keywords, line numbers, type suffixes) @@ -104,21 +120,27 @@ The runtime library (I/O, strings, number formatting, math) is a `no_std` Rust c Key design choices: - No IR of its own: the AST is lowered directly to Cranelift's +- One code generator for both executables and the JIT, so the two cannot + disagree; the test suite runs every program both ways - A portable `no_std` Rust runtime over the C library - GW-BASIC type semantics (division always returns Double) ## Requirements - Rust 1.96 or later -- Linux and macOS: the C compiler/linker (`cc`) with the C library, to link - programs -- Windows: the MSVC linker (`link.exe`) and its C runtime, to link programs +- To build executables, the platform's linker: `cc` with the C library on + Linux and macOS, the MSVC linker (`link.exe`) and its C runtime on Windows. + `xbasic64 run` needs no linker ## Platforms -- Linux (x86-64, aarch64) -- Windows (x86-64) -- macOS (arm64) +| OS | Architecture | Executables | `xbasic64 run` | +|---------|-----------------|-------------|----------------| +| Linux | x86-64, aarch64 | yes | yes | +| Windows | x86-64 | yes | yes | +| macOS | arm64 | yes | yes | + +The compiler targets the machine it runs on. ## License From 773a2da018b67085faed2ec939f756434aaefd8e Mon Sep 17 00:00:00 2001 From: Jeff Garzik Date: Sat, 10 Oct 2026 21:43:58 +0000 Subject: [PATCH 7/7] rt: open files non-inheritable on Windows; fix libc_sig's doc comment The intermittent Windows CI failure -- "Permission denied" from inside the runtime's unit tests, first seen at 4ad0166 -- is now named, by the panic its fatal error makes in a test: a_field_window_outlives_its_file closing a file and opening it again. The C library's handles are inheritable by default on Windows, and the unit tests that expect a fatal error run in child processes, beside the file tests. A child started while a file was open for writing kept a write handle of its own to it, so reopening the file -- which refuses other writers (_SH_DENYWR, as the assembly runtime's CreateFileA did) -- failed for as long as the child lived. sys::open now adds MSVC's N mode flag, so the handle stays in the process that opened it. The same would have bitten a program the compiler runs, had the compiler started a child. an_open_file_is_not_inherited_by_a_child reproduces it: under Wine it failed with Permission denied before, and passes now. Also: LIBC_FUNCTIONS had taken libc_sig's doc comment above its own; each has its own again (review feedback). Co-Authored-By: Claude Opus 5.5 --- rt/src/file.rs | 25 +++++++++++++++++++++++++ rt/src/sys.rs | 20 +++++++++++++++++++- src/codegen/mod.rs | 2 +- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/rt/src/file.rs b/rt/src/file.rs index 91ea707..56e376f 100644 --- a/rt/src/file.rs +++ b/rt/src/file.rs @@ -1343,6 +1343,31 @@ mod tests { ); } + /// A file the program has open is not handed to a child process the + /// compiler or a test starts meanwhile. On Windows the C library's + /// handles were inheritable, so a child started while a file was open + /// for writing kept its own write handle to it; the file, closed and + /// opened again (which refuses other writers), was then "Permission + /// denied" for as long as the child lived. CI hit it when the unit tests + /// that run in child processes ran beside a file test. + #[cfg(windows)] + #[test] + fn an_open_file_is_not_inherited_by_a_child() { + let _g = serial(); + let (_dir, path) = temp("inherit.txt"); + open(&path, OUTPUT, 9); + put(9, "x"); + // A child that outlives the close, started while the file is open. + let mut child = std::process::Command::new("cmd") + .args(["/c", "ping -n 3 127.0.0.1 >nul"]) + .spawn() + .unwrap(); + unsafe { _rt_file_close(9) }; + open(&path, OUTPUT, 9); + unsafe { _rt_file_close(9) }; + let _ = child.wait(); + } + /// A FIELD variable is a window onto its file's record buffer, and /// outlives the OPEN that made it: a re-OPEN of the number, sequential or /// RANDOM, freed the buffer, and LSET then wrote freed memory and PRINT diff --git a/rt/src/sys.rs b/rt/src/sys.rs index 8a345a8..7f8d4b3 100644 --- a/rt/src/sys.rs +++ b/rt/src/sys.rs @@ -550,11 +550,29 @@ mod windows { /// assembly runtime's CreateFileA did with FILE_SHARE_READ: a second /// writer is refused, which is "Permission denied". /// + /// And not inherited (`N`): the C library's handles are inheritable by + /// default, so a child process started while the file was open -- by a + /// test, or by the compiler around a program it runs -- kept a write + /// handle of its own, and the program's next OPEN of the file, refusing + /// other writers, was "Permission denied" while the child lived. + /// /// # Safety /// Both must be NUL-terminated. pub unsafe fn open(name: *const c_char, mode: *const c_char) -> *mut FILE { const _SH_DENYWR: c_int = 0x20; - unsafe { _fsopen(name, mode, _SH_DENYWR) } + // The mode, then N. Modes are a few letters ("r+b" at most). + let mut with_n = [0u8; 8]; + let mut n = 0; + while n < with_n.len() - 2 { + let c = unsafe { *mode.add(n) } as u8; + if c == 0 { + break; + } + with_n[n] = c; + n += 1; + } + with_n[n] = b'N'; + unsafe { _fsopen(name, with_n.as_ptr().cast(), _SH_DENYWR) } } /// Why the open just made failed. errno alone cannot say: the CRT maps diff --git a/src/codegen/mod.rs b/src/codegen/mod.rs index 403b681..17d7161 100644 --- a/src/codegen/mod.rs +++ b/src/codegen/mod.rs @@ -538,13 +538,13 @@ fn rt_type(t: rtsig::Ty) -> Type { } } -/// The C library functions generated code calls, with their signatures. /// The C library functions generated code calls, all of them. pub(crate) const LIBC_FUNCTIONS: [&str; 12] = [ "pow", "sin", "cos", "tan", "atan", "exp", "log", "calloc", "realloc", "free", "memset", "memcpy", ]; +/// The C library functions generated code calls, with their signatures. fn libc_sig(name: &str) -> (&'static [Type], &'static [Type]) { debug_assert!(LIBC_FUNCTIONS.contains(&name), "{name} is not listed"); const F: &[Type] = &[types::F64];