From 069a4dfeee9fbddc439e80ea35380ffe24eeb2f7 Mon Sep 17 00:00:00 2001 From: mattsu Date: Fri, 19 Jun 2026 08:58:01 +0900 Subject: [PATCH 1/2] pr: prepare page construction for streaming input Replace the whole-buffer page parser with a chunk-aware PageBuilder while preserving the existing buffered output path. This isolates page boundary and line-number handling before the streaming path is enabled. --- src/uu/pr/src/pr.rs | 220 +++++++++++++++++++++++++++----------------- 1 file changed, 135 insertions(+), 85 deletions(-) diff --git a/src/uu/pr/src/pr.rs b/src/uu/pr/src/pr.rs index 0671742cb3c..2c7f940b198 100644 --- a/src/uu/pr/src/pr.rs +++ b/src/uu/pr/src/pr.rs @@ -122,6 +122,16 @@ impl FileLine { } } +struct PageLine { + line_number: usize, + content: Vec, +} + +struct InputPage { + page_number: usize, + lines: Vec, +} + struct ColumnModeOptions { width: usize, columns: usize, @@ -967,104 +977,144 @@ fn pr(path: &str, options: &OutputOptions) -> Result { Ok(0) } -/// Group lines of a file into pages. -/// -/// Returns a list of the form `(page_num, lines)`. -/// -fn get_pages(options: &OutputOptions, file_id: usize, buf: &[u8]) -> Vec<(usize, Vec)> { - let start_page = options.start_page; - let end_page = options.end_page; - let lines_needed_per_page = lines_to_read_for_page(options); - - // Keep a running total of the number of lines read, starting with - // 0 or another specified number. - let mut line_num = get_start_line_number(options); - - // We will collect each page into a list of pages, along with - // its page number. - let mut pages: Vec<(usize, Vec)> = vec![]; - - // We will build each page iteratively, since one page may - // contain multiple lines and may be interrupted by either a - // form feed or by reaching a line limit. - let mut page = vec![]; - let mut page_num = 0; - - // Remember the index of the end of the last line to use as the - // beginning of the next line. - let mut prev = 0; - - // Search for either the form feed character `\f` or the newline - // character `\n`. The newline character marks the end of a line, - // and a page comprises several lines. A form feed character marks - // the end of a page regardless of how many lines have been read. - for i in memchr::memchr2_iter(FF, NL, buf) { - if buf[i] == FF { - // Treat everything up to (but not including) the form feed - // character as the last line of the page. - if i > 0 && i == prev && buf[i - 1] == NL { - // If the file has the pattern `\n\f`, don't treat the - // `\f` as its own line; instead ignore the empty line. - } else { - let file_line = - FileLine::from_buf(file_id, page_num, line_num, &buf[prev..i], options); - page.push(file_line); - } +struct PageBuilder<'a> { + options: &'a OutputOptions, + lines_needed_per_page: usize, + line_number: usize, + page_number: usize, + page: Vec, + pending: Vec, + current_line_has_content: bool, + last_delimiter: Option, +} - // Remember where the last line ended. - prev = i + 1; - - // The page is finished, so we add it to the list of - // pages and clear the `page` buffer for the next - // iteration. - // - // TODO Optimization opportunity: don't bother pushing - // lines and pages if we aren't going to display it. - if start_page <= page_num + 1 && end_page.is_none_or(|e| page_num < e) { - pages.push((page_num, page.clone())); - } - page_num += 1; - page.clear(); - } else { - // Add everything up to (but not including) the newline - // character as one line of the page. - if i > 0 && i == prev && buf[i - 1] == FF { - // If the file has the pattern `\f\n`, don't treat the - // `\n` as its own line; instead ignore the empty line. - } else { - let file_line = - FileLine::from_buf(file_id, page_num, line_num, &buf[prev..i], options); - page.push(file_line); - line_num += 1; - } +impl<'a> PageBuilder<'a> { + fn new(options: &'a OutputOptions) -> Self { + Self { + options, + lines_needed_per_page: lines_to_read_for_page(options), + line_number: get_start_line_number(options), + page_number: 0, + page: Vec::new(), + pending: Vec::new(), + current_line_has_content: false, + last_delimiter: None, + } + } + + fn push_chunk(&mut self, chunk: &[u8]) -> Vec { + let mut pages = Vec::new(); + let mut prev = 0; + + for index in memchr::memchr2_iter(FF, NL, chunk) { + self.pending.extend_from_slice(&chunk[prev..index]); + let delimiter = chunk[index]; - // Remember where the last line ended. - prev = i + 1; + if delimiter == FF { + if !(self.pending.is_empty() + && !self.current_line_has_content + && matches!(self.last_delimiter, Some(NL))) + { + self.push_pending_line(); + } + self.last_delimiter = Some(FF); + if let Some(page) = self.finish_page() { + pages.push(page); + } + } else { + if !(self.pending.is_empty() + && !self.current_line_has_content + && matches!(self.last_delimiter, Some(FF))) + { + self.push_pending_line(); + self.line_number += 1; + } + self.last_delimiter = Some(NL); - // If the page is finished, add it to the list of pages - // and clear the `page` buffer for the next iteration. - if page.len() >= lines_needed_per_page { - if start_page <= page_num + 1 && end_page.is_none_or(|e| page_num < e) { - pages.push((page_num, page.clone())); + if self.page.len() >= self.lines_needed_per_page { + if let Some(page) = self.finish_page() { + pages.push(page); + } } - page_num += 1; - page.clear(); } + + prev = index + 1; + } + + self.pending.extend_from_slice(&chunk[prev..]); + if !self.pending.is_empty() { + self.last_delimiter = None; } + + pages } - // Consider all trailing bytes as the last line. - if prev < buf.len() { - let file_line = FileLine::from_buf(file_id, page_num, line_num, &buf[prev..], options); - page.push(file_line); + fn finish(mut self) -> Vec { + if !self.pending.is_empty() || self.current_line_has_content { + self.push_pending_line(); + } + + if self.page.is_empty() { + Vec::new() + } else { + self.finish_page().into_iter().collect() + } } - // Consider all trailing lines as the last page. - if !page.is_empty() && start_page <= page_num + 1 && end_page.is_none_or(|e| page_num < e) { - pages.push((page_num, page.clone())); + fn push_pending_line(&mut self) { + self.page.push(PageLine { + line_number: self.line_number, + content: std::mem::take(&mut self.pending), + }); + self.current_line_has_content = false; } + fn finish_page(&mut self) -> Option { + let page_number = self.page_number; + let lines = std::mem::take(&mut self.page); + self.page_number += 1; + + if self.page_is_in_range(page_number) { + Some(InputPage { page_number, lines }) + } else { + None + } + } + + fn page_is_in_range(&self, page_number: usize) -> bool { + self.options.start_page <= page_number + 1 + && self.options.end_page.is_none_or(|end| page_number < end) + } +} + +/// Group lines of a file into pages. +/// +/// Returns a list of the form `(page_num, lines)`. +/// +fn get_pages(options: &OutputOptions, file_id: usize, buf: &[u8]) -> Vec<(usize, Vec)> { + let mut builder = PageBuilder::new(options); + let mut pages = builder.push_chunk(buf); + pages.extend(builder.finish()); + pages + .into_iter() + .map(|page| { + let lines = page + .lines + .into_iter() + .map(|line| { + FileLine::from_buf( + file_id, + page.page_number, + line.line_number, + &line.content, + options, + ) + }) + .collect(); + (page.page_number, lines) + }) + .collect() } /// Key used to group lines together according to their file and page number. From d54d2f9c9dc3450c9673f63c025acd601aaf856d Mon Sep 17 00:00:00 2001 From: mattsu Date: Fri, 19 Jun 2026 08:58:17 +0900 Subject: [PATCH 2/2] pr: stream simple non-file inputs Process simple layouts from pipes, stdin, and character devices incrementally instead of buffering unbounded input. Reuse PageBuilder for pagination, preserve buffered fallback behavior, and cover streaming devices, FIFO parity, invalid UTF-8, and zero-content-line cases. --- Cargo.lock | 1 + src/uu/pr/Cargo.toml | 1 + src/uu/pr/src/pr.rs | 189 ++++++++++++++++++++++++++++++++++++++- tests/by-util/test_pr.rs | 122 ++++++++++++++++++++++++- 4 files changed, 311 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 2b93e066919..325e94d5339 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3864,6 +3864,7 @@ dependencies = [ "itertools 0.14.0", "memchr", "regex", + "rustix", "thiserror 2.0.18", "uucore", ] diff --git a/src/uu/pr/Cargo.toml b/src/uu/pr/Cargo.toml index 5a464bfda7f..5cc190e555a 100644 --- a/src/uu/pr/Cargo.toml +++ b/src/uu/pr/Cargo.toml @@ -27,6 +27,7 @@ memchr = { workspace = true } regex = { workspace = true } thiserror = { workspace = true } fluent = { workspace = true } +rustix = { workspace = true, features = ["fs"] } [[bin]] name = "pr" diff --git a/src/uu/pr/src/pr.rs b/src/uu/pr/src/pr.rs index 2c7f940b198..bd9fe30908a 100644 --- a/src/uu/pr/src/pr.rs +++ b/src/uu/pr/src/pr.rs @@ -11,7 +11,9 @@ use itertools::Itertools; use regex::Regex; use std::ffi::OsStr; use std::fs::metadata; -use std::io::{Read, Write, stderr, stdin, stdout}; +use std::io::{BufWriter, Read, Write, stderr, stdin, stdout}; +#[cfg(unix)] +use std::os::fd::AsFd; use std::str::Utf8Error; use std::string::FromUtf8Error; use std::time::SystemTime; @@ -938,6 +940,164 @@ fn read_to_end(path: &str) -> Result, std::io::Error> { } } +fn should_use_streaming_pr(path: &str, options: &OutputOptions) -> bool { + let is_simple_layout = options.number.is_none() + && options.expand_tabs.is_none() + && options.column_mode_options.is_none() + && options.merge_files_print.is_none() + && !options.join_lines + && options.line_width.is_none() + && options.offset_spaces.is_empty(); + + if !is_simple_layout { + return false; + } + + if lines_to_read_for_page(options) == 0 { + return false; + } + + if path == FILE_STDIN { + return should_stream_stdin(); + } + + metadata(path).is_ok_and(|meta| !meta.file_type().is_file()) +} + +#[cfg(unix)] +fn should_stream_stdin() -> bool { + rustix::fs::fstat(stdin().as_fd()) + .is_ok_and(|stat| !rustix::fs::FileType::from_raw_mode(stat.st_mode).is_file()) +} + +#[cfg(not(unix))] +fn should_stream_stdin() -> bool { + false +} + +fn write_stream_page_header( + out: &mut impl Write, + options: &OutputOptions, + page: usize, +) -> Result<(), std::io::Error> { + let line_separator = options.line_separator.as_bytes(); + for line in header_content(options, page) { + out.write_all(line.as_bytes())?; + out.write_all(line_separator)?; + } + Ok(()) +} + +fn write_stream_page_trailer( + out: &mut impl Write, + options: &OutputOptions, + lines_in_page: usize, +) -> Result<(), std::io::Error> { + let content_line_separator = options.content_line_separator.as_bytes(); + if !options.form_feed_used { + // `print_page`/`write_columns` emits blank-content separators until + // the page reaches `lines_needed_per_page`. + let lines_needed_per_page = lines_to_read_for_page(options); + for _ in lines_in_page..lines_needed_per_page { + out.write_all(content_line_separator)?; + } + } + + let line_separator = options.line_separator.as_bytes(); + let trailer = trailer_content(options); + for (index, line) in trailer.iter().enumerate() { + out.write_all(line.as_bytes())?; + if index + 1 != trailer.len() { + out.write_all(line_separator)?; + } + } + out.write_all(options.page_separator_char.as_bytes())?; + Ok(()) +} + +fn write_stream_page( + out: &mut impl Write, + options: &OutputOptions, + page: &InputPage, +) -> Result<(), std::io::Error> { + write_stream_page_header(out, options, page.page_number + 1)?; + write_stream_page_body_and_trailer(out, options, page) +} + +fn write_stream_page_body_and_trailer( + out: &mut impl Write, + options: &OutputOptions, + page: &InputPage, +) -> Result<(), std::io::Error> { + let content_line_separator = options.content_line_separator.as_bytes(); + for line in &page.lines { + out.write_all(&line.content)?; + out.write_all(content_line_separator)?; + } + + write_stream_page_trailer(out, options, page.lines.len()) +} + +fn pr_stream_simple_with_reader( + mut reader: R, + options: &OutputOptions, +) -> Result { + let out = stdout(); + let mut out = BufWriter::new(out.lock()); + let mut page_builder = PageBuilder::new(options); + let mut streamed_current_page = false; + + let mut buf = [0_u8; 8192]; + loop { + let n = reader.read(&mut buf)?; + if n == 0 { + break; + } + + for page in page_builder.push_chunk(&buf[..n]) { + if streamed_current_page { + write_stream_page_body_and_trailer(&mut out, options, &page)?; + streamed_current_page = false; + } else { + write_stream_page(&mut out, options, &page)?; + } + } + + if let Some(pending) = page_builder.take_pending_for_streaming() { + if page_builder.current_page_in_range() { + if !streamed_current_page { + write_stream_page_header(&mut out, options, page_builder.page_number + 1)?; + streamed_current_page = true; + } + out.write_all(&pending)?; + } + } + + if page_builder.should_stop() { + break; + } + } + + if !page_builder.should_stop() { + for page in page_builder.finish() { + if streamed_current_page { + write_stream_page_body_and_trailer(&mut out, options, &page)?; + streamed_current_page = false; + } else { + write_stream_page(&mut out, options, &page)?; + } + } + } + + out.flush()?; + Ok(0) +} + +fn pr_stream_simple(path: &str, options: &OutputOptions) -> Result { + let file = std::fs::File::open(path)?; + pr_stream_simple_with_reader(file, options) +} + fn apply_expand_tab(chunk: &mut Vec, byte: u8, expand_options: &ExpandTabsOptions) { if byte == expand_options.input_char as u8 { // If the byte encountered is the input char we use width to calculate @@ -960,6 +1120,13 @@ fn apply_expand_tab(chunk: &mut Vec, byte: u8, expand_options: &ExpandTabsOp } fn pr(path: &str, options: &OutputOptions) -> Result { + if should_use_streaming_pr(path, options) { + if path == FILE_STDIN { + return pr_stream_simple_with_reader(stdin(), options); + } + return pr_stream_simple(path, options); + } + // Read the entire contents of the file into a buffer. // // TODO Read incrementally. @@ -1061,6 +1228,26 @@ impl<'a> PageBuilder<'a> { } } + fn should_stop(&self) -> bool { + self.options + .end_page + .is_some_and(|end| self.page_number >= end) + } + + fn current_page_in_range(&self) -> bool { + self.page_is_in_range(self.page_number) + } + + fn take_pending_for_streaming(&mut self) -> Option> { + if self.pending.is_empty() { + None + } else { + self.current_line_has_content = true; + self.last_delimiter = None; + Some(std::mem::take(&mut self.pending)) + } + } + fn push_pending_line(&mut self) { self.page.push(PageLine { line_number: self.line_number, diff --git a/tests/by-util/test_pr.rs b/tests/by-util/test_pr.rs index 5c60d879aaa..c81a982e8d4 100644 --- a/tests/by-util/test_pr.rs +++ b/tests/by-util/test_pr.rs @@ -2,11 +2,15 @@ // // For the full copyright and license information, please view the LICENSE // file that was distributed with this source code. -// spell-checker:ignore (ToDO) Sdivide ading +// spell-checker:ignore (ToDO) Sdivide ading IRWXU use jiff::{Timestamp, ToSpan}; use regex::Regex; +#[cfg(unix)] +use std::fmt::Write as _; use std::fs::metadata; +#[cfg(unix)] +use uutests::at_and_ts; use uutests::util::UCommand; use uutests::{at_and_ucmd, new_ucmd}; @@ -45,6 +49,36 @@ fn valid_last_modified_template_vars(from: Timestamp) -> Vec