From bf2645df8b7ffc571d3891f67844eb19e8f46fa7 Mon Sep 17 00:00:00 2001 From: "steeven.herlant" Date: Wed, 13 May 2026 16:15:04 +0200 Subject: [PATCH 1/3] feat(OutputBuffer): expose scanner buffer size configuration OutputBuffer.Lines() used bufio.NewScanner with its default 64KB token limit. Lines exceeding that size caused silent truncation: s.Scan() returned false with bufio.ErrTooLong, but the error was never checked, so all subsequent output was lost without any indication. - Adds SetScannerBufferSize(n int) on OutputBuffer to override the limit - Adds Err() on OutputBuffer to surface scanner errors to callers - Propagates Options.LineBufferSize (already used by OutputStream) to OutputBuffer in NewCmdOptions, for both Buffered and CombinedOutput modes - Propagates the buffer size through Clone() - Adds tests covering the truncation behaviour and the new API --- cmd.go | 57 ++++++++++++++++++++++++---- cmd_test.go | 107 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 157 insertions(+), 7 deletions(-) diff --git a/cmd.go b/cmd.go index ac989b3..dc8f99a 100644 --- a/cmd.go +++ b/cmd.go @@ -205,10 +205,17 @@ func NewCmdOptions(options Options, name string, args ...string) *Cmd { if options.Buffered { c.stdoutBuf = NewOutputBuffer() c.stderrBuf = NewOutputBuffer() + if options.LineBufferSize > 0 { + c.stdoutBuf.SetScannerBufferSize(int(options.LineBufferSize)) + c.stderrBuf.SetScannerBufferSize(int(options.LineBufferSize)) + } } if options.CombinedOutput { c.stdoutBuf = NewOutputBuffer() + if options.LineBufferSize > 0 { + c.stdoutBuf.SetScannerBufferSize(int(options.LineBufferSize)) + } c.stderrBuf = nil } @@ -239,11 +246,18 @@ func NewCmdOptions(options Options, name string, args ...string) *Cmd { // of the original object is lost. Cmd is one-use only, so if you need to restart // a Cmd, you need to Clone it. func (c *Cmd) Clone() *Cmd { + var lineBufferSize uint + if c.stdoutBuf != nil { + lineBufferSize = uint(c.stdoutBuf.scannerBufSize) + } else if c.stdoutStream != nil { + lineBufferSize = uint(c.stdoutStream.bufSize) + } clone := NewCmdOptions( Options{ Buffered: c.stdoutBuf != nil, CombinedOutput: c.stdoutBuf != nil, Streaming: c.stdoutStream != nil, + LineBufferSize: lineBufferSize, }, c.Name, c.Args..., @@ -582,20 +596,41 @@ func (c *Cmd) run(in io.Reader) { // While runnableCmd is running, call stdout.Lines() to read all output // currently written. type OutputBuffer struct { - buf *bytes.Buffer - lines []string + buf *bytes.Buffer + lines []string + scannerBufSize int + scanErr error *sync.Mutex } // NewOutputBuffer creates a new output buffer. The buffer is unbounded and safe // for multiple goroutines to read while the command is running by calling Lines. func NewOutputBuffer() *OutputBuffer { - out := &OutputBuffer{ - buf: &bytes.Buffer{}, - lines: []string{}, - Mutex: &sync.Mutex{}, + return &OutputBuffer{ + buf: &bytes.Buffer{}, + lines: []string{}, + scannerBufSize: bufio.MaxScanTokenSize, + Mutex: &sync.Mutex{}, } - return out +} + +// SetScannerBufferSize sets the maximum token size for the bufio.Scanner used +// by Lines(). The default is bufio.MaxScanTokenSize (64KB). Increase this value +// if Lines() truncates output because a line exceeds that limit (Err() will +// return a non-nil error in that case). Must be called before the command starts. +func (rw *OutputBuffer) SetScannerBufferSize(n int) { + rw.Lock() + rw.scannerBufSize = n + rw.Unlock() +} + +// Err returns the last error set by Lines(), if any. A non-nil error indicates +// that output was truncated because a line exceeded the scanner buffer size. +// Use SetScannerBufferSize to increase the limit. +func (rw *OutputBuffer) Err() error { + rw.Lock() + defer rw.Unlock() + return rw.scanErr } // Write makes OutputBuffer implement the io.Writer interface. Do not call @@ -610,14 +645,22 @@ func (rw *OutputBuffer) Write(p []byte) (n int, err error) { // Lines returns lines of output written by the Cmd. It is safe to call while // the Cmd is running and after it has finished. Subsequent calls returns more // lines, if more lines were written. "\r\n" are stripped from the lines. +// +// If a line exceeds the scanner buffer size (set via SetScannerBufferSize, +// default bufio.MaxScanTokenSize / 64KB), scanning stops and Err() returns +// the error. Use SetScannerBufferSize to increase the limit. func (rw *OutputBuffer) Lines() []string { rw.Lock() // Scanners are io.Readers which effectively destroy the buffer by reading // to EOF. So once we scan the buf to lines, the buf is empty again. s := bufio.NewScanner(rw.buf) + s.Buffer(make([]byte, rw.scannerBufSize), rw.scannerBufSize) for s.Scan() { rw.lines = append(rw.lines, s.Text()) } + if err := s.Err(); err != nil { + rw.scanErr = err + } rw.Unlock() return rw.lines } diff --git a/cmd_test.go b/cmd_test.go index 0d51cdc..6d1a7ab 100644 --- a/cmd_test.go +++ b/cmd_test.go @@ -1479,3 +1479,110 @@ func TestCmdLineBufferIncrease(t *testing.T) { } <-catStderrStatus } + +func TestOutputBufferDefaultScannerBufSize(t *testing.T) { + // Default scanner buffer size is bufio.MaxScanTokenSize (64KB). + // A line within that limit must be returned correctly. + buf := cmd.NewOutputBuffer() + line := bytes.Repeat([]byte("x"), 1000) + line = append(line, '\n') + buf.Write(line) + lines := buf.Lines() + if len(lines) != 1 { + t.Fatalf("expected 1 line, got %d", len(lines)) + } + if len(lines[0]) != 1000 { + t.Errorf("expected line of 1000 bytes, got %d", len(lines[0])) + } + if err := buf.Err(); err != nil { + t.Errorf("expected no error, got: %v", err) + } +} + +func TestOutputBufferLongLineTruncation(t *testing.T) { + // A line exceeding bufio.MaxScanTokenSize (64KB) with the default buffer + // must set Err() to a non-nil error, and Lines() must stop at that point. + buf := cmd.NewOutputBuffer() + + longLine := bytes.Repeat([]byte("x"), 65*1024) // 65KB > 64KB default + longLine = append(longLine, '\n') + after := []byte("after\n") + buf.Write(longLine) + buf.Write(after) + + lines := buf.Lines() + if buf.Err() == nil { + t.Error("expected a non-nil error from Err() after long line, got nil") + } + for _, l := range lines { + if l == "after" { + t.Error("lines after the oversized line should not appear when truncated") + } + } +} + +func TestOutputBufferSetScannerBufferSize(t *testing.T) { + // SetScannerBufferSize allows reading lines larger than 64KB. + buf := cmd.NewOutputBuffer() + buf.SetScannerBufferSize(256 * 1024) // 256KB + + longLine := bytes.Repeat([]byte("y"), 65*1024) // 65KB + longLine = append(longLine, '\n') + buf.Write(longLine) + buf.Write([]byte("after\n")) + + lines := buf.Lines() + if err := buf.Err(); err != nil { + t.Errorf("expected no error with enlarged buffer, got: %v", err) + } + if len(lines) != 2 { + t.Fatalf("expected 2 lines, got %d", len(lines)) + } + if len(lines[0]) != 65*1024 { + t.Errorf("expected first line of %d bytes, got %d", 65*1024, len(lines[0])) + } + if lines[1] != "after" { + t.Errorf("expected second line 'after', got %q", lines[1]) + } +} + +func TestOutputBufferLineBufferSizeOption(t *testing.T) { + // Options.LineBufferSize propagates to OutputBuffer when Buffered is true. + const bufSize = 256 * 1024 // 256KB + p := cmd.NewCmdOptions( + cmd.Options{Buffered: true, LineBufferSize: bufSize}, + "echo", + string(bytes.Repeat([]byte("z"), 65*1024)), // 65KB argument → one long output line + ) + status := <-p.Start() + if status.Error != nil { + t.Fatalf("command error: %v", status.Error) + } + if len(status.Stdout) != 1 { + t.Fatalf("expected 1 line of stdout, got %d", len(status.Stdout)) + } + if len(status.Stdout[0]) != 65*1024 { + t.Errorf("expected stdout line of %d bytes, got %d", 65*1024, len(status.Stdout[0])) + } +} + +func TestOutputBufferClonePropagatesScannerBufSize(t *testing.T) { + // Clone must propagate LineBufferSize to the cloned Cmd's OutputBuffer. + const bufSize = 256 * 1024 + p := cmd.NewCmdOptions( + cmd.Options{Buffered: true, LineBufferSize: bufSize}, + "echo", + string(bytes.Repeat([]byte("z"), 65*1024)), + ) + clone := p.Clone() + status := <-clone.Start() + if status.Error != nil { + t.Fatalf("clone command error: %v", status.Error) + } + if len(status.Stdout) != 1 { + t.Fatalf("expected 1 line from clone stdout, got %d", len(status.Stdout)) + } + if len(status.Stdout[0]) != 65*1024 { + t.Errorf("expected clone stdout line of %d bytes, got %d", 65*1024, len(status.Stdout[0])) + } +} From e5b097f056b5e126145ba101a26e9d7985bce1b4 Mon Sep 17 00:00:00 2001 From: "steeven.herlant" Date: Wed, 13 May 2026 16:18:37 +0200 Subject: [PATCH 2/3] feat(Cmd): propagate scanner overflow error into Status.Error When OutputBuffer.Lines() encounters a line exceeding the scanner buffer size, the error is now surfaced in Status.Error once the command finishes, so callers using the Cmd API detect the truncation without having to access OutputBuffer directly. --- cmd.go | 7 ++++++- cmd_test.go | 14 ++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/cmd.go b/cmd.go index dc8f99a..156ed11 100644 --- a/cmd.go +++ b/cmd.go @@ -389,11 +389,16 @@ func (c *Cmd) Status() Status { if !c.final { if c.stdoutBuf != nil { c.status.Stdout = c.stdoutBuf.Lines() + if err := c.stdoutBuf.Err(); err != nil && c.status.Error == nil { + c.status.Error = err + } c.stdoutBuf = nil // release buffers - } if c.stderrBuf != nil { c.status.Stderr = c.stderrBuf.Lines() + if err := c.stderrBuf.Err(); err != nil && c.status.Error == nil { + c.status.Error = err + } c.stderrBuf = nil // release buffers } c.final = true diff --git a/cmd_test.go b/cmd_test.go index 6d1a7ab..a023420 100644 --- a/cmd_test.go +++ b/cmd_test.go @@ -1566,6 +1566,20 @@ func TestOutputBufferLineBufferSizeOption(t *testing.T) { } } +func TestCmdStatusErrorOnScannerOverflow(t *testing.T) { + // When a line exceeds the scanner buffer size, status.Error must be non-nil + // so the caller can detect the truncation without inspecting OutputBuffer directly. + p := cmd.NewCmdOptions( + cmd.Options{Buffered: true}, // default 64KB limit + "echo", + string(bytes.Repeat([]byte("x"), 65*1024)), // 65KB > 64KB + ) + status := <-p.Start() + if status.Error == nil { + t.Error("expected status.Error to be non-nil on scanner overflow, got nil") + } +} + func TestOutputBufferClonePropagatesScannerBufSize(t *testing.T) { // Clone must propagate LineBufferSize to the cloned Cmd's OutputBuffer. const bufSize = 256 * 1024 From 1e9138180b16adbd1df358eecf99cc5333aa9412 Mon Sep 17 00:00:00 2001 From: "steeven.herlant" Date: Tue, 19 May 2026 10:19:27 +0200 Subject: [PATCH 3/3] chore: rename module path to github.com/orus-io/cmd --- cmd_test.go | 2 +- cmd_windows_test.go | 2 +- examples/blocking-buffered/main.go | 2 +- examples/blocking-streaming/main.go | 2 +- go.mod | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd_test.go b/cmd_test.go index a023420..35a1031 100644 --- a/cmd_test.go +++ b/cmd_test.go @@ -12,7 +12,7 @@ import ( "testing" "time" - "github.com/go-cmd/cmd" + "github.com/orus-io/cmd" "github.com/go-test/deep" ) diff --git a/cmd_windows_test.go b/cmd_windows_test.go index cc33c81..1fd5c25 100644 --- a/cmd_windows_test.go +++ b/cmd_windows_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "github.com/go-cmd/cmd" + "github.com/orus-io/cmd" "github.com/go-test/deep" ) diff --git a/examples/blocking-buffered/main.go b/examples/blocking-buffered/main.go index 4b3d5e3..61bb7db 100644 --- a/examples/blocking-buffered/main.go +++ b/examples/blocking-buffered/main.go @@ -3,7 +3,7 @@ package main import ( "fmt" - "github.com/go-cmd/cmd" + "github.com/orus-io/cmd" ) func main() { diff --git a/examples/blocking-streaming/main.go b/examples/blocking-streaming/main.go index f065aa3..e95f328 100644 --- a/examples/blocking-streaming/main.go +++ b/examples/blocking-streaming/main.go @@ -6,7 +6,7 @@ import ( "fmt" "os" - "github.com/go-cmd/cmd" + "github.com/orus-io/cmd" ) func main() { diff --git a/go.mod b/go.mod index eda6304..3f138dd 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module github.com/go-cmd/cmd +module github.com/orus-io/cmd go 1.20