oc rsync: Add --last flag

The flag can be used to only sync N most recently modified files.

The flag is mutually exclusive to:
  * --include
  * --exclude
  * --delete
  * --watch

It is also ignored when the source is a local directory when using tar,
because the implementation doesn't allow to select particular files.

Generally when there is any problem when using --last, the error is
ignored and sync happens as if the flag was not specified.

Regarding implementation details, oc rsync performs an extras step when
--last is specified, and that is discovering relevant files to select.
This is done using manual directory walking when local, for remote the
remote executor is used to invoke a shell using find+sort+head.

The resulting filenames are then passed to --files-from for rsync,
for tar they are simply passed to the command as arguments.

Tests were added for testing the discovery mechanism, the rest has been
tested manually. oc rsync is poorly unit-tested in general.

Assisted-by: Claude Code

Author: tchap

pkg/cli/rsync/copy_rsync.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"path/filepath"
 	"strings"
 
 	"github.com/spf13/cobra"
@@ -27,6 +28,9 @@ type rsyncStrategy struct {
 	LocalExecutor  executor
 	RemoteExecutor executor
 	podChecker     podChecker
+
+	Last          uint
+	fileDiscovery fileDiscoverer
 }
 
 // DefaultRsyncRemoteShellToUse generates an command to create a remote shell.
@@ -75,15 +79,45 @@ func NewRsyncStrategy(o *RsyncOptions) CopyStrategy {
 		RemoteExecutor: newRemoteExecutor(o),
 		LocalExecutor:  newLocalExecutor(),
 		podChecker:     podAPIChecker{o.Client, o.Namespace, podName, o.ContainerName, o.Quiet, o.ErrOut},
+		Last:           o.Last,
+		fileDiscovery:  o.fileDiscovery,
 	}
 }
 
 func (r *rsyncStrategy) Copy(source, destination *PathSpec, out, errOut io.Writer) error {
 	klog.V(3).Infof("Copying files with rsync")
+
+	// In case --last is specified, discover the right files and pass them to rsync as an explicit list.
+	var (
+		in  io.Reader
+		dst = destination.RsyncPath()
+	)
+	if r.Last > 0 {
+		filenames, err := r.fileDiscovery.DiscoverFiles(source.Path, r.Last)
+		if err != nil {
+			klog.Infof("Warning: failed to apply --last filtering: %v", err)
+		} else {
+			var b bytes.Buffer
+			for _, filename := range filenames {
+				b.WriteString(filename)
+				b.WriteRune('\n')
+			}
+			in = &b
+
+			// Make dst compatible with what rsync does without --last.
+			dst = filepath.Join(dst, filepath.Base(source.Path))
+
+			klog.V(3).Infof("Applied --last=%d to rsync strategy: using %d files", r.Last, len(filenames))
+		}
+	}
+
 	cmd := append([]string{"rsync"}, r.Flags...)
-	cmd = append(cmd, "-e", r.RshCommand, source.RsyncPath(), destination.RsyncPath())
+	if in != nil {
+		cmd = append(cmd, "--files-from", "-")
+	}
+	cmd = append(cmd, "-e", r.RshCommand, source.RsyncPath(), dst)
 	errBuf := &bytes.Buffer{}
-	err := r.LocalExecutor.Execute(cmd, nil, out, errBuf)
+	err := r.LocalExecutor.Execute(cmd, in, out, errBuf)
 	if isExitError(err) {
 		// Check if pod exists
 		if podCheckErr := r.podChecker.CheckPod(); podCheckErr != nil {

pkg/cli/rsync/copy_rsyncd.go

@@ -7,6 +7,7 @@ import (
 	"io"
 	"math/rand"
 	"net"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -67,6 +68,9 @@ type rsyncDaemonStrategy struct {
 	PortForwarder  forwarder
 	LocalExecutor  executor
 
+	Last          uint
+	fileDiscovery fileDiscoverer
+
 	daemonPIDFile   string
 	daemonPort      int
 	localPort       int
@@ -220,20 +224,54 @@ func (s *rsyncDaemonStrategy) stopPortForward() {
 
 func (s *rsyncDaemonStrategy) copyUsingDaemon(source, destination *PathSpec, out, errOut io.Writer) error {
 	klog.V(3).Infof("Copying files with rsync daemon")
+
+	// In case --last is specified, discover the right files and pass them to rsync as an explicit list.
+	var (
+		in  io.Reader
+		dst string
+	)
+	if destination.Local() {
+		dst = destination.RsyncPath()
+	} else {
+		dst = destination.Path
+	}
+	if s.Last > 0 {
+		filenames, err := s.fileDiscovery.DiscoverFiles(source.Path, s.Last)
+		if err != nil {
+			klog.Infof("Warning: failed to apply --last filtering: %v", err)
+		} else {
+			var b bytes.Buffer
+			for _, filename := range filenames {
+				b.WriteString(filename)
+				b.WriteRune('\n')
+			}
+			in = &b
+
+			// Make dst compatible with what rsync does without --last.
+			dst = filepath.Join(dst, filepath.Base(source.Path))
+
+			klog.V(3).Infof("Applied --last=%d to rsync-daemon strategy: using %d files", s.Last, len(filenames))
+		}
+	}
+
 	cmd := append([]string{"rsync"}, s.Flags...)
+	if in != nil {
+		cmd = append(cmd, "--files-from", "-")
+	}
+
 	var sourceArg, destinationArg string
 	if source.Local() {
 		sourceArg = source.RsyncPath()
 	} else {
 		sourceArg = localRsyncURL(s.localPort, remoteLabel, source.Path)
 	}
 	if destination.Local() {
-		destinationArg = destination.RsyncPath()
+		destinationArg = dst
 	} else {
-		destinationArg = localRsyncURL(s.localPort, remoteLabel, destination.Path)
+		destinationArg = localRsyncURL(s.localPort, remoteLabel, dst)
 	}
 	cmd = append(cmd, sourceArg, destinationArg)
-	err := s.LocalExecutor.Execute(cmd, nil, out, errOut)
+	err := s.LocalExecutor.Execute(cmd, in, out, errOut)
 	if err != nil {
 		// Determine whether rsync is present in the pod container
 		testRsyncErr := executeWithLogging(s.RemoteExecutor, testRsyncCommand)
@@ -297,6 +335,8 @@ func NewRsyncDaemonStrategy(o *RsyncOptions) CopyStrategy {
 		RemoteExecutor: remoteExec,
 		LocalExecutor:  newLocalExecutor(),
 		PortForwarder:  forwarder,
+		Last:           o.Last,
+		fileDiscovery:  o.fileDiscovery,
 	}
 }
 

pkg/cli/rsync/copy_tar.go

@@ -30,6 +30,7 @@ type tarStrategy struct {
 	RemoteExecutor executor
 	Includes       []string
 	Excludes       []string
+	NoRecursion    bool
 	IgnoredFlags   []string
 	Flags          []string
 }
@@ -44,15 +45,37 @@ func NewTarStrategy(o *RsyncOptions) CopyStrategy {
 
 	remoteExec := newRemoteExecutor(o)
 
+	// Handle --last option by discovering N most recently modified files.
+	includes := append([]string(nil), o.RsyncInclude...)
+	noRecursion := false
+	if o.Last > 0 {
+		if o.Source.Local() {
+			klog.Info("Warning: --last flag is ignored when creating a local tar file")
+		} else {
+			filenames, err := o.fileDiscovery.DiscoverFiles(o.Source.Path, o.Last)
+			if err != nil {
+				klog.Infof("Warning: failed to apply --last filtering: %v", err)
+			} else {
+				// Replace any existing includes with our filtered list.
+				if len(filenames) > 0 {
+					includes = filenames
+					noRecursion = true
+					klog.V(3).Infof("Applied --last=%d to tar strategy: using %d files", o.Last, len(filenames))
+				}
+			}
+		}
+	}
+
 	return &tarStrategy{
 		Quiet:          o.Quiet,
 		Delete:         o.Delete,
-		Includes:       o.RsyncInclude,
+		Includes:       includes,
 		Excludes:       o.RsyncExclude,
 		Tar:            tarHelper,
 		RemoteExecutor: remoteExec,
 		IgnoredFlags:   ignoredFlags,
 		Flags:          tarFlagsFromOptions(o),
+		NoRecursion:    noRecursion,
 	}
 }
 
@@ -145,7 +168,7 @@ func (r *tarStrategy) Copy(source, destination *PathSpec, out, errOut io.Writer)
 	} else {
 		klog.V(4).Infof("Creating local tar file %s from remote path %s", tmp.Name(), source.Path)
 		errBuf := &bytes.Buffer{}
-		err = tarRemote(r.RemoteExecutor, source.Path, r.Includes, r.Excludes, tmp, errBuf)
+		err = tarRemote(r.RemoteExecutor, source.Path, r.Includes, r.Excludes, r.NoRecursion, tmp, errBuf)
 		if err != nil {
 			if checkTar(r.RemoteExecutor) != nil {
 				return strategySetupError("tar not available in container")
@@ -198,7 +221,7 @@ func (r *tarStrategy) String() string {
 	return "tar"
 }
 
-func tarRemote(exec executor, sourceDir string, includes, excludes []string, out, errOut io.Writer) error {
+func tarRemote(exec executor, sourceDir string, includes, excludes []string, noRecursion bool, out, errOut io.Writer) error {
 	klog.V(4).Infof("Tarring %s remotely", sourceDir)
 
 	exclude := []string{}
@@ -220,7 +243,11 @@ func tarRemote(exec executor, sourceDir string, includes, excludes []string, out
 			include = append(include, path.Join(path.Base(sourceDir), pattern))
 		}
 
-		cmd = []string{"tar", "-C", path.Dir(sourceDir), "-c", path.Base(sourceDir)}
+		cmd = []string{"tar", "-C", path.Dir(sourceDir)}
+		if noRecursion {
+			cmd = append(cmd, "--no-recursion")
+		}
+		cmd = append(cmd, "-c", path.Base(sourceDir))
 		cmd = append(cmd, append(include, exclude...)...)
 	}
 	klog.V(4).Infof("Remote tar command: %s", strings.Join(cmd, " "))

pkg/cli/rsync/copy_tar_test.go

@@ -0,0 +1,77 @@
+package rsync
+
+import (
+	"errors"
+	"sort"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+)
+
+// TestNewTarStrategy_FileDiscovery tests the specific file discovery logic in NewTarStrategy.
+func TestNewTarStrategy_FileDiscovery(t *testing.T) {
+	testCases := []struct {
+		name             string
+		originalIncludes []string
+		discoveredFiles  []string
+		discoveryError   error
+		expectedIncludes []string
+	}{
+		{
+			name:             "discovery finds files - replaces original includes",
+			originalIncludes: []string{"*.log", "*.txt"},
+			discoveredFiles:  []string{"newest.log", "middle.log", "oldest.log"},
+			expectedIncludes: []string{"newest.log", "middle.log", "oldest.log"},
+		},
+		{
+			name:             "discovery finds no files - keeps original includes",
+			originalIncludes: []string{"*.log", "*.txt"},
+			discoveredFiles:  []string{},
+			expectedIncludes: []string{"*.log", "*.txt"},
+		},
+		{
+			name:             "discovery fails - keeps original includes",
+			originalIncludes: []string{"*.log", "*.txt"},
+			discoveryError:   errors.New("command failed"),
+			expectedIncludes: []string{"*.log", "*.txt"},
+		},
+		{
+			name:             "no original includes but discovery finds files",
+			originalIncludes: []string{},
+			discoveredFiles:  []string{"file1.txt", "file2.txt"},
+			expectedIncludes: []string{"file1.txt", "file2.txt"},
+		},
+		{
+			name:             "no original includes and no discovery",
+			originalIncludes: []string{},
+			discoveredFiles:  []string{},
+			expectedIncludes: nil,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Init the strategy.
+			options := &RsyncOptions{
+				RsyncInclude: tc.originalIncludes,
+				Last:         3, // Enable file discovery
+				Source:       &PathSpec{PodName: "test-pod", Path: "/test/path"},
+				Destination:  &PathSpec{Path: "/local/path"},
+				fileDiscovery: &mockFileDiscoverer{
+					files: tc.discoveredFiles,
+					err:   tc.discoveryError,
+				},
+			}
+
+			strategy := NewTarStrategy(options).(*tarStrategy)
+
+			// Verify the result matches expectations.
+			sort.Strings(strategy.Includes)
+			sort.Strings(tc.expectedIncludes)
+			if !cmp.Equal(strategy.Includes, tc.expectedIncludes) {
+				t.Errorf("expected includes mismatch: \n%s\n",
+					cmp.Diff(tc.expectedIncludes, strategy.Includes))
+			}
+		})
+	}
+}

pkg/cli/rsync/discovery.go

@@ -0,0 +1,7 @@
+package rsync
+
+// fileDiscoverer discovers files at the given path,
+// limiting the list to lastN most recently modified files.
+type fileDiscoverer interface {
+	DiscoverFiles(basePath string, lastN uint) ([]string, error)
+}

pkg/cli/rsync/discovery_local.go

@@ -0,0 +1,65 @@
+package rsync
+
+import (
+	"fmt"
+	"os"
+	"sort"
+	"time"
+
+	"k8s.io/klog/v2"
+)
+
+// localFileDiscoverer implements fileDiscoverer interface for local directories.
+type localFileDiscoverer struct{}
+
+func newLocalFileDiscoverer() localFileDiscoverer {
+	return localFileDiscoverer{}
+}
+
+func (discoverer localFileDiscoverer) DiscoverFiles(basePath string, last uint) ([]string, error) {
+	klog.V(4).Infof("Discovering files in local directory %s (last = %d)", basePath, last)
+
+	entries, err := os.ReadDir(basePath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read directory %s: %w", basePath, err)
+	}
+
+	type fileInfo struct {
+		name    string
+		modTime time.Time
+	}
+
+	files := make([]fileInfo, 0, len(entries))
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue // Skip directories, only process regular files.
+		}
+
+		info, err := entry.Info()
+		if err != nil {
+			return nil, fmt.Errorf("failed to get file info for %s: %w", entry.Name(), err)
+		}
+
+		files = append(files, fileInfo{
+			name:    entry.Name(),
+			modTime: info.ModTime(),
+		})
+	}
+
+	// Sort by modification time (newest first).
+	sort.Slice(files, func(i, j int) bool {
+		return files[i].modTime.After(files[j].modTime)
+	})
+
+	// Limit to the latest N files.
+	if len(files) > int(last) {
+		files = files[:last]
+	}
+
+	// Extract just the file names (relative paths).
+	result := make([]string, len(files))
+	for i, file := range files {
+		result[i] = file.name
+	}
+	return result, nil
+}