From 83fe438b849d1938782d4e3898ced496b45d3d19 Mon Sep 17 00:00:00 2001 From: Arthur Date: Fri, 3 Jul 2026 16:52:16 +0200 Subject: [PATCH] fix(photos): chunk exiftool calls to prevent timeout on large imports Exiftool was invoked once with all file paths as CLI arguments. On batches with large files (PSD, TIF), this exceeded the timeout and could hit OS argument length limits. - Process 200 files per chunk with paths piped via stdin (-@ -) - Use -fast flag to skip full file reads (metadata headers only) - Gracefully fall back to filename patterns on timeout --- stacklets/photos/cli/import.py | 47 +++++++++++++++++++++++----------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/stacklets/photos/cli/import.py b/stacklets/photos/cli/import.py index 8621a4a..d596965 100644 --- a/stacklets/photos/cli/import.py +++ b/stacklets/photos/cli/import.py @@ -249,28 +249,45 @@ def _parse_exiftool_output(stdout): return values +_EXIFTOOL_CHUNK = 200 + + def _extract_years(source_root, rel_paths): """Return a dict mapping rel_path -> year string. - Uses exiftool in batch mode for speed. Falls back to filename patterns - for files without EXIF dates. + Uses exiftool in batch mode for speed, processing files in chunks to + avoid timeouts on large batches. File paths are passed via stdin (-@ -) + to avoid OS argument length limits. Falls back to filename patterns for + files without EXIF dates. """ years = {} if shutil.which("exiftool"): - full_paths = [str(Path(source_root) / rel) for rel in rel_paths] - result = subprocess.run( - ["exiftool", "-DateTimeOriginal", "-s3", "-f"] + full_paths, - capture_output=True, text=True, timeout=300, - ) - values = _parse_exiftool_output(result.stdout) - for rel, val in zip(rel_paths, values): - if val and val != "-" and len(val) >= 4: - y = val[:4] - if y.isdigit() and 1990 <= int(y) <= 2039: - years[rel] = y - continue - years[rel] = _year_from_filename(Path(rel).name) + for i in range(0, len(rel_paths), _EXIFTOOL_CHUNK): + chunk_rels = rel_paths[i:i + _EXIFTOOL_CHUNK] + full_paths = [str(Path(source_root) / rel) for rel in chunk_rels] + argfile = "\n".join(full_paths) + "\n" + try: + result = subprocess.run( + ["exiftool", "-fast", "-DateTimeOriginal", "-s3", "-f", + "-@", "-"], + input=argfile, capture_output=True, text=True, timeout=120, + ) + values = _parse_exiftool_output(result.stdout) + except subprocess.TimeoutExpired: + print(f" ! exiftool timed out on chunk {i // _EXIFTOOL_CHUNK + 1}," + " falling back to filenames", file=sys.stderr) + values = [] + for rel, val in zip(chunk_rels, values): + if val and val != "-" and len(val) >= 4: + y = val[:4] + if y.isdigit() and 1990 <= int(y) <= 2039: + years[rel] = y + continue + years[rel] = _year_from_filename(Path(rel).name) + remaining = chunk_rels[len(values):] + for rel in remaining: + years[rel] = _year_from_filename(Path(rel).name) else: for rel in rel_paths: years[rel] = _year_from_filename(Path(rel).name)