fix: remove redundant and duplicate codes

Author: Chamberlain0w0

scripts/compare_loss.py

@@ -9,24 +9,7 @@
 import sys
 from pathlib import Path
 from argparse import ArgumentParser
-
-
-def collect_log_files(base_dir):
-    """Collect comparable training logs keyed by basename."""
-    files = {}
-    duplicates = {}
-
-    for path in base_dir.rglob('*.log'):
-        if path.name.startswith('build') or path.name.endswith('_profile.log'):
-            continue
-
-        key = path.name
-        if key in files:
-            duplicates.setdefault(key, [files[key]]).append(path)
-            continue
-        files[key] = path
-
-    return files, duplicates
+from compare_utils import collect_log_files, exit_if_duplicate_logs
 
 def get_dtype_from_filename(filename):
     """Determine dtype from filename. Returns 'bfloat16' or 'fp32'."""
@@ -82,18 +65,8 @@ def main():
 
     files1, duplicates1 = collect_log_files(args.dir1)
     files2, duplicates2 = collect_log_files(args.dir2)
-
-    if duplicates1:
-        print(f"Found duplicate log basenames in {args.dir1.resolve()}, cannot compare safely:")
-        for name, paths in sorted(duplicates1.items()):
-            print(f"  {name}: {', '.join(str(p.relative_to(args.dir1)) for p in paths)}")
-        sys.exit(1)
-
-    if duplicates2:
-        print(f"Found duplicate log basenames in {args.dir2.resolve()}, cannot compare safely:")
-        for name, paths in sorted(duplicates2.items()):
-            print(f"  {name}: {', '.join(str(p.relative_to(args.dir2)) for p in paths)}")
-        sys.exit(1)
+    exit_if_duplicate_logs(args.dir1, duplicates1)
+    exit_if_duplicate_logs(args.dir2, duplicates2)
 
     only_in_1 = set(files1.keys()) - set(files2.keys())
     only_in_2 = set(files2.keys()) - set(files1.keys())

scripts/compare_tps.py

@@ -9,24 +9,7 @@
 import sys
 from pathlib import Path
 from argparse import ArgumentParser
-
-
-def collect_log_files(base_dir):
-    """Collect comparable training logs keyed by basename."""
-    files = {}
-    duplicates = {}
-
-    for path in base_dir.rglob('*.log'):
-        if path.name.startswith('build') or path.name.endswith('_profile.log'):
-            continue
-
-        key = path.name
-        if key in files:
-            duplicates.setdefault(key, [files[key]]).append(path)
-            continue
-        files[key] = path
-
-    return files, duplicates
+from compare_utils import collect_log_files, exit_if_duplicate_logs
 
 def parse_log(file_path):
     """Extract step -> tok/s mapping from log file."""
@@ -75,18 +58,8 @@ def main():
 
     files1, duplicates1 = collect_log_files(args.dir1)
     files2, duplicates2 = collect_log_files(args.dir2)
-
-    if duplicates1:
-        print(f"Found duplicate log basenames in {args.dir1.resolve()}, cannot compare safely:")
-        for name, paths in sorted(duplicates1.items()):
-            print(f"  {name}: {', '.join(str(p.relative_to(args.dir1)) for p in paths)}")
-        sys.exit(1)
-
-    if duplicates2:
-        print(f"Found duplicate log basenames in {args.dir2.resolve()}, cannot compare safely:")
-        for name, paths in sorted(duplicates2.items()):
-            print(f"  {name}: {', '.join(str(p.relative_to(args.dir2)) for p in paths)}")
-        sys.exit(1)
+    exit_if_duplicate_logs(args.dir1, duplicates1)
+    exit_if_duplicate_logs(args.dir2, duplicates2)
 
     only_in_1 = set(files1.keys()) - set(files2.keys())
     only_in_2 = set(files2.keys()) - set(files1.keys())

scripts/run_models_and_profile.bash

@@ -5,20 +5,29 @@ set -o pipefail
 
 usage() {
     cat <<'EOF'
-Usage: run_models_and_profile.bash [config_file] [--only-run tag1,tag2]
+Usage: run_models_and_profile.bash [--test-config path] [--only-run tag1,tag2]
 
 Options:
+  --test-config PATH  Path to test config JSON. Default: test_config.json.
   --only-run TAGS   Only run the specified tag groups, separated by commas.
   -h, --help        Show this help message.
 EOF
 }
 
 CONFIG_FILE="test_config.json"
 ONLY_RUN_TAGS=""
-CONFIG_FILE_SET="no"
 
 while [[ $# -gt 0 ]]; do
     case "$1" in
+        --test-config)
+            [[ $# -lt 2 ]] && { echo "Error: --test-config requires a file path."; exit 1; }
+            CONFIG_FILE="$2"
+            shift 2
+            ;;
+        --test-config=*)
+            CONFIG_FILE="${1#*=}"
+            shift
+            ;;
         --only-run)
             [[ $# -lt 2 ]] && { echo "Error: --only-run requires a comma-separated tag list."; exit 1; }
             ONLY_RUN_TAGS="$2"
@@ -38,14 +47,9 @@ while [[ $# -gt 0 ]]; do
             exit 1
             ;;
         *)
-            if [[ "$CONFIG_FILE_SET" == "yes" ]]; then
-                echo "Error: Multiple config files provided."
-                usage
-                exit 1
-            fi
-            CONFIG_FILE="$1"
-            CONFIG_FILE_SET="yes"
-            shift
+            echo "Error: Unknown positional argument: $1"
+            usage
+            exit 1
             ;;
     esac
 done