Update

Author: svekars

.github/workflows/_build-tutorials-base.yml

@@ -210,7 +210,7 @@ jobs:
         if: always()
         shell: bash
         run: |
-          python -m tools.deprecation_checker.api_report \
+          python3 -m tools.deprecation_checker.api_report \
             --build-log _build/build.log \
             -o _build/api_report.md \
             --create-issue || true

.jenkins/build.sh

@@ -164,13 +164,8 @@ elif [[ "${JOB_TYPE}" == "manager" ]]; then
   # Step 5.1: Run post-processing script on .ipynb files:
   python .jenkins/post_process_notebooks.py
 
-  # Step 5.2: Merge build logs from all workers and create/update GitHub Issue
-  mkdir -p _build
-  for ((worker_id=1;worker_id<NUM_WORKERS+1;worker_id++)); do
-    awsv2 s3 cp s3://${BUCKET_NAME}/${BUILD_PREFIX}/${COMMIT_ID}/build_log_${worker_id}.txt _build/build_log_${worker_id}.txt || true
-  done
-  cat _build/build_log_*.txt > _build/build.log 2>/dev/null || true
-  python -m tools.deprecation_checker.api_report --build-log _build/build.log -o _build/api_report.md --create-issue || true
+  # Step 5.2: API deprecation report is generated by the GHA workflow after
+  # merging worker build logs from artifacts. See _build-tutorials-base.yml.
 
   # Step 6: Copy generated HTML files and static files to S3
   7z a manager.7z docs

tools/deprecation_checker/api_report.py

@@ -30,7 +30,7 @@
 from pathlib import Path
 from typing import List
 
-from .build_warning_parser import BuildWarning, is_tutorial_source, parse_log
+from .build_warning_parser import BuildWarning, classify_dependency, is_tutorial_source, parse_log
 
 # --------------------------------------------------------------------------- #
 # Constants
@@ -117,11 +117,32 @@ def generate_report(warnings: List[BuildWarning]) -> str:
             "",
         ]
 
-    if other_warnings:
+    # Classify dependency warnings
+    pytorch_warnings = [w for w in other_warnings if classify_dependency(w.file) == "pytorch"]
+    pytorch_lib_warnings = [w for w in other_warnings if classify_dependency(w.file) == "pytorch_libs"]
+    third_party_warnings = [w for w in other_warnings if classify_dependency(w.file) == "third_party"]
+
+    if pytorch_warnings:
+        parts += [
+            "## PyTorch warnings",
+            "",
+            _findings_section(pytorch_warnings),
+            "",
+        ]
+
+    if pytorch_lib_warnings:
+        parts += [
+            "## PyTorch libraries warnings",
+            "",
+            _findings_section(pytorch_lib_warnings),
+            "",
+        ]
+
+    if third_party_warnings:
         parts += [
-            "## Warnings from dependencies / non-tutorial code",
+            "## Third-party dependency warnings",
             "",
-            _findings_section(other_warnings),
+            _findings_section(third_party_warnings),
             "",
         ]
 

tools/deprecation_checker/build_warning_parser.py

@@ -11,11 +11,15 @@
 from pathlib import Path
 from typing import List
 
+# Strip ANSI escape sequences and carriage-return progress lines
+_ANSI_RE = re.compile(r"\x1b\[[0-9;]*[A-Za-z]")
+
 # Matches standard Python warning output:
 #   /path/to/file.py:42: DeprecationWarning: some message
+#   <unknown>:2: DeprecationWarning: invalid escape sequence '\s'
 # The message may span continuation lines (indented), but we grab the first line.
 _WARNING_RE = re.compile(
-    r"^(?P<path>.+?\.py):(?P<lineno>\d+):\s+"
+    r"(?P<path>/?[^\s:]+\.py|<unknown>):(?P<lineno>\d+):\s+"
     r"(?P<category>DeprecationWarning|FutureWarning):\s+"
     r"(?P<message>.+)$"
 )
@@ -68,14 +72,20 @@ def parse_log(log_path: str | Path) -> List[BuildWarning]:
     repeating the same warning 50 times.
     """
     log_path = Path(log_path)
-    if not log_path.exists():
+    try:
+        text = log_path.read_text(errors="replace")
+    except FileNotFoundError:
         return []
 
     seen: dict[tuple[str, str], BuildWarning] = {}
     warnings: list[BuildWarning] = []
 
-    for line in log_path.read_text(errors="replace").splitlines():
-        m = _WARNING_RE.match(line)
+    for line in text.splitlines():
+        # Strip ANSI escapes and split on \r to handle progress-line overwriting
+        line = _ANSI_RE.sub("", line)
+        if "\r" in line:
+            line = line.rsplit("\r", 1)[-1]
+        m = _WARNING_RE.search(line)
         if m is None:
             continue
 
@@ -101,3 +111,39 @@ def parse_log(log_path: str | Path) -> List[BuildWarning]:
 def is_tutorial_source(path: str) -> bool:
     """Return True if *path* belongs to a known tutorial source directory."""
     return any(path.startswith(d) for d in _SOURCE_DIRS)
+
+
+# Package prefixes that belong to PyTorch core
+_PYTORCH_CORE_PACKAGES = (
+    "/torch/",
+    "torch/",
+)
+
+# Package prefixes for PyTorch ecosystem libraries
+_PYTORCH_LIB_PACKAGES = (
+    "/torchvision/",
+    "/torchaudio/",
+    "/torchtext/",
+    "/torchrl/",
+    "/tensordict/",
+    "/torchdata/",
+    "/torchtune/",
+    "/torchtitan/",
+    "/functorch/",
+    "/torch_xla/",
+    "/executorch/",
+)
+
+
+def classify_dependency(path: str) -> str:
+    """Classify a non-tutorial warning path into a dependency category.
+
+    Returns one of: ``"pytorch"``, ``"pytorch_libs"``, ``"third_party"``.
+    """
+    for prefix in _PYTORCH_CORE_PACKAGES:
+        if prefix in path:
+            return "pytorch"
+    for prefix in _PYTORCH_LIB_PACKAGES:
+        if prefix in path:
+            return "pytorch_libs"
+    return "third_party"