Add GPU reproduction scripts, eval outputs, and validation sets

.gitignore

@@ -0,0 +1,16 @@
+# Python cache
+__pycache__/
+*.pyc
+
+# Resumable-download temp files
+*.dl
+
+# Large model checkpoints (>100MB, exceed GitHub's file limit).
+# Hosted via GitHub Release assets and on HuggingFace
+# (embedded-language-flows/ELF-B-de-en-torch, ELF-B-xsum-torch) instead.
+ckpts/ELF-B-de-en-torch/
+ckpts/ELF-B-xsum-torch/
+
+# From-scratch pilot training checkpoints (large; keep eval metrics only)
+outputs/scratch_deen_base/
+outputs/scratch_deen_geom/

ckpts/wmt14_de-en_validation_t5/dataset_info.json

@@ -0,0 +1,101 @@
+{
+  "builder_name": "parquet",
+  "citation": "",
+  "config_name": "de-en",
+  "dataset_name": "wmt14",
+  "dataset_size": 1359920533,
+  "description": "",
+  "download_checksums": {
+    "hf://datasets/wmt/wmt14@b199e406369ec1b7634206d3ded5ba45de2fe696/de-en/train-00000-of-00003.parquet": {
+      "num_bytes": 279527247,
+      "checksum": null
+    },
+    "hf://datasets/wmt/wmt14@b199e406369ec1b7634206d3ded5ba45de2fe696/de-en/train-00001-of-00003.parquet": {
+      "num_bytes": 264970514,
+      "checksum": null
+    },
+    "hf://datasets/wmt/wmt14@b199e406369ec1b7634206d3ded5ba45de2fe696/de-en/train-00002-of-00003.parquet": {
+      "num_bytes": 272987200,
+      "checksum": null
+    },
+    "hf://datasets/wmt/wmt14@b199e406369ec1b7634206d3ded5ba45de2fe696/de-en/validation-00000-of-00001.parquet": {
+      "num_bytes": 473798,
+      "checksum": null
+    },
+    "hf://datasets/wmt/wmt14@b199e406369ec1b7634206d3ded5ba45de2fe696/de-en/test-00000-of-00001.parquet": {
+      "num_bytes": 508753,
+      "checksum": null
+    }
+  },
+  "download_size": 818467512,
+  "features": {
+    "input": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "target": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "index": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "condition_input_ids": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "condition_sequence_length": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "input_ids": {
+      "feature": {
+        "dtype": "int32",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "sequence_length": {
+      "dtype": "int64",
+      "_type": "Value"
+    }
+  },
+  "homepage": "",
+  "license": "",
+  "size_in_bytes": 2178388045,
+  "splits": {
+    "train": {
+      "name": "train",
+      "num_bytes": 1358406800,
+      "num_examples": 4508785,
+      "shard_lengths": [
+        1521929,
+        1721928,
+        1264928
+      ],
+      "dataset_name": "wmt14"
+    },
+    "validation": {
+      "name": "validation",
+      "num_bytes": 736407,
+      "num_examples": 3000,
+      "dataset_name": "wmt14"
+    },
+    "test": {
+      "name": "test",
+      "num_bytes": 777326,
+      "num_examples": 3003,
+      "dataset_name": "wmt14"
+    }
+  },
+  "version": {
+    "version_str": "0.0.0",
+    "major": 0,
+    "minor": 0,
+    "patch": 0
+  }
+}

ckpts/wmt14_de-en_validation_t5/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "5f50308c846b2747",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

ckpts/xsum_validation_t5/dataset_info.json

@@ -0,0 +1,88 @@
+{
+  "builder_name": "parquet",
+  "citation": "",
+  "config_name": "default",
+  "dataset_name": "xsum",
+  "dataset_size": 532255381,
+  "description": "",
+  "download_checksums": {
+    "hf://datasets/EdinburghNLP/xsum@7d4d486c2f8ef850b1a11aead99b894ff3dd7da9/data/train-00000-of-00001.parquet": {
+      "num_bytes": 299612457,
+      "checksum": null
+    },
+    "hf://datasets/EdinburghNLP/xsum@7d4d486c2f8ef850b1a11aead99b894ff3dd7da9/data/validation-00000-of-00001.parquet": {
+      "num_bytes": 16447668,
+      "checksum": null
+    },
+    "hf://datasets/EdinburghNLP/xsum@7d4d486c2f8ef850b1a11aead99b894ff3dd7da9/data/test-00000-of-00001.parquet": {
+      "num_bytes": 16731226,
+      "checksum": null
+    }
+  },
+  "download_size": 332791351,
+  "features": {
+    "input": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "target": {
+      "dtype": "string",
+      "_type": "Value"
+    },
+    "index": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "condition_input_ids": {
+      "feature": {
+        "dtype": "int64",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "condition_sequence_length": {
+      "dtype": "int64",
+      "_type": "Value"
+    },
+    "input_ids": {
+      "feature": {
+        "dtype": "int32",
+        "_type": "Value"
+      },
+      "_type": "List"
+    },
+    "sequence_length": {
+      "dtype": "int64",
+      "_type": "Value"
+    }
+  },
+  "homepage": "",
+  "license": "",
+  "size_in_bytes": 865046732,
+  "splits": {
+    "train": {
+      "name": "train",
+      "num_bytes": 479206363,
+      "num_examples": 204045,
+      "dataset_name": "xsum"
+    },
+    "validation": {
+      "name": "validation",
+      "num_bytes": 26292877,
+      "num_examples": 11332,
+      "dataset_name": "xsum"
+    },
+    "test": {
+      "name": "test",
+      "num_bytes": 26756141,
+      "num_examples": 11334,
+      "dataset_name": "xsum"
+    }
+  },
+  "version": {
+    "version_str": "0.0.0",
+    "major": 0,
+    "minor": 0,
+    "patch": 0
+  }
+}

ckpts/xsum_validation_t5/state.json

@@ -0,0 +1,13 @@
+{
+  "_data_files": [
+    {
+      "filename": "data-00000-of-00001.arrow"
+    }
+  ],
+  "_fingerprint": "f934ca16a682895c",
+  "_format_columns": null,
+  "_format_kwargs": {},
+  "_format_type": null,
+  "_output_all_columns": false,
+  "_split": null
+}

download_train_data.sh

@@ -0,0 +1,80 @@
+#!/usr/bin/env bash
+# One-off resilient fetch of the De-En TRAINING set through the flaky root
+# proxy (127.0.0.1:7899). Same strategy as resilient_download.sh: git clone
+# (LFS smudge skipped) for filenames/pointers + resumable curl on /resolve/.
+# Re-run safe; loops forever until every file is verified complete.
+export http_proxy=http://127.0.0.1:7899 https_proxy=http://127.0.0.1:7899 no_proxy=127.0.0.1,localhost
+export GIT_LFS_SKIP_SMUDGE=1
+DEST=/sxl/sxl_code/ELF_pytorch/ckpts
+mkdir -p "$DEST"
+HFCO=https://huggingface.co
+
+log(){ echo "[$(date +%H:%M:%S)] $*"; }
+
+DATA_REPOS=("embedded-language-flows/wmt14_de-en_train_t5")
+
+clone_pointers(){ # repo dir
+  local repo=$1
+  local dir=$2
+  [ -d "$dir/.git" ] && { log "pointers present: $repo"; return 0; }
+  local url="$HFCO/datasets/$repo"
+  while true; do
+    rm -rf "$dir"
+    if git clone --depth 1 "$url" "$dir" >/tmp/clone_train.$$.log 2>&1; then
+      log "cloned pointers: $repo"; return 0
+    fi
+    log "clone $repo failed: $(tail -1 /tmp/clone_train.$$.log | cut -c1-90); retry 15s"
+    sleep 15
+  done
+}
+
+is_pointer(){ head -c 50 "$1" 2>/dev/null | grep -q 'git-lfs.github.com'; }
+
+fetch_one(){ # repo dir relpath
+  local repo=$1
+  local dir=$2
+  local rel=$3
+  local ptr="$dir/$rel" final="$dir/$rel" dl="$dir/$rel.dl"
+  if [ -f "$final" ] && ! is_pointer "$final"; then log "ok (cached) $rel"; return 0; fi
+  local want oid
+  want=$(grep -E '^size '      "$ptr" | awk '{print $2}')
+  oid=$(grep  -E '^oid sha256:' "$ptr" | sed 's/.*sha256://')
+  local base="$HFCO/datasets/$repo/resolve/main"
+  mkdir -p "$(dirname "$dl")"
+  while true; do
+    local have; have=$(stat -c%s "$dl" 2>/dev/null || echo 0)
+    if [ -n "$want" ] && [ "$have" = "$want" ]; then break; fi
+    log "GET $rel have=$have/${want:-?}"
+    curl --proxy http://127.0.0.1:7899 --connect-timeout 8 --max-time 3600 \
+         -C - -L -s -S --speed-limit 2000 --speed-time 30 \
+         -o "$dl" "$base/$rel" 2>/tmp/curl_train.$$.err || true
+    have=$(stat -c%s "$dl" 2>/dev/null || echo 0)
+    if [ -n "$want" ] && [ "$have" = "$want" ]; then break; fi
+    log "  partial $rel have=$have/${want:-?} ($(tail -1 /tmp/curl_train.$$.err 2>/dev/null|cut -c1-50)); retry 8s"
+    sleep 8
+  done
+  if [ -n "$oid" ]; then
+    local got; got=$(sha256sum "$dl" | awk '{print $1}')
+    if [ "$got" != "$oid" ]; then log "CHECKSUM MISMATCH $rel; redownloading"; rm -f "$dl"; fetch_one "$@"; return; fi
+    log "sha256 ok $rel"
+  fi
+  mv -f "$dl" "$final"
+  log "DONE $rel ($(stat -c%s "$final"))"
+}
+
+process_repo(){ # repo
+  local repo=$1
+  local dir="$DEST/$(basename "$repo")"
+  clone_pointers "$repo" "$dir"
+  local f
+  while read -r f; do
+    [ -z "$f" ] && continue
+    if is_pointer "$dir/$f"; then fetch_one "$repo" "$dir" "$f"; fi
+  done < <(cd "$dir" && git ls-files)
+  log "REPO COMPLETE: $repo -> $dir"
+}
+
+log "=== download_train_data start (proxy 127.0.0.1:7899) ==="
+for r in "${DATA_REPOS[@]}"; do process_repo "$r"; done
+log "=== TRAIN DATA DOWNLOADED ==="
+touch "$DEST/.TRAIN_DATA_DONE"