From 1605c3f12ea1ee4f4904c9527f5a89d63175b85c Mon Sep 17 00:00:00 2001 From: Caio Lins Date: Sun, 25 Oct 2026 13:53:46 +0900 Subject: [PATCH] tooling: move LINE reverse-engineering agent skills to beeper-line-skills They are optional and now live at https://github.com/clins1994/beeper-line-skills, maintained separately. Remove .agents/, the .claude/commands symlink and the capture/parse/redact scripts, and add an optional pointer to README.md and AGENTS.md. Co-Authored-By: Claude Sonnet 5.5 --- .agents/commands/line-analyze.md | 48 --- .agents/commands/line-capture.md | 38 -- .agents/commands/line-implement.md | 93 ----- .agents/skills/line-implement/REFERENCE.md | 170 --------- .agents/skills/line-implement/SKILL.md | 72 ---- .claude/commands | 1 - AGENTS.md | 10 + README.md | 13 + scripts/line-cdp-capture.py | 387 --------------------- scripts/line-chrome-capture.sh | 361 ------------------- scripts/parse-line-traffic.py | 336 ------------------ scripts/redact-line-capture.py | 143 -------- 12 files changed, 23 insertions(+), 1649 deletions(-) delete mode 100644 .agents/commands/line-analyze.md delete mode 100644 .agents/commands/line-capture.md delete mode 100644 .agents/commands/line-implement.md delete mode 100644 .agents/skills/line-implement/REFERENCE.md delete mode 100644 .agents/skills/line-implement/SKILL.md delete mode 120000 .claude/commands delete mode 100755 scripts/line-cdp-capture.py delete mode 100755 scripts/line-chrome-capture.sh delete mode 100755 scripts/parse-line-traffic.py delete mode 100755 scripts/redact-line-capture.py diff --git a/.agents/commands/line-analyze.md b/.agents/commands/line-analyze.md deleted file mode 100644 index 98554177..00000000 --- a/.agents/commands/line-analyze.md +++ /dev/null @@ -1,48 +0,0 @@ -Parse and analyze a LINE Chrome Extension network capture. - -## Input - -Use the provided path to a net-log JSON file. If no path is provided, default to -`/tmp/line-chrome-extension-capture/line-net-log.json`. - -## What to do - -1. Run the parser: - ``` - python3 scripts/parse-line-traffic.py - ``` - This extracts LINE-related HTTP requests from the Chrome net-log and writes a `-parsed.json` file. - -2. Read the parsed output JSON file. - -3. Analyze the traffic and present findings organized by category: - - - **Thrift RPC** (`/api/talk/thrift/`): The main LINE API. Requests use LINE's Thrift-over-HTTP protocol. The `x-lhm` header indicates the Thrift method name. Compare against methods in `pkg/line/methods.go`. - - - **Long-poll / SSE** (`/api/talk/long-polling/`, `/api/operation/receive`): Real-time event delivery. LF1 is used during login verification, JQ for login polling, and SSE for ongoing message/operation streaming. Compare against `pkg/line/sse.go` and `pkg/line/client.go`. - - - **Auth** (`/api/auth/`): Token refresh and login flows. Compare against login methods in `pkg/line/client.go` (loginV2, confirmE2EELogin, tokenRefresh). - - - **Media/OBS** (`obs.line-apps.com`): Media upload/download. Check `x-obs-params` header for operation details, `x-obs-oid` in responses. Compare against `pkg/line/client.go` (UploadOBS, DownloadOBS). - - - **Stickers/CDN** (`stickershop`, `scdn`): Sticker image downloads. - - - **Secondary API** (`legy-jp.line-apps.com`): Link previews and page info. Compare against `pkg/line/client.go` (GetPageInfo). - -4. For each interesting request, explain: - - What the bridge equivalent is (or note if it's not implemented yet) - - The request/response structure - - Any headers that carry protocol-specific meaning - -5. Highlight any **unimplemented** endpoints or behaviors that differ from what the bridge currently does. Cross-reference with `pkg/line/methods.go` and `pkg/connector/`. - -6. After presenting the analysis, ask the user if they want to clean up the capture files (`/tmp/line-chrome-extension-capture/`). Warn that the raw net-log and parsed output contain sensitive data (auth tokens in `X-Line-Access` headers, message IDs, etc.) and should not be left on disk or committed to the repository. If the user agrees, delete the entire capture directory. - -## LINE protocol context - -- The bridge identifies as LINE Chrome Extension: header `x-line-application` contains the app version string. -- Thrift RPC uses HTTP POST to `line-chrome-gw.line-apps.com/api/talk/thrift/Talk//`. -- Auth token is in `x-line-access` header. -- E2EE messages have `chunks` field containing encrypted content (base64 segments). -- Operations have numeric `type` codes — see `pkg/connector/sync.go` and `pkg/connector/handle_message.go` for the operation type switch. -- Body content in the parsed output may be base64-encoded binary (Thrift serialization). Note the method name from headers rather than trying to decode the binary body. diff --git a/.agents/commands/line-capture.md b/.agents/commands/line-capture.md deleted file mode 100644 index b0ecd69c..00000000 --- a/.agents/commands/line-capture.md +++ /dev/null @@ -1,38 +0,0 @@ -Launch Chrome with network capture enabled for LINE Chrome Extension analysis. - -## What to do - -1. **Close any running Chrome instances first** (Chrome only supports one instance per profile with remote debugging). Warn the user if Chrome is running. - -2. Run the capture script: - ``` - bash scripts/line-chrome-capture.sh /tmp/line-chrome-extension-capture - ``` - This launches Chrome with: - - Remote debugging on port 9222 - - Full network logging to `/tmp/line-chrome-extension-capture/line-net-log.json` - - A persistent capture-only Chrome profile at - `~/.cache/line-chrome-extension-capture/chrome-debug-profile` - - Auto-loaded LINE and Codex extensions staged from the user's normal Chrome - profile into `~/.cache/line-chrome-extension-capture/extensions` - - A LINE extension version preflight: - - repo version from `pkg/line/client.go` `ExtensionVersion` - - installed/staged Chrome extension version from `manifest.json` - - a warning if those versions differ - - First-run Chrome prompts pre-skipped: - - "Make Google Chrome the default browser" stays unchecked - - "Automatically send usage statistics and crash reports" stays unchecked - - Chrome/Sync sign-in is skipped; use "Stay Signed Out" if it appears - - If the LINE extension is missing and the user asked for setup, relaunch with - `LINE_CAPTURE_AUTO_LOAD_EXTENSIONS=0`, open the official Web Store listing - for `ophjlpahpchlmihnnnihgmmeilfjmjjc`, verify LINE / LY Corporation, click - Add to Chrome, and accept the standard "Add extension" confirmation - -3. Tell the user: - - Open the LINE Chrome Extension (browser toolbar icon) - - Log in manually if the capture profile is not already logged in - - Perform whatever flow they want to analyze - - When done, quit Chrome completely (Cmd+Q) - -4. Once Chrome is closed, the net-log file is ready. Continue with the - line-analyze workflow. diff --git a/.agents/commands/line-implement.md b/.agents/commands/line-implement.md deleted file mode 100644 index 09bf18b1..00000000 --- a/.agents/commands/line-implement.md +++ /dev/null @@ -1,93 +0,0 @@ -Implement a matrix-line feature by capturing and mimicking LINE Chrome Extension behavior. - -## Input - -Use the provided feature goal, for example: - -``` -Support Matrix group avatar changes by matching LINE Chrome Extension traffic. -``` - -## What to do - -1. **Understand the goal and the current bridge first.** - - Read `AGENTS.md`/`CLAUDE.md`, `docs/protocol/README.md`, relevant endpoint docs, and `docs/protocol/gap-analysis.md`. - - Inspect the likely implementation surface in `pkg/line/`, `pkg/connector/`, and existing tests. - - If existing protocol docs already answer the feature, implement from docs and code without a new capture. - -2. **Write a capture contract before launching Chrome.** - Capture only the missing evidence: - - exact user-visible LINE action to perform - - expected Talk/Auth/OBS/SSE endpoints and method names - - request argument shape and ordering - - response wrapper shape, headers, and error behavior - - resulting SSE/operation events, revisions, message metadata, and local state changes - - what values the bridge must persist or expose afterward - -3. **Check LINE Chrome Extension version drift.** - - Compare the installed/staged extension `manifest.json` version against - `pkg/line/client.go` `ExtensionVersion`. - - Also check literal `x-line-application` versions in `pkg/line/client.go` - and the default `clientVersion` in `pkg/runner.go`. - - The capture launcher prints the repo and Chrome extension versions and - warns on mismatch. Do not silently mix capture evidence from a different - LINE Chrome Extension version. - -4. **Security preflight.** - - Treat raw captures as secrets: they can contain access tokens, cookies, MIDs, message contents, contact data, and E2EE metadata. - - Use a fresh `/tmp/line-chrome-extension-capture/-` directory with `umask 077`. - - Do not commit raw capture files, Chrome profiles, screenshots, QR codes, or parsed JSON. - - Do not request, store, or automate the LINE password. - - The user performs LINE login manually, including email/password, QR, PIN, 2FA, and mobile approval. - - Do not run the bridge and LINE Chrome Extension simultaneously unless the user explicitly accepts session invalidation. - - Prefer the persistent capture Chrome profile from `scripts/line-chrome-capture.sh`. Use the user's normal Chrome profile only after explicit approval. - -5. **Run the capture.** - - Check for running Chrome and `matrix-line` processes; warn before closing or stopping anything. - - Start capture: - ``` - umask 077 - CAPTURE_DIR="/tmp/line-chrome-extension-capture/$(date +%Y%m%d-%H%M%S)-" - bash scripts/line-chrome-capture.sh "$CAPTURE_DIR" - ``` - - Use Codex Chrome automation when available to drive the already-logged-in extension UI. If it cannot access the capture profile or extension UI, use Computer Use. - - The launcher stages clean unpacked LINE and Codex extension copies from the user's normal Chrome profile and auto-loads them when possible. - - The launcher pre-skips Chrome's first-run prompts: default browser is off, - usage/crash reporting is off, and Chrome/Sync sign-in should be skipped. - If the sign-in screen still appears, choose "Stay Signed Out". - - If Chrome blocks unpacked loading or LINE is missing from the capture - profile, relaunch setup with `LINE_CAPTURE_AUTO_LOAD_EXTENSIONS=0`, open - the official Web Store listing - `https://chromewebstore.google.com/detail/line/ophjlpahpchlmihnnnihgmmeilfjmjjc`, - verify LINE / LY Corporation, click Add to Chrome, and accept the standard - "Add extension" confirmation when the user has requested setup. - - If Chrome is not logged in, pause and ask the user to complete LINE login manually in the capture browser. - - Perform the smallest LINE UI flow that satisfies the capture contract, then quit Chrome completely. - -6. **Parse, redact, and analyze.** - - Prefer CDP output because it includes SSE messages: - ``` - python3 scripts/parse-line-traffic.py "$CAPTURE_DIR/line-cdp-capture.json" "$CAPTURE_DIR/line-cdp-capture-parsed.json" - python3 scripts/redact-line-capture.py "$CAPTURE_DIR/line-cdp-capture-parsed.json" - ``` - - Read raw files only locally and only when redacted output loses information needed for implementation. - - Quote or document only redacted snippets. - - Compare against `pkg/line/methods.go`, `pkg/line/client.go`, `pkg/line/sse.go`, and the relevant `pkg/connector/` flow. - - Produce an evidence table: Chrome behavior, bridge behavior, implementation delta, files to change. - -7. **Implement religiously.** - - Match the Chrome Extension's endpoint path, service/method name, argument order, headers, request body shape, response handling, and fallback behavior. - - Preserve existing bridge architecture: Thrift/API code in `pkg/line`, Matrix-facing behavior in `pkg/connector`, E2EE behavior in `pkg/e2ee`/`pkg/ltsm`. - - Update protocol docs only with redacted request/response/event evidence. - - Add or update focused tests for parsing, request construction, event handling, persistence, and fallback/error behavior. - -8. **Verify and clean up.** - - Run: - ``` - go fmt ./... - goimports -local "github.com/beeper/line" -w . - go test ./... - go vet $(go list ./... | grep -v /ltsm) - ``` - - If available, run `staticcheck $(go list ./... | grep -v /ltsm)`. - - Before finishing, report where raw captures live and ask whether to delete them. Prefer keeping only redacted notes/docs. diff --git a/.agents/skills/line-implement/REFERENCE.md b/.agents/skills/line-implement/REFERENCE.md deleted file mode 100644 index 649075ea..00000000 --- a/.agents/skills/line-implement/REFERENCE.md +++ /dev/null @@ -1,170 +0,0 @@ -# Line Implement Reference - -## Repo Orientation - -Start with: - -- `AGENTS.md` / `CLAUDE.md` -- `.agents/commands/line-capture.md` -- `.agents/commands/line-analyze.md` -- `.agents/commands/line-implement.md` -- `docs/protocol/README.md` -- `docs/protocol/endpoints/README.md` -- `docs/protocol/gap-analysis.md` -- `pkg/line/methods.go`, `pkg/line/client.go`, `pkg/line/sse.go` -- likely `pkg/connector/*` files for the Matrix-facing feature - -Do not capture if the current protocol docs already provide enough evidence. -Use `.agents` as the repo-local source of truth. `.claude/commands` is a -compatibility symlink for Claude-style command discovery when present. - -## Capture Contract - -Before launching Chrome, write down: - -- the exact LINE UI action to perform -- fixture requirements such as DM/group, `c...`/`r...` chat MID type, media type, - old/new state, or Letter Sealing state -- expected endpoint families: TalkService, AuthService, LoginQrCode, OBS, SSE, - CDN, or secondary APIs -- unknowns needed for implementation: method name, URL, argument order, request - body shape, response wrapper, headers, operation events, revision changes, - persisted IDs, and fallback/error behavior - -Keep the UI flow narrow. Avoid broad exploratory captures unless the protocol -surface is genuinely unknown. - -## Version Preflight - -Before trusting capture evidence, compare the repo's LINE Chrome Extension -version with the installed extension: - -```bash -rg -n 'ExtensionVersion|x-line-application|clientVersion' pkg/line/client.go pkg/runner.go -find "$HOME/.cache/line-chrome-extension-capture/chrome-debug-profile/Default/Extensions/ophjlpahpchlmihnnnihgmmeilfjmjjc" \ - -maxdepth 2 -type f -name manifest.json -print -``` - -The canonical repo version is `pkg/line/client.go` `ExtensionVersion`. Also -check hardcoded `x-line-application` header strings in `pkg/line/client.go` and -the default `clientVersion` in `pkg/runner.go`; those must not drift from the -extension manifest version. The capture launcher prints repo and Chrome LINE -versions at startup and warns on mismatch. If versions differ, do not silently -mix evidence: warn the user and either update the repo version constants/headers -or record that the capture was taken from a different LINE Chrome Extension -version. - -## Security Rules - -Raw LINE captures are secrets. They may include access tokens, cookies, session -IDs, MIDs, contacts, message bodies, media object IDs, QR/login artifacts, and -E2EE metadata. - -- Use `/tmp/line-chrome-extension-capture/-` with `umask 077`. -- Prefer the persistent dedicated Chrome profile created by `scripts/line-chrome-capture.sh` - at `~/.cache/line-chrome-extension-capture/chrome-debug-profile`. -- Ask before using the user's normal Chrome profile. -- Ask before stopping a running bridge or invalidating an active LINE session. -- Keep remote debugging local and close Chrome after capture. -- If the user asks to set up the capture browser, installing the official LINE - Chrome Extension is in scope. Verify extension ID - `ophjlpahpchlmihnnnihgmmeilfjmjjc`, listing name LINE, and publisher/offered - by LY Corporation before clicking Add to Chrome and accepting the standard - "Add extension" confirmation. -- Do not request, store, or automate LINE credentials. -- Never ask the user to paste their LINE password into chat, command arguments, - files, or terminal output. -- If the capture browser is not logged in, pause and ask the user to complete - LINE login manually in Chrome, including email/password, QR, PIN, 2FA, and - mobile approval. -- Never commit raw captures, Chrome profiles, screenshots, QR codes, parsed JSON, - or redacted files unless the user explicitly asks for sanitized docs. -- Quote/document only redacted snippets. - -## Capture Commands - -Use CDP capture first because it includes SSE event bodies: - -```bash -umask 077 -CAPTURE_DIR="/tmp/line-chrome-extension-capture/$(date +%Y%m%d-%H%M%S)-feature-slug" -bash scripts/line-chrome-capture.sh "$CAPTURE_DIR" -``` - -The launcher stages clean unpacked copies of the LINE and Codex Chrome extension -directories from the user's normal Chrome profile when they are installed there, -then auto-loads those staged copies. When LINE is already installed in the -persistent capture profile, the launcher uses that installed copy and only -auto-loads other staged extensions; in that mode it must not pass -`--disable-extensions-except` for only the staged helpers, because that disables -the installed LINE extension. Override detection with -`LINE_CHROME_EXTENSION_DIR` or `CODEX_CHROME_EXTENSION_DIR` when needed. -If unpacked loading is blocked by Chrome, use the persistent capture profile to -open the official LINE Chrome Web Store listing and install it there; start this -setup run with `LINE_CAPTURE_AUTO_LOAD_EXTENSIONS=0` so Chrome is not also -loading a staged unpacked extension with the same ID. Accept the expected -extension confirmation only after the user has requested setup. -It also pre-seeds the dedicated capture profile so Chrome first-run/sign-in -prompts do not interrupt capture: default-browser opt-in is off, usage/crash -reporting is off, and Chrome/Sync stays signed out. If Chrome still shows the -sign-in screen, choose "Stay Signed Out" and continue to LINE login only. - -When Chrome closes: - -```bash -python3 scripts/parse-line-traffic.py "$CAPTURE_DIR/line-cdp-capture.json" "$CAPTURE_DIR/line-cdp-capture-parsed.json" -python3 scripts/redact-line-capture.py "$CAPTURE_DIR/line-cdp-capture-parsed.json" -``` - -Use the net-log only as a backup: - -```bash -python3 scripts/parse-line-traffic.py "$CAPTURE_DIR/line-net-log.json" "$CAPTURE_DIR/line-net-log-parsed.json" -python3 scripts/redact-line-capture.py "$CAPTURE_DIR/line-net-log-parsed.json" -``` - -## Analysis Checklist - -Build an evidence table with: - -- ordered requests and timestamps -- endpoint URL and service/method -- meaningful headers such as `x-lhm`, `x-lpv`, `x-lsr`, OBS headers, and content type -- request argument shape and exact order -- response status, wrapper code/message/data, and important headers -- SSE/operation event type, params, revisions, message metadata, LOC_KEYs, and - localRev/chat revision changes -- Chrome behavior vs bridge behavior -- implementation delta and files to change - -For Thrift payloads, prefer structured decoded data when available. If only -opaque binary/base64 is available, use the URL, service/method, headers, and -observable response/event behavior, then map to existing Go request builders. - -## Implementation Rules - -- Put LINE API/client behavior in `pkg/line`. -- Put Matrix bridge behavior and portal/message handling in `pkg/connector`. -- Keep E2EE behavior in `pkg/e2ee` and avoid editing generated `pkg/ltsm/wbc_generated.go`. -- Match Chrome's method names, endpoint paths, arg order, header semantics, - body shape, retry/fallback behavior, and event interpretation. -- Prefer existing request helpers and local patterns over new abstractions. -- Add protocol docs for new endpoint behavior using only redacted evidence. -- Add focused tests for request construction, response parsing, event handling, - persistence, and user-visible bridge behavior. - -## Verification - -Run the relevant subset, then the broader checks when practical: - -```bash -go fmt ./... -goimports -local "github.com/beeper/line" -w . -go test ./... -go vet $(go list ./... | grep -v /ltsm) -staticcheck $(go list ./... | grep -v /ltsm) -``` - -If a command is unavailable or too slow, report that clearly. - -At the end, state where raw captures remain and ask whether to delete them. diff --git a/.agents/skills/line-implement/SKILL.md b/.agents/skills/line-implement/SKILL.md deleted file mode 100644 index d4d0a8a2..00000000 --- a/.agents/skills/line-implement/SKILL.md +++ /dev/null @@ -1,72 +0,0 @@ ---- -name: line-implement -description: Plans and implements matrix-line bridge features by inspecting bridge code, deriving the missing LINE Chrome Extension evidence, securely capturing/analyzing traffic, and coding behavior to match Chrome. Use when implementing LINE protocol features, mimicking LINE Chrome Extension APIs, or investigating matrix-line behavior from network captures. ---- - -# Line Implement - -## Quick Start - -Use this for the Matrix LINE bridge when a feature goal depends on exact LINE -Chrome Extension behavior. - -1. Confirm the repo is `beeper-line`/`matrix-line-messenger`; read `AGENTS.md`. -2. Treat `.agents` as the repo-local source of truth. Read - `.agents/commands/line-implement.md` if present. `.claude/commands` may exist - only as a compatibility symlink to `.agents/commands`. -3. Load [REFERENCE.md](REFERENCE.md) for the full workflow. -4. Check LINE Chrome Extension version drift before capture: compare the repo - `pkg/line/client.go` `ExtensionVersion` and hardcoded LINE application - header versions against the installed extension `manifest.json`. -5. First inspect code and `docs/protocol`; capture only unknown behavior. -6. Treat all raw captures as secrets and work from redacted outputs whenever possible. - -## Required Loop - -1. Convert the user's feature goal into a capture contract. -2. Compare existing protocol docs and bridge implementation. -3. Use secure Chrome/CDP capture only for missing evidence. -4. Parse and redact captures before quoting, documenting, or sharing details. -5. Implement the bridge to match Chrome's endpoint, headers, argument shape, - response handling, events, and fallback behavior. -6. Run focused tests plus repo formatting/linting commands. - -## Automation - -For Chrome UI work, prefer the Codex Chrome connector when it can access the -needed browser/profile. If it cannot operate the LINE extension UI, use -Computer Use. - -The capture launcher should pre-skip Chrome first-run prompts. Keep the default -browser checkbox off, usage/crash reporting off, and Chrome/Sync signed out. If -Chrome still shows the sign-in page, choose "Stay Signed Out"; only LINE login is -manual. - -Before trusting a capture, check the installed LINE Chrome Extension version -against the repo version. The repo source is `pkg/line/client.go` -`ExtensionVersion`; also scan for literal `x-line-application` versions in -`pkg/line/client.go` and the default client version in `pkg/runner.go`. The -installed extension version comes from -`~/.cache/line-chrome-extension-capture/chrome-debug-profile/Default/Extensions/ophjlpahpchlmihnnnihgmmeilfjmjjc/*/manifest.json` -or the staged extension manifest. If they differ, warn the user and either -update the repo version constants/headers or note that the evidence is from a -different Chrome Extension version. - -The capture launcher stages clean unpacked LINE/Codex extension copies from the -user's normal Chrome profile before loading them, because Chrome Web Store -install directories contain metadata that unpacked-extension loading rejects. -If LINE is already installed in the persistent capture profile, the launcher -uses that installed copy instead of injecting a staged LINE copy. -When installing LINE from the Chrome Web Store into the persistent capture -profile, run the launcher with `LINE_CAPTURE_AUTO_LOAD_EXTENSIONS=0` so the -Web Store is not fighting an already command-line-loaded extension ID. - -If the LINE extension is missing from the capture Chrome profile and the user -has asked to set it up, open the official Chrome Web Store listing for extension -ID `ophjlpahpchlmihnnnihgmmeilfjmjjc` (LINE, offered by LY Corporation), click -Add to Chrome, and accept the standard "Add extension" confirmation. Do not -accept prompts for any other extension or publisher without a new user request. - -Do not request, store, or automate LINE credentials. If the extension is not -logged in, pause and ask the user to complete login manually in the capture -browser, including email/password, QR, PIN, 2FA, and mobile approval. diff --git a/.claude/commands b/.claude/commands deleted file mode 120000 index 1ea3574a..00000000 --- a/.claude/commands +++ /dev/null @@ -1 +0,0 @@ -../.agents/commands \ No newline at end of file diff --git a/AGENTS.md b/AGENTS.md index fad5dd3a..7c49e559 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -80,3 +80,13 @@ Message encryption is hybrid: ## Logging Uses `zerolog`. Pre-commit hooks enforce: no `Msgf` (use `Msg` with structured fields), use `Stringer` interface where applicable. + +## Optional Agent Skills + +Not required for anything in this repo. If you want agent help reverse +engineering the LINE Chrome Extension (capturing and analyzing its traffic, and +later structuring the evidence), there is an optional set of skills in +https://github.com/clins1994/beeper-line-skills. It is maintained separately by +@clins1994 and may change independently. Install with +`npx skills add clins1994/beeper-line-skills`, then run the skills from inside +this checkout. diff --git a/README.md b/README.md index 732232fe..aed51534 100644 --- a/README.md +++ b/README.md @@ -336,3 +336,16 @@ automatically. For details on how the bridge handles different Letter Sealing combinations, see [readme/LETTER_SEALING.md](readme/LETTER_SEALING.md). + +## Optional: agent skills for reverse engineering + +This is completely optional. If you want help figuring out what the LINE Chrome +Extension does (capturing and analyzing its traffic), and later an opinionated +way to structure the evidence for that kind of work, you can try +[clins1994/beeper-line-skills](https://github.com/clins1994/beeper-line-skills). +Nothing in this repo needs it. It is maintained separately by +[@clins1994](https://github.com/clins1994) and will change on its own schedule. + +```bash +npx skills add clins1994/beeper-line-skills +``` diff --git a/scripts/line-cdp-capture.py b/scripts/line-cdp-capture.py deleted file mode 100755 index 142befe9..00000000 --- a/scripts/line-cdp-capture.py +++ /dev/null @@ -1,387 +0,0 @@ -#!/usr/bin/env python3 -"""Capture LINE traffic via Chrome DevTools Protocol. - -Connects to Chrome's remote debugging port and captures: -- SSE EventSource messages (the critical piece net-log misses) -- Thrift RPC request/response bodies -- All LINE-domain request headers - -Usage: - # First, launch Chrome with --remote-debugging-port=9222 - # Then: - python3 scripts/line-cdp-capture.py [output.json] - - # Or use the wrapper: - bash scripts/line-chrome-capture.sh [output_dir] -""" - -import json -import sys -import os -import signal -import time -import base64 -import threading -from collections import defaultdict -from datetime import datetime, timezone -from urllib.parse import urlparse - -import websocket - -LINE_DOMAINS = [ - "line-chrome-gw.line-apps.com", - "obs.line-apps.com", - "legy-jp.line-apps.com", - "gd2.line.naver.jp", - "gwx.line.naver.jp", - "gw.line.naver.jp", - "api.line.me", - "stickershop.line-scdn.net", - "profile.line-scdn.net", - "obs.line-scdn.net", - "scdn.line-apps.com", -] - -CDP_PORT = 9222 - - -def is_line_url(url: str) -> bool: - for domain in LINE_DOMAINS: - if domain in url: - return True - return False - - -def timestamp() -> str: - return datetime.now(timezone.utc).isoformat() - - -class CDPCapture: - def __init__(self, output_path: str): - self.output_path = output_path - self.ws = None - self.msg_id = 0 - self.lock = threading.Lock() - - # Captured data - self.requests = {} # requestId -> request info - self.sse_messages = [] # EventSource messages - self.thrift_responses = {} # requestId -> response body - self.request_bodies = {} # requestId -> request body - self.pending_body_requests = {} # msg_id -> (requestId, session_id) - self.sessions = {} # sessionId -> targetInfo - - self.running = True - - def next_id(self) -> int: - with self.lock: - self.msg_id += 1 - return self.msg_id - - def send(self, method: str, params: dict = None) -> int: - mid = self.next_id() - msg = {"id": mid, "method": method} - if params: - msg["params"] = params - self.ws.send(json.dumps(msg)) - return mid - - def connect(self): - """Connect to Chrome CDP browser target and auto-attach to all targets. - - Extensions run in their own target (service_worker/background_page), - so we connect at the browser level and use Target.setAutoAttach to - intercept network events from every target including extensions. - """ - import urllib.request - - # Get browser-level WebSocket URL - try: - resp = urllib.request.urlopen(f"http://127.0.0.1:{CDP_PORT}/json/version") - version_info = json.loads(resp.read()) - ws_url = version_info["webSocketDebuggerUrl"] - except Exception as e: - print(f"Cannot connect to Chrome CDP on port {CDP_PORT}: {e}", file=sys.stderr) - print("Make sure Chrome is running with --remote-debugging-port=9222", file=sys.stderr) - sys.exit(1) - - # Also list targets for debugging - try: - resp = urllib.request.urlopen(f"http://127.0.0.1:{CDP_PORT}/json") - targets = json.loads(resp.read()) - for t in targets: - print(f" Target: [{t.get('type')}] {t.get('title', '?')} ({t.get('url', '')})", file=sys.stderr) - except Exception: - pass - - print(f"Connecting to browser target: {ws_url}", file=sys.stderr) - - self.ws = websocket.WebSocket() - self.ws.connect(ws_url) - - # Auto-attach to ALL targets (pages, extensions, service workers, etc.) - # This makes Network events from extension contexts flow through our connection. - self.send("Target.setAutoAttach", { - "autoAttach": True, - "waitForDebuggerOnStart": False, - "flatten": True, - }) - - # Also discover existing targets - self.send("Target.setDiscoverTargets", { - "discover": True, - }) - - print("CDP connected to browser. Waiting for targets...", file=sys.stderr) - - def send_to_session(self, session_id: str, method: str, params: dict = None) -> int: - """Send a CDP command to a specific session (attached target).""" - mid = self.next_id() - msg = {"id": mid, "method": method, "sessionId": session_id} - if params: - msg["params"] = params - self.ws.send(json.dumps(msg)) - return mid - - def handle_event(self, msg: dict): - method = msg.get("method", "") - params = msg.get("params", {}) - session_id = msg.get("sessionId", "") - - # Handle target attachment — enable Network on each new target - if method == "Target.attachedToTarget": - target_info = params.get("targetInfo", {}) - sid = params.get("sessionId", "") - ttype = target_info.get("type", "") - title = target_info.get("title", "") - url = target_info.get("url", "") - print(f" Attached to [{ttype}] {title} ({url})", file=sys.stderr) - - # Enable Network capture on this target - self.send_to_session(sid, "Network.enable", { - "maxTotalBufferSize": 100 * 1024 * 1024, - "maxResourceBufferSize": 10 * 1024 * 1024, - }) - # Track session -> target mapping - self.sessions[sid] = target_info - return - - if method == "Target.detachedFromTarget": - sid = params.get("sessionId", "") - target = self.sessions.pop(sid, {}) - print(f" Detached from [{target.get('type', '?')}] {target.get('title', '?')}", file=sys.stderr) - return - - if method == "Network.requestWillBeSent": - self._on_request(params, session_id) - elif method == "Network.responseReceived": - self._on_response(params, session_id) - elif method == "Network.loadingFinished": - self._on_loading_finished(params, session_id) - elif method == "Network.eventSourceMessageReceived": - self._on_sse_message(params) - elif method == "Network.dataReceived": - # Track streaming data chunks - pass - - def handle_response(self, msg: dict): - mid = msg.get("id") - if mid in self.pending_body_requests: - request_id, session_id = self.pending_body_requests.pop(mid) - result = msg.get("result", {}) - body = result.get("body", "") - is_b64 = result.get("base64Encoded", False) - if body: - self.thrift_responses[request_id] = { - "body": body, - "base64Encoded": is_b64, - "timestamp": timestamp(), - } - - def _on_request(self, params: dict, session_id: str = ""): - request = params.get("request", {}) - url = request.get("url", "") - if not is_line_url(url): - return - - request_id = params.get("requestId") - headers = request.get("headers", {}) - - self.requests[request_id] = { - "url": url, - "method": request.get("method", "GET"), - "headers": headers, - "timestamp": timestamp(), - "has_post_data": request.get("hasPostData", False), - "session_id": session_id, - } - - # Capture POST body for Thrift calls - if request.get("hasPostData") and "/thrift/" in url: - post_data = request.get("postData", "") - if post_data: - self.request_bodies[request_id] = post_data - - count = len(self.requests) - if count % 10 == 1: - print(f" [{count} LINE requests captured]", file=sys.stderr) - - def _on_response(self, params: dict, session_id: str = ""): - request_id = params.get("requestId") - if request_id not in self.requests: - return - - response = params.get("response", {}) - self.requests[request_id]["status"] = response.get("status") - self.requests[request_id]["response_headers"] = response.get("headers", {}) - self.requests[request_id]["mime_type"] = response.get("mimeType", "") - - def _on_loading_finished(self, params: dict, session_id: str = ""): - request_id = params.get("requestId") - if request_id not in self.requests: - return - - req = self.requests[request_id] - url = req.get("url", "") - - # Fetch response body for Thrift RPC and JSON responses - # Must use the same session that owns the request - target_session = req.get("session_id", session_id) - if "/thrift/" in url or req.get("mime_type", "") == "application/json": - if target_session: - mid = self.send_to_session(target_session, "Network.getResponseBody", {"requestId": request_id}) - else: - mid = self.send("Network.getResponseBody", {"requestId": request_id}) - self.pending_body_requests[mid] = (request_id, target_session) - - def _on_sse_message(self, params: dict): - """This is the key event — captures each SSE message as it arrives.""" - request_id = params.get("requestId") - url = "" - if request_id in self.requests: - url = self.requests[request_id].get("url", "") - - event_name = params.get("eventName", "") - event_id = params.get("eventId", "") - data = params.get("data", "") - - entry = { - "timestamp": timestamp(), - "requestId": request_id, - "url": url, - "eventName": event_name, - "eventId": event_id, - "data": data, - } - - # Try to parse the data as JSON - try: - entry["parsed"] = json.loads(data) - except (json.JSONDecodeError, TypeError): - pass - - self.sse_messages.append(entry) - - # Log it live - preview = data[:120] if data else "(empty)" - print(f" SSE [{event_name or 'message'}]: {preview}", file=sys.stderr) - - def save(self): - """Write captured data to output file.""" - # Build structured output - requests_list = [] - for rid, req in self.requests.items(): - entry = dict(req) - entry.pop("session_id", None) # internal, don't export - if rid in self.thrift_responses: - entry["response_body"] = self.thrift_responses[rid] - if rid in self.request_bodies: - entry["request_body"] = self.request_bodies[rid] - requests_list.append(entry) - - output = { - "capture_method": "cdp", - "capture_time": timestamp(), - "requests_count": len(requests_list), - "sse_messages_count": len(self.sse_messages), - "requests": requests_list, - "sse_messages": self.sse_messages, - } - - with open(self.output_path, "w") as f: - json.dump(output, f, indent=2, ensure_ascii=False) - - print(f"\nCapture saved to: {self.output_path}", file=sys.stderr) - print(f" Requests: {len(requests_list)}", file=sys.stderr) - print(f" SSE messages: {len(self.sse_messages)}", file=sys.stderr) - - def run(self): - """Main event loop — read CDP messages until Chrome closes or Ctrl+C.""" - print("Listening for LINE traffic... (Ctrl+C or close Chrome to stop)\n", file=sys.stderr) - - while self.running: - try: - raw = self.ws.recv() - if not raw: - break - msg = json.loads(raw) - - if "method" in msg: - self.handle_event(msg) - elif "id" in msg: - self.handle_response(msg) - - except websocket.WebSocketConnectionClosedException: - print("\nChrome closed connection.", file=sys.stderr) - break - except KeyboardInterrupt: - print("\nStopping capture...", file=sys.stderr) - break - except Exception as e: - print(f"Error: {e}", file=sys.stderr) - continue - - self.save() - - -def wait_for_chrome(timeout: int = 60) -> bool: - """Wait for Chrome CDP to become available.""" - import urllib.request - - print(f"Waiting for Chrome CDP on port {CDP_PORT}...", file=sys.stderr) - start = time.time() - while time.time() - start < timeout: - try: - resp = urllib.request.urlopen(f"http://127.0.0.1:{CDP_PORT}/json", timeout=2) - targets = json.loads(resp.read()) - if targets: - return True - except Exception: - pass - time.sleep(1) - return False - - -def main(): - output_path = sys.argv[1] if len(sys.argv) > 1 else "/tmp/line-chrome-extension-capture/line-cdp-capture.json" - - # Ensure output directory exists - os.makedirs(os.path.dirname(output_path), exist_ok=True) - - if not wait_for_chrome(): - print("Timed out waiting for Chrome. Is it running with --remote-debugging-port=9222?", file=sys.stderr) - sys.exit(1) - - capture = CDPCapture(output_path) - - # Handle SIGTERM gracefully - def handle_signal(signum, frame): - capture.running = False - signal.signal(signal.SIGTERM, handle_signal) - - capture.connect() - capture.run() - - -if __name__ == "__main__": - main() diff --git a/scripts/line-chrome-capture.sh b/scripts/line-chrome-capture.sh deleted file mode 100755 index 101967ed..00000000 --- a/scripts/line-chrome-capture.sh +++ /dev/null @@ -1,361 +0,0 @@ -#!/bin/bash -# Launch Chrome with CDP-based network capture for LINE extension analysis. -# -# Usage: -# ./scripts/line-chrome-capture.sh [output_dir] -# -# This starts Chrome with remote debugging and runs a CDP listener that -# captures all LINE traffic including SSE EventSource message bodies. -# -# Uses a persistent separate Chrome profile at: -# ${LINE_CAPTURE_CHROME_PROFILE:-$HOME/.cache/line-chrome-extension-capture/chrome-debug-profile} -# On first run, you'll need to log in to LINE. The profile persists across -# capture output directories. -# -# The capture profile is pre-seeded to skip Chrome's first-run prompts: it does -# not make Chrome the default browser, does not send usage/crash reports, and -# stays signed out of Chrome/Sync. LINE login is still manual. -# -# If LINE/Codex Chrome extensions are already installed in your normal Chrome -# Default profile, this script stages clean unpacked copies into: -# ${LINE_CAPTURE_EXTENSION_CACHE_DIR:-$HOME/.cache/line-chrome-extension-capture/extensions} -# and auto-loads those copies into the capture profile. If LINE is already -# installed in the persistent capture profile, the script uses that installation -# instead of injecting a staged copy. Override detection with: -# LINE_CHROME_EXTENSION_DIR=/path/to/LINE/3.7.2_0 -# CODEX_CHROME_EXTENSION_DIR=/path/to/Codex/1.1.5_0 -# Disable staged extension auto-loading with: -# LINE_CAPTURE_AUTO_LOAD_EXTENSIONS=0 -# This is useful when installing LINE from the Chrome Web Store into the -# persistent capture profile. -# -# When done, close Chrome (Cmd+Q). The capture files will be at: -# /line-cdp-capture.json — CDP capture (SSE + request bodies) -# /line-net-log.json — Chrome net-log (low-level backup) - -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -OUTPUT_DIR="${1:-/tmp/line-chrome-extension-capture}" -mkdir -p "$OUTPUT_DIR" -NETLOG_FILE="$OUTPUT_DIR/line-net-log.json" -CDP_FILE="$OUTPUT_DIR/line-cdp-capture.json" -DEBUG_PROFILE="${LINE_CAPTURE_CHROME_PROFILE:-$HOME/.cache/line-chrome-extension-capture/chrome-debug-profile}" -EXTENSION_CACHE_DIR="${LINE_CAPTURE_EXTENSION_CACHE_DIR:-$HOME/.cache/line-chrome-extension-capture/extensions}" -AUTO_LOAD_EXTENSIONS="${LINE_CAPTURE_AUTO_LOAD_EXTENSIONS:-1}" - -seed_chrome_capture_profile() { - local profile_dir="$1" - local default_dir="$profile_dir/Default" - - mkdir -p "$default_dir" - chmod 700 "$profile_dir" 2>/dev/null || true - chmod 700 "$default_dir" 2>/dev/null || true - - # Chrome uses this sentinel for first-run completion. The JSON preferences - # below keep the choices explicit and readable for future maintainers. - touch "$profile_dir/First Run" - - python3 - "$profile_dir" <<'PY' -import json -import sys -from pathlib import Path - -profile = Path(sys.argv[1]) - -def merge(dst, src): - for key, value in src.items(): - if isinstance(value, dict) and isinstance(dst.get(key), dict): - merge(dst[key], value) - else: - dst[key] = value - return dst - -def update_json(path, values): - try: - data = json.loads(path.read_text()) if path.exists() else {} - if not isinstance(data, dict): - data = {} - except json.JSONDecodeError: - data = {} - merge(data, values) - path.write_text(json.dumps(data, indent=2, sort_keys=True) + "\n") - path.chmod(0o600) - -update_json(profile / "Local State", { - "browser": { - "check_default_browser": False, - "has_seen_welcome_page": True, - }, - "distribution": { - "import_bookmarks": False, - "import_history": False, - "make_chrome_default": False, - "skip_first_run_ui": True, - "suppress_first_run_default_browser_prompt": True, - }, - "signin": { - "allowed": False, - "allowed_on_next_startup": False, - }, - "sync": { - "suppress_start": True, - }, - "user_experience_metrics": { - "reporting_enabled": False, - }, -}) - -update_json(profile / "Default" / "Preferences", { - "browser": { - "check_default_browser": False, - "has_seen_welcome_page": True, - }, - "credentials_enable_autosignin": False, - "credentials_enable_service": False, - "distribution": { - "import_bookmarks": False, - "import_history": False, - "make_chrome_default": False, - "skip_first_run_ui": True, - "suppress_first_run_default_browser_prompt": True, - }, - "profile": { - "exit_type": "Normal", - "exited_cleanly": True, - "name": "LINE Capture", - }, - "signin": { - "allowed": False, - "allowed_on_next_startup": False, - }, - "sync": { - "requested": False, - "suppress_start": True, - }, -}) -PY -} - -find_latest_extension_dir() { - local extension_id="$1" - local base="$HOME/Library/Application Support/Google/Chrome/Default/Extensions/$extension_id" - if [ -d "$base" ]; then - find "$base" -mindepth 1 -maxdepth 1 -type d | sort -V | tail -n 1 - fi -} - -find_latest_profile_extension_dir() { - local profile_dir="$1" - local extension_id="$2" - local base="$profile_dir/Default/Extensions/$extension_id" - if [ -d "$base" ]; then - find "$base" -mindepth 1 -maxdepth 1 -type d | sort -V | tail -n 1 - fi -} - -read_manifest_version() { - local extension_dir="$1" - if [ -n "${extension_dir:-}" ] && [ -f "$extension_dir/manifest.json" ]; then - python3 - "$extension_dir/manifest.json" <<'PY' -import json -import sys -from pathlib import Path - -try: - print(json.loads(Path(sys.argv[1]).read_text()).get("version", "")) -except Exception: - print("") -PY - fi -} - -find_repo_extension_version() { - local client_file="$SCRIPT_DIR/../pkg/line/client.go" - if [ -f "$client_file" ]; then - python3 - "$client_file" <<'PY' -import re -import sys -from pathlib import Path - -match = re.search(r'ExtensionVersion\s*=\s*"([^"]+)"', Path(sys.argv[1]).read_text()) -print(match.group(1) if match else "") -PY - fi -} - -stage_extension_dir() { - local extension_id="$1" - local source_dir="$2" - - if [ -z "${source_dir:-}" ] || [ ! -f "$source_dir/manifest.json" ]; then - return 0 - fi - - local version - version="$(python3 - "$source_dir/manifest.json" <<'PY' -import json -import sys -from pathlib import Path - -try: - version = json.loads(Path(sys.argv[1]).read_text()).get("version", "unknown") -except Exception: - version = "unknown" -print("".join(c if c.isalnum() or c in ".-_" else "_" for c in str(version))) -PY -)" - - local dest="$EXTENSION_CACHE_DIR/$extension_id/$version" - mkdir -p "$dest" - - # Chrome Web Store extension directories contain _metadata/, which Chrome - # rejects when loading as unpacked. Stage a clean copy for capture runs. - if command -v rsync >/dev/null 2>&1; then - rsync -a --delete --exclude '_metadata' "$source_dir/" "$dest/" - else - local tmp="$dest.tmp" - rm -rf "$tmp" - mkdir -p "$tmp" - (cd "$source_dir" && tar --exclude './_metadata' -cf - .) | (cd "$tmp" && tar -xf -) - rm -rf "$dest" - mv "$tmp" "$dest" - fi - chmod -R go-rwx "$EXTENSION_CACHE_DIR" 2>/dev/null || true - - echo "$dest" -} - -LINE_EXTENSION_DIR="" -CODEX_EXTENSION_DIR="" -LINE_PROFILE_EXTENSION_DIR="$(find_latest_profile_extension_dir "$DEBUG_PROFILE" ophjlpahpchlmihnnnihgmmeilfjmjjc)" -LINE_PROFILE_EXTENSION_VERSION="$(read_manifest_version "$LINE_PROFILE_EXTENSION_DIR")" -if [ "$AUTO_LOAD_EXTENSIONS" != "0" ]; then - if [ -z "${LINE_PROFILE_EXTENSION_DIR:-}" ]; then - LINE_EXTENSION_SOURCE_DIR="${LINE_CHROME_EXTENSION_DIR:-$(find_latest_extension_dir ophjlpahpchlmihnnnihgmmeilfjmjjc)}" - LINE_EXTENSION_DIR="$(stage_extension_dir ophjlpahpchlmihnnnihgmmeilfjmjjc "$LINE_EXTENSION_SOURCE_DIR")" - fi - CODEX_EXTENSION_SOURCE_DIR="${CODEX_CHROME_EXTENSION_DIR:-$(find_latest_extension_dir hehggadaopoacecdllhhajmbjkdcmajg)}" - CODEX_EXTENSION_DIR="$(stage_extension_dir hehggadaopoacecdllhhajmbjkdcmajg "$CODEX_EXTENSION_SOURCE_DIR")" -fi -LINE_STAGED_EXTENSION_VERSION="$(read_manifest_version "$LINE_EXTENSION_DIR")" -LINE_CHROME_EXTENSION_VERSION="${LINE_PROFILE_EXTENSION_VERSION:-$LINE_STAGED_EXTENSION_VERSION}" -REPO_EXTENSION_VERSION="$(find_repo_extension_version)" - -EXTENSION_DIRS=() -if [ -n "${LINE_EXTENSION_DIR:-}" ] && [ -f "$LINE_EXTENSION_DIR/manifest.json" ]; then - EXTENSION_DIRS+=("$LINE_EXTENSION_DIR") -fi -if [ -n "${CODEX_EXTENSION_DIR:-}" ] && [ -f "$CODEX_EXTENSION_DIR/manifest.json" ]; then - EXTENSION_DIRS+=("$CODEX_EXTENSION_DIR") -fi - -CHROME="${CHROME:-/Applications/Google Chrome.app/Contents/MacOS/Google Chrome}" -if [ ! -x "$CHROME" ]; then - echo "Chrome not found at $CHROME" - echo "Set CHROME env var to your Chrome binary path." - exit 1 -fi - -FIRST_RUN=false -if [ ! -d "$DEBUG_PROFILE" ]; then - FIRST_RUN=true - mkdir -p "$DEBUG_PROFILE" -fi -seed_chrome_capture_profile "$DEBUG_PROFILE" - -echo "==> Starting Chrome with CDP + net-log capture" -echo " CDP output: $CDP_FILE" -echo " Net-log: $NETLOG_FILE" -echo " Profile: $DEBUG_PROFILE" -if [ -n "${LINE_PROFILE_EXTENSION_DIR:-}" ]; then - echo " LINE profile extension: $LINE_PROFILE_EXTENSION_DIR" -fi -if [ -n "${REPO_EXTENSION_VERSION:-}" ]; then - echo " Repo LINE version: $REPO_EXTENSION_VERSION" -fi -if [ -n "${LINE_CHROME_EXTENSION_VERSION:-}" ]; then - echo " Chrome LINE version: $LINE_CHROME_EXTENSION_VERSION" -fi -if [ -n "${REPO_EXTENSION_VERSION:-}" ] && [ -n "${LINE_CHROME_EXTENSION_VERSION:-}" ] && [ "$REPO_EXTENSION_VERSION" != "$LINE_CHROME_EXTENSION_VERSION" ]; then - echo " WARNING: repo ExtensionVersion ($REPO_EXTENSION_VERSION) does not match Chrome extension ($LINE_CHROME_EXTENSION_VERSION)." -fi -if [ ${#EXTENSION_DIRS[@]} -gt 0 ]; then - echo " Extensions:" - for ext_dir in "${EXTENSION_DIRS[@]}"; do - echo " - $ext_dir" - done -else - echo " Extensions: none auto-loaded" -fi -echo "" -if [ "$FIRST_RUN" = true ]; then - echo " FIRST RUN: Chrome first-run/sign-in prompts are pre-skipped." - echo " Log in to LINE manually in the capture browser." - if [ ${#EXTENSION_DIRS[@]} -eq 0 ]; then - echo " LINE extension was not auto-detected; install it from the Chrome Web Store first." - fi - echo " The profile persists across capture dirs — you only need to do this once." - echo "" -fi -echo " 1. Open the LINE Chrome Extension and do your flow" -echo " 2. When done, close Chrome (Cmd+Q)" -echo "" - -# Chrome 146+ requires --user-data-dir for remote debugging. -CHROME_ARGS=( - --no-first-run - --no-default-browser-check - --disable-sync - --disable-features=SignInPromo,SigninInterception,ChromeSigninIntercept,DiceWebSigninInterception - --remote-debugging-port=9222 - --remote-allow-origins="*" - --user-data-dir="$DEBUG_PROFILE" - --log-net-log="$NETLOG_FILE" - --net-log-capture-mode=IncludeSensitive -) - -if [ ${#EXTENSION_DIRS[@]} -gt 0 ]; then - IFS=, - EXTENSION_ARG="${EXTENSION_DIRS[*]}" - unset IFS - if [ -z "${LINE_PROFILE_EXTENSION_DIR:-}" ]; then - CHROME_ARGS+=(--disable-extensions-except="$EXTENSION_ARG") - fi - CHROME_ARGS+=(--load-extension="$EXTENSION_ARG") -fi - -START_URL="${LINE_CAPTURE_START_URL:-}" -if [ -z "$START_URL" ] && [ -n "${LINE_PROFILE_EXTENSION_DIR:-}" ] && [ -f "$LINE_PROFILE_EXTENSION_DIR/manifest.json" ]; then - START_URL="chrome-extension://ophjlpahpchlmihnnnihgmmeilfjmjjc/index.html" -fi -if [ -z "$START_URL" ] && [ -n "${LINE_EXTENSION_DIR:-}" ] && [ -f "$LINE_EXTENSION_DIR/manifest.json" ]; then - START_URL="chrome-extension://ophjlpahpchlmihnnnihgmmeilfjmjjc/index.html" -fi -if [ -z "$START_URL" ]; then - START_URL="about:blank" -fi - -"$CHROME" "${CHROME_ARGS[@]}" "$START_URL" 2>/dev/null & -CHROME_PID=$! - -# Start the CDP capture (it waits for Chrome to be ready) -python3 "$SCRIPT_DIR/line-cdp-capture.py" "$CDP_FILE" & -CDP_PID=$! - -# Wait for Chrome to exit -wait $CHROME_PID 2>/dev/null || true - -# Give CDP listener a moment to finish saving -sleep 1 - -# Stop CDP capture if still running -if kill -0 $CDP_PID 2>/dev/null; then - kill $CDP_PID 2>/dev/null - wait $CDP_PID 2>/dev/null || true -fi - -echo "" -echo "==> Chrome closed. Captures saved:" -[ -f "$CDP_FILE" ] && echo " CDP: $CDP_FILE ($(du -h "$CDP_FILE" | cut -f1))" -[ -f "$NETLOG_FILE" ] && echo " Net-log: $NETLOG_FILE ($(du -h "$NETLOG_FILE" | cut -f1))" diff --git a/scripts/parse-line-traffic.py b/scripts/parse-line-traffic.py deleted file mode 100755 index dd21a688..00000000 --- a/scripts/parse-line-traffic.py +++ /dev/null @@ -1,336 +0,0 @@ -#!/usr/bin/env python3 -"""Parse LINE network captures (CDP or net-log format). - -Usage: - python3 scripts/parse-line-traffic.py [output.json] - -Supports two input formats: -- CDP capture (from line-cdp-capture.py): has SSE message bodies, Thrift response bodies -- Chrome net-log: lower-level, lacks streaming body data - -Extracts HTTP requests/responses to LINE domains and writes a structured -JSON summary suitable for analysis by Claude or other tools. -""" - -import json -import sys -import os -from collections import defaultdict - -LINE_DOMAINS = [ - "line-chrome-gw.line-apps.com", - "obs.line-apps.com", - "legy-jp.line-apps.com", - "gd2.line.naver.jp", - "gwx.line.naver.jp", - "gw.line.naver.jp", - "api.line.me", - "stickershop.line-scdn.net", - "profile.line-scdn.net", - "obs.line-scdn.net", - "scdn.line-apps.com", -] - - -def is_line_domain(url: str) -> bool: - """Check if URL belongs to a LINE domain.""" - for domain in LINE_DOMAINS: - if domain in url: - return True - return False - - -def parse_netlog(filepath: str) -> dict: - """Parse Chrome net-log JSON and extract LINE traffic.""" - print(f"Reading {filepath}...", file=sys.stderr) - file_size = os.path.getsize(filepath) - print(f"File size: {file_size / 1024 / 1024:.1f} MB", file=sys.stderr) - - with open(filepath, "r") as f: - data = json.load(f) - - constants = data.get("constants", {}) - events = data.get("events", []) - - # Build reverse maps from numeric IDs to names - event_types = {} - if "logEventTypes" in constants: - for name, num in constants["logEventTypes"].items(): - event_types[num] = name - - source_types = {} - if "logSourceType" in constants: - for name, num in constants["logSourceType"].items(): - source_types[num] = name - - phases = {} - if "logEventPhase" in constants: - for name, num in constants["logEventPhase"].items(): - phases[num] = name - - # Group events by source ID - sources = defaultdict(list) - for event in events: - sid = event.get("source", {}).get("id") - if sid is not None: - sources[sid].append(event) - - # Extract URL_REQUEST sources that hit LINE domains - line_requests = [] - for sid, evts in sources.items(): - source_type = evts[0].get("source", {}).get("type") - source_name = source_types.get(source_type, str(source_type)) - - if source_name != "URL_REQUEST": - continue - - # Find the URL - url = None - method = None - status_code = None - request_headers = {} - response_headers = {} - request_body = None - response_body_parts = [] - - for evt in evts: - etype = event_types.get(evt.get("type"), "") - params = evt.get("params", {}) - - if etype == "URL_REQUEST_START_JOB" or etype == "REQUEST_ALIVE": - if "url" in params: - url = params["url"] - if "method" in params: - method = params["method"] - - elif etype == "HTTP_TRANSACTION_SEND_REQUEST_HEADERS": - if "headers" in params: - for h in params["headers"]: - if ": " in h: - k, v = h.split(": ", 1) - request_headers[k] = v - elif h.startswith("GET ") or h.startswith("POST ") or h.startswith("PUT "): - parts = h.split(" ") - method = parts[0] - if "line" in params: - line_parts = params["line"].split(" ") - if len(line_parts) >= 1: - method = line_parts[0] - - elif etype == "HTTP_TRANSACTION_READ_RESPONSE_HEADERS": - if "headers" in params: - for h in params["headers"]: - if h.startswith("HTTP/"): - parts = h.split(" ") - if len(parts) >= 2: - try: - status_code = int(parts[1]) - except ValueError: - pass - elif ": " in h: - k, v = h.split(": ", 1) - response_headers[k] = v - - elif etype == "URL_REQUEST_JOB_BYTES_READ": - if "bytes" in params: - response_body_parts.append(params["bytes"]) - - elif etype == "UPLOAD_DATA_STREAM_READ": - if "bytes" in params: - request_body = params["bytes"] - - if not url or not is_line_domain(url): - continue - - # Try to decode base64 body content - response_body = "".join(response_body_parts) if response_body_parts else None - - entry = { - "url": url, - "method": method or "GET", - } - if status_code: - entry["status"] = status_code - if request_headers: - # Filter to interesting headers - interesting = { - k: v for k, v in request_headers.items() - if k.lower() in ( - "content-type", "x-line-access", "x-line-application", - "x-lhm", "x-lpv", "x-lsr", "x-obs-params", - "x-line-channeltoken", "accept", - ) - } - if interesting: - entry["request_headers"] = interesting - if response_headers: - interesting = { - k: v for k, v in response_headers.items() - if k.lower() in ( - "content-type", "x-obs-oid", "x-line-next-access-token", - "x-line-error-code", "x-line-error-message", - ) - } - if interesting: - entry["response_headers"] = interesting - if request_body: - entry["request_body_b64"] = request_body - if response_body: - entry["response_body_b64"] = response_body - - line_requests.append(entry) - - return { - "total_events": len(events), - "total_sources": len(sources), - "line_requests_count": len(line_requests), - "line_requests": line_requests, - } - - -def categorize_requests(requests: list) -> dict: - """Group requests by API category.""" - categories = defaultdict(list) - for req in requests: - url = req["url"] - if "/api/talk/thrift/" in url: - categories["thrift_rpc"].append(req) - elif "/api/talk/long-polling/" in url or "/api/operation/receive" in url: - categories["long_poll_sse"].append(req) - elif "/api/auth/" in url: - categories["auth"].append(req) - elif "obs.line-apps.com" in url: - categories["media_obs"].append(req) - elif "stickershop" in url or "scdn" in url: - categories["stickers_cdn"].append(req) - elif "/sc/api/" in url or "legy" in url: - categories["secondary_api"].append(req) - else: - categories["other"].append(req) - return dict(categories) - - -def is_cdp_capture(data: dict) -> bool: - """Check if the input is a CDP capture (vs net-log).""" - return data.get("capture_method") == "cdp" - - -def parse_cdp_capture(data: dict) -> dict: - """Parse a CDP capture file into the same format as parse_netlog.""" - requests = data.get("requests", []) - sse_messages = data.get("sse_messages", []) - - line_requests = [] - for req in requests: - url = req.get("url", "") - entry = { - "url": url, - "method": req.get("method", "GET"), - } - if "status" in req: - entry["status"] = req["status"] - - # Extract interesting request headers - headers = req.get("headers", {}) - interesting_req = {} - for k, v in headers.items(): - if k.lower() in ( - "content-type", "x-line-access", "x-line-application", - "x-lhm", "x-lpv", "x-lsr", "x-obs-params", - "x-line-channeltoken", "accept", - ): - interesting_req[k] = v - if interesting_req: - entry["request_headers"] = interesting_req - - # Extract interesting response headers - resp_headers = req.get("response_headers", {}) - interesting_resp = {} - for k, v in resp_headers.items(): - if k.lower() in ( - "content-type", "x-obs-oid", "x-line-next-access-token", - "x-line-error-code", "x-line-error-message", - ): - interesting_resp[k] = v - if interesting_resp: - entry["response_headers"] = interesting_resp - - # Include response body if available - if "response_body" in req: - body_info = req["response_body"] - body = body_info.get("body", "") - if body_info.get("base64Encoded"): - entry["response_body_b64"] = body - else: - # Try to parse as JSON for readability - try: - entry["response_body"] = json.loads(body) - except (json.JSONDecodeError, TypeError): - entry["response_body_text"] = body - - # Include request body if available - if "request_body" in req: - entry["request_body_text"] = req["request_body"] - - entry["timestamp"] = req.get("timestamp", "") - line_requests.append(entry) - - return { - "capture_method": "cdp", - "total_requests": len(requests), - "line_requests_count": len(line_requests), - "sse_messages_count": len(sse_messages), - "line_requests": line_requests, - "sse_messages": sse_messages, - } - - -def main(): - if len(sys.argv) < 2: - print(__doc__, file=sys.stderr) - sys.exit(1) - - input_path = sys.argv[1] - output_path = sys.argv[2] if len(sys.argv) > 2 else None - - if not os.path.exists(input_path): - print(f"File not found: {input_path}", file=sys.stderr) - sys.exit(1) - - print(f"Reading {input_path}...", file=sys.stderr) - with open(input_path, "r") as f: - data = json.load(f) - - if is_cdp_capture(data): - result = parse_cdp_capture(data) - else: - result = parse_netlog(input_path) - - # Add categorized view - result["by_category"] = categorize_requests(result["line_requests"]) - - # Summary - print(f"\n=== LINE Traffic Summary ===", file=sys.stderr) - print(f"Capture method: {result.get('capture_method', 'net-log')}", file=sys.stderr) - print(f"LINE requests found: {result['line_requests_count']}", file=sys.stderr) - if result.get("sse_messages_count"): - print(f"SSE messages captured: {result['sse_messages_count']}", file=sys.stderr) - for cat, reqs in result["by_category"].items(): - print(f" {cat}: {len(reqs)}", file=sys.stderr) - - output_json = json.dumps(result, indent=2, ensure_ascii=False) - - if output_path: - with open(output_path, "w") as f: - f.write(output_json) - print(f"\nOutput written to: {output_path}", file=sys.stderr) - else: - # Default output location - default_out = input_path.replace(".json", "-parsed.json") - with open(default_out, "w") as f: - f.write(output_json) - print(f"\nOutput written to: {default_out}", file=sys.stderr) - - -if __name__ == "__main__": - main() diff --git a/scripts/redact-line-capture.py b/scripts/redact-line-capture.py deleted file mode 100755 index 8c20f481..00000000 --- a/scripts/redact-line-capture.py +++ /dev/null @@ -1,143 +0,0 @@ -#!/usr/bin/env python3 -"""Redact sensitive values from LINE capture JSON files. - -Usage: - python3 scripts/redact-line-capture.py input.json [output.json] - -The default output path is input-redacted.json. This helper preserves JSON -structure for protocol analysis while removing credentials, cookies, tokens, -session identifiers, email addresses, and obvious long secrets. -""" - -from __future__ import annotations - -import hashlib -import json -import os -import re -import sys -from typing import Any - - -SENSITIVE_KEYS = { - "authorization", - "cookie", - "set-cookie", - "x-hmac", - "x-line-access", - "x-line-channeltoken", - "x-line-next-access-token", - "x-line-refresh-token", - "x-line-session-id", - "x-obs-params", - "password", - "passwd", - "email", - "mail", - "token", - "access_token", - "refresh_token", - "session", - "certificate", -} - -SENSITIVE_KEY_FRAGMENTS = ( - "auth", - "cookie", - "credential", - "password", - "secret", - "session", - "token", -) - -EMAIL_RE = re.compile(r"(?i)[a-z0-9._%+-]+@[a-z0-9.-]+\.[a-z]{2,}") -JWT_RE = re.compile(r"eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}") -MID_RE = re.compile(r"\b([ucr][0-9a-f]{20,})\b", re.IGNORECASE) -LONG_SECRET_RE = re.compile(r"^[A-Za-z0-9+/=_-]{96,}$") - - -def marker(kind: str, value: str) -> str: - digest = hashlib.sha256(value.encode("utf-8", errors="ignore")).hexdigest()[:12] - return f"" - - -def key_is_sensitive(key: str) -> bool: - normalized = key.lower() - if normalized in SENSITIVE_KEYS: - return True - return any(fragment in normalized for fragment in SENSITIVE_KEY_FRAGMENTS) - - -def redact_string(value: str, force: str | None = None) -> str: - if not value: - return value - if force: - return marker(force, value) - - value = JWT_RE.sub(lambda match: marker("jwt", match.group(0)), value) - value = EMAIL_RE.sub(lambda match: marker("email", match.group(0)), value) - value = MID_RE.sub(lambda match: match.group(1)[0].lower() + marker("mid", match.group(1)), value) - - stripped = value.strip() - if LONG_SECRET_RE.match(stripped): - return marker("opaque", stripped) - return value - - -def redact(value: Any, parent_key: str = "") -> Any: - if isinstance(value, dict): - redacted = {} - for key, child in value.items(): - key_text = str(key) - if key_is_sensitive(key_text): - if isinstance(child, (dict, list)): - redacted[key] = marker(key_text.lower(), json.dumps(child, sort_keys=True, default=str)) - else: - redacted[key] = redact_string(str(child), key_text.lower()) - else: - redacted[key] = redact(child, key_text) - return redacted - - if isinstance(value, list): - return [redact(item, parent_key) for item in value] - - if isinstance(value, str): - return redact_string(value) - - return value - - -def default_output_path(input_path: str) -> str: - base, ext = os.path.splitext(input_path) - return f"{base}-redacted{ext or '.json'}" - - -def main() -> int: - if len(sys.argv) not in (2, 3): - print(__doc__, file=sys.stderr) - return 1 - - input_path = sys.argv[1] - output_path = sys.argv[2] if len(sys.argv) == 3 else default_output_path(input_path) - - with open(input_path, "r", encoding="utf-8") as infile: - data = json.load(infile) - - redacted = redact(data) - - with open(output_path, "w", encoding="utf-8") as outfile: - json.dump(redacted, outfile, indent=2, ensure_ascii=False) - outfile.write("\n") - - try: - os.chmod(output_path, 0o600) - except OSError: - pass - - print(f"Redacted capture written to: {output_path}", file=sys.stderr) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main())