diff --git a/NEWS b/NEWS index 2b01df6..40951ce 100644 --- a/NEWS +++ b/NEWS @@ -1,3 +1,13 @@ +Hyperflux (unreleased) +====================== + +--extract-scan can now find any file type, not just streaming video. By default +it looks for video and audio; pass --scan-ext=iso,zip,mkv,flac (any extensions, +any type) to scan for those instead, which also works on a plain directory +listing. When a page has more than one matching file, flux opens a multi-select +so you choose which ones go into the generated config. + + Hyperflux 2.1.0 (2026-06-19) ============================ diff --git a/README.md b/README.md index 580bfdb..88c8f0f 100644 --- a/README.md +++ b/README.md @@ -41,6 +41,11 @@ discovery is content-driven: it follows promising links even without a standard `/watch` or `/play` pattern, stays within the series you scan, and ignores ad and onclick/popunder traps. +By default the scan looks for video and audio. Pass `--scan-ext=iso,zip,mkv,flac` +to find any file type instead (disk images, archives, anything), which also works +on a plain directory listing. With more than one file it opens a multi-select so +you choose which ones go into the config. + The scan prints the config and stashes a pending copy instead of activating it. When you are happy with it, run the `flux --save-config ` line it suggests to install it active. Use `-o FILE` to write straight to a path instead. diff --git a/src/scan.c b/src/scan.c index 1a91bd7..df557b9 100644 --- a/src/scan.c +++ b/src/scan.c @@ -140,27 +140,6 @@ url_host(const char *url) return h; } -/* Return the first non-empty path segment of `url` as a malloc'd string, or - * NULL if none exists (e.g. "https://host/" or no scheme). */ -static char * -url_first_path_segment(const char *url) -{ - if (!url) - return NULL; - const char *sep = strstr(url, "://"); - if (!sep) - return NULL; - const char *path = sep + 3; - path += strcspn(path, "/?#"); /* skip authority */ - if (*path != '/') - return NULL; - path++; /* skip leading slash */ - size_t n = strcspn(path, "/?#"); - if (n == 0) - return NULL; - return scan_strndup(path, n); -} - /* ---- ad-domain blocklist --------------------------------------------- */ /* Small built-in list of ad/tracker hosts; a candidate on (or under) any of @@ -222,27 +201,77 @@ scan_is_ad_host(const char *host) /* ---- candidate kind sniff -------------------------------------------- */ -/* Classify a media URL by its path extension. Returns 1 for a recognised media - * URL (kind set), 0 for anything else. */ +/* Default extension table (exts == NULL): video + audio + HLS. HLS extensions + * classify as SCAN_KIND_HLS; every other entry is a direct file. The audio set + * widens the scan past the old video/HLS-only bias. See #17. */ +static const struct { const char *ext; scan_kind_t kind; } scan_default_exts[] = { + { "m3u8", SCAN_KIND_HLS }, + { "m3u", SCAN_KIND_HLS }, + /* video */ + { "mp4", SCAN_KIND_FILE }, + { "webm", SCAN_KIND_FILE }, + { "mkv", SCAN_KIND_FILE }, + { "mov", SCAN_KIND_FILE }, + { "m4v", SCAN_KIND_FILE }, + { "avi", SCAN_KIND_FILE }, + { "flv", SCAN_KIND_FILE }, + { "ts", SCAN_KIND_FILE }, + { "m2ts", SCAN_KIND_FILE }, + { "ogv", SCAN_KIND_FILE }, + /* audio */ + { "mp3", SCAN_KIND_FILE }, + { "flac", SCAN_KIND_FILE }, + { "aac", SCAN_KIND_FILE }, + { "opus", SCAN_KIND_FILE }, + { "ogg", SCAN_KIND_FILE }, + { "oga", SCAN_KIND_FILE }, + { "wav", SCAN_KIND_FILE }, + { "m4a", SCAN_KIND_FILE }, + { "weba", SCAN_KIND_FILE }, + { "wma", SCAN_KIND_FILE }, + { "aiff", SCAN_KIND_FILE }, + { "alac", SCAN_KIND_FILE }, +}; + +/* Classify a media URL by its path extension, using the active extension set + * (custom set from the ctx, or the default table when exts == NULL). The custom + * set matches ONLY those extensions, all SCAN_KIND_FILE. Match is on the path + * before ?# and case-insensitive. Returns 1 for a recognised URL (kind set), + * 0 for anything else. Tokens have no leading '.'. */ static int -classify_url(const char *url, scan_kind_t *kind) +classify_url_ext(const char *url, const char *const *exts, size_t nexts, + scan_kind_t *kind) { size_t pathlen = strcspn(url, "?#"); - static const struct { const char *ext; scan_kind_t kind; } exts[] = { - { ".m3u8", SCAN_KIND_HLS }, - { ".m3u", SCAN_KIND_HLS }, - { ".mp4", SCAN_KIND_FILE }, - { ".webm", SCAN_KIND_FILE }, - { ".mkv", SCAN_KIND_FILE }, - { ".mov", SCAN_KIND_FILE }, - { ".m4v", SCAN_KIND_FILE }, - }; - for (size_t i = 0; i < sizeof(exts) / sizeof(exts[0]); i++) { - size_t el = strlen(exts[i].ext); - if (pathlen >= el && - strncasecmp(url + pathlen - el, exts[i].ext, el) == 0) { - *kind = exts[i].kind; + /* Custom set: any matching extension is a plain file. */ + if (exts) { + for (size_t i = 0; i < nexts; i++) { + size_t el = strlen(exts[i]); + /* Need a '.' immediately before the extension on the path. */ + if (el == 0 || pathlen < el + 1) + continue; + if (url[pathlen - el - 1] != '.') + continue; + if (strncasecmp(url + pathlen - el, exts[i], el) == 0) { + *kind = SCAN_KIND_FILE; + return 1; + } + } + return 0; + } + + /* Default set: video + audio + HLS. */ + for (size_t i = 0; i < sizeof(scan_default_exts) / sizeof(scan_default_exts[0]); + i++) { + size_t el = strlen(scan_default_exts[i].ext); + if (pathlen < el + 1) + continue; + if (url[pathlen - el - 1] != '.') + continue; + if (strncasecmp(url + pathlen - el, scan_default_exts[i].ext, el) + == 0) { + *kind = scan_default_exts[i].kind; return 1; } } @@ -265,8 +294,19 @@ struct scan_ctx { /* page0 body for the filename-as-param heuristic. Points into caller's * buffer; NOT owned. NULL outside of discover_chain recursion. */ const char *page0_body; + /* Optional extension filter (NULL/0 = default video+audio+HLS bias). + * Borrowed from the caller; NOT owned. See #17. */ + const char *const *exts; + size_t nexts; }; +/* Classify against the ctx's active extension set. */ +static int +classify_url(struct scan_ctx *ctx, const char *url, scan_kind_t *kind) +{ + return classify_url_ext(url, ctx->exts, ctx->nexts, kind); +} + /* Record `url` as visited; returns 1 if already seen, 0 if newly added/unrecordable. * Visited set is best-effort dedup: SCAN_MAX_FETCHES is the real termination bound. */ static int @@ -293,12 +333,28 @@ add_candidate(struct scan_ctx *ctx, const char *raw_ref, scan_ctx_t context) if (!raw_ref || !*raw_ref) return 0; + /* Reject any ref containing control characters (CR, LF, tab, etc.). + * A crafted href with an embedded newline could inject extra directives + * into the emitted config. See #17. */ + for (const char *p = raw_ref; *p; p++) + if (iscntrl((unsigned char)*p)) + return 0; + char *url = extractor_resolve_url(ctx->page_url, raw_ref); if (!url) return -1; + /* Also reject a resolved URL that contains control characters (defensive: + * extractor_resolve_url should never produce one, but be explicit). */ + for (const char *p = url; *p; p++) { + if (iscntrl((unsigned char)*p)) { + free(url); + return 0; + } + } + scan_kind_t kind; - if (!classify_url(url, &kind)) { + if (!classify_url(ctx, url, &kind)) { free(url); return 0; /* not a media URL we handle */ } @@ -325,10 +381,20 @@ add_candidate(struct scan_ctx *ctx, const char *raw_ref, scan_ctx_t context) return 0; /* cap: ignore further candidates */ } + /* Store the raw ref so scan_emit_config_selection can build an alternation + * that matches what is ACTUALLY in the page body (relative hrefs on + * directory listings don't appear as absolute URLs). See #17. */ + char *rr = scan_strdup(raw_ref); + if (!rr) { + free(url); + return -1; + } + char *host = url_host(url); /* may be NULL: best effort */ scan_candidate_t *c = &r->cands[r->ncands]; memset(c, 0, sizeof(*c)); c->url = url; + c->raw_ref = rr; c->host = host; c->kind = kind; c->context = context; @@ -415,12 +481,8 @@ static const struct scan_pattern scan_patterns[] = { SCAN_CTX_PLAYER }, { "[\"']source[\"'][[:space:]]*:[[:space:]]*[\"']([^\"']+)[\"']", SCAN_CTX_PLAYER }, - /* Catch-all: any bare media URL in the text (unknown context). - * '&' is excluded from the PATH class so " in HTML-attribute JSON - * doesn't bleed across into a later .mp4, but the optional query/fragment - * allows '&' so signed URLs like ?token=abc&expires=123 are not truncated. */ - { "(https?:[^\"'[:space:]<>()&]+\\.(m3u8|mp4|webm)([?#][^\"'[:space:]<>()]*)?)", - SCAN_CTX_UNKNOWN }, + /* The bare-URL catch-all is built dynamically from the active extension + * set in collect_dynamic_ext (so audio/custom extensions are covered). */ }; /* The "og:video" two-pattern set captures group 2 in the first variant; to keep @@ -429,6 +491,89 @@ static const struct scan_pattern scan_patterns[] = { static const char *const scan_og_video_ere = "property=[\"']og:video[^\"']*[\"'][^>]*content=[\"']([^\"']+)[\"']"; +/* Build an ERE-escaped extension alternation ("mp4|webm|m3u8") from the active + * extension set into `dst`. Tokens are alnum so the escape is defensive. The + * default set uses scan_default_exts; a custom set uses ctx->exts. Returns the + * number of tokens written (0 on an empty/overflowing set). */ +static size_t +build_ext_alternation(struct scan_ctx *ctx, char *dst, size_t dlen) +{ + size_t o = 0, ntok = 0; + dst[0] = '\0'; + + size_t n = ctx->exts ? ctx->nexts + : sizeof(scan_default_exts) / sizeof(scan_default_exts[0]); + for (size_t i = 0; i < n; i++) { + const char *ext = ctx->exts ? ctx->exts[i] + : scan_default_exts[i].ext; + if (!ext || !*ext) + continue; + if (ntok > 0) { + if (o + 1 >= dlen) + return 0; + dst[o++] = '|'; + } + for (const char *p = ext; *p; p++) { + /* ERE-escape metacharacters defensively; {} included so a + * crafted token can't form a brace quantifier. See #17. */ + if (strchr(".^$*+?()|[]{}\\", *p)) { + if (o + 2 >= dlen) + return 0; + dst[o++] = '\\'; + } + if (o + 1 >= dlen) + return 0; + dst[o++] = *p; + } + ntok++; + } + dst[o] = '\0'; + return ntok; +} + +/* Build the dynamic capture EREs from the active extension set and run each + * over `text`, capturing group 1 as a candidate. Covers a bare-URL catch-all + * and href/src catch-alls for directory listings and relative links. Each ERE + * is regcomp-verified inside collect_regex; an empty/bad set is skipped. + * Returns 0 on success, -1 on OOM. */ +static int +collect_dynamic_ext(struct scan_ctx *ctx, const char *text) +{ + char alt[1024]; + if (build_ext_alternation(ctx, alt, sizeof(alt)) == 0) + return 0; /* empty/overflowing set: nothing to do */ + + /* Bare-URL catch-all: '&' excluded from the PATH class (so " in + * attribute JSON doesn't bleed), allowed in the optional query tail. The + * URL is group 1. See #17 / FIX from #scan FIX C. */ + char bare[1200]; + int n = snprintf(bare, sizeof(bare), + "(https?:[^\"'[:space:]<>()&]+\\.(%s)([?#][^\"'[:space:]<>()]*)?)", + alt); + if (n > 0 && (size_t)n < sizeof(bare)) + if (collect_regex(ctx, text, bare, SCAN_CTX_UNKNOWN) < 0) + return -1; + + /* href/src catch-alls for directory listings and relative links. POSIX + * ERE has no non-capturing group, so use two patterns each with the URL + * as group 1; add_candidate resolves relative -> absolute. */ + char href_ere[1200]; + n = snprintf(href_ere, sizeof(href_ere), + "href=[\"']([^\"'<>]+\\.(%s))[\"']", alt); + if (n > 0 && (size_t)n < sizeof(href_ere)) + if (collect_regex(ctx, text, href_ere, SCAN_CTX_UNKNOWN) < 0) + return -1; + + char src_ere[1200]; + n = snprintf(src_ere, sizeof(src_ere), + "src=[\"']([^\"'<>]+\\.(%s))[\"']", alt); + if (n > 0 && (size_t)n < sizeof(src_ere)) + if (collect_regex(ctx, text, src_ere, SCAN_CTX_UNKNOWN) < 0) + return -1; + + return 0; +} + /* Find same-origin