Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
103 changes: 74 additions & 29 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,17 @@ jobs:
echo "${out}"
echo "${out}" | grep -q '^OK: all'

- name: Unit test (tui window/row math)
run: |
set -euo pipefail
# Pure window/row-count helpers (the breakage-prone math). The
# interactive prompt is not unit-tested; these are. No autotools.
"${CC}" -D_DEFAULT_SOURCE -Wall -Wextra -g \
src/tui.c src/test_tui.c -o /tmp/test_tui
out="$(/tmp/test_tui)"
echo "${out}"
echo "${out}" | grep -q '^OK: all'

- name: Unit test (hls)
run: |
set -euo pipefail
Expand Down Expand Up @@ -403,27 +414,27 @@ jobs:
}
trap cleanup EXIT

# A series page listing three episode pages; each episode page embeds
# a <source> pointing at a distinct media file. The config 'list's the
# episode hrefs; the per-episode pipeline resolves each <source>.
head -c 4096 /dev/urandom > "${srvdir}/ep1.bin"
head -c 8192 /dev/urandom > "${srvdir}/ep2.bin"
head -c 16384 /dev/urandom > "${srvdir}/ep3.bin"
# A series page listing a LARGE number of episode pages (N=60) to
# prove there is NO cap and the last episode stays reachable. Each
# episode page embeds a <source> pointing at a distinct media file.
# The config 'list's the episode hrefs; the per-episode pipeline
# resolves each <source>.
N=60
mkdir -p "${srvdir}/play"
for i in 1 2 3; do
cat > "${srvdir}/play/Ep0${i}.html" <<HTML
{
echo '<!doctype html><html><body>'
for i in $(seq 1 "${N}"); do
ii="$(printf '%02d' "${i}")"
head -c $((1024 + i)) /dev/urandom > "${srvdir}/ep${i}.bin"
cat > "${srvdir}/play/Ep${ii}.html" <<HTML
<!doctype html><html><body>
<video><source src="/ep${i}.bin" type="video/mp4"></video>
</body></html>
HTML
done
cat > "${srvdir}/series.html" <<'HTML'
<!doctype html><html><body>
<a data-num="1" href="/play/Ep01.html">1</a>
<a data-num="2" href="/play/Ep02.html">2</a>
<a data-num="3" href="/play/Ep03.html">3</a>
</body></html>
HTML
echo "<a data-num=\"${i}\" href=\"/play/Ep${ii}.html\">${i}</a>"
done
echo '</body></html>'
} > "${srvdir}/series.html"

port="$(python3 -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()')"
python3 -m http.server "${port}" \
Expand Down Expand Up @@ -453,32 +464,53 @@ jobs:
cmp -s "$1" "$2" || { echo "byte mismatch: $2" >&2; exit 1; }
}

# --episodes 1,3 : exactly episodes 1 and 3, byte-exact; 2 absent.
# --episodes over a LARGE list: a spread including the LAST episode
# (60) proves no truncation. Exactly {1,3,60} land; 2 is absent.
out="${workdir}/sel"
mkdir -p "${out}"
"${flux}" -q --episodes 1,3 -o "${out}/" \
"${flux}" -q --episodes 1,3,60 -o "${out}/" \
"http://127.0.0.1:${port}/series.html"
assert_match "${srvdir}/ep1.bin" "${out}/ep01-ep1.bin"
assert_match "${srvdir}/ep3.bin" "${out}/ep03-ep3.bin"
test ! -e "${out}/ep02-ep2.bin" \
assert_match "${srvdir}/ep1.bin" "${out}/ep1.bin"
assert_match "${srvdir}/ep3.bin" "${out}/ep3.bin"
assert_match "${srvdir}/ep60.bin" "${out}/ep60.bin"
test ! -e "${out}/ep2.bin" \
|| { echo "episode 2 should not have been downloaded" >&2; exit 1; }

# --all : every episode, byte-exact.
# Re-run with ep1 already present: it must be SKIPPED, the newly
# requested ep2 downloaded. Proves on-disk status + skip-if-present.
"${flux}" --episodes 1,2 -o "${out}/" \
"http://127.0.0.1:${port}/series.html" > "${workdir}/rerun.log" 2>&1
grep -q "already on disk, skipping" "${workdir}/rerun.log" \
|| { echo "re-run did not skip the on-disk episode" >&2;
cat "${workdir}/rerun.log" >&2; exit 1; }
assert_match "${srvdir}/ep2.bin" "${out}/ep2.bin"

# --all : every one of the N episodes, byte-exact; count matches N.
alldir="${workdir}/all"
mkdir -p "${alldir}"
"${flux}" -q --all -o "${alldir}/" \
"http://127.0.0.1:${port}/series.html"
assert_match "${srvdir}/ep1.bin" "${alldir}/ep01-ep1.bin"
assert_match "${srvdir}/ep2.bin" "${alldir}/ep02-ep2.bin"
assert_match "${srvdir}/ep3.bin" "${alldir}/ep03-ep3.bin"

# A bad --episodes spec must be rejected with a non-zero exit.
if "${flux}" -q --episodes 9 -o "${out}/" \
assert_match "${srvdir}/ep1.bin" "${alldir}/ep1.bin"
assert_match "${srvdir}/ep30.bin" "${alldir}/ep30.bin"
assert_match "${srvdir}/ep60.bin" "${alldir}/ep60.bin"
got="$(ls "${alldir}" | wc -l)"
test "${got}" -eq "${N}" \
|| { echo "expected ${N} files, got ${got}" >&2; exit 1; }

# --all RE-RUN: everything is present, so 0 downloaded, N skipped.
"${flux}" --all -o "${alldir}/" \
"http://127.0.0.1:${port}/series.html" > "${workdir}/all2.log" 2>&1
grep -qE "0 downloaded, ${N} skipped" "${workdir}/all2.log" \
|| { echo "re-run --all should skip everything" >&2;
tail -3 "${workdir}/all2.log" >&2; exit 1; }

# A bad --episodes spec (out of range over N) must exit non-zero.
if "${flux}" -q --episodes 99 -o "${out}/" \
"http://127.0.0.1:${port}/series.html" 2>/dev/null; then
echo "bad --episodes spec was accepted" >&2; exit 1
fi

echo "Series e2e OK: --episodes 1,3 + --all byte-exact, bad spec rejected"
echo "Series e2e OK: N=${N} no cap, --episodes/--all byte-exact, on-disk skip + re-run, bad spec rejected"

sanitizers:
name: sanitizers
Expand Down Expand Up @@ -513,6 +545,19 @@ jobs:
echo "${out}"
echo "${out}" | grep -q '^OK: all'

- name: ASan/UBSan unit test (tui window/row math)
run: |
set -euo pipefail
# Pure window/row-count helpers under sanitizers.
clang -D_DEFAULT_SOURCE -Wall -Wextra -g \
-fsanitize=address,undefined -fno-sanitize-recover=all \
src/tui.c src/test_tui.c -o /tmp/test_tui_san
export ASAN_OPTIONS=detect_leaks=1:abort_on_error=1
export UBSAN_OPTIONS=print_stacktrace=1:halt_on_error=1
out="$(/tmp/test_tui_san)"
echo "${out}"
echo "${out}" | grep -q '^OK: all'

- name: ASan/UBSan unit test (hls)
run: |
set -euo pipefail
Expand Down
17 changes: 17 additions & 0 deletions ChangeLog
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,23 @@ kept unchanged for the record, including their original author names.



Version: 2.0.0, 2026-06-18

[ Hyperflux contributors ]

* series: a 'list' config opens an interactive multi-select episode TUI with on-disk
status (missing pre-selected, 'a' selects all); batch download skips files already
present; episodes keep their remote filename; --all and --episodes select
non-interactively
* tui: skills-styled multi-select (no ncurses) with windowed scrolling and no list
cap, wrapping-aware redraw, SIGWINCH-safe, terminal restored on every exit
* extractor: interpolate {var} in var/list regex arguments (slug-anchored episode
lists); animeworld.conf is now a series config; epid charset accepts '-' and '_'
* packaging: bundled extractor configs install active into
/usr/share/hyperflux/extractors (deb/rpm/apk and the native rpm spec)
* scan: --extract-scan saves the generated config into the active user dir by
default; HTML no-match hint

Version: 1.1.0, 2026-06-18

[ Hyperflux contributors ]
Expand Down
27 changes: 27 additions & 0 deletions NEWS
Original file line number Diff line number Diff line change
@@ -1,3 +1,30 @@
Hyperflux 2.0.0 (2026-06-18)
============================

Download a whole series, not just one file. Point flux at any episode page of a
supported site and it opens an interactive picker listing every episode: the ones
already on disk are ticked, the rest start selected, and one key selects them all.
Confirm and flux downloads only what's missing, keeping each file under the same
name it has on the site, so re-running later grabs just the new episodes.

* The episode picker is a clean terminal UI: arrow keys or j/k to move, space to
toggle, "a" to select all, enter to download, q to quit. It scrolls through long
series with no cap and redraws cleanly when the terminal is resized.

* Extractor configs now work out of the box: the bundled config installs active in a
system directory, so a fresh install resolves matching pages with no setup, and
"flux --extract-scan <url>" writes the config it generates straight into your
active config dir. A download that turns out to be a web page now hints you to run
--extract-scan instead of silently saving the HTML.

* Config regexes can interpolate captured variables, which is what lets a config
enumerate a series' episodes.

Everything from 1.x still applies: a page that matches no config is a normal direct
download, HLS streams are fetched in parallel and assembled, and packages are on the
apt/dnf/apk Cloudsmith repositories.


Hyperflux 1.1.0 (2026-06-18)
============================

Expand Down
2 changes: 1 addition & 1 deletion VERSION
Original file line number Diff line number Diff line change
@@ -1 +1 @@
1.1.0 2026-06-18T12:57:14Z
2.0.0 2026-06-18T17:16:13Z
24 changes: 21 additions & 3 deletions examples/extractors/animeworld.conf
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@
#
# Then just run: flux 'https://www.animeworld.ac/play/anime-name.xxxx/AbC123'
#
# This is a SERIES config: the play page lists every episode of the series, so
# `flux <episode-url>` opens an episode multi-select and downloads what you pick.
#
# SECURITY NOTE: an extractor config issues arbitrary HTTP requests with the
# headers it declares. Treat shared configs like scripts and review them before
# use. Hyperflux never fetches configs from a URL and never forwards your
Expand All @@ -21,9 +24,24 @@ name animeworld
# Multiple `match` lines are OR-ed. POSIX Extended Regular Expression.
match animeworld\.[a-z]+/play/

# The episode id is the last path segment of the page URL.
# `var <name> <- <source> regex <ERE>` captures group 1 of the ERE.
var epid <- url regex /play/[^/]+/([A-Za-z0-9]+)
# --- series setup (runs once on the page; skipped per-episode) ---------------
#
# The play page of any episode lists ALL episodes of that series. Capture the
# series slug ("anime-name.animeid") from the page URL, fetch the page, then
# `list` only the /play/<slug>/<epid> hrefs so just THIS series is captured.
# A {var} in a var/list regex is interpolated against the store before the
# regex compiles, so {slug} anchors the list to this series.
var slug <- url regex /play/([^/]+)/
get page <- {url}
list eps <- page regex href="(/play/{slug}/[A-Za-z0-9_-]+)"

# --- per-episode pipeline (runs once per selected episode, with {url} bound to
# that episode's page URL; the directives above are skipped here) -------------

# The episode id is the last path segment of the episode URL. epids may contain
# '-' and '_' (e.g. "VbGT-F"), so the charset is [A-Za-z0-9_-]+, not just
# alphanumerics.
var epid <- url regex /play/[^/]+/([A-Za-z0-9_-]+)

# The landing HTML does NOT contain the media URL; the player loads it from an
# internal API. `get` performs an HTTP GET and names the response body so later
Expand Down
2 changes: 1 addition & 1 deletion po/Makevars
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ subdir = po
top_builddir = ..

# These options get passed to xgettext.
XGETTEXT_OPTIONS = --keyword=_ --keyword=N_ --omit-header
XGETTEXT_OPTIONS = --from-code=UTF-8 --keyword=_ --keyword=N_ --omit-header

# This is the copyright holder that gets inserted into the header of the
# $(DOMAIN).pot file. Set this to the copyright holder of the surrounding
Expand Down
40 changes: 32 additions & 8 deletions src/extractor.c
Original file line number Diff line number Diff line change
Expand Up @@ -886,16 +886,24 @@ extractor_run(const extractor_t *ex, const char *page_url,
d->name, d->source);
goto done;
}
/* Interpolate {var}s in the regex before it compiles. */
char *ere = interpolate(d->arg, &st, err);
if (!ere)
goto done;
char *cap = NULL;
int r = regex_capture1(d->arg, src, &cap, err);
if (r < 0)
int r = regex_capture1(ere, src, &cap, err);
if (r < 0) {
free(ere);
goto done;
}
if (r == 0) {
ext_set_err(err, ex->path, 0,
"var '%s': regex /%s/ did not match source '%s'",
d->name, d->arg, d->source);
d->name, ere, d->source);
free(ere);
goto done;
}
free(ere);
if (store_set(&st, d->name, cap) < 0) {
ext_set_err(err, ex->path, 0,
"var '%s': store full", d->name);
Expand Down Expand Up @@ -1191,23 +1199,31 @@ extractor_list_episodes(const extractor_t *ex, const char *page_url,
d->name, d->source);
goto done;
}
/* Interpolate {var}s (e.g. a slug) before the regex
* compiles, so the list can be anchored to this series. */
char *ere = interpolate(d->arg, &st, err);
if (!ere)
goto done;
char *cerr = NULL;
int rc = regex_capture_all(d->arg, src,
int rc = regex_capture_all(ere, src,
EXTRACTOR_MAX_EPISODES,
&raw, &nraw, &cerr);
if (rc < 0) {
if (err)
*err = cerr;
else
free(cerr);
free(ere);
goto done;
}
if (nraw == 0) {
ext_set_err(err, ex->path, 0,
"list '%s': regex /%s/ matched no items in source '%s'",
d->name, d->arg, d->source);
d->name, ere, d->source);
free(ere);
goto done;
}
free(ere);
break; /* one 'list' per config; stop at the first */
} else if (d->kind == EXT_DIR_VAR) {
const char *src = store_get(&st, d->source,
Expand All @@ -1218,16 +1234,24 @@ extractor_list_episodes(const extractor_t *ex, const char *page_url,
d->name, d->source);
goto done;
}
/* Interpolate {var}s in the regex before it compiles. */
char *ere = interpolate(d->arg, &st, err);
if (!ere)
goto done;
char *cap = NULL;
int r = regex_capture1(d->arg, src, &cap, err);
if (r < 0)
int r = regex_capture1(ere, src, &cap, err);
if (r < 0) {
free(ere);
goto done;
}
if (r == 0) {
ext_set_err(err, ex->path, 0,
"var '%s': regex /%s/ did not match source '%s'",
d->name, d->arg, d->source);
d->name, ere, d->source);
free(ere);
goto done;
}
free(ere);
if (store_set(&st, d->name, cap) < 0) {
ext_set_err(err, ex->path, 0,
"var '%s': store full", d->name);
Expand Down
Loading
Loading