docs: update README and CHANGELOG for v1.0.6 proxy features

Author: Inside4ndroid

CHANGELOG.md

@@ -2,6 +2,47 @@
 
 All notable changes to this project will be documented in this file.
 
+## [1.0.6] - 2025-09-20
+
+### Removed
+- MoviesClub provider (multi-server scraping complexities & Turnstile challenge; deprecated permanently)
+- Xprime provider (upstream Xprime.tv offline due to security changes)
+ - 4khdhub provider: all `.zip` archive links are now omitted entirely (previous releases experimented with stripping the extension which produced non‑playable pseudo‑MKV links)
+
+### Added
+- Structured multi-server debug instrumentation (session summaries, per-fetch metrics, pattern counters)
+- Turnstile challenge detection & bypass attempt scaffold (synthetic `/rcp_verify` token posting)
+- Optional internal stream proxy (`enableProxy` flag): mounts `/m3u8-proxy`, `/ts-proxy`, `/sub-proxy` with playlist + segment + subtitle handling and segment prefetch cache.
+ - Proxy range management features:
+   - `clampOpen` (default on) – caps ambiguous open‑ended `bytes=0-` requests to a bounded initial window (`openChunkKB`, default 4096 KB)
+   - `progressiveOpen` (default on) – incremental expansion of the head range on successive `bytes=0-` requests instead of a single huge span
+   - `initChunkKB` (default 512 KB) – size of the synthetic initial 206 response when neither clamp/progressive produce a range and `noSynth` is not set
+   - `tailPrefetch` (default on) + `tailPrefetchKB` (default 256 KB) – asynchronous fetch & in‑memory cache of the file tail to satisfy rapid VLC tail probes
+   - `force200` (opt‑in) – normalizes upstream 206 responses to 200 for diagnostics
+   - `noSynth` (opt‑in) – disables synthetic initial partial response generation
+ - Tail prefetch TTL cleanup task (30 min window) and in‑memory maps for: segment cache, open range clamp, progressive growth, and tail buffers
+ - Host routing overrides: `pixeldrain.*` & `video-downloads.googleusercontent.com` are forced through `/ts-proxy` (extensionless or ambiguous content)
+
+### Changed
+- Centralized multi-server request headers with realistic `sec-ch-ua*` & `Sec-Fetch-*` values
+- Added retry, rotating User-Agent, and cookie jar logic to multi-server fetch pipeline
+- Showbox provider priority map updated after Xprime removal
+- README/Docs trimmed to reflect current active providers only
+- When `enableProxy` is active, stream response objects have their original `headers` field removed (proxy handles all required headers internally).
+ - 4khdhub provider now filters out archive endpoints instead of attempting extension normalization (prevents feeding ZIP files to players)
+ - Open‑ended range handling improved to reduce VLC negotiation loops by throttling first‑pass read size and growing progressively
+ - Synthetic initial partial response is automatically suppressed when `progressiveOpen` is active (real range growth preferred)
+
+### Fixed
+- Ensured multi-server fallback attempts (direct rcp player/m3u8 extraction) operate with improved diagnostics
+ - Eliminated repeated VLC tail probe stalls caused by archive masquerading as video content (root cause was filtered by dropping `.zip` URLs)
+
+### Documentation
+ - Updated README version badge to 1.0.6 and provider list (removed MoviesClub & Xprime, clarified active providers list)
+ - Added proxy tuning parameter reference (clamp/progressive/tail prefetch, synthetic partial, force200) and host override notes
+ - Expanded explanation that per‑stream headers are stripped when proxying is enabled
+
+
 ## [1.0.5] - 2025-09-19
 
 ### Improved
@@ -10,13 +51,11 @@ All notable changes to this project will be documented in this file.
 - Tightened URL validation: removed unconditional trust for `r2.dev`; validation logic now consistent across hosts.
 - Host distribution instrumentation logs final hostname counts for easier diagnostics.
 - Preserved HubCloud worker `.zip` links by stripping the `.zip` extension instead of discarding them (enables direct playback attempts).
-- MoviesClub provider: Added automatic `Origin`/`Referer` headers for `vidora.stream` sources to prevent 403 responses.
 
 ### Notes
 - `r2.dev` links are always removed from final output; no env flag required.
 
 ### Documentation
-- README: Updated Providers section, unified schema example, MoviesClub listed, registry mapping instructions, and badge version.
 
 ## [1.0.4] - 2025-09-18
 
@@ -70,6 +109,8 @@ All notable changes to this project will be documented in this file.
 ## [1.0.0] - 2025-09-16
 - Initial stable release.
 
+[1.0.6]: https://github.com/Inside4ndroid/TMDB-Embed-API/compare/v1.0.5...v1.0.6
+[1.0.5]: https://github.com/Inside4ndroid/TMDB-Embed-API/compare/v1.0.4...v1.0.5
 [1.0.3]: https://github.com/Inside4ndroid/TMDB-Embed-API/compare/v1.0.2...v1.0.3
 [1.0.4]: https://github.com/Inside4ndroid/TMDB-Embed-API/compare/v1.0.3...v1.0.4
 [1.0.2]: https://github.com/Inside4ndroid/TMDB-Embed-API/compare/v1.0.1...v1.0.2

README.md

@@ -6,7 +6,7 @@
   <img src="https://img.shields.io/badge/Node.js-18%2B-brightgreen?style=flat" />
   <img src="https://img.shields.io/badge/Status-Active-success?style=flat" />
   <img src="https://img.shields.io/badge/License-MIT-blue?style=flat" />
-  <img src="https://img.shields.io/badge/Version-1.0.5-informational?style=flat" />
+  <img src="https://img.shields.io/badge/Version-1.0.6-informational?style=flat" />
   <img src="https://img.shields.io/docker/pulls/inside4ndroid/tmdb-embed-api?label=Docker%20Pulls&style=flat" />
 </p>
 
@@ -34,7 +34,7 @@
 
 ## ✨ Features
 - **Multi‑TMDB Key Rotation** – Supply multiple API keys; one is chosen randomly per request.
-- **Provider Aggregation** – Pluggable providers (Showbox, 4khdhub, MoviesMod, MP4Hydra, VidZee, Vixsrc, Xprime, MoviesClub) with per‑provider enable toggles + default selection.
+- **Provider Aggregation** – Pluggable providers (Showbox, 4khdhub, MoviesMod, MP4Hydra, VidZee, Vixsrc, UHDMovies) with per‑provider enable toggles + default selection.
 - **🔥 Plugin System** – Drop new provider files in `providers/` and add its exported function to the registry map (`providers/registry.js` → `providerFunctionMap`).
 - **Dynamic Filtering** – Minimum quality presets, custom JSON quality map, codec exclusion rules (presets + JSON).
 - **Runtime Overrides UI** – Fully interactive web admin at `/` (login protected) writing to `utils/user-config.json`.
@@ -43,6 +43,23 @@
 - **Config Propagation** – Overrides mirrored to `process.env` for legacy compatibility (no `.env` required after first save).
 - **Back‑Navigation Safe** – Cache-control + visibility/session revalidation.
 - **Extensible** – Simple drop-in provider plugin system.
+- **Optional Stream Proxy Layer** – When enabled, rewrites returned stream URLs so HLS playlists, TS segments, and subtitles are served through internal endpoints (`/m3u8-proxy`, `/ts-proxy`, `/sub-proxy`) allowing uniform headers, origin shielding, and optional segment caching.
+  When active the API omits per-stream `headers` objects from responses (they're no longer needed by clients) to avoid leaking upstream header requirements.
+
+  Proxy Tuning Parameters (query flags accepted by `/ts-proxy` – defaults shown):
+  - `clampOpen` (on) – If a client sends an ambiguous `Range: bytes=0-`, constrain it to an initial window of `openChunkKB` (default 4096 KB) to avoid huge first reads.
+  - `openChunkKB=4096` – Size (KB) used for both clamp window and each progressive expansion increment.
+  - `progressiveOpen` (on) – Grow successive ambiguous head requests (`bytes=0-`) incrementally instead of one large span. Maintains a per‑URL expansion map.
+  - `initChunkKB=512` – Size used for a synthetic initial partial (206) when no client range is provided and progressive growth is disabled. Capped 64–2048 KB.
+  - `noSynth=1` – Disable synthetic initial partial generation (forces pass‑through behavior).
+  - `force200=1` – Normalize upstream 206 responses to 200 (diagnostics / edge player testing).
+  - `tailPrefetch` (on) – Enable asynchronous tail fetch of the file’s last bytes to satisfy rapid player tail probes.
+  - `tailPrefetchKB=256` – Tail window size (64–2048 KB). Cached in memory with TTL cleanup.
+  Behavior Notes:
+  - Synthetic partials auto‑disable when `progressiveOpen` is active (real progressive ranges preferred).
+  - Player tail probes (e.g., VLC metadata scans) are accelerated by the cached tail window.
+  - Forced 200 mode strips `Content-Range` to emulate full responses for troubleshooting.
+  - Host Overrides: `pixeldrain.*` and `video-downloads.googleusercontent.com` URLs are routed through `/ts-proxy` regardless of extension to ensure correct range + MIME handling.
 
 ---
 
@@ -217,9 +234,8 @@ The API supports a plugin system. Drop a new provider file in the `providers/` f
 - `mp4hydra` - MP4Hydra streams
 - `vidzee` - VidZee streams
 - `vixsrc` - Vixsrc streams
-- `xprime` - Xprime streams
 - `uhdmovies` - UHD Movies streams
-- `moviesclub` - MoviesClub streams
+
 
 ### Adding a New Provider
 1. **Create** `providers/yourprovider.js` with your stream fetching logic
@@ -234,9 +250,7 @@ The API supports a plugin system. Drop a new provider file in the `providers/` f
      'MP4Hydra.js': 'getMP4HydraStreams',
      'VidZee.js': 'getVidZeeStreams',
      'vixsrc.js': 'getVixsrcStreams',
-     'xprime.js': 'getXprimeStreams',
-     'uhdmovies.js': 'getUHDMoviesStreams',
-     'moviesclub.js': 'getMoviesClubStreams',
+  'uhdmovies.js': 'getUHDMoviesStreams',
      'yourprovider.js': 'getYourproviderStreams'
    };
    ```
@@ -299,6 +313,21 @@ Aggregate endpoint auto-resolves IMDb when needed and merges provider output bef
 
 Filtering passes through `applyFilters` to enforce min quality + codec exclusions.
 
+> Note: When the `enableProxy` flag is turned on, provider-specific request headers are stripped from each stream object before responding. Clients should use the proxied URL directly without adding custom Referer/Origin headers.
+
+---
+
+## ⚙️ Configuration Flags (Advanced Panel)
+| Flag | Default | Purpose |
+|------|---------|---------|
+| disableCache | false | Disables internal caches (Showbox + proxy segment cache) |
+| enablePStreamApi | true | Enables Showbox PStream-specific handling |
+| disableUrlValidation | false | Skip general URL pattern validation checks |
+| disable4khdhubUrlValidation | false | Skip 4khdhub-specific URL validation |
+| enableProxy | false | Mounts proxy routes and rewrites stream URLs through them |
+
+Toggle `enableProxy` to activate the internal proxy. This adds lightweight playlist/segment/subtitle rewriting without modifying provider code. Disable it to return direct upstream URLs.
+
 ---
 
 ## 🧩 Quality & Codec Filtering

providers/moviesclub.js

@@ -3,6 +3,19 @@
 
 const axios = require('axios');
 const JsUnpacker = require('../utils/jsunpack');
+const { config } = require('../utils/config');
+
+function applyMoviesClubProxy(url) {
+  try {
+    const base = config.moviesclubProxy;
+    if (!base || typeof base !== 'string') return url;
+    // Ensure base ends with either = or a delimiter allowing raw append
+    const proxied = base + encodeURIComponent(url);
+    return proxied;
+  } catch {
+    return url;
+  }
+}
 
 async function getMoviesClubStreams(tmdbId, mediaType, season, episode) {
   console.log(`[moviesclub] Fetching streams for ${mediaType} tmdbId=${tmdbId} season=${season} episode=${episode}`);
@@ -52,7 +65,7 @@ async function getMoviesClubStreams(tmdbId, mediaType, season, episode) {
     // Check if this is a cdn.moviesapi.club embed URL with multiple servers
     if (embedUrl.includes('cdn.moviesapi.club')) {
       console.log('[moviesclub] Detected cdn.moviesapi.club embed URL, handling multiple servers');
-      return await handleMultipleServers(embedUrl, mediaType, season, episode, tmdbId);
+      return null;
     }
 
     // Handle regular embed URLs (existing logic)
@@ -104,11 +117,12 @@ async function handleMultipleServers(embedUrl, mediaType, season, episode, tmdbI
 
           // If it's a URL, use it directly
           if (decodedSrc.startsWith('http')) {
+            const rawUrl = decodedSrc;
             const stream = {
               name: `MoviesClub - Direct Stream`,
               title: `${mediaType === 'movie' ? 'Movie' : `S${season.toString().padStart(2, '0')}E${episode.toString().padStart(2, '0')}`} ${tmdbId} - MoviesClub Direct`,
-              url: decodedSrc,
-              quality: extractQualityFromUrl(decodedSrc) || '720p'
+              url: applyMoviesClubProxy(rawUrl),
+              quality: extractQualityFromUrl(rawUrl) || '720p'
             };
             streams.push(stream);
             console.log(`[moviesclub] Added direct stream from iframe`);
@@ -209,11 +223,12 @@ async function handleMultipleServers(embedUrl, mediaType, season, episode, tmdbI
         console.log(`[moviesclub] Extracted stream URL: ${streamUrl.substring(0, 100)}...`);
 
         // Create stream object
+        const rawUrl = streamUrl;
         const stream = {
           name: `MoviesClub - ${serverName}`,
           title: `${mediaType === 'movie' ? 'Movie' : `S${season.toString().padStart(2, '0')}E${episode.toString().padStart(2, '0')}`} ${tmdbId} - ${serverName}`,
-          url: streamUrl,
-          quality: extractQualityFromUrl(streamUrl) || '720p',
+          url: applyMoviesClubProxy(rawUrl),
+          quality: extractQualityFromUrl(rawUrl) || '720p',
           headers: {}
         };
 
@@ -304,20 +319,17 @@ async function handleSingleServer(embedUrl, mediaType, season, episode, tmdbId)
 
     // Extract streams from the configuration
     if (playerConfig.file) {
+      const rawUrl = playerConfig.file;
       const stream = {
         name: `MoviesClub - ${playerConfig.title || 'Server 1'}`,
         title: playerConfig.title || `${mediaType === 'movie' ? 'Movie' : `S${season.toString().padStart(2, '0')}E${episode.toString().padStart(2, '0')}`} ${tmdbId} - MoviesClub`,
-        url: playerConfig.file,
-        quality: extractQualityFromUrl(playerConfig.file) || '720p',
-        headers: (() => {
-          if (playerConfig.file.includes('vidora.stream')) {
-            return {
-              Origin: 'https://vidora.stream',
-              Referer: 'https://vidora.stream/'
-            };
-          }
-          return {};
-        })()
+        url: applyMoviesClubProxy(rawUrl),
+        quality: extractQualityFromUrl(rawUrl) || '720p',
+        // Always include vidora headers for single-server results to prevent 403 playback issues
+        headers: {
+          Origin: 'https://vidora.stream',
+          Referer: 'https://vidora.stream/'
+        }
       };
 
       // Add poster if available
@@ -387,11 +399,12 @@ function extractStreamsFromServer(serverData, serverName, mediaType, season, epi
         console.log(`[moviesclub] Server response is not JSON, treating as direct URL: ${serverData.substring(0, 100)}`);
         // Treat as direct stream URL
         if (serverData.startsWith('http')) {
+          const rawUrl = serverData;
           const stream = {
             name: `MoviesClub - ${serverName}`,
             title: `${mediaType === 'movie' ? 'Movie' : `S${season.toString().padStart(2, '0')}E${episode.toString().padStart(2, '0')}`} ${tmdbId} - ${serverName}`,
-            url: serverData,
-            quality: extractQualityFromUrl(serverData) || '720p',
+            url: applyMoviesClubProxy(rawUrl),
+            quality: extractQualityFromUrl(rawUrl) || '720p',
             headers: serverData.includes('vidora.stream') ? { Origin: 'https://vidora.stream', Referer: 'https://vidora.stream/' } : {}
           };
           streams.push(stream);
@@ -409,12 +422,13 @@ function extractStreamsFromServer(serverData, serverName, mediaType, season, epi
         if (serverData[key] && typeof serverData[key] === 'string' && serverData[key].startsWith('http')) {
           console.log(`[moviesclub] Found stream URL in key '${key}': ${serverData[key].substring(0, 100)}`);
 
+          const rawUrl = serverData[key];
           const stream = {
             name: `MoviesClub - ${serverName}`,
             title: `${mediaType === 'movie' ? 'Movie' : `S${season.toString().padStart(2, '0')}E${episode.toString().padStart(2, '0')}`} ${tmdbId} - ${serverName}`,
-            url: serverData[key],
-            quality: extractQualityFromUrl(serverData[key]) || '720p',
-            headers: serverData[key].includes('vidora.stream') ? { Origin: 'https://vidora.stream', Referer: 'https://vidora.stream/' } : {}
+            url: applyMoviesClubProxy(rawUrl),
+            quality: extractQualityFromUrl(rawUrl) || '720p',
+            headers: rawUrl.includes('vidora.stream') ? { Origin: 'https://vidora.stream', Referer: 'https://vidora.stream/' } : {}
           };
 
           // Add poster if available

public/config.html

@@ -182,6 +182,7 @@ <h1>Paths</h1>
         <p class="hint">Provider enable/disable & feature flags apply immediately. Click toggles to enable (highlighted) or disable (dim).</p>
         <div class="adv-group">
           <div class="field small"><label>Showbox Cache Dir <input type="text" name="adv_showboxCacheDir" placeholder="/data/showbox-cache"/></label></div>
+          <div class="field small"><label>MoviesClub Proxy URL <p class="hint">Optional. Example: <code>https://winter-surf-e49f.inside4ndroid-techsup.workers.dev/?destination=</code><br/>Leave blank for direct links. The raw stream URL will be appended (URL encoded).</p><input type="text" name="adv_moviesclubProxy" placeholder="https://proxy/?destination=" /></label></div>
         </div>
       </section>
       <section id="panel-live" class="panel">

public/index.js

@@ -174,6 +174,7 @@ function fillForm(data){
 	advBools.forEach(([cfgKey, elName])=>{ const el = f.elements[elName]; if(!el) return; const src = (override[cfgKey]!==undefined? override[cfgKey] : merged[cfgKey]); if(el.type==='checkbox'){ el.checked = !!src; } else { el.value = src? 'true':''; } });
 	const advTexts = [
 	['showboxCacheDir','adv_showboxCacheDir']
+	,['moviesclubProxy','adv_moviesclubProxy']
 	];
 	advTexts.forEach(([cfgKey, elName])=>{ const el = f.elements[elName]; if(!el) return; const val = override[cfgKey]!==undefined? override[cfgKey] : merged[cfgKey]; el.value = val || ''; });
 	// Sync checkbox style advanced flags
@@ -257,6 +258,7 @@ async function save(){
 	};
 	Object.entries(boolMap).forEach(([formName,cfgKey])=>{ const el = f.elements[formName]; if(el) payload[cfgKey] = !!el.checked; });
 	const textMap = { adv_showboxCacheDir:'showboxCacheDir' };
+	textMap.adv_moviesclubProxy = 'moviesclubProxy';
 	Object.entries(textMap).forEach(([formName,cfgKey])=>{ const el = f.elements[formName]; if(!el) return; const v = el.value.trim(); payload[cfgKey] = v? v : null; });
 	const tmdbHidden = f.elements['tmdbApiKeysHidden'];
 	if (tmdbHidden){

utils/config.js

@@ -113,6 +113,11 @@ function normalizeConfig(base) {
   // Proxy features removed; always use direct connections
   if (cfg.disableUrlValidation === undefined) cfg.disableUrlValidation = false;
   if (cfg.disable4khdhubUrlValidation === undefined) cfg.disable4khdhubUrlValidation = false;
+  // MoviesClub optional proxy base (string URL with trailing ?destination= or similar)
+  if (cfg.moviesclubProxy !== undefined && typeof cfg.moviesclubProxy === 'string') {
+    cfg.moviesclubProxy = cfg.moviesclubProxy.trim();
+    if (!cfg.moviesclubProxy) delete cfg.moviesclubProxy;
+  }
   return cfg;
 }
 
@@ -152,6 +157,8 @@ function applyConfigToEnv(cfg){
   delete process.env.VIDZEE_PROXY_URL;
   delete process.env.VIDSRC_PROXY_URL;
   delete process.env.MOVIESMOD_PROXY_URL;
+  // Mirror moviesclub proxy for potential legacy usage
+  if (cfg.moviesclubProxy) process.env.MOVIESCLUB_PROXY_URL = cfg.moviesclubProxy; else delete process.env.MOVIESCLUB_PROXY_URL;
   if (cfg.defaultRegion) process.env.FEBBOX_REGION = cfg.defaultRegion; // alias
 }