diff --git a/web-ui/src/components/player/video-player.tsx b/web-ui/src/components/player/video-player.tsx index 5e9e39de..a5ff76b6 100644 --- a/web-ui/src/components/player/video-player.tsx +++ b/web-ui/src/components/player/video-player.tsx @@ -75,6 +75,9 @@ export function VideoPlayer({ const stablePlaybackTimeoutRef = useRef(0); // Whether to auto-play after player recreation (true for initial load and "go live") const shouldAutoPlayRef = useRef(true); + // Whether the pause was an explicit user action (vs. the OS pausing on backgrounding). + // Used to decide if playback should auto-resume when the page returns to foreground. + const userPausedRef = useRef(false); // Digit input state const [digitBuffer, setDigitBuffer] = useState(""); @@ -107,6 +110,7 @@ export function VideoPlayer({ if (goingLive && playMode === "live" && video) { // "Go Live": jump to the end of buffered range, resume playback and liveSync + userPausedRef.current = false; video.play(); const buffered = video.buffered; if (buffered.length > 0) { @@ -137,8 +141,10 @@ export function VideoPlayer({ const video = videoRef.current; if (video) { if (video.paused) { + userPausedRef.current = false; video.play(); } else { + userPausedRef.current = true; video.pause(); } } @@ -278,6 +284,42 @@ export function VideoPlayer({ player?.setLiveSync(playMode === "live"); }, [playMode, player]); + // Media Session: lock screen / control center metadata (esp. useful during PiP playback) + useEffect(() => { + if (!("mediaSession" in navigator)) return; + if (!channel) { + navigator.mediaSession.metadata = null; + return; + } + navigator.mediaSession.metadata = new MediaMetadata({ + title: currentProgram?.title || channel.name, + artist: currentProgram?.title ? channel.name : channel.group, + artwork: channel.logo ? [{ src: channel.logo }] : [], + }); + }, [channel, currentProgram]); + + useEffect(() => { + if (!("mediaSession" in navigator)) return; + navigator.mediaSession.playbackState = isPlaying ? "playing" : "paused"; + }, [isPlaying]); + + // Media Session action handlers (lock screen / control center play & pause) + useEffect(() => { + if (!("mediaSession" in navigator)) return; + navigator.mediaSession.setActionHandler("play", () => { + userPausedRef.current = false; + videoRef.current?.play(); + }); + navigator.mediaSession.setActionHandler("pause", () => { + userPausedRef.current = true; + videoRef.current?.pause(); + }); + return () => { + navigator.mediaSession.setActionHandler("play", null); + navigator.mediaSession.setActionHandler("pause", null); + }; + }, []); + // Load segments whenever they change (channel switch, seek, retry — all go through here) const handleLoadSegments = useEffectEvent((newSegments: PlayerSegment[]) => { if (!newSegments.length || !player) return; @@ -298,6 +340,7 @@ export function VideoPlayer({ if (shouldAutoPlayRef.current) { const video = videoRef.current; if (video) { + userPausedRef.current = false; const playPromise = video.play(); if (playPromise) { playPromise @@ -397,6 +440,55 @@ export function VideoPlayer({ setIsPiP(false); }); + // Foreground recovery: iOS pauses web media when the page goes to background + // without PiP, and may even tear down the whole media pipeline (MediaSource + // close + decode error). When the page becomes visible again, resume playback — + // rebuilding the stream when the old session is dead or stale. + const handleVisibilityChange = useEffectEvent(() => { + if (document.visibilityState !== "visible") return; + const video = videoRef.current; + if (!video || !player || error || needsUserInteraction) return; + // PiP keeps playing in background; nothing to recover + if (document.pictureInPictureElement) return; + // Respect an explicit user pause; only recover from OS-initiated interruptions + if (userPausedRef.current) return; + + // Media element died in background (MediaSource closed / decode error). + // Note: video.paused may still report false in this state. + const mediaDead = video.error !== null; + const behindLiveMs = Date.now() - (streamStartTime.getTime() + currentVideoTime * 1000); + + if (playMode === "live" && (mediaDead || behindLiveMs > 10000)) { + // Dead session or stale buffer — rebuild the stream at the live edge + console.log("Reloading at live edge after background suspension"); + shouldAutoPlayRef.current = true; + onSeek?.(new Date()); + return; + } + + if (mediaDead) { + // Catchup: rebuild the stream at the current position + console.log("Reloading at current position after background suspension"); + shouldAutoPlayRef.current = true; + onSeek?.(new Date(streamStartTime.getTime() + currentVideoTime * 1000)); + return; + } + + if (video.paused) { + video.play()?.catch((err: Error) => { + if (err.name === "NotAllowedError") { + setNeedsUserInteraction(true); + } + }); + } + }); + + useEffect(() => { + const handler = () => handleVisibilityChange(); + document.addEventListener("visibilitychange", handler); + return () => document.removeEventListener("visibilitychange", handler); + }, []); + const handleKeyDown = useEffectEvent((e: KeyboardEvent) => { if (e.target instanceof HTMLInputElement || e.target instanceof HTMLTextAreaElement) { return; @@ -614,6 +706,7 @@ export function VideoPlayer({ if (!videoRef.current) return; setNeedsUserInteraction(false); setIsPlaying(true); + userPausedRef.current = false; videoRef.current.play()?.catch((err: Error) => { console.error("Play error after user interaction:", err); setError(`${t("failedToPlay")}: ${err.message}`); diff --git a/web-ui/src/mpegts/config.ts b/web-ui/src/mpegts/config.ts index f5f72680..181e937a 100644 --- a/web-ui/src/mpegts/config.ts +++ b/web-ui/src/mpegts/config.ts @@ -29,8 +29,8 @@ export interface PlayerConfig { export const defaultConfig: PlayerConfig = { liveSync: true, - liveSyncMaxLatency: 1.2, - liveSyncTargetLatency: 0.8, + liveSyncMaxLatency: 3, + liveSyncTargetLatency: 1.5, liveSyncPlaybackRate: 1.2, fixAudioTimestampGap: true, diff --git a/web-ui/src/mpegts/core/media-info.ts b/web-ui/src/mpegts/core/media-info.ts deleted file mode 100644 index d1068c98..00000000 --- a/web-ui/src/mpegts/core/media-info.ts +++ /dev/null @@ -1,82 +0,0 @@ -class MediaInfo { - mimeType: string | null; - duration: number | null; - - hasAudio: boolean | null; - hasVideo: boolean | null; - audioCodec: string | null; - videoCodec: string | null; - audioDataRate: number | null; - videoDataRate: number | null; - - audioSampleRate: number | null; - audioChannelCount: number | null; - - width: number | null; - height: number | null; - fps: number | null; - profile: string | null; - level: string | null; - refFrames: number | null; - chromaFormat: string | null; - sarNum: number | null; - sarDen: number | null; - - segments: MediaInfo[] | null; - segmentCount: number | null; - - constructor() { - this.mimeType = null; - this.duration = null; - - this.hasAudio = null; - this.hasVideo = null; - this.audioCodec = null; - this.videoCodec = null; - this.audioDataRate = null; - this.videoDataRate = null; - - this.audioSampleRate = null; - this.audioChannelCount = null; - - this.width = null; - this.height = null; - this.fps = null; - this.profile = null; - this.level = null; - this.refFrames = null; - this.chromaFormat = null; - this.sarNum = null; - this.sarDen = null; - - this.segments = null; - this.segmentCount = null; - } - - isComplete(): boolean { - const audioInfoComplete = - this.hasAudio === false || - (this.hasAudio === true && - this.audioCodec != null && - this.audioSampleRate != null && - this.audioChannelCount != null); - - const videoInfoComplete = - this.hasVideo === false || - (this.hasVideo === true && - this.videoCodec != null && - this.width != null && - this.height != null && - this.fps != null && - this.profile != null && - this.level != null && - this.refFrames != null && - this.chromaFormat != null && - this.sarNum != null && - this.sarDen != null); - - return this.mimeType != null && audioInfoComplete && videoInfoComplete; - } -} - -export default MediaInfo; diff --git a/web-ui/src/mpegts/demux/base-demuxer.ts b/web-ui/src/mpegts/demux/base-demuxer.ts deleted file mode 100644 index 76cd6c8e..00000000 --- a/web-ui/src/mpegts/demux/base-demuxer.ts +++ /dev/null @@ -1,51 +0,0 @@ -import type MediaInfo from "../core/media-info"; -import type { KLVData } from "./klv"; -import type { PESPrivateData, PESPrivateDataDescriptor } from "./pes-private-data"; -import type { PGSData } from "./pgs-data"; -import type { SCTE35Data } from "./scte35"; -import type { SMPTE2038Data } from "./smpte2038"; - -type OnErrorCallback = (type: string, info: string) => void; -type OnMediaInfoCallback = (mediaInfo: MediaInfo) => void; -type OnTrackMetadataCallback = (type: string, metadata: unknown) => void; -type OnDataAvailableCallback = (audioTrack: unknown, videoTrack: unknown) => void; -type OnTimedID3MetadataCallback = (timed_id3_data: PESPrivateData) => void; -type onPGSSubitleDataCallback = (pgs_data: PGSData) => void; -type OnSynchronousKLVMetadataCallback = (synchronous_klv_data: KLVData) => void; -type OnAsynchronousKLVMetadataCallback = (asynchronous_klv_data: PESPrivateData) => void; -type OnSMPTE2038MetadataCallback = (smpte2038_data: SMPTE2038Data) => void; -type OnSCTE35MetadataCallback = (scte35_data: SCTE35Data) => void; -type OnPESPrivateDataCallback = (private_data: PESPrivateData) => void; -type OnPESPrivateDataDescriptorCallback = (private_data_descriptor: PESPrivateDataDescriptor) => void; - -export default abstract class BaseDemuxer { - public onError: OnErrorCallback | null = null; - public onMediaInfo: OnMediaInfoCallback | null = null; - public onTrackMetadata: OnTrackMetadataCallback | null = null; - public onDataAvailable: OnDataAvailableCallback | null = null; - public onTimedID3Metadata: OnTimedID3MetadataCallback | null = null; - public onPGSSubtitleData: onPGSSubitleDataCallback | null = null; - public onSynchronousKLVMetadata: OnSynchronousKLVMetadataCallback | null = null; - public onAsynchronousKLVMetadata: OnAsynchronousKLVMetadataCallback | null = null; - public onSMPTE2038Metadata: OnSMPTE2038MetadataCallback | null = null; - public onSCTE35Metadata: OnSCTE35MetadataCallback | null = null; - public onPESPrivateData: OnPESPrivateDataCallback | null = null; - public onPESPrivateDataDescriptor: OnPESPrivateDataDescriptorCallback | null = null; - - public destroy(): void { - this.onError = null; - this.onMediaInfo = null; - this.onTrackMetadata = null; - this.onDataAvailable = null; - this.onTimedID3Metadata = null; - this.onPGSSubtitleData = null; - this.onSynchronousKLVMetadata = null; - this.onAsynchronousKLVMetadata = null; - this.onSMPTE2038Metadata = null; - this.onSCTE35Metadata = null; - this.onPESPrivateData = null; - this.onPESPrivateDataDescriptor = null; - } - - abstract parseChunks(chunk: ArrayBuffer, byteStart: number): number; -} diff --git a/web-ui/src/mpegts/demux/klv.ts b/web-ui/src/mpegts/demux/klv.ts deleted file mode 100644 index 8b941b2f..00000000 --- a/web-ui/src/mpegts/demux/klv.ts +++ /dev/null @@ -1,40 +0,0 @@ -export class KLVData { - pid!: number; - stream_id!: number; - pts?: number; - dts?: number; - access_units!: AccessUnit[]; - data!: Uint8Array; - len!: number; -} - -type AccessUnit = { - service_id: number; - sequence_number: number; - flags: number; - data: Uint8Array; -}; - -export const klv_parse = (data: Uint8Array) => { - const result: AccessUnit[] = []; - - let offset = 0; - while (offset + 5 < data.byteLength) { - const service_id = data[offset + 0]; - const sequence_number = data[offset + 1]; - const flags = data[offset + 2]; - const au_size = (data[offset + 3] << 8) | (data[offset + 4] << 0); - const au_data = data.slice(offset + 5, offset + 5 + au_size); - - result.push({ - service_id, - sequence_number, - flags, - data: au_data, - }); - - offset += 5 + au_size; - } - - return result; -}; diff --git a/web-ui/src/mpegts/demux/pat-pmt-pes.ts b/web-ui/src/mpegts/demux/pat-pmt-pes.ts index 67254ff8..e54d5809 100644 --- a/web-ui/src/mpegts/demux/pat-pmt-pes.ts +++ b/web-ui/src/mpegts/demux/pat-pmt-pes.ts @@ -17,9 +17,6 @@ export enum StreamType { kLOASAAC = 0x11, kAC3 = 0x81, kEAC3 = 0x87, - kMetadata = 0x15, - kSCTE35 = 0x86, - kPGS = 0x90, kH264 = 0x1b, kH265 = 0x24, } @@ -38,55 +35,20 @@ export class PMT { common_pids: { h264: number | undefined; h265: number | undefined; - av1: number | undefined; adts_aac: number | undefined; loas_aac: number | undefined; - opus: number | undefined; ac3: number | undefined; eac3: number | undefined; mp3: number | undefined; } = { h264: undefined, h265: undefined, - av1: undefined, adts_aac: undefined, loas_aac: undefined, - opus: undefined, ac3: undefined, eac3: undefined, mp3: undefined, }; - - pes_private_data_pids: { - [pid: number]: boolean; - } = {}; - - timed_id3_pids: { - [pid: number]: boolean; - } = {}; - - pgs_pids: { - [pid: number]: boolean; - } = {}; - pgs_langs: { - [pid: number]: string; - } = {}; - - synchronous_klv_pids: { - [pid: number]: boolean; - } = {}; - - asynchronous_klv_pids: { - [pid: number]: boolean; - } = {}; - - scte_35_pids: { - [pid: number]: boolean; - } = {}; - - smpte2038_pids: { - [oid: number]: boolean; - } = {}; } export interface ProgramToPMTMap { diff --git a/web-ui/src/mpegts/demux/pes-private-data.ts b/web-ui/src/mpegts/demux/pes-private-data.ts deleted file mode 100644 index 2c0ead39..00000000 --- a/web-ui/src/mpegts/demux/pes-private-data.ts +++ /dev/null @@ -1,16 +0,0 @@ -// ISO/IEC 13818-1 PES packets containing private data (stream_type=0x06) -export class PESPrivateData { - pid!: number; - stream_id!: number; - pts?: number; - dts?: number; - nearest_pts?: number; - data!: Uint8Array; - len!: number; -} - -export class PESPrivateDataDescriptor { - pid!: number; - stream_type!: number; - descriptor!: Uint8Array; -} diff --git a/web-ui/src/mpegts/demux/pgs-data.ts b/web-ui/src/mpegts/demux/pgs-data.ts deleted file mode 100644 index b72f683c..00000000 --- a/web-ui/src/mpegts/demux/pgs-data.ts +++ /dev/null @@ -1,10 +0,0 @@ -// ISO/IEC 13818-1 PES packets containing private data (stream_type=0x06) -export class PGSData { - pid!: number; - stream_id!: number; - pts?: number; - dts?: number; - lang!: string; - data!: Uint8Array; - len!: number; -} diff --git a/web-ui/src/mpegts/demux/scte35.ts b/web-ui/src/mpegts/demux/scte35.ts deleted file mode 100644 index d8164759..00000000 --- a/web-ui/src/mpegts/demux/scte35.ts +++ /dev/null @@ -1,772 +0,0 @@ -import ExpGolomb from "./exp-golomb"; - -export type SCTE35Data = - | { - splice_command_type: SCTE35CommandType.kSpliceInsert; - pts?: number; - nearest_pts?: number; - auto_return?: boolean; - duraiton?: number; - detail: SCTE35Detail; - data: Uint8Array; - } - | { - splice_command_type: SCTE35CommandType.kTimeSignal; - pts?: number; - nearest_pts?: number; - detail: SCTE35Detail; - data: Uint8Array; - } - | { - splice_command_type: - | SCTE35CommandType.kSpliceNull - | SCTE35CommandType.kBandwidthReservation - | SCTE35CommandType.kSpliceSchedule - | SCTE35CommandType.kPrivateCommand; - pts: undefined; - nearest_pts?: number; - detail: SCTE35Detail; - data: Uint8Array; - }; - -type SCTE35Detail = - | { - table_id: number; - section_syntax_indicator: boolean; - private_indicator: boolean; - section_length: number; - protocol_version: number; - encrypted_packet: boolean; - encryption_algorithm: number; - pts_adjustment: number; - cw_index: number; - tier: number; - splice_command_length: number; - splice_command_type: SCTE35CommandType.kSpliceNull; - splice_command: SpliceNull; - descriptor_loop_length: number; - splice_descriptors: SpliceDescriptor[]; - E_CRC32?: number; - CRC32: number; - } - | { - table_id: number; - section_syntax_indicator: boolean; - private_indicator: boolean; - section_length: number; - protocol_version: number; - encrypted_packet: boolean; - encryption_algorithm: number; - pts_adjustment: number; - cw_index: number; - tier: number; - splice_command_length: number; - splice_command_type: SCTE35CommandType.kSpliceSchedule; - splice_command: SpliceSchedule; - descriptor_loop_length: number; - splice_descriptors: SpliceDescriptor[]; - E_CRC32?: number; - CRC32: number; - } - | { - table_id: number; - section_syntax_indicator: boolean; - private_indicator: boolean; - section_length: number; - protocol_version: number; - encrypted_packet: boolean; - encryption_algorithm: number; - pts_adjustment: number; - cw_index: number; - tier: number; - splice_command_length: number; - splice_command_type: SCTE35CommandType.kSpliceInsert; - splice_command: SpliceInsert; - descriptor_loop_length: number; - splice_descriptors: SpliceDescriptor[]; - E_CRC32?: number; - CRC32: number; - } - | { - table_id: number; - section_syntax_indicator: boolean; - private_indicator: boolean; - section_length: number; - protocol_version: number; - encrypted_packet: boolean; - encryption_algorithm: number; - pts_adjustment: number; - cw_index: number; - tier: number; - splice_command_length: number; - splice_command_type: SCTE35CommandType.kTimeSignal; - splice_command: TimeSignal; - descriptor_loop_length: number; - splice_descriptors: SpliceDescriptor[]; - E_CRC32?: number; - CRC32: number; - } - | { - table_id: number; - section_syntax_indicator: boolean; - private_indicator: boolean; - section_length: number; - protocol_version: number; - encrypted_packet: boolean; - encryption_algorithm: number; - pts_adjustment: number; - cw_index: number; - tier: number; - splice_command_length: number; - splice_command_type: SCTE35CommandType.kBandwidthReservation; - splice_command: BandwidthReservation; - descriptor_loop_length: number; - splice_descriptors: SpliceDescriptor[]; - E_CRC32?: number; - CRC32: number; - } - | { - table_id: number; - section_syntax_indicator: boolean; - private_indicator: boolean; - section_length: number; - protocol_version: number; - encrypted_packet: boolean; - encryption_algorithm: number; - pts_adjustment: number; - cw_index: number; - tier: number; - splice_command_length: number; - splice_command_type: SCTE35CommandType.kPrivateCommand; - splice_command: PrivateCommand; - descriptor_loop_length: number; - splice_descriptors: SpliceDescriptor[]; - E_CRC32?: number; - CRC32: number; - }; - -export enum SCTE35CommandType { - kSpliceNull = 0x0, - kSpliceSchedule = 0x4, - kSpliceInsert = 0x5, - kTimeSignal = 0x6, - kBandwidthReservation = 0x07, - kPrivateCommand = 0xff, -} - -type SpliceTime = { - time_specified_flag: boolean; - pts_time?: number; -}; - -const parseSpliceTime = (reader: ExpGolomb): SpliceTime => { - const time_specified_flag = reader.readBool(); - - if (!time_specified_flag) { - reader.readBits(7); - return { time_specified_flag }; - } else { - reader.readBits(6); - const pts_time = reader.readBits(31) * 4 + reader.readBits(2); - return { - time_specified_flag, - pts_time, - }; - } -}; - -type BreakDuration = { - auto_return: boolean; - duration: number; -}; -const parseBreakDuration = (reader: ExpGolomb): BreakDuration => { - const auto_return = reader.readBool(); - reader.readBits(6); - const duration = reader.readBits(31) * 4 + reader.readBits(2); - return { - auto_return, - duration, - }; -}; - -type SpliceInsertComponent = { - component_tag: number; - splice_time?: SpliceTime; -}; -const parseSpliceInsertComponent = (splice_immediate_flag: boolean, reader: ExpGolomb): SpliceInsertComponent => { - const component_tag = reader.readBits(8); - if (splice_immediate_flag) { - return { component_tag }; - } - - const splice_time = parseSpliceTime(reader); - return { - component_tag, - splice_time, - }; -}; -type SpliceScheduleEventComponent = { - component_tag: number; - utc_splice_time: number; -}; -const parseSpliceScheduleEventComponent = (reader: ExpGolomb): SpliceScheduleEventComponent => { - const component_tag = reader.readBits(8); - const utc_splice_time = reader.readBits(32); - return { - component_tag, - utc_splice_time, - }; -}; - -type SpliceScheduleEvent = { - splice_event_id: number; - splice_event_cancel_indicator: boolean; - out_of_network_indicator?: boolean; - program_splice_flag?: boolean; - duration_flag?: boolean; - utc_splice_time?: number; - component_count?: number; - components?: SpliceScheduleEventComponent[]; - break_duration?: BreakDuration; - unique_program_id?: number; - avail_num?: number; - avails_expected?: number; -}; -const parseSpliceScheduleEvent = (reader: ExpGolomb): SpliceScheduleEvent => { - const splice_event_id = reader.readBits(32); - const splice_event_cancel_indicator = reader.readBool(); - reader.readBits(7); - - const spliceScheduleEvent: SpliceScheduleEvent = { - splice_event_id, - splice_event_cancel_indicator, - }; - - if (splice_event_cancel_indicator) { - return spliceScheduleEvent; - } - - spliceScheduleEvent.out_of_network_indicator = reader.readBool(); - spliceScheduleEvent.program_splice_flag = reader.readBool(); - spliceScheduleEvent.duration_flag = reader.readBool(); - reader.readBits(5); - - if (spliceScheduleEvent.program_splice_flag) { - spliceScheduleEvent.utc_splice_time = reader.readBits(32); - } else { - spliceScheduleEvent.component_count = reader.readBits(8); - spliceScheduleEvent.components = []; - for (let i = 0; i < spliceScheduleEvent.component_count; i++) { - spliceScheduleEvent.components.push(parseSpliceScheduleEventComponent(reader)); - } - } - - if (spliceScheduleEvent.duration_flag) { - spliceScheduleEvent.break_duration = parseBreakDuration(reader); - } - - spliceScheduleEvent.unique_program_id = reader.readBits(16); - spliceScheduleEvent.avail_num = reader.readBits(8); - spliceScheduleEvent.avails_expected = reader.readBits(8); - - return spliceScheduleEvent; -}; - -type SpliceNull = Record; -type SpliceSchedule = { - splice_count: number; - events: SpliceScheduleEvent[]; -}; -type SpliceInsert = { - splice_event_id: number; - splice_event_cancel_indicator: boolean; - out_of_network_indicator?: boolean; - program_splice_flag?: boolean; - duration_flag?: boolean; - splice_immediate_flag?: boolean; - splice_time?: SpliceTime; - component_count?: number; - components?: SpliceInsertComponent[]; - break_duration?: BreakDuration; - unique_program_id?: number; - avail_num?: number; - avails_expected?: number; -}; -type TimeSignal = { - splice_time: SpliceTime; -}; -type BandwidthReservation = Record; -type PrivateCommand = { - identifier: string; - private_data: ArrayBuffer; -}; - -type SpliceCommand = SpliceNull | SpliceSchedule | SpliceInsert | TimeSignal | BandwidthReservation | PrivateCommand; - -const parseSpliceNull = (): SpliceNull => { - return {}; -}; -const parseSpliceSchedule = (reader: ExpGolomb): SpliceSchedule => { - const splice_count = reader.readBits(8); - const events: SpliceScheduleEvent[] = []; - for (let i = 0; i < splice_count; i++) { - events.push(parseSpliceScheduleEvent(reader)); - } - return { - splice_count, - events, - }; -}; -const parseSpliceInsert = (reader: ExpGolomb): SpliceInsert => { - const splice_event_id = reader.readBits(32); - const splice_event_cancel_indicator = reader.readBool(); - reader.readBits(7); - - const spliceInsert: SpliceInsert = { - splice_event_id, - splice_event_cancel_indicator, - }; - - if (splice_event_cancel_indicator) { - return spliceInsert; - } - - spliceInsert.out_of_network_indicator = reader.readBool(); - spliceInsert.program_splice_flag = reader.readBool(); - spliceInsert.duration_flag = reader.readBool(); - spliceInsert.splice_immediate_flag = reader.readBool(); - reader.readBits(4); - - if (spliceInsert.program_splice_flag && !spliceInsert.splice_immediate_flag) { - spliceInsert.splice_time = parseSpliceTime(reader); - } - if (!spliceInsert.program_splice_flag) { - spliceInsert.component_count = reader.readBits(8); - spliceInsert.components = []; - for (let i = 0; i < spliceInsert.component_count; i++) { - spliceInsert.components.push(parseSpliceInsertComponent(spliceInsert.splice_immediate_flag, reader)); - } - } - - if (spliceInsert.duration_flag) { - spliceInsert.break_duration = parseBreakDuration(reader); - } - - spliceInsert.unique_program_id = reader.readBits(16); - spliceInsert.avail_num = reader.readBits(8); - spliceInsert.avails_expected = reader.readBits(8); - - return spliceInsert; -}; -const parseTimeSignal = (reader: ExpGolomb): TimeSignal => { - return { - splice_time: parseSpliceTime(reader), - }; -}; -const parseBandwidthReservation = (): BandwidthReservation => { - return {}; -}; -const parsePrivateCommand = (splice_command_length: number, reader: ExpGolomb): PrivateCommand => { - const identifier = String.fromCharCode( - reader.readBits(8), - reader.readBits(8), - reader.readBits(8), - reader.readBits(8), - ); - const data = new Uint8Array(splice_command_length - 4); - for (let i = 0; i < splice_command_length - 4; i++) { - data[i] = reader.readBits(8); - } - - return { - identifier, - private_data: data.buffer, - }; -}; - -type Descriptor = { - descriptor_tag: number; - descriptor_length: number; - identifier: string; -}; -type AvailDescriptor = Descriptor & { - provider_avail_id: number; -}; -const parseAvailDescriptor = ( - descriptor_tag: number, - descriptor_length: number, - identifier: string, - reader: ExpGolomb, -): AvailDescriptor => { - const provider_avail_id = reader.readBits(32); - - return { - descriptor_tag, - descriptor_length, - identifier, - provider_avail_id, - }; -}; -type DTMFDescriptor = Descriptor & { - preroll: number; - dtmf_count: number; - DTMF_char: string; -}; -const parseDTMFDescriptor = ( - descriptor_tag: number, - descriptor_length: number, - identifier: string, - reader: ExpGolomb, -): DTMFDescriptor => { - const preroll = reader.readBits(8); - const dtmf_count = reader.readBits(3); - reader.readBits(5); - let DTMF_char = ""; - for (let i = 0; i < dtmf_count; i++) { - DTMF_char += String.fromCharCode(reader.readBits(8)); - } - - return { - descriptor_tag, - descriptor_length, - identifier, - preroll, - dtmf_count, - DTMF_char, - }; -}; -type SegmentationDescriptorComponent = { - component_tag: number; - pts_offset: number; -}; -const parseSegmentationDescriptorComponent = (reader: ExpGolomb): SegmentationDescriptorComponent => { - const component_tag = reader.readBits(8); - reader.readBits(7); - const pts_offset = reader.readBits(31) * 4 + reader.readBits(2); - return { - component_tag, - pts_offset, - }; -}; -type SegmentationDescriptor = Descriptor & { - segmentation_event_id: number; - segmentation_event_cancel_indicator: boolean; - program_segmentation_flag?: boolean; - segmentation_duration_flag?: boolean; - delivery_not_restricted_flag?: boolean; - web_delivery_allowed_flag?: boolean; - no_regional_blackout_flag?: boolean; - archive_allowed_flag?: boolean; - device_restrictions?: number; - component_count?: number; - components?: SegmentationDescriptorComponent[]; - segmentation_duration?: number; - segmentation_upid_type?: number; - segmentation_upid_length?: number; - segmentation_upid?: ArrayBuffer; - segmentation_type_id?: number; - segment_num?: number; - segments_expected?: number; - sub_segment_num?: number; - sub_segments_expected?: number; -}; -const parseSegmentationDescriptor = ( - descriptor_tag: number, - descriptor_length: number, - identifier: string, - reader: ExpGolomb, -): SegmentationDescriptor => { - const segmentation_event_id = reader.readBits(32); - const segmentation_event_cancel_indicator = reader.readBool(); - reader.readBits(7); - - const segmentationDescriptor: SegmentationDescriptor = { - descriptor_tag, - descriptor_length, - identifier, - segmentation_event_id, - segmentation_event_cancel_indicator, - }; - - if (segmentation_event_cancel_indicator) { - return segmentationDescriptor; - } - - segmentationDescriptor.program_segmentation_flag = reader.readBool(); - segmentationDescriptor.segmentation_duration_flag = reader.readBool(); - segmentationDescriptor.delivery_not_restricted_flag = reader.readBool(); - - if (!segmentationDescriptor.delivery_not_restricted_flag) { - segmentationDescriptor.web_delivery_allowed_flag = reader.readBool(); - segmentationDescriptor.no_regional_blackout_flag = reader.readBool(); - segmentationDescriptor.archive_allowed_flag = reader.readBool(); - segmentationDescriptor.device_restrictions = reader.readBits(2); - } else { - reader.readBits(5); - } - - if (!segmentationDescriptor.program_segmentation_flag) { - segmentationDescriptor.component_count = reader.readBits(8); - segmentationDescriptor.components = []; - for (let i = 0; i < segmentationDescriptor.component_count; i++) { - segmentationDescriptor.components.push(parseSegmentationDescriptorComponent(reader)); - } - } - - if (segmentationDescriptor.segmentation_duration_flag) { - segmentationDescriptor.segmentation_duration = reader.readBits(40); - } - - segmentationDescriptor.segmentation_upid_type = reader.readBits(8); - segmentationDescriptor.segmentation_upid_length = reader.readBits(8); - { - const upid = new Uint8Array(segmentationDescriptor.segmentation_upid_length); - for (let i = 0; i < segmentationDescriptor.segmentation_upid_length; i++) { - upid[i] = reader.readBits(8); - } - segmentationDescriptor.segmentation_upid = upid.buffer; - } - segmentationDescriptor.segmentation_type_id = reader.readBits(8); - segmentationDescriptor.segment_num = reader.readBits(8); - segmentationDescriptor.segments_expected = reader.readBits(8); - if ( - segmentationDescriptor.segmentation_type_id === 0x34 || - segmentationDescriptor.segmentation_type_id === 0x36 || - segmentationDescriptor.segmentation_type_id === 0x38 || - segmentationDescriptor.segmentation_type_id === 0x3a - ) { - segmentationDescriptor.sub_segment_num = reader.readBits(8); - segmentationDescriptor.sub_segments_expected = reader.readBits(8); - } - - return segmentationDescriptor; -}; -type TimeDescriptor = Descriptor & { - TAI_seconds: number; - TAI_ns: number; - UTC_offset: number; -}; -const parseTimeDescriptor = ( - descriptor_tag: number, - descriptor_length: number, - identifier: string, - reader: ExpGolomb, -): TimeDescriptor => { - const TAI_seconds = reader.readBits(48); - const TAI_ns = reader.readBits(32); - const UTC_offset = reader.readBits(16); - - return { - descriptor_tag, - descriptor_length, - identifier, - TAI_seconds, - TAI_ns, - UTC_offset, - }; -}; -type AudioDescriptorComponent = { - component_tag: number; - ISO_code: string; - Bit_Stream_Mode: number; - Num_Channels: number; - Full_Srvc_Audio: boolean; -}; -const parseAudioDescriptorComponent = (reader: ExpGolomb): AudioDescriptorComponent => { - const component_tag = reader.readBits(8); - const ISO_code = String.fromCharCode(reader.readBits(8), reader.readBits(8), reader.readBits(8)); - const Bit_Stream_Mode = reader.readBits(3); - const Num_Channels = reader.readBits(4); - const Full_Srvc_Audio = reader.readBool(); - - return { - component_tag, - ISO_code, - Bit_Stream_Mode, - Num_Channels, - Full_Srvc_Audio, - }; -}; -type AudioDescriptor = Descriptor & { - audio_count: number; - components: AudioDescriptorComponent[]; -}; -const parseAudioDescriptor = ( - descriptor_tag: number, - descriptor_length: number, - identifier: string, - reader: ExpGolomb, -): AudioDescriptor => { - const audio_count = reader.readBits(4); - const components: AudioDescriptorComponent[] = []; - for (let i = 0; i < audio_count; i++) { - components.push(parseAudioDescriptorComponent(reader)); - } - - return { - descriptor_tag, - descriptor_length, - identifier, - audio_count, - components, - }; -}; - -type SpliceDescriptor = AvailDescriptor | DTMFDescriptor | SegmentationDescriptor | TimeDescriptor | AudioDescriptor; - -export const readSCTE35 = (data: Uint8Array): SCTE35Data => { - const reader = new ExpGolomb(data); - - const table_id = reader.readBits(8); - const section_syntax_indicator = reader.readBool(); - const private_indicator = reader.readBool(); - reader.readBits(2); - const section_length = reader.readBits(12); - const protocol_version = reader.readBits(8); - const encrypted_packet = reader.readBool(); - const encryption_algorithm = reader.readBits(6); - const pts_adjustment = reader.readBits(31) * 4 + reader.readBits(2); - const cw_index = reader.readBits(8); - const tier = reader.readBits(12); - const splice_command_length = reader.readBits(12); - const splice_command_type = reader.readBits(8) as SCTE35CommandType; - - let splice_command: SpliceCommand | null = null; - if (splice_command_type === SCTE35CommandType.kSpliceNull) { - splice_command = parseSpliceNull(); - } else if (splice_command_type === SCTE35CommandType.kSpliceSchedule) { - splice_command = parseSpliceSchedule(reader); - } else if (splice_command_type === SCTE35CommandType.kSpliceInsert) { - splice_command = parseSpliceInsert(reader); - } else if (splice_command_type === SCTE35CommandType.kTimeSignal) { - splice_command = parseTimeSignal(reader); - } else if (splice_command_type === SCTE35CommandType.kBandwidthReservation) { - splice_command = parseBandwidthReservation(); - } else if (splice_command_type === SCTE35CommandType.kPrivateCommand) { - splice_command = parsePrivateCommand(splice_command_length, reader); - } else { - reader.readBits(splice_command_length * 8); - } - - const splice_descriptors: SpliceDescriptor[] = []; - - const descriptor_loop_length = reader.readBits(16); - for (let length = 0; length < descriptor_loop_length; ) { - const descriptor_tag = reader.readBits(8); - const descriptor_length = reader.readBits(8); - const identifier = String.fromCharCode( - reader.readBits(8), - reader.readBits(8), - reader.readBits(8), - reader.readBits(8), - ); - - if (descriptor_tag === 0x00) { - splice_descriptors.push(parseAvailDescriptor(descriptor_tag, descriptor_length, identifier, reader)); - } else if (descriptor_tag === 0x01) { - splice_descriptors.push(parseDTMFDescriptor(descriptor_tag, descriptor_length, identifier, reader)); - } else if (descriptor_tag === 0x02) { - splice_descriptors.push(parseSegmentationDescriptor(descriptor_tag, descriptor_length, identifier, reader)); - } else if (descriptor_tag === 0x03) { - splice_descriptors.push(parseTimeDescriptor(descriptor_tag, descriptor_length, identifier, reader)); - } else if (descriptor_tag === 0x04) { - splice_descriptors.push(parseAudioDescriptor(descriptor_tag, descriptor_length, identifier, reader)); - } else { - reader.readBits((descriptor_length - 4) * 8); - } - - length += 2 + descriptor_length; - } - - const E_CRC32 = encrypted_packet ? reader.readBits(32) : undefined; - const CRC32 = reader.readBits(32); - - const detail = { - table_id, - section_syntax_indicator, - private_indicator, - section_length, - protocol_version, - encrypted_packet, - encryption_algorithm, - pts_adjustment, - cw_index, - tier, - splice_command_length, - splice_command_type, - splice_command, - descriptor_loop_length, - splice_descriptors, - E_CRC32, - CRC32, - } as SCTE35Detail; - - if (splice_command_type === SCTE35CommandType.kSpliceInsert) { - const spliceInsert = splice_command as SpliceInsert; - - if (spliceInsert.splice_event_cancel_indicator) { - return { - splice_command_type, - detail, - data, - }; - } else if (spliceInsert.program_splice_flag && !spliceInsert.splice_immediate_flag) { - const auto_return = spliceInsert.duration_flag ? spliceInsert.break_duration?.auto_return : undefined; - const duraiton = spliceInsert.duration_flag ? (spliceInsert.break_duration?.duration ?? 0) / 90 : undefined; - - if (spliceInsert.splice_time?.time_specified_flag) { - return { - splice_command_type, - pts: (pts_adjustment + (spliceInsert.splice_time?.pts_time ?? 0)) % 2 ** 33, - auto_return, - duraiton, - detail, - data, - }; - } else { - return { - splice_command_type, - auto_return, - duraiton, - detail, - data, - }; - } - } else { - const auto_return = spliceInsert.duration_flag ? spliceInsert.break_duration?.auto_return : undefined; - const duraiton = spliceInsert.duration_flag ? (spliceInsert.break_duration?.duration ?? 0) / 90 : undefined; - - return { - splice_command_type, - auto_return, - duraiton, - detail, - data, - }; - } - } else if (splice_command_type === SCTE35CommandType.kTimeSignal) { - const timeSignal = splice_command as TimeSignal; - - if (timeSignal.splice_time.time_specified_flag) { - return { - splice_command_type, - pts: (pts_adjustment + (timeSignal.splice_time.pts_time ?? 0)) % 2 ** 33, - detail, - data, - }; - } else { - return { - splice_command_type, - detail, - data, - }; - } - } else { - return { - splice_command_type, - pts: undefined, - detail, - data, - } as SCTE35Data; - } -}; diff --git a/web-ui/src/mpegts/demux/smpte2038.ts b/web-ui/src/mpegts/demux/smpte2038.ts deleted file mode 100644 index 42817d0c..00000000 --- a/web-ui/src/mpegts/demux/smpte2038.ts +++ /dev/null @@ -1,98 +0,0 @@ -import ExpGolomb from "./exp-golomb"; - -export class SMPTE2038Data { - pid!: number; - stream_id!: number; - pts?: number; - dts?: number; - nearest_pts?: number; - ancillaries!: AncillaryData[]; - data!: Uint8Array; - len!: number; -} - -type AncillaryData = { - yc_indicator: boolean; - line_number: number; - horizontal_offset: number; - did: number; - sdid: number; - user_data: Uint8Array; - description: string; - information: Record; -}; - -export const smpte2038parse = (data: Uint8Array) => { - const gb = new ExpGolomb(data); - let readBits = 0; - - const ancillaries: AncillaryData[] = []; - while (true) { - const zero = gb.readBits(6); - readBits += 6; - if (zero !== 0) { - break; - } - const YC_indicator = gb.readBool(); - readBits += 1; - const line_number = gb.readBits(11); - readBits += 11; - const horizontal_offset = gb.readBits(12); - readBits += 12; - const data_ID = gb.readBits(10) & 0xff; - readBits += 10; - const data_SDID = gb.readBits(10) & 0xff; - readBits += 10; - const data_count = gb.readBits(10) & 0xff; - readBits += 10; - const user_data = new Uint8Array(data_count); - for (let i = 0; i < data_count; i++) { - const user_data_word = gb.readBits(10) & 0xff; - readBits += 10; - user_data[i] = user_data_word; - } - const _checksum_word = gb.readBits(10); - readBits += 10; - - let description = "User Defined"; - const information: Record = {}; - if (data_ID === 0x41) { - if (data_SDID === 0x07) { - description = "SCTE-104"; - } - } else if (data_ID === 0x5f) { - if (data_SDID === 0xdc) { - description = "ARIB STD-B37 (1SEG)"; - } else if (data_SDID === 0xdd) { - description = "ARIB STD-B37 (ANALOG)"; - } else if (data_SDID === 0xde) { - description = "ARIB STD-B37 (SD)"; - } else if (data_SDID === 0xdf) { - description = "ARIB STD-B37 (HD)"; - } - } else if (data_ID === 0x61) { - if (data_SDID === 0x01) { - description = "EIA-708"; - } else if (data_SDID === 0x02) { - description = "EIA-608"; - } - } - - ancillaries.push({ - yc_indicator: YC_indicator, - line_number, - horizontal_offset, - did: data_ID, - sdid: data_SDID, - user_data, - description, - information, - }); - gb.readBits(8 - ((readBits - Math.floor(readBits / 8)) % 8)); - readBits += (8 - (readBits - Math.floor(readBits / 8))) % 8; - } - - gb.destroy(); - - return ancillaries; -}; diff --git a/web-ui/src/mpegts/demux/ts-demuxer.ts b/web-ui/src/mpegts/demux/ts-demuxer.ts index b6d81736..eb86d3f2 100644 --- a/web-ui/src/mpegts/demux/ts-demuxer.ts +++ b/web-ui/src/mpegts/demux/ts-demuxer.ts @@ -1,9 +1,7 @@ -import MediaInfo from "../core/media-info"; import { IllegalStateException } from "../utils/exception"; import Log from "../utils/logger"; import { AACADTSParser, type AACFrame, AACLOASParser, AudioSpecificConfig, type LOASAACFrame } from "./aac"; import { AC3Config, type AC3Frame, AC3Parser, EAC3Config, type EAC3Frame, EAC3Parser } from "./ac3"; -import BaseDemuxer from "./base-demuxer"; import { AVCDecoderConfigurationRecord, H264AnnexBParser, @@ -19,7 +17,6 @@ import { HEVCDecoderConfigurationRecord, } from "./h265"; import H265Parser from "./h265-parser"; -import { KLVData, klv_parse } from "./klv"; import { MP3Data } from "./mp3"; import { type MPEG4AudioObjectTypes, MPEG4SamplingFrequencies, type MPEG4SamplingFrequencyIndex } from "./mpeg4-audio"; import { @@ -32,12 +29,16 @@ import { SliceQueue, StreamType, } from "./pat-pmt-pes"; -import { PESPrivateData, PESPrivateDataDescriptor } from "./pes-private-data"; -import { PGSData } from "./pgs-data"; -import { readSCTE35 } from "./scte35"; -import { SMPTE2038Data, smpte2038parse } from "./smpte2038"; import SPSParser from "./sps-parser"; +export interface TSProbeResult { + match: boolean; + needMoreData?: boolean; + consumed?: number; + ts_packet_size?: number; + sync_offset?: number; +} + interface TSSliceMisc { pid: number; file_position: number; @@ -75,12 +76,6 @@ type EAC3AudioMetadata = { channel_mode: number; num_blks: number; }; -type OpusAudioMetadata = { - codec: "opus"; - channel_count: number; - channel_config_code: number; - sample_rate: number; -}; type MP3AudioMetadata = { codec: "mp3"; object_type: number; @@ -100,25 +95,29 @@ type AudioData = codec: "ec-3"; data: EAC3Frame; } - | { - codec: "opus"; - meta: OpusAudioMetadata; - } | { codec: "mp3"; data: MP3Data; }; -class TSDemuxer extends BaseDemuxer { +export type OnErrorCallback = (type: string, info: string) => void; +export type OnTrackMetadataCallback = (type: string, metadata: unknown) => void; +export type OnDataAvailableCallback = (audioTrack: unknown, videoTrack: unknown) => void; + +class TSDemuxer { private readonly TAG: string = "TSDemuxer"; - private ts_packet_size_!: number; - private sync_offset_!: number; - private first_parse_: boolean = true; - private media_info_ = new MediaInfo(); + public onError: OnErrorCallback | null = null; + public onTrackMetadata: OnTrackMetadataCallback | null = null; + public onDataAvailable: OnDataAvailableCallback | null = null; + /** Software audio decode support (MP2) */ + public onRawAudioData: ((frame: { codec: "mp2"; data: Uint8Array; pts: number }) => void) | null = null; + + private ts_packet_size_: number; + private sync_offset_: number; + private first_parse_: boolean = true; - private timescale_ = 90; - private duration_ = 0; + private readonly timescale_ = 90; private pat_!: PAT; private current_program_!: number; @@ -133,22 +132,15 @@ class TSDemuxer extends BaseDemuxer { vps: H265NaluHVC1 | undefined; sps: H264NaluAVC1 | H265NaluHVC1 | undefined; pps: H264NaluAVC1 | H265NaluHVC1 | undefined; - av1c: Uint8Array | undefined; details: Record; } = { vps: undefined, sps: undefined, pps: undefined, - av1c: undefined, details: {} as Record, }; - private audio_metadata_: - | AACAudioMetadata - | AC3AudioMetadata - | EAC3AudioMetadata - | OpusAudioMetadata - | MP3AudioMetadata = { + private audio_metadata_: AACAudioMetadata | AC3AudioMetadata | EAC3AudioMetadata | MP3AudioMetadata = { codec: undefined as unknown as "aac", audio_object_type: undefined as unknown as MPEG4AudioObjectTypes, sampling_freq_index: undefined as unknown as MPEG4SamplingFrequencyIndex, @@ -156,7 +148,6 @@ class TSDemuxer extends BaseDemuxer { channel_config: undefined as unknown as number, }; - private last_pcr_: number | undefined; private last_pcr_base_: number = NaN; private timestamp_offset_: number = 0; @@ -170,8 +161,6 @@ class TSDemuxer extends BaseDemuxer { private video_metadata_changed_ = false; private loas_previous_frame: LOASAACFrame | null = null; - // Software audio decode support - public onRawAudioData: ((frame: { codec: "mp2"; data: Uint8Array; pts: number }) => void) | null = null; private soft_decode_audio_codec_: "mp2" | null = null; private video_track_ = { @@ -193,17 +182,14 @@ class TSDemuxer extends BaseDemuxer { this.timestamp_offset_ = value; } - public constructor(probe_data: Record, _config: unknown) { - super(); - + public constructor(probe_data: TSProbeResult) { this.ts_packet_size_ = probe_data.ts_packet_size as number; this.sync_offset_ = probe_data.sync_offset as number; } public destroy() { - this.media_info_ = null as unknown as MediaInfo; - this.pes_slice_queues_ = null as unknown as PIDToSliceQueues; - this.section_slice_queues_ = null as unknown as PIDToSliceQueues; + this.pes_slice_queues_ = {}; + this.section_slice_queues_ = {}; this.video_metadata_ = null as unknown as typeof this.video_metadata_; this.audio_metadata_ = null as unknown as typeof this.audio_metadata_; @@ -212,19 +198,20 @@ class TSDemuxer extends BaseDemuxer { this.video_track_ = null as unknown as typeof this.video_track_; this.audio_track_ = null as unknown as typeof this.audio_track_; + this.onError = null; + this.onTrackMetadata = null; + this.onDataAvailable = null; this.onRawAudioData = null; this.soft_decode_audio_codec_ = null; - - super.destroy(); } - public static probe(buffer: ArrayBuffer) { + public static probe(buffer: ArrayBuffer): TSProbeResult { const data = new Uint8Array(buffer); let sync_offset = -1; let ts_packet_size = 188; if (data.byteLength <= 3 * ts_packet_size) { - return { needMoreData: true }; + return { match: false, needMoreData: true }; } while (sync_offset === -1) { @@ -275,20 +262,9 @@ class TSDemuxer extends BaseDemuxer { }; } - public bindDataSource(loader: Record) { - loader.onDataArrival = this.parseChunks.bind(this); - return this; - } - - public resetMediaInfo() { - this.media_info_ = new MediaInfo(); - } - public parseChunks(chunk: ArrayBuffer, byte_start: number): number { - if (!this.onError || !this.onMediaInfo || !this.onTrackMetadata || !this.onDataAvailable) { - throw new IllegalStateException( - "onError & onMediaInfo & onTrackMetadata & onDataAvailable callback must be specified", - ); + if (!this.onError || !this.onTrackMetadata || !this.onDataAvailable) { + throw new IllegalStateException("onError & onTrackMetadata & onDataAvailable callback must be specified"); } let offset = 0; @@ -315,7 +291,6 @@ class TSDemuxer extends BaseDemuxer { } const payload_unit_start_indicator = (data[1] & 0x40) >>> 6; - const _transport_priority = (data[1] & 0x20) >>> 5; const pid = ((data[1] & 0x1f) << 8) | data[2]; const adaptation_field_control = (data[3] & 0x30) >>> 4; const continuity_conunter = data[3] & 0x0f; @@ -335,10 +310,8 @@ class TSDemuxer extends BaseDemuxer { const PCR_flag = (data[5] & 0x10) >>> 4; if (PCR_flag) { - const pcr_base = this.getPcrBase(data); - const pcr_extension = ((data[10] & 0x01) << 8) | data[11]; - const pcr = pcr_base * 300 + pcr_extension; - this.last_pcr_ = pcr; + // track PCR base for pts/dts wraparound detection + this.getPcrBase(data); } } if (adaptation_field_control === 0x02 || 5 + adaptation_field_length === 188) { @@ -358,10 +331,8 @@ class TSDemuxer extends BaseDemuxer { if (adaptation_field_control === 0x01 || adaptation_field_control === 0x03) { if ( pid === 0 || // PAT (pid === 0) - pid === this.current_pmt_pid_ || // PMT - (this.pmt_ !== undefined && this.pmt_.pid_stream_type[pid] === StreamType.kSCTE35) + pid === this.current_pmt_pid_ // PMT ) { - // SCTE35 const ts_payload_length = 188 - ts_payload_start_index; this.handleSectionSlice(chunk, offset + ts_payload_start_index, ts_payload_length, { @@ -380,18 +351,11 @@ class TSDemuxer extends BaseDemuxer { if ( pid === this.pmt_.common_pids.h264 || pid === this.pmt_.common_pids.h265 || - pid === this.pmt_.common_pids.av1 || pid === this.pmt_.common_pids.adts_aac || pid === this.pmt_.common_pids.loas_aac || pid === this.pmt_.common_pids.ac3 || pid === this.pmt_.common_pids.eac3 || - pid === this.pmt_.common_pids.opus || - pid === this.pmt_.common_pids.mp3 || - this.pmt_.pes_private_data_pids[pid] === true || - this.pmt_.timed_id3_pids[pid] === true || - this.pmt_.pgs_pids[pid] === true || - this.pmt_.synchronous_klv_pids[pid] === true || - this.pmt_.asynchronous_klv_pids[pid] === true + pid === this.pmt_.common_pids.mp3 ) { this.handlePESSlice(chunk, offset + ts_payload_start_index, ts_payload_length, { pid, @@ -434,7 +398,7 @@ class TSDemuxer extends BaseDemuxer { if (slice_queue.total_length === slice_queue.expected_length) { this.emitSectionSlices(slice_queue, misc); } else { - this.clearSlices(slice_queue, misc); + this.clearSlices(slice_queue); } } @@ -464,7 +428,7 @@ class TSDemuxer extends BaseDemuxer { if (slice_queue.total_length === slice_queue.expected_length) { this.emitSectionSlices(slice_queue, misc); } else if (slice_queue.total_length >= slice_queue.expected_length) { - this.clearSlices(slice_queue, misc); + this.clearSlices(slice_queue); } i += remain_section.byteLength; @@ -481,7 +445,7 @@ class TSDemuxer extends BaseDemuxer { if (slice_queue.total_length === slice_queue.expected_length) { this.emitSectionSlices(slice_queue, misc); } else if (slice_queue.total_length >= slice_queue.expected_length) { - this.clearSlices(slice_queue, misc); + this.clearSlices(slice_queue); } } } @@ -490,7 +454,6 @@ class TSDemuxer extends BaseDemuxer { const data = new Uint8Array(buffer, offset, length); const packet_start_code_prefix = (data[0] << 16) | (data[1] << 8) | data[2]; - const _stream_id = data[3]; const PES_packet_length = (data[4] << 8) | data[5]; if (misc.payload_unit_start_indicator) { @@ -509,7 +472,7 @@ class TSDemuxer extends BaseDemuxer { if (slice_queue.expected_length === 0 || slice_queue.expected_length === slice_queue.total_length) { this.emitPESSlices(slice_queue, misc); } else { - this.clearSlices(slice_queue, misc); + this.clearSlices(slice_queue); } } @@ -535,7 +498,7 @@ class TSDemuxer extends BaseDemuxer { if (slice_queue.expected_length > 0 && slice_queue.expected_length === slice_queue.total_length) { this.emitPESSlices(slice_queue, misc); } else if (slice_queue.expected_length > 0 && slice_queue.expected_length < slice_queue.total_length) { - this.clearSlices(slice_queue, misc); + this.clearSlices(slice_queue); } } @@ -578,7 +541,7 @@ class TSDemuxer extends BaseDemuxer { this.parsePES(pes_data); } - private clearSlices(slice_queue: SliceQueue, _misc: TSSliceMisc): void { + private clearSlices(slice_queue: SliceQueue): void { slice_queue.slices = []; slice_queue.expected_length = -1; slice_queue.total_length = 0; @@ -592,8 +555,6 @@ class TSDemuxer extends BaseDemuxer { this.parsePAT(data); } else if (pid === this.current_pmt_pid_) { this.parsePMT(data); - } else if (this.pmt_?.scte_35_pids[pid]) { - this.parseSCTE35(data); } } @@ -609,126 +570,77 @@ class TSDemuxer extends BaseDemuxer { } if ( - stream_id !== 0xbc && // program_stream_map - stream_id !== 0xbe && // padding_stream - stream_id !== 0xbf && // private_stream_2 - stream_id !== 0xf0 && // ECM - stream_id !== 0xf1 && // EMM - stream_id !== 0xff && // program_stream_directory - stream_id !== 0xf2 && // DSMCC - stream_id !== 0xf8 + stream_id === 0xbc || // program_stream_map + stream_id === 0xbe || // padding_stream + stream_id === 0xbf || // private_stream_2 + stream_id === 0xf0 || // ECM + stream_id === 0xf1 || // EMM + stream_id === 0xff || // program_stream_directory + stream_id === 0xf2 || // DSMCC + stream_id === 0xf8 ) { - const _PES_scrambling_control = (data[6] & 0x30) >>> 4; - const PTS_DTS_flags = (data[7] & 0xc0) >>> 6; - const PES_header_data_length = data[8]; + return; + } - let pts: number | undefined; - let dts: number | undefined; + const PTS_DTS_flags = (data[7] & 0xc0) >>> 6; + const PES_header_data_length = data[8]; - if (PTS_DTS_flags === 0x02 || PTS_DTS_flags === 0x03) { - pts = this.getTimestamp(data, 9); - dts = PTS_DTS_flags === 0x03 ? this.getTimestamp(data, 14) : pts; - } + let pts: number | undefined; + let dts: number | undefined; - const payload_start_index = 6 + 3 + PES_header_data_length; - let payload_length: number; + if (PTS_DTS_flags === 0x02 || PTS_DTS_flags === 0x03) { + pts = this.getTimestamp(data, 9); + dts = PTS_DTS_flags === 0x03 ? this.getTimestamp(data, 14) : pts; + } - if (PES_packet_length !== 0) { - if (PES_packet_length < 3 + PES_header_data_length) { - Log.v(this.TAG, `Malformed PES: PES_packet_length < 3 + PES_header_data_length`); - return; - } - payload_length = PES_packet_length - 3 - PES_header_data_length; - } else { - // PES_packet_length === 0 - payload_length = data.byteLength - payload_start_index; + const payload_start_index = 6 + 3 + PES_header_data_length; + let payload_length: number; + + if (PES_packet_length !== 0) { + if (PES_packet_length < 3 + PES_header_data_length) { + Log.v(this.TAG, `Malformed PES: PES_packet_length < 3 + PES_header_data_length`); + return; } + payload_length = PES_packet_length - 3 - PES_header_data_length; + } else { + // PES_packet_length === 0 + payload_length = data.byteLength - payload_start_index; + } - const payload = data.subarray(payload_start_index, payload_start_index + payload_length); + const payload = data.subarray(payload_start_index, payload_start_index + payload_length); - switch (pes_data.stream_type) { - case StreamType.kMPEG1Audio: - case StreamType.kMPEG2Audio: - this.parseMP3Payload(payload, pts); - break; - case StreamType.kPESPrivateData: - if (this.pmt_.common_pids.av1 === pes_data.pid) { - // this.parseAV1Payload( - // payload, - // pts, - // dts, - // pes_data.file_position, - // pes_data.random_access_indicator - // ); - } else if (this.pmt_.common_pids.opus === pes_data.pid) { - this.parseOpusPayload(payload, pts); - } else if (this.pmt_.common_pids.ac3 === pes_data.pid) { - this.parseAC3Payload(payload, pts); - } else if (this.pmt_.common_pids.eac3 === pes_data.pid) { - this.parseEAC3Payload(payload, pts); - } else if (this.pmt_.asynchronous_klv_pids[pes_data.pid]) { - this.parseAsynchronousKLVMetadataPayload(payload, pes_data.pid, stream_id); - } else if (this.pmt_.smpte2038_pids[pes_data.pid]) { - this.parseSMPTE2038MetadataPayload(payload, pts, dts, pes_data.pid, stream_id); - } else { - this.parsePESPrivateDataPayload(payload, pts, dts, pes_data.pid, stream_id); - } - break; - case StreamType.kADTSAAC: - this.parseADTSAACPayload(payload, pts); - break; - case StreamType.kLOASAAC: - this.parseLOASAACPayload(payload, pts); - break; - case StreamType.kAC3: + switch (pes_data.stream_type) { + case StreamType.kMPEG1Audio: + case StreamType.kMPEG2Audio: + this.parseMP3Payload(payload, pts); + break; + case StreamType.kPESPrivateData: + if (this.pmt_.common_pids.ac3 === pes_data.pid) { this.parseAC3Payload(payload, pts); - break; - case StreamType.kEAC3: + } else if (this.pmt_.common_pids.eac3 === pes_data.pid) { this.parseEAC3Payload(payload, pts); - break; - case StreamType.kMetadata: - if (this.pmt_.timed_id3_pids[pes_data.pid]) { - this.parseTimedID3MetadataPayload(payload, pts, dts, pes_data.pid, stream_id); - } else if (this.pmt_.synchronous_klv_pids[pes_data.pid]) { - this.parseSynchronousKLVMetadataPayload(payload, pts, dts, pes_data.pid, stream_id); - } - break; - case StreamType.kPGS: - this.parsePGSPayload(payload, pts, dts, pes_data.pid, stream_id, this.pmt_.pgs_langs[pes_data.pid]); - break; - case StreamType.kH264: - this.parseH264Payload(payload, pts, dts, pes_data.file_position, pes_data.random_access_indicator); - break; - case StreamType.kH265: - this.parseH265Payload(payload, pts, dts, pes_data.file_position, pes_data.random_access_indicator); - break; - default: - break; - } - } else if ( - stream_id === 0xbc || // program_stream_map - stream_id === 0xbf || // private_stream_2 - stream_id === 0xf0 || // ECM - stream_id === 0xf1 || // EMM - stream_id === 0xff || // program_stream_directory - stream_id === 0xf2 || // DSMCC_stream - stream_id === 0xf8 - ) { - // ITU-T Rec. H.222.1 type E stream - if (pes_data.stream_type === StreamType.kPESPrivateData) { - const payload_start_index = 6; - let payload_length: number; - - if (PES_packet_length !== 0) { - payload_length = PES_packet_length; - } else { - // PES_packet_length === 0 - payload_length = data.byteLength - payload_start_index; } - - const payload = data.subarray(payload_start_index, payload_start_index + payload_length); - this.parsePESPrivateDataPayload(payload, undefined, undefined, pes_data.pid, stream_id); - } + break; + case StreamType.kADTSAAC: + this.parseADTSAACPayload(payload, pts); + break; + case StreamType.kLOASAAC: + this.parseLOASAACPayload(payload, pts); + break; + case StreamType.kAC3: + this.parseAC3Payload(payload, pts); + break; + case StreamType.kEAC3: + this.parseEAC3Payload(payload, pts); + break; + case StreamType.kH264: + this.parseH264Payload(payload, pts, dts, pes_data.file_position, pes_data.random_access_indicator); + break; + case StreamType.kH265: + this.parseH265Payload(payload, pts, dts, pes_data.file_position); + break; + default: + break; } } @@ -741,11 +653,9 @@ class TSDemuxer extends BaseDemuxer { const section_length = ((data[1] & 0x0f) << 8) | data[2]; - const _transport_stream_id = (data[3] << 8) | data[4]; const version_number = (data[5] & 0x3e) >>> 1; const current_next_indicator = data[5] & 0x01; const section_number = data[6]; - const _last_section_number = data[7]; let pat: PAT | null = null; @@ -809,7 +719,6 @@ class TSDemuxer extends BaseDemuxer { const version_number = (data[5] & 0x3e) >>> 1; const current_next_indicator = data[5] & 0x01; const section_number = data[6]; - const _last_section_number = data[7]; let pmt: PMT | null = null; @@ -844,7 +753,6 @@ class TSDemuxer extends BaseDemuxer { pmt.common_pids.loas_aac || pmt.common_pids.ac3 || pmt.common_pids.eac3 || - pmt.common_pids.opus || pmt.common_pids.mp3; if (stream_type === StreamType.kH264 && !already_has_video) { @@ -864,134 +772,26 @@ class TSDemuxer extends BaseDemuxer { !already_has_audio ) { pmt.common_pids.mp3 = elementary_PID; - } else if (stream_type === StreamType.kPESPrivateData) { - pmt.pes_private_data_pids[elementary_PID] = true; - if (ES_info_length > 0) { - // parse descriptor for PES private data - for (let offset = i + 5; offset < i + 5 + ES_info_length; ) { - const tag = data[offset + 0]; - const length = data[offset + 1]; - if (tag === 0x05) { - // Registration Descriptor - const registration = String.fromCharCode(...Array.from(data.subarray(offset + 2, offset + 2 + length))); - - if (registration === "VANC") { - pmt.smpte2038_pids[elementary_PID] = true; - } else if (registration === "AC-3" && !already_has_audio) { - pmt.common_pids.ac3 = elementary_PID; // DVB AC-3 (FIXME: NEED VERIFY) - } else if (registration === "EC-3" && !already_has_audio) { - pmt.common_pids.eac3 = elementary_PID; // DVB EAC-3 (FIXME: NEED VERIFY) - } else if (registration === "AV01") { - pmt.common_pids.av1 = elementary_PID; - } else if (registration === "Opus") { - pmt.common_pids.opus = elementary_PID; - } else if (registration === "KLVA") { - pmt.asynchronous_klv_pids[elementary_PID] = true; - } - } else if (tag === 0x7f) { - // DVB extension descriptor - if (elementary_PID === pmt.common_pids.opus) { - const ext_desc_tag = data[offset + 2]; - let channel_config_code: number | null = null; - if (ext_desc_tag === 0x80) { - // User defined (provisional Opus) - channel_config_code = data[offset + 3]; - } - - if (channel_config_code == null) { - Log.e(this.TAG, `Not Supported Opus channel count.`); - continue; - } - - const meta = { - codec: "opus", - channel_count: (channel_config_code & 0x0f) === 0 ? 2 : channel_config_code & 0x0f, - channel_config_code, - sample_rate: 48000, - } as const; - const sample = { - codec: "opus", - meta, - } as const; - - if (this.audio_init_segment_dispatched_ === false) { - this.audio_metadata_ = meta; - this.dispatchAudioInitSegment(sample); - } else if (this.detectAudioMetadataChange(sample)) { - // flush stashed frames before notify new AudioSpecificConfig - this.dispatchAudioMediaSegment(); - // notify new AAC AudioSpecificConfig - this.dispatchAudioInitSegment(sample); - } - } - } else if (tag === 0x80) { - if (elementary_PID === pmt.common_pids.av1) { - this.video_metadata_.av1c = data.subarray(offset + 2, offset + 2 + length); - } - } else if (tag === 0x82) { - pmt.common_pids.ac3 = elementary_PID; - } else if (tag === 0x7a) { - pmt.common_pids.eac3 = elementary_PID; - } - - offset += 2 + length; - } - // provide descriptor for PES private data via callback - const descriptors = data.subarray(i + 5, i + 5 + ES_info_length); - this.dispatchPESPrivateDataDescriptor(elementary_PID, stream_type, descriptors); - } - } else if (stream_type === StreamType.kMetadata) { - if (ES_info_length > 0) { - // parse descriptor for PES private data - for (let offset = i + 5; offset < i + 5 + ES_info_length; ) { - const tag = data[offset + 0]; - const length = data[offset + 1]; - - if (tag === 0x26) { - const metadata_application_format = (data[offset + 2] << 8) | (data[offset + 3] << 0); - let metadata_application_format_identifier: string | null = null; - if (metadata_application_format === 0xffff) { - metadata_application_format_identifier = String.fromCharCode( - ...Array.from(data.subarray(offset + 4, offset + 4 + 4)), - ); - } - const metadata_format = data[offset + 4 + (metadata_application_format === 0xffff ? 4 : 0)]; - let metadata_format_identifier: string | null = null; - if (metadata_format === 0xff) { - const pad = 4 + (metadata_application_format === 0xffff ? 4 : 0) + 1; - metadata_format_identifier = String.fromCharCode( - ...Array.from(data.subarray(offset + pad, offset + pad + 4)), - ); - } - - if (metadata_application_format_identifier === "ID3 " && metadata_format_identifier === "ID3 ") { - pmt.timed_id3_pids[elementary_PID] = true; - } else if (metadata_format_identifier === "KLVA") { - pmt.synchronous_klv_pids[elementary_PID] = true; - } - } - - offset += 2 + length; - } - } - } else if (stream_type === StreamType.kSCTE35) { - pmt.scte_35_pids[elementary_PID] = true; - } else if (stream_type === StreamType.kPGS) { - pmt.pgs_langs[elementary_PID] = "und"; - if (ES_info_length > 0) { - // parse descriptor - for (let offset = i + 5; offset < i + 5 + ES_info_length; ) { - const tag = data[offset + 0]; - const length = data[offset + 1]; - if (tag === 0x0a) { - // ISO_639_LANGUAGE_DESCRIPTOR - const lang = String.fromCharCode(...Array.from(data.slice(offset + 2, offset + 5))); - pmt.pgs_langs[elementary_PID] = lang; + } else if (stream_type === StreamType.kPESPrivateData && ES_info_length > 0) { + // parse descriptors to detect DVB AC-3 / E-AC-3 in private PES + for (let offset = i + 5; offset < i + 5 + ES_info_length; ) { + const tag = data[offset + 0]; + const length = data[offset + 1]; + if (tag === 0x05) { + // Registration Descriptor + const registration = String.fromCharCode(...Array.from(data.subarray(offset + 2, offset + 2 + length))); + if (registration === "AC-3" && !already_has_audio) { + pmt.common_pids.ac3 = elementary_PID; // DVB AC-3 + } else if (registration === "EC-3" && !already_has_audio) { + pmt.common_pids.eac3 = elementary_PID; // DVB EAC-3 } - offset += 2 + length; + } else if (tag === 0x82) { + pmt.common_pids.ac3 = elementary_PID; + } else if (tag === 0x7a) { + pmt.common_pids.eac3 = elementary_PID; } + offset += 2 + length; } - pmt.pgs_pids[elementary_PID] = true; } i += 5 + ES_info_length; @@ -1002,14 +802,14 @@ class TSDemuxer extends BaseDemuxer { Log.v(this.TAG, `Parsed first PMT: ${JSON.stringify(pmt)}`); } this.pmt_ = pmt; - if (pmt.common_pids.h264 || pmt.common_pids.h265 || pmt.common_pids.av1) { + if (pmt.common_pids.h264 || pmt.common_pids.h265) { this.has_video_ = true; } if ( pmt.common_pids.adts_aac || pmt.common_pids.loas_aac || pmt.common_pids.ac3 || - pmt.common_pids.opus || + pmt.common_pids.eac3 || pmt.common_pids.mp3 ) { this.has_audio_ = true; @@ -1017,104 +817,6 @@ class TSDemuxer extends BaseDemuxer { } } - private parseSCTE35(data: Uint8Array): void { - const scte35 = readSCTE35(data); - - if (scte35.pts !== undefined) { - const pts_ms = Math.floor(scte35.pts / this.timescale_); - scte35.pts = pts_ms; - } else { - scte35.nearest_pts = this.getNearestTimestampMilliseconds(); - } - - if (this.onSCTE35Metadata) { - this.onSCTE35Metadata(scte35); - } - } - - // private parseAV1Payload( - // data: Uint8Array, - // pts: number, - // dts: number, - // file_position: number, - // random_access_indicator: number - // ) { - // let av1_in_ts_parser = new AV1OBUInMpegTsParser(data); - // let payload: Uint8Array | null = null; - // let units: { data: Uint8Array }[] = []; - // let length = 0; - // let keyframe = false; - - // let details = null; - // while ((payload = av1_in_ts_parser.readNextOBUPayload()) != null) { - // details = AV1OBUParser.parseOBUs(payload, this.video_metadata_.details); - - // if (details && details.keyframe === true) { - // if (!this.video_init_segment_dispatched_) { - // const av1c = new Uint8Array( - // new ArrayBuffer( - // this.video_metadata_.av1c.byteLength + - // details.sequence_header_data.byteLength - // ) - // ); - // av1c.set(this.video_metadata_.av1c, 0); - // av1c.set( - // details.sequence_header_data, - // this.video_metadata_.av1c.byteLength - // ); - // details.av1c = av1c; - - // this.video_metadata_.details = details; - // this.dispatchVideoInitSegment(); - // } else if (this.detectVideoMetadataChange(null, details) === true) { - // this.video_metadata_changed_ = true; - // // flush stashed frames before changing codec metadata - // this.dispatchVideoMediaSegment(); - - // const av1c = new Uint8Array( - // new ArrayBuffer( - // this.video_metadata_.av1c.byteLength + - // details.sequence_header_data.byteLength - // ) - // ); - // av1c.set(this.video_metadata_.av1c, 0); - // av1c.set( - // details.sequence_header_data, - // this.video_metadata_.av1c.byteLength - // ); - // details.av1c = av1c; - // // notify new codec metadata (maybe changed) - // this.dispatchVideoInitSegment(); - // } - // } - // this.video_metadata_.details = details; - - // //if (this.video_init_segment_dispatched_) { - // keyframe ||= details.keyframe; - // units.push({ data: payload }); - // length += payload.byteLength; - // //} - // } - - // let pts_ms = Math.floor(pts / this.timescale_); - // let dts_ms = Math.floor(dts / this.timescale_); - - // if (units.length) { - // let track = this.video_track_; - // let av1_sample = { - // units, - // length, - // isKeyframe: keyframe, - // dts: dts_ms, - // pts: pts_ms, - // cts: pts_ms - dts_ms, - // file_position, - // }; - // track.samples.push(av1_sample); - // track.length += length; - // } - // } - private parseH264Payload( data: Uint8Array, pts: number | undefined, @@ -1138,14 +840,13 @@ class TSDemuxer extends BaseDemuxer { if (!this.video_init_segment_dispatched_) { this.video_metadata_.sps = nalu_avc1; this.video_metadata_.details = details; - } else if (this.detectVideoMetadataChange(nalu_avc1, details) === true) { + } else if (this.detectVideoMetadataChange(details) === true) { Log.v(this.TAG, `H264: Critical h264 metadata has been changed, attempt to re-generate InitSegment`); this.video_metadata_changed_ = true; this.video_metadata_ = { vps: undefined, sps: nalu_avc1, pps: undefined, - av1c: undefined, details: details, }; } @@ -1199,13 +900,7 @@ class TSDemuxer extends BaseDemuxer { } } - private parseH265Payload( - data: Uint8Array, - pts: number | undefined, - dts: number | undefined, - file_position: number, - _random_access_indicator: number, - ) { + private parseH265Payload(data: Uint8Array, pts: number | undefined, dts: number | undefined, file_position: number) { const annexb_parser = new H265AnnexBParser(data); let nalu_payload: H265NaluPayload | null = null; const units: { type: H265NaluType; data: Uint8Array }[] = []; @@ -1233,14 +928,13 @@ class TSDemuxer extends BaseDemuxer { ...this.video_metadata_.details, ...details, }; - } else if (this.detectVideoMetadataChange(nalu_hvc1, details) === true) { + } else if (this.detectVideoMetadataChange(details) === true) { Log.v(this.TAG, `H265: Critical h265 metadata has been changed, attempt to re-generate InitSegment`); this.video_metadata_changed_ = true; this.video_metadata_ = { vps: undefined, sps: nalu_hvc1, pps: undefined, - av1c: undefined, details: details, }; } @@ -1301,10 +995,7 @@ class TSDemuxer extends BaseDemuxer { } } - private detectVideoMetadataChange( - _new_sps: H264NaluAVC1 | H265NaluHVC1, - new_details: Record, - ): boolean { + private detectVideoMetadataChange(new_details: Record): boolean { const old_details = this.video_metadata_.details; if (new_details.codec_mimetype !== old_details.codec_mimetype) { Log.v( @@ -1361,7 +1052,7 @@ class TSDemuxer extends BaseDemuxer { meta.type = "video"; meta.id = this.video_track_.id; meta.timescale = 1000; - meta.duration = this.duration_; + meta.duration = 0; const codec_size = details.codec_size as Record; const present_size = details.present_size as Record; @@ -1386,12 +1077,7 @@ class TSDemuxer extends BaseDemuxer { meta.codec = details.codec_mimetype; - if (this.video_metadata_.av1c) { - meta.av1c = this.video_metadata_.av1c; - if (this.video_init_segment_dispatched_ === false) { - Log.v(this.TAG, `Generated first AV1 for mimeType: ${meta.codec}`); - } - } else if (this.video_metadata_.vps) { + if (this.video_metadata_.vps) { const vps_without_header = this.video_metadata_.vps.data.subarray(4); const sps_without_header = this.video_metadata_.sps?.data.subarray(4); const pps_without_header = this.video_metadata_.pps?.data.subarray(4); @@ -1425,30 +1111,6 @@ class TSDemuxer extends BaseDemuxer { this.onTrackMetadata?.("video", meta); this.video_init_segment_dispatched_ = true; this.video_metadata_changed_ = false; - - // notify new MediaInfo - const mi = this.media_info_; - mi.hasVideo = true; - mi.width = meta.codecWidth as number | null; - mi.height = meta.codecHeight as number | null; - mi.fps = frame_rate.fps as number; - mi.profile = meta.profile as string | null; - mi.level = meta.level as string | null; - mi.refFrames = details.ref_frames as unknown as number | null; - mi.chromaFormat = details.chroma_format_string as unknown as string | null; - mi.sarNum = sar_ratio.width as number; - mi.sarDen = sar_ratio.height as number; - mi.videoCodec = meta.codec as string | null; - - if (mi.hasAudio && mi.audioCodec) { - mi.mimeType = `video/mp2t; codecs="${mi.videoCodec},${mi.audioCodec}"`; - } else { - mi.mimeType = `video/mp2t; codecs="${mi.videoCodec}"`; - } - - if (mi.isComplete()) { - this.onMediaInfo?.(mi); - } } private dispatchVideoMediaSegment() { @@ -1714,9 +1376,8 @@ class TSDemuxer extends BaseDemuxer { }; this.dispatchAudioInitSegment(audio_sample); } else if (this.detectAudioMetadataChange(audio_sample)) { - // flush stashed frames before notify new AudioSpecificConfig + // flush stashed frames before notify new config this.dispatchAudioMediaSegment(); - // notify new AAC AudioSpecificConfig this.dispatchAudioInitSegment(audio_sample); } @@ -1759,7 +1420,7 @@ class TSDemuxer extends BaseDemuxer { if (this.audio_metadata_.codec === "ec-3") { if (pts === undefined && this.audio_last_sample_pts_ !== undefined) { - ref_sample_duration = ((256 * this.audio_metadata_.num_blks) / this.audio_metadata_.sampling_frequency) * 1000; // TODO: AEC3 BLK + ref_sample_duration = ((256 * this.audio_metadata_.num_blks) / this.audio_metadata_.sampling_frequency) * 1000; base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; } else if (pts === undefined) { Log.w(this.TAG, `EAC3: Unknown pts`); @@ -1774,7 +1435,7 @@ class TSDemuxer extends BaseDemuxer { eac3_frame = adts_parser.readNextEAC3Frame(); while (eac3_frame != null) { - ref_sample_duration = (1536 / eac3_frame.sampling_frequency) * 1000; // TODO: EAC3 BLK + ref_sample_duration = (1536 / eac3_frame.sampling_frequency) * 1000; const audio_sample = { codec: "ec-3", data: eac3_frame, @@ -1791,9 +1452,8 @@ class TSDemuxer extends BaseDemuxer { }; this.dispatchAudioInitSegment(audio_sample); } else if (this.detectAudioMetadataChange(audio_sample)) { - // flush stashed frames before notify new AudioSpecificConfig + // flush stashed frames before notify new config this.dispatchAudioMediaSegment(); - // notify new AAC AudioSpecificConfig this.dispatchAudioInitSegment(audio_sample); } @@ -1820,71 +1480,6 @@ class TSDemuxer extends BaseDemuxer { } } - private parseOpusPayload(data: Uint8Array, pts: number | undefined) { - if (this.has_video_ && !this.video_init_segment_dispatched_) { - // If first video IDR frame hasn't been detected, - // Wait for first IDR frame and video init segment being dispatched - return; - } - - let ref_sample_duration: number; - let base_pts_ms!: number; - - if (pts !== undefined) { - base_pts_ms = pts / this.timescale_; - } - if (this.audio_metadata_.codec === "opus") { - if (pts === undefined && this.audio_last_sample_pts_ !== undefined) { - ref_sample_duration = 20; - base_pts_ms = this.audio_last_sample_pts_ + ref_sample_duration; - } else if (pts === undefined) { - Log.w(this.TAG, `Opus: Unknown pts`); - return; - } - } - - let sample_pts_ms = base_pts_ms; - let last_sample_pts_ms: number | undefined; - - for (let offset = 0; offset < data.length; ) { - ref_sample_duration = 20; - - const opus_pending_trim_start = (data[offset + 1] & 0x10) !== 0; - const trim_end = (data[offset + 1] & 0x08) !== 0; - let index = offset + 2; - let size = 0; - - while (data[index] === 0xff) { - size += 255; - index += 1; - } - size += data[index]; - index += 1; - index += opus_pending_trim_start ? 2 : 0; - index += trim_end ? 2 : 0; - - last_sample_pts_ms = sample_pts_ms; - const sample_pts_ms_int = Math.floor(sample_pts_ms); - const sample = data.slice(index, index + size); - - const opus_sample = { - unit: sample, - length: sample.byteLength, - pts: sample_pts_ms_int, - dts: sample_pts_ms_int, - }; - this.audio_track_.samples.push(opus_sample); - this.audio_track_.length += sample.byteLength; - - sample_pts_ms += ref_sample_duration; - offset = index + size; - } - - if (last_sample_pts_ms) { - this.audio_last_sample_pts_ = last_sample_pts_ms; - } - } - private parseMP3Payload(data: Uint8Array, pts: number | undefined) { if (this.has_video_ && !this.video_init_segment_dispatched_) { // If first video IDR frame hasn't been detected, @@ -1895,22 +1490,16 @@ class TSDemuxer extends BaseDemuxer { const _mpegAudioV10SampleRateTable = [44100, 48000, 32000, 0]; const _mpegAudioV20SampleRateTable = [22050, 24000, 16000, 0]; const _mpegAudioV25SampleRateTable = [11025, 12000, 8000, 0]; - const _mpegAudioL1BitRateTable = [0, 32, 64, 96, 128, 160, 192, 224, 256, 288, 320, 352, 384, 416, 448, -1]; - const _mpegAudioL2BitRateTable = [0, 32, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, 384, -1]; - const _mpegAudioL3BitRateTable = [0, 32, 40, 48, 56, 64, 80, 96, 112, 128, 160, 192, 224, 256, 320, -1]; const ver = (data[1] >>> 3) & 0x03; const layer = (data[1] & 0x06) >> 1; - const bitrate_index = (data[2] & 0xf0) >>> 4; const sampling_freq_index = (data[2] & 0x0c) >>> 2; const channel_mode = (data[3] >>> 6) & 0x03; const channel_count = channel_mode !== 3 ? 2 : 1; let sample_rate = 0; - let _bit_rate = 0; let object_type = 34; // Layer-3, listed in MPEG-4 Audio Object Types - const _codec = "mp3"; switch (ver) { case 0: // MPEG 2.5 sample_rate = _mpegAudioV25SampleRateTable[sampling_freq_index]; @@ -1926,21 +1515,12 @@ class TSDemuxer extends BaseDemuxer { switch (layer) { case 1: // Layer 3 object_type = 34; - if (bitrate_index < _mpegAudioL3BitRateTable.length) { - _bit_rate = _mpegAudioL3BitRateTable[bitrate_index]; - } break; case 2: // Layer 2 object_type = 33; - if (bitrate_index < _mpegAudioL2BitRateTable.length) { - _bit_rate = _mpegAudioL2BitRateTable[bitrate_index]; - } break; case 3: // Layer 1 object_type = 32; - if (bitrate_index < _mpegAudioL1BitRateTable.length) { - _bit_rate = _mpegAudioL1BitRateTable[bitrate_index]; - } break; } @@ -1952,7 +1532,7 @@ class TSDemuxer extends BaseDemuxer { Log.i(this.TAG, `MP2 audio detected, enabling software decode`); } - // Dispatch audio init segment (as MP3) for MediaInfo + // Dispatch audio init segment (as MP3) for track metadata const mp3sample = new MP3Data(); mp3sample.object_type = object_type; mp3sample.sample_rate = sample_rate; @@ -2007,9 +1587,8 @@ class TSDemuxer extends BaseDemuxer { }; this.dispatchAudioInitSegment(audio_sample); } else if (this.detectAudioMetadataChange(audio_sample)) { - // flush stashed frames before notify new AudioSpecificConfig + // flush stashed frames before notify new config this.dispatchAudioMediaSegment(); - // notify new AAC AudioSpecificConfig this.dispatchAudioInitSegment(audio_sample); } @@ -2098,24 +1677,6 @@ class TSDemuxer extends BaseDemuxer { ); return true; } - } else if (sample.codec === "opus" && this.audio_metadata_.codec === "opus") { - const data = sample.meta; - - if (data.sample_rate !== this.audio_metadata_.sample_rate) { - Log.v( - this.TAG, - `Opus: SamplingFrequencyIndex changed from ${this.audio_metadata_.sample_rate} to ${data.sample_rate}`, - ); - return true; - } - - if (data.channel_count !== this.audio_metadata_.channel_count) { - Log.v( - this.TAG, - `Opus: Channel count changed from ${this.audio_metadata_.channel_count} to ${data.channel_count}`, - ); - return true; - } } else if (sample.codec === "mp3" && this.audio_metadata_.codec === "mp3") { const data = sample.data; if (data.object_type !== this.audio_metadata_.object_type) { @@ -2148,7 +1709,7 @@ class TSDemuxer extends BaseDemuxer { meta.type = "audio"; meta.id = this.audio_track_.id; meta.timescale = 1000; - meta.duration = this.duration_; + meta.duration = 0; if (this.audio_metadata_.codec === "aac") { if (sample.codec !== "aac") { @@ -2184,15 +1745,7 @@ class TSDemuxer extends BaseDemuxer { meta.originalCodec = ec3_config.original_codec_mimetype; meta.config = ec3_config.config; meta.refSampleDuration = - ((256 * ec3_config.num_blks) / (meta.audioSampleRate as number)) * (meta.timescale as number); // TODO: blk size - } else if (this.audio_metadata_.codec === "opus") { - meta.audioSampleRate = this.audio_metadata_.sample_rate; - meta.channelCount = this.audio_metadata_.channel_count; - meta.channelConfigCode = this.audio_metadata_.channel_config_code; - meta.codec = "opus"; - meta.originalCodec = "opus"; - meta.config = undefined; - meta.refSampleDuration = 20; + ((256 * ec3_config.num_blks) / (meta.audioSampleRate as number)) * (meta.timescale as number); } else if (this.audio_metadata_.codec === "mp3") { meta.audioSampleRate = this.audio_metadata_.sample_rate; meta.channelCount = this.audio_metadata_.channel_count; @@ -2217,7 +1770,7 @@ class TSDemuxer extends BaseDemuxer { type: "audio", id: this.audio_track_.id, timescale: 1000, - duration: this.duration_, + duration: 0, audioSampleRate: sampleRate, channelCount: channelCount, codec: "mp4a.40.2", @@ -2232,212 +1785,6 @@ class TSDemuxer extends BaseDemuxer { } this.audio_init_segment_dispatched_ = true; this.video_metadata_changed_ = false; - - // notify new MediaInfo - const mi = this.media_info_; - mi.hasAudio = true; - mi.audioCodec = meta.originalCodec as string | null; - mi.audioSampleRate = meta.audioSampleRate as number | null; - mi.audioChannelCount = meta.channelCount as number | null; - - if (mi.hasVideo && mi.videoCodec) { - mi.mimeType = `video/mp2t; codecs="${mi.videoCodec},${mi.audioCodec}"`; - } else { - mi.mimeType = `video/mp2t; codecs="${mi.audioCodec}"`; - } - - if (mi.isComplete()) { - this.onMediaInfo?.(mi); - } - } - - private dispatchPESPrivateDataDescriptor(pid: number, stream_type: number, descriptor: Uint8Array) { - const desc = new PESPrivateDataDescriptor(); - desc.pid = pid; - desc.stream_type = stream_type; - desc.descriptor = descriptor; - - if (this.onPESPrivateDataDescriptor) { - this.onPESPrivateDataDescriptor(desc); - } - } - - private parsePESPrivateDataPayload( - data: Uint8Array, - pts: number | undefined, - dts: number | undefined, - pid: number, - stream_id: number, - ) { - const private_data = new PESPrivateData(); - - private_data.pid = pid; - private_data.stream_id = stream_id; - private_data.len = data.byteLength; - private_data.data = data; - - if (pts !== undefined) { - const pts_ms = Math.floor(pts / this.timescale_); - private_data.pts = pts_ms; - } else { - private_data.nearest_pts = this.getNearestTimestampMilliseconds(); - } - - if (dts !== undefined) { - const dts_ms = Math.floor(dts / this.timescale_); - private_data.dts = dts_ms; - } - - if (this.onPESPrivateData) { - this.onPESPrivateData(private_data); - } - } - - private parseTimedID3MetadataPayload( - data: Uint8Array, - pts: number | undefined, - dts: number | undefined, - pid: number, - stream_id: number, - ) { - const timed_id3_metadata = new PESPrivateData(); - - timed_id3_metadata.pid = pid; - timed_id3_metadata.stream_id = stream_id; - timed_id3_metadata.len = data.byteLength; - timed_id3_metadata.data = data; - - if (pts !== undefined) { - const pts_ms = Math.floor(pts / this.timescale_); - timed_id3_metadata.pts = pts_ms; - } - - if (dts !== undefined) { - const dts_ms = Math.floor(dts / this.timescale_); - timed_id3_metadata.dts = dts_ms; - } - - if (this.onTimedID3Metadata) { - this.onTimedID3Metadata(timed_id3_metadata); - } - } - - private parsePGSPayload( - data: Uint8Array, - pts: number | undefined, - dts: number | undefined, - pid: number, - stream_id: number, - lang: string, - ) { - const pgs_data = new PGSData(); - - pgs_data.pid = pid; - pgs_data.lang = lang; - pgs_data.stream_id = stream_id; - pgs_data.len = data.byteLength; - pgs_data.data = data; - - if (pts !== undefined) { - const pts_ms = Math.floor(pts / this.timescale_); - pgs_data.pts = pts_ms; - } - - if (dts !== undefined) { - const dts_ms = Math.floor(dts / this.timescale_); - pgs_data.dts = dts_ms; - } - - if (this.onPGSSubtitleData) { - this.onPGSSubtitleData(pgs_data); - } - } - - private parseSynchronousKLVMetadataPayload( - data: Uint8Array, - pts: number | undefined, - dts: number | undefined, - pid: number, - stream_id: number, - ) { - const synchronous_klv_metadata = new KLVData(); - - synchronous_klv_metadata.pid = pid; - synchronous_klv_metadata.stream_id = stream_id; - synchronous_klv_metadata.len = data.byteLength; - synchronous_klv_metadata.data = data; - - if (pts !== undefined) { - const pts_ms = Math.floor(pts / this.timescale_); - synchronous_klv_metadata.pts = pts_ms; - } - - if (dts !== undefined) { - const dts_ms = Math.floor(dts / this.timescale_); - synchronous_klv_metadata.dts = dts_ms; - } - - synchronous_klv_metadata.access_units = klv_parse(data); - - if (this.onSynchronousKLVMetadata) { - this.onSynchronousKLVMetadata(synchronous_klv_metadata); - } - } - - private parseAsynchronousKLVMetadataPayload(data: Uint8Array, pid: number, stream_id: number) { - const asynchronous_klv_metadata = new PESPrivateData(); - - asynchronous_klv_metadata.pid = pid; - asynchronous_klv_metadata.stream_id = stream_id; - asynchronous_klv_metadata.len = data.byteLength; - asynchronous_klv_metadata.data = data; - - if (this.onAsynchronousKLVMetadata) { - this.onAsynchronousKLVMetadata(asynchronous_klv_metadata); - } - } - - private parseSMPTE2038MetadataPayload( - data: Uint8Array, - pts: number | undefined, - dts: number | undefined, - pid: number, - stream_id: number, - ) { - const smpte2038_data = new SMPTE2038Data(); - - smpte2038_data.pid = pid; - smpte2038_data.stream_id = stream_id; - smpte2038_data.len = data.byteLength; - smpte2038_data.data = data; - - if (pts !== undefined) { - const pts_ms = Math.floor(pts / this.timescale_); - smpte2038_data.pts = pts_ms; - } - smpte2038_data.nearest_pts = this.getNearestTimestampMilliseconds(); - - if (dts !== undefined) { - const dts_ms = Math.floor(dts / this.timescale_); - smpte2038_data.dts = dts_ms; - } - - smpte2038_data.ancillaries = smpte2038parse(data); - if (this.onSMPTE2038Metadata) { - this.onSMPTE2038Metadata(smpte2038_data); - } - } - - private getNearestTimestampMilliseconds(): number | undefined { - // Prefer using last audio sample pts if audio track exists - if (this.audio_last_sample_pts_ !== undefined) { - return Math.floor(this.audio_last_sample_pts_); - } else if (this.last_pcr_ !== undefined) { - // Fallback to PCR time if audio track doesn't exist - const pcr_time_ms = Math.floor(this.last_pcr_ / 300 / this.timescale_); - return pcr_time_ms; - } - return undefined; } private getPcrBase(data: Uint8Array): number { diff --git a/web-ui/src/mpegts/hls/fmp4.ts b/web-ui/src/mpegts/hls/fmp4.ts new file mode 100644 index 00000000..eecf7289 --- /dev/null +++ b/web-ui/src/mpegts/hls/fmp4.ts @@ -0,0 +1,247 @@ +/** + * Minimal ISO BMFF box parsing for the fMP4 passthrough path: + * - codec string extraction from an init segment (moov) + * - per-track timescales (mdhd) and media segment start time (moof/tfdt) + */ + +interface BoxRange { + type: string; + /** offset of the box itself (including header) */ + boxStart: number; + /** payload start offset (after the box header) */ + start: number; + /** payload end offset (exclusive) */ + end: number; +} + +function readBoxes(data: Uint8Array, start: number, end: number): BoxRange[] { + const boxes: BoxRange[] = []; + const view = new DataView(data.buffer, data.byteOffset, data.byteLength); + let offset = start; + while (offset + 8 <= end) { + let size = view.getUint32(offset); + const type = String.fromCharCode(data[offset + 4], data[offset + 5], data[offset + 6], data[offset + 7]); + let headerSize = 8; + if (size === 1) { + if (offset + 16 > end) break; + size = view.getUint32(offset + 8) * 0x100000000 + view.getUint32(offset + 12); + headerSize = 16; + } else if (size === 0) { + size = end - offset; + } + if (size < headerSize || offset + size > end) break; + boxes.push({ type, boxStart: offset, start: offset + headerSize, end: offset + size }); + offset += size; + } + return boxes; +} + +function findBox(data: Uint8Array, start: number, end: number, type: string): BoxRange | null { + return readBoxes(data, start, end).find((b) => b.type === type) ?? null; +} + +/** Probe whether the buffer looks like the start of an ISO BMFF (fMP4) stream. */ +export function probeFmp4(buffer: ArrayBuffer): boolean { + const data = new Uint8Array(buffer); + if (data.byteLength < 8) return false; + const type = String.fromCharCode(data[4], data[5], data[6], data[7]); + return ["ftyp", "styp", "moov", "moof", "sidx", "emsg", "prft", "free"].includes(type); +} + +export function containsMoov(data: Uint8Array): boolean { + return readBoxes(data, 0, data.byteLength).some((b) => b.type === "moov"); +} + +/** Split a self-initializing segment into the init part (up to the first moof) and the media part. */ +export function splitInitFromSegment(data: Uint8Array): { init: Uint8Array; media: Uint8Array } { + for (const box of readBoxes(data, 0, data.byteLength)) { + if (box.type === "moof") { + return { init: data.subarray(0, box.boxStart), media: data.subarray(box.boxStart) }; + } + } + return { init: data, media: data.subarray(data.byteLength) }; +} + +export interface InitSegmentInfo { + codecs: string[]; + /** trackId -> timescale */ + timescales: Map; +} + +const hex2 = (v: number) => v.toString(16).padStart(2, "0"); + +function hevcCodecString(prefix: "hvc1" | "hev1", hvcc: Uint8Array): string { + // ISO/IEC 14496-15 Annex E codec string from HEVCDecoderConfigurationRecord + const profileSpace = (hvcc[1] >> 6) & 0x03; + const tierFlag = (hvcc[1] >> 5) & 0x01; + const profileIdc = hvcc[1] & 0x1f; + const compat = (hvcc[2] << 24) | (hvcc[3] << 16) | (hvcc[4] << 8) | hvcc[5]; + // reverse bit order of the 32-bit compatibility flags + let reversed = 0; + for (let i = 0; i < 32; i++) { + reversed = (reversed << 1) | ((compat >>> i) & 1); + } + const levelIdc = hvcc[12]; + let result = `${prefix}.${["", "A", "B", "C"][profileSpace]}${profileIdc}.${(reversed >>> 0).toString(16)}.${ + tierFlag ? "H" : "L" + }${levelIdc}`; + // constraint bytes, trailing zero bytes omitted + const constraints = Array.from(hvcc.subarray(6, 12)); + while (constraints.length > 0 && constraints[constraints.length - 1] === 0) { + constraints.pop(); + } + for (const byte of constraints) { + result += `.${byte.toString(16).toUpperCase()}`; + } + return result; +} + +function mp4aCodecString(data: Uint8Array, entry: BoxRange): string { + const esds = findBox(data, entry.start + 28, entry.end, "esds"); + if (esds) { + // walk MPEG-4 descriptors: version(4) then ES_Descriptor(tag 3) > DecoderConfigDescriptor(tag 4) + let offset = esds.start + 4; + const readDescriptor = (): { tag: number; size: number; start: number } | null => { + if (offset >= esds.end) return null; + const tag = data[offset++]; + let size = 0; + let byte: number; + do { + byte = data[offset++]; + size = (size << 7) | (byte & 0x7f); + } while (byte & 0x80 && offset < esds.end); + return { tag, size, start: offset }; + }; + const es = readDescriptor(); + if (es && es.tag === 0x03) { + offset = es.start + 3; // ES_ID(2) + flags(1), assume no optional fields + const dec = readDescriptor(); + if (dec && dec.tag === 0x04) { + const oti = data[dec.start]; + offset = dec.start + 13; // objectTypeIndication(1) + streamType/bufferSize(4) + bitrates(8) + const asc = readDescriptor(); + if (asc && asc.tag === 0x05 && asc.size >= 1) { + const aot = data[asc.start] >> 3; + return `mp4a.${hex2(oti)}.${aot}`; + } + return `mp4a.${hex2(oti)}`; + } + } + } + return "mp4a.40.2"; +} + +function sampleEntryCodec(data: Uint8Array, entry: BoxRange): string | null { + switch (entry.type) { + case "avc1": + case "avc3": { + // VisualSampleEntry header is 78 bytes + const avcc = findBox(data, entry.start + 78, entry.end, "avcC"); + if (avcc) { + return `${entry.type}.${hex2(data[avcc.start + 1])}${hex2(data[avcc.start + 2])}${hex2(data[avcc.start + 3])}`; + } + return entry.type; + } + case "hvc1": + case "hev1": { + const hvcc = findBox(data, entry.start + 78, entry.end, "hvcC"); + if (hvcc) { + return hevcCodecString(entry.type, data.subarray(hvcc.start, hvcc.end)); + } + return entry.type; + } + case "mp4a": + return mp4aCodecString(data, entry); + case "ac-3": + case "ec-3": + return entry.type; + case ".mp3": + return "mp3"; + default: + return null; + } +} + +export function parseInitSegment(data: Uint8Array): InitSegmentInfo { + const codecs: string[] = []; + const timescales = new Map(); + + const moov = findBox(data, 0, data.byteLength, "moov"); + if (!moov) { + return { codecs, timescales }; + } + + for (const trak of readBoxes(data, moov.start, moov.end)) { + if (trak.type !== "trak") continue; + + let trackId = -1; + const tkhd = findBox(data, trak.start, trak.end, "tkhd"); + if (tkhd) { + const version = data[tkhd.start]; + const idOffset = tkhd.start + (version === 1 ? 20 : 12); + trackId = (data[idOffset] << 24) | (data[idOffset + 1] << 16) | (data[idOffset + 2] << 8) | data[idOffset + 3]; + } + + const mdia = findBox(data, trak.start, trak.end, "mdia"); + if (!mdia) continue; + + const mdhd = findBox(data, mdia.start, mdia.end, "mdhd"); + if (mdhd && trackId >= 0) { + const version = data[mdhd.start]; + const tsOffset = mdhd.start + (version === 1 ? 20 : 12); + const timescale = + ((data[tsOffset] << 24) | (data[tsOffset + 1] << 16) | (data[tsOffset + 2] << 8) | data[tsOffset + 3]) >>> 0; + timescales.set(trackId, timescale); + } + + const minf = findBox(data, mdia.start, mdia.end, "minf"); + const stbl = minf && findBox(data, minf.start, minf.end, "stbl"); + const stsd = stbl && findBox(data, stbl.start, stbl.end, "stsd"); + if (stsd) { + // stsd payload: version+flags(4) + entry_count(4), then sample entries as boxes + for (const entry of readBoxes(data, stsd.start + 8, stsd.end)) { + const codec = sampleEntryCodec(data, entry); + if (codec) { + codecs.push(codec); + } + } + } + } + + return { codecs, timescales }; +} + +/** Earliest baseMediaDecodeTime across trafs of the first moof, in seconds. */ +export function getSegmentStartTime(data: Uint8Array, timescales: Map): number | null { + const moof = findBox(data, 0, data.byteLength, "moof"); + if (!moof) return null; + + let earliest: number | null = null; + for (const traf of readBoxes(data, moof.start, moof.end)) { + if (traf.type !== "traf") continue; + + const tfhd = findBox(data, traf.start, traf.end, "tfhd"); + const tfdt = findBox(data, traf.start, traf.end, "tfdt"); + if (!tfhd || !tfdt) continue; + + const trackId = + (data[tfhd.start + 4] << 24) | (data[tfhd.start + 5] << 16) | (data[tfhd.start + 6] << 8) | data[tfhd.start + 7]; + const timescale = timescales.get(trackId); + if (!timescale) continue; + + const version = data[tfdt.start]; + const view = new DataView(data.buffer, data.byteOffset, data.byteLength); + let baseTime: number; + if (version === 1) { + baseTime = view.getUint32(tfdt.start + 4) * 0x100000000 + view.getUint32(tfdt.start + 8); + } else { + baseTime = view.getUint32(tfdt.start + 4); + } + + const seconds = baseTime / timescale; + if (earliest === null || seconds < earliest) { + earliest = seconds; + } + } + return earliest; +} diff --git a/web-ui/src/mpegts/hls/hls-source.ts b/web-ui/src/mpegts/hls/hls-source.ts new file mode 100644 index 00000000..227a902b --- /dev/null +++ b/web-ui/src/mpegts/hls/hls-source.ts @@ -0,0 +1,240 @@ +import type { PlayerConfig } from "../config"; +import Log from "../utils/logger"; +import type { SegmentMeta, SegmentSource } from "../worker/segment-source"; +import { type HlsMediaPlaylist, parseM3U8 } from "./m3u8"; + +export interface HlsInfo { + live: boolean; + targetDuration: number; + totalDuration: number; + /** CODECS attribute from the multivariant playlist, if any. */ + codecs?: string; +} + +const TAG = "HlsSource"; +/** Start playback this many segments away from the live edge. */ +const LIVE_EDGE_SEGMENTS = 3; +const MAX_REFRESH_FAILURES = 5; + +/** SegmentSource driven by an HLS media playlist (with live refresh). */ +export class HlsSource implements SegmentSource { + onInfo: ((info: HlsInfo) => void) | null = null; + + private url: string; + private readonly config: PlayerConfig; + private readonly abort = new AbortController(); + private destroyed = false; + + private live = true; + private ended = false; + private targetDuration = 6; + private totalDuration = 0; + private codecs: string | undefined; + + private segments: SegmentMeta[] = []; + private nextIndex = 0; + /** Media sequence number of the next segment to ingest from playlist refreshes. */ + private nextMediaSequence = -1; + /** Accumulated timeline position for the next appended segment, in seconds. */ + private timelinePos = 0; + private initialized = false; + /** Force a remuxer reset on the next returned segment (initial load / after seek). */ + private resetPending = true; + private refreshFailures = 0; + private lastPlaylistHadNews = true; + /** Playlist content already fetched during HLS detection, consumed on the first load. */ + private preloaded: { text: string; url: string } | null; + + constructor(url: string, config: PlayerConfig, preloaded?: { text: string; url: string }) { + this.url = preloaded?.url ?? url; + this.config = config; + this.preloaded = preloaded ?? null; + } + + get info(): HlsInfo { + return { + live: this.live, + targetDuration: this.targetDuration, + totalDuration: this.totalDuration, + codecs: this.codecs, + }; + } + + async next(): Promise { + if (!this.initialized) { + await this.initialize(); + } + + while (!this.destroyed) { + if (this.nextIndex < this.segments.length) { + const meta = this.segments[this.nextIndex++]; + if (this.resetPending) { + this.resetPending = false; + return { ...meta, resetRemuxer: true }; + } + return meta; + } + if (this.ended) { + return null; + } + await this.refresh(); + } + return null; + } + + /** Reposition to the segment containing `seconds` (VOD/EVENT only). */ + seek(seconds: number): void { + let index = 0; + for (let i = 0; i < this.segments.length; i++) { + if (this.segments[i].start <= seconds) { + index = i; + } else { + break; + } + } + this.nextIndex = index; + this.resetPending = true; + } + + destroy(): void { + this.destroyed = true; + this.abort.abort(); + } + + private async initialize(): Promise { + const playlist = await this.fetchPlaylist(); + if (playlist === null) { + throw new Error("HLS playlist load failed"); + } + + this.ingest(playlist); + + if (this.live) { + // Start near the live edge and rebase the timeline so playback starts at 0 + this.nextIndex = Math.max(0, this.segments.length - LIVE_EDGE_SEGMENTS); + const base = this.segments[this.nextIndex]?.start ?? 0; + if (base > 0) { + this.segments = this.segments.map((s) => ({ ...s, start: s.start - base })); + this.timelinePos -= base; + } + } + + this.initialized = true; + this.onInfo?.(this.info); + } + + private ingest(playlist: HlsMediaPlaylist): void { + this.live = playlist.live; + this.ended = !playlist.live; + if (playlist.targetDuration > 0) { + this.targetDuration = playlist.targetDuration; + } + + let newSegments = 0; + for (const seg of playlist.segments) { + if (this.nextMediaSequence !== -1 && seg.mediaSequence < this.nextMediaSequence) { + continue; // already ingested + } + // Detect skipped segments (playlist advanced faster than we refreshed) + const skipped = this.nextMediaSequence !== -1 && seg.mediaSequence > this.nextMediaSequence; + if (skipped) { + Log.w(TAG, `Missed HLS segments: expected sequence ${this.nextMediaSequence}, got ${seg.mediaSequence}`); + } + + this.segments.push({ + url: seg.url, + start: this.timelinePos, + duration: seg.duration, + timestampBase: 0, + resetRemuxer: seg.discontinuity || skipped, + initUrl: seg.initUrl, + }); + this.timelinePos += seg.duration; + this.nextMediaSequence = seg.mediaSequence + 1; + newSegments++; + + // Trim consumed history to bound memory on long-running live streams + if (this.live && this.nextIndex > 64) { + const drop = this.nextIndex - 32; + this.segments.splice(0, drop); + this.nextIndex -= drop; + } + } + + this.lastPlaylistHadNews = newSegments > 0; + this.totalDuration = playlist.totalDuration; + } + + private async refresh(): Promise { + // Per spec: reload after targetDuration; after an unchanged playlist, retry after half of it + const delay = (this.lastPlaylistHadNews ? this.targetDuration : this.targetDuration / 2) * 1000; + await this.sleep(delay); + if (this.destroyed) return; + + const playlist = await this.fetchPlaylist(); + if (playlist) { + this.ingest(playlist); + } + } + + /** Fetch and parse the playlist (resolving a multivariant playlist to its best variant). */ + private async fetchPlaylist(): Promise { + while (!this.destroyed) { + try { + const playlist = await this.fetchOnce(this.url); + if (playlist.kind === "multivariant") { + const best = [...playlist.variants].sort((a, b) => b.bandwidth - a.bandwidth)[0]; + if (!best) { + throw new Error("Multivariant playlist contains no variants"); + } + this.codecs = best.codecs; + this.url = best.url; + continue; // fetch the selected media playlist + } + this.refreshFailures = 0; + return playlist; + } catch (e) { + if (this.destroyed) return null; + this.refreshFailures++; + Log.w(TAG, `Playlist load failed (${this.refreshFailures}/${MAX_REFRESH_FAILURES}): ${(e as Error).message}`); + if (this.refreshFailures >= MAX_REFRESH_FAILURES) { + throw e; + } + await this.sleep((this.targetDuration / 2) * 1000); + } + } + return null; + } + + private async fetchOnce(url: string) { + if (this.preloaded) { + const { text, url: baseUrl } = this.preloaded; + this.preloaded = null; + return parseM3U8(text, baseUrl); + } + const response = await fetch(url, { + headers: this.config.headers, + signal: this.abort.signal, + referrerPolicy: (this.config.referrerPolicy as ReferrerPolicy | undefined) ?? "no-referrer-when-downgrade", + }); + if (!response.ok) { + throw new Error(`HTTP ${response.status} ${response.statusText}`); + } + const text = await response.text(); + return parseM3U8(text, response.url || url); + } + + private sleep(ms: number): Promise { + return new Promise((resolve) => { + const timer = setTimeout(resolve, ms); + this.abort.signal.addEventListener( + "abort", + () => { + clearTimeout(timer); + resolve(); + }, + { once: true }, + ); + }); + } +} diff --git a/web-ui/src/mpegts/hls/m3u8.ts b/web-ui/src/mpegts/hls/m3u8.ts new file mode 100644 index 00000000..9c689a9a --- /dev/null +++ b/web-ui/src/mpegts/hls/m3u8.ts @@ -0,0 +1,143 @@ +/** + * Minimal m3u8 playlist parser. Supports only what the player needs: + * - Media playlist: EXTINF, EXT-X-TARGETDURATION, EXT-X-MEDIA-SEQUENCE, + * EXT-X-DISCONTINUITY, EXT-X-ENDLIST, EXT-X-MAP + * - Multivariant playlist: EXT-X-STREAM-INF (BANDWIDTH / CODECS) + * + * EXT-X-PLAYLIST-TYPE is ignored: any playlist without EXT-X-ENDLIST (including + * EVENT) is treated as live and keeps refreshing. + * + * Explicitly unsupported: LL-HLS, EXT-X-MEDIA renditions, encryption, byteranges. + */ + +export interface HlsPlaylistSegment { + url: string; + duration: number; + mediaSequence: number; + discontinuity: boolean; + initUrl?: string; +} + +export interface HlsMediaPlaylist { + kind: "media"; + /** true when the playlist has no EXT-X-ENDLIST (will keep refreshing). */ + live: boolean; + targetDuration: number; + mediaSequence: number; + segments: HlsPlaylistSegment[]; + totalDuration: number; +} + +export interface HlsVariant { + url: string; + bandwidth: number; + codecs?: string; +} + +export interface HlsMultivariantPlaylist { + kind: "multivariant"; + variants: HlsVariant[]; +} + +export type HlsPlaylist = HlsMediaPlaylist | HlsMultivariantPlaylist; + +/** Parse attribute list like `BANDWIDTH=1280000,CODECS="avc1.4d401f,mp4a.40.2"`. */ +function parseAttributes(input: string): Record { + const attrs: Record = {}; + const re = /([A-Z0-9-]+)=("[^"]*"|[^,]*)/g; + let match = re.exec(input); + while (match !== null) { + let value = match[2]; + if (value.startsWith('"') && value.endsWith('"')) { + value = value.slice(1, -1); + } + attrs[match[1]] = value; + match = re.exec(input); + } + return attrs; +} + +export function parseM3U8(text: string, baseUrl: string): HlsPlaylist { + const lines = text.split(/\r?\n/).map((l) => l.trim()); + if (!lines.some((l) => l.startsWith("#EXTM3U"))) { + throw new Error("Not a valid m3u8 playlist"); + } + + if (lines.some((l) => l.startsWith("#EXT-X-STREAM-INF:"))) { + return parseMultivariant(lines, baseUrl); + } + return parseMedia(lines, baseUrl); +} + +function parseMultivariant(lines: string[], baseUrl: string): HlsMultivariantPlaylist { + const variants: HlsVariant[] = []; + let pending: { bandwidth: number; codecs?: string } | null = null; + + for (const line of lines) { + if (line.startsWith("#EXT-X-STREAM-INF:")) { + const attrs = parseAttributes(line.slice("#EXT-X-STREAM-INF:".length)); + pending = { + bandwidth: Number.parseInt(attrs.BANDWIDTH ?? "0", 10) || 0, + codecs: attrs.CODECS, + }; + } else if (pending && line.length > 0 && !line.startsWith("#")) { + variants.push({ url: new URL(line, baseUrl).href, ...pending }); + pending = null; + } + } + + return { kind: "multivariant", variants }; +} + +function parseMedia(lines: string[], baseUrl: string): HlsMediaPlaylist { + const segments: HlsPlaylistSegment[] = []; + let targetDuration = 0; + let mediaSequence = 0; + let ended = false; + let pendingDuration: number | null = null; + let pendingDiscontinuity = false; + let currentInitUrl: string | undefined; + let totalDuration = 0; + + for (const line of lines) { + if (line.startsWith("#EXTINF:")) { + pendingDuration = Number.parseFloat(line.slice("#EXTINF:".length)) || 0; + } else if (line.startsWith("#EXT-X-TARGETDURATION:")) { + targetDuration = Number.parseFloat(line.slice("#EXT-X-TARGETDURATION:".length)) || 0; + } else if (line.startsWith("#EXT-X-MEDIA-SEQUENCE:")) { + mediaSequence = Number.parseInt(line.slice("#EXT-X-MEDIA-SEQUENCE:".length), 10) || 0; + } else if (line.startsWith("#EXT-X-DISCONTINUITY")) { + // also matches EXT-X-DISCONTINUITY-SEQUENCE; harmless for our use + if (line === "#EXT-X-DISCONTINUITY") { + pendingDiscontinuity = true; + } + } else if (line.startsWith("#EXT-X-MAP:")) { + const attrs = parseAttributes(line.slice("#EXT-X-MAP:".length)); + if (attrs.URI) { + currentInitUrl = new URL(attrs.URI, baseUrl).href; + } + } else if (line.startsWith("#EXT-X-ENDLIST")) { + ended = true; + } else if (line.length > 0 && !line.startsWith("#") && pendingDuration !== null) { + segments.push({ + url: new URL(line, baseUrl).href, + duration: pendingDuration, + mediaSequence: mediaSequence + segments.length, + discontinuity: pendingDiscontinuity, + initUrl: currentInitUrl, + }); + totalDuration += pendingDuration; + pendingDuration = null; + pendingDiscontinuity = false; + } + } + + return { + kind: "media", + live: !ended, + targetDuration, + mediaSequence, + segments, + totalDuration, + }; +} diff --git a/web-ui/src/mpegts/index.ts b/web-ui/src/mpegts/index.ts index f253cba3..43209dbc 100644 --- a/web-ui/src/mpegts/index.ts +++ b/web-ui/src/mpegts/index.ts @@ -1,5 +1,4 @@ import { defaultConfig, type PlayerConfig } from "./config"; -import { createHlsPlayer } from "./player/hls-player"; import { createMpegtsPlayer } from "./player/mpegts-player"; import type { Player, PlayerError, PlayerEventMap, PlayerImpl, PlayerSegment } from "./types"; @@ -22,83 +21,43 @@ export function createPlayer(video: HTMLVideoElement, config?: Partial void>(); const audioSuspendedHandlers = new Set<() => void>(); - // Cached impls — created on demand, kept alive across type switches - const cache: Record = {}; - let activeType: string | null = null; - let lastSegments: PlayerSegment[] = []; - - function setHandlers(impl: PlayerImpl): void { - impl.onError = (e) => { - for (const h of errorHandlers) { - h(e); - } - }; - impl.onAudioSuspended = () => { - for (const h of audioSuspendedHandlers) { - h(); - } - }; - } - - function getOrCreateImpl(type: "mpegts" | "hls"): PlayerImpl { - if (!cache[type]) { - const impl = - type === "hls" - ? createHlsPlayer(video, fullConfig, seekHandlers) - : createMpegtsPlayer(video, fullConfig, seekHandlers); - setHandlers(impl); - cache[type] = impl; - } - return cache[type]; - } - - function switchTo(type: "mpegts" | "hls"): PlayerImpl { - if (activeType === type && cache[type]) { - return cache[type]; - } - - // Suspend current active impl (release video element, keep resources) - if (activeType && cache[activeType]) { - cache[activeType].suspend(); + let impl: PlayerImpl | null = null; + + function getImpl(): PlayerImpl { + if (!impl) { + impl = createMpegtsPlayer(video, fullConfig, seekHandlers); + impl.onError = (e) => { + for (const h of errorHandlers) { + h(e); + } + }; + impl.onAudioSuspended = () => { + for (const h of audioSuspendedHandlers) { + h(); + } + }; } - - activeType = type; - return getOrCreateImpl(type); - } - - function setupHLSDetection(impl: PlayerImpl): void { - impl.onHLSDetected = () => { - if (destroyed || !lastSegments.length) return; - const hlsImpl = switchTo("hls"); - hlsImpl.loadSegments(lastSegments); - }; + return impl; } return { loadSegments(segments: PlayerSegment[]) { if (destroyed || !segments.length) return; - lastSegments = segments; - const impl = switchTo("mpegts"); - setupHLSDetection(impl); - impl.loadSegments(segments); + getImpl().loadSegments(segments); }, seek(seconds: number) { - if (activeType) cache[activeType]?.seek(seconds); + impl?.seek(seconds); }, setLiveSync(enabled: boolean) { - for (const impl of Object.values(cache)) { - impl.setLiveSync(enabled); - } + impl?.setLiveSync(enabled); }, destroy() { destroyed = true; - for (const impl of Object.values(cache)) { - impl.destroy(); - } - activeType = null; + impl?.destroy(); + impl = null; }, on(event: K, handler: PlayerEventMap[K]) { diff --git a/web-ui/src/mpegts/io/fetch-loader.ts b/web-ui/src/mpegts/io/fetch-loader.ts index 2ad08f46..c3dc275a 100644 --- a/web-ui/src/mpegts/io/fetch-loader.ts +++ b/web-ui/src/mpegts/io/fetch-loader.ts @@ -1,5 +1,4 @@ import type { PlayerConfig } from "../config"; -import Browser from "../utils/browser"; import { IllegalStateException, RuntimeException } from "../utils/exception"; import Log from "../utils/logger"; @@ -58,7 +57,8 @@ class FetchLoader { onSeeked: (() => void) | null; onError: ((type: string, info: LoaderErrorInfo) => void) | null; onComplete: ((extraData: unknown) => void) | null; - onHLSDetected: (() => void) | null; + /** Called with the playlist text and its final (post-redirect) URL when the response is an HLS playlist. */ + onHLSDetected: ((text: string, url: string) => void) | null; // --- config / data source --- private _config: PlayerConfig; @@ -284,10 +284,13 @@ class FetchLoader { // detect HLS content-type before processing body const ct = res.headers.get("Content-Type")?.toLowerCase() ?? ""; if (ct.includes("mpegurl") || ct.includes("m3u")) { - res.body?.cancel(); this._status = LoaderStatus.kIdle; - this.onHLSDetected?.(); - return; + // Read the body so the already-fetched playlist can be reused (avoids a duplicate request) + return res.text().then((text) => { + if (!this._requestAbort) { + this.onHLSDetected?.(text, res.url || sourceURL); + } + }); } // content-length @@ -369,11 +372,6 @@ class FetchLoader { const errCode = typeof err.code === "number" ? err.code : -1; const errMsg = typeof err.message === "string" ? err.message : ""; - if (errCode === 11 && Browser.msedge) { - // InvalidStateError on Microsoft Edge – ignore - return; - } - this._status = LoaderStatus.kError; let type: string; let info: LoaderErrorInfo; @@ -396,13 +394,11 @@ class FetchLoader { private _abortFetch(): void { this._requestAbort = true; - if (this._status !== LoaderStatus.kBuffering || !Browser.chrome) { - if (this._abortController) { - try { - this._abortController.abort(); - } catch (_e) { - /* swallow */ - } + if (this._abortController) { + try { + this._abortController.abort(); + } catch (_e) { + /* swallow */ } } } diff --git a/web-ui/src/mpegts/player/hls-player.ts b/web-ui/src/mpegts/player/hls-player.ts deleted file mode 100644 index 0b81bd66..00000000 --- a/web-ui/src/mpegts/player/hls-player.ts +++ /dev/null @@ -1,94 +0,0 @@ -import type { PlayerConfig } from "../config"; -import type { PlayerImpl, PlayerSegment } from "../types"; -import { isBuffered } from "./mpegts-player"; - -export function createHlsPlayer( - video: HTMLVideoElement, - _config: PlayerConfig, - seekHandlers: Set<(s: number) => void>, -): PlayerImpl { - let segments: PlayerSegment[] = []; - let currentIndex = 0; - let listenersBound = false; - - function onVideoError(): void { - impl.onError?.({ category: "media", detail: video.error?.message ?? "Unknown HLS error" }); - } - - function onVideoEnded(): void { - const nextIndex = currentIndex + 1; - if (nextIndex < segments.length) { - currentIndex = nextIndex; - video.src = segments[nextIndex].url; - video.play(); - } - } - - function bindListeners(): void { - if (listenersBound) return; - video.addEventListener("error", onVideoError); - video.addEventListener("ended", onVideoEnded); - listenersBound = true; - } - - function unbindListeners(): void { - if (!listenersBound) return; - video.removeEventListener("error", onVideoError); - video.removeEventListener("ended", onVideoEnded); - listenersBound = false; - } - - const impl: PlayerImpl = { - onError: null, - - loadSegments(newSegments: PlayerSegment[]) { - segments = newSegments; - currentIndex = 0; - bindListeners(); - const wasPlaying = !video.paused; - video.src = segments[0].url; - if (wasPlaying) { - video.play(); - } - }, - - setLiveSync(_enabled: boolean) { - // HLS live sync is handled natively by the browser - }, - - seek(seconds: number) { - if (isBuffered(video, seconds) || isHLSSeekable(video, seconds)) { - video.currentTime = seconds; - } else { - for (const h of seekHandlers) { - h(seconds); - } - } - }, - - suspend() { - segments = []; - currentIndex = 0; - video.removeAttribute("src"); - video.load(); - }, - - destroy() { - impl.suspend(); - unbindListeners(); - }, - }; - - return impl; -} - -function isHLSSeekable(video: HTMLVideoElement, seconds: number): boolean { - const seekable = video.seekable; - if (!seekable) return false; - for (let i = 0; i < seekable.length; i++) { - if (seconds >= seekable.start(i) && seconds <= seekable.end(i)) { - return true; - } - } - return false; -} diff --git a/web-ui/src/mpegts/player/live-sync.ts b/web-ui/src/mpegts/player/live-sync.ts index 528f0894..55d48af6 100644 --- a/web-ui/src/mpegts/player/live-sync.ts +++ b/web-ui/src/mpegts/player/live-sync.ts @@ -3,6 +3,11 @@ import Log from "../utils/logger"; const TAG = "LiveSync"; +/** Each live-edge underrun raises the latency floor by this much (seconds). */ +const UNDERRUN_BACKOFF_STEP = 1; +/** Upper bound for the adaptive latency increase (seconds). */ +const UNDERRUN_BACKOFF_MAX = 6; + /** Sets up live latency synchronization by adjusting playbackRate on timeupdate events. */ export function setupLiveSync(video: HTMLMediaElement, config: PlayerConfig): () => void { if (config.liveSync) { @@ -15,6 +20,13 @@ export function setupLiveSync(video: HTMLMediaElement, config: PlayerConfig): () ); } + // Adaptive backoff: every genuine live-edge underrun raises the latency we + // are willing to tolerate, so devices with bursty data delivery (e.g. iOS + // Safari, where fetch chunks arrive in 1-2s batches) settle at a latency + // they can sustain instead of rebuffering in a loop. Devices that never + // underrun keep the configured low latency. + let extraLatency = 0; + function onTimeUpdate(): void { if (!config.liveSync) return; @@ -24,13 +36,13 @@ export function setupLiveSync(video: HTMLMediaElement, config: PlayerConfig): () const bufferedEnd = buffered.end(buffered.length - 1); const latency = bufferedEnd - video.currentTime; - if (latency > config.liveSyncMaxLatency) { + if (latency > config.liveSyncMaxLatency + extraLatency) { const targetRate = Math.min(2, Math.max(1, config.liveSyncPlaybackRate)); if (targetRate !== video.playbackRate) { Log.v(TAG, `Video playback rate set to ${targetRate}`); - video.playbackRate = Math.min(2, Math.max(1, config.liveSyncPlaybackRate)); + video.playbackRate = targetRate; } - } else if (latency <= config.liveSyncTargetLatency) { + } else if (latency <= config.liveSyncTargetLatency + extraLatency) { if (video.playbackRate !== 1 && video.playbackRate !== 0) { video.playbackRate = 1; Log.v(TAG, "Video playback rate reset to 1"); @@ -39,11 +51,38 @@ export function setupLiveSync(video: HTMLMediaElement, config: PlayerConfig): () // else: between target and max, keep current playbackRate } + function onWaiting(): void { + if (!config.liveSync) return; + + // Only count genuine live-edge underruns (playback caught up with the end + // of the buffer), not startup waits or seeks into unbuffered regions. + const buffered = video.buffered; + const atLiveEdge = buffered.length > 0 && buffered.end(buffered.length - 1) - video.currentTime < 0.5; + if (!atLiveEdge) return; + + // Reset any boost immediately: timeupdate stops firing during the stall, so + // the regular latency check cannot run, and staying boosted would starve + // playback again as soon as it resumes. + if (video.playbackRate !== 1 && video.playbackRate !== 0) { + video.playbackRate = 1; + } + + if (extraLatency < UNDERRUN_BACKOFF_MAX) { + extraLatency = Math.min(extraLatency + UNDERRUN_BACKOFF_STEP, UNDERRUN_BACKOFF_MAX); + } + Log.w( + TAG, + `Live-edge underrun, raising latency tolerance: target ${(config.liveSyncTargetLatency + extraLatency).toFixed(1)}s, max ${(config.liveSyncMaxLatency + extraLatency).toFixed(1)}s`, + ); + } + video.addEventListener("timeupdate", onTimeUpdate); + video.addEventListener("waiting", onWaiting); return () => { Log.v(TAG, "Video playback rate reset to 1, live sync disabled"); video.removeEventListener("timeupdate", onTimeUpdate); + video.removeEventListener("waiting", onWaiting); video.playbackRate = 1; }; } @@ -53,7 +92,13 @@ export function setupLiveSync(video: HTMLMediaElement, config: PlayerConfig): () * If the video is stalled or hasn't received canplay and the currentTime is before * the first buffered range, seek to the start of the buffered range. */ -export function setupStartupStallJumper(video: HTMLMediaElement): () => void { +export interface StallJumper { + /** Re-run stall detection. Call whenever buffered ranges change (e.g. after a SourceBuffer append). */ + check(): void; + destroy(): void; +} + +export function setupStartupStallJumper(video: HTMLMediaElement): StallJumper { let canplayReceived = false; function onCanPlay(): void { @@ -68,10 +113,7 @@ export function setupStartupStallJumper(video: HTMLMediaElement): () => void { const target = buffered.start(0); Log.w(TAG, `Playback stuck at ${video.currentTime}, seeking to ${target}`); video.currentTime = target; - video.removeEventListener("progress", onProgress); } - } else { - video.removeEventListener("progress", onProgress); } } @@ -79,17 +121,14 @@ export function setupStartupStallJumper(video: HTMLMediaElement): () => void { detectAndFix(true); } - function onProgress(): void { - detectAndFix(); - } - video.addEventListener("canplay", onCanPlay); video.addEventListener("stalled", onStalled); - video.addEventListener("progress", onProgress); - return () => { - video.removeEventListener("canplay", onCanPlay); - video.removeEventListener("stalled", onStalled); - video.removeEventListener("progress", onProgress); + return { + check: () => detectAndFix(), + destroy: () => { + video.removeEventListener("canplay", onCanPlay); + video.removeEventListener("stalled", onStalled); + }, }; } diff --git a/web-ui/src/mpegts/player/mpegts-player.ts b/web-ui/src/mpegts/player/mpegts-player.ts index 9d48dbb2..7534c096 100644 --- a/web-ui/src/mpegts/player/mpegts-player.ts +++ b/web-ui/src/mpegts/player/mpegts-player.ts @@ -1,11 +1,37 @@ import { PCMAudioPlayer } from "../audio/pcm-audio-player"; import type { PlayerConfig } from "../config"; import type { PlayerImpl, PlayerSegment } from "../types"; +import Log from "../utils/logger"; import type { WorkerCommand, WorkerEvent } from "../worker/messages"; import TransmuxWorker from "../worker/transmux-worker.ts?worker&inline"; -import { setupLiveSync, setupStartupStallJumper } from "./live-sync"; +import { type StallJumper, setupLiveSync, setupStartupStallJumper } from "./live-sync"; import { createMSE, type MSE } from "./mse"; +const TAG = "Player"; + +/** Attach verbose listeners to media element events for diagnosing playback stalls. */ +function setupVideoDebugLogs(video: HTMLVideoElement): () => void { + const events = ["loadedmetadata", "canplay", "playing", "waiting", "stalled", "pause", "seeking", "seeked", "error"]; + const handler = (e: Event) => { + const buffered: string[] = []; + for (let i = 0; i < video.buffered.length; i++) { + buffered.push(`${video.buffered.start(i).toFixed(2)}-${video.buffered.end(i).toFixed(2)}`); + } + Log.v( + TAG, + `video event '${e.type}': currentTime=${video.currentTime.toFixed(2)}, readyState=${video.readyState}, paused=${video.paused}, buffered=[${buffered.join(",")}]${e.type === "error" ? `, error=${video.error?.code}:${video.error?.message}` : ""}`, + ); + }; + for (const ev of events) { + video.addEventListener(ev, handler); + } + return () => { + for (const ev of events) { + video.removeEventListener(ev, handler); + } + }; +} + /** Check if a given time position is within any buffered range of the video element. */ export function isBuffered(video: HTMLMediaElement, seconds: number): boolean { const buffered = video.buffered; @@ -17,6 +43,10 @@ export function isBuffered(video: HTMLMediaElement, seconds: number): boolean { return false; } +/** Forward buffer watermarks for HLS VOD/EVENT: pause fetching when far ahead of playback. */ +const VOD_FORWARD_BUFFER_PAUSE = 30; +const VOD_FORWARD_BUFFER_RESUME = 15; + export function createMpegtsPlayer( video: HTMLVideoElement, config: PlayerConfig, @@ -27,10 +57,15 @@ export function createMpegtsPlayer( let workerInitialized = false; let pendingSegments: PlayerSegment[] | null = null; let destroyLiveSync: (() => void) | null = null; - let destroyStallJumper: (() => void) | null = null; + let stallJumper: StallJumper | null = null; let mseGeneration = 0; let liveSyncEnabled = config.liveSync; + // HLS playlist info reported by the worker (null when playing a non-HLS source) + let hlsInfo: { live: boolean; totalDuration: number } | null = null; + let watermarkTimer: ReturnType | null = null; + let watermarkPaused = false; + // PCM audio player for software-decoded audio (MP2) let pcmPlayer: PCMAudioPlayer | null = null; let pcmPlayerInitPromise: Promise | null = null; @@ -75,10 +110,12 @@ export function createMpegtsPlayer( case "complete": mse?.endOfStream(); break; - case "media-info": - break; - case "hls-detected": - impl.onHLSDetected?.(); + case "hls-info": + hlsInfo = { live: msg.live, totalDuration: msg.totalDuration }; + if (!msg.live) { + mse?.setDuration(msg.totalDuration); + startWatermarkThrottle(); + } break; case "pcm-audio-data": { const player = ensurePCMPlayer(); @@ -100,6 +137,38 @@ export function createMpegtsPlayer( return worker; } + /** Throttle fetching for HLS VOD/EVENT: pause the worker when buffered far ahead of playback. */ + function startWatermarkThrottle(): void { + if (watermarkTimer) return; + watermarkTimer = setInterval(() => { + const buffered = video.buffered; + // Measure forward buffer within the range containing currentTime; after a seek + // to an unbuffered position, stale ranges further ahead must not count. + let ahead = 0; + for (let i = 0; i < buffered.length; i++) { + if (video.currentTime >= buffered.start(i) && video.currentTime <= buffered.end(i)) { + ahead = buffered.end(i) - video.currentTime; + break; + } + } + if (!watermarkPaused && ahead > VOD_FORWARD_BUFFER_PAUSE) { + watermarkPaused = true; + worker?.postMessage({ type: "pause" } satisfies WorkerCommand); + } else if (watermarkPaused && ahead < VOD_FORWARD_BUFFER_RESUME) { + watermarkPaused = false; + worker?.postMessage({ type: "resume" } satisfies WorkerCommand); + } + }, 1000); + } + + function stopWatermarkThrottle(): void { + if (watermarkTimer) { + clearInterval(watermarkTimer); + watermarkTimer = null; + } + watermarkPaused = false; + } + function loadInWorker(segments: PlayerSegment[]): void { const w = ensureWorker(); if (!workerInitialized) { @@ -130,6 +199,45 @@ export function createMpegtsPlayer( worker?.postMessage(cmd); }; + mse.onBufferAvailable = () => { + // Don't resume while the VOD watermark throttle is intentionally holding the worker + if (watermarkPaused) return; + const cmd: WorkerCommand = { type: "resume" }; + worker?.postMessage(cmd); + }; + + mse.onStartStreaming = () => { + if (watermarkPaused) return; + const cmd: WorkerCommand = { type: "resume" }; + worker?.postMessage(cmd); + }; + + // Note: onEndStreaming intentionally does NOT pause the worker. For continuous + // live TS streams, pausing aborts the in-flight fetch and resumes via a Range + // request, which restarts a live stream mid-flow and corrupts the timeline. + // The MSE layer already defers appends while ManagedMediaSource streaming=false. + + // Buffered ranges change exactly on SourceBuffer updateend; re-check for startup + // stalls there (iOS does not reliably fire progress/stalled on the media element). + mse.onBufferedChange = () => stallJumper?.check(); + + mse.onSourceClose = () => { + // The UA killed the media pipeline (e.g. iOS reclaiming resources in + // background). Stop fetching — this session cannot be revived. + const cmd: WorkerCommand = { type: "pause" }; + worker?.postMessage(cmd); + if (document.visibilityState === "visible") { + // Unexpected closure while visible: surface as an error so the app retries + impl.onError?.({ + category: "media", + detail: "MediaSourceClosed", + info: "MediaSource was closed unexpectedly", + }); + } + // When hidden, stay quiet: retrying in background would fail repeatedly and + // exhaust the app's retry budget. The app reloads the stream on foreground. + }; + mse.onError = (info) => { impl.onError?.({ category: "media", @@ -139,12 +247,16 @@ export function createMpegtsPlayer( }; } + let destroyVideoDebugLogs: (() => void) | null = null; + function initLiveHelpers(): void { if (!destroyLiveSync && liveSyncEnabled) { destroyLiveSync = setupLiveSync(video, config); } - destroyStallJumper?.(); - destroyStallJumper = setupStartupStallJumper(video); + stallJumper?.destroy(); + stallJumper = setupStartupStallJumper(video); + destroyVideoDebugLogs?.(); + destroyVideoDebugLogs = setupVideoDebugLogs(video); } const impl: PlayerImpl = { @@ -152,6 +264,8 @@ export function createMpegtsPlayer( loadSegments(segments: PlayerSegment[]) { mseGeneration++; + hlsInfo = null; + stopWatermarkThrottle(); if (mse) { mse.destroy(); mse = null; @@ -176,6 +290,17 @@ export function createMpegtsPlayer( seek(seconds: number) { if (isBuffered(video, seconds)) { video.currentTime = seconds; + } else if (hlsInfo && !hlsInfo.live) { + // HLS VOD/EVENT: reposition inside the playlist (worker reschedules segments) + const cmd: WorkerCommand = { type: "seek", seconds }; + worker?.postMessage(cmd); + // The watermark throttle may be holding the worker paused; the seek target + // needs data now, so resume immediately (the throttle re-pauses if needed) + if (watermarkPaused) { + watermarkPaused = false; + worker?.postMessage({ type: "resume" } satisfies WorkerCommand); + } + video.currentTime = seconds; } else { for (const h of seekHandlers) { h(seconds); @@ -184,6 +309,8 @@ export function createMpegtsPlayer( }, suspend() { + stopWatermarkThrottle(); + hlsInfo = null; if (mse) { mse.destroy(); mse = null; @@ -191,8 +318,10 @@ export function createMpegtsPlayer( destroyPCMPlayer(); destroyLiveSync?.(); destroyLiveSync = null; - destroyStallJumper?.(); - destroyStallJumper = null; + stallJumper?.destroy(); + stallJumper = null; + destroyVideoDebugLogs?.(); + destroyVideoDebugLogs = null; }, destroy() { diff --git a/web-ui/src/mpegts/player/mse.ts b/web-ui/src/mpegts/player/mse.ts index 0d7cef8e..194f86f2 100644 --- a/web-ui/src/mpegts/player/mse.ts +++ b/web-ui/src/mpegts/player/mse.ts @@ -1,5 +1,4 @@ import type { PlayerConfig } from "../config"; -import Browser from "../utils/browser"; import Log from "../utils/logger"; type Track = "video" | "audio"; @@ -31,9 +30,21 @@ export interface MSE { open(onOpen: () => void): void; appendInit(track: Track, data: ArrayBuffer, codec: string, container: string): void; appendMedia(track: Track, data: ArrayBuffer): void; + /** Set the MediaSource duration (e.g. from an HLS VOD playlist) so seeks beyond buffered data are not clamped. */ + setDuration(seconds: number): void; endOfStream(): void; destroy(): void; onBufferFull: (() => void) | null; + /** Fired when buffer space becomes available again after a previous onBufferFull. */ + onBufferAvailable: (() => void) | null; + /** Fired after each SourceBuffer update completes (buffered ranges may have changed). */ + onBufferedChange: (() => void) | null; + /** ManagedMediaSource: UA wants more media data appended (streaming → true). */ + onStartStreaming: (() => void) | null; + /** ManagedMediaSource: UA has enough buffered data (streaming → false). */ + onEndStreaming: (() => void) | null; + /** Fired when the MediaSource is closed by the UA (not by destroy), e.g. iOS reclaiming the media pipeline in background. */ + onSourceClose: (() => void) | null; onError: ((info: { code: number; msg: string }) => void) | null; } @@ -46,6 +57,7 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { let mediaSource: MSEMediaSource | null = null; let objectURL: string | null = null; + let destroying = false; const sourceBuffers: Record = { video: null, audio: null }; const pendingSegments: Record = { video: [], audio: [] }; @@ -55,6 +67,7 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { let isBufferFull = false; let hasPendingEos = false; + let pendingDuration: number | null = null; // Deferred init segments: queued before sourceopen fires let pendingSourceBufferInit: { track: Track; data: ArrayBuffer; codec: string; container: string }[] = []; @@ -98,6 +111,63 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { } } + let appendGateLogged = false; + + function canAppendToManagedSource(): boolean { + // ManagedMediaSource only accepts appends while streaming === true. + // When the property is absent (regular MediaSource), always allow appends. + const allowed = !useManagedMediaSource || mediaSource?.streaming !== false; + if (!allowed && !appendGateLogged) { + appendGateLogged = true; + Log.v( + TAG, + `Appends deferred: MMS streaming=false (pending video:${pendingSegments.video.length} audio:${pendingSegments.audio.length})`, + ); + } + return allowed; + } + + function bufferedSummary(): string { + const fmt = (sb: SourceBuffer | null): string => { + if (!sb) return "-"; + const parts: string[] = []; + for (let i = 0; i < sb.buffered.length; i++) { + parts.push(`${sb.buffered.start(i).toFixed(2)}-${sb.buffered.end(i).toFixed(2)}`); + } + return parts.join(",") || "empty"; + }; + return `video[${fmt(sourceBuffers.video)}] audio[${fmt(sourceBuffers.audio)}]`; + } + + function flushPendingSourceBufferInit(): void { + if (pendingSourceBufferInit.length === 0 || mediaSource?.readyState !== "open") { + return; + } + const pendings = pendingSourceBufferInit; + pendingSourceBufferInit = []; + for (const pending of pendings) { + createSourceBuffer(pending.track, pending.codec, pending.container); + lastInitSegments[pending.track] = { + data: pending.data, + codec: pending.codec, + container: pending.container, + }; + } + } + + function tryAppendPending(): void { + if (mediaSource?.readyState !== "open" || !canAppendToManagedSource()) { + return; + } + if (hasPendingRemoveRanges()) { + doRemoveRanges(); + return; + } + if (hasPendingSegments()) { + doAppendSegments(); + } + } + function doAppendSegments(): void { const tracks: Track[] = ["video", "audio"]; for (const track of tracks) { @@ -105,9 +175,8 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { if (!sb || sb.updating) { continue; } - // If ManagedMediaSource and streaming is false, do not append - if (mediaSource?.streaming === false) { - continue; + if (!canAppendToManagedSource()) { + return; } if (pendingSegments[track].length > 0) { @@ -120,7 +189,10 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { try { sb.appendBuffer(segment); - isBufferFull = false; + if (isBufferFull) { + isBufferFull = false; + mse.onBufferAvailable?.(); + } } catch (error: unknown) { pendingSegments[track].unshift(segment); if ((error as DOMException).code === 22) { @@ -191,13 +263,36 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { } } + function tryApplyDuration(): void { + if (pendingDuration === null || mediaSource?.readyState !== "open") { + return; + } + if (sourceBuffers.video?.updating || sourceBuffers.audio?.updating) { + return; // retried on the next updateend + } + try { + if (!(mediaSource.duration >= pendingDuration)) { + mediaSource.duration = pendingDuration; + } + pendingDuration = null; + } catch (error: unknown) { + Log.w(TAG, `Failed to set duration: ${(error as Error).message}`); + } + } + function onSourceBufferUpdateEnd(): void { + mse.onBufferedChange?.(); + tryApplyDuration(); if (hasPendingRemoveRanges()) { doRemoveRanges(); } else if (hasPendingSegments()) { - doAppendSegments(); + tryAppendPending(); } else if (hasPendingEos) { mse.endOfStream(); + } else if (isBufferFull) { + // All queued segments drained and removals finished — buffer has room again + isBufferFull = false; + mse.onBufferAvailable?.(); } } @@ -210,14 +305,10 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { return; } - let adjustedCodec = codec; - if (adjustedCodec === "opus" && Browser.safari) { - adjustedCodec = "Opus"; - } - let mimeType = container; - if (adjustedCodec && adjustedCodec.length > 0) { - mimeType += `;codecs=${adjustedCodec}`; + if (codec && codec.length > 0) { + // Quoting is required when the codec list contains commas (muxed fMP4 renditions) + mimeType += `;codecs="${codec}"`; } if (mimeType !== mimeTypes[track]) { @@ -250,6 +341,11 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { const mse: MSE = { onBufferFull: null, + onBufferAvailable: null, + onBufferedChange: null, + onStartStreaming: null, + onEndStreaming: null, + onSourceClose: null, onError: null, open(onOpen: () => void): void { @@ -274,24 +370,8 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { Log.v(TAG, "MediaSource onSourceOpen"); ms.removeEventListener("sourceopen", onSourceOpenHandler); - // Process deferred init segments - if (pendingSourceBufferInit.length > 0) { - const pendings = pendingSourceBufferInit; - pendingSourceBufferInit = []; - for (const pending of pendings) { - createSourceBuffer(pending.track, pending.codec, pending.container); - lastInitSegments[pending.track] = { - data: pending.data, - codec: pending.codec, - container: pending.container, - }; - } - } - - // There may be pending media segments; append them - if (hasPendingSegments()) { - doAppendSegments(); - } + flushPendingSourceBufferInit(); + tryAppendPending(); sourceOpenCallback?.(); sourceOpenCallback = null; @@ -313,6 +393,16 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { mediaSource.removeEventListener("qualitychange", onQualityChangeHandler); } } + // The SourceBuffers are detached now; accessing them (even .buffered) + // throws InvalidStateError. Drop the refs so queued appends become no-ops. + sourceBuffers.video = null; + sourceBuffers.audio = null; + mimeTypes.video = null; + mimeTypes.audio = null; + if (!destroying) { + Log.w(TAG, "MediaSource closed unexpectedly (e.g. reclaimed by the OS in background)"); + mse.onSourceClose?.(); + } }; ms.addEventListener("sourceopen", onSourceOpenHandler); @@ -321,10 +411,18 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { if (useManagedMediaSource) { onStartStreamingHandler = () => { - Log.v(TAG, "ManagedMediaSource onStartStreaming"); + appendGateLogged = false; + Log.v( + TAG, + `ManagedMediaSource onStartStreaming, pending video:${pendingSegments.video.length} audio:${pendingSegments.audio.length}, buffered: ${bufferedSummary()}`, + ); + flushPendingSourceBufferInit(); + tryAppendPending(); + mse.onStartStreaming?.(); }; onEndStreamingHandler = () => { - Log.v(TAG, "ManagedMediaSource onEndStreaming"); + Log.v(TAG, `ManagedMediaSource onEndStreaming, buffered: ${bufferedSummary()}`); + mse.onEndStreaming?.(); }; onQualityChangeHandler = () => { Log.v(TAG, "ManagedMediaSource onQualityChange"); @@ -335,14 +433,12 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { ms.addEventListener("qualitychange", onQualityChangeHandler); } - // Attach MediaSource to video element + // Attach MediaSource to video element (blob URL for both MSE and MMS per spec examples) if (useManagedMediaSource) { - (video as unknown as Record).disableRemotePlayback = true; - (video as unknown as Record).srcObject = ms; - } else { - objectURL = URL.createObjectURL(ms as unknown as MediaSource); - video.src = objectURL; + video.disableRemotePlayback = true; } + objectURL = URL.createObjectURL(ms as unknown as MediaSource); + video.src = objectURL; }, appendInit(track: Track, data: ArrayBuffer, codec: string, container: string): void { @@ -352,39 +448,34 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { return; } - let adjustedCodec = codec; - if (adjustedCodec === "opus" && Browser.safari) { - adjustedCodec = "Opus"; - } - - const mimePreview = adjustedCodec ? `${container};codecs=${adjustedCodec}` : container; + const mimePreview = codec ? `${container};codecs="${codec}"` : container; Log.v(TAG, `Received Initialization Segment, mimeType: ${mimePreview}`); - lastInitSegments[track] = { data, codec: adjustedCodec, container }; - - const firstInit = !mimeTypes[track]; - createSourceBuffer(track, adjustedCodec, container); + lastInitSegments[track] = { data, codec, container }; + createSourceBuffer(track, codec, container); pendingSegments[track].push(data); - - if (!firstInit) { - const sb = sourceBuffers[track]; - if (sb && !sb.updating) { - doAppendSegments(); - } - } + tryAppendPending(); }, appendMedia(track: Track, data: ArrayBuffer): void { pendingSegments[track].push(data); + // After the MediaSource closes (e.g. iOS background reclaim), the + // SourceBuffers are dead — touching them throws InvalidStateError. + if (mediaSource?.readyState !== "open") { + return; + } + if (needCleanupSourceBuffer()) { doCleanupSourceBuffer(); } - const sb = sourceBuffers[track]; - if (sb && !sb.updating && !hasPendingRemoveRanges()) { - doAppendSegments(); - } + tryAppendPending(); + }, + + setDuration(seconds: number): void { + pendingDuration = seconds; + tryApplyDuration(); }, endOfStream(): void { @@ -405,6 +496,7 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { }, destroy(): void { + destroying = true; if (mediaSource) { const ms = mediaSource; const tracks: Track[] = ["video", "audio"]; @@ -463,6 +555,7 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { pendingSourceBufferInit = []; isBufferFull = false; hasPendingEos = false; + pendingDuration = null; mediaSource = null; } @@ -471,13 +564,14 @@ export function createMSE(video: HTMLVideoElement, config: PlayerConfig): MSE { objectURL = null; } - if (useManagedMediaSource) { - (video as unknown as Record).srcObject = null; - } else { - video.removeAttribute("src"); - } + video.removeAttribute("src"); mse.onBufferFull = null; + mse.onBufferAvailable = null; + mse.onBufferedChange = null; + mse.onStartStreaming = null; + mse.onEndStreaming = null; + mse.onSourceClose = null; mse.onError = null; sourceOpenCallback = null; }, diff --git a/web-ui/src/mpegts/remux/mp4-generator.ts b/web-ui/src/mpegts/remux/mp4-generator.ts index fa2c8d7b..627dd091 100644 --- a/web-ui/src/mpegts/remux/mp4-generator.ts +++ b/web-ui/src/mpegts/remux/mp4-generator.ts @@ -35,10 +35,8 @@ interface MP4Meta { duration: number; avcc?: Uint8Array; hvcc?: Uint8Array; - av1c?: Uint8Array; audioSampleRate?: number; channelCount?: number; - channelConfigCode?: number; config?: number[] | Uint8Array; originalCodec?: string; refSampleDuration?: number; @@ -46,8 +44,6 @@ interface MP4Meta { codecHeight?: number; presentWidth?: number; presentHeight?: number; - sampleSize?: number; - littleEndian?: boolean; } interface MP4Constants { @@ -89,8 +85,6 @@ class MP4 { hdlr: [], hvc1: [], hvcC: [], - av01: [], - av1C: [], mdat: [], mdhd: [], mdia: [], @@ -117,14 +111,7 @@ class MP4 { tkhd: [], vmhd: [], smhd: [], - chnl: [], ".mp3": [], - Opus: [], - dOps: [], - fLaC: [], - dfLa: [], - ipcm: [], - pcmC: [], "ac-3": [], dac3: [], "ec-3": [], @@ -679,19 +666,11 @@ class MP4 { return MP4.box(MP4.types.stsd, MP4.constants.STSD_PREFIX, MP4.ac3(meta)); } else if (meta.codec === "ec-3") { return MP4.box(MP4.types.stsd, MP4.constants.STSD_PREFIX, MP4.ec3(meta)); - } else if (meta.codec === "opus") { - return MP4.box(MP4.types.stsd, MP4.constants.STSD_PREFIX, MP4.Opus(meta)); - } else if (meta.codec === "flac") { - return MP4.box(MP4.types.stsd, MP4.constants.STSD_PREFIX, MP4.fLaC(meta)); - } else if (meta.codec === "ipcm") { - return MP4.box(MP4.types.stsd, MP4.constants.STSD_PREFIX, MP4.ipcm(meta)); } // else: aac -> mp4a return MP4.box(MP4.types.stsd, MP4.constants.STSD_PREFIX, MP4.mp4a(meta)); } else if (meta.type === "video" && meta.codec.startsWith("hvc1")) { return MP4.box(MP4.types.stsd, MP4.constants.STSD_PREFIX, MP4.hvc1(meta)); - } else if (meta.type === "video" && meta.codec.startsWith("av01")) { - return MP4.box(MP4.types.stsd, MP4.constants.STSD_PREFIX, MP4.av01(meta)); } else { return MP4.box(MP4.types.stsd, MP4.constants.STSD_PREFIX, MP4.avc1(meta)); } @@ -894,249 +873,6 @@ class MP4 { return MP4.box(MP4.types.esds, data); } - static Opus(meta: MP4Meta): Uint8Array { - const channelCount = meta.channelCount || 0; - const sampleRate = meta.audioSampleRate || 0; - - const data = new Uint8Array([ - 0x00, - 0x00, - 0x00, - 0x00, // reserved(4) - 0x00, - 0x00, - 0x00, - 0x01, // reserved(2) + data_reference_index(2) - 0x00, - 0x00, - 0x00, - 0x00, // reserved: 2 * 4 bytes - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - channelCount, // channelCount(2) - 0x00, - 0x10, // sampleSize(2) - 0x00, - 0x00, - 0x00, - 0x00, // reserved(4) - (sampleRate >>> 8) & 0xff, // Audio sample rate - sampleRate & 0xff, - 0x00, - 0x00, - ]); - - return MP4.box(MP4.types.Opus, data, MP4.dOps(meta)); - } - - static dOps(meta: MP4Meta): Uint8Array { - const channelCount = meta.channelCount || 0; - const channelConfigCode = meta.channelConfigCode || 0; - const sampleRate = meta.audioSampleRate || 0; - - if (meta.config) { - return MP4.box(MP4.types.dOps, meta.config as Uint8Array); - } - - let mapping: number[] = []; - switch (channelConfigCode) { - case 0x01: - case 0x02: - mapping = [0x0]; - break; - case 0x00: // dualmono - mapping = [0xff, 1, 1, 0, 1]; - break; - case 0x80: // dualmono - mapping = [0xff, 2, 0, 0, 1]; - break; - case 0x03: - mapping = [0x01, 2, 1, 0, 2, 1]; - break; - case 0x04: - mapping = [0x01, 2, 2, 0, 1, 2, 3]; - break; - case 0x05: - mapping = [0x01, 3, 2, 0, 4, 1, 2, 3]; - break; - case 0x06: - mapping = [0x01, 4, 2, 0, 4, 1, 2, 3, 5]; - break; - case 0x07: - mapping = [0x01, 4, 2, 0, 4, 1, 2, 3, 5, 6]; - break; - case 0x08: - mapping = [0x01, 5, 3, 0, 6, 1, 2, 3, 4, 5, 7]; - break; - case 0x82: - mapping = [0x01, 1, 2, 0, 1]; - break; - case 0x83: - mapping = [0x01, 1, 3, 0, 1, 2]; - break; - case 0x84: - mapping = [0x01, 1, 4, 0, 1, 2, 3]; - break; - case 0x85: - mapping = [0x01, 1, 5, 0, 1, 2, 3, 4]; - break; - case 0x86: - mapping = [0x01, 1, 6, 0, 1, 2, 3, 4, 5]; - break; - case 0x87: - mapping = [0x01, 1, 7, 0, 1, 2, 3, 4, 5, 6]; - break; - case 0x88: - mapping = [0x01, 1, 8, 0, 1, 2, 3, 4, 5, 6, 7]; - break; - } - - const data = new Uint8Array([ - 0x00, // Version (1) - channelCount, // OutputChannelCount: 2 - 0x00, - 0x00, // PreSkip: 2 - (sampleRate >>> 24) & 0xff, // Audio sample rate: 4 - (sampleRate >>> 17) & 0xff, - (sampleRate >>> 8) & 0xff, - (sampleRate >>> 0) & 0xff, - 0x00, - 0x00, // Global Gain : 2 - ...mapping, - ]); - return MP4.box(MP4.types.dOps, data); - } - - static fLaC(meta: MP4Meta): Uint8Array { - const channelCount = meta.channelCount || 0; - const sampleRate = Math.min(meta.audioSampleRate || 0, 65535); - const sampleSize = meta.sampleSize || 0; - - const data = new Uint8Array([ - 0x00, - 0x00, - 0x00, - 0x00, // reserved(4) - 0x00, - 0x00, - 0x00, - 0x01, // reserved(2) + data_reference_index(2) - 0x00, - 0x00, - 0x00, - 0x00, // reserved: 2 * 4 bytes - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - channelCount, // channelCount(2) - 0x00, - sampleSize, // sampleSize(2) - 0x00, - 0x00, - 0x00, - 0x00, // reserved(4) - (sampleRate >>> 8) & 0xff, // Audio sample rate - sampleRate & 0xff, - 0x00, - 0x00, - ]); - - return MP4.box(MP4.types.fLaC, data, MP4.dfLa(meta)); - } - - static dfLa(meta: MP4Meta): Uint8Array { - const data = new Uint8Array([ - 0x00, - 0x00, - 0x00, - 0x00, // version, flag - ...(meta.config as number[]), - ]); - return MP4.box(MP4.types.dfLa, data); - } - - static ipcm(meta: MP4Meta): Uint8Array { - const channelCount = meta.channelCount || 0; - const sampleRate = Math.min(meta.audioSampleRate || 0, 65535); - const sampleSize = meta.sampleSize || 0; - - const data = new Uint8Array([ - 0x00, - 0x00, - 0x00, - 0x00, // reserved(4) - 0x00, - 0x00, - 0x00, - 0x01, // reserved(2) + data_reference_index(2) - 0x00, - 0x00, - 0x00, - 0x00, // reserved: 2 * 4 bytes - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - channelCount, // channelCount(2) - 0x00, - sampleSize, // sampleSize(2) - 0x00, - 0x00, - 0x00, - 0x00, // reserved(4) - (sampleRate >>> 8) & 0xff, // Audio sample rate - sampleRate & 0xff, - 0x00, - 0x00, - ]); - - if (meta.channelCount === 1) { - return MP4.box(MP4.types.ipcm, data, MP4.pcmC(meta)); - } else { - return MP4.box(MP4.types.ipcm, data, MP4.chnl(meta), MP4.pcmC(meta)); - } - } - - static chnl(meta: MP4Meta): Uint8Array { - const data = new Uint8Array([ - 0x00, - 0x00, - 0x00, - 0x00, // version, flag - 0x01, // Channel Based Layout - meta.channelCount || 0, // AudioConfiguration - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, // omittedChannelsMap - ]); - return MP4.box(MP4.types.chnl, data); - } - - static pcmC(meta: MP4Meta): Uint8Array { - const littleEndian = meta.littleEndian ? 0x01 : 0x00; - const sampleSize = meta.sampleSize || 0; - const data = new Uint8Array([ - 0x00, - 0x00, - 0x00, - 0x00, // version, flag - littleEndian, - sampleSize, - ]); - return MP4.box(MP4.types.pcmC, data); - } - static avc1(meta: MP4Meta): Uint8Array { if (meta.avcc == null) { throw new Error("MP4: avcc is required for avc1 box"); @@ -1319,97 +1055,6 @@ class MP4 { return MP4.box(MP4.types.hvc1, data, MP4.box(MP4.types.hvcC, hvcc)); } - static av01(meta: MP4Meta): Uint8Array { - if (meta.av1c == null) { - throw new Error("MP4: av1c is required for av01 box"); - } - const av1c = meta.av1c; - const width = meta.codecWidth || 192, - height = meta.codecHeight || 108; - - const data = new Uint8Array([ - 0x00, - 0x00, - 0x00, - 0x00, // reserved(4) - 0x00, - 0x00, - 0x00, - 0x01, // reserved(2) + data_reference_index(2) - 0x00, - 0x00, - 0x00, - 0x00, // pre_defined(2) + reserved(2) - 0x00, - 0x00, - 0x00, - 0x00, // pre_defined: 3 * 4 bytes - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - (width >>> 8) & 0xff, // width: 2 bytes - width & 0xff, - (height >>> 8) & 0xff, // height: 2 bytes - height & 0xff, - 0x00, - 0x48, - 0x00, - 0x00, // horizresolution: 4 bytes - 0x00, - 0x48, - 0x00, - 0x00, // vertresolution: 4 bytes - 0x00, - 0x00, - 0x00, - 0x00, // reserved: 4 bytes - 0x00, - 0x01, // frame_count - 0x0a, // strlen - 0x78, - 0x71, - 0x71, - 0x2f, // compressorname: 32 bytes - 0x66, - 0x6c, - 0x76, - 0x2e, - 0x6a, - 0x73, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x00, - 0x18, // depth - 0xff, - 0xff, // pre_defined = -1 - ]); - return MP4.box(MP4.types.av01, data, MP4.box(MP4.types.av1C, av1c)); - } - // Movie Extends box static mvex(meta: MP4Meta): Uint8Array { return MP4.box(MP4.types.mvex, MP4.trex(meta)); diff --git a/web-ui/src/mpegts/remux/mp4-remuxer.ts b/web-ui/src/mpegts/remux/mp4-remuxer.ts index 03c95e2c..6fbf6442 100644 --- a/web-ui/src/mpegts/remux/mp4-remuxer.ts +++ b/web-ui/src/mpegts/remux/mp4-remuxer.ts @@ -1,5 +1,5 @@ import { MediaSegmentInfo, MediaSegmentInfoList, SampleInfo } from "../core/media-segment-info"; -import Browser from "../utils/browser"; +import { isFirefox, isSafari } from "../utils/browser"; import { IllegalStateException } from "../utils/exception"; import Log from "../utils/logger"; import AAC from "./aac-silent"; @@ -100,6 +100,7 @@ class MP4Remuxer { private _dtsBase: number; private _dtsBaseInited: boolean; + private _dtsBaseOffset: number; private _audioDtsBase: number; private _videoDtsBase: number; private _audioNextDts: number | undefined; @@ -116,8 +117,6 @@ class MP4Remuxer { private _onInitSegment: InitSegmentCallback | null; private _onMediaSegment: MediaSegmentCallback | null; - private _forceFirstIDR: boolean; - private _fillSilentAfterSeek: boolean; private _mp3UseMpegAudio: boolean; private _fillAudioTimestampGap: boolean; @@ -131,6 +130,7 @@ class MP4Remuxer { this._dtsBase = -1; this._dtsBaseInited = false; + this._dtsBaseOffset = 0; this._audioDtsBase = Infinity; this._videoDtsBase = Infinity; this._audioNextDts = undefined; @@ -147,22 +147,8 @@ class MP4Remuxer { this._onInitSegment = null; this._onMediaSegment = null; - // Workaround for chrome < 50: Always force first sample as a Random Access Point in media segment - // see https://bugs.chromium.org/p/chromium/issues/detail?id=229412 - const browser = Browser as Record; - const version = browser.version as Record | undefined; - this._forceFirstIDR = !!( - browser.chrome && - version && - (version.major < 50 || (version.major === 50 && version.build < 2661)) - ); - - // Workaround for IE11/Edge: Fill silent aac frame after keyframe-seeking - // Make audio beginDts equals with video beginDts, in order to fix seek freeze - this._fillSilentAfterSeek = !!(browser.msedge || browser.msie); - // While only FireFox supports 'audio/mp4, codecs="mp3"', use 'audio/mpeg' for chrome, safari, ... - this._mp3UseMpegAudio = !browser.firefox; + this._mp3UseMpegAudio = !isFirefox; this._fillAudioTimestampGap = this._config.fixAudioTimestampGap || false; @@ -228,12 +214,12 @@ class MP4Remuxer { this._silentAudioLastDts = undefined; } - seek(_originalDts: number): void { - this._audioStashedLastSample = null; - this._videoStashedLastSample = null; - this._videoSegmentInfoList?.clear(); - this._audioSegmentInfoList?.clear(); - this._silentAudioLastDts = undefined; + /** + * Position the output timeline: the first remuxed sample will be emitted at `offsetMs` + * instead of 0. Must be called before any samples are remuxed. + */ + setDtsBaseOffset(offsetMs: number): void { + this._dtsBaseOffset = offsetMs; } remux(audioTrack: DemuxTrack | undefined, videoTrack: DemuxTrack | undefined): void { @@ -428,6 +414,7 @@ class MP4Remuxer { } else { this._dtsBase = Math.min(this._audioDtsBase, this._videoDtsBase); } + this._dtsBase -= this._dtsBaseOffset; this._dtsBaseInited = true; } @@ -491,8 +478,6 @@ class MP4Remuxer { const mpegRawTrack = this._audioMeta.codec === "mp3" && this._mp3UseMpegAudio; const firstSegmentAfterSeek = this._dtsBaseInited && this._audioNextDts === undefined; - let insertPrefixSilentFrame = false; - if (!samples || samples.length === 0) { return; } @@ -547,11 +532,6 @@ class MP4Remuxer { // this._audioNextDts == undefined if (this._audioSegmentInfoList?.isEmpty()) { dtsCorrection = 0; - if (this._fillSilentAfterSeek && !this._videoSegmentInfoList?.isEmpty()) { - if (this._audioMeta.originalCodec !== "mp3") { - insertPrefixSilentFrame = true; - } - } } else { const prevSample = this._audioSegmentInfoList?.getLastSampleBefore(firstSampleOriginalDts); if (prevSample != null) { @@ -568,24 +548,6 @@ class MP4Remuxer { } } - if (insertPrefixSilentFrame) { - // align audio segment beginDts to match with current video segment's beginDts - const firstSampleDts = firstSampleOriginalDts - (dtsCorrection as number); - const videoSegment = this._videoSegmentInfoList?.getLastSegmentBefore(firstSampleOriginalDts); - if (videoSegment != null && videoSegment.beginDts < firstSampleDts) { - const silentUnit = AAC.getSilentFrame(this._audioMeta.originalCodec ?? "", this._audioMeta.channelCount ?? 0); - if (silentUnit) { - const dts = videoSegment.beginDts; - const silentFrameDuration = firstSampleDts - videoSegment.beginDts; - Log.v(this.TAG, `InsertPrefixSilentAudio: dts: ${dts}, duration: ${silentFrameDuration}`); - samples.unshift({ unit: silentUnit, dts: dts, pts: dts }); - mdatBytes += silentUnit.byteLength; - } // silentUnit == null: Cannot generate, skip - } else { - insertPrefixSilentFrame = false; - } - } - const mp4Samples: MP4Sample[] = []; // Correct dts for each sample, and calculate sample duration. Then output to mp4Samples @@ -621,7 +583,7 @@ class MP4Remuxer { } else if ( dtsCorrection >= maxAudioFramesDrift * refSampleDuration && this._fillAudioTimestampGap && - !(Browser as Record).safari + !isSafari ) { // Silent frame generation, if large timestamp gap detected && config.fixAudioTimestampGap needFillSilentFrames = true; @@ -993,14 +955,6 @@ class MP4Remuxer { track.samples = mp4Samples; track.sequenceNumber++; - // workaround for chrome < 50: force first sample as a random access point - // see https://bugs.chromium.org/p/chromium/issues/detail?id=229412 - if (this._forceFirstIDR) { - const flags = mp4Samples[0].flags; - flags.dependsOn = 2; - flags.isNonSync = 0; - } - const moofbox = MP4.moof(track as unknown as import("./mp4-generator").MP4Track, firstDts); track.samples = []; track.length = 0; diff --git a/web-ui/src/mpegts/types.ts b/web-ui/src/mpegts/types.ts index 44ba2bd1..b2ac6ba3 100644 --- a/web-ui/src/mpegts/types.ts +++ b/web-ui/src/mpegts/types.ts @@ -28,7 +28,6 @@ export interface Player { /** Internal player implementation interface */ export interface PlayerImpl { onError: ((error: PlayerError) => void) | null; - onHLSDetected?: (() => void) | null; onAudioSuspended?: (() => void) | null; loadSegments(segments: PlayerSegment[]): void; seek(seconds: number): void; diff --git a/web-ui/src/mpegts/utils/browser.ts b/web-ui/src/mpegts/utils/browser.ts index 2610279a..664cc29f 100644 --- a/web-ui/src/mpegts/utils/browser.ts +++ b/web-ui/src/mpegts/utils/browser.ts @@ -1,126 +1,7 @@ -export interface BrowserVersion { - major: number; - string: string; - minor?: number; - build?: number; -} +const ua = self.navigator?.userAgent.toLowerCase() ?? ""; -export interface BrowserInfo { - [key: string]: boolean | string | BrowserVersion | undefined; - version?: BrowserVersion; - name?: string; - platform?: string; -} +/** Safari (excluding Chrome/Chromium which also contain "safari" in UA). */ +export const isSafari = ua.includes("safari") && !ua.includes("chrome") && !ua.includes("android"); -const Browser: BrowserInfo = {}; - -function detect(): void { - // modified from jquery-browser-plugin - - const ua = self.navigator.userAgent.toLowerCase(); - - const match = - /(edge)\/([\w.]+)/.exec(ua) || - /(opr)[/]([\w.]+)/.exec(ua) || - /(chrome)[ /]([\w.]+)/.exec(ua) || - /(iemobile)[/]([\w.]+)/.exec(ua) || - /(version)(applewebkit)[ /]([\w.]+).*(safari)[ /]([\w.]+)/.exec(ua) || - /(webkit)[ /]([\w.]+).*(version)[ /]([\w.]+).*(safari)[ /]([\w.]+)/.exec(ua) || - /(webkit)[ /]([\w.]+)/.exec(ua) || - /(opera)(?:.*version|)[ /]([\w.]+)/.exec(ua) || - /(msie) ([\w.]+)/.exec(ua) || - (ua.indexOf("trident") >= 0 && /(rv)(?::| )([\w.]+)/.exec(ua)) || - (ua.indexOf("compatible") < 0 && /(firefox)[ /]([\w.]+)/.exec(ua)) || - []; - - const platform_match = - /(ipad)/.exec(ua) || - /(ipod)/.exec(ua) || - /(windows phone)/.exec(ua) || - /(iphone)/.exec(ua) || - /(kindle)/.exec(ua) || - /(android)/.exec(ua) || - /(windows)/.exec(ua) || - /(mac)/.exec(ua) || - /(linux)/.exec(ua) || - /(cros)/.exec(ua) || - []; - - const matched = { - browser: match[5] || match[3] || match[1] || "", - version: match[2] || match[4] || "0", - majorVersion: match[4] || match[2] || "0", - platform: platform_match[0] || "", - }; - - const browser: BrowserInfo = {}; - if (matched.browser) { - browser[matched.browser] = true; - - const versionArray = matched.majorVersion.split("."); - browser.version = { - major: parseInt(matched.majorVersion, 10), - string: matched.version, - }; - if (versionArray.length > 1) { - browser.version.minor = parseInt(versionArray[1], 10); - } - if (versionArray.length > 2) { - browser.version.build = parseInt(versionArray[2], 10); - } - } - - if (matched.platform) { - browser[matched.platform] = true; - } - - if (browser.chrome || browser.opr || browser.safari) { - browser.webkit = true; - } - - // MSIE. IE11 has 'rv' identifer - if (browser.rv || browser.iemobile) { - if (browser.rv) { - delete browser.rv; - } - const msie = "msie"; - matched.browser = msie; - browser[msie] = true; - } - - // Microsoft Edge - if (browser.edge) { - delete browser.edge; - const msedge = "msedge"; - matched.browser = msedge; - browser[msedge] = true; - } - - // Opera 15+ - if (browser.opr) { - const opera = "opera"; - matched.browser = opera; - browser[opera] = true; - } - - // Stock android browsers are marked as Safari - if (browser.safari && browser.android) { - const android = "android"; - matched.browser = android; - browser[android] = true; - } - - browser.name = matched.browser; - browser.platform = matched.platform; - - for (const key in Browser) { - if (Object.hasOwn(Browser, key)) { - delete Browser[key]; - } - } - Object.assign(Browser, browser); -} - -detect(); - -export default Browser; +/** Firefox — the only browser supporting 'audio/mp4; codecs="mp3"'. */ +export const isFirefox = ua.includes("firefox"); diff --git a/web-ui/src/mpegts/utils/typedarray-equality.ts b/web-ui/src/mpegts/utils/typedarray-equality.ts deleted file mode 100644 index d9ceb7c1..00000000 --- a/web-ui/src/mpegts/utils/typedarray-equality.ts +++ /dev/null @@ -1,50 +0,0 @@ -function isAligned16(a: Uint8Array): boolean { - return a.byteOffset % 2 === 0 && a.byteLength % 2 === 0; -} - -function isAligned32(a: Uint8Array): boolean { - return a.byteOffset % 4 === 0 && a.byteLength % 4 === 0; -} - -function compareArray(a: Uint8Array | Uint16Array | Uint32Array, b: Uint8Array | Uint16Array | Uint32Array): boolean { - for (let i = 0; i < a.length; i++) { - if (a[i] !== b[i]) { - return false; - } - } - return true; -} - -function equal8(a: Uint8Array, b: Uint8Array): boolean { - return compareArray(a, b); -} - -function equal16(a: Uint8Array, b: Uint8Array): boolean { - const a16 = new Uint16Array(a.buffer, a.byteOffset, a.byteLength / 2); - const b16 = new Uint16Array(b.buffer, b.byteOffset, b.byteLength / 2); - return compareArray(a16, b16); -} - -function equal32(a: Uint8Array, b: Uint8Array): boolean { - const a32 = new Uint32Array(a.buffer, a.byteOffset, a.byteLength / 4); - const b32 = new Uint32Array(b.buffer, b.byteOffset, b.byteLength / 4); - return compareArray(a32, b32); -} - -function buffersAreEqual(a: Uint8Array, b: Uint8Array): boolean { - if (a.byteLength !== b.byteLength) { - return false; - } - - if (isAligned32(a) && isAligned32(b)) { - return equal32(a, b); - } - - if (isAligned16(a) && isAligned16(b)) { - return equal16(a, b); - } - - return equal8(a, b); -} - -export default buffersAreEqual; diff --git a/web-ui/src/mpegts/utils/utf8-conv.ts b/web-ui/src/mpegts/utils/utf8-conv.ts deleted file mode 100644 index c712f492..00000000 --- a/web-ui/src/mpegts/utils/utf8-conv.ts +++ /dev/null @@ -1,67 +0,0 @@ -function checkContinuation(uint8array: Uint8Array, start: number, checkLength: number): boolean { - const array = uint8array; - if (start + checkLength < array.length) { - while (checkLength--) { - if ((array[++start] & 0xc0) !== 0x80) return false; - } - return true; - } else { - return false; - } -} - -function decodeUTF8(uint8array: Uint8Array): string { - const out: string[] = []; - const input = uint8array; - let i = 0; - const length = uint8array.length; - - while (i < length) { - if (input[i] < 0x80) { - out.push(String.fromCharCode(input[i])); - ++i; - continue; - } else if (input[i] < 0xc0) { - // fallthrough - } else if (input[i] < 0xe0) { - if (checkContinuation(input, i, 1)) { - const ucs4 = ((input[i] & 0x1f) << 6) | (input[i + 1] & 0x3f); - if (ucs4 >= 0x80) { - out.push(String.fromCharCode(ucs4 & 0xffff)); - i += 2; - continue; - } - } - } else if (input[i] < 0xf0) { - if (checkContinuation(input, i, 2)) { - const ucs4 = ((input[i] & 0xf) << 12) | ((input[i + 1] & 0x3f) << 6) | (input[i + 2] & 0x3f); - if (ucs4 >= 0x800 && (ucs4 & 0xf800) !== 0xd800) { - out.push(String.fromCharCode(ucs4 & 0xffff)); - i += 3; - continue; - } - } - } else if (input[i] < 0xf8) { - if (checkContinuation(input, i, 3)) { - let ucs4 = - ((input[i] & 0x7) << 18) | - ((input[i + 1] & 0x3f) << 12) | - ((input[i + 2] & 0x3f) << 6) | - (input[i + 3] & 0x3f); - if (ucs4 > 0x10000 && ucs4 < 0x110000) { - ucs4 -= 0x10000; - out.push(String.fromCharCode((ucs4 >>> 10) | 0xd800)); - out.push(String.fromCharCode((ucs4 & 0x3ff) | 0xdc00)); - i += 4; - continue; - } - } - } - out.push(String.fromCharCode(0xfffd)); - ++i; - } - - return out.join(""); -} - -export default decodeUTF8; diff --git a/web-ui/src/mpegts/worker/messages.ts b/web-ui/src/mpegts/worker/messages.ts index b61681f9..fa04d491 100644 --- a/web-ui/src/mpegts/worker/messages.ts +++ b/web-ui/src/mpegts/worker/messages.ts @@ -5,6 +5,7 @@ export type WorkerCommand = | { type: "init"; segments: PlayerSegment[]; config: PlayerConfig; gen: number } | { type: "start" } | { type: "load-segments"; segments: PlayerSegment[]; gen: number } + | { type: "seek"; seconds: number } | { type: "pause" } | { type: "resume" } | { type: "destroy" }; @@ -12,8 +13,7 @@ export type WorkerCommand = export type WorkerEvent = | { type: "init-segment"; track: "video" | "audio"; data: ArrayBuffer; codec: string; container: string; gen: number } | { type: "media-segment"; track: "video" | "audio"; data: ArrayBuffer; gen: number } - | { type: "media-info"; info: unknown; gen: number } | { type: "complete"; gen: number } | { type: "error"; category: "io" | "demux"; detail: string; info?: string; gen: number } - | { type: "hls-detected"; gen: number } + | { type: "hls-info"; live: boolean; totalDuration: number; gen: number } | { type: "pcm-audio-data"; pcm: ArrayBuffer; channels: number; sampleRate: number; pts: number; gen: number }; diff --git a/web-ui/src/mpegts/worker/pipeline.ts b/web-ui/src/mpegts/worker/pipeline.ts index 17f33e13..1f8fc7bd 100644 --- a/web-ui/src/mpegts/worker/pipeline.ts +++ b/web-ui/src/mpegts/worker/pipeline.ts @@ -1,13 +1,15 @@ import type { PlayerConfig } from "../config"; import { createDefaultConfig } from "../config"; -import MediaInfo from "../core/media-info"; import { WorkerAudioDecoder } from "../decoder/worker-audio-decoder"; import DemuxErrors from "../demux/demux-errors"; import TSDemuxer from "../demux/ts-demuxer"; -import FetchLoader from "../io/fetch-loader"; +import { containsMoov, parseInitSegment, probeFmp4, splitInitFromSegment } from "../hls/fmp4"; +import { type HlsInfo, HlsSource } from "../hls/hls-source"; +import FetchLoader, { LoaderErrors } from "../io/fetch-loader"; import MP4Remuxer from "../remux/mp4-remuxer"; import type { PlayerSegment } from "../types"; import Log from "../utils/logger"; +import { type SegmentMeta, type SegmentSource, StaticSegmentSource } from "./segment-source"; export interface PipelineCallbacks { onInitSegment: ( @@ -29,20 +31,32 @@ export interface PipelineCallbacks { }, ) => void; onLoadingComplete: () => void; - onMediaInfo: (mediaInfo: unknown) => void; onIOError: (type: string, info: { code: number; msg: string }) => void; onDemuxError: (type: string, info: string) => void; - onHLSDetected: () => void; + onHlsInfo: (info: HlsInfo) => void; onPCMAudioData: (pcm: Float32Array, channels: number, sampleRate: number, pts: number) => void; } -interface InternalSegment { - duration: number; - url: string; - timestampBase: number; - cors: boolean; - withCredentials: boolean; - referrerPolicy?: ReferrerPolicy; +class LoadError extends Error { + constructor( + public errorType: string, + public info: { code: number; msg: string }, + ) { + super(info.msg); + } +} + +const HLS_URL_RE = /\.m3u8?($|\?)/i; + +/** Sentinel rejection value for intentionally cancelled segment loads. */ +const CANCELLED = Symbol("cancelled"); + +/** Copy a Uint8Array view into a standalone (transferable) ArrayBuffer. */ +function toArrayBuffer(view: Uint8Array): ArrayBuffer { + if (view.byteOffset === 0 && view.byteLength === view.buffer.byteLength) { + return view.buffer as ArrayBuffer; + } + return view.slice().buffer as ArrayBuffer; } class Pipeline { @@ -51,81 +65,129 @@ class Pipeline { private _config: PlayerConfig; private _callbacks: PipelineCallbacks; - private _segments: InternalSegment[]; - private _currentSegmentIndex: number; + private _initialSegments: PlayerSegment[]; + + /** Increments to invalidate the currently running load loop. */ + private _runId = 0; + + private _source: SegmentSource | null = null; + private _hlsSource: HlsSource | null = null; + + private _demuxer: TSDemuxer | null = null; + private _remuxer: MP4Remuxer | null = null; + private _ioctl: FetchLoader | null = null; + /** Settles the in-flight segment load promise (so a cancelled loop can exit). */ + private _cancelLoad: (() => void) | null = null; + + private _paused = false; + private _resumeGate: (() => void) | null = null; + /** + * Discrete segment sources (HLS, catchup lists) load one short URL per iteration. + * For these, pause only gates between segments — never abort an in-flight fetch + * (which shows up as immediate cancel/retry on later segments). + */ + private _discreteSegments = false; + + /** dts offset (ms) to apply when the remuxer is next created (HLS discontinuity / seek). */ + private _pendingDtsOffsetMs = 0; + + // --- fMP4 passthrough state --- + private _fmp4Mode = false; + private _fmp4InitSent = false; + private _fmp4Chunks: Uint8Array[] = []; + private _lastInitUrl: string | null = null; - private _mediaInfo: MediaInfo | null; - private _demuxer: TSDemuxer | null; - private _remuxer: MP4Remuxer | null; - private _ioctl: FetchLoader | null; private _workerAudioDecoder: WorkerAudioDecoder | null = null; private _workerAudioDecoderInitPromise: Promise | null = null; constructor(segments: PlayerSegment[], config: PlayerConfig, callbacks: PipelineCallbacks) { this._callbacks = callbacks; this._config = { ...createDefaultConfig(), ...config }; - - this._segments = this._buildSegments(segments); - this._currentSegmentIndex = 0; - - this._mediaInfo = null; - this._demuxer = null; - this._remuxer = null; - this._ioctl = null; - } - - private _buildSegments(playerSegments: PlayerSegment[]): InternalSegment[] { - let totalDuration = 0; - const segments: InternalSegment[] = playerSegments.map((seg) => { - const duration = seg.duration ?? 0; - const internal: InternalSegment = { - duration, - url: seg.url, - timestampBase: totalDuration, - cors: true, - withCredentials: false, - }; - if (this._config.referrerPolicy) { - internal.referrerPolicy = this._config.referrerPolicy as ReferrerPolicy; - } - totalDuration += duration; - return internal; - }); - return segments; + this._initialSegments = segments; } start(): void { - this._loadSegment(0); + this._load(this._initialSegments); } - stop(): void { - this._internalAbort(); + loadSegments(newSegments: PlayerSegment[]): void { + this._load(newSegments); } pause(): void { - if (this._ioctl?.isWorking()) { - this._ioctl.pause(); + this._paused = true; + // Continuous single-URL TS streams can pause mid-fetch and resume via Range. + if (!this._discreteSegments) { + this._ioctl?.pause(); } } resume(): void { - if (this._ioctl?.isPaused()) { - this._ioctl.resume(); + this._paused = false; + if (!this._discreteSegments) { + this._ioctl?.resume(); } + this._resumeGate?.(); + this._resumeGate = null; } - loadSegments(newSegments: PlayerSegment[]): void { - // Stop current loading - this._internalAbort(); + /** Seek within an HLS VOD/EVENT playlist. No-op for live or non-HLS sources. */ + seek(seconds: number): void { + if (!this._hlsSource || this._hlsSource.info.live) { + return; + } + this._runId++; + this._abortCurrentLoad(); + this._fmp4Chunks = []; + this._hlsSource.seek(seconds); + void this._run(this._runId); + } + + destroy(): void { + this._runId++; + this._teardown(); + if (this._workerAudioDecoder) { + this._workerAudioDecoder.destroy(); + this._workerAudioDecoder = null; + } + this._workerAudioDecoderInitPromise = null; + } + + // ---- Private methods ---- + + private _load(segments: PlayerSegment[]): void { + this._runId++; + this._teardown(); + + // Reset WASM audio decoder state (clear stale mdct/qmf from previous stream) + this._workerAudioDecoder?.reset(); - // Reset internal state - this._mediaInfo = null; + const url = segments[0]?.url ?? ""; + this._discreteSegments = segments.length > 1 || HLS_URL_RE.test(url); + if (segments.length === 1 && HLS_URL_RE.test(url)) { + // Fast path: known playlist URL, skip the content-type detection round-trip + this._startHls(url); + } else { + this._source = new StaticSegmentSource(segments); + void this._run(this._runId); + } + } - // Setup new segments - this._segments = this._buildSegments(newSegments); - this._currentSegmentIndex = 0; + private _startHls(url: string, preloaded?: { text: string; url: string }): void { + this._discreteSegments = true; + const hls = new HlsSource(url, this._config, preloaded); + hls.onInfo = (info) => this._callbacks.onHlsInfo(info); + this._hlsSource = hls; + this._source = hls; + void this._run(this._runId); + } - // Destroy demuxer and remuxer for clean state (handles codec/container changes) + /** Stop all loading and demux/remux state, keeping the worker reusable. */ + private _teardown(): void { + this._abortCurrentLoad(); + this._source?.destroy(); + this._source = null; + this._hlsSource = null; if (this._demuxer) { this._demuxer.destroy(); this._demuxer = null; @@ -134,21 +196,101 @@ class Pipeline { this._remuxer.destroy(); this._remuxer = null; } - - // Reset WASM audio decoder state (clear stale mdct/qmf from previous stream) - this._workerAudioDecoder?.reset(); - - // Start from segment 0 — will re-probe format and recreate demuxer+remuxer - this._loadSegment(0); + this._pendingDtsOffsetMs = 0; + this._fmp4Mode = false; + this._fmp4InitSent = false; + this._fmp4Chunks = []; + this._lastInitUrl = null; + this._paused = false; + this._resumeGate?.(); + this._resumeGate = null; } - destroy(): void { - this._mediaInfo = null; - + private _abortCurrentLoad(): void { if (this._ioctl) { this._ioctl.destroy(); this._ioctl = null; } + this._cancelLoad?.(); + this._cancelLoad = null; + } + + /** Block until unpaused or this run is superseded (seek / reload). */ + private async _waitIfPaused(runId: number): Promise { + while (this._paused && this._runId === runId) { + await new Promise((resolve) => { + this._resumeGate = resolve; + }); + } + return this._runId === runId; + } + + // ---- Load loop ---- + + private async _run(runId: number): Promise { + const source = this._source; + if (!source) return; + + while (this._runId === runId) { + if (!(await this._waitIfPaused(runId))) return; + + let meta: SegmentMeta | null; + try { + meta = await source.next(); + } catch (e) { + if (this._runId === runId) { + Log.e(this.TAG, `Segment source failed: ${(e as Error).message}`); + this._callbacks.onIOError(LoaderErrors.EXCEPTION, { code: -1, msg: (e as Error).message }); + } + return; + } + if (this._runId !== runId) return; + + if (!meta) { + this._remuxer?.flushStashedSamples(); + this._callbacks.onLoadingComplete(); + return; + } + + try { + if (meta.resetRemuxer) { + this._resetTransmux(meta.start); + } + if (meta.initUrl && meta.initUrl !== this._lastInitUrl) { + if (!(await this._waitIfPaused(runId))) return; + await this._loadFmp4Init(meta.initUrl, runId); + if (this._runId !== runId) return; + this._lastInitUrl = meta.initUrl; + } + if (!(await this._waitIfPaused(runId))) return; + await this._loadSegment(meta); + if (this._runId !== runId) return; + + if (this._fmp4Mode) { + this._flushFmp4Segment(); + } + // HLS TS segments carry continuous timestamps: keep the stashed samples so the + // remuxer splices segments seamlessly. Static (catchup) segments each restart + // their own timeline, so flush between them. + if (!this._hlsSource) { + this._remuxer?.flushStashedSamples(); + } + } catch (e) { + if (this._runId !== runId || e === CANCELLED) return; + if (e instanceof LoadError) { + Log.e(this.TAG, `IOException: type = ${e.errorType}, code = ${e.info.code}, msg = ${e.info.msg}`); + this._callbacks.onIOError(e.errorType, e.info); + } else { + Log.e(this.TAG, `Segment load failed: ${(e as Error).message}`); + this._callbacks.onIOError(LoaderErrors.EXCEPTION, { code: -1, msg: (e as Error).message }); + } + return; + } + } + } + + /** Destroy demuxer + remuxer so the next segment re-anchors the output timeline at `startSeconds`. */ + private _resetTransmux(startSeconds: number): void { if (this._demuxer) { this._demuxer.destroy(); this._demuxer = null; @@ -157,98 +299,95 @@ class Pipeline { this._remuxer.destroy(); this._remuxer = null; } - if (this._workerAudioDecoder) { - this._workerAudioDecoder.destroy(); - this._workerAudioDecoder = null; - } - this._workerAudioDecoderInitPromise = null; + this._pendingDtsOffsetMs = startSeconds * 1000; } - // ---- Private methods ---- - - private _loadSegment(segmentIndex: number): void { - this._currentSegmentIndex = segmentIndex; - const segment = this._segments[segmentIndex]; - - const dataSource = { - url: segment.url, - cors: segment.cors, - withCredentials: segment.withCredentials, - referrerPolicy: segment.referrerPolicy, - }; - - const ioctl = new FetchLoader(dataSource, this._config, segmentIndex); + private _loadSegment(meta: SegmentMeta): Promise { + const ioctl = new FetchLoader( + { + url: meta.url, + cors: true, + withCredentials: false, + referrerPolicy: this._config.referrerPolicy as ReferrerPolicy | undefined, + }, + this._config, + ); this._ioctl = ioctl; - ioctl.onError = this._onIOException.bind(this); - ioctl.onSeeked = this._onIOSeeked.bind(this); - ioctl.onComplete = this._onIOComplete.bind(this) as (extraData: unknown) => void; - ioctl.onHLSDetected = () => this._callbacks.onHLSDetected(); - - ioctl.onDataArrival = this._onInitChunkArrival.bind(this); - ioctl.open(); + return new Promise((resolve, reject) => { + this._cancelLoad = () => reject(CANCELLED); + + ioctl.onError = (type, info) => reject(new LoadError(type, info)); + ioctl.onSeeked = () => this._remuxer?.insertDiscontinuity(); + ioctl.onComplete = () => resolve(); + ioctl.onHLSDetected = (text, url) => { + // Playlist served from a non-.m3u8 URL: switch the pipeline to the HLS source, + // reusing the playlist content we already downloaded + this._runId++; + reject(CANCELLED); + this._startHls(meta.url, { text, url }); + }; + ioctl.onDataArrival = (data, byteStart) => this._onProbeChunk(meta, data, byteStart); + ioctl.open(); + }).finally(() => { + ioctl.destroy(); + if (this._ioctl === ioctl) { + this._ioctl = null; + this._cancelLoad = null; + } + }); } - private _internalAbort(): void { - if (this._ioctl) { - this._ioctl.destroy(); - this._ioctl = null; + /** First-chunk handler: probe the container format, then hand off to the right path. */ + private _onProbeChunk(meta: SegmentMeta, data: ArrayBuffer, byteStart: number): number { + if (this._fmp4Mode) { + return this._onFmp4Chunk(data); } - } - private _onInitChunkArrival(data: ArrayBuffer, byteStart: number): number { const probeData = TSDemuxer.probe(data); - - if (!(probeData as Record).match) { - if (!(probeData as Record).needMoreData) { - Log.e(this.TAG, "Non MPEG-TS, Unsupported media type!"); - Promise.resolve().then(() => { - this._internalAbort(); - }); - this._callbacks.onDemuxError(DemuxErrors.FORMAT_UNSUPPORTED, "Non MPEG-TS, Unsupported media type!"); + if (probeData.match) { + this._setupTSDemuxerRemuxer(probeData, meta); + if (this._ioctl && this._demuxer) { + this._ioctl.onDataArrival = this._demuxer.parseChunks.bind(this._demuxer); } - return 0; + return this._demuxer?.parseChunks(data, byteStart) ?? 0; } - this._setupTSDemuxerRemuxer(probeData); - - // Set timestampBase for multi-segment time continuity - const segment = this._segments[this._currentSegmentIndex]; - if (segment && this._demuxer) { - this._demuxer.timestampBase = segment.timestampBase * 90000; // seconds → 90kHz ticks + if (probeFmp4(data)) { + this._fmp4Mode = true; + if (this._ioctl) { + this._ioctl.onDataArrival = (chunk) => this._onFmp4Chunk(chunk); + } + return this._onFmp4Chunk(data); } - // Switch from probe handler to direct demuxer parsing for subsequent chunks - if (this._ioctl && this._demuxer) { - (this._ioctl as unknown as Record).onDataArrival = this._demuxer.parseChunks.bind(this._demuxer); + if (!probeData.needMoreData) { + Log.e(this.TAG, "Unsupported media type (neither MPEG-TS nor fMP4)"); + Promise.resolve().then(() => this._abortCurrentLoad()); + this._callbacks.onDemuxError(DemuxErrors.FORMAT_UNSUPPORTED, "Unsupported media type!"); } - - return this._demuxer?.parseChunks(data, byteStart) ?? 0; + return 0; } - private _setupTSDemuxerRemuxer(probeData: unknown): void { + // ---- MPEG-TS path ---- + + private _setupTSDemuxerRemuxer(probeData: unknown, meta: SegmentMeta): void { if (this._demuxer) { this._demuxer.destroy(); } - const demuxer = new TSDemuxer(probeData as Record, this._config); + const demuxer = new TSDemuxer(probeData as ConstructorParameters[0]); this._demuxer = demuxer; if (!this._remuxer) { this._remuxer = new MP4Remuxer(this._config); + if (this._pendingDtsOffsetMs !== 0) { + this._remuxer.setDtsBaseOffset(this._pendingDtsOffsetMs); + this._pendingDtsOffsetMs = 0; + } } demuxer.onError = this._onDemuxException.bind(this); - demuxer.onMediaInfo = this._onMediaInfo.bind(this); - - // Metadata event callbacks: ignored (not forwarded to web-ui) - demuxer.onTimedID3Metadata = () => {}; - demuxer.onPGSSubtitleData = () => {}; - demuxer.onSynchronousKLVMetadata = () => {}; - demuxer.onAsynchronousKLVMetadata = () => {}; - demuxer.onSMPTE2038Metadata = () => {}; - demuxer.onSCTE35Metadata = () => {}; - demuxer.onPESPrivateDataDescriptor = () => {}; - demuxer.onPESPrivateData = () => {}; + demuxer.timestampBase = meta.timestampBase * 90000; // seconds → 90kHz ticks // Set up software audio decode callback when MP2 WASM URL is configured if (this._config.wasmDecoders.mp2) { @@ -257,96 +396,95 @@ class Pipeline { }; } - (this._remuxer as MP4Remuxer).bindDataSource( - this._demuxer as unknown as { + this._remuxer.bindDataSource( + demuxer as unknown as { onDataAvailable: (...args: unknown[]) => void; onTrackMetadata: (...args: unknown[]) => void; }, ); - (this._demuxer as TSDemuxer).bindDataSource(this._ioctl as unknown as Record); - this._remuxer.onInitSegment = this._onRemuxerInitSegmentArrival.bind(this); - this._remuxer.onMediaSegment = this._onRemuxerMediaSegmentArrival.bind( - this, - ) as unknown as typeof this._remuxer.onMediaSegment; - } - - private _onMediaInfo(mediaInfo: MediaInfo): void { - if (this._mediaInfo == null) { - // Store first segment's mediainfo as global mediaInfo - this._mediaInfo = Object.assign({}, mediaInfo) as MediaInfo; - this._mediaInfo.segments = []; - this._mediaInfo.segmentCount = this._segments.length; - Object.setPrototypeOf(this._mediaInfo, MediaInfo.prototype); - } - - const segmentInfo = Object.assign({}, mediaInfo) as MediaInfo; - Object.setPrototypeOf(segmentInfo, MediaInfo.prototype); - (this._mediaInfo.segments as MediaInfo[])[this._currentSegmentIndex] = segmentInfo; - - // Notify mediaInfo update - this._reportSegmentMediaInfo(this._currentSegmentIndex); + this._remuxer.onInitSegment = (type, initSegment) => { + this._callbacks.onInitSegment(type, initSegment as unknown as Parameters[1]); + }; + this._remuxer.onMediaSegment = (type, mediaSegment) => { + this._callbacks.onMediaSegment( + type, + mediaSegment as unknown as Parameters[1], + ); + }; } - private _onIOSeeked(): void { - (this._remuxer as MP4Remuxer).insertDiscontinuity(); + private _onDemuxException(type: string, info: string): void { + Log.e(this.TAG, `DemuxException: type = ${type}, info = ${info}`); + this._callbacks.onDemuxError(type, info); } - private _onIOComplete(extraData: number): void { - const segmentIndex = extraData; - const nextSegmentIndex = segmentIndex + 1; + // ---- fMP4 passthrough path ---- - if (nextSegmentIndex < this._segments.length) { - this._internalAbort(); - if (this._remuxer) { - this._remuxer.flushStashedSamples(); - } - this._loadSegment(nextSegmentIndex); - } else { - if (this._remuxer) { - this._remuxer.flushStashedSamples(); - } - this._callbacks.onLoadingComplete(); + private async _loadFmp4Init(initUrl: string, runId: number): Promise { + this._fmp4Mode = true; + const response = await fetch(initUrl, { + headers: this._config.headers, + referrerPolicy: (this._config.referrerPolicy as ReferrerPolicy | undefined) ?? "no-referrer-when-downgrade", + }); + if (this._runId !== runId) return; + if (!response.ok) { + throw new LoadError(LoaderErrors.HTTP_STATUS_CODE_INVALID, { code: response.status, msg: response.statusText }); } + const data = new Uint8Array(await response.arrayBuffer()); + // Superseded mid-fetch (seek/reload/destroy): don't append a stale init segment + if (this._runId !== runId) return; + this._sendFmp4Init(data); } - private _onIOException(type: string, info: { code: number; msg: string }): void { - Log.e(this.TAG, `IOException: type = ${type}, code = ${info.code}, msg = ${info.msg}`); - this._callbacks.onIOError(type, info); - } - - private _onDemuxException(type: string, info: string): void { - Log.e(this.TAG, `DemuxException: type = ${type}, info = ${info}`); - this._callbacks.onDemuxError(type, info); - } - - private _onRemuxerInitSegmentArrival(type: string, initSegment: unknown): void { - this._callbacks.onInitSegment( - type, - initSegment as { - type: string; - container: string; - codec?: string; - data?: ArrayBuffer; - }, - ); + private _sendFmp4Init(data: Uint8Array): void { + const codec = this._hlsSource?.info.codecs ?? parseInitSegment(data).codecs.join(","); + this._callbacks.onInitSegment("video", { + type: "video", + container: "video/mp4", + codec, + data: toArrayBuffer(data), + }); + this._fmp4InitSent = true; } - private _onRemuxerMediaSegmentArrival(type: string, mediaSegment: Record): void { - this._callbacks.onMediaSegment(type, mediaSegment as { type: string; data?: ArrayBuffer }); + private _onFmp4Chunk(data: ArrayBuffer): number { + this._fmp4Chunks.push(new Uint8Array(data)); + return data.byteLength; } - private _reportSegmentMediaInfo(segmentIndex: number): void { - const segmentInfo = this._mediaInfo?.segments?.[segmentIndex]; - const exportInfo: Record = Object.assign({}, segmentInfo) as unknown as Record; + /** Forward a fully buffered fMP4 segment to MSE (extracting the init part on first use). */ + private _flushFmp4Segment(): void { + if (this._fmp4Chunks.length === 0) { + return; + } + const total = this._fmp4Chunks.reduce((sum, c) => sum + c.byteLength, 0); + const segment = new Uint8Array(total); + let offset = 0; + for (const chunk of this._fmp4Chunks) { + segment.set(chunk, offset); + offset += chunk.byteLength; + } + this._fmp4Chunks = []; - exportInfo.duration = this._mediaInfo?.duration; - exportInfo.segmentCount = this._mediaInfo?.segmentCount; - delete exportInfo.segments; + let media: Uint8Array = segment; + if (!this._fmp4InitSent) { + if (!containsMoov(segment)) { + this._callbacks.onDemuxError(DemuxErrors.FORMAT_ERROR, "fMP4 stream has no initialization segment (moov)"); + return; + } + const parts = splitInitFromSegment(segment); + this._sendFmp4Init(parts.init); + media = parts.media; + } - this._callbacks.onMediaInfo(exportInfo); + if (media.byteLength > 0) { + this._callbacks.onMediaSegment("video", { type: "video", data: toArrayBuffer(media) }); + } } + // ---- MP2 software audio decode ---- + private _handleRawAudioFrame(frame: { codec: "mp2"; data: Uint8Array; pts: number }): void { // Lazily create WorkerAudioDecoder on first raw audio frame if (!this._workerAudioDecoder) { diff --git a/web-ui/src/mpegts/worker/segment-source.ts b/web-ui/src/mpegts/worker/segment-source.ts new file mode 100644 index 00000000..fc13a8f3 --- /dev/null +++ b/web-ui/src/mpegts/worker/segment-source.ts @@ -0,0 +1,50 @@ +import type { PlayerSegment } from "../types"; + +export interface SegmentMeta { + url: string; + /** Position of this segment on the output media timeline, in seconds. */ + start: number; + /** Segment duration in seconds (0 if unknown / live). */ + duration: number; + /** Value added to demuxed PTS/DTS (seconds). Used by the static (catchup) path where each URL restarts its own timeline. */ + timestampBase: number; + /** Destroy and recreate the remuxer before this segment, re-anchoring output at `start` (HLS discontinuity / seek). */ + resetRemuxer: boolean; + /** fMP4 initialization segment URL (HLS EXT-X-MAP), if any. */ + initUrl?: string; +} + +export interface SegmentSource { + /** Returns the next segment to load, or null when the stream has ended. May wait (e.g. live playlist refresh). */ + next(): Promise; + destroy(): void; +} + +/** Plain URL-list source: the existing mpegts / catchup multi-segment path. */ +export class StaticSegmentSource implements SegmentSource { + private index = 0; + private readonly metas: SegmentMeta[]; + + constructor(segments: PlayerSegment[]) { + let start = 0; + this.metas = segments.map((seg) => { + const meta: SegmentMeta = { + url: seg.url, + start, + duration: seg.duration ?? 0, + timestampBase: start, + resetRemuxer: false, + }; + start += seg.duration ?? 0; + return meta; + }); + } + + next(): Promise { + return Promise.resolve(this.metas[this.index++] ?? null); + } + + destroy(): void { + this.index = this.metas.length; + } +} diff --git a/web-ui/src/mpegts/worker/transmux-worker.ts b/web-ui/src/mpegts/worker/transmux-worker.ts index 29aa7346..2e5e3384 100644 --- a/web-ui/src/mpegts/worker/transmux-worker.ts +++ b/web-ui/src/mpegts/worker/transmux-worker.ts @@ -37,17 +37,14 @@ function createPipeline(segments: PlayerSegment[], config: PlayerConfig): Pipeli onLoadingComplete() { post({ type: "complete", gen }); }, - onMediaInfo(info) { - post({ type: "media-info", info, gen }); - }, onIOError(type, info) { post({ type: "error", category: "io", detail: type, info: info.msg, gen }); }, onDemuxError(type, info) { post({ type: "error", category: "demux", detail: type, info, gen }); }, - onHLSDetected() { - post({ type: "hls-detected", gen }); + onHlsInfo(info) { + post({ type: "hls-info", live: info.live, totalDuration: info.totalDuration, gen }); }, onPCMAudioData(pcm, channels, sampleRate, pts) { const buffer = pcm.buffer as ArrayBuffer; @@ -73,6 +70,9 @@ self.addEventListener("message", (e: MessageEvent) => { gen = cmd.gen; pipeline?.loadSegments(cmd.segments); break; + case "seek": + pipeline?.seek(cmd.seconds); + break; case "pause": pipeline?.pause(); break;