[timecode] Add new timecode module and initial VFR support #168

Add new Timecode type to replace FrameTimecode. The new type supports VFR by storing exact timing information.

Rough out how support for this will look in VideoStream by using the iterator protocol. This is only implemented
for OpenCV and PyAV currently, and both have some minor drawbacks.

Author: Breakthrough

scenedetect/__init__.py

@@ -56,7 +56,7 @@
 
 # Used for module identification and when printing version & about info
 # (e.g. calling `scenedetect version` or `scenedetect about`).
-__version__ = "0.6.6"
+__version__ = "0.7-dev0"
 
 init_logger()
 logger = getLogger("pyscenedetect")

scenedetect/backends/opencv.py

@@ -20,14 +20,22 @@
 import math
 import os.path
 import typing as ty
+from fractions import Fraction
 from logging import getLogger
 
 import cv2
 import numpy as np
 
 from scenedetect.frame_timecode import MAX_FPS_DELTA, FrameTimecode
 from scenedetect.platform import get_file_name
-from scenedetect.video_stream import FrameRateUnavailable, SeekError, VideoOpenFailure, VideoStream
+from scenedetect.timecode import Timecode
+from scenedetect.video_stream import (
+    FrameRateUnavailable,
+    SeekError,
+    VideoFrame,
+    VideoOpenFailure,
+    VideoStream,
+)
 
 logger = getLogger("pyscenedetect")
 
@@ -264,6 +272,30 @@ def reset(self):
         self._cap.release()
         self._open_capture(self._frame_rate)
 
+    def __next__(self):
+        # NOTE: POS_FRAMES starts from 0 before any frames are read.
+        read, image = self._cap.read()
+        if not read:
+            raise StopIteration()
+        # We can only query CAP_PROP_PTS if this uses the ffmpeg backend,  however it doesn't seem
+        # to work correctly. Quite frequently consecutive frames return the same PTS. We might need
+        # to just abandon using PTS with OpenCV and rely on milliseconds. This will still result
+        # in occasional off-by-one errors for VFR videos, but better than the status quo.
+        #
+        # We should also add a config option so users can specify if OpenCV should use fixed or
+        # variable timing (i.e. if we should use CAP_PROP_POS_MSEC or CAP_PROP_POS_FRAMES for
+        # timestamp calculation).
+        USE_PTS = False
+        if USE_PTS:
+            pts = self._cap.get(cv2.CAP_PROP_PTS)
+            time_base = Fraction.from_float(self._cap.get(cv2.CAP_PROP_FPS))
+            time_base = Fraction(numerator=time_base.denominator, denominator=time_base.numerator)
+        else:
+            pts = self._cap.get(cv2.CAP_PROP_POS_MSEC)
+            time_base = Fraction(1, 1000)
+        timecode = Timecode(pts=round(pts), time_base=time_base)
+        return VideoFrame(image=image, timecode=timecode)
+
     def read(self, decode: bool = True, advance: bool = True) -> ty.Union[np.ndarray, bool]:
         """Read and decode the next frame as a np.ndarray. Returns False when video ends,
         or the maximum number of decode attempts has passed.

scenedetect/backends/pyav.py

@@ -19,7 +19,8 @@
 
 from scenedetect.frame_timecode import MAX_FPS_DELTA, FrameTimecode
 from scenedetect.platform import get_file_name
-from scenedetect.video_stream import FrameRateUnavailable, VideoOpenFailure, VideoStream
+from scenedetect.timecode import Timecode
+from scenedetect.video_stream import FrameRateUnavailable, VideoFrame, VideoOpenFailure, VideoStream
 
 logger = getLogger("pyscenedetect")
 
@@ -263,6 +264,19 @@ def reset(self):
         except Exception as ex:
             raise VideoOpenFailure() from ex
 
+    def __next__(self) -> VideoFrame:
+        # TODO: On the VFR test video, we seem to only decode 1979 frames instead of 1980. See what
+        # the issue could be.
+        try:
+            frame = next(self._container.decode(video=0))
+        except av.error.EOFError as ex:
+            if not self._handle_eof():
+                raise StopIteration() from ex
+            return next(self)  # *NOTE*: self._handle_eof must ensure we won't recurse again.
+        image = frame.to_ndarray(format="bgr24")
+        timecode = Timecode(pts=frame.pts, time_base=frame.time_base)
+        return VideoFrame(image=image, timecode=timecode)
+
     def read(self, decode: bool = True, advance: bool = True) -> ty.Union[np.ndarray, bool]:
         """Read and decode the next frame as a np.ndarray. Returns False when video ends.
 

scenedetect/timecode.py

@@ -0,0 +1,49 @@
+#
+#            PySceneDetect: Python-Based Video Scene Detector
+#   -------------------------------------------------------------------
+#     [  Site:    https://scenedetect.com                           ]
+#     [  Docs:    https://scenedetect.com/docs/                     ]
+#     [  Github:  https://github.com/Breakthrough/PySceneDetect/    ]
+#
+# Copyright (C) 2014-2025 Brandon Castellano <http://www.bcastell.com>.
+# PySceneDetect is licensed under the BSD 3-Clause License; see the
+# included LICENSE file, or visit one of the above pages for details.
+#
+"""``scenedetect.timecode`` Module
+
+This module contains types and functions for handling video timecodes, including parsing user input
+and timecode format conversion.
+"""
+
+from dataclasses import dataclass
+from fractions import Fraction
+
+
+# TODO(@Breakthrough): Add conversion from Timecode -> FrameTimecode for backwards compatibility.
+# TODO(@Breakthrough): How should we deal with frame numbers? We might need to detect if a video is
+# VFR or not, and if so, either omit them or always start them from 0 regardless of the start seek.
+# With PyAV we can probably assume the video is VFR if the guessed rate of the stream differs
+# from the average rate.
+#
+# Each backend has slight nuances we have to take into account:
+#   - PyAV: Does not include a position in frames, we can probably estimate it. Need to also compare
+#     with how OpenCV handles this. It also seems to fail to decode the last frame. This library
+#     provides the most accurate timing information however.
+#   - OpenCV: Lacks any kind of timebase, only provides position in milliseconds and as frames.
+#     This is probably sufficient, since we could just use 1ms as a timebase.
+#   - MoviePy: Assumes fixed framerate and doesn't include timing information. Fixing this is
+#     probably not feasible, so we should make sure the docs warn users about this.
+#
+#
+@dataclass
+class Timecode:
+    """Timing information associated with a given frame."""
+
+    pts: int
+    """Presentation timestamp of the frame in units of `time_base`."""
+    time_base: Fraction
+    """The base unit in which `pts` is measured."""
+
+    @property
+    def seconds(self) -> float:
+        return float(self.time_base * self.pts)

scenedetect/video_stream.py

@@ -33,14 +33,12 @@
 
 import typing as ty
 from abc import ABC, abstractmethod
+from dataclasses import dataclass
 
 import numpy as np
 
 from scenedetect.frame_timecode import FrameTimecode
-
-##
-## VideoStream Exceptions
-##
+from scenedetect.timecode import Timecode
 
 
 class SeekError(Exception):
@@ -79,6 +77,14 @@ def __init__(self):
 ##
 
 
+@dataclass
+class VideoFrame:
+    """Data returned when reading/decoding a frame from a video."""
+
+    image: np.ndarray
+    timecode: Timecode
+
+
 class VideoStream(ABC):
     """Interface which all video backends must implement."""
 
@@ -175,7 +181,31 @@ def frame_number(self) -> int:
     #
     # Abstract Methods
     #
+    def __iter__(self) -> ty.Iterable[VideoFrame]:
+        return self
+
+    def __next__(self) -> VideoFrame:
+        """Read and decode the next frame from the current seek position.
+
+        Raises:
+            StopIteration: The next frame could not be decoded (i.e. the stream ended).
+        """
+        # TODO(v0.7): Make this an abstract method when it is implemented for all backends.
+        raise NotImplementedError()
+
+    def skip(self) -> ty.Optional[Timecode]:
+        """Advance the stream to the next frame without decoding the frame data. *May* be faster in
+        cases where the image data for a given frame isn't required.
+
+        Returns:
+            The `Timecode` of the frame that was skipped, or None if it could not be decoded (i.e.
+            the stream ended).
+        """
+        # TODO(v0.7): Make this an abstract method when it is implemented for all backends.
+        raise NotImplementedError()
 
+    # TODO(v0.7): Mark this as deprecated in lieu of `__next__` and `skip`. See if there is a way
+    # we can change this to no longer be an abstract method, but to instead use the above methods.
     @abstractmethod
     def read(self, decode: bool = True, advance: bool = True) -> ty.Union[np.ndarray, bool]:
         """Read and decode the next frame as a np.ndarray. Returns False when video ends.

tests/conftest.py

@@ -109,6 +109,12 @@ def test_movie_clip() -> str:
     return check_exists("tests/resources/goldeneye.mp4")
 
 
+@pytest.fixture
+def test_vfr_video() -> str:
+    """Movie clip containing fast cut, but encoded as variable framerate."""
+    return check_exists("tests/resources/goldeneye-vfr.mp4")
+
+
 @pytest.fixture
 def corrupt_video_file() -> str:
     """Video containing a corrupted frame causing a decode failure."""