Audio subtitles

CMakeLists.txt

@@ -1166,12 +1166,12 @@ if (NOT DEFINED IMPACTO_DISABLE_FFMPEG)
     list(APPEND Impacto_Src
         src/video/ffmpegplayer.cpp
         src/video/ffmpegstream.cpp
-        src/video/ffmpegsubtitlehelper.cpp
+        src/subtitle/ffmpegsubtitlehelper.cpp
     )
     list(APPEND Impacto_Header
         src/video/ffmpegplayer.h
         src/video/ffmpegstream.h
-        src/video/ffmpegsubtitlehelper.h
+        src/subtitle/ffmpegsubtitlehelper.h
     )
 
     if (NOT VCPKG_TOOLCHAIN)

src/audio/audiosystem.cpp

@@ -190,6 +190,33 @@ void AudioInit() {
   IsInit = true;
 }
 
+void AudioSubtitlesStart(AudioChannel* channel) {
+  using Profile::Subtitle::SubtitleMappings;
+  std::optional<Subtitle::SubtitlePlayer> player;
+
+  if (channel->GetGroup() != AudioChannelGroup::ACG_BGM) return;
+  AudioStream const* bgmStream = channel->GetStream();
+
+  auto bgmSubtitleMapItr = SubtitleMappings.find("bgm");
+  if (bgmSubtitleMapItr == SubtitleMappings.end()) return;
+  auto mappingsItr =
+      bgmSubtitleMapItr->second.find(bgmStream->GetBaseStream()->Meta.FileName);
+  if (mappingsItr == bgmSubtitleMapItr->second.end()) {
+    mappingsItr =
+        bgmSubtitleMapItr->second.find(bgmStream->GetBaseStream()->Meta.Id);
+  }
+  if (mappingsItr == bgmSubtitleMapItr->second.end()) return;
+
+  SubtitlePlayers[channel->GetId() - AC_BGM0].emplace(Profile::DesignWidth,
+                                                      Profile::DesignHeight);
+  int trackId = 0;
+  for (auto const& subFile : mappingsItr->second) {
+    if (!subFile.Path) continue;
+    SubtitlePlayers[channel->GetId() - AC_BGM0]->AddTrackFile(
+        trackId++, subFile.Type, *subFile.Path, subFile.Config);
+  }
+}
+
 void PlayInGroup(AudioChannelGroup group, std::string const& mountpoint,
                  uint32_t fileId, bool loop, float fadeIn) {
   AudioChannel* channel = GetNextChannelInGroup(group);
@@ -223,5 +250,34 @@ void AudioUpdate(float dt) {
   }
 }
 
+void AudioSubtitlesUpdate() {
+  for (int i = 0; i < ssize(SubtitlePlayers); i++) {
+    if (!SubtitlePlayers[i]) continue;
+    auto const* currentChannel = Channels[i + AC_BGM0].get();
+    switch (currentChannel->GetState()) {
+      case ACS_Stopped:
+        SubtitlePlayers[i].reset();
+        break;
+      case ACS_Paused:
+        break;
+      case ACS_Playing:
+      case ACS_FadingIn:
+      case ACS_FadingOut: {
+        auto durationSec =
+            std::chrono::duration<float>{currentChannel->PositionInSeconds()};
+        SubtitlePlayers[i]->UpdateElapsedTime(
+            std::chrono::duration_cast<Video::Clock::Microseconds>(
+                durationSec));
+      }
+    }
+  }
+}
+
+void AudioSubtitlesRender() {
+  for (auto& subtitlePlayer : SubtitlePlayers) {
+    if (subtitlePlayer) subtitlePlayer->Render();
+  }
+}
+
 }  // namespace Audio
 }  // namespace Impacto

src/audio/audiosystem.h

@@ -1,10 +1,12 @@
 #pragma once
 
+#include <memory>
+
 #include "../impacto.h"
 #include "../log.h"
 #include "audiochannel.h"
 #include "audiobackend.h"
-#include <memory>
+#include "../subtitle/subtitlesystem.h"
 
 namespace Impacto {
 namespace Audio {
@@ -13,12 +15,19 @@ void AudioInit();
 void AudioUpdate(float dt);
 void AudioShutdown();
 
+void AudioSubtitlesStart(AudioChannel* channel);
+void AudioSubtitlesUpdate();
+void AudioSubtitlesRender();
+
 inline AudioBackend* Backend = nullptr;
 
 inline float MasterVolume = 1.0f;
 inline std::array<float, ACG_Count> GroupVolumes;
 inline std::array<std::unique_ptr<AudioChannel>, AC_Count> Channels;
 
+// 3 subtitle players for 3 bgm channels
+inline std::array<std::optional<Subtitle::SubtitlePlayer>, 3> SubtitlePlayers;
+
 template <AudioChannelId... Ids>
 struct ChannelGroupDef {
   static constexpr std::array<AudioChannelId, sizeof...(Ids)> ChannelIds{

src/audio/openal/openalaudiochannel.cpp

@@ -49,6 +49,8 @@ void OpenALAudioChannel::Play(std::unique_ptr<AudioStream> stream, bool loop,
   FadeProgress = FadeDuration == 0 ? 1.0f : 0.0f;
   State = FadeDuration == 0 ? ACS_Playing : ACS_FadingIn;
   Looping = loop;
+
+  AudioSubtitlesStart(this);
 }
 
 void OpenALAudioChannel::Stop(float fadeOutDuration) {

src/game.cpp

@@ -333,6 +333,11 @@ void Update(float dt) {
     Video::VideoUpdate(dt);
   }
 
+  if (+Profile::GameFeatures & +GameFeature::Audio &&
+      +Profile::GameFeatures & +GameFeature::Subtitles) {
+    Audio::AudioSubtitlesUpdate();
+  }
+
   if (+Profile::GameFeatures & +GameFeature::Scene3D) {
     Renderer->Scene->Update(dt);
   }
@@ -512,6 +517,11 @@ static void RenderMain() {
         (ScrWork[SW_MOVIEPRI] == 3000 || ScrWork[SW_MOVIEPRI] == 4000)))) {
     Video::VideoRender(ScrWork[SW_MOVIEALPHA] / 256.0f);
   }
+
+  if (+Profile::GameFeatures & +GameFeature::Audio &&
+      +Profile::GameFeatures & +GameFeature::Subtitles) {
+    Audio::AudioSubtitlesRender();
+  }
   MaskCapture.SetHasEffects(false);
 }
 

src/profile/profile_internal.h

@@ -54,6 +54,12 @@ T EnsureGet();
 template <typename T>
 T EnsureGetKey();
 
+template <typename T>
+  requires requires(T x) {
+    { std::variant{x} } -> std::same_as<T>;
+  }
+T EnsureGetKey();
+
 template <typename T>
 bool TryGetMember(char const* name, T& out);
 
@@ -132,6 +138,12 @@ template <typename T>
   }
 std::optional<T> TryGet();
 
+template <typename T>
+  requires requires(T x) {
+    { std::variant{x} } -> std::same_as<T>;
+  }
+std::optional<T> TryGet();
+
 template <typename E>
   requires std::is_enum_v<E>
 std::optional<E> TryGet();
@@ -156,8 +168,45 @@ inline T EnsureGetKey() {
     return lua_tostring(LuaState, -2);
   else if constexpr (std::is_integral_v<T> && !std::is_same_v<T, bool>)
     return (T)lua_tointeger(LuaState, -2);
-  else
+  else if constexpr (requires(T x) {
+                       {
+                         std::variant { x }
+                       } -> std::same_as<T>;
+                     }) {
+    lua_rotate(LuaState, -2, 1);
+    T result = EnsureGet<T>();
+    lua_rotate(LuaState, -2, -1);
+    return result;
+
+  } else
+    static_assert(sizeof(T*) == 0, "Invalid Key Type");
+}
+
+template <typename T>
+  requires requires(T x) {
+    { std::variant{x} } -> std::same_as<T>;
+  }
+inline T EnsureGetKey() {
+  static constexpr auto isKeyType = []<typename K>() {
+    return is_any_of_v<K, char const*, std::string_view, std::string> ||
+           (std::is_integral_v<K> && !std::is_same_v<K, bool>);
+  };
+  static constexpr auto variantTypeChecker = []<std::size_t... Is>(
+                                                 std::index_sequence<Is...>) {
+    return (
+        isKeyType.template operator()<std::variant_alternative_t<Is, T>>() ||
+        ...);
+  };
+
+  if constexpr (variantTypeChecker.template operator()(
+                    std::make_index_sequence<std::variant_size_v<T>>{})) {
+    lua_rotate(LuaState, -2, 1);
+    T result = EnsureGet<T>();
+    lua_rotate(LuaState, -2, -1);
+    return result;
+  } else {
     static_assert(sizeof(T*) == 0, "Invalid Key Type");
+  }
 }
 
 template <typename T>
@@ -234,7 +283,7 @@ inline std::optional<T> TryGet() {
       if (ec == std::errc{}) {
         return outNumber;
       }
-      ImpLog(LogLevel::Fatal, LogChannel::Profile,
+      ImpLog(LogLevel::Warning, LogChannel::Profile,
              "Error encountered converting {:s} to number: {:s}\n", inputStr,
              std::make_error_code(ec).message());
     } else {
@@ -247,7 +296,7 @@ inline std::optional<T> TryGet() {
       if (endPtr != inputStr.data()) {
         return outNumber;
       }
-      ImpLog(LogLevel::Fatal, LogChannel::Profile,
+      ImpLog(LogLevel::Warning, LogChannel::Profile,
              "Error encountered converting {:s} to number: {:s}\n", inputStr,
              std::error_code{errno, std::generic_category()}.message());
     }
@@ -410,6 +459,30 @@ inline std::optional<T> TryGet() {
   return result;
 }
 
+template <typename T>
+  requires requires(T x) {
+    { std::variant{x} } -> std::same_as<T>;
+  }
+inline std::optional<T> TryGet() {
+  const auto variantElemHandler =
+      [&]<std::size_t I>(std::optional<T>& variantOpt) {
+        using ElemType = std::variant_alternative_t<I, T>;
+        auto elemOpt = TryGet<ElemType>();
+        if (elemOpt) variantOpt = std::move(*elemOpt);
+        return elemOpt.has_value();
+      };
+
+  const auto variantFetcher =
+      [&]<std::size_t... Is>(std::index_sequence<Is...>) {
+        std::optional<T> variantOpt;
+
+        (variantElemHandler.template operator()<Is>(variantOpt) || ...);
+        return variantOpt;
+      };
+
+  return variantFetcher(std::make_index_sequence<std::variant_size_v<T>>{});
+}
+
 template <typename T>
   requires is_std_array<T>::value
 inline std::optional<T> TryGet() {
@@ -442,8 +515,8 @@ inline std::optional<T> TryGet() {
 template <typename E>
   requires std::is_enum_v<E>
 inline std::optional<E> TryGet() {
-  const auto optUnderlying = TryGet<std::underlying_type_t<E>>();
   using MagicEnumRange = magic_enum::customize::enum_range<E>;
+  const auto optUnderlying = TryGet<std::underlying_type_t<E>>();
   if (optUnderlying) {
     if constexpr (requires { MagicEnumRange::is_flags; }) {
       if constexpr (MagicEnumRange::is_flags)

src/profile/subtitle.h

@@ -21,7 +21,10 @@ struct SubtitleTrackFile {
 };
 
 using SubtitleTrackFiles = std::vector<SubtitleTrackFile>;
-inline ankerl::unordered_dense::map<std::string, SubtitleTrackFiles>
+using SubtitleMountMapping =
+    ankerl::unordered_dense::map<std::variant<uint32_t, std::string>,
+                                 SubtitleTrackFiles>;
+inline ankerl::unordered_dense::map<std::string, SubtitleMountMapping>
     SubtitleMappings;
 inline std::vector<std::string> SubtitleFontsDir;
 

src/shaders/opengl/SubtitleGlyph_frag.glsl

@@ -7,6 +7,6 @@ uniform sampler2D CoverageMap;
 
 void main() {
     float mask = texture(CoverageMap, uv).r;
-    color = tint * mask;
-    color.rgb = clamp(color.rgb, 0.0, 1.0);
+    float alpha = tint.a * mask;
+    color = vec4(tint.rgb * alpha, alpha);
 }

src/video/ffmpegsubtitlehelper.cpp → src/subtitle/ffmpegsubtitlehelper.cpp

@@ -2,7 +2,7 @@
 
 #include <avcpp/packet.h>
 
-namespace Impacto::Video {
+namespace Impacto::Subtitle {
 // avcpp codeccontext.cpp
 std::pair<int, const std::error_category*> make_error_pair(av::Errors errc) {
   return std::make_pair(static_cast<int>(errc), &av::avcpp_category());
@@ -137,4 +137,4 @@ SubtitleData SubtitleDecoderContext::decode(const av::Packet& packet,
   return decodeSubtitle(ec, packet, offset, &decodedBytes);
 }
 
-}  // namespace Impacto::Video
+}  // namespace Impacto::Subtitle

src/video/ffmpegsubtitlehelper.h → src/subtitle/ffmpegsubtitlehelper.h

@@ -6,7 +6,7 @@
 // Since these are basically extension classes for avcpp, we'll use their naming
 // conventions for it for consistency
 
-namespace Impacto::Video {
+namespace Impacto::Subtitle {
 
 struct SubtitleData {
   SubtitleData() = default;
@@ -153,4 +153,4 @@ class SubtitleDecoderContext
                               size_t* decodedBytes);
 };
 
-}  // namespace Impacto::Video
+}  // namespace Impacto::Subtitle

src/subtitle/subtitlesystem.cpp

@@ -119,7 +119,7 @@ void SubtitlePlayer::PushEntry(int trackId, SubtitleEntry entry) {
   backend->AddSubtitleEntry(trackId, std::move(entry));
 }
 
-void SubtitlePlayer::Update(Video::Clock::Microseconds elapsedTime) {
+void SubtitlePlayer::UpdateElapsedTime(Video::Clock::Microseconds elapsedTime) {
   for (auto& backend : Backends) {
     if (backend) {
       backend->Update(elapsedTime);

src/subtitle/subtitlesystem.h

@@ -36,7 +36,7 @@ class SubtitlePlayer {
                     std::string const& path,
                     Profile::SubtitleConfigType config);
   void PushEntry(int trackId, SubtitleEntry entry);
-  void Update(Video::Clock::Microseconds elapsedTime);
+  void UpdateElapsedTime(Video::Clock::Microseconds elapsedTime);
   void Render();
 
  protected: