diff --git a/.gitmodules b/.gitmodules index 7430bb873ca..c817dc52781 100644 --- a/.gitmodules +++ b/.gitmodules @@ -28,3 +28,6 @@ [submodule "Externals/sse2rvv"] path = Externals/sse2rvv url = https://github.com/pattonkan/sse2rvv.git +[submodule "Externals/freetype"] + path = Externals/freetype + url = https://github.com/ChugunovRoman/freetype.git diff --git a/Externals/CMakeLists.txt b/Externals/CMakeLists.txt index 4b438f03b6a..1b8d014fe38 100644 --- a/Externals/CMakeLists.txt +++ b/Externals/CMakeLists.txt @@ -15,6 +15,15 @@ add_compile_options(${XRAY_DISABLE_WARNINGS}) add_subdirectory(GameSpy) add_subdirectory(imgui-proj) +# FreeType: disable optional dependencies to minimize build complexity +set(FT_DISABLE_ZLIB ON CACHE BOOL "" FORCE) +set(FT_DISABLE_BZIP2 ON CACHE BOOL "" FORCE) +set(FT_DISABLE_PNG ON CACHE BOOL "" FORCE) +set(FT_DISABLE_HARFBUZZ ON CACHE BOOL "" FORCE) +set(FT_DISABLE_BROTLI ON CACHE BOOL "" FORCE) + +add_subdirectory(freetype) + if (NOT TARGET xrLuabind) message(FATAL_ERROR "You probably have downloaded OpenXRay sources as ZIP archive, you can't do that. Use git to clone the repository.\n" diff --git a/Externals/freetype b/Externals/freetype new file mode 160000 index 00000000000..f025a8c92c0 --- /dev/null +++ b/Externals/freetype @@ -0,0 +1 @@ +Subproject commit f025a8c92c09d55a43d696985913005add4907e0 diff --git a/Externals/freetype.vcxproj b/Externals/freetype.vcxproj new file mode 100644 index 00000000000..24d46ac59bc --- /dev/null +++ b/Externals/freetype.vcxproj @@ -0,0 +1,80 @@ + + + + + + + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B} + freetype + + + + + + + StaticLibrary + + + + + + + + + + + + FT2_BUILD_LIBRARY;%(PreprocessorDefinitions) + NotUsing + TurnOffAllWarnings + $(xrExternals)freetype\include;%(AdditionalIncludeDirectories) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/Externals/freetype.vcxproj.filters b/Externals/freetype.vcxproj.filters new file mode 100644 index 00000000000..72dc09126a5 --- /dev/null +++ b/Externals/freetype.vcxproj.filters @@ -0,0 +1,208 @@ + + + + + {f2a4c9c1-3f5b-4a7e-9c8d-1e2f3a4b5c6d} + + + {a1b2c3d4-e5f6-7890-abcd-ef1234567890} + + + {b2c3d4e5-f6a7-8901-bcde-f23456789012} + + + {c1d2e3f4-a5b6-7890-cdef-1234567890ab} + + + {d2e3f4a5-b6c7-8901-defa-2345678901bc} + + + {c3d4e5f6-a7b8-9012-cdef-345678901234} + + + {e3f4a5b6-c7d8-9012-fabc-3456789012cd} + + + {f4a5b6c7-d8e9-0123-abcd-4567890123de} + + + {a5b6c7d8-e9f0-1234-bcde-5678901234ef} + + + {b6c7d8e9-f0a1-2345-cdef-6789012345f0} + + + {c7d8e9f0-a1b2-3456-defa-789012345601} + + + {d8e9f0a1-b2c3-4567-efab-890123456712} + + + {d4e5f6a7-b8c9-0123-defa-456789012345} + + + {e5f6a7b8-c9d0-1234-efab-567890123456} + + + {f6a7b8c9-d0e1-2345-fabc-678901234567} + + + {a7b8c9d0-e1f2-3456-abcd-789012345678} + + + {e9f0a1b2-c3d4-5678-0123-901234567823} + + + {b8c9d0e1-f2a3-4567-bcde-890123456789} + + + {c9d0e1f2-a3b4-5678-cdef-901234567890} + + + {f0a1b2c3-d4e5-6789-1234-012345678934} + + + {d0e1f2a3-b4c5-6789-defa-012345678901} + + + {e1f2a3b4-c5d6-7890-efab-123456789012} + + + {f2a3b4c5-d6e7-8901-fabc-234567890123} + + + {a3b4c5d6-e7f8-9012-abcd-345678901234} + + + {b4c5d6e7-f8a9-0123-bcde-456789012345} + + + + + src\autofit + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\base + + + src\bdf + + + src\cache + + + src\cff + + + src\cid + + + src\gzip + + + src\hvf + + + src\lzw + + + src\pcf + + + src\pfr + + + src\psaux + + + src\pshinter + + + src\psnames + + + src\raster + + + src\sfnt + + + src\smooth + + + src\sdf + + + src\svg + + + src\truetype + + + src\type1 + + + src\type42 + + + src\winfonts + + + builds\windows + + + builds\windows + + + diff --git a/res/fsgame.ltx b/res/fsgame.ltx index 4082c5030bd..08ff070d12a 100644 --- a/res/fsgame.ltx +++ b/res/fsgame.ltx @@ -19,10 +19,11 @@ $game_textures$ = true| true| $game_data$| textures\ $game_config$ = true| false| $game_data$| configs\ $game_weathers$ = true| false| $game_config$| environment\weathers $game_weather_effects$ = true| false| $game_config$| environment\weather_effects +$game_fonts$ = true| false| $game_data$| fonts\ $textures$ = true| true| $game_data$| textures\ $level$ = false| false| $game_levels$ $game_scripts$ = true| false| $game_data$| scripts\| *.script| Game script files $logs$ = true| false| $app_data_root$| logs\ $screenshots$ = true| false| $app_data_root$| screenshots\ $game_saves$ = true| false| $app_data_root$| savedgames\ -$downloads$ = false| false| $app_data_root$ \ No newline at end of file +$downloads$ = false| false| $app_data_root$ diff --git a/res/gamedata/fonts/DroidSans.ttf b/res/gamedata/fonts/DroidSans.ttf new file mode 100644 index 00000000000..767c63ad000 Binary files /dev/null and b/res/gamedata/fonts/DroidSans.ttf differ diff --git a/res/gamedata/fonts/editors/RobotoMono.ttf b/res/gamedata/fonts/editors/RobotoMono.ttf new file mode 100644 index 00000000000..6df2b253603 Binary files /dev/null and b/res/gamedata/fonts/editors/RobotoMono.ttf differ diff --git a/res/gamedata/fonts/eng/ARIALN.TTF b/res/gamedata/fonts/eng/ARIALN.TTF new file mode 100644 index 00000000000..49bb696062f Binary files /dev/null and b/res/gamedata/fonts/eng/ARIALN.TTF differ diff --git a/res/gamedata/fonts/eng/ARIALNB.TTF b/res/gamedata/fonts/eng/ARIALNB.TTF new file mode 100644 index 00000000000..631931e5304 Binary files /dev/null and b/res/gamedata/fonts/eng/ARIALNB.TTF differ diff --git a/res/gamedata/fonts/eng/ARIALNBI.TTF b/res/gamedata/fonts/eng/ARIALNBI.TTF new file mode 100644 index 00000000000..4704de205a8 Binary files /dev/null and b/res/gamedata/fonts/eng/ARIALNBI.TTF differ diff --git a/res/gamedata/fonts/eng/ARIALNI.TTF b/res/gamedata/fonts/eng/ARIALNI.TTF new file mode 100644 index 00000000000..1c42b10a70c Binary files /dev/null and b/res/gamedata/fonts/eng/ARIALNI.TTF differ diff --git a/res/gamedata/fonts/eng/Graffiti1CTT.ttf b/res/gamedata/fonts/eng/Graffiti1CTT.ttf new file mode 100644 index 00000000000..80b44687bd3 Binary files /dev/null and b/res/gamedata/fonts/eng/Graffiti1CTT.ttf differ diff --git a/res/gamedata/fonts/eng/Letterica.ttf b/res/gamedata/fonts/eng/Letterica.ttf new file mode 100644 index 00000000000..4be0bae1650 Binary files /dev/null and b/res/gamedata/fonts/eng/Letterica.ttf differ diff --git a/res/gamedata/fonts/eng/Roboto.ttf b/res/gamedata/fonts/eng/Roboto.ttf new file mode 100644 index 00000000000..ddf4bfacb39 Binary files /dev/null and b/res/gamedata/fonts/eng/Roboto.ttf differ diff --git a/res/gamedata/fonts/eng/RobotoMono-Bold.ttf b/res/gamedata/fonts/eng/RobotoMono-Bold.ttf new file mode 100644 index 00000000000..d8841280b87 Binary files /dev/null and b/res/gamedata/fonts/eng/RobotoMono-Bold.ttf differ diff --git a/res/gamedata/fonts/eng/RobotoMono-Light.ttf b/res/gamedata/fonts/eng/RobotoMono-Light.ttf new file mode 100644 index 00000000000..4893662d796 Binary files /dev/null and b/res/gamedata/fonts/eng/RobotoMono-Light.ttf differ diff --git a/res/gamedata/fonts/eng/RobotoMono-Medium.ttf b/res/gamedata/fonts/eng/RobotoMono-Medium.ttf new file mode 100644 index 00000000000..f6c149a2035 Binary files /dev/null and b/res/gamedata/fonts/eng/RobotoMono-Medium.ttf differ diff --git a/res/gamedata/fonts/eng/RobotoMono.ttf b/res/gamedata/fonts/eng/RobotoMono.ttf new file mode 100644 index 00000000000..6df2b253603 Binary files /dev/null and b/res/gamedata/fonts/eng/RobotoMono.ttf differ diff --git a/res/gamedata/fonts/eng/arial.ttf b/res/gamedata/fonts/eng/arial.ttf new file mode 100644 index 00000000000..8682d94623e Binary files /dev/null and b/res/gamedata/fonts/eng/arial.ttf differ diff --git a/res/gamedata/fonts/eng/arialbd.ttf b/res/gamedata/fonts/eng/arialbd.ttf new file mode 100644 index 00000000000..a6037e6865d Binary files /dev/null and b/res/gamedata/fonts/eng/arialbd.ttf differ diff --git a/res/gamedata/fonts/eng/arialbi.ttf b/res/gamedata/fonts/eng/arialbi.ttf new file mode 100644 index 00000000000..6a1fa0fa035 Binary files /dev/null and b/res/gamedata/fonts/eng/arialbi.ttf differ diff --git a/res/gamedata/fonts/eng/ariali.ttf b/res/gamedata/fonts/eng/ariali.ttf new file mode 100644 index 00000000000..380199780fc Binary files /dev/null and b/res/gamedata/fonts/eng/ariali.ttf differ diff --git a/res/gamedata/fonts/eng/ariblk.ttf b/res/gamedata/fonts/eng/ariblk.ttf new file mode 100644 index 00000000000..e7ae345aa00 Binary files /dev/null and b/res/gamedata/fonts/eng/ariblk.ttf differ diff --git a/res/gamedata/fonts/eng/consola.ttf b/res/gamedata/fonts/eng/consola.ttf new file mode 100644 index 00000000000..e881ca4b59f Binary files /dev/null and b/res/gamedata/fonts/eng/consola.ttf differ diff --git a/res/gamedata/fonts/eng/consolab.ttf b/res/gamedata/fonts/eng/consolab.ttf new file mode 100644 index 00000000000..77f5d6052e1 Binary files /dev/null and b/res/gamedata/fonts/eng/consolab.ttf differ diff --git a/res/gamedata/fonts/eng/consolai.ttf b/res/gamedata/fonts/eng/consolai.ttf new file mode 100644 index 00000000000..2de4de8a990 Binary files /dev/null and b/res/gamedata/fonts/eng/consolai.ttf differ diff --git a/res/gamedata/fonts/eng/consolaz.ttf b/res/gamedata/fonts/eng/consolaz.ttf new file mode 100644 index 00000000000..d9df2110fee Binary files /dev/null and b/res/gamedata/fonts/eng/consolaz.ttf differ diff --git a/res/gamedata/fonts/eng/default.ttf b/res/gamedata/fonts/eng/default.ttf new file mode 100644 index 00000000000..4be0bae1650 Binary files /dev/null and b/res/gamedata/fonts/eng/default.ttf differ diff --git a/res/gamedata/fonts/eng/robotocondensed.ttf b/res/gamedata/fonts/eng/robotocondensed.ttf new file mode 100644 index 00000000000..74f029c7c46 Binary files /dev/null and b/res/gamedata/fonts/eng/robotocondensed.ttf differ diff --git a/res/gamedata/fonts/rus/ARIALN.TTF b/res/gamedata/fonts/rus/ARIALN.TTF new file mode 100644 index 00000000000..49bb696062f Binary files /dev/null and b/res/gamedata/fonts/rus/ARIALN.TTF differ diff --git a/res/gamedata/fonts/rus/ARIALNB.TTF b/res/gamedata/fonts/rus/ARIALNB.TTF new file mode 100644 index 00000000000..631931e5304 Binary files /dev/null and b/res/gamedata/fonts/rus/ARIALNB.TTF differ diff --git a/res/gamedata/fonts/rus/ARIALNBI.TTF b/res/gamedata/fonts/rus/ARIALNBI.TTF new file mode 100644 index 00000000000..4704de205a8 Binary files /dev/null and b/res/gamedata/fonts/rus/ARIALNBI.TTF differ diff --git a/res/gamedata/fonts/rus/ARIALNI.TTF b/res/gamedata/fonts/rus/ARIALNI.TTF new file mode 100644 index 00000000000..1c42b10a70c Binary files /dev/null and b/res/gamedata/fonts/rus/ARIALNI.TTF differ diff --git a/res/gamedata/fonts/rus/Graffiti1CTT.ttf b/res/gamedata/fonts/rus/Graffiti1CTT.ttf new file mode 100644 index 00000000000..80b44687bd3 Binary files /dev/null and b/res/gamedata/fonts/rus/Graffiti1CTT.ttf differ diff --git a/res/gamedata/fonts/rus/Letterica.ttf b/res/gamedata/fonts/rus/Letterica.ttf new file mode 100644 index 00000000000..4be0bae1650 Binary files /dev/null and b/res/gamedata/fonts/rus/Letterica.ttf differ diff --git a/res/gamedata/fonts/rus/Roboto.ttf b/res/gamedata/fonts/rus/Roboto.ttf new file mode 100644 index 00000000000..ddf4bfacb39 Binary files /dev/null and b/res/gamedata/fonts/rus/Roboto.ttf differ diff --git a/res/gamedata/fonts/rus/RobotoMono-Bold.ttf b/res/gamedata/fonts/rus/RobotoMono-Bold.ttf new file mode 100644 index 00000000000..d8841280b87 Binary files /dev/null and b/res/gamedata/fonts/rus/RobotoMono-Bold.ttf differ diff --git a/res/gamedata/fonts/rus/RobotoMono-Light.ttf b/res/gamedata/fonts/rus/RobotoMono-Light.ttf new file mode 100644 index 00000000000..4893662d796 Binary files /dev/null and b/res/gamedata/fonts/rus/RobotoMono-Light.ttf differ diff --git a/res/gamedata/fonts/rus/RobotoMono-Medium.ttf b/res/gamedata/fonts/rus/RobotoMono-Medium.ttf new file mode 100644 index 00000000000..f6c149a2035 Binary files /dev/null and b/res/gamedata/fonts/rus/RobotoMono-Medium.ttf differ diff --git a/res/gamedata/fonts/rus/RobotoMono.ttf b/res/gamedata/fonts/rus/RobotoMono.ttf new file mode 100644 index 00000000000..6df2b253603 Binary files /dev/null and b/res/gamedata/fonts/rus/RobotoMono.ttf differ diff --git a/res/gamedata/fonts/rus/arial.ttf b/res/gamedata/fonts/rus/arial.ttf new file mode 100644 index 00000000000..8682d94623e Binary files /dev/null and b/res/gamedata/fonts/rus/arial.ttf differ diff --git a/res/gamedata/fonts/rus/arialbd.ttf b/res/gamedata/fonts/rus/arialbd.ttf new file mode 100644 index 00000000000..a6037e6865d Binary files /dev/null and b/res/gamedata/fonts/rus/arialbd.ttf differ diff --git a/res/gamedata/fonts/rus/arialbi.ttf b/res/gamedata/fonts/rus/arialbi.ttf new file mode 100644 index 00000000000..6a1fa0fa035 Binary files /dev/null and b/res/gamedata/fonts/rus/arialbi.ttf differ diff --git a/res/gamedata/fonts/rus/ariali.ttf b/res/gamedata/fonts/rus/ariali.ttf new file mode 100644 index 00000000000..380199780fc Binary files /dev/null and b/res/gamedata/fonts/rus/ariali.ttf differ diff --git a/res/gamedata/fonts/rus/ariblk.ttf b/res/gamedata/fonts/rus/ariblk.ttf new file mode 100644 index 00000000000..e7ae345aa00 Binary files /dev/null and b/res/gamedata/fonts/rus/ariblk.ttf differ diff --git a/res/gamedata/fonts/rus/consola.ttf b/res/gamedata/fonts/rus/consola.ttf new file mode 100644 index 00000000000..e881ca4b59f Binary files /dev/null and b/res/gamedata/fonts/rus/consola.ttf differ diff --git a/res/gamedata/fonts/rus/consolab.ttf b/res/gamedata/fonts/rus/consolab.ttf new file mode 100644 index 00000000000..77f5d6052e1 Binary files /dev/null and b/res/gamedata/fonts/rus/consolab.ttf differ diff --git a/res/gamedata/fonts/rus/consolai.ttf b/res/gamedata/fonts/rus/consolai.ttf new file mode 100644 index 00000000000..2de4de8a990 Binary files /dev/null and b/res/gamedata/fonts/rus/consolai.ttf differ diff --git a/res/gamedata/fonts/rus/consolaz.ttf b/res/gamedata/fonts/rus/consolaz.ttf new file mode 100644 index 00000000000..d9df2110fee Binary files /dev/null and b/res/gamedata/fonts/rus/consolaz.ttf differ diff --git a/res/gamedata/fonts/rus/default.ttf b/res/gamedata/fonts/rus/default.ttf new file mode 100644 index 00000000000..4be0bae1650 Binary files /dev/null and b/res/gamedata/fonts/rus/default.ttf differ diff --git a/res/gamedata/fonts/rus/robotocondensed.ttf b/res/gamedata/fonts/rus/robotocondensed.ttf new file mode 100644 index 00000000000..74f029c7c46 Binary files /dev/null and b/res/gamedata/fonts/rus/robotocondensed.ttf differ diff --git a/src/Common/FSMacros.hpp b/src/Common/FSMacros.hpp index 4aecb23accd..1951187db34 100644 --- a/src/Common/FSMacros.hpp +++ b/src/Common/FSMacros.hpp @@ -11,6 +11,7 @@ #define _game_meshes_ "$game_meshes$" #define _game_shaders_ "$game_shaders$" #define _game_config_ "$game_config$" +#define _game_fonts_ "$game_fonts$" // editor path definition #define _server_root_ "$server_root$" diff --git a/src/Include/xrRender/FontRender.h b/src/Include/xrRender/FontRender.h index a2dc9d5c31b..a7f2868c0b7 100644 --- a/src/Include/xrRender/FontRender.h +++ b/src/Include/xrRender/FontRender.h @@ -10,6 +10,7 @@ class IFontRender virtual ~IFontRender() {} virtual void Initialize(cpcstr cShader, cpcstr cTexture) = 0; virtual void OnRender(CGameFont& owner) = 0; + virtual void CreateFontAtlas(u32 width, u32 height, const char* name, void* bitmap) {} }; #endif // FontRender_included diff --git a/src/Layers/xrRender/dxFontRender.cpp b/src/Layers/xrRender/dxFontRender.cpp index 5db8321b3fa..5a22e38a2b7 100644 --- a/src/Layers/xrRender/dxFontRender.cpp +++ b/src/Layers/xrRender/dxFontRender.cpp @@ -1,12 +1,9 @@ #include "stdafx.h" - #include "dxFontRender.h" - #include "xrEngine/GameFont.h" -#include "xrCore/Text/StringConversion.hpp" +#include "xrCore/Text/Utf8Utils.hpp" extern ENGINE_API bool g_bRendering; -extern ENGINE_API Fvector2 g_current_font_scale; namespace xray::render::RENDER_NAMESPACE { @@ -14,10 +11,14 @@ dxFontRender::~dxFontRender() { pShader.destroy(); pGeom.destroy(); + pTexture.destroy(); } void dxFontRender::Initialize(cpcstr cShader, cpcstr cTexture) { + if (pTexture._get() == nullptr) + pTexture.create(cTexture); + pShader.create(cShader, cTexture); pGeom.create(FVF::F_TL, RImplementation.Vertex.Buffer(), RImplementation.QuadIB); } @@ -25,186 +26,236 @@ void dxFontRender::Initialize(cpcstr cShader, cpcstr cTexture) void dxFontRender::OnRender(CGameFont& owner) { VERIFY(g_bRendering); - if (pShader) - RCache.set_Shader(pShader); - if (!(owner.uFlags & CGameFont::fsValid)) - { - R_ASSERT(pShader); - R_constant* C = RCache.get_c(c_sbase)._get(); // get sampler - CTexture* T = RCache.get_ActiveTexture(C ? C->samp.index : 0); - R_ASSERT(T); - owner.vTS.set((int)T->get_Width(), (int)T->get_Height()); - owner.fTCHeight = owner.fHeight / float(owner.vTS.y); - owner.uFlags |= CGameFont::fsValid; - } + // Guard against an uninitialized texture (e.g. a legacy font whose .dds/.ini + // failed to load). Skip rendering rather than crash on a null dereference. + if (pTexture == nullptr || pShader == nullptr) + return; - for (u32 i = 0; i < owner.strings.size();) - { - // calculate first-fit - int count = 1; + RCache.set_Shader(pShader); - u32 length = owner.smart_strlen(owner.strings[i].string); - auto [actionsCount, actionsLength] = owner.get_actions_text_length(owner.strings[i].string); - length += actionsLength - actionsCount * 2; + auto fWidth = (float)std::max(m_atlasWidth ? m_atlasWidth : pTexture->get_Width(), 4u); + auto fHeight = (float)std::max(m_atlasHeight ? m_atlasHeight : pTexture->get_Height(), 4u); - while ((i + count) < owner.strings.size()) + for (CGameFont::String& str : owner.strings) + { + if (str.string[0]) { - u32 L = owner.smart_strlen(owner.strings[i + count].string); - auto [aC, aL] = owner.get_actions_text_length(owner.strings[i].string); - L += aL - aC * 2; + const bool isUtf8 = XRay::Utf8::IsValid(str.string); + const char* text = isUtf8 ? str.string : str.string_utf8.c_str(); - if ((L + length) < MAX_MB_CHARS) + // Single pass over the text to count codepoints and (when needed for + // alignment) accumulate the pixel width. This avoids a separate + // full-string scan in owner.WidthOf(text) below. + int codepointCount = 0; + float textWidth = 0.0f; + const bool needWidth = (str.align == CGameFont::alCenter || str.align == CGameFont::alRight); + if (needWidth) { - count++; - length += L; + const float spacing = owner.GetLetterSpacing(); + for (const char* p = text; *p;) + { + size_t cpLen = 0; + const int cp = (int)XRay::Utf8::Decode(p, cpLen); + p += cpLen ? cpLen : 1; + ++codepointCount; + textWidth += owner.WidthOf(cp) + spacing; + } + if (codepointCount > 0) + textWidth -= spacing; } else - break; - } - - // lock AGP memory - u32 vOffset; - FVF::TL* v = (FVF::TL*)RImplementation.Vertex.Lock(length * 4, pGeom.stride(), vOffset); - FVF::TL* start = v; + { + codepointCount = (int)XRay::Utf8::LengthCodepoints(text); + } - // fill vertices - u32 last = i + count; - for (; i < last; i++) - { - CGameFont::String& PS = owner.strings[i]; - xr_wide_char wsStr[MAX_MB_CHARS]; + // lock AGP memory + u32 vOffset; + FVF::TL* vertexes = (FVF::TL*)RImplementation.Vertex.Lock(codepointCount * 4, pGeom.stride(), vOffset); + FVF::TL* start = vertexes; - const u16 len = owner.IsMultibyte() ? mbhMulti2Wide(wsStr, nullptr, MAX_MB_CHARS, PS.string) : xr_strlen(PS.string); + float X = float(iFloor(str.x)); + float Y = float(iFloor(str.y)); + float Y2 = Y + str.height; - if (len) + if (str.align) { - float X = float(iFloor(PS.x)); - float Y = float(iFloor(PS.y)); - float S = PS.height * g_current_font_scale.y; - float Y2 = Y + S; - float fSize = 0; - - if (PS.align) - fSize = owner.IsMultibyte() ? owner.SizeOf_(wsStr) : owner.SizeOf_(PS.string); + const float width = needWidth ? textWidth : (float)owner.WidthOf(text); - switch (PS.align) + switch (str.align) { - case CGameFont::alCenter: X -= (iFloor(fSize * 0.5f)) * g_current_font_scale.x; break; - case CGameFont::alRight: X -= iFloor(fSize); break; + case CGameFont::alCenter: + X -= iFloor(width * 0.5f); + break; + case CGameFont::alRight: + X -= iFloor(width); + break; } + } - const u32 clr = PS.c; - u32 clr2 = PS.c; + u32 clr, clr2; + clr2 = clr = str.c; + if (str.gradient) + clr2 = str.gradientColor; - if (owner.uFlags & CGameFont::fsGradient) - { - const u32 r = color_get_R(clr) / 2; - const u32 g = color_get_G(clr) / 2; - const u32 b = color_get_B(clr) / 2; - const u32 a = color_get_A(clr); - clr2 = color_rgba(r, g, b, a); - } + X -= 0.5f; + Y -= 0.5f; + Y2 -= 0.5f; + + bool firstGlyph = true; + // Apply the global text scale to per-glyph metrics here too, so the + // rendered advance/offset matches what WidthOf() reported for layout + // and alignment. Without this, alRight/alCenter text drifts to the + // right and glyph baselines look uneven when scale != 1.0. + const float scale = g_text_scale; + for (const char* p = text; *p;) + { + size_t cpLen = 0; + const int cp = (int)XRay::Utf8::Decode(p, cpLen); + p += cpLen ? cpLen : 1; + + const CGameFont::Glyph* glyphInfo = owner.GetGlyphInfo(cp); + if (glyphInfo == nullptr) + continue; + + if (!firstGlyph) + X += glyphInfo->Abc.abcA * scale; + firstGlyph = false; + + float GlyphY = Y + glyphInfo->yOffset * scale; + float GlyphY2 = Y2 + glyphInfo->yOffset * scale; -#ifndef USE_DX9 // Vertex shader will cancel a DX9 correction, so make fake offset - X -= 0.5f; - Y -= 0.5f; - Y2 -= 0.5f; -#endif // !USE_DX9 + float X2 = X + glyphInfo->Abc.abcB * scale; - for (u16 j = 0; j < len; j++) + float u1 = float(glyphInfo->TextureCoord.left) / fWidth; + float u2 = float(glyphInfo->TextureCoord.right) / fWidth; + + float v1 = float(glyphInfo->TextureCoord.top) / fHeight; + float v2 = float(glyphInfo->TextureCoord.bottom) / fHeight; + + if (str.gradientMode == CGameFont::gm_horz) + { + vertexes->set(X, GlyphY2, clr, u1, v2); + ++vertexes; + vertexes->set(X, GlyphY, clr, u1, v1); + ++vertexes; + vertexes->set(X2, GlyphY2, clr2, u2, v2); + ++vertexes; + vertexes->set(X2, GlyphY, clr2, u2, v1); + ++vertexes; + } + else if (str.gradientMode == CGameFont::gm_back) + { + vertexes->set(X, GlyphY2, clr2, u1, v2); + ++vertexes; + vertexes->set(X, GlyphY, clr2, u1, v1); + ++vertexes; + vertexes->set(X2, GlyphY2, clr, u2, v2); + ++vertexes; + vertexes->set(X2, GlyphY, clr, u2, v1); + ++vertexes; + } + else if (str.gradientMode == CGameFont::gm_down) { - if (owner.IsMultibyte()) - { - if (wsStr[1 + j] == GAME_ACTION_MARK) - { - static_assert(kLASTACTION < type_max, "Modify the code to have more than 255 actions."); - ++j; - const EGameActions actionId = static_cast(wsStr[1 + j]); - - cpcstr binding = GetActionBinding(actionId); - - xr_wide_char wideBinding[MAX_MB_CHARS]; - const auto bindingLen = mbhMulti2Wide(wideBinding, nullptr, MAX_MB_CHARS, binding); - - for (size_t k = 0; k < bindingLen; ++k) - { - const Fvector l = owner.GetCharTC(wideBinding[1 + k]); - ImprintChar(l, owner, v, X, Y2, clr2, Y, clr, wsStr, j); - } - } - else - { - const Fvector l = owner.GetCharTC(wsStr[1 + j]); - ImprintChar(l, owner, v, X, Y2, clr2, Y, clr, wsStr, j); - } - } - else - { - if (PS.string[j] == GAME_ACTION_MARK) - { - static_assert(kLASTACTION < type_max, "Modify the code to have more than 255 actions."); - ++j; - const EGameActions actionId = static_cast(PS.string[j]); - - pcstr binding = GetActionBinding(actionId); - - while (binding[0]) - { - const Fvector l = owner.GetCharTC((u16)(u8)binding[0]); - ImprintChar(l, owner, v, X, Y2, clr2, Y, clr, wsStr, j); - ++binding; - } - } - else - { - const Fvector l = owner.GetCharTC((u16)(u8)PS.string[j]); - ImprintChar(l, owner, v, X, Y2, clr2, Y, clr, wsStr, j); - } - } + vertexes->set(X, GlyphY2, clr, u1, v2); + ++vertexes; + vertexes->set(X, GlyphY, clr2, u1, v1); + ++vertexes; + vertexes->set(X2, GlyphY2, clr, u2, v2); + ++vertexes; + vertexes->set(X2, GlyphY, clr2, u2, v1); + ++vertexes; } + else + { + vertexes->set(X, GlyphY2, clr2, u1, v2); + ++vertexes; + vertexes->set(X, GlyphY, clr, u1, v1); + ++vertexes; + vertexes->set(X2, GlyphY2, clr2, u2, v2); + ++vertexes; + vertexes->set(X2, GlyphY, clr2, u2, v1); + ++vertexes; + } + X = X2 + glyphInfo->Abc.abcC * scale + owner.GetLetterSpacing() * scale; } - } - // Unlock and draw - u32 vCount = (u32)(v - start); - RImplementation.Vertex.Unlock(vCount, pGeom.stride()); - if (vCount) - { - RCache.set_Geometry(pGeom); - RCache.Render(D3DPT_TRIANGLELIST, vOffset, 0, vCount, 0, vCount / 2); + // Unlock and draw + u32 vertexesCount = (u32)(vertexes - start); + RImplementation.Vertex.Unlock(vertexesCount, pGeom.stride()); + + if (vertexesCount > 0) + { + RCache.set_Geometry(pGeom); + RCache.Render(D3DPT_TRIANGLELIST, vOffset, 0, vertexesCount, 0, vertexesCount / 2); + } } } } -inline void dxFontRender::ImprintChar(Fvector l, const CGameFont& owner, FVF::TL*& v, float& X, float Y2, u32 clr2, float Y, u32 clr, xr_wide_char* wsStr, int j) +void dxFontRender::CreateFontAtlas(u32 width, u32 height, const char* name, void* bitmap) { - float scw = l.z * g_current_font_scale.x; + m_atlasWidth = width; + m_atlasHeight = height; + Msg("* Font atlas '%s' created: %dx%d", name, width, height); - float fTCWidth = l.z / owner.vTS.x; +#if !defined(USE_OGL) + ID3DTexture2D* pSurface = nullptr; +#endif + ZoneScoped; - if (!fis_zero(l.z)) - { - //float tu = (l.x / owner.vTS.x) + (0.5f / owner.vTS.x); - //float tv = (l.y / owner.vTS.y) + (0.5f / owner.vTS.y); - float tu = (l.x / owner.vTS.x); - float tv = (l.y / owner.vTS.y); - - v->set(X, Y2, clr2, tu, tv + owner.fTCHeight); - v++; - v->set(X, Y, clr, tu, tv); - v++; - v->set(X + scw, Y2, clr2, tu + fTCWidth, tv + owner.fTCHeight); - v++; - v->set(X + scw, Y, clr, tu + fTCWidth, tv); - v++; - } - X += scw * owner.vInterval.x; - if (owner.IsMultibyte()) - { - X -= 2; - if (IsNeedSpaceCharacter(wsStr[1 + j])) - X += owner.fXStep; - } +#if defined(USE_DX11) + D3D_TEXTURE2D_DESC descFontAtlas; + ZeroMemory(&descFontAtlas, sizeof(D3D_TEXTURE2D_DESC)); + descFontAtlas.Width = width; + descFontAtlas.Height = height; + descFontAtlas.MipLevels = 1; + descFontAtlas.ArraySize = 1; + descFontAtlas.SampleDesc.Count = 1; + descFontAtlas.SampleDesc.Quality = 0; + descFontAtlas.Format = DXGI_FORMAT_B8G8R8A8_UNORM; + descFontAtlas.Usage = D3D_USAGE_IMMUTABLE; + descFontAtlas.BindFlags = D3D11_BIND_SHADER_RESOURCE; + descFontAtlas.CPUAccessFlags = 0; + descFontAtlas.MiscFlags = 0; + + D3D_SUBRESOURCE_DATA FontData; + FontData.pSysMem = bitmap; + FontData.SysMemSlicePitch = 0; + FontData.SysMemPitch = width * 4; + + R_CHK(HW.pDevice->CreateTexture2D(&descFontAtlas, &FontData, &pSurface)); +#elif defined(USE_OGL) + const GLuint oldTextureID = static_cast(m_oglTextureID); + if (oldTextureID != 0) + glDeleteTextures(1, &oldTextureID); + GLuint pTextureID = 0; + glGenTextures(1, &pTextureID); + m_oglTextureID = static_cast(pTextureID); + glBindTexture(GL_TEXTURE_2D, pTextureID); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_LINEAR); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_S, GL_CLAMP_TO_EDGE); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_WRAP_T, GL_CLAMP_TO_EDGE); + glTexImage2D(GL_TEXTURE_2D, 0, GL_RGBA8, width, height, 0, GL_BGRA, GL_UNSIGNED_BYTE, bitmap); + glBindTexture(GL_TEXTURE_2D, 0); +#else + D3DLOCKED_RECT LockedRect = {}; + R_CHK(HW.pDevice->CreateTexture(width, height, 1, 0, D3DFMT_A8R8G8B8, D3DPOOL_MANAGED, &pSurface, nullptr)); + R_CHK(pSurface->LockRect(0, &LockedRect, nullptr, 0)); + + memcpy(LockedRect.pBits, bitmap, width * height * 4); + + R_CHK(pSurface->UnlockRect(0)); +#endif + + pTexture.create(name); + +#if defined(USE_OGL) + pTexture->surface_set(GL_TEXTURE_2D, pTextureID); +#else + pTexture->surface_set(pSurface); + _RELEASE(pSurface); +#endif } } // namespace xray::render::RENDER_NAMESPACE diff --git a/src/Layers/xrRender/dxFontRender.h b/src/Layers/xrRender/dxFontRender.h index 28747a60d0f..ee4444e7780 100644 --- a/src/Layers/xrRender/dxFontRender.h +++ b/src/Layers/xrRender/dxFontRender.h @@ -1,7 +1,6 @@ #pragma once #include "Include/xrRender/FontRender.h" - #include "xrEngine/GameFont.h" namespace xray::render::RENDER_NAMESPACE @@ -14,12 +13,15 @@ class dxFontRender : public IFontRender void Initialize(cpcstr cShader, cpcstr cTexture) override; void OnRender(CGameFont& owner) override; - -private: - inline void ImprintChar(Fvector l, const CGameFont& owner, FVF::TL*& v, float& X, float Y2, u32 clr2, float Y, u32 clr, xr_wide_char* wsStr, int j); + void CreateFontAtlas(u32 width, u32 height, const char* name, void* bitmap) override; private: ref_shader pShader; ref_geom pGeom; + ref_texture pTexture; + + u32 m_atlasWidth = 0; + u32 m_atlasHeight = 0; + u32 m_oglTextureID = 0; }; } // namespace xray::render::RENDER_NAMESPACE diff --git a/src/Layers/xrRenderPC_GL/CMakeLists.txt b/src/Layers/xrRenderPC_GL/CMakeLists.txt index 2f333dcafcc..8dfbb0f2469 100644 --- a/src/Layers/xrRenderPC_GL/CMakeLists.txt +++ b/src/Layers/xrRenderPC_GL/CMakeLists.txt @@ -412,6 +412,7 @@ target_link_libraries(xrRender_GL xrParticles xrScriptEngine xrImGui + freetype ) target_compile_definitions(xrRender_GL diff --git a/src/Layers/xrRenderPC_GL/xrRender_GL.vcxproj b/src/Layers/xrRenderPC_GL/xrRender_GL.vcxproj index dbdba4f3809..175e26af24b 100644 --- a/src/Layers/xrRenderPC_GL/xrRender_GL.vcxproj +++ b/src/Layers/xrRenderPC_GL/xrRender_GL.vcxproj @@ -25,7 +25,7 @@ - $(SolutionDir)Layers\xrRender_R2;$(xrSdkDir)include\DirectXMesh;$(xrExternals)imgui;$(xrExternals)OpenAutomate\inc;$(xrExternals)gli;$(xrExternals)gli\external;$(xrExternals)AGS_SDK;$(xrExternals)glslang\glslang\Public;$(xrExternals)sse2neon\include;%(AdditionalIncludeDirectories) + $(SolutionDir)Layers\xrRender_R2;$(xrSdkDir)include\DirectXMesh;$(xrExternals)imgui;$(xrExternals)OpenAutomate\inc;$(xrExternals)gli;$(xrExternals)gli\external;$(xrExternals)AGS_SDK;$(xrExternals)glslang\glslang\Public;$(xrExternals)sse2neon\include;$(xrExternals)freetype\include;%(AdditionalIncludeDirectories) _USRDLL;XRRENDER_GL_EXPORTS;USE_OGL;RENDER_NAMESPACE=render_gl;%(PreprocessorDefinitions) /bigobj %(AdditionalOptions) @@ -405,6 +405,9 @@ {c8fbc3ce-d6de-4fc3-bc15-7b647614db09} + + {78b079bd-9fc7-4b9e-b4a6-96da0f00248b} + {2c419512-6eee-4707-bc51-2e834855552e} diff --git a/src/Layers/xrRenderPC_R4/CMakeLists.txt b/src/Layers/xrRenderPC_R4/CMakeLists.txt index 12e2072b7bf..b502b0d6dc5 100644 --- a/src/Layers/xrRenderPC_R4/CMakeLists.txt +++ b/src/Layers/xrRenderPC_R4/CMakeLists.txt @@ -32,6 +32,7 @@ target_link_libraries(xrRenderPC_R4 xrScriptEngine xrAPI xrMiscMath + freetype ) target_compile_definitions(xrRenderPC_R4 diff --git a/src/Layers/xrRenderPC_R4/xrRender_R4.vcxproj b/src/Layers/xrRenderPC_R4/xrRender_R4.vcxproj index 60c99d142ba..45fa987b8ca 100644 --- a/src/Layers/xrRenderPC_R4/xrRender_R4.vcxproj +++ b/src/Layers/xrRenderPC_R4/xrRender_R4.vcxproj @@ -25,7 +25,7 @@ - $(SolutionDir)Layers\xrRender_R2;$(xrExternals)imgui;$(xrExternals)OpenAutomate\inc;$(xrSdkDir)include\nvapi;$(xrSdkDir)include\DirectXMesh;$(xrExternals)AGS_SDK;%(AdditionalIncludeDirectories) + $(SolutionDir)Layers\xrRender_R2;$(xrExternals)imgui;$(xrExternals)OpenAutomate\inc;$(xrSdkDir)include\nvapi;$(xrSdkDir)include\DirectXMesh;$(xrExternals)AGS_SDK;$(xrExternals)freetype\include;%(AdditionalIncludeDirectories) _USRDLL;XRRENDER_R4_EXPORTS;USE_DX11;RENDER_NAMESPACE=render_r4;%(PreprocessorDefinitions) /bigobj %(AdditionalOptions) @@ -462,6 +462,9 @@ {c8fbc3ce-d6de-4fc3-bc15-7b647614db09} + + {78b079bd-9fc7-4b9e-b4a6-96da0f00248b} + {2c419512-6eee-4707-bc51-2e834855552e} diff --git a/src/engine.sln b/src/engine.sln index b69bc97976b..5329d4513b2 100644 --- a/src/engine.sln +++ b/src/engine.sln @@ -84,6 +84,8 @@ EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xrWeatherEngine", "editors\xrWeatherEngine\xrWeatherEngine.vcxproj", "{FABD5BBF-EA6F-4633-A34F-95CDC553CA36}" EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "imgui", "..\Externals\imgui.vcxproj", "{C8FBC3CE-D6DE-4FC3-BC15-7B647614DB09}" +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "freetype", "..\Externals\freetype.vcxproj", "{78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}" +EndProject EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "xrLuaFix", "..\Externals\xrLuaFix\xrLuaFix.vcxproj", "{4E076FEB-DC78-4BF7-99C7-AC8267247989}" EndProject @@ -1039,6 +1041,38 @@ Global {C8FBC3CE-D6DE-4FC3-BC15-7B647614DB09}.Release|x64.Build.0 = Release|x64 {C8FBC3CE-D6DE-4FC3-BC15-7B647614DB09}.Release|x86.ActiveCfg = Release|Win32 {C8FBC3CE-D6DE-4FC3-BC15-7B647614DB09}.Release|x86.Build.0 = Release|Win32 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Debug|ARM.ActiveCfg = Debug|ARM + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Debug|ARM.Build.0 = Debug|ARM + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Debug|ARM64.ActiveCfg = Debug|ARM64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Debug|ARM64.Build.0 = Debug|ARM64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Debug|x64.ActiveCfg = Debug|x64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Debug|x64.Build.0 = Debug|x64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Debug|x86.ActiveCfg = Debug|Win32 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Debug|x86.Build.0 = Debug|Win32 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Mixed|ARM.ActiveCfg = Mixed|ARM + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Mixed|ARM.Build.0 = Mixed|ARM + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Mixed|ARM64.ActiveCfg = Mixed|ARM64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Mixed|ARM64.Build.0 = Mixed|ARM64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Mixed|x64.ActiveCfg = Mixed|x64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Mixed|x64.Build.0 = Mixed|x64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Mixed|x86.ActiveCfg = Mixed|Win32 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Mixed|x86.Build.0 = Mixed|Win32 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release Master Gold|ARM.ActiveCfg = Release Master Gold|ARM + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release Master Gold|ARM.Build.0 = Release Master Gold|ARM + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release Master Gold|ARM64.ActiveCfg = Release Master Gold|ARM64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release Master Gold|ARM64.Build.0 = Release Master Gold|ARM64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release Master Gold|x64.ActiveCfg = Release Master Gold|x64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release Master Gold|x64.Build.0 = Release Master Gold|x64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release Master Gold|x86.ActiveCfg = Release Master Gold|Win32 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release Master Gold|x86.Build.0 = Release Master Gold|Win32 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release|ARM.ActiveCfg = Release|ARM + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release|ARM.Build.0 = Release|ARM + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release|ARM64.ActiveCfg = Release|ARM64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release|ARM64.Build.0 = Release|ARM64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release|x64.ActiveCfg = Release|x64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release|x64.Build.0 = Release|x64 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release|x86.ActiveCfg = Release|Win32 + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B}.Release|x86.Build.0 = Release|Win32 {4E076FEB-DC78-4BF7-99C7-AC8267247989}.Debug|ARM.ActiveCfg = Debug|ARM {4E076FEB-DC78-4BF7-99C7-AC8267247989}.Debug|ARM.Build.0 = Debug|ARM {4E076FEB-DC78-4BF7-99C7-AC8267247989}.Debug|ARM64.ActiveCfg = Debug|ARM64 @@ -1155,6 +1189,7 @@ Global {DAFD6D98-3DCD-40AD-BC85-EC71797B6767} = {2BFC806B-CE92-4EA4-8FE8-5F2EA54BA348} {FABD5BBF-EA6F-4633-A34F-95CDC553CA36} = {CB0C814D-FB4E-453B-B7A0-716F4A1EACA4} {C8FBC3CE-D6DE-4FC3-BC15-7B647614DB09} = {2BFC806B-CE92-4EA4-8FE8-5F2EA54BA348} + {78B079BD-9FC7-4B9E-B4A6-96DA0F00248B} = {2BFC806B-CE92-4EA4-8FE8-5F2EA54BA348} {4E076FEB-DC78-4BF7-99C7-AC8267247989} = {49438080-78B8-4056-BDCB-4DACAD652C21} {CFF9F0DD-C2FC-424C-800B-BFBA35003932} = {2BFC806B-CE92-4EA4-8FE8-5F2EA54BA348} EndGlobalSection diff --git a/src/xrCore/CMakeLists.txt b/src/xrCore/CMakeLists.txt index 9a1741bee04..e7c4860c59e 100644 --- a/src/xrCore/CMakeLists.txt +++ b/src/xrCore/CMakeLists.txt @@ -394,6 +394,8 @@ target_sources_grouped( FILES Text/StringConversion.cpp Text/StringConversion.hpp + Text/Utf8Utils.cpp + Text/Utf8Utils.hpp ) target_sources_grouped( diff --git a/src/xrCore/FS.cpp b/src/xrCore/FS.cpp index af923d279d8..eb7fbfd85c2 100644 --- a/src/xrCore/FS.cpp +++ b/src/xrCore/FS.cpp @@ -10,6 +10,7 @@ #elif defined(XR_PLATFORM_POSIX) #include #endif +#include "xrCore/Text/Utf8Utils.hpp" #include #include @@ -80,14 +81,19 @@ void VerifyPath(pcstr path) CopyMemory(tmp, path, i); tmp[i] = 0; convert_path_separators(tmp); - _mkdir(tmp); +#if defined(XR_PLATFORM_WINDOWS) + _wmkdir(XRay::Utf8::ToWide(tmp).c_str()); +#else + mkdir(tmp, 0777); +#endif } } static int open_internal(pcstr fn, int& handle) { #if defined(XR_PLATFORM_WINDOWS) - return (_sopen_s(&handle, fn, _O_RDONLY | _O_BINARY, _SH_DENYNO, _S_IREAD)); + const std::wstring wfn = XRay::Utf8::ToWide(fn); + return (_wsopen_s(&handle, wfn.c_str(), _O_RDONLY | _O_BINARY, _SH_DENYNO, _S_IREAD)); #elif defined(XR_PLATFORM_POSIX) pstr conv_fn = xr_strdup(fn); convert_path_separators(conv_fn); @@ -142,7 +148,12 @@ void FileCompress(pcstr fn, pcstr sign, void* data, size_t size) MARK M; mk_mark(M, sign); +#if defined(XR_PLATFORM_WINDOWS) + const std::wstring wfn = XRay::Utf8::ToWide(fn); + int H = _wopen(wfn.c_str(), O_BINARY | O_CREAT | O_WRONLY | O_TRUNC, S_IREAD | S_IWRITE); +#else int H = _open(fn, O_BINARY | O_CREAT | O_WRONLY | O_TRUNC, S_IREAD | S_IWRITE); +#endif R_ASSERT2(H > 0, fn); std::ignore = _write(H, &M, 8); _writeLZ(H, data, size); @@ -154,7 +165,12 @@ void* FileDecompress(pcstr fn, pcstr sign, size_t* size) MARK M, F; mk_mark(M, sign); +#if defined(XR_PLATFORM_WINDOWS) + const std::wstring wfn = XRay::Utf8::ToWide(fn); + int H = _wopen(wfn.c_str(), O_BINARY | O_RDONLY); +#else int H = _open(fn, O_BINARY | O_RDONLY); +#endif R_ASSERT2(H > 0, fn); std::ignore = _read(H, &F, 8); if (strncmp(M, F, 8) != 0) @@ -489,6 +505,90 @@ CFileReader::CFileReader(pcstr name) }; CFileReader::~CFileReader() { xr_free(data); }; //--------------------------------------------------- +// virtual stream (memory-mapped), optional open without fatal assert +#if defined(XR_PLATFORM_WINDOWS) +CVirtualFileReader::CVirtualFileReader(void* hf, void* hm, char* mapped, size_t sz, pcstr dbgName) + : IReader(mapped, sz), hSrcFile(hf), hSrcMap(hm) +{ +#ifdef FS_DEBUG + register_file_mapping(data, Size, dbgName); +#endif +} +#elif defined(XR_PLATFORM_POSIX) +CVirtualFileReader::CVirtualFileReader(int fd, char* mapped, size_t sz, pcstr dbgName) + : IReader(mapped, sz) +{ + hSrcFile = fd; +#ifdef FS_DEBUG + register_file_mapping(data, Size, dbgName); +#endif +} +#endif + +CVirtualFileReader* CVirtualFileReader::TryOpen(pcstr cFileName) +{ +#if defined(XR_PLATFORM_WINDOWS) + const std::wstring wFileName = XRay::Utf8::ToWide(cFileName); + void* const hFile = + (void*)CreateFileW(wFileName.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, 0, 0); + if (hFile == (void*)INVALID_HANDLE_VALUE) + { + Msg("! CVirtualFileReader::TryOpen: CreateFile failed [%s] err=%s", cFileName, + xrDebug::ErrorToString(static_cast(GetLastError()))); + return nullptr; + } + LARGE_INTEGER size{}; + if (!GetFileSizeEx((HANDLE)hFile, &size) || size.QuadPart == 0) + { + Msg("! CVirtualFileReader::TryOpen: bad size [%s]", cFileName); + CloseHandle((HANDLE)hFile); + return nullptr; + } + void* const hMap = (void*)CreateFileMapping((HANDLE)hFile, 0, PAGE_READONLY, 0, 0, 0); + if (hMap == (void*)INVALID_HANDLE_VALUE) + { + CloseHandle((HANDLE)hFile); + return nullptr; + } + char* mapped = (char*)MapViewOfFile((HANDLE)hMap, FILE_MAP_READ, 0, 0, 0); + if (!mapped) + { + CloseHandle((HANDLE)hMap); + CloseHandle((HANDLE)hFile); + return nullptr; + } + // Private ctor: cannot use xr_new(...) from header (access from non-friend template). + void* const mem = Memory.mem_alloc(sizeof(CVirtualFileReader)); + return new (mem) CVirtualFileReader(hFile, hMap, mapped, static_cast(size.QuadPart), cFileName); +#elif defined(XR_PLATFORM_POSIX) + pstr conv_path = xr_strdup(cFileName); + if (!conv_path) + return nullptr; + convert_path_separators(conv_path); + const int fd = ::open(conv_path, O_RDONLY); + xr_free(conv_path); + if (fd == -1) + return nullptr; + struct stat file_info{}; + if (::fstat(fd, &file_info) != 0 || file_info.st_size <= 0) + { + ::close(fd); + return nullptr; + } + const size_t sz = static_cast(file_info.st_size); + char* data = (char*)::mmap(NULL, sz, PROT_READ, MAP_SHARED, fd, 0); + if (!data || data == MAP_FAILED) + { + ::close(fd); + return nullptr; + } + void* const mem = Memory.mem_alloc(sizeof(CVirtualFileReader)); + return new (mem) CVirtualFileReader(fd, data, sz, cFileName); +#else +# error Select or add implementation for your platform +#endif +} +//--------------------------------------------------- // compressed stream CCompressedReader::CCompressedReader(const char* name, const char* sign) { @@ -500,7 +600,8 @@ CVirtualFileRW::CVirtualFileRW(pcstr cFileName) { #if defined(XR_PLATFORM_WINDOWS) // Open the file - hSrcFile = CreateFile(cFileName, GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, 0, OPEN_EXISTING, 0, 0); + const std::wstring wFileName = XRay::Utf8::ToWide(cFileName); + hSrcFile = CreateFileW(wFileName.c_str(), GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, 0, OPEN_EXISTING, 0, 0); R_ASSERT3(hSrcFile != INVALID_HANDLE_VALUE, cFileName, xrDebug::ErrorToString(GetLastError())); LARGE_INTEGER size; @@ -555,7 +656,8 @@ CVirtualFileReader::CVirtualFileReader(pcstr cFileName) { #if defined(XR_PLATFORM_WINDOWS) // Open the file - hSrcFile = CreateFile(cFileName, GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, 0, 0); + const std::wstring wFileName = XRay::Utf8::ToWide(cFileName); + hSrcFile = CreateFileW(wFileName.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, 0, 0); R_ASSERT3(hSrcFile != INVALID_HANDLE_VALUE, cFileName, xrDebug::ErrorToString(GetLastError())); LARGE_INTEGER size; diff --git a/src/xrCore/FS_internal.h b/src/xrCore/FS_internal.h index 8f325515ccf..50638eed415 100644 --- a/src/xrCore/FS_internal.h +++ b/src/xrCore/FS_internal.h @@ -10,6 +10,10 @@ #if defined(XR_PLATFORM_WINDOWS) #include #include +#include +#include "xrCore/Text/Utf8Utils.hpp" + +inline std::wstring Utf8ToWidePath(pcstr path) { return XRay::Utf8::ToWide(path); } #endif #if defined(XR_PLATFORM_BSD) @@ -33,6 +37,41 @@ class CFileWriter final : public IWriter VerifyPath(fName.c_str()); pstr conv_fn = xr_strdup(name); convert_path_separators(conv_fn); +#if defined(XR_PLATFORM_WINDOWS) + const std::wstring wide_fn = Utf8ToWidePath(conv_fn); + const wchar_t* wide_name = wide_fn.c_str(); + if (exclusive) + { + const int handle = _wsopen(wide_name, _O_WRONLY | _O_TRUNC | _O_CREAT | _O_BINARY, SH_DENYWR); + if (handle != -1) + { + hf = _wfdopen(handle, L"wb"); + if (!hf) + { + _close(handle); + string1024 error; + xr_strerror(errno, error, sizeof(error)); + Msg("! _wfdopen failed after _wsopen: '%s'. Error: '%s'.", conv_fn, error); + } + } + else + { + string1024 error; + xr_strerror(errno, error, sizeof(error)); + Msg("! Can't create file (exclusive): '%s'. Error: '%s'.", conv_fn, error); + } + } + else + { + hf = _wfopen(wide_name, L"wb"); + if (hf == 0) + { + string1024 error; + xr_strerror(errno, error, sizeof(error)); + Msg("! Can't write file: '%s'. Error: '%s'.", conv_fn, error); + } + } +#else if (exclusive) { const int handle = _sopen(conv_fn, _O_WRONLY | _O_TRUNC | _O_CREAT | _O_BINARY, SH_DENYWR); @@ -56,6 +95,7 @@ class CFileWriter final : public IWriter Msg("! Can't write file: '%s'. Error: '%s'.", conv_fn, error); } } +#endif xr_free(conv_fn); } @@ -66,11 +106,11 @@ class CFileWriter final : public IWriter fclose(hf); // release RO attrib #if defined(XR_PLATFORM_WINDOWS) - u32 dwAttr = GetFileAttributes(fName.c_str()); + u32 dwAttr = GetFileAttributesW(XRay::Utf8::ToWide(fName.c_str()).c_str()); if ((dwAttr != u32(-1)) && (dwAttr & FILE_ATTRIBUTE_READONLY)) { dwAttr &= ~FILE_ATTRIBUTE_READONLY; - SetFileAttributes(fName.c_str(), dwAttr); + SetFileAttributesW(XRay::Utf8::ToWide(fName.c_str()).c_str(), dwAttr); } #endif } @@ -153,8 +193,15 @@ class CVirtualFileReader final : public IReader # error Select or add implementation for your platform #endif +#if defined(XR_PLATFORM_WINDOWS) + CVirtualFileReader(void* hf, void* hm, char* mapped, size_t sz, pcstr dbgName); +#elif defined(XR_PLATFORM_POSIX) + CVirtualFileReader(int fd, char* mapped, size_t sz, pcstr dbgName); +#endif + public: CVirtualFileReader(pcstr cFileName); + static CVirtualFileReader* TryOpen(pcstr cFileName); ~CVirtualFileReader() override; }; diff --git a/src/xrCore/LocatorAPI.cpp b/src/xrCore/LocatorAPI.cpp index b53178946c0..e5b0aa76ec6 100644 --- a/src/xrCore/LocatorAPI.cpp +++ b/src/xrCore/LocatorAPI.cpp @@ -9,6 +9,7 @@ #include #include #include +#include "xrCore/Text/Utf8Utils.hpp" #elif defined(XR_PLATFORM_POSIX) #include #include @@ -469,19 +470,27 @@ void CLocatorAPI::LoadArchive(archive& A, pcstr entrypoint) void CLocatorAPI::archive::open() { - struct stat file_info; #if defined(XR_PLATFORM_WINDOWS) // Open the file if (hSrcFile && hSrcMap) return; - hSrcFile = CreateFile(path.c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, 0, nullptr); + hSrcFile = CreateFileW(XRay::Utf8::ToWide(path.c_str()).c_str(), GENERIC_READ, FILE_SHARE_READ | FILE_SHARE_WRITE, nullptr, OPEN_EXISTING, 0, nullptr); R_ASSERT(hSrcFile != INVALID_HANDLE_VALUE); hSrcMap = CreateFileMapping(hSrcFile, nullptr, PAGE_READONLY, 0, 0, nullptr); R_ASSERT(hSrcMap != INVALID_HANDLE_VALUE); - stat(path.c_str(), &file_info); - modif = file_info.st_mtime; + + FILETIME ftCreate{}, ftAccess{}, ftWrite{}; + GetFileTime(hSrcFile, &ftCreate, &ftAccess, &ftWrite); + const ULARGE_INTEGER writeTime{ { ftWrite.dwLowDateTime, ftWrite.dwHighDateTime } }; + // Convert Windows FILETIME (100-ns intervals since 1601) to Unix seconds + modif = static_cast((writeTime.QuadPart - 116444736000000000ULL) / 10000000ULL); + + LARGE_INTEGER fileSize{}; + R_ASSERT(GetFileSizeEx(hSrcFile, &fileSize)); + size = static_cast(fileSize.QuadPart); #elif defined(XR_PLATFORM_POSIX) + struct stat file_info; // Open the file if (hSrcFile) return; @@ -497,10 +506,10 @@ void CLocatorAPI::archive::open() modif = file_info.st_mtim.tv_sec; # endif xr_free(conv_path); + size = file_info.st_size; #else # error Select or add implementation for your platform #endif - size = file_info.st_size; R_ASSERT(size > 0); } @@ -601,7 +610,7 @@ bool ignore_name(const char* _name) return false; } -void CLocatorAPI::ProcessOne(pcstr path, const _finddata_t& entry) +void CLocatorAPI::ProcessOne(pcstr path, const xr_finddata_t& entry) { ZoneScoped; @@ -653,7 +662,8 @@ void CLocatorAPI::ProcessOne(pcstr path, const _finddata_t& entry) bool ignore_path(pcstr _path) { #if defined(XR_PLATFORM_WINDOWS) - HANDLE h = CreateFile(_path, 0, 0, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_READONLY | FILE_FLAG_NO_BUFFERING, nullptr); + const std::wstring wpath = XRay::Utf8::ToWide(_path); + HANDLE h = CreateFileW(wpath.c_str(), 0, 0, nullptr, OPEN_EXISTING, FILE_ATTRIBUTE_READONLY | FILE_FLAG_NO_BUFFERING, nullptr); if (h != INVALID_HANDLE_VALUE) { @@ -691,13 +701,16 @@ bool CLocatorAPI::Recurse(pcstr path) return true; xr_strcpy(scanPath, sizeof scanPath, path); xr_strcat(scanPath, "*"); - _finddata_t findData; + xr_finddata_t findData; convert_path_separators(scanPath); #ifdef XR_PLATFORM_WINDOWS - intptr_t handle = _findfirst(scanPath, &findData); + const std::wstring wscanPath = XRay::Utf8::ToWide(scanPath); + _wfinddata64i32_t wfindData{}; + intptr_t handle = _wfindfirst(wscanPath.c_str(), &wfindData); if (handle == -1) return false; #elif defined(XR_PLATFORM_POSIX) + glob_t globbuf; globbuf.gl_offs = 256; @@ -717,7 +730,15 @@ bool CLocatorAPI::Recurse(pcstr path) while (done != -1) { #if defined(XR_PLATFORM_WINDOWS) - // do nothing + findData.attrib = wfindData.attrib; + findData.size = static_cast<_fsize_t>(wfindData.size); + findData.time_create = wfindData.time_create; + findData.time_access = wfindData.time_access; + findData.time_write = wfindData.time_write; + { + const xr_string utf8Name = XRay::Utf8::FromWide(wfindData.name); + xr_strcpy(findData.name, sizeof findData.name, utf8Name.c_str()); + } #elif defined(XR_PLATFORM_POSIX) xr_strcpy(findData.name, globbuf.gl_pathv[handle - done]); struct stat fi; @@ -764,7 +785,7 @@ bool CLocatorAPI::Recurse(pcstr path) if (!ignore) rec_files.push_back(findData); #ifdef XR_PLATFORM_WINDOWS - done = _findnext(handle, &findData); + done = _wfindnext(handle, &wfindData); #elif defined(XR_PLATFORM_POSIX) done--; #else @@ -781,7 +802,7 @@ bool CLocatorAPI::Recurse(pcstr path) size_t newSize = rec_files.size(); if (newSize > oldSize) { - std::sort(rec_files.begin() + oldSize, rec_files.end(), [](const _finddata_t& x, const _finddata_t& y) + std::sort(rec_files.begin() + oldSize, rec_files.end(), [](const xr_finddata_t& x, const xr_finddata_t& y) { return xr_strcmp(x.name, y.name) < 0; }); @@ -1668,13 +1689,13 @@ void CLocatorAPI::w_close(IWriter*& S) if (bReg) { #if defined(XR_PLATFORM_WINDOWS) - struct _stat st; - _stat(fname, &st); - Register(fname, VFS_STANDARD_FILE, 0, 0, st.st_size, st.st_size, (u32)st.st_mtime); + struct _stat st{}; + if (_wstat(XRay::Utf8::ToWide(fname).c_str(), &st) == 0) + Register(fname, VFS_STANDARD_FILE, 0, 0, st.st_size, st.st_size, (u32)st.st_mtime); #elif defined(XR_PLATFORM_POSIX) - struct stat st; - ::stat(fname, &st); - Register(fname, VFS_STANDARD_FILE, 0, 0, st.st_size, st.st_size, (u32)st.st_mtime); + struct stat st{}; + if (::stat(fname, &st) == 0) + Register(fname, VFS_STANDARD_FILE, 0, 0, st.st_size, st.st_size, (u32)st.st_mtime); #else # error Select or add implementation for your platform #endif @@ -1740,7 +1761,11 @@ bool CLocatorAPI::dir_delete(pcstr initial, pcstr nm, bool remove_files) const char* end_symbol = r_it->name + xr_strlen(r_it->name) - 1; if (*end_symbol == _DELIMITER) { - _rmdir(r_it->name); +#if defined(XR_PLATFORM_WINDOWS) + _wrmdir(XRay::Utf8::ToWide(r_it->name).c_str()); +#else + rmdir(r_it->name); +#endif m_files.erase(*r_it); } } @@ -1759,7 +1784,13 @@ void CLocatorAPI::file_delete(pcstr path, pcstr nm) if (I != m_files.end()) { // remove file +#if defined(XR_PLATFORM_WINDOWS) + _wunlink(XRay::Utf8::ToWide(I->name).c_str()); +#elif defined(XR_PLATFORM_POSIX) xr_unlink(I->name); +#else +# error Select or add implementation for your platform +#endif auto str = pstr(I->name); xr_free(str); m_files.erase(I); @@ -1794,7 +1825,13 @@ void CLocatorAPI::file_rename(pcstr src, pcstr dest, bool overwrite) { if (!overwrite) return; +#if defined(XR_PLATFORM_WINDOWS) + _wunlink(XRay::Utf8::ToWide(D->name).c_str()); +#elif defined(XR_PLATFORM_POSIX) xr_unlink(D->name); +#else +# error Select or add implementation for your platform +#endif auto str = pstr(D->name); xr_free(str); m_files.erase(D); @@ -1813,7 +1850,13 @@ void CLocatorAPI::file_rename(pcstr src, pcstr dest, bool overwrite) VerifyPath(dest); pstr conv_dest = xr_strdup(dest); convert_path_separators(conv_dest); +#if defined(XR_PLATFORM_WINDOWS) + _wrename(XRay::Utf8::ToWide(src).c_str(), XRay::Utf8::ToWide(conv_dest).c_str()); +#elif defined(XR_PLATFORM_POSIX) rename(src, conv_dest); +#else +# error Select or add implementation for your platform +#endif xr_free(conv_dest); } } @@ -1990,7 +2033,12 @@ bool CLocatorAPI::can_write_to_folder(pcstr path) string_path temp; pcstr fn = "$!#%TEMP%#!$.$$$"; strconcat(sizeof temp, temp, path, path[xr_strlen(path) - 1] != _DELIMITER ? DELIMITER : "", fn); +#if defined(XR_PLATFORM_WINDOWS) + const std::wstring wtemp = XRay::Utf8::ToWide(temp); + FILE* hf = _wfopen(wtemp.c_str(), L"wb"); +#else FILE* hf = fopen(temp, "wb"); +#endif if (hf == nullptr) return false; fclose(hf); @@ -2009,7 +2057,12 @@ bool CLocatorAPI::can_write_to_alias(pcstr path) bool CLocatorAPI::can_modify_file(pcstr fname) { +#if defined(XR_PLATFORM_WINDOWS) + const std::wstring wfname = XRay::Utf8::ToWide(fname); + FILE* hf = _wfopen(wfname.c_str(), L"r+b"); +#else FILE* hf = fopen(fname, "r+b"); +#endif if (hf) { fclose(hf); diff --git a/src/xrCore/LocatorAPI.h b/src/xrCore/LocatorAPI.h index 6bd1c269be5..97a63b0fcbf 100644 --- a/src/xrCore/LocatorAPI.h +++ b/src/xrCore/LocatorAPI.h @@ -14,7 +14,17 @@ #include "xrCommon/predicates.h" #include "Common/Noncopyable.hpp" -#if defined(XR_PLATFORM_POSIX) +#if defined(XR_PLATFORM_WINDOWS) +struct xr_finddata_t +{ + unsigned attrib; + __time64_t time_create; + __time64_t time_access; + __time64_t time_write; + _fsize_t size; + char name[1024]; +}; +#elif defined(XR_PLATFORM_POSIX) #include #define _A_HIDDEN 0x02 #define _A_SUBDIR 0x00000010 @@ -48,6 +58,8 @@ struct _finddata32_t _fsize_t size; char name[FILENAME_MAX]; }; + +typedef _finddata_t xr_finddata_t; #endif class CStreamReader; @@ -150,7 +162,7 @@ class XRCORE_API CLocatorAPI : Noncopyable using files_set = xr_set; using files_it = files_set::iterator; - using FFVec = xr_vector<_finddata_t>; + using FFVec = xr_vector; FFVec rec_files; int m_iLockRescan; @@ -165,7 +177,7 @@ class XRCORE_API CLocatorAPI : Noncopyable const file* RegisterExternal(pcstr name); const file* Register(pcstr name, size_t vfs, u32 crc, u32 ptr, u32 size_real, u32 size_compressed, u32 modif); void ProcessArchive(pcstr path); - void ProcessOne(pcstr path, const _finddata_t& entry); + void ProcessOne(pcstr path, const xr_finddata_t& entry); bool Recurse(pcstr path); files_it file_find_it(pcstr n); diff --git a/src/xrCore/Media/Image.cpp b/src/xrCore/Media/Image.cpp index ba1aa75d140..912565cd7e0 100644 --- a/src/xrCore/Media/Image.cpp +++ b/src/xrCore/Media/Image.cpp @@ -2,6 +2,9 @@ #include "Image.hpp" #include +#if defined(XR_PLATFORM_WINDOWS) +#include "xrCore/Text/Utf8Utils.hpp" +#endif using namespace XRay::Media; @@ -18,7 +21,12 @@ Image::~Image() void Image::SaveTGA(const char* name, ImageDataFormat format, bool align) { +#if defined(XR_PLATFORM_WINDOWS) + const std::wstring wname = XRay::Utf8::ToWide(name); + FILE* file = _wfopen(wname.c_str(), L"wb"); +#else FILE* file = std::fopen(name, "wb"); +#endif auto writerFunc = [&](void* data, size_t dataSize) { std::fwrite(data, dataSize, 1, file); }; SaveTGA(writerFunc, format, align); std::fclose(file); diff --git a/src/xrCore/Text/StringConversion.cpp b/src/xrCore/Text/StringConversion.cpp index dc4522773ce..b638f3038d0 100644 --- a/src/xrCore/Text/StringConversion.cpp +++ b/src/xrCore/Text/StringConversion.cpp @@ -1,177 +1,3 @@ #include "stdafx.h" - -#include "StringConversion.hpp" - -#include -#include - -#define BITS1_MASK 0x80 // 10000000b -#define BITS2_MASK 0xC0 // 11000000b -#define BITS3_MASK 0xE0 // 11100000b -#define BITS4_MASK 0xF0 // 11110000b - -#define BITS1_EXP 0x00 // 00000000b -#define BITS2_EXP 0x80 // 10000000b -#define BITS3_EXP 0xC0 // 11000000b -#define BITS4_EXP 0xE0 // 11100000b - -// MB_DUMB_CONVERSION - ignore non UTF-8 compliant sequences, as-is conversion -#define MB_DUMB_CONVERSION - -#ifdef MB_DUMB_CONVERSION - -u16 mbhMulti2WideDumb(xr_wide_char* WideStr, xr_wide_char* WidePos, u16 WideStrSize, const char* MultiStr) -{ - u16 spos = 0, dpos = 0; - u8 b1; - xr_wide_char wc = 0; - - VERIFY(MultiStr); - - if (!MultiStr[0]) - return 0; - - if (WideStr || WidePos) - VERIFY2(((WideStrSize > 0) && (WideStrSize < 0xFFFF)), make_string("'WideStrSize'=%hu", WideStrSize)); - - while ((b1 = MultiStr[spos++]) != 0x00) - { - if (WidePos) - WidePos[dpos] = spos; - - dpos++; - - wc = b1; - - if (WideStr) - { - VERIFY2((dpos < WideStrSize), make_string("S1: '%s',%hu<%hu", MultiStr, dpos, WideStrSize)); - WideStr[dpos] = wc; - } - } - - if (WidePos) - WidePos[dpos] = spos; - - if (WideStr) - { - VERIFY2((dpos < WideStrSize), make_string("S2: '%s',%hu<%hu", MultiStr, dpos, WideStrSize)); - WideStr[dpos + 1] = 0x0000; - } - - if (WideStr) - WideStr[0] = dpos; - - return dpos; -} - -#endif // MB_DUMB_CONVERSION - -u16 mbhMulti2Wide(xr_wide_char* WideStr, xr_wide_char* WidePos, u16 WideStrSize, const char* MultiStr) -{ - u16 spos = 0; - u16 dpos = 0; - u8 b1, b2, b3; - xr_wide_char wc = 0; - - VERIFY(MultiStr); - - if (!MultiStr[0]) - return 0; - - if (WideStr || WidePos) - VERIFY2(((WideStrSize > 0) && (WideStrSize < 0xFFFF)), make_string("'WideStrSize'=%hu", WideStrSize)); - - while ((b1 = MultiStr[spos]) != 0x00) - { - if (WidePos) - WidePos[dpos] = spos; - - spos++; - - if ((b1 & BITS1_MASK) == BITS1_EXP) - { - wc = b1; - } - else if ((b1 & BITS3_MASK) == BITS3_EXP) - { - b2 = MultiStr[spos++]; -#ifdef MB_DUMB_CONVERSION - if (!(b2 && ((b2 & BITS2_MASK) == BITS2_EXP))) - return mbhMulti2WideDumb(WideStr, WidePos, WideStrSize, MultiStr); -#else - VERIFY2((b2 && ((b2 & BITS2_MASK) == BITS2_EXP)), - make_string("B2: '%s',@%hu,[%hc][%hc]", MultiStr, spos, b1, b2)); -#endif - wc = ((b1 & ~BITS3_MASK) << 6) | (b2 & ~BITS2_MASK); - } - else if ((b1 & BITS4_MASK) == BITS4_EXP) - { - b2 = MultiStr[spos++]; -#ifdef MB_DUMB_CONVERSION - if (!(b2 && ((b2 & BITS2_MASK) == BITS2_EXP))) - return mbhMulti2WideDumb(WideStr, WidePos, WideStrSize, MultiStr); -#else - VERIFY2((b2 && ((b2 & BITS2_MASK) == BITS2_EXP)), - make_string("B31: '%s',@%hu,[%hc][%hc]", MultiStr, spos, b1, b2)); -#endif - b3 = MultiStr[spos++]; -#ifdef MB_DUMB_CONVERSION - if (!(b3 && ((b3 & BITS2_MASK) == BITS2_EXP))) - return mbhMulti2WideDumb(WideStr, WidePos, WideStrSize, MultiStr); -#else - VERIFY2((b3 && ((b3 & BITS2_MASK) == BITS2_EXP)), - make_string("B32: '%s',@%hu,[%hc][%hc][%hc]", MultiStr, spos, b1, b2, b3)); -#endif - wc = ((b1 & ~BITS4_MASK) << 12) | ((b2 & ~BITS2_MASK) << 6) | (b3 & ~BITS2_MASK); - } - else - { -#ifdef MB_DUMB_CONVERSION - return mbhMulti2WideDumb(WideStr, WidePos, WideStrSize, MultiStr); -#else - VERIFY2(0, make_string("B1: '%s',@%hu,[%hc]", MultiStr, spos, b1)); -#endif - } - - dpos++; - - if (WideStr) - { - VERIFY2((dpos < WideStrSize), make_string("S1: '%s',%hu<%hu", MultiStr, dpos, WideStrSize)); - WideStr[dpos] = wc; - } - } - - if (WidePos) - WidePos[dpos] = spos; - - if (WideStr) - { - VERIFY2((dpos < WideStrSize), make_string("S2: '%s',%hu<%hu", MultiStr, dpos, WideStrSize)); - WideStr[dpos + 1] = 0x0000; - } - - if (WideStr) - WideStr[0] = dpos; - - return dpos; -} - -xr_string StringFromUTF8(const char* in, const std::locale& locale) -{ - using wcvt = std::wstring_convert, wchar_t>; - auto wstr = wcvt{}.from_bytes(in); - xr_string result(wstr.size(), '\0'); - std::use_facet>(locale).narrow(wstr.data(), wstr.data() + wstr.size(), '?', &result[0]); - return result; -} - -xr_string StringToUTF8(const char* in, const std::locale& locale) -{ - using wcvt = std::wstring_convert, wchar_t>; - std::wstring wstr(xr_strlen(in), L'\0'); - std::use_facet>(locale).widen(in, in + xr_strlen(in), &wstr[0]); - std::string result = wcvt{}.to_bytes(wstr.data(), wstr.data() + wstr.size()); - return result.data(); -} +// All functions in this file have been removed as dead code (Stage 8 cleanup). +// Word-break helpers remain in StringConversion.hpp (used by GameFont::SplitByWidth). diff --git a/src/xrCore/Text/StringConversion.hpp b/src/xrCore/Text/StringConversion.hpp index 798e18a376b..3d84bacf19b 100644 --- a/src/xrCore/Text/StringConversion.hpp +++ b/src/xrCore/Text/StringConversion.hpp @@ -1,11 +1,7 @@ #pragma once #include "xrCore/xrCore.h" -typedef u16 xr_wide_char; // Don't replace it with wchar_t, it won't work correctly, I don't know why. - -#define MAX_MB_CHARS 4096 - -XRCORE_API u16 mbhMulti2Wide(xr_wide_char* WideStr, xr_wide_char* WidePos, u16 WideStrSize, const char* MultiStr); +using xr_wide_char = u16; // used by word-break helpers in GameFont IC bool IsNeedSpaceCharacter(xr_wide_char wc) { @@ -29,6 +25,3 @@ IC bool IsAlphaCharacter(xr_wide_char wc) ((wc >= 0x0061) && (wc <= 0x007A)) || ((wc >= 0xFF10) && (wc <= 0xFF19)) || ((wc >= 0xFF21) && (wc <= 0xFF3A)) || ((wc >= 0xFF41) && (wc <= 0xFF5A))); } - -XRCORE_API xr_string StringFromUTF8(const char* string, const std::locale& locale); -XRCORE_API xr_string StringToUTF8(const char* string, const std::locale& locale); diff --git a/src/xrCore/Text/Utf8Utils.cpp b/src/xrCore/Text/Utf8Utils.cpp new file mode 100644 index 00000000000..199fed81b3b --- /dev/null +++ b/src/xrCore/Text/Utf8Utils.cpp @@ -0,0 +1,554 @@ +#include "stdafx.h" +#include "Utf8Utils.hpp" + +#if defined(XR_PLATFORM_WINDOWS) +#include +#endif + +#include +#include + +namespace XRay +{ +namespace Utf8 +{ +size_t SequenceLength(u8 leadingByte) +{ + if ((leadingByte & 0x80) == 0x00) + return 1; + if ((leadingByte & 0xE0) == 0xC0) + return 2; + if ((leadingByte & 0xF0) == 0xE0) + return 3; + if ((leadingByte & 0xF8) == 0xF0) + return 4; + return 0; // Invalid leading byte +} + +pcstr Next(pcstr pos) +{ + if (!pos || !*pos) + return pos; + + const size_t len = SequenceLength(static_cast(*pos)); + if (len == 0) + return pos + 1; // Invalid byte, skip one byte to avoid infinite loops + + for (size_t i = 1; i < len; ++i) + { + if (!pos[i] || !IsContinuationByte(static_cast(pos[i]))) + return pos + 1; // Truncated or invalid sequence, skip one byte + } + + return pos + len; +} + +pcstr Prev(pcstr start, pcstr pos) +{ + if (!start || !pos || pos <= start) + return start; + + pcstr prev = pos - 1; + // Skip continuation bytes + while (prev > start && IsContinuationByte(static_cast(*prev))) + --prev; + + // Validate that we landed on a valid leading byte + const size_t len = SequenceLength(static_cast(*prev)); + if (len == 0 || prev + len > pos) + { + // Invalid sequence. Keep moving back over continuation bytes so we never + // return a continuation byte as a "codepoint start" (callers that only + // step once would otherwise land inside a multi-byte sequence). + pcstr fallback = pos - 1; + while (fallback > start && IsContinuationByte(static_cast(*fallback))) + --fallback; + return fallback; + } + + return prev; +} + +u32 Decode(pcstr pos, size_t& outLength) +{ + outLength = 0; + if (!pos || !*pos) + return 0; + + const u8 b1 = static_cast(*pos); + const size_t len = SequenceLength(b1); + + if (len == 1) + { + outLength = 1; + return b1; + } + + if (len == 2) + { + const u8 b2 = static_cast(pos[1]); + if (b2 && IsContinuationByte(b2)) + { + const u32 cp = ((b1 & 0x1F) << 6) | (b2 & 0x3F); + // Reject overlong sequences (e.g. 0xC0 0x80 encoding U+0000). + if (cp < 0x80) + { + outLength = 1; + return REPLACEMENT_CHARACTER; + } + outLength = 2; + return cp; + } + } + else if (len == 3) + { + const u8 b2 = static_cast(pos[1]); + if (b2 && IsContinuationByte(b2)) + { + const u8 b3 = static_cast(pos[2]); + if (b3 && IsContinuationByte(b3)) + { + const u32 cp = ((b1 & 0x0F) << 12) | ((b2 & 0x3F) << 6) | (b3 & 0x3F); + // Reject overlong sequences and surrogates + if (cp < 0x800 || (cp >= 0xD800 && cp <= 0xDFFF)) + { + outLength = 1; + return REPLACEMENT_CHARACTER; + } + outLength = 3; + return cp; + } + } + } + else if (len == 4) + { + const u8 b2 = static_cast(pos[1]); + if (b2 && IsContinuationByte(b2)) + { + const u8 b3 = static_cast(pos[2]); + if (b3 && IsContinuationByte(b3)) + { + const u8 b4 = static_cast(pos[3]); + if (b4 && IsContinuationByte(b4)) + { + const u32 cp = ((b1 & 0x07) << 18) | ((b2 & 0x3F) << 12) | ((b3 & 0x3F) << 6) | (b4 & 0x3F); + if (cp < 0x10000 || cp > 0x10FFFF) + { + outLength = 1; + return REPLACEMENT_CHARACTER; + } + outLength = 4; + return cp; + } + } + } + } + + outLength = 1; + return REPLACEMENT_CHARACTER; +} + +size_t LengthCodepoints(pcstr str) +{ + size_t count = 0; + while (str && *str) + { + str = Next(str); + ++count; + } + return count; +} + +pcstr Advance(pcstr pos, size_t codepoints) +{ + if (!pos || !*pos || codepoints == 0) + return pos; + + while (codepoints > 0 && *pos) + { + pos = Next(pos); + --codepoints; + } + + return pos; +} + +size_t DistanceCodepoints(pcstr start, pcstr end) +{ + if (!start || !end || start >= end) + return 0; + + size_t count = 0; + while (*start && start < end) + { + start = Next(start); + ++count; + } + + return count; +} + +bool IsValid(pcstr str) +{ + if (!str) + return true; // Empty/null is valid + + while (*str) + { + size_t len = 0; + const u32 cp = Decode(str, len); + if (cp == REPLACEMENT_CHARACTER && len == 1 && SequenceLength(static_cast(*str)) != 1) + return false; + str += len; + } + return true; +} + +pcstr StripBom(pcstr str) +{ + if (!str) + return nullptr; + if (static_cast(str[0]) == 0xEF && static_cast(str[1]) == 0xBB && static_cast(str[2]) == 0xBF) + return str + 3; + return str; +} + +void ToLowerAscii(pstr str) +{ + if (!str) + return; + while (*str) + { + if (*str >= 'A' && *str <= 'Z') + *str += 'a' - 'A'; + ++str; + } +} + +void ToUpperAscii(pstr str) +{ + if (!str) + return; + while (*str) + { + if (*str >= 'a' && *str <= 'z') + *str -= 'a' - 'A'; + ++str; + } +} + +std::wstring ToWide(pcstr str) +{ + std::wstring result; + if (!str) + return result; + + result.reserve(xr_strlen(str)); // Worst case: all ASCII + + while (*str) + { + size_t len = 0; + const u32 cp = Decode(str, len); + str += len; + + if (cp <= 0xFFFF) + { + result.push_back(static_cast(cp)); + } + else + { + // Surrogate pair for non-BMP (should not happen for us, but handle correctly) + const u32 code = cp - 0x10000; + result.push_back(static_cast(0xD800 + (code >> 10))); + result.push_back(static_cast(0xDC00 + (code & 0x3FF))); + } + } + + return result; +} + +xr_string FromWide(const wchar_t* str) +{ + xr_string result; + if (!str) + return result; + + const size_t len = std::wcslen(str); + result.reserve(len * 3); // Worst case: 3 bytes per wchar_t for BMP + + for (size_t i = 0; i < len; ++i) + { + const wchar_t wc = str[i]; + if (wc <= 0x7F) + { + result.push_back(static_cast(wc)); + } + else if (wc <= 0x7FF) + { + result.push_back(static_cast(0xC0 | (wc >> 6))); + result.push_back(static_cast(0x80 | (wc & 0x3F))); + } + else if (wc >= 0xD800 && wc <= 0xDBFF && i + 1 < len) + { + // Surrogate pair + const wchar_t low = str[++i]; + if (low >= 0xDC00 && low <= 0xDFFF) + { + const u32 cp = 0x10000 + ((wc - 0xD800) << 10) + (low - 0xDC00); + result.push_back(static_cast(0xF0 | (cp >> 18))); + result.push_back(static_cast(0x80 | ((cp >> 12) & 0x3F))); + result.push_back(static_cast(0x80 | ((cp >> 6) & 0x3F))); + result.push_back(static_cast(0x80 | (cp & 0x3F))); + } + else + { + result.push_back(static_cast(0xEF)); + result.push_back(static_cast(0xBF)); + result.push_back(static_cast(0xBD)); + } + } + else if (wc >= 0xDC00 && wc <= 0xDFFF) + { + // Lone low surrogate + result.push_back(static_cast(0xEF)); + result.push_back(static_cast(0xBF)); + result.push_back(static_cast(0xBD)); + } + else + { + result.push_back(static_cast(0xE0 | (wc >> 12))); + result.push_back(static_cast(0x80 | ((wc >> 6) & 0x3F))); + result.push_back(static_cast(0x80 | (wc & 0x3F))); + } + } + + return result; +} + +namespace +{ +// CP1251 to Unicode mapping for bytes 0x80..0xBF. +static const wchar_t CP1251ToUnicodeTable[64] = +{ + 0x0402, 0x0403, 0x201A, 0x0453, 0x201E, 0x2026, 0x2020, 0x2021, + 0x20AC, 0x2030, 0x0409, 0x2039, 0x040A, 0x040C, 0x040B, 0x040F, + 0x0452, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, + 0xFFFD, 0x2122, 0x0459, 0x203A, 0x045A, 0x045C, 0x045B, 0x045F, + 0x00A0, 0x040E, 0x045E, 0x0408, 0x00A4, 0x0490, 0x00A6, 0x00A7, + 0x0401, 0x00A9, 0x0404, 0x00AB, 0x00AC, 0x00AD, 0x00AE, 0x0407, + 0x00B0, 0x00B1, 0x0406, 0x0456, 0x0491, 0x00B5, 0x00B6, 0x00B7, + 0x0451, 0x2116, 0x0454, 0x00BB, 0x0458, 0x0405, 0x0455, 0x0457, +}; + +static wchar_t CP1251ByteToUnicode(u8 byte) +{ + if (byte < 0x80) + return static_cast(byte); + + if (byte < 0xC0) + return CP1251ToUnicodeTable[byte - 0x80]; + + // 0xC0..0xFF map directly to U+0410..U+044F + return static_cast(byte - 0xC0 + 0x0410); +} + +static u8 UnicodeToCP1251Byte(wchar_t wc) +{ + if (wc < 0x80) + return static_cast(wc); + + if (wc >= 0x0410 && wc <= 0x044F) + return static_cast(wc - 0x0410 + 0xC0); + + for (int i = 0; i < 64; ++i) + { + if (CP1251ToUnicodeTable[i] == wc) + return static_cast(0x80 + i); + } + + return static_cast('?'); +} + +static xr_string CP1251ToUtf8(pcstr str) +{ + const size_t len = xr_strlen(str); + std::wstring wide; + wide.reserve(len); + + for (size_t i = 0; i < len; ++i) + wide.push_back(CP1251ByteToUnicode(static_cast(str[i]))); + + return FromWide(wide.c_str()); +} + +static xr_string Utf8ToCP1251(pcstr str) +{ + xr_string result; + result.reserve(xr_strlen(str)); + + pcstr p = str; + while (*p) + { + size_t len = 0; + const u32 cp = Decode(p, len); + if (len == 0) + { + ++p; + result.push_back('?'); + continue; + } + + result.push_back(static_cast(UnicodeToCP1251Byte(static_cast(cp)))); + p += len; + } + + return result; +} +} // namespace + +// Public single-byte CP1251 -> Unicode codepoint mapping (uses the internal +// table above). Exposed so the font atlas builder can re-key legacy CP1251 +// glyphs without duplicating the translation table. +u32 CP1251ByteToCodepoint(u8 byte) { return static_cast(CP1251ByteToUnicode(byte)); } + +xr_string FromACP(pcstr str) +{ + if (!str || !str[0]) + return xr_string(); + +#if defined(XR_PLATFORM_WINDOWS) + // ACP on Windows may already be UTF-8 (system locale option) or a legacy code page. + // If the input is valid UTF-8, keep it unchanged. + if (IsValid(str)) + return xr_string(str); + + const int wideLen = MultiByteToWideChar(CP_ACP, 0, str, -1, nullptr, 0); + if (wideLen <= 0) + return xr_string(); + + std::wstring wide; + wide.resize(wideLen); + MultiByteToWideChar(CP_ACP, 0, str, -1, wide.data(), wideLen); + return FromWide(wide.c_str()); +#else + // POSIX has no ACP. Assume UTF-8 first, then fall back to CP1251 for legacy content. + if (IsValid(str)) + return xr_string(str); + + return CP1251ToUtf8(str); +#endif +} + +xr_string FromCP1251(pcstr str) +{ + if (!str || !str[0]) + return xr_string(); + +#if defined(XR_PLATFORM_WINDOWS) + constexpr UINT CP_1251 = 1251; + const int wideLen = MultiByteToWideChar(CP_1251, 0, str, -1, nullptr, 0); + if (wideLen <= 0) + return xr_string(); + + std::wstring wide; + wide.resize(wideLen); + MultiByteToWideChar(CP_1251, 0, str, -1, wide.data(), wideLen); + return FromWide(wide.c_str()); +#else + return CP1251ToUtf8(str); +#endif +} + +xr_string ToCP1251(pcstr str) +{ + if (!str || !str[0]) + return xr_string(); + +#if defined(XR_PLATFORM_WINDOWS) + constexpr UINT CP_1251 = 1251; + const std::wstring wide = ToWide(str); + const int cpLen = WideCharToMultiByte(CP_1251, 0, wide.c_str(), -1, nullptr, 0, nullptr, nullptr); + if (cpLen <= 0) + return xr_string(); + + xr_string result; + result.resize(cpLen); + WideCharToMultiByte(CP_1251, 0, wide.c_str(), -1, result.data(), cpLen, nullptr, nullptr); + return result; +#else + return Utf8ToCP1251(str); +#endif +} + +xr_string Sanitize(pcstr str) +{ + xr_string result; + if (!str) + return result; + + while (*str) + { + size_t len = 0; + const u32 cp = Decode(str, len); + if (cp == REPLACEMENT_CHARACTER && len == 1 && SequenceLength(static_cast(*str)) != 1) + { + // Invalid sequence, insert replacement character + result.push_back(static_cast(0xEF)); + result.push_back(static_cast(0xBF)); + result.push_back(static_cast(0xBD)); + ++str; + } + else + { + result.append(str, len); + str += len; + } + } + + return result; +} + +xr_string FixTextInputEncoding(pcstr text) +{ + if (!text || !text[0]) + return xr_string(); + + // If SDL already gives us valid UTF-8, nothing to fix. + if (IsValid(text)) + return xr_string(text); + +#if defined(XR_PLATFORM_WINDOWS) + // SDL on Windows may deliver characters encoded in the current keyboard + // layout's ANSI code page instead of UTF-8. Convert from that code page. + const HKL hkl = GetKeyboardLayout(0); + const LANGID langId = LOWORD(hkl); + const LCID lcid = MAKELCID(langId, SORT_DEFAULT); + + char cpBuf[16] = {}; + if (GetLocaleInfoA(lcid, LOCALE_IDEFAULTANSICODEPAGE, cpBuf, sizeof(cpBuf))) + { + const UINT cp = static_cast(atoi(cpBuf)); + if (cp != 0) + { + const int wideLen = MultiByteToWideChar(cp, 0, text, -1, nullptr, 0); + if (wideLen > 0) + { + std::wstring wide; + wide.resize(wideLen); + if (MultiByteToWideChar(cp, 0, text, -1, wide.data(), wideLen) > 0) + // FromWide() stops at the null terminator written by MultiByteToWideChar, + // so there is no need to resize it away (matches FromACP/FromCP1251). + return FromWide(wide.c_str()); + } + } + } +#else + // Non-Windows: SDL text input is already UTF-8; return it unchanged. + return xr_string(text); +#endif + + // Fallback: return sanitized UTF-8. + return Sanitize(text); +} +} // namespace Utf8 +} // namespace XRay diff --git a/src/xrCore/Text/Utf8Utils.hpp b/src/xrCore/Text/Utf8Utils.hpp new file mode 100644 index 00000000000..242bb854c95 --- /dev/null +++ b/src/xrCore/Text/Utf8Utils.hpp @@ -0,0 +1,94 @@ +#pragma once + +#include "xrCore/xrCore.h" +#include "xrCommon/xr_string.h" + +// Basic UTF-8 utilities. Supports only valid UTF-8 sequences (BMP is enough for our needs). +// All functions assume null-terminated strings unless noted otherwise. + +namespace XRay +{ +namespace Utf8 +{ +// Returns true if the byte is a valid UTF-8 continuation byte (10xxxxxx). +inline bool IsContinuationByte(u8 byte) { return (byte & 0xC0) == 0x80; } + +// Returns true if the byte is a valid UTF-8 leading byte. +inline bool IsLeadingByte(u8 byte) +{ + return (byte & 0x80) == 0x00 || // 0xxxxxxx (ASCII) + (byte & 0xE0) == 0xC0 || // 110xxxxx + (byte & 0xF0) == 0xE0 || // 1110xxxx + (byte & 0xF8) == 0xF0; // 11110xxx +} + +// Returns the number of bytes in the UTF-8 sequence starting with the given byte. +// Returns 0 for continuation bytes and invalid leading bytes. +XRCORE_API size_t SequenceLength(u8 leadingByte); + +// Returns pointer to the start of the next UTF-8 codepoint. +// If the current byte is not a leading byte, advances one byte forward (safe fallback). +XRCORE_API pcstr Next(pcstr pos); + +// Returns pointer to the start of the previous UTF-8 codepoint. +// start must be the beginning of the string. Returns start if pos == start. +XRCORE_API pcstr Prev(pcstr start, pcstr pos); + +// Decodes one UTF-8 codepoint starting at pos. +// Returns the codepoint value and writes the number of bytes into outLength. +// Returns REPLACEMENT_CHARACTER (U+FFFD) and outLength=1 on invalid sequence. +XRCORE_API u32 Decode(pcstr pos, size_t& outLength); + +// Counts the number of UTF-8 codepoints in the string. +XRCORE_API size_t LengthCodepoints(pcstr str); + +// Advances the pointer by the given number of codepoints. +// Stops at the end of the string. Returns pos if count is 0 or pos is null/empty. +XRCORE_API pcstr Advance(pcstr pos, size_t codepoints); + +// Counts the number of UTF-8 codepoints between start and end (end must be >= start). +XRCORE_API size_t DistanceCodepoints(pcstr start, pcstr end); + +// Returns true if the entire string is valid UTF-8. +XRCORE_API bool IsValid(pcstr str); + +// Strips UTF-8 BOM (EF BB BF) from the beginning of the string if present. +XRCORE_API pcstr StripBom(pcstr str); + +// ASCII-only lowercase. Does not touch non-ASCII bytes, so it is safe for UTF-8. +XRCORE_API void ToLowerAscii(pstr str); + +// ASCII-only uppercase. Does not touch non-ASCII bytes, so it is safe for UTF-8. +XRCORE_API void ToUpperAscii(pstr str); + +// Converts UTF-8 string to UTF-16 (std::wstring). Useful for WinAPI interop. +XRCORE_API std::wstring ToWide(pcstr str); + +// Converts UTF-16 string (wchar_t*) to UTF-8. +XRCORE_API xr_string FromWide(const wchar_t* str); + +// Converts a string from the system default ANSI code page to UTF-8. +XRCORE_API xr_string FromACP(pcstr str); + +// Converts a string from Windows-1251 (legacy game scripts/text) to UTF-8. +XRCORE_API xr_string FromCP1251(pcstr str); + +// Converts a UTF-8 string to Windows-1251. +XRCORE_API xr_string ToCP1251(pcstr str); + +// Maps a single CP1251 byte (0x00..0xFF) to its Unicode codepoint. +// Used by the font atlas builder to re-key glyphs of legacy Windows-1251 TTFs. +XRCORE_API u32 CP1251ByteToCodepoint(u8 byte); + +// Replaces invalid UTF-8 bytes with U+FFFD replacement character. +XRCORE_API xr_string Sanitize(pcstr str); + +// Windows: SDL_TEXTINPUT may return characters encoded in the current keyboard layout's +// ANSI code page instead of UTF-8. This helper detects invalid UTF-8 and converts from +// the layout's code page. On non-Windows it returns the input unchanged. +XRCORE_API xr_string FixTextInputEncoding(pcstr text); + +constexpr u32 REPLACEMENT_CHARACTER = 0xFFFD; +constexpr u32 MAX_BMP_CODEPOINT = 0xFFFF; +} // namespace Utf8 +} // namespace XRay diff --git a/src/xrCore/XML/XMLDocument.cpp b/src/xrCore/XML/XMLDocument.cpp index 77e7581af99..d3753492529 100644 --- a/src/xrCore/XML/XMLDocument.cpp +++ b/src/xrCore/XML/XMLDocument.cpp @@ -1,6 +1,7 @@ #include "stdafx.h" #include "XMLDocument.hpp" +#include "xrCore/Text/Utf8Utils.hpp" pcstr UI_PATH = UI_PATH_DEFAULT; pcstr UI_PATH_WITH_DELIMITER = UI_PATH_DEFAULT_WITH_DELIMITER; @@ -89,6 +90,16 @@ void ParseFile(pcstr path, CMemoryWriter& W, IReader* F, XMLDocument* xml, bool } }; + // Strip UTF-8 BOM if present at the beginning of this file. + // Each included file is appended to the document buffer, so its BOM must be + // removed before parsing to avoid embedded zero-width characters. + if (F->tell() == 0 && F->length() >= 3) + { + const u8* data = static_cast(F->pointer()); + if (data[0] == 0xEF && data[1] == 0xBB && data[2] == 0xBF) + F->seek(3); + } + while (!F->eof()) { string4096 str; @@ -183,6 +194,34 @@ bool XMLDocument::Load(pcstr path, pcstr xml_filename, bool fatal) bool XMLDocument::Set(pcstr text, bool fatal) { R_ASSERT(text != nullptr); + + // Strip UTF-8 BOM if present + text = XRay::Utf8::StripBom(text); + + // Auto-detect legacy CP1251 XML files and transcode them to UTF-8. + // This keeps compatibility with old mod content while the engine works internally in UTF-8. + xr_string convertedText; + if (!XRay::Utf8::IsValid(text)) + { + Msg("! XML file '%s' is not valid UTF-8, converting from Windows-1251 to UTF-8", m_xml_file_name); + convertedText = XRay::Utf8::FromCP1251(text); + + // Update the XML declaration encoding if it claims windows-1251 + const char windows1251[] = "encoding=\"windows-1251\""; + size_t pos = convertedText.find(windows1251); + if (pos != xr_string::npos) + convertedText.replace(pos, std::size(windows1251) - 1, "encoding=\"utf-8\""); + else + { + const char windows1251sq[] = "encoding='windows-1251'"; + pos = convertedText.find(windows1251sq); + if (pos != xr_string::npos) + convertedText.replace(pos, std::size(windows1251sq) - 1, "encoding='utf-8'"); + } + + text = convertedText.c_str(); + } + m_Doc.Parse(&m_Doc, text); if (m_Doc.Error()) diff --git a/src/xrCore/file_stream_reader.cpp b/src/xrCore/file_stream_reader.cpp index d7660b142c5..f76042aa1e0 100644 --- a/src/xrCore/file_stream_reader.cpp +++ b/src/xrCore/file_stream_reader.cpp @@ -1,14 +1,16 @@ #include "stdafx.h" #include "file_stream_reader.h" -#if defined(XR_PLATFORM_POSIX) +#if defined(XR_PLATFORM_WINDOWS) +#include "xrCore/Text/Utf8Utils.hpp" +#elif defined(XR_PLATFORM_POSIX) #include #endif void CFileStreamReader::construct(pcstr file_name, const size_t& window_size) { #if defined(XR_PLATFORM_WINDOWS) - m_file_handle = CreateFile(file_name, GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr); + m_file_handle = CreateFileW(XRay::Utf8::ToWide(file_name).c_str(), GENERIC_READ, FILE_SHARE_READ, nullptr, OPEN_EXISTING, 0, nullptr); VERIFY(m_file_handle != INVALID_HANDLE_VALUE); LARGE_INTEGER size; diff --git a/src/xrCore/os_clipboard.cpp b/src/xrCore/os_clipboard.cpp index e7b92d7c5c6..980a867add3 100644 --- a/src/xrCore/os_clipboard.cpp +++ b/src/xrCore/os_clipboard.cpp @@ -11,25 +11,14 @@ #include "stdafx.h" #include "os_clipboard.h" -#include "Text/StringConversion.hpp" +#include "Text/Utf8Utils.hpp" #include -#include - -void os_clipboard::copy_to_clipboard(pcstr buf, bool alreadyUTF8 /*= false*/) +void os_clipboard::copy_to_clipboard(pcstr buf, bool /*alreadyUTF8*/ /*= true*/) { - int result; - if (alreadyUTF8) - { - result = SDL_SetClipboardText(buf); - } - else - { - static std::locale locale(""); - xr_string string = StringToUTF8(buf, locale); - result = SDL_SetClipboardText(string.c_str()); - } + // All internal strings are UTF-8 now, so pass them directly to SDL. + const int result = SDL_SetClipboardText(buf); if (result < 0) { Msg("! Failed to copy text to the clipboard: %s", SDL_GetError()); @@ -53,20 +42,33 @@ void os_clipboard::paste_from_clipboard(pstr buffer, size_t buffer_size) return; } - static std::locale locale(""); - const xr_string string = StringFromUTF8(clipData, locale); + strncpy_s(buffer, buffer_size, clipData, buffer_size - 1); SDL_free(clipData); - strncpy_s(buffer, buffer_size, string.c_str(), buffer_size - 1); + // If strncpy_s truncated inside a multi-byte UTF-8 sequence, the buffer ends + // with a stray continuation byte of an incomplete codepoint. Trim only that + // incomplete tail back to its leading byte. NOTE: we must NOT trim every + // trailing continuation byte — a completed multi-byte codepoint (e.g. Cyrillic + // 'т' = D1 82) also ends in a continuation byte and must be preserved. + const size_t pastedLength = xr_strlen(buffer); + if (pastedLength > 0 && XRay::Utf8::IsContinuationByte(static_cast(buffer[pastedLength - 1]))) + { + pcstr leading = XRay::Utf8::Prev(buffer, buffer + pastedLength); + const size_t leadingIdx = leading - buffer; + const size_t seqLen = XRay::Utf8::SequenceLength(static_cast(*leading)); + // Trim only when the leading byte's sequence is actually cut off. + if (seqLen > 0 && leadingIdx + seqLen > pastedLength) + buffer[leadingIdx] = '\0'; + } const size_t length = xr_strlen(buffer); for (size_t i = 0; i < length; ++i) { - const char c = buffer[i]; - if ((std::isprint(c, locale) == 0 && c != char(-1)) || c == '\t' || c == '\n') // "я" = -1 - { + const unsigned char c = static_cast(buffer[i]); + // Allow printable ASCII, UTF-8 continuation bytes and UTF-8 leading bytes. + // Only reject control characters except tab and newline. + if (c < 0x20 && c != '\t' && c != '\n') buffer[i] = ' '; - } } } @@ -93,20 +95,17 @@ void os_clipboard::update_clipboard(pcstr string) return; } - static std::locale locale(""); - const xr_string stringInUTF8 = StringToUTF8(string, locale); - const size_t clipLength = xr_strlen(clipData); - const size_t stringLength = stringInUTF8.size(); + const size_t stringLength = xr_strlen(string); const size_t bufferSize = (clipLength + stringLength + 1) * sizeof(char); pstr buffer = (pstr)xr_alloca(bufferSize); xr_strcpy(buffer, bufferSize, clipData); // copy the clipboard - xr_strcat(buffer, bufferSize, stringInUTF8.c_str()); // copy the new string + xr_strcat(buffer, bufferSize, string); // copy the new string SDL_free(clipData); - copy_to_clipboard(buffer, true); + copy_to_clipboard(buffer); } diff --git a/src/xrCore/os_clipboard.h b/src/xrCore/os_clipboard.h index 0d011a003f0..95beb006134 100644 --- a/src/xrCore/os_clipboard.h +++ b/src/xrCore/os_clipboard.h @@ -11,7 +11,7 @@ namespace os_clipboard { -XRCORE_API void copy_to_clipboard(pcstr buf, bool alreadyUTF8 = false); +XRCORE_API void copy_to_clipboard(pcstr buf, bool alreadyUTF8 = true); XRCORE_API void paste_from_clipboard(pstr buf, size_t buf_size); XRCORE_API void update_clipboard(pcstr str); } // namespace os_clipboard diff --git a/src/xrCore/xrCore.cpp b/src/xrCore/xrCore.cpp index ae9d57e386b..76b09bd4a6e 100644 --- a/src/xrCore/xrCore.cpp +++ b/src/xrCore/xrCore.cpp @@ -6,6 +6,7 @@ #if defined(XR_PLATFORM_WINDOWS) #include #include +#include "xrCore/Text/Utf8Utils.hpp" #pragma comment(lib, "winmm.lib") #elif defined(XR_PLATFORM_POSIX) #include @@ -222,12 +223,15 @@ void xrCore::Initialize(pcstr _ApplicationName, pcstr commandLine, bool init_fs, { string_path c_name; sscanf(strstr(Core.Params, "-wf ") + 4, "%[^ ] ", c_name); - SetCurrentDirectory(c_name); + SetCurrentDirectoryW(XRay::Utf8::ToWide(c_name).c_str()); } #endif #if defined(XR_PLATFORM_WINDOWS) - GetCurrentDirectory(sizeof(WorkingPath), WorkingPath); + wchar_t tempWorkingPath[sizeof(WorkingPath)]{}; + GetCurrentDirectoryW(sizeof(tempWorkingPath) / sizeof(wchar_t), tempWorkingPath); + const xr_string utf8WorkingPath = XRay::Utf8::FromWide(tempWorkingPath); + xr_strcpy(WorkingPath, sizeof(WorkingPath), utf8WorkingPath.c_str()); #elif defined(XR_PLATFORM_POSIX) getcwd(WorkingPath, sizeof(WorkingPath)); #else diff --git a/src/xrCore/xrCore.vcxproj b/src/xrCore/xrCore.vcxproj index c339cfb7c39..ee492c55e3a 100644 --- a/src/xrCore/xrCore.vcxproj +++ b/src/xrCore/xrCore.vcxproj @@ -98,6 +98,7 @@ call .GitInfo.cmd + @@ -203,6 +204,7 @@ call .GitInfo.cmd + diff --git a/src/xrCore/xrCore.vcxproj.filters b/src/xrCore/xrCore.vcxproj.filters index e783e07cf4e..19e4779aabd 100644 --- a/src/xrCore/xrCore.vcxproj.filters +++ b/src/xrCore/xrCore.vcxproj.filters @@ -303,6 +303,9 @@ Text + + Text + FS @@ -674,6 +677,9 @@ Text + + Text + FS diff --git a/src/xrCore/xr_ini.cpp b/src/xrCore/xr_ini.cpp index c4249f1b756..86771e1ef2d 100644 --- a/src/xrCore/xr_ini.cpp +++ b/src/xrCore/xr_ini.cpp @@ -259,7 +259,10 @@ XRCORE_API bool _parse(pstr dest, pcstr src) { while (*src) { - if (isspace((u8)*src)) + // Only ASCII whitespace is stripped outside quoted strings. + // Using iswspace() on UTF-8 bytes would treat some continuation bytes + // (e.g. 0xA0 NBSP) as whitespace and corrupt multi-byte codepoints. + if (*src == ' ' || *src == '\t' || *src == '\r' || *src == '\n') { if (bInsideSTR) { @@ -267,7 +270,7 @@ XRCORE_API bool _parse(pstr dest, pcstr src) continue; } - while (*src && isspace(*src)) + while (*src && (*src == ' ' || *src == '\t' || *src == '\r' || *src == '\n')) ++src; continue; @@ -409,10 +412,24 @@ void CInifile::Load(IReader* F, pcstr path, allow_include_func_t allow_include_f string4096 str2; bool bInsideSTR = false; + bool bFirstLine = true; while (!F->eof()) { F->r_string(str, sizeof str); + + // Strip UTF-8 BOM at the beginning of the file + if (bFirstLine && + static_cast(str[0]) == 0xEF && + static_cast(str[1]) == 0xBB && + static_cast(str[2]) == 0xBF) + { + const size_t len = xr_strlen(str); + if (len >= 3) + memmove(str, str + 3, len - 2); // includes null terminator + } + bFirstLine = false; + _Trim(str); pstr comm = strchr(str, ';'); pstr comm_1 = strchr(str, '/'); diff --git a/src/xrCore/xrsharedmem.cpp b/src/xrCore/xrsharedmem.cpp index a2140afe2d9..835d6567b9b 100644 --- a/src/xrCore/xrsharedmem.cpp +++ b/src/xrCore/xrsharedmem.cpp @@ -2,6 +2,10 @@ #pragma hdrstop // huh? #include "Threading/Lock.hpp" +#if defined(XR_PLATFORM_WINDOWS) +#include "xrCore/Text/Utf8Utils.hpp" +#endif + using namespace std; XRCORE_API smem_container* g_pSharedMemoryContainer = nullptr; @@ -82,7 +86,11 @@ void smem_container::clean() void smem_container::dump() { ScopeLock scope(&lock); +#if defined(XR_PLATFORM_WINDOWS) + FILE* F = _wfopen(L"x:\\$smem_dump$.txt", L"w"); +#else FILE* F = fopen("x:\\$smem_dump$.txt", "w"); +#endif for (auto& v : container) fprintf(F, "%4u : crc[%6x], %u bytes\n", v->dwReference, v->dwCRC, v->dwLength); fclose(F); diff --git a/src/xrCore/xrstring.cpp b/src/xrCore/xrstring.cpp index e498aaf6426..da0364b3376 100644 --- a/src/xrCore/xrstring.cpp +++ b/src/xrCore/xrstring.cpp @@ -8,6 +8,10 @@ #include "FS_impl.h" #include +#if defined(XR_PLATFORM_WINDOWS) +#include "xrCore/Text/Utf8Utils.hpp" +#endif + XRCORE_API str_container* g_pStringContainer = nullptr; #if 1 @@ -220,7 +224,11 @@ void str_container::verify() const void str_container::dump() const { impl->cs.Enter(); +#if defined(XR_PLATFORM_WINDOWS) + FILE* F = _wfopen(L"d:\\$str_dump$.txt", L"w"); +#else FILE* F = fopen("d:\\$str_dump$.txt", "w"); +#endif impl->dump(F); fclose(F); impl->cs.Leave(); @@ -402,7 +410,11 @@ void str_container::dump() impl->cs.Enter(); str_container_impl::cdb::iterator it = impl->container.begin(); str_container_impl::cdb::iterator end = impl->container.end(); +#if defined(XR_PLATFORM_WINDOWS) + FILE* F = _wfopen(L"d:\\$str_dump$.txt", L"w"); +#else FILE* F = fopen("d:\\$str_dump$.txt", "w"); +#endif for (; it != end; it++) fprintf( F, "ref[%4d]-len[%3d]-crc[%8X] : %s\n", (*it)->dwReference, (*it)->dwLength, (*it)->dwCRC, (*it)->value); diff --git a/src/xrEngine/CMakeLists.txt b/src/xrEngine/CMakeLists.txt index 6ee391fa94c..99cb8558283 100644 --- a/src/xrEngine/CMakeLists.txt +++ b/src/xrEngine/CMakeLists.txt @@ -413,6 +413,7 @@ target_include_directories(xrEngine "${CMAKE_SOURCE_DIR}/src" "${CMAKE_SOURCE_DIR}/Externals/imgui" "${CMAKE_SOURCE_DIR}/Externals/OpenAutomate/inc" + "${CMAKE_SOURCE_DIR}/Externals/freetype/include" ) target_link_libraries(xrEngine @@ -432,6 +433,7 @@ target_link_libraries(xrEngine Ogg::Ogg Theora::Theora SDL2::SDL2 + freetype ) target_compile_definitions(xrEngine diff --git a/src/xrEngine/Device_imgui.cpp b/src/xrEngine/Device_imgui.cpp index d4ad7842c55..4dc5a0f1f59 100644 --- a/src/xrEngine/Device_imgui.cpp +++ b/src/xrEngine/Device_imgui.cpp @@ -35,6 +35,55 @@ void CRenderDevice::InitializeImGui() io.ConfigNavMoveSetMousePos = true; + // Load a font with Cyrillic glyph support for the console. + // The default ImGui font (ProggyClean) only covers U+0020-U+00FF, + // so Cyrillic characters (U+0400-U+04FF) render as '?'. + { + // Try several candidate locations; the first one that exists wins. + string_path fontPath; + bool found = false; + const char* candidates[] = + { + "fonts" DELIMITER "DroidSans.ttf", + "fonts" DELIMITER "rus" DELIMITER "arial.ttf", + "fonts" DELIMITER "eng" DELIMITER "arial.ttf", + }; + for (const char* cand : candidates) + { + FS.update_path(fontPath, "$game_data$", cand); + if (FS.exist(fontPath)) + { + found = true; + break; + } + } + + if (found) + { + // Build glyph ranges manually (GetGlyphRangesCyrillic is obsolete since 1.92) + ImFontGlyphRangesBuilder builder; + ImWchar ranges[] = + { + 0x0020, 0x00FF, // Basic Latin + Latin Supplement + 0x0400, 0x04FF, // Cyrillic + 0x0500, 0x052F, // Cyrillic Supplement + 0, + }; + builder.AddRanges(ranges); + ImVector glyphRanges; + builder.BuildRanges(&glyphRanges); + io.Fonts->AddFontFromFileTTF(fontPath, 16.0f, nullptr, glyphRanges.Data); + } + else + { + // No Cyrillic-capable TTF found: ImGui will fall back to its built-in + // font, which cannot render Cyrillic (shown as '?'). Warn so the missing + // asset is easy to diagnose instead of being a silent rendering bug. + Msg("! ImGui: no Cyrillic TTF font found in $game_data$\\fonts\\; " + "console/UI overlay text in Russian will render as '?'"); + } + } + string_path fName; FS.update_path(fName, "$app_data_root$", io.IniFilename); convert_path_separators(fName); diff --git a/src/xrEngine/FDemoRecord.cpp b/src/xrEngine/FDemoRecord.cpp index a9ce7dfc34f..447185a3570 100644 --- a/src/xrEngine/FDemoRecord.cpp +++ b/src/xrEngine/FDemoRecord.cpp @@ -60,46 +60,11 @@ Fbox get_level_screenshot_bound() return res; } -pcstr GetFontTexName(pcstr section) -{ - static const char* tex_names[] = { "texture800", "texture", "texture1600" }; - int def_idx = 1; // default 1024x768 - int idx = def_idx; - -#if 0 - u32 w = Device.dwWidth; - - if (w <= 800) - idx = 0; - else if (w <= 1280) - idx = 1; - else - idx = 2; -#else - u32 h = Device.dwHeight; - - if (h <= 600) - idx = 0; - else if (h <= 1024) - idx = 1; - else - idx = 2; -#endif - - while (idx >= 0) - { - if (pSettings->line_exist(section, tex_names[idx])) - return pSettings->r_string(section, tex_names[idx]); - --idx; - } - return pSettings->r_string(section, tex_names[def_idx]); -} - CDemoRecord::CDemoRecord(const char* name, float life_time) : CEffectorCam(cefDemo, life_time /*,false*/), m_speed(speed_1), m_angle_speed(speed_1), - m_Font(pSettings->r_string(DEMO_RECORD_HELP_FONT, "shader"), GetFontTexName(DEMO_RECORD_HELP_FONT)) + m_Font(DEMO_RECORD_HELP_FONT) { Device.seqRender.Add(this, REG_PRIORITY_LOW - 1000); diff --git a/src/xrEngine/GameFont.cpp b/src/xrEngine/GameFont.cpp index add453d0661..8e750aa56c8 100644 --- a/src/xrEngine/GameFont.cpp +++ b/src/xrEngine/GameFont.cpp @@ -1,297 +1,1055 @@ #include "stdafx.h" #pragma hdrstop +#if defined(XR_PLATFORM_WINDOWS) +#include +#endif + #include "GameFont.h" #include "xr_level_controller.h" #include "xrCore/Text/StringConversion.hpp" #include "Render.h" #include "StringTable/StringTable.h" +#include "Include/xrRender/RenderFactory.h" +#include "Include/xrRender/FontRender.h" + +#include +#include + +#include extern ENGINE_API bool g_bRendering; -ENGINE_API Fvector2 g_current_font_scale = {1.0f, 1.0f}; -#include "Include/xrRender/RenderFactory.h" -#include "Include/xrRender/FontRender.h" +// Global text scale multiplier (cvar `ui_text_scale`). Multiplies the width and +// height returned by the font on the draw path so that the whole in-game text can +// be resized at runtime without rebuilding the FreeType atlas. +ENGINE_API float g_text_scale = 0.85f; + +// Font renderer selector (cvar `r_font_legacy`). 0 = FreeType TTF, 1 = +// legacy X-Ray bitmap fonts. Read on each font (re)initialization. +ENGINE_API int g_font_legacy = 0; + +FT_Library FreetypeLib = nullptr; +bool CGameFont::bFreetypeInitialized = false; -CGameFont::CGameFont(pcstr section, u8 flags) +namespace { - pFontRender = GEnv.RenderFactory->CreateFontRender(); - fCurrentHeight = 0.0f; - fXStep = 0.0f; - fYStep = 0.0f; - uFlags = flags; - Initialize(pSettings->r_string(section, "shader"), pSettings->r_string(section, "texture")); + constexpr u32 TextureDimension = 4096; +} // namespace + +void CGameFont::InitializeFreetype() +{ + FT_Error Error = FT_Init_FreeType(&FreetypeLib); + R_ASSERT2(Error == 0, "FreeType initialize failed"); +} + +static inline float DI2PX(float x) +{ + return float(iFloor((x + 1) * float(Device.dwWidth) * 0.5f)); +} +static inline float DI2PY(float y) +{ + return float(iFloor((y + 1) * float(Device.dwHeight) * 0.5f)); +} + +CGameFont::CGameFont(pcstr section, u8 flags) : LineSpacing(1.f), LetterSpacing(0.f) +{ +#ifdef DEBUG + m_font_name = section; + Msg("* Init font %s", section); +#endif + + uFlags = flags | fsMultibyte; + + // Read font name + if (pSettings->line_exist(section, "name")) + { + xr_string nameStr = pSettings->r_string(section, "name"); + XRay::Utf8::ToLowerAscii(nameStr.data()); + Data.Name = xr_strdup(nameStr.c_str()); + } + + if (!Data.Name || !xr_strlen(Data.Name)) + Data.Name = xr_strdup(section); + + Data.TextureName = xr_strdup(section); + + // Set font shader and style + Data.Shader = xr_strdup(pSettings->r_string(section, "shader")); + + Data.Style = nullptr; + if (pSettings->line_exist(section, "style")) + Data.Style = xr_strdup(pSettings->r_string(section, "style")); + + // Read font size + Data.Size = 14; if (pSettings->line_exist(section, "size")) + Data.Size = (u16)std::min(pSettings->r_u32(section, "size"), u16(-1)); + + if (pSettings->line_exist(section, "opentype")) + Data.OpenType = pSettings->r_bool(section, "opentype"); + + if (pSettings->line_exist(section, "letter_spacing")) + SetLetterSpacing(pSettings->r_float(section, "letter_spacing")); + + if (pSettings->line_exist(section, "line_spacing")) + SetLineSpacing(pSettings->r_float(section, "line_spacing")); + + // Init + pFontRender = GEnv.RenderFactory->CreateFontRender(); + Prepare(Data.Name, Data.Shader, Data.Style, Data.Size); +} + +CGameFont::~CGameFont() +{ + // Shading + if (OurFont) { - float sz = pSettings->r_float(section, "size"); - if (uFlags & fsDeviceIndependent) - SetHeightI(sz); - else - SetHeight(sz); + FT_Done_Face(OurFont); + OurFont = nullptr; + } + + if (m_fontReader) + { + FS.r_close(m_fontReader); + m_fontReader = nullptr; } - if (pSettings->line_exist(section, "interval")) - SetInterval(pSettings->r_fvector2(section, "interval")); + + GEnv.RenderFactory->DestroyFontRender(pFontRender); + pFontRender = nullptr; + + xr_free(Data.Shader); + xr_free(Data.Style); + xr_free(Data.Name); + xr_free(Data.TextureName); } -CGameFont::CGameFont(pcstr shader, pcstr texture, u8 flags) +void CGameFont::ReInit() { - pFontRender = GEnv.RenderFactory->CreateFontRender(); - fCurrentHeight = 0.0f; - fXStep = 0.0f; - fYStep = 0.0f; - uFlags = flags; - Initialize(shader, texture); + Prepare(Data.Name, Data.Shader, Data.Style, Data.Size); } -void CGameFont::Initialize(pcstr cShader, pcstr cTextureName) +void CGameFont::Prepare(pcstr name, pcstr shader, pcstr style, u32 size) { - string_path cTexture; + Initialize2(name, shader, style, size); +} - pcstr _lang = StringTable().GetCurrentFontPrefix().c_str(); - bool is_di = strstr(cTextureName, "ui_font_hud_01") || strstr(cTextureName, "ui_font_hud_02") || - strstr(cTextureName, "ui_font_console_02"); - if (_lang && !is_di) - strconcat(cTexture, cTextureName, _lang); - else - xr_strcpy(cTexture, sizeof(cTexture), cTextureName); +static xr_vector split(const xr_string& s, char delim) +{ + xr_vector elems; + std::stringstream ss(s.c_str()); + xr_string item; + while (std::getline(ss, item, delim)) + elems.push_back(item); + return std::move(elems); +} + +// Resolve the legacy bitmap font texture name. Priority: +// 1. Explicit "legacy_texture" field in the section (fonts.ltx). +// 2. Hardcoded section->texture table (old engine naming, no common pattern). +// 3. Heuristic "ui_font__" as a last resort. +// The .dds/.ini files live in $game_textures$ alongside the old engine layout. +static void resolve_legacy_texture(pcstr section, pcstr name, string_path& outTexture) +{ + // 1. Explicit override in fonts.ltx (recommended for any new section). + pcstr explicitTex = READ_IF_EXISTS(pSettings, r_string, section, "legacy_texture", nullptr); + if (explicitTex && explicitTex[0]) + { + xr_strcpy(outTexture, sizeof(outTexture), explicitTex); + return; + } - uFlags &= ~fsValid; - vTS.set(1.f, 1.f); // обязательно !!! + // 2. Hardcoded table for the GW sections. Old font texture names have no + // common naming pattern (arial_14_1024, graff_19_1024, letter_16_1024, + // hud_01, console_02), so a table is the only reliable mapping. + // Paths are relative to $game_textures$ (the old engine stored fonts in textures\ui\). + struct SectionToTex { pcstr section; pcstr tex; }; + static const SectionToTex table[] = + { + { "stat_font", "ui\\ui_font_hud_01" }, + { "hud_font_medium", "ui\\ui_font_hud_02" }, + { "hud_font_di", "ui\\ui_font_console_02" }, + { "hud_font_di2", "ui\\ui_font_console_02" }, + { "ui_font_arial_14", "ui\\ui_font_arial_14_1024" }, + { "ui_font_graffiti19_russian", "ui\\ui_font_graff_19_1024" }, + { "ui_font_graffiti22_russian", "ui\\ui_font_graff_22_1024" }, + { "ui_font_graff_32", "ui\\ui_font_graff_32_1024" }, + { "ui_font_graff_40", "ui\\ui_font_graff_40_1024" }, + { "ui_font_graff_50", "ui\\ui_font_graff_50_1024" }, + { "ui_font_letterica16_russian", "ui\\ui_font_letter_16_1024" }, + { "ui_font_letterica18_russian", "ui\\ui_font_letter_18_1024" }, + { "ui_font_letter_25", "ui\\ui_font_letter_25_1024" }, + { "ui_font_letterica16", "ui\\ui_font_letter_16_1024" }, + { "ui_font_letterica18", "ui\\ui_font_letter_18_1024" }, + { "ui_font_letterica25", "ui\\ui_font_letter_25_1024" }, + { "ui_font_graffiti19", "ui\\ui_font_graff_19_1024" }, + { "ui_font_graffiti22", "ui\\ui_font_graff_22_1024" }, + }; + for (const auto& entry : table) + { + if (xr_strcmp(section, entry.section) == 0) + { + xr_strcpy(outTexture, sizeof(outTexture), entry.tex); + return; + } + } + + // 3. Heuristic fallback (may not match real files; emit a hint to the log). + xr_string base = "ui\\ui_font_"; + base += name; + XRay::Utf8::ToLowerAscii(base.data()); + base += "_1024"; + xr_strcpy(outTexture, sizeof(outTexture), base.c_str()); + Msg("~ resolve_legacy_texture: section '%s' not in table, heuristic '%s'", section, outTexture); +} + +void CGameFont::InitializeLegacy(pcstr name, pcstr shader, pcstr /*style*/, u32 size) +{ + Data.IsBitmap = true; + Data.HasUnicodeCharmap = false; // metrics are CP1251-keyed; we translate to Unicode + Data.OpenType = false; - eCurrentAlignment = alLeft; - vInterval.set(1.f, 1.f); + GlyphData.clear(); + Data.Size = static_cast(size); - strings.reserve(128); + // 1. Resolve the texture + .ini path (located in $game_textures$, old layout). + string_path textureName; + resolve_legacy_texture(Data.TextureName ? Data.TextureName : name, name, textureName); - // check ini exist - string_path fn, buf; - xr_strcpy(buf, cTexture); - if (strext(buf)) - *strext(buf) = 0; - R_ASSERT2(FS.exist(fn, "$game_textures$", buf, ".ini"), fn); - CInifile* ini = CInifile::Create(fn); + string_path iniPath; + if (!FS.exist(iniPath, "$game_textures$", textureName, ".ini")) + { + // Hard failure: do NOT silently return, otherwise pTexture stays null + // and the renderer crashes with a null dereference. Give a clear message. + R_ASSERT3(false, "r_font_legacy: no legacy font .ini found for section. " + "Check fonts.ltx 'legacy_texture' or disable r_font_legacy.", + Data.TextureName ? Data.TextureName : name); + } - nNumChars = 0x100; - TCMap = (Fvector*)xr_realloc((void*)TCMap, nNumChars * sizeof(Fvector)); + Msg("~ Legacy font '%s' -> texture '%s', ini '%s'", name, textureName, iniPath); + + // 2. Read the .ini glyph metrics. The old engine supported four .ini formats; + // we handle all of them and convert each into the UTF-8 GlyphData map. + // All formats are keyed by CP1251 byte (0..255), so every glyph is stored + // under its Unicode codepoint via CP1251ByteToCodepoint(). + CInifile* ini = CInifile::Create(iniPath, true); // read-only + + // Helper: store a glyph under its Unicode codepoint. + // - TextureCoord: pixel rect of the glyph bitmap in the .dds atlas. + // - advance: total horizontal step to the next glyph. In the old engine this + // was the per-glyph width (right-left); the new renderer derives the step + // from abcA+abcB+abcC, so we put the bitmap width into abcB and the + // remaining advance into abcC. + // NOTE: the space glyph MUST be stored too — otherwise it has no advance and + // words run together. The old engine rendered space as an (empty) atlas cell + // with a real advance; we do the same. + auto addGlyph = [&](u8 cp1251Byte, LONG left, LONG top, LONG right, LONG bottom, float advance) + { + const int codepoint = static_cast(XRay::Utf8::CP1251ByteToCodepoint(cp1251Byte)); + const LONG bitmapWidth = std::max(0, right - left); + Glyph glyph{}; + glyph.TextureCoord = { left, top, right, bottom }; + glyph.Abc.abcA = 0; + glyph.Abc.abcB = static_cast(bitmapWidth); + // post-bearing = advance - bitmapWidth, so abcA+abcB+abcC == advance. + glyph.Abc.abcC = static_cast(std::max(0.0f, advance - bitmapWidth)); + glyph.yOffset = 0; + GlyphData[codepoint] = glyph; + }; + + float fontHeight = 16.0f; // sensible default if no section provides it + char keyBuf[16]; if (ini->section_exist("mb_symbol_coords")) { - nNumChars = 0x10000; - TCMap = (Fvector*)xr_realloc((void*)TCMap, nNumChars * sizeof(Fvector)); - uFlags |= fsMultibyte; - fHeight = ini->r_float("mb_symbol_coords", "height"); + // Multibyte format: 5-digit keys (codepoint index, 0..0xFFFF). Each row is + // "u, v, right" (r_fvector3). Here the key IS the codepoint directly (no + // CP1251 translation) — the atlas was built for Unicode BMP already. + // The old engine set TCMap.z = 1 + right - left (note the +1), used for both + // the quad width and the advance. The quad texture spans [left..right]. + fontHeight = ini->r_float("mb_symbol_coords", "height"); + for (u32 cp = 0; cp <= 0xFFFF; ++cp) + { + xr_sprintf(keyBuf, sizeof(keyBuf), "%05d", cp); + if (!ini->line_exist("mb_symbol_coords", keyBuf)) + continue; + const Fvector3 v = ini->r_fvector3("mb_symbol_coords", keyBuf); + // For mb format the key is already a codepoint; store it directly. + // advance = 1 + (right - left), matching the old engine's TCMap.z. + const float advance = 1.0f + std::max(0.0f, v.z - v.x); + Glyph glyph{}; + glyph.TextureCoord = { static_cast(v.x), static_cast(v.y), static_cast(v.z), static_cast(v.y + fontHeight) }; + glyph.Abc.abcA = 0; + glyph.Abc.abcB = static_cast(advance); + glyph.Abc.abcC = 0; + glyph.yOffset = 0; + GlyphData[static_cast(cp)] = glyph; + } + } + else if (ini->section_exist("symbol_coords")) + { + // 256-entry format keyed by CP1251 byte. Row is "left, top, right" (r_fvector3). + // The old engine set TCMap.z = right - left (width_correction was commented out: + // `float d = 0.0f;`), and used that value for BOTH the quad width and the advance. + // This is proportional spacing — narrow glyphs advance less, wide glyphs more. + fontHeight = ini->r_float("symbol_coords", "height"); + for (u32 cp1251Byte = 0; cp1251Byte < 256; ++cp1251Byte) + { + xr_sprintf(keyBuf, sizeof(keyBuf), "%03d", cp1251Byte); + if (!ini->line_exist("symbol_coords", keyBuf)) + continue; + const Fvector3 v = ini->r_fvector3("symbol_coords", keyBuf); + const LONG left = static_cast(v.x); + const LONG top = static_cast(v.y); + const LONG right = static_cast(v.z); + // advance = right - left (matches old engine exactly; no width_correction). + addGlyph(static_cast(cp1251Byte), left, top, right, + static_cast(top + fontHeight), static_cast(right - left)); + } + } + else if (ini->section_exist("char widths")) + { + // Per-character widths on a uniform grid (cpl = 16). Row is a single width + // value which IS the advance (the old engine used it directly as TCMap.z). + fontHeight = ini->r_float("char widths", "height"); + const int cpl = 16; + for (u32 cp1251Byte = 0; cp1251Byte < 256; ++cp1251Byte) + { + xr_sprintf(keyBuf, sizeof(keyBuf), "%d", cp1251Byte); + if (!ini->line_exist("char widths", keyBuf)) + continue; + const float w = ini->r_float("char widths", keyBuf); + const LONG left = static_cast((cp1251Byte % cpl) * fontHeight); + const LONG top = static_cast((cp1251Byte / cpl) * fontHeight); + addGlyph(static_cast(cp1251Byte), left, top, static_cast(left + w), + static_cast(top + fontHeight), w); + } + } + else if (ini->section_exist("font_size")) + { + // Monospace grid: every glyph has the same width/height. advance = width. + fontHeight = ini->r_float("font_size", "height"); + const float width = ini->r_float("font_size", "width"); + const int cpl = ini->r_s32("font_size", "cpl"); + for (u32 cp1251Byte = 0; cp1251Byte < 256; ++cp1251Byte) + { + const LONG left = static_cast((cp1251Byte % cpl) * width); + const LONG top = static_cast((cp1251Byte / cpl) * fontHeight); + addGlyph(static_cast(cp1251Byte), left, top, static_cast(left + width), + static_cast(top + fontHeight), width); + } + } + else + { + // No recognized section — fall back to a monospace 8x8 grid so the font + // is at least usable rather than empty. + Msg("! r_font_legacy: no recognized metrics section in '%s', using 8x8 fallback", iniPath); + fontHeight = 8.0f; + const float width = 8.0f; + const int cpl = 16; + for (u32 cp1251Byte = 0; cp1251Byte < 256; ++cp1251Byte) + { + const LONG left = static_cast((cp1251Byte % cpl) * width); + const LONG top = static_cast((cp1251Byte / cpl) * fontHeight); + addGlyph(static_cast(cp1251Byte), left, top, static_cast(left + width), + static_cast(top + fontHeight), width); + } + } + + CInifile::Destroy(ini); + + // 3. Match the resolution/DPI scaling applied to FreeType TTF fonts (Initialize2) + // so legacy bitmap fonts end up the same visual size at the same ui_text_scale. + // TTF bakes res_scale*ppi_scale into the rasterized atlas; for bitmap fonts we + // scale only the on-screen advance/quad width and line height. The atlas texture + // coordinates must stay in the original .dds pixel space, otherwise glyphs sample + // the wrong atlas cells and text appears garbled. + const bool is_res_depend = !!READ_IF_EXISTS(pSettings, r_bool, Data.TextureName, "res_depend", TRUE); + const bool is_dpi_depend = !!READ_IF_EXISTS(pSettings, r_bool, Data.TextureName, "dpi_depend", !is_res_depend); + float res_scale = 1.0f; + float ppi_scale = 1.0f; + if (is_res_depend && Device.dwHeight > 0) + res_scale = float(Device.dwHeight) / 900.0f; +#if defined(XR_PLATFORM_WINDOWS) + if (is_dpi_depend) + { + HDC hDCScreen = GetDC(NULL); + const float Hmm = (float)GetDeviceCaps(hDCScreen, VERTSIZE); + const float Wmm = (float)GetDeviceCaps(hDCScreen, HORZSIZE); + const float Hpx = (float)GetDeviceCaps(hDCScreen, VERTRES); + const float Wpx = (float)GetDeviceCaps(hDCScreen, HORZRES); + ReleaseDC(NULL, hDCScreen); + int ppi = int(25.4f * sqrt(Hpx * Hpx + Wpx * Wpx) / sqrt(Hmm * Hmm + Wmm * Wmm)); + if (ppi > 0) + ppi_scale = float(ppi) / 92.0f; + } +#endif + const float totalScale = res_scale * ppi_scale; + if (totalScale != 1.0f) + { + // Bitmap fonts use a fixed-resolution .dds atlas. Scaling the texture + // coordinates would sample the wrong atlas cells and produce garbled + // glyphs. Only the on-screen advance/quad width and line height are + // scaled so legacy fonts match the visual size of FreeType TTF fonts. + for (auto& kv : GlyphData) + { + kv.second.Abc.abcA = static_cast(kv.second.Abc.abcA * totalScale); + kv.second.Abc.abcB = static_cast(kv.second.Abc.abcB * totalScale); + kv.second.Abc.abcC = static_cast(kv.second.Abc.abcC * totalScale); + } + fontHeight *= totalScale; + } + + // 4. Load the .dds texture via the standard texturing path (the engine + // decompresses DXT5 itself — no manual decoding needed). + pFontRender->Initialize(shader, textureName); + + // 5. Set the font height metric so CurrentHeight_() / line spacing work. + fCurrentHeight = fontHeight; + + Msg("* Legacy font '%s' loaded: %u glyphs, height=%.1f, scale=%.2f", + name, (u32)GlyphData.size(), fontHeight, totalScale); +} + +void CGameFont::Initialize2(pcstr name, pcstr shader, pcstr style, u32 size) +{ + // Font renderer selection: + // - `r_font_legacy` == 1 forces the legacy X-Ray bitmap fonts (.dds + .ini). + // - otherwise FreeType TTF is used, but if no TTF/OTF files exist in + // $game_fonts$, we also fall back to legacy automatically. + // The legacy path converts CP1251-keyed metrics into the UTF-8 GlyphData map + // and reuses the same UTF-8 renderer (dxFontRender::OnRender) — the old + // texture-based rendering code (ImprintChar/TCMap) is never restored. + const bool forceLegacy = g_font_legacy; + bool noTtf = false; + if (!forceLegacy) + { + FS_FileSet ttfFiles; + const size_t ttfCount = FS.file_list(ttfFiles, _game_fonts_, FS_ListFiles, "*.ttf"); + FS_FileSet otfFiles; + const size_t otfCount = FS.file_list(otfFiles, _game_fonts_, FS_ListFiles, "*.otf"); + noTtf = (ttfCount == 0 && otfCount == 0); + } + + if (forceLegacy || noTtf) + { + if (forceLegacy) + Msg("~ r_font_legacy = 1, using legacy bitmap font for '%s'", name); + else + Msg("~ No TTF/OTF fonts in $game_fonts$, using legacy bitmap font for '%s'", name); + InitializeLegacy(name, shader, style, size); + return; + } + + if (!bFreetypeInitialized) + { + InitializeFreetype(); + bFreetypeInitialized = true; + } + + if (OurFont) + { + FT_Done_Face(OurFont); + OurFont = nullptr; + } + if (m_fontReader) + { + FS.r_close(m_fontReader); + m_fontReader = nullptr; + } + + GlyphData.clear(); + + ZeroMemory(&Style, sizeof(Style)); + Data.Size = static_cast(size); + + if (style != nullptr) + { + xr_string StyleDesc(style); + xr_vector StyleTokens = split(StyleDesc, '|'); + for (const xr_string& token : StyleTokens) + { + if (token == "bold") + Style.bold = 1; + else if (token == "italic") + Style.italic = 1; + else if (token == "underline") + Style.underline = 1; + else if (token == "strike") + Style.strike = 1; + } + } + xr_vector FontBitmap; + const u32 bitmapSize = TextureDimension * TextureDimension; + FontBitmap.assign(bitmapSize, 0); + + int ppi = 96; + float res_scale = 1.0f; + float ppi_scale = 1.0f; - fXStep = ceil(fHeight / 2.0f); + // Font scaling flags are platform-independent; read them before the + // platform-specific PPI block so resolution-based scaling works on POSIX too. + const bool is_res_depend = !!READ_IF_EXISTS(pSettings, r_bool, Data.TextureName, "res_depend", TRUE); + const bool is_dpi_depend = !!READ_IF_EXISTS(pSettings, r_bool, Data.TextureName, "dpi_depend", !is_res_depend); - // Searching for the first valid character + // Resolution-dependent scaling is the same on every platform: scale relative + // to the reference 900px height. Without this, fonts stay tiny on 4K on POSIX. + if (is_res_depend && Device.dwHeight > 0) + res_scale = float(Device.dwHeight) / 900.0f; - Fvector vFirstValid = {0, 0, 0}; +#if defined(XR_PLATFORM_WINDOWS) + HDC hDCScreen = GetDC(NULL); - if (ini->line_exist("mb_symbol_coords", "09608")) + auto Hmm = (float)GetDeviceCaps(hDCScreen, VERTSIZE); + auto Wmm = (float)GetDeviceCaps(hDCScreen, HORZSIZE); + auto Hpx = (float)GetDeviceCaps(hDCScreen, VERTRES); + auto Wpx = (float)GetDeviceCaps(hDCScreen, HORZRES); + + ReleaseDC(NULL, hDCScreen); + + ppi = int(25.4f * sqrt(Hpx * Hpx + Wpx * Wpx) / sqrt(Hmm * Hmm + Wmm * Wmm)); + if (ppi <= 0) + ppi = 96; + + if (is_dpi_depend) + ppi_scale = float(ppi) / 92.0f; +#endif + + auto fHeight = float(size) * res_scale * ppi_scale; + if (fHeight < 1.0f) + fHeight = 1.0f; + + pcstr fontPrefix = StringTable().GetCurrentFontPrefix().c_str(); + if (!fontPrefix || !fontPrefix[0]) + { + fontPrefix = StringTable().GetCurrentLanguage().c_str(); + if (!fontPrefix || !fontPrefix[0]) { - Fvector v = ini->r_fvector3("mb_symbol_coords", "09608"); - vFirstValid.set(v.x, v.y, 1 + v[2] - v[0]); + Msg("! Font prefix/language is not set, falling back to 'eng'"); + fontPrefix = "eng"; } else - for (u32 i = 0; i < nNumChars; i++) + { + Msg("! Font prefix is not set, using current language '%s'", fontPrefix); + } + } + + auto try_open_font = [&](pcstr prefix, pcstr fontName, string_path& outPath) -> IReader* + { + xr_string path = prefix; + path += "\\"; + path += fontName; + path += ".ttf"; + + FS.update_path(outPath, _game_fonts_, path.c_str()); + return FS.r_open(outPath); + }; + + // Case-insensitive fallback for POSIX file systems. Font names in fonts.ltx + // are lowercased before lookup (see ToLowerAscii above), but the actual TTF + // files on disk may use mixed case (e.g. "Graffiti1CTT.ttf"). On case- + // sensitive file systems the direct open then fails. Walk the font folder + // once and pick the entry that matches the requested path case-insensitively. + // FS_File::name is already stored lower-case, so the comparison is cheap. + auto try_open_font_ci = [&](pcstr prefix, pcstr fontName, string_path& outPath) -> IReader* + { + if (IReader* direct = try_open_font(prefix, fontName, outPath)) + return direct; + + FS_FileSet files; + if (FS.file_list(files, _game_fonts_, FS_ListFiles, "*.ttf") == 0) + return nullptr; + + // file_list names are relative to $game_fonts$ and lower-case, e.g. "rus\arial.ttf". + xr_string wanted = xr_string(prefix) + _DELIMITER + fontName + ".ttf"; + XRay::Utf8::ToLowerAscii(wanted.data()); + for (const FS_File& f : files) + { + if (f.name == wanted) { - xr_sprintf(buf, sizeof(buf), "%05d", i); - if (ini->line_exist("mb_symbol_coords", buf)) + FS.update_path(outPath, _game_fonts_, f.name.c_str()); + if (IReader* r = FS.r_open(outPath)) { - Fvector v = ini->r_fvector3("mb_symbol_coords", buf); - vFirstValid.set(v.x, v.y, 1 + v[2] - v[0]); - break; + Msg("~ Font: case-insensitive match '%s'", f.name.c_str()); + return r; } } + } + return nullptr; + }; + + string_path FullPath{}; + IReader* FontFile = try_open_font_ci(fontPrefix, name, FullPath); + if (FontFile == nullptr && xr_stricmp(fontPrefix, "eng") != 0) + { + Msg("! Can't open font '%s' in prefix '%s', trying 'eng'", name, fontPrefix); + FontFile = try_open_font_ci("eng", name, FullPath); + } + + if (FontFile == nullptr) + { + Msg("! Can't open font file %s", name); + + FontFile = try_open_font_ci(fontPrefix, "default", FullPath); + if (FontFile == nullptr && xr_stricmp(fontPrefix, "eng") != 0) + FontFile = try_open_font_ci("eng", "default", FullPath); + + R_ASSERT3(FontFile != nullptr, "Can't find default font for prefix: %s", fontPrefix); + } - // Filling entire character table + FT_Error FTError = FT_New_Memory_Face(FreetypeLib, (FT_Byte*)FontFile->pointer(), FontFile->length(), 0, &OurFont); + R_ASSERT3(FTError == 0, "FT_New_Memory_Face returned error", FullPath); + + m_fontReader = FontFile; + + // Try to use the Unicode charmap. If the font only provides a legacy + // Windows-1251 cmap (common for old STALKER TTFs), we will translate + // character codes to Unicode codepoints when building the atlas. + Data.HasUnicodeCharmap = (FT_Select_Charmap(OurFont, FT_ENCODING_UNICODE) == 0); + if (!Data.HasUnicodeCharmap) + Msg("! Font '%s' has no Unicode charmap, using CP1251 translation", name); + + u32 TargetX = 0; + u32 TargetY = 0; + u32 TargetX2 = 0; + u32 TargetY2 = 0; + + FT_Size_RequestRec req; + req.type = FT_SIZE_REQUEST_TYPE_CELL; + req.width = 0; + req.height = (uint32_t)(fHeight * 64.0f); + req.horiResolution = 0; + req.vertResolution = 0; + FTError = FT_Request_Size(OurFont, &req); + if (FTError != 0) + { + Msg("! FT_Request_Size failed for '%s' (height=%.2f), falling back to FT_Set_Pixel_Sizes", name, fHeight); + FTError = FT_Set_Pixel_Sizes(OurFont, 0, size); + R_ASSERT3(FTError == 0, "FT_Set_Pixel_Sizes returned error", FullPath); + } + +#define FT_CEIL(X) (((X + 63) & -64) / 64) + + float FontSizeInPixels = (float)(OurFont->size->metrics.ascender - OurFont->size->metrics.descender) / 64.0f; + if (FontSizeInPixels < 1.0f) + FontSizeInPixels = float(size); + + u32 CurrentRowHeight = (u32)FontSizeInPixels; + + auto CopyGlyphImageToAtlas = [this, &FontBitmap, &TargetX, &TargetX2, &TargetY, &TargetY2, &CurrentRowHeight, FontSizeInPixels](FT_Bitmap& GlyphBitmap) + { + const u32 glyphWidth = GlyphBitmap.width; + const u32 glyphRows = GlyphBitmap.rows; + + if (glyphWidth > TextureDimension || glyphRows > TextureDimension) + { + Msg("! Glyph too large: %dx%d, skipping", glyphWidth, glyphRows); + return; + } - for (u32 i = 0; i < nNumChars; i++) + TargetX2 = TargetX + glyphWidth; + if (TargetX2 > TextureDimension) { - xr_sprintf(buf, sizeof(buf), "%05d", i); - if (ini->line_exist("mb_symbol_coords", buf)) + TargetX = 0; + TargetX2 = glyphWidth; + TargetY += CurrentRowHeight + 5; + CurrentRowHeight = (u32)FontSizeInPixels; + } + + TargetY2 = TargetY + std::max((u32)FontSizeInPixels, glyphRows); + CurrentRowHeight = std::max(CurrentRowHeight, TargetY2 - TargetY); + R_ASSERT2(TargetY2 <= TextureDimension, "Font too large, or dimension texture is too small"); + + const u32 TargetYSaved = TargetY; + if (glyphRows <= (u32)FontSizeInPixels) + TargetY = TargetY + (u32)(FontSizeInPixels - (float)glyphRows); + + const u32 yEnd = std::min(TargetY2, TargetY + glyphRows); + + const int pitch = GlyphBitmap.pitch; + u8* srcRow = GlyphBitmap.buffer; + if (pitch < 0 && glyphRows > 0) + srcRow += pitch * (glyphRows - 1); + + switch (GlyphBitmap.pixel_mode) + { + case FT_PIXEL_MODE_GRAY: + { + for (u32 y = TargetY; y < yEnd; ++y, srcRow += pitch) { - Fvector v = ini->r_fvector3("mb_symbol_coords", buf); - TCMap[i].set(v.x, v.y, 1 + v[2] - v[0]); + for (u32 x = TargetX; x < TargetX2; ++x) + { + const u8 SourcePixel = srcRow[x - TargetX]; + + u32 FinalPixel = SourcePixel; + FinalPixel |= (SourcePixel << 8); + FinalPixel |= (SourcePixel << 16); + FinalPixel |= (SourcePixel << 24); + + FontBitmap[(y * TextureDimension) + x] = FinalPixel; + } } - else - TCMap[i] = vFirstValid; // "unassigned" unprintable characters } + break; + case FT_PIXEL_MODE_MONO: + { + for (u32 y = TargetY; y < yEnd; ++y, srcRow += pitch) + { + for (u32 x = TargetX; x < TargetX2; ++x) + { + const u32 sx = x - TargetX; + const u8 SourcePixel = (srcRow[sx >> 3] >> (7 - (sx & 7))) & 1 ? 0xFF : 0x00; - // Special case for space - TCMap[0x0020].set(0, 0, 0); - // Special case for ideographic space - TCMap[0x3000].set(0, 0, 0); - } - else if (ini->section_exist("symbol_coords")) - { - float d = 0.0f; - //. if(ini->section_exist("width_correction")) - //. d = ini->r_float("width_correction", "value"); + u32 FinalPixel = SourcePixel; + FinalPixel |= (SourcePixel << 8); + FinalPixel |= (SourcePixel << 16); + FinalPixel |= (SourcePixel << 24); - fHeight = ini->r_float("symbol_coords", "height"); - for (u32 i = 0; i < nNumChars; i++) + FontBitmap[(y * TextureDimension) + x] = FinalPixel; + } + } + } + break; + case FT_PIXEL_MODE_BGRA: { - xr_sprintf(buf, sizeof(buf), "%03d", i); - Fvector v = ini->r_fvector3("symbol_coords", buf); - TCMap[i].set(v.x, v.y, v[2] - v[0] + d); + for (u32 y = TargetY; y < yEnd; ++y, srcRow += pitch) + { + const u32* SourceImage = (const u32*)srcRow; + for (u32 x = TargetX; x < TargetX2; ++x) + { + const u32 SourcePixel = SourceImage[x - TargetX]; + + const u8 Alpha = (SourcePixel & 0x000000FF); + const u8 Red = (SourcePixel & 0x0000FF00) >> 8; + const u8 Green = (SourcePixel & 0x00FF0000) >> 16; + const u8 Blue = (SourcePixel & 0xFF000000) >> 24; + + u32 FinalPixel = Alpha; + FinalPixel |= (u32(Blue) << 8); + FinalPixel |= (u32(Green) << 16); + FinalPixel |= (u32(Red) << 24); + + FontBitmap[(y * TextureDimension) + x] = FinalPixel; + } + } } - } - else + break; + default: + Msg("! Unsupported font bitmap pixel mode: %d", GlyphBitmap.pixel_mode); + break; + } + + TargetY = TargetYSaved; + }; + + u32 index = 0; + + auto LoadGlyph = [&](FT_UInt glyphIndex, FT_ULong charCode) { - if (ini->section_exist("char widths")) + // We only support the Basic Multilingual Plane (BMP). + if (charCode > 0xFFFF) + return; + + // FreeType already gave us the glyph index for the current charmap. + // Use it directly instead of FT_Get_Char_Index, which would interpret + // the value against the active charmap again. + int storageCode = static_cast(charCode); + if (!Data.HasUnicodeCharmap) { - fHeight = ini->r_float("char widths", "height"); - int cpl = 16; - for (u32 i = 0; i < nNumChars; i++) + // Legacy Windows-1251 font: re-key the glyph under the Unicode + // codepoint that corresponds to the CP1251 byte the font uses. + // Bytes 0x00..0xFF map 1:1; any native code >0xFF has no reliable + // CP1251 mapping (and cannot be addressed via UTF-8 text), so skip it + // instead of storing it under a key the renderer would never look up. + if (charCode <= 0xFF) + storageCode = static_cast(XRay::Utf8::CP1251ByteToCodepoint(static_cast(charCode))); + else { - xr_sprintf(buf, sizeof(buf), "%d", i); - float w = ini->r_float("char widths", buf); - TCMap[i].set((i % cpl) * fHeight, (i / cpl) * fHeight, w); + Msg("! Font '%s': non-Unicode glyph with native code 0x%04X (>0xFF) " + "is not addressable via UTF-8, skipping", name, charCode); + return; } } - else + + FTError = FT_Load_Glyph(OurFont, glyphIndex, FT_LOAD_RENDER | FT_LOAD_TARGET_NORMAL); + if (FTError != 0) { - R_ASSERT(ini->section_exist("font_size")); - fHeight = ini->r_float("font_size", "height"); - float width = ini->r_float("font_size", "width"); - const int cpl = ini->r_s32("font_size", "cpl"); - for (u32 i = 0; i < nNumChars; i++) - TCMap[i].set((i % cpl) * width, (i / cpl) * fHeight, width); + Msg("! FT_Load_Glyph failed for glyph %u (char 0x%04X) in font '%s': %d", + glyphIndex, charCode, FullPath, FTError); + return; } + FT_GlyphSlot Glyph = OurFont->glyph; + FT_Glyph_Metrics& GlyphMetrics = Glyph->metrics; + + CopyGlyphImageToAtlas(Glyph->bitmap); + + RECT region; + region.left = TargetX; + region.right = TargetX + Glyph->bitmap.width; + region.top = TargetY; + region.bottom = long(TargetY2); + + ABC widths; + widths.abcA = FT_CEIL(GlyphMetrics.horiBearingX); + widths.abcB = Glyph->bitmap.width; + widths.abcC = FT_CEIL(GlyphMetrics.horiAdvance) - widths.abcB - widths.abcA; + + int GlyphTopScanlineOffset = int(FontSizeInPixels - Glyph->bitmap.rows); + int yOffset = -Glyph->bitmap_top - GlyphTopScanlineOffset; + yOffset += (int)FontSizeInPixels; // Return back to the center pos + yOffset -= (int)(FontSizeInPixels / 4); + + GlyphData[storageCode] = { region, widths, yOffset }; + + TargetX = TargetX2; + TargetX += 4; + }; + + auto charCode = FT_Get_First_Char(OurFont, &index); + while (index != 0) + { + LoadGlyph(index, charCode); + charCode = FT_Get_Next_Char(OurFont, charCode, &index); } - fCurrentHeight = fHeight; + fCurrentHeight = FontSizeInPixels; - CInifile::Destroy(ini); + // Account for the last row's actual height when sizing the atlas. + TargetY2 = std::max(TargetY2, TargetY + CurrentRowHeight); - // Shading - pFontRender->Initialize(cShader, cTexture); + string128 textureName; + xr_sprintf(textureName, sizeof(textureName), "$user$%s", Data.TextureName ? Data.TextureName : Data.Name); + + auto TargetDemensionY = 16u; + while (TargetDemensionY < TargetY2) + TargetDemensionY *= 2u; + + Msg("* Font '%s' metrics: fHeight=%.2f FontSizeInPixels=%.2f ppi=%d res_scale=%.3f ppi_scale=%.3f atlas=%dx%d used=%d", + Data.Name, fHeight, FontSizeInPixels, ppi, res_scale, ppi_scale, TextureDimension, TargetDemensionY, TargetY2); + R_ASSERT2(TargetDemensionY <= TextureDimension, "Font too large, or dimension texture is too small"); + + pFontRender->CreateFontAtlas(TextureDimension, TargetDemensionY, textureName, FontBitmap.data()); + + pFontRender->Initialize(shader, textureName); } -CGameFont::~CGameFont() +void CGameFont::OutSet(float x, float y) { - if (TCMap) - xr_free(TCMap); + fCurrentX = x; + fCurrentY = y; +} - // Shading - GEnv.RenderFactory->DestroyFontRender(pFontRender); +void CGameFont::OutSetI(float x, float y) { OutSet(DI2PX(x), DI2PY(y)); } + +void CGameFont::OnRender() +{ + ZoneScoped; + ScopeLock g(&s_cs); + if (!strings.empty()) + { + pFontRender->OnRender(*this); + strings.clear(); + } } -static inline float DI2PX(float x) +float CGameFont::SizeOf_(int cChar) { - return float(iFloor((x + 1) * float(Device.dwWidth) * 0.5f)); + if (cChar < 0) + cChar = u8(cChar); + + return WidthOf(cChar); } -static inline float DI2PY(float y) + +float CGameFont::SizeOf_(pcstr s) { - return float(iFloor((y + 1) * float(Device.dwHeight) * 0.5f)); + return WidthOf(s); } -void CGameFont::OutSet(float x, float y) +float CGameFont::CurrentHeight_() { return fCurrentHeight * vInterval.y * g_text_scale; } + +void CGameFont::SetHeight(float S) { fCurrentHeight = S; } + +void CGameFont::SetHeightI(float S) { - fCurrentX = x; - fCurrentY = y; + fCurrentHeight = S * Device.dwHeight; } -void CGameFont::OutSetI(float x, float y) { OutSet(DI2PX(x), DI2PY(y)); } -u32 CGameFont::smart_strlen(pcstr S) { return (IsMultibyte() ? mbhMulti2Wide(NULL, NULL, 0, S) : xr_strlen(S)); } +const CGameFont::Glyph* CGameFont::GetGlyphInfo(int ch) +{ + auto symbolInfoIterator = GlyphData.find(ch); + if (symbolInfoIterator == GlyphData.end()) + return nullptr; -std::pair CGameFont::get_actions_text_length(pcstr s) + return &symbolInfoIterator->second; +} + +float CGameFont::WidthOf(int ch) +{ + if (ch == '\t' || ch == '\n') + return 0.f; + + if (const Glyph* glyphInfo = GetGlyphInfo(ch)) + // Apply the global text scale here (terminal width helper): WidthOf(pcstr) + // sums per-codepoint widths through this method, so scaling once here + // keeps both single-char and whole-string measurements consistent. + return float(glyphInfo->Abc.abcA + glyphInfo->Abc.abcB + glyphInfo->Abc.abcC) * g_text_scale; + + return 0.f; +} + +float CGameFont::WidthOf(pcstr str) { - u32 count = 0; - u32 length = 0; + if (!str || !str[0]) return 0; + + float size = 0; + const float spacing = GetLetterSpacing(); - while (s[0]) + if (XRay::Utf8::IsValid(str)) { - if (s[0] == GAME_ACTION_MARK) + for (const char* p = str; *p;) { - static_assert(kLASTACTION < type_max, "Modify the code to have more than 255 actions."); - ++s; - const EGameActions actionId = static_cast(s[0]); - - cpcstr binding = GetActionBinding(actionId); - length += xr_strlen(binding); - ++count; + size_t len = 0; + const int cp = (int)XRay::Utf8::Decode(p, len); + p += len ? len : 1; + size += WidthOf(cp); + size += spacing; } - ++s; + size -= spacing; } - return { count, length }; + else + { + const int length = (int)xr_strlen(str); + for (int i = 0; i < length; i++) + { + size += WidthOf((u8)str[i]); + size += spacing; + } + size -= spacing; + } + + return size; } -void CGameFont::OnRender() +u32 CGameFont::smart_strlen(pcstr S) { - if (!strings.empty()) - { - pFontRender->OnRender(*this); - strings.clear(); - } + if (!S || !S[0]) + return 0; + + // All TTF/OTF fonts are multibyte; count Unicode codepoints for UTF-8 input. + if (IsMultibyte() && XRay::Utf8::IsValid(S)) + return static_cast(XRay::Utf8::LengthCodepoints(S)); + + return static_cast(xr_strlen(S)); } u16 CGameFont::GetCutLengthPos(float fTargetWidth, pcstr pszText) { VERIFY(pszText); - xr_wide_char wsStr[MAX_MB_CHARS], wsPos[MAX_MB_CHARS]; - float fCurWidth = 0.0f, fDelta = 0.0f; + float fCurWidth = 0.0f; + size_t lastPos = 0; - u16 len = mbhMulti2Wide(wsStr, wsPos, MAX_MB_CHARS, pszText); + // Fallback for non-UTF-8 strings: iterate bytes. + if (!XRay::Utf8::IsValid(pszText)) + { + for (pcstr p = pszText; *p; ++p) + { + const float fDelta = WidthOf(static_cast(*p)); + if ((fCurWidth + fDelta) > fTargetWidth) + break; + + fCurWidth += fDelta; + lastPos = static_cast(p - pszText) + 1; + } + return static_cast(lastPos); + } - u16 i = 1; - for (; i <= len; i++) + for (pcstr p = pszText; *p;) { - fDelta = GetCharTC(wsStr[i]).z - 2; + size_t len = 0; + const u32 cp = XRay::Utf8::Decode(p, len); + const pcstr next = p + (len ? len : 1); - if (IsNeedSpaceCharacter(wsStr[i])) - fDelta += fXStep; + float fDelta = WidthOf(static_cast(cp)); + if (IsNeedSpaceCharacter(static_cast(cp))) + fDelta += WidthOf(' '); if ((fCurWidth + fDelta) > fTargetWidth) break; - else - fCurWidth += fDelta; + + fCurWidth += fDelta; + lastPos = static_cast(next - pszText); + p = next; } - return wsPos[i - 1]; + return static_cast(lastPos); } u16 CGameFont::SplitByWidth(u16* puBuffer, u16 uBufferSize, float fTargetWidth, pcstr pszText) { VERIFY(puBuffer && uBufferSize && pszText); - xr_wide_char wsStr[MAX_MB_CHARS], wsPos[MAX_MB_CHARS]; - float fCurWidth = 0.0f, fDelta = 0.0f; - u16 nLines = 0; + // Fallback for non-UTF-8 strings: iterate bytes. + if (!XRay::Utf8::IsValid(pszText)) + { + float fCurWidth = 0.0f; + u16 nLines = 0; - u16 len = mbhMulti2Wide(wsStr, wsPos, MAX_MB_CHARS, pszText); + for (pcstr p = pszText; *p; ++p) + { + const float fDelta = WidthOf(static_cast(*p)); + + if ((fCurWidth + fDelta) > fTargetWidth && nLines < uBufferSize) + { + fCurWidth = fDelta; + puBuffer[nLines++] = static_cast(p - pszText); + } + else + fCurWidth += fDelta; + } + return nLines; + } + + float fCurWidth = 0.0f; + u16 nLines = 0; + pcstr p = pszText; + u32 prevCp = 0; + bool hasPrev = false; - for (u16 i = 1; i <= len; i++) + while (*p) { - fDelta = GetCharTC(wsStr[i]).z - 2; + size_t len = 0; + const u32 cp = XRay::Utf8::Decode(p, len); + const pcstr next = p + (len ? len : 1); - if (IsNeedSpaceCharacter(wsStr[i])) - fDelta += fXStep; + float fDelta = WidthOf(static_cast(cp)); + if (IsNeedSpaceCharacter(static_cast(cp))) + fDelta += WidthOf(' '); - if (((fCurWidth + fDelta) > fTargetWidth) && // overlength - (!IsBadStartCharacter(wsStr[i])) && // can start with this character - (i < len) && // is not the last character - ((i > 1) && (!IsBadEndCharacter(wsStr[i - 1]))) // && // do not stop the string on a "bad" character - // ( ( i > 1 ) && ( ! ( ( IsAlphaCharacter( wsStr[ i - 1 ] ) ) && ( IsAlphaCharacter( wsStr[ i ] ) ) ) ) ) - // // do - // not split numbers or words - ) + const bool canBreak = hasPrev && + ((fCurWidth + fDelta) > fTargetWidth) && + (!IsBadStartCharacter(static_cast(cp))) && + (*next != '\0') && + (!IsBadEndCharacter(static_cast(prevCp))); + + if (canBreak && nLines < uBufferSize) { fCurWidth = fDelta; - VERIFY(nLines < uBufferSize); - puBuffer[nLines++] = wsPos[i - 1]; + puBuffer[nLines++] = static_cast(p - pszText); } else fCurWidth += fDelta; + + prevCp = cp; + hasPrev = true; + p = next; } return nLines; } -void CGameFont::MasterOut(bool bCheckDevice, bool bUseCoords, bool bScaleCoords, bool bUseSkip, float _x, float _y, - float _skip, pcstr fmt, va_list p) +void CGameFont::MasterOut(bool bCheckDevice, bool bUseCoords, bool bScaleCoords, bool bUseSkip, + float _x, float _y, float _skip, pcstr fmt, va_list p) { if (bCheckDevice && (!Device.b_is_Active)) return; @@ -301,150 +1059,120 @@ void CGameFont::MasterOut(bool bCheckDevice, bool bUseCoords, bool bScaleCoords, rs.x = (bUseCoords ? (bScaleCoords ? (DI2PX(_x)) : _x) : fCurrentX); rs.y = (bUseCoords ? (bScaleCoords ? (DI2PY(_y)) : _y) : fCurrentY); rs.c = dwCurrentColor; - rs.height = fCurrentHeight; + rs.gradientColor = dwGradientColor; + rs.height = fCurrentHeight * g_text_scale; rs.align = eCurrentAlignment; - int vs_sz = vsnprintf(rs.string, sizeof(rs.string), fmt, p); - // VERIFY( ( vs_sz != -1 ) && ( rs.string[ vs_sz ] == '\0' ) ); + rs.gradient = fGradientEnabled; + rs.gradientMode = fGradientMode; + int vs_sz = vsnprintf(rs.string, sizeof(rs.string), fmt, p); rs.string[sizeof(rs.string) - 1] = 0; if (vs_sz == -1) - { return; - } - - if (vs_sz) - strings.push_back(rs); - - if (bUseSkip) - OutSkip(_skip); -} - -#define MASTER_OUT(CHECK_DEVICE, USE_COORDS, SCALE_COORDS, USE_SKIP, X, Y, SKIP, FMT) \ - \ -{ \ - va_list p; \ - va_start(p, fmt); \ - MasterOut(CHECK_DEVICE, USE_COORDS, SCALE_COORDS, USE_SKIP, X, Y, SKIP, FMT, p); \ - va_end(p); \ - \ -} -void __cdecl CGameFont::OutI(float _x, float _y, pcstr fmt, ...) -{ - MASTER_OUT(false, true, true, false, _x, _y, 0.0f, fmt); -}; - -void __cdecl CGameFont::Out(float _x, float _y, pcstr fmt, ...) -{ - MASTER_OUT(true, true, false, false, _x, _y, 0.0f, fmt); -}; - -void __cdecl CGameFont::OutNext(pcstr fmt, ...) { MASTER_OUT(true, false, false, true, 0.0f, 0.0f, 1.0f, fmt); }; -void CGameFont::OutNextVA(pcstr format, va_list args) -{ - MasterOut(true, false, false, true, 0.0f, 0.0f, 1.0f, format, args); -} - -void CGameFont::OutSkip(float val) { fCurrentY += val * CurrentHeight_(); } -float CGameFont::SizeOf_(const char cChar) -{ - return (GetCharTC((u16)(u8)(((IsMultibyte() && cChar == ' ')) ? 0 : cChar)).z * vInterval.x); -} - -float CGameFont::SizeOf_(pcstr s) -{ - if (!(s && s[0])) - return 0; - - if (IsMultibyte()) + // Expand GAME_ACTION_MARK sequences (0x1B + action_id) to key names + // This was previously only done in the wide-char path (SizeOf_(xr_wide_char*)), + // but the UTF-8 FreeType renderer needs it here. { - xr_wide_char wsStr[MAX_MB_CHARS]; - - mbhMulti2Wide(wsStr, NULL, MAX_MB_CHARS, s); - - return SizeOf_(wsStr); - } + char expanded[sizeof(rs.string)]; + char* dst = expanded; + const char* src = rs.string; + const char* const dst_end = expanded + sizeof(expanded) - 1; - const size_t len = xr_strlen(s); - float X = 0; - if (len) - { - for (size_t j = 0; j < len; j++) + while (*src && dst < dst_end) { - if (s[j] == GAME_ACTION_MARK) + if (*src == GAME_ACTION_MARK) { + ++src; + if (!*src) break; // malformed: marker at end of string static_assert(kLASTACTION < type_max, "Modify the code to have more than 255 actions."); - ++j; - const EGameActions actionId = static_cast(s[j]); + const EGameActions actionId = static_cast(static_cast(*src)); + ++src; - pcstr binding = GetActionBinding(actionId); - while (binding[0]) + cpcstr binding = GetActionBinding(actionId); + if (binding) { - X += GetCharTC((u16)(u8)binding[0]).z; - ++binding; + for (const char* b = binding; *b && dst < dst_end; ++b) + *dst++ = *b; } } else { - X += GetCharTC((u16)(u8)s[j]).z; + *dst++ = *src++; } } + *dst = 0; + xr_strcpy(rs.string, sizeof(rs.string), expanded); } - return (X * vInterval.x); -} - -float CGameFont::SizeOf_(const xr_wide_char* wsStr) -{ - if (!(wsStr && wsStr[0])) - return 0; - const u16 len = wsStr[0]; - float X = 0.0f; - - if (len) + if (!XRay::Utf8::IsValid(rs.string)) { - for (u16 j = 1; j <= len; j++) + // If vsnprintf truncated a UTF-8 string, the last codepoint may be cut in half. + // Try to recover a valid UTF-8 prefix by trimming up to 3 trailing bytes. + // If that does not help, the source is likely CP1251 — convert it. + const xr_string original(rs.string); + size_t len = original.size(); + bool recovered = false; + while (len > 0 && original.size() - len < 3) { - if (wsStr[j] == GAME_ACTION_MARK) - { - static_assert(kLASTACTION < type_max, "Modify the code to have more than 255 actions."); - ++j; - const EGameActions actionId = static_cast(wsStr[j]); - - cpcstr binding = GetActionBinding(actionId); - - xr_wide_char wideBinding[MAX_MB_CHARS]; - const u16 bindingLen = mbhMulti2Wide(wideBinding, nullptr, MAX_MB_CHARS, binding); - - for (u16 i = 1; i <= bindingLen; ++i) - { - float fDelta = GetCharTC(wideBinding[i]).z - 2; - if (IsNeedSpaceCharacter(wsStr[j])) - fDelta += fXStep; - X += fDelta; - } - } - else + --len; + rs.string[len] = 0; + if (XRay::Utf8::IsValid(rs.string)) { - float fDelta = GetCharTC(wsStr[j]).z - 2; - if (IsNeedSpaceCharacter(wsStr[j])) - fDelta += fXStep; - X += fDelta; + recovered = true; + break; } } + + if (!recovered) + { + // The source bytes are not valid UTF-8. Treat them as CP1251 (legacy + // content or a caller that still passes narrow strings) and convert. + // Log it: this path hides real bugs where callers feed CP1251 instead + // of UTF-8, so make it visible instead of silently "fixing" the text. + Msg("! CGameFont: non-UTF-8 text passed to Out*() (treated as CP1251): \"%s\"", original.c_str()); + strncpy_s(rs.string, sizeof(rs.string), original.c_str(), original.size()); + rs.string[sizeof(rs.string) - 1] = 0; + rs.string_utf8 = XRay::Utf8::FromCP1251(rs.string); + } } - return (X * vInterval.x); + + if (vs_sz) + { + ScopeLock g(&s_cs); + strings.push_back(rs); + } + + if (bUseSkip) + OutSkip(_skip); } -float CGameFont::CurrentHeight_() { return fCurrentHeight * vInterval.y; } -void CGameFont::SetHeightI(float S) +#define MASTER_OUT(CHECK_DEVICE, USE_COORDS, SCALE_COORDS, USE_SKIP, X, Y, SKIP, FMT) \ +{ \ + va_list p; \ + va_start(p, fmt); \ + MasterOut(CHECK_DEVICE, USE_COORDS, SCALE_COORDS, USE_SKIP, X, Y, SKIP, FMT, p); \ + va_end(p); \ +} + +void __cdecl CGameFont::OutI(float _x, float _y, pcstr fmt, ...) { - VERIFY(uFlags & fsDeviceIndependent); - fCurrentHeight = S * Device.dwHeight; -}; + MASTER_OUT(false, true, true, false, _x, _y, 0.0f, fmt); +} -void CGameFont::SetHeight(float S) +void __cdecl CGameFont::Out(float _x, float _y, pcstr fmt, ...) { - VERIFY(uFlags & fsDeviceIndependent); - fCurrentHeight = S; -}; + MASTER_OUT(true, true, false, false, _x, _y, 0.0f, fmt); +} + +void __cdecl CGameFont::OutNext(pcstr fmt, ...) +{ + MASTER_OUT(true, false, false, true, 0.0f, 0.0f, LineSpacing, fmt); +} + +void CGameFont::OutNextVA(pcstr format, va_list args) +{ + MasterOut(true, false, false, true, 0.0f, 0.0f, LineSpacing, format, args); +} + +void CGameFont::OutSkip(float val) { fCurrentY += val * CurrentHeight_(); } diff --git a/src/xrEngine/GameFont.h b/src/xrEngine/GameFont.h index 68ead3b1943..c4f6b394be5 100644 --- a/src/xrEngine/GameFont.h +++ b/src/xrEngine/GameFont.h @@ -3,20 +3,19 @@ #include "xrEngine/IGameFont.hpp" #include "xrCommon/xr_vector.h" #include "xrCore/_vector3d.h" -#ifdef DEBUG -#include "xrCore/xrstring.h" -#endif +#include "xrCore/Text/Utf8Utils.hpp" +#include "xrCore/Threading/Lock.hpp" +#include "xrCore/Threading/ScopeLock.hpp" + +struct FT_FaceRec_; +using FT_Face = FT_FaceRec_*; namespace xray::render { -namespace render_r4 -{ -class dxFontRender; -} -namespace render_gl -{ -class dxFontRender; -} +namespace render_r4 { class dxFontRender; } +namespace render_gl { class dxFontRender; } +namespace render_r2 { class dxFontRender; } +namespace render_r3 { class dxFontRender; } } // namespace xray::render class IFontRender; @@ -25,74 +24,175 @@ class ENGINE_API CGameFont final : public IGameFont { friend class xray::render::render_r4::dxFontRender; friend class xray::render::render_gl::dxFontRender; + friend class xray::render::render_r2::dxFontRender; + friend class xray::render::render_r3::dxFontRender; + +public: + enum EGradientMode + { + gm_vert = 0, + gm_horz = 1, + gm_back = 2, + gm_down = 3, + gm_count + }; + + struct ABC + { + int abcA; + u32 abcB; + int abcC; + }; + + struct Glyph + { + RECT TextureCoord; + ABC Abc; + int yOffset; + }; private: struct String { - string1024 string; + string2048 string; + xr_string string_utf8; float x, y; float height; u32 c; EAligment align; + bool gradient; + EGradientMode gradientMode; + u32 gradientColor; }; -protected: - Fvector2 vHalfPixel; - Ivector2 vTS; - EAligment eCurrentAlignment; - u8 uFlags; - u32 dwCurrentColor; - float fCurrentHeight; - float fCurrentX, fCurrentY; - Fvector2 vInterval; - float fHeight; - float fXStep; - float fYStep; - float fTCHeight; - u32 nNumChars{ 256 }; - xr_vector strings; - Fvector* TCMap{}; - IFontRender* pFontRender; + struct BaseData + { + bool OpenType = false; + bool HasUnicodeCharmap = false; // true if FreeType selected a Unicode cmap + bool IsBitmap = false; // true: legacy bitmap font (.dds + .ini), no FreeType atlas + u16 Size = 14; + const char* Name = nullptr; // font family file name (e.g. "arial") + const char* Shader = nullptr; + const char* Style = nullptr; + const char* TextureName = nullptr; // unique atlas texture name (e.g. section "font_default") + }; + + BaseData Data; -protected: - const Fvector& GetCharTC(u16 c) const { return TCMap[c]; } + float fCurrentHeight = 0.0f; + float fCurrentX = 0.0f; + float fCurrentY = 0.0f; + bool fGradientEnabled = false; + EGradientMode fGradientMode = gm_vert; + + u8 uFlags = 0; + u32 dwCurrentColor = 0; + u32 dwGradientColor = 0; + + EAligment eCurrentAlignment = alLeft; + Fvector2 vInterval{ 1.f, 1.f }; + + Lock s_cs; + xr_vector strings; + IFontRender* pFontRender = nullptr; public: CGameFont(pcstr section, u8 flags = 0); - CGameFont(pcstr shader, pcstr texture, u8 flags = 0); virtual ~CGameFont(); - virtual void Initialize(pcstr shader, pcstr texture) override; - virtual void SetColor(u32 C) override { dwCurrentColor = C; } - virtual u32 GetColor() const override { return dwCurrentColor; } - virtual void SetHeightI(float S) override; - virtual void SetHeight(float S) override; - virtual float GetHeight() const override { return fCurrentHeight; }; - virtual void SetInterval(float x, float y) override { vInterval.set(x, y); }; - virtual void SetInterval(const Fvector2& v) override { vInterval.set(v); }; - virtual void SetAligment(EAligment aligment) override { eCurrentAlignment = aligment; } - virtual float SizeOf_(pcstr s) override; - virtual float SizeOf_(const xr_wide_char* wsStr) override; - virtual float SizeOf_(const char cChar); // only ANSII - virtual float CurrentHeight_() override; - virtual void OutSetI(float x, float y) override; - virtual void OutSet(float x, float y) override; - virtual Fvector2 GetPosition() const override { return {fCurrentX, fCurrentY}; } - virtual void MasterOut(bool bCheckDevice, bool bUseCoords, bool bScaleCoords, bool bUseSkip, float _x, float _y, - float _skip, pcstr fmt, va_list p) override; - virtual u32 smart_strlen(pcstr S) override; - virtual std::pair get_actions_text_length(pcstr s) override; // returns count of actions plus total text size of all actions - virtual bool IsMultibyte() const override { return (uFlags & fsMultibyte); }; - virtual u16 SplitByWidth(u16* puBuffer, u16 uBufferSize, float fTargetWidth, pcstr pszText) override; - virtual u16 GetCutLengthPos(float fTargetWidth, pcstr pszText) override; - virtual void OutI(float _x, float _y, pcstr fmt, ...) override; - virtual void Out(float _x, float _y, pcstr fmt, ...) override; - virtual void OutNext(pcstr fmt, ...) override; - virtual void OutNextVA(pcstr format, va_list args) override; - virtual void OutSkip(float val = 1.f) override; - virtual void OnRender() override; - virtual void Clear() override { strings.clear(); } + void ReInit(); + + void SetColor(u32 C) override { dwCurrentColor = C; } + u32 GetColor() const override { return dwCurrentColor; } + void SetGradientColor(u32 C) { dwGradientColor = C; } + + void SetHeight(float S) override; + void SetHeightI(float S) override; + float GetHeight() const override { return fCurrentHeight; } + + void SetAligment(EAligment aligment) override { eCurrentAlignment = aligment; } + + void SetInterval(float x, float y) override { vInterval.set(x, y); } + void SetInterval(const Fvector2& v) override { vInterval.set(v); } + + float SizeOf_(pcstr s) override; + + float CurrentHeight_() override; + + void OutSetI(float x, float y) override; + void OutSet(float x, float y) override; + Fvector2 GetPosition() const override { return { fCurrentX, fCurrentY }; } + + void MasterOut(bool bCheckDevice, bool bUseCoords, bool bScaleCoords, bool bUseSkip, + float _x, float _y, float _skip, pcstr fmt, va_list p) override; + + u32 smart_strlen(pcstr S) override; + + bool IsMultibyte() const override { return (uFlags & fsMultibyte) != 0; } + + u16 SplitByWidth(u16* puBuffer, u16 uBufferSize, float fTargetWidth, pcstr pszText) override; + u16 GetCutLengthPos(float fTargetWidth, pcstr pszText) override; + + void OutI(float _x, float _y, pcstr fmt, ...) override; + void Out(float _x, float _y, pcstr fmt, ...) override; + void OutNext(pcstr fmt, ...) override; + void OutNextVA(pcstr format, va_list args) override; + void OutSkip(float val = 1.f) override; + + void OnRender() override; + void Clear() override { strings.clear(); } + + inline u32 GetSize() const { return Data.Size; } + inline float GetLetterSpacing() const { return LetterSpacing; } + inline void SetLetterSpacing(float spacing) { LetterSpacing = spacing; } + inline float GetLineSpacing() const { return LineSpacing; } + inline void SetLineSpacing(float spacing) { LineSpacing = spacing; } + + const Glyph* GetGlyphInfo(int ch); + float WidthOf(int ch); + float WidthOf(pcstr str); + float SizeOf_(int cChar); // internal single-codepoint width, used by rendering pipeline + +private: + float LetterSpacing = 0.f; + float LineSpacing = 1.f; + + struct StyleBits + { + u32 bold : 1; + u32 italic : 1; + u32 underline : 1; + u32 strike : 1; + } Style{}; + + FT_Face OurFont = nullptr; + IReader* m_fontReader = nullptr; + xr_map GlyphData; + + void Prepare(pcstr name, pcstr shader, pcstr style, u32 size); + void Initialize2(pcstr name, pcstr shader, pcstr style, u32 size); + // Legacy bitmap-font fallback: loads a .dds + .ini font, converts CP1251-keyed + // glyph metrics into the UTF-8 GlyphData map, and reuses the UTF-8 renderer. + // Invoked from Initialize2 when no TTF/OTF fonts are present in $game_fonts$. + void InitializeLegacy(pcstr name, pcstr shader, pcstr style, u32 size); + + static bool bFreetypeInitialized; + static void InitializeFreetype(); + #ifdef DEBUG +public: shared_str m_font_name; #endif }; + +// Global text scale multiplier (controlled by the `ui_text_scale` console variable). +// Applied on the draw path (WidthOf/CurrentHeight_), so changes take effect +// immediately in runtime without rebuilding the FreeType font atlas. +ENGINE_API extern float g_text_scale; + +// Font renderer selector (controlled by the `r_font_legacy` console variable). +// 0 (default) = FreeType TTF/OTF rendering (UTF-8 native). +// 1 = legacy X-Ray bitmap fonts (.dds + .ini), CP1251 metrics +// converted to Unicode codepoints on load. +// Changes apply on the next font (re)initialization (e.g. vid_restart / UI reset). +ENGINE_API extern int g_font_legacy; diff --git a/src/xrEngine/IGameFont.hpp b/src/xrEngine/IGameFont.hpp index a5b973fd6f7..6abc58deb87 100644 --- a/src/xrEngine/IGameFont.hpp +++ b/src/xrEngine/IGameFont.hpp @@ -6,8 +6,6 @@ class XR_NOVTABLE IGameFont { - friend class dxFontRender; - public: enum EAligment : u8 { @@ -26,7 +24,6 @@ class XR_NOVTABLE IGameFont public: virtual ~IGameFont() = 0; - virtual void Initialize(pcstr shader, pcstr texture) = 0; virtual void SetColor(u32 C) = 0; virtual u32 GetColor() const = 0; virtual void SetHeightI(float S) = 0; @@ -36,8 +33,6 @@ class XR_NOVTABLE IGameFont virtual void SetInterval(const Fvector2& v) = 0; virtual void SetAligment(EAligment aligment) = 0; virtual float SizeOf_(pcstr s) = 0; - virtual float SizeOf_(const xr_wide_char* wsStr) = 0; - virtual float SizeOf_(const char cChar) = 0; // only ANSI virtual float CurrentHeight_() = 0; virtual void OutSetI(float x, float y) = 0; virtual void OutSet(float x, float y) = 0; @@ -45,7 +40,6 @@ class XR_NOVTABLE IGameFont virtual void MasterOut(bool bCheckDevice, bool bUseCoords, bool bScaleCoords, bool bUseSkip, float _x, float _y, float _skip, pcstr fmt, va_list p) = 0; virtual u32 smart_strlen(pcstr S) = 0; - virtual std::pair get_actions_text_length(pcstr str) = 0; // returns count of actions plus total text size of all actions virtual bool IsMultibyte() const = 0; virtual u16 SplitByWidth(u16* puBuffer, u16 uBufferSize, float fTargetWidth, pcstr pszText) = 0; virtual u16 GetCutLengthPos(float fTargetWidth, pcstr pszText) = 0; diff --git a/src/xrEngine/line_edit_control.cpp b/src/xrEngine/line_edit_control.cpp index 10f49c4f581..6fb2b8fbab6 100644 --- a/src/xrEngine/line_edit_control.cpp +++ b/src/xrEngine/line_edit_control.cpp @@ -10,7 +10,7 @@ #include "xrCore/os_clipboard.h" #include "xrCore/buffer_vector.h" -#include "xrCore/Text/StringConversion.hpp" +#include "xrCore/Text/Utf8Utils.hpp" #include "Common/object_broker.h" #include "xr_input.h" @@ -18,13 +18,11 @@ #include -#include - ENGINE_API float g_console_sensitive = 0.15f; namespace text_editor { -static bool terminate_char(char c, bool check_space = false) +static bool terminate_char(u32 c, bool check_space = false) { switch (c) { @@ -263,12 +261,14 @@ void line_edit_control::remove_callback(int dik) xr_delete(m_actions[dik]); } -void line_edit_control::insert_character(char c) +void line_edit_control::insert_utf8_codepoint(pcstr cp, size_t len) { - VERIFY(m_inserted_pos < (m_buffer_size - 1 /*trailing zero*/)); - m_inserted[m_inserted_pos ] = c; - m_inserted[m_inserted_pos + 1] = 0; - m_inserted_pos++; + if (m_inserted_pos + len >= (m_buffer_size - 1 /*trailing zero*/)) + return; + for (size_t i = 0; i < len; ++i) + m_inserted[m_inserted_pos + i] = cp[i]; + m_inserted[m_inserted_pos + len] = 0; + m_inserted_pos += len; } void line_edit_control::clear_inserted() { m_inserted[0] = m_inserted[1] = 0; m_inserted_pos = 0; } @@ -280,17 +280,37 @@ void line_edit_control::set_edit(pcstr str) strncpy_s(m_edit_str, m_buffer_size, str, str_size); m_edit_str[str_size] = 0; + // Some UI defaults / scripts still pass CP1251 strings. Normalize to UTF-8 internally. + if (!XRay::Utf8::IsValid(m_edit_str)) + { + const xr_string utf8 = XRay::Utf8::FromCP1251(m_edit_str); + str_size = std::min(utf8.size(), m_buffer_size - 1); + strncpy_s(m_edit_str, m_buffer_size, utf8.c_str(), str_size); + m_edit_str[str_size] = 0; + } + + // Make sure the stored string does not end in the middle of a UTF-8 codepoint. + while (str_size > 0 && !XRay::Utf8::IsValid(m_edit_str)) + { + --str_size; + m_edit_str[str_size] = 0; + } + m_cur_pos = str_size; m_select_start = m_cur_pos; m_accel = 1.0f; update_bufs(); } -bool line_edit_control::char_is_allowed(char c) +bool line_edit_control::char_is_allowed(u32 codepoint) { + // Reject control characters (below U+0020). + if (codepoint < 0x20) + return false; + if (m_current_mode == im_number_only) { - switch (c) + switch (codepoint) { case '7': case '8': case '9': case '4': case '5': case '6': @@ -301,7 +321,7 @@ bool line_edit_control::char_is_allowed(char c) return false; } } - switch (c) + switch (codepoint) { case '\'': case '\"': // ' and " case '\\': case '/': // \ and / @@ -371,13 +391,29 @@ void line_edit_control::on_text_input(const char *text) clear_inserted(); compute_positions(); - static std::locale locale(""); - const auto str = StringFromUTF8(text, locale); + // Pasted text may still be in CP1251; convert it once if needed. + xr_string inputText; + pcstr p = text; + if (text && text[0] && !XRay::Utf8::IsValid(text)) + { + inputText = XRay::Utf8::FromCP1251(text); + p = inputText.c_str(); + } - for (char c : str) + while (*p != 0) { - if (char_is_allowed(c)) - insert_character(c); + size_t len = 0; + const u32 codepoint = XRay::Utf8::Decode(p, len); + // Skip replacement character from stray invalid bytes rather than inserting raw bytes. + if (codepoint == XRay::Utf8::REPLACEMENT_CHARACTER && len == 1 && + XRay::Utf8::SequenceLength(static_cast(*p)) != 1) + { + ++p; + continue; + } + if (char_is_allowed(codepoint)) + insert_utf8_codepoint(p, len); + p += len; } add_inserted_text(); @@ -478,7 +514,13 @@ void line_edit_control::update_bufs() m_buf3[0] = 0; const size_t edit_size = xr_strlen(m_edit_str); - const u8 ds = (m_cursor_view && m_insert_mode && m_p2 < edit_size) ? 1 : 0; + size_t ds = 0; + if (m_cursor_view && m_insert_mode && m_p2 < edit_size) + { + size_t len = 0; + XRay::Utf8::Decode(m_edit_str + m_p2, len); + ds = len; + } strncpy_s(m_buf0, m_buffer_size, m_edit_str, m_cur_pos); strncpy_s(m_buf1, m_buffer_size, m_edit_str, m_p1); strncpy_s(m_buf2, m_buffer_size, m_edit_str + m_p1, m_p2 - m_p1 + ds); @@ -516,10 +558,19 @@ void line_edit_control::add_inserted_text() { m_inserted[m_buffer_size - 1 - m_p1] = 0; new_size = xr_strlen(m_inserted); + // trim back to the leading byte so we do not split a UTF-8 codepoint + while (new_size > 0 && XRay::Utf8::IsContinuationByte(static_cast(m_inserted[new_size - 1]))) + m_inserted[--new_size] = 0; } strncpy_s(buf + m_p1, m_buffer_size - m_p1, m_inserted, _min(new_size, m_buffer_size - m_p1)); // part 2 - const u8 ds = (m_insert_mode && m_p2 < old_edit_size) ? 1 : 0; + size_t ds = 0; + if (m_insert_mode && m_p2 < old_edit_size) + { + size_t len = 0; + XRay::Utf8::Decode(m_edit_str + m_p2, len); + ds = len; + } strncpy_s(buf + m_p1 + new_size, m_buffer_size - (m_p1 + new_size), m_edit_str + m_p2 + ds, _min(old_edit_size - m_p2 - ds, m_buffer_size - m_p1 - new_size)); // part 3 buf[m_buffer_size] = 0; @@ -553,7 +604,7 @@ void line_edit_control::copy_to_clipboard() void line_edit_control::paste_from_clipboard() { os_clipboard::paste_from_clipboard(m_inserted, m_buffer_size - 1); - m_inserted_pos += xr_strlen(m_inserted); + m_inserted_pos = xr_strlen(m_inserted); } void line_edit_control::cut_to_clipboard() { @@ -587,14 +638,23 @@ void line_edit_control::delete_selected(bool back) { if (back) { - u8 dp = ((m_p1 == m_p2) && m_p1 > 0) ? 1 : 0; + size_t dp = 0; + if ((m_p1 == m_p2) && m_p1 > 0) + dp = m_edit_str + m_p1 - XRay::Utf8::Prev(m_edit_str, m_edit_str + m_p1); + strncpy_s(m_undo_buf, m_buffer_size, m_edit_str + m_p1 - dp, m_p2 - m_p1 + dp); strncpy_s(m_edit_str + m_p1 - dp, m_buffer_size - (m_p1 - dp), m_edit_str + m_p2, edit_len - m_p2); m_cur_pos = m_p1 - dp; } else { - u8 dn = ((m_p1 == m_p2) && m_p2 < edit_len) ? 1 : 0; + size_t dn = 0; + if ((m_p1 == m_p2) && m_p2 < edit_len) + { + size_t len = 0; + XRay::Utf8::Decode(m_edit_str + m_p2, len); + dn = len; + } strncpy_s(m_undo_buf, m_buffer_size, m_edit_str + m_p1, m_p2 - m_p1 + dn); strncpy_s(m_edit_str + m_p1, m_buffer_size - m_p1, m_edit_str + m_p2 + dn, edit_len - m_p2 - dn); m_cur_pos = m_p1; @@ -632,43 +692,79 @@ void line_edit_control::move_pos_end() { m_cur_pos = xr_strlen(m_edit_str); } void line_edit_control::move_pos_left() { if (m_cur_pos > 0) - --m_cur_pos; + m_cur_pos = XRay::Utf8::Prev(m_edit_str, m_edit_str + m_cur_pos) - m_edit_str; +} +void line_edit_control::move_pos_right() +{ + m_cur_pos = XRay::Utf8::Next(m_edit_str + m_cur_pos) - m_edit_str; } -void line_edit_control::move_pos_right() { ++m_cur_pos; } void line_edit_control::move_pos_left_word() { - size_t i = m_cur_pos > 0 ? m_cur_pos - 1 : 0; - - while (i > 0 && m_edit_str[i] == ' ') - --i; + pcstr const edit_str = m_edit_str; + pcstr p = edit_str + m_cur_pos; - if (i > 0 && !terminate_char(m_edit_str[i])) + // skip whitespace to the left, landing on the first non-space character + while (p > edit_str) { - while (i > 0 && !terminate_char(m_edit_str[i], true)) - --i; + pcstr prev = XRay::Utf8::Prev(edit_str, p); + size_t len = 0; + const u32 cp = XRay::Utf8::Decode(prev, len); + if (cp != ' ') + { + p = prev; + break; + } + p = prev; + } - if (i > 0) - ++i; + // skip non-terminating characters to the left + if (p > edit_str) + { + size_t len = 0; + if (!terminate_char(XRay::Utf8::Decode(p, len))) + { + while (p > edit_str) + { + pcstr prev = XRay::Utf8::Prev(edit_str, p); + size_t prev_len = 0; + const u32 cp = XRay::Utf8::Decode(prev, prev_len); + if (terminate_char(cp, true)) + break; + p = prev; + } + } } - m_cur_pos = i; + m_cur_pos = p - edit_str; } void line_edit_control::move_pos_right_word() { - const size_t edit_len = xr_strlen(m_edit_str); - size_t i = m_cur_pos + 1; - - while (i < edit_len && !terminate_char(m_edit_str[i], true)) - ++i; + pcstr const edit_str = m_edit_str; + pcstr p = XRay::Utf8::Next(edit_str + m_cur_pos); + pcstr const end = edit_str + xr_strlen(edit_str); - //while (i < edit_len && terminate_char(m_edit_str[i])) - // ++i; + // skip non-terminating characters to the right + while (p < end) + { + size_t len = 0; + const u32 cp = XRay::Utf8::Decode(p, len); + if (terminate_char(cp, true)) + break; + p += len; + } - while (i < edit_len && m_edit_str[i] == ' ') - ++i; + // skip whitespace to the right + while (p < end) + { + size_t len = 0; + const u32 cp = XRay::Utf8::Decode(p, len); + if (cp != ' ') + break; + p += len; + } - m_cur_pos = i; + m_cur_pos = p - edit_str; } void line_edit_control::compute_positions() @@ -686,7 +782,14 @@ void line_edit_control::compute_positions() m_p2 = m_select_start; } -void line_edit_control::clamp_cur_pos() { clamp(m_cur_pos, 0, xr_strlen(m_edit_str)); } +void line_edit_control::clamp_cur_pos() +{ + clamp(m_cur_pos, 0, xr_strlen(m_edit_str)); + + // If m_cur_pos points to a continuation byte, move it back to the leading byte. + while (m_cur_pos > 0 && XRay::Utf8::IsContinuationByte(static_cast(m_edit_str[m_cur_pos]))) + --m_cur_pos; +} void line_edit_control::SwitchKL() { cpcstr hint = SDL_GetHint(SDL_HINT_GRAB_KEYBOARD); diff --git a/src/xrEngine/line_edit_control.h b/src/xrEngine/line_edit_control.h index d9b105ab39b..4129365f0f5 100644 --- a/src/xrEngine/line_edit_control.h +++ b/src/xrEngine/line_edit_control.h @@ -71,7 +71,7 @@ class ENGINE_API line_edit_control void assign_callback(int const dik, key_state state, Callback const& callback); void remove_callback(int dik); - void insert_character(char c); + void insert_utf8_codepoint(pcstr cp, size_t len); bool get_key_state(key_state mask) const { return mask ? !!m_key_state.test(mask) : true; } void set_key_state(key_state mask, bool value) { m_key_state.set(mask, value); } @@ -86,7 +86,7 @@ class ENGINE_API line_edit_control void set_selected_mode(bool status) { m_unselected_mode = !status; } bool get_selected_mode() const { return !m_unselected_mode; } - bool char_is_allowed(char c); + bool char_is_allowed(u32 codepoint); private: line_edit_control(line_edit_control const&); diff --git a/src/xrEngine/x_ray.cpp b/src/xrEngine/x_ray.cpp index 20d2e576e1b..4103130ffe6 100644 --- a/src/xrEngine/x_ray.cpp +++ b/src/xrEngine/x_ray.cpp @@ -27,12 +27,10 @@ #endif #ifdef XR_PLATFORM_WINDOWS -#include - #include "DiscordGameSDK/discord.h" #define USE_DISCORD_INTEGRATION -#include "xrCore/Text/StringConversion.hpp" +#include "xrCore/Text/Utf8Utils.hpp" #endif // global variables @@ -526,12 +524,10 @@ void CApplication::InitializeDiscord() if (core) { - const std::locale locale(""); - discord::Activity activity{}; activity.SetType(discord::ActivityType::Playing); activity.SetApplicationId(DISCORD_APP_ID); - activity.SetState(StringToUTF8(Core.ApplicationTitle, locale).c_str()); + activity.SetState(XRay::Utf8::FromACP(Core.ApplicationTitle).c_str()); activity.GetAssets().SetLargeImage("logo"); core->ActivityManager().UpdateActivity(activity, nullptr); diff --git a/src/xrEngine/xrEngine.vcxproj b/src/xrEngine/xrEngine.vcxproj index f6b0554cee3..2695c050852 100644 --- a/src/xrEngine/xrEngine.vcxproj +++ b/src/xrEngine/xrEngine.vcxproj @@ -21,7 +21,7 @@ - $(xrExternals)imgui;%(AdditionalIncludeDirectories) + $(xrExternals)imgui;$(xrExternals)freetype\include;%(AdditionalIncludeDirectories) ENGINE_BUILD;INGAME_EDITOR;%(PreprocessorDefinitions) @@ -231,6 +231,9 @@ {c8fbc3ce-d6de-4fc3-bc15-7b647614db09} + + {78b079bd-9fc7-4b9e-b4a6-96da0f00248b} + {2c419512-6eee-4707-bc51-2e834855552e} diff --git a/src/xrEngine/xr_input.cpp b/src/xrEngine/xr_input.cpp index b8caddd9ab1..59f0b8cfcc2 100644 --- a/src/xrEngine/xr_input.cpp +++ b/src/xrEngine/xr_input.cpp @@ -5,9 +5,7 @@ #include "IInputReceiver.h" #include "GameFont.h" #include "XR_IOConsole.h" -#include "xrCore/Text/StringConversion.hpp" - -#include +#include "xrCore/Text/Utf8Utils.hpp" CInput* pInput = nullptr; @@ -299,28 +297,51 @@ void CInput::KeyUpdate() case SDL_KEYDOWN: if (event.key.repeat) continue; - cbStack.back()->IR_OnKeyboardPress(event.key.keysym.scancode); + { + ZoneScopedN("CInput::KeyUpdate/IR_OnKeyboardPress"); + ZoneTextF("#%d/%d scancode %d", i, count, (int)event.key.keysym.scancode); + cbStack.back()->IR_OnKeyboardPress(event.key.keysym.scancode); + } break; case SDL_KEYUP: + { + ZoneScopedN("CInput::KeyUpdate/IR_OnKeyboardRelease"); cbStack.back()->IR_OnKeyboardRelease(event.key.keysym.scancode); - break; + } + break; case SDL_TEXTINPUT: if (cnt != textInputCounter) continue; // if input target changed, skip this frame - cbStack.back()->IR_OnTextInput(event.text.text); + { + ZoneScopedN("CInput::KeyUpdate/IR_OnTextInput"); + const xr_string utf8Text = XRay::Utf8::FixTextInputEncoding(event.text.text); + cbStack.back()->IR_OnTextInput(utf8Text.c_str()); + } break; case SDL_KEYMAPCHANGED: + { + ZoneScopedN("CInput::KeyUpdate/KeyMapChanged_Process"); seqKeyMapChanged.Process(); - break; + } + break; } } - for (u32 i = 0; i < COUNT_KB_BUTTONS; ++i) - if (keyboardState[i]) + { + ZoneScopedN("CInput::KeyUpdate/IR_OnKeyboardHold_scan"); + u32 held_keys = 0; + for (u32 i = 0; i < COUNT_KB_BUTTONS; ++i) + { + if (!keyboardState[i]) + continue; + ++held_keys; cbStack.back()->IR_OnKeyboardHold(i); + } + ZoneTextF("held_keys %u / scancodes", held_keys); + } } bool ControllerState::attitude_changed() const @@ -519,14 +540,13 @@ void CInput::ControllerUpdate() bool KbdKeyToButtonName(const int dik, xr_string& result) { - static std::locale locale(""); - if (dik >= 0) { cpcstr name = SDL_GetKeyName(SDL_GetKeyFromScancode((SDL_Scancode)dik)); if (name && name[0]) { - result = StringFromUTF8(name, locale); + // SDL returns key names in UTF-8; keep them as-is for the UTF-8 UI. + result = name; return true; } } diff --git a/src/xrEngine/xr_ioc_cmd.cpp b/src/xrEngine/xr_ioc_cmd.cpp index 7403be7f93d..34171555237 100644 --- a/src/xrEngine/xr_ioc_cmd.cpp +++ b/src/xrEngine/xr_ioc_cmd.cpp @@ -11,6 +11,7 @@ #include "xr_object.h" #include "xr_object_list.h" +#include "GameFont.h" xr_vector VidQualityToken; @@ -782,6 +783,13 @@ void CCC_Register() CMD2(CCC_Gamma, "rs_c_gamma", &ps_gamma); CMD2(CCC_Gamma, "rs_c_brightness", &ps_brightness); CMD2(CCC_Gamma, "rs_c_contrast", &ps_contrast); + + // Global UI text scale multiplier (applied at draw time, no atlas rebuild). + CMD4(CCC_Float, "ui_text_scale", &g_text_scale, 0.5f, 3.0f); + + // Font renderer selector: 0 = FreeType TTF/OTF (default), 1 = legacy X-Ray + // bitmap fonts (.dds + .ini). Apply via vid_restart / UI reset. + CMD4(CCC_Integer, "r_font_legacy", &g_font_legacy, 0, 1); // CMD4(CCC_Integer, "rs_vb_size", &rsDVB_Size, 32, 4096); // CMD4(CCC_Integer, "rs_ib_size", &rsDIB_Size, 32, 4096); diff --git a/src/xrGame/ui/UIPdaKillMessage.cpp b/src/xrGame/ui/UIPdaKillMessage.cpp index 125a6533651..ab643160d8e 100644 --- a/src/xrGame/ui/UIPdaKillMessage.cpp +++ b/src/xrGame/ui/UIPdaKillMessage.cpp @@ -51,7 +51,7 @@ float CUIPdaKillMessage::InitText(CUIStatic& refStatic, float x, ColoredName& in return 0.0f; CGameFont* pFont = refStatic.GetFont(); - float _eps = pFont->SizeOf_(' '); + float _eps = pFont->SizeOf_(" "); UI().ClientToScreenScaledWidth(_eps); // add one letter float height = pFont->CurrentHeight_(); diff --git a/src/xrNetServer/NET_Server.cpp b/src/xrNetServer/NET_Server.cpp index 9f78c70198f..2106dbbb2bd 100644 --- a/src/xrNetServer/NET_Server.cpp +++ b/src/xrNetServer/NET_Server.cpp @@ -523,7 +523,6 @@ HRESULT IPureServer::net_Handler(u32 dwMessageType, PVOID pMessage) CHK_DX(_hr); //string64 cname; - //CHK_DX(WideCharToMultiByte( CP_ACP, 0, Pinfo->pwszName, -1, cname, sizeof(cname) , nullptr, nullptr ) ); SClientConnectData cl_data; //xr_strcpy(cl_data.name, cname); diff --git a/src/xrScriptEngine/script_engine.cpp b/src/xrScriptEngine/script_engine.cpp index e4271648808..88d912c7644 100644 --- a/src/xrScriptEngine/script_engine.cpp +++ b/src/xrScriptEngine/script_engine.cpp @@ -313,6 +313,16 @@ bool CScriptEngine::parse_namespace(pcstr caNamespaceName, pstr b, size_t b_size bool CScriptEngine::load_buffer( lua_State* L, LPCSTR caBuffer, size_t tSize, LPCSTR caScriptName, LPCSTR caNameSpaceName) { + // Strip UTF-8 BOM if present + if (tSize >= 3 && + static_cast(caBuffer[0]) == 0xEF && + static_cast(caBuffer[1]) == 0xBB && + static_cast(caBuffer[2]) == 0xBF) + { + caBuffer += 3; + tSize -= 3; + } + int l_iErrorCode; if (caNameSpaceName && xr_strcmp(GlobalNamespace, caNameSpaceName)) { diff --git a/src/xrServerEntities/object_factory_spawner.cpp b/src/xrServerEntities/object_factory_spawner.cpp index 9765f957b21..a9d1e4633a7 100644 --- a/src/xrServerEntities/object_factory_spawner.cpp +++ b/src/xrServerEntities/object_factory_spawner.cpp @@ -9,6 +9,11 @@ #include "alife_smart_terrain_registry.h" +#include "xrGame/ui/UIInventoryUtilities.h" +#include "xrCore/Text/Utf8Utils.hpp" + +using namespace InventoryUtilities; + extern CSE_Abstract* CALifeSimulator__spawn_item2(CALifeSimulator* self, pcstr section, const Fvector& position, u32 level_vertex_id, GameGraph::_GRAPH_ID game_vertex_id, ALife::_OBJECT_ID id_parent); @@ -257,11 +262,10 @@ void CObjectFactory::on_tool_frame() { case DisplayGameNames: { - const std::locale locale(""); if (cpcstr inv_name = pSettings->read_if_exists(section->Name.c_str(), "inv_name", nullptr)) { const auto translated = StringTable().translate(inv_name); - name = StringToUTF8(translated.c_str(), locale); + name = XRay::Utf8::FromACP(translated.c_str()); break; } if (cpcstr character_profile = pSettings->read_if_exists(section->Name.c_str(), "character_profile", nullptr)) @@ -274,7 +278,7 @@ void CObjectFactory::on_tool_frame() if (character_name[0] && !strstr(character_name, "GENERATE_NAME")) { const auto translated = StringTable().translate(character_name); - name = StringToUTF8(translated.c_str(), locale); + name = XRay::Utf8::FromACP(translated.c_str()); break; } } diff --git a/src/xrUICore/EditBox/UICustomEdit.cpp b/src/xrUICore/EditBox/UICustomEdit.cpp index bab1dfdf6ec..b4abfa22d1c 100644 --- a/src/xrUICore/EditBox/UICustomEdit.cpp +++ b/src/xrUICore/EditBox/UICustomEdit.cpp @@ -1,6 +1,7 @@ #include "pch.hpp" #include "UICustomEdit.h" #include "Lines/UILines.h" +#include "xrCore/Text/Utf8Utils.hpp" #include "xrEngine/line_edit_control.h" #include "xrEngine/xr_input.h" @@ -96,7 +97,7 @@ void CUICustomEdit::SetPasswordMode(bool mode) { TextItemControl()->SetPasswordM void CUICustomEdit::SendMessage(CUIWindow* pWnd, s16 msg, void* pData) { - //кто-то другой захватил клавиатуру + // someone else captured the keyboard if (msg == WINDOW_KEYBOARD_CAPTURE_LOST && m_bInputFocus) { CaptureFocus(false); @@ -173,53 +174,91 @@ void CUICustomEdit::Draw() if (ec().need_update() || m_force_update) { - float ui_width = GetWidth(); + const float ui_width = GetWidth(); - LPCSTR cursor_str = ec().str_before_cursor(); - u32 cursor_str_size = xr_strlen(cursor_str); + pcstr cursor_str = ec().str_before_cursor(); - LPCSTR istr = cursor_str; + // Scroll left: skip whole codepoints until the tail fits in the edit box. + pcstr istr = cursor_str; float str_length = font->SizeOf_(istr); UI().ClientToScreenScaledWidth(str_length); - - u32 ix = 0; - while ((str_length > ui_width) && (ix < cursor_str_size)) + while (str_length > ui_width && *istr) { - istr = cursor_str + ix; + istr = XRay::Utf8::Next(istr); str_length = font->SizeOf_(istr); UI().ClientToScreenScaledWidth(str_length); - ++ix; } - istr = cursor_str + ix; - LPCSTR astr = ec().str_edit() + ix; - u32 str_size = xr_strlen(ec().str_edit()); - u32 jx = 1; - strncpy_s(m_out_str, sizeof(m_out_str), astr, jx); + // The visible part of the full edit string starts at the same codepoint position + // as the scrolled prefix (istr). Use codepoint distance to avoid landing on a + // continuation byte if the edit buffer and the prefix buffer ever differ. + const size_t skipped_codepoints = XRay::Utf8::DistanceCodepoints(cursor_str, istr); + pcstr astr = XRay::Utf8::Advance(ec().str_edit(), skipped_codepoints); - str_length = font->SizeOf_(m_out_str); - UI().ClientToScreenScaledWidth(str_length); + // Guard against a misaligned start if the edit buffer ever ends with an incomplete codepoint. + while (*astr && XRay::Utf8::IsContinuationByte(static_cast(*astr))) + ++astr; - while ((str_length < ui_width) && (jx < str_size - ix)) + // Grow the visible substring by whole codepoints until it no longer fits. + const size_t max_out_bytes = sizeof(m_out_str) - 1; + size_t visible_bytes = 0; + for (pcstr p = astr; *p; p = XRay::Utf8::Next(p)) { - strncpy_s(m_out_str, sizeof(m_out_str), astr, jx); + size_t candidate_bytes = XRay::Utf8::Next(p) - astr; + if (candidate_bytes > max_out_bytes) + { + candidate_bytes = max_out_bytes; + strncpy_s(m_out_str, sizeof(m_out_str), astr, candidate_bytes); + m_out_str[candidate_bytes] = '\0'; + + str_length = font->SizeOf_(m_out_str); + UI().ClientToScreenScaledWidth(str_length); + + if (visible_bytes == 0 || str_length < ui_width) + visible_bytes = candidate_bytes; + break; + } + + strncpy_s(m_out_str, sizeof(m_out_str), astr, candidate_bytes); + m_out_str[candidate_bytes] = '\0'; + str_length = font->SizeOf_(m_out_str); UI().ClientToScreenScaledWidth(str_length); - ++jx; + + if (str_length < ui_width) + { + visible_bytes = candidate_bytes; + } + else if (visible_bytes == 0) + { + visible_bytes = candidate_bytes; + break; + } + else + { + break; + } + } + strncpy_s(m_out_str, sizeof(m_out_str), astr, visible_bytes); + m_out_str[visible_bytes] = '\0'; + + // Defensive: the visible substring must not end in the middle of a codepoint. + // Use IsValid() rather than a trailing-continuation check: a completed + // multi-byte codepoint (e.g. Cyrillic 'т' = D1 82) legitimately ends in a + // continuation byte and must NOT be trimmed here. + while (visible_bytes > 0 && !XRay::Utf8::IsValid(m_out_str)) + { + --visible_bytes; + m_out_str[visible_bytes] = '\0'; } - strncpy_s(m_out_str, sizeof(m_out_str), astr, jx); TextItemControl()->SetText(m_out_str); if (TextItemControl()->IsPasswordMode()) { - string256 passText; - shared_str str(istr); - int sz = (int)str.size(); - for (int i = 0; i < sz; i++) - passText[i] = '*'; - passText[sz] = 0; - m_dx_cur = font->SizeOf_(passText); // cursor_str + const size_t sz = XRay::Utf8::LengthCodepoints(istr); + xr_string passText(sz, '*'); + m_dx_cur = font->SizeOf_(passText.c_str()); // cursor_str } else m_dx_cur = font->SizeOf_(istr); // cursor_str diff --git a/src/xrUICore/FontManager/FontManager.cpp b/src/xrUICore/FontManager/FontManager.cpp index 1a037f28bea..626594307d9 100644 --- a/src/xrUICore/FontManager/FontManager.cpp +++ b/src/xrUICore/FontManager/FontManager.cpp @@ -4,9 +4,9 @@ CFontManager::CFontManager() { - m_all_fonts.push_back(&pFontMedium); // used cpp - m_all_fonts.push_back(&pFontDI); // used cpp - m_all_fonts.push_back(&pFontArial14); // used xml + m_all_fonts.push_back(&pFontMedium); + m_all_fonts.push_back(&pFontDI); + m_all_fonts.push_back(&pFontArial14); m_all_fonts.push_back(&pFontGraffiti19Russian); m_all_fonts.push_back(&pFontGraffiti22Russian); m_all_fonts.push_back(&pFontLetterica16Russian); @@ -26,66 +26,30 @@ CFontManager::CFontManager() void CFontManager::InitializeFonts() { - InitializeFont(pFontMedium, "hud_font_medium"); + InitializeFont(pFontMedium, "hud_font_medium", 0); InitializeFont(pFontDI, "hud_font_di", CGameFont::fsGradient | CGameFont::fsDeviceIndependent); - InitializeFont(pFontArial14, "ui_font_arial_14"); - InitializeFont(pFontGraffiti19Russian, "ui_font_graffiti19_russian"); - InitializeFont(pFontGraffiti22Russian, "ui_font_graffiti22_russian"); - InitializeFont(pFontLetterica16Russian, "ui_font_letterica16_russian"); - InitializeFont(pFontLetterica18Russian, "ui_font_letterica18_russian"); - InitializeFont(pFontGraffiti32Russian, "ui_font_graff_32"); - InitializeFont(pFontGraffiti50Russian, "ui_font_graff_50"); - InitializeFont(pFontLetterica25, "ui_font_letter_25"); + InitializeFont(pFontArial14, "ui_font_arial_14", 0); + InitializeFont(pFontGraffiti19Russian, "ui_font_graffiti19_russian", 0); + InitializeFont(pFontGraffiti22Russian, "ui_font_graffiti22_russian", 0); + InitializeFont(pFontLetterica16Russian, "ui_font_letterica16_russian", 0); + InitializeFont(pFontLetterica18Russian, "ui_font_letterica18_russian", 0); + InitializeFont(pFontGraffiti32Russian, "ui_font_graff_32", 0); + InitializeFont(pFontGraffiti50Russian, "ui_font_graff_50", 0); + InitializeFont(pFontLetterica25, "ui_font_letter_25", 0); InitializeFont(pFontStat, "stat_font", CGameFont::fsDeviceIndependent); } -LPCSTR CFontManager::GetFontTexName(LPCSTR section) -{ - constexpr pcstr tex_names[] = { "texture800", "texture", "texture1600" }; - int def_idx = 1; // default 1024x768 - int idx = def_idx; - - u32 h = Device.dwHeight; - - if (h <= 600) - idx = 0; - else if (h <= 1024) - idx = 1; - else - idx = 2; - - while (idx >= 0) - { - if (pSettings->line_exist(section, tex_names[idx])) - return pSettings->r_string(section, tex_names[idx]); - --idx; - } - return pSettings->r_string(section, tex_names[def_idx]); -} - void CFontManager::InitializeFont(CGameFont*& F, LPCSTR section, u32 flags) { - LPCSTR font_tex_name = GetFontTexName(section); - R_ASSERT(font_tex_name); - - LPCSTR sh_name = pSettings->r_string(section, "shader"); if (!F) - F = xr_new(sh_name, font_tex_name, flags); + F = xr_new(section, (u8)flags); else - F->Initialize(sh_name, font_tex_name); + F->ReInit(); #ifdef DEBUG F->m_font_name = section; #endif - if (pSettings->line_exist(section, "size")) - { - const float sz = pSettings->r_float(section, "size"); - if (flags & CGameFont::fsDeviceIndependent) - F->SetHeightI(sz); - else - F->SetHeight(sz); - } if (pSettings->line_exist(section, "interval")) F->SetInterval(pSettings->r_fvector2(section, "interval")); } diff --git a/src/xrUICore/FontManager/FontManager.h b/src/xrUICore/FontManager/FontManager.h index 505ce49b73c..fdce8d6a0fc 100644 --- a/src/xrUICore/FontManager/FontManager.h +++ b/src/xrUICore/FontManager/FontManager.h @@ -26,7 +26,6 @@ struct XRUICORE_API CFontManager : public CUIResetNotifier void InitializeFonts(); void InitializeFont(CGameFont*& F, LPCSTR section, u32 flags = 0); - LPCSTR GetFontTexName(LPCSTR section); void OnUIReset() override; }; diff --git a/src/xrUICore/Lines/UILines.cpp b/src/xrUICore/Lines/UILines.cpp index 1e8ee2579c7..e7efd2c35ac 100644 --- a/src/xrUICore/Lines/UILines.cpp +++ b/src/xrUICore/Lines/UILines.cpp @@ -10,10 +10,65 @@ #include "UILines.h" #include "XML/UIXmlInitBase.h" #include "xrCore/Text/StringConversion.hpp" +#include "xrCore/Text/Utf8Utils.hpp" constexpr auto COLOR_TAG_BEGIN = "%c["; constexpr auto COLOR_TAG_END = ']'; +namespace +{ +constexpr size_t MAX_MB_CHARS = 4096; + +// Same rules as CUILine::ProcessNewLines: LTX "\\n" plus real CR/LF (UTF-8: 0x0D/0x0A are single-byte). +bool next_line_break_in_scan(pcstr pszSearch, size_t& out_off, size_t& out_skip) +{ + out_off = 0; + out_skip = 0; + pcstr best = nullptr; + size_t best_skip = 0; + auto consider = [&](pcstr p, size_t sk) + { + if (!p) + return; + if (!best || p < best) + { + best = p; + best_skip = sk; + } + }; + + consider(strstr(pszSearch, "\\n"), 2); + + for (pcstr q = pszSearch; *q;) + { + const unsigned char c = static_cast(*q); + if (c == '\r' && q[1] == '\n') + { + consider(q, 2); + q += 2; + } + else if (c == '\r') + { + consider(q, 1); + ++q; + } + else if (c == '\n') + { + consider(q, 1); + ++q; + } + else + ++q; + } + + if (!best) + return false; + out_off = static_cast(best - pszSearch); + out_skip = best_skip; + return true; +} +} // namespace + CUILines::CUILines() { uFlags.set(flNeedReparse, false); @@ -63,11 +118,23 @@ void CUILines::SetText(const char* text) void CUILines::SetTextST(LPCSTR str_id) { SetText(StringTable().translate(str_id).c_str()); } LPCSTR CUILines::GetText() const { return m_text.c_str(); } void CUILines::Reset() { m_lines.clear(); } -float get_str_width(CGameFont* pFont, char ch) + +// Copy up to 'count' bytes from 'src' into 'dest' and trim any trailing +// incomplete UTF-8 codepoint. This is a safety net for SplitByWidth returning +// a position that falls in the middle of a multi-byte codepoint. +static size_t copy_valid_utf8(char* dest, size_t destsz, pcstr src, size_t count) { - float ll = pFont->SizeOf_(ch); - UI().ClientToScreenScaledWidth(ll); - return ll; + size_t copy_len = std::min(count, destsz - 1); + strncpy_s(dest, destsz, src, copy_len); + dest[copy_len] = '\0'; + + while (copy_len > 0 && !XRay::Utf8::IsValid(dest)) + { + --copy_len; + dest[copy_len] = '\0'; + } + + return copy_len; } void CUILines::ParseText(bool force) @@ -90,142 +157,77 @@ void CUILines::ParseText(bool force) if (uFlags.test(flRecognizeNewLine)) { - if (m_pFont->IsMultibyte()) + CUILine tmp_line; + const size_t vsz = line.m_subLines.size(); + VERIFY(vsz); + for (size_t i = 0; i < vsz; i++) { - CUILine tmp_line; - const size_t vsz = line.m_subLines.size(); - VERIFY(vsz); - for (size_t i = 0; i < vsz; i++) + const u32 tcolor = line.m_subLines[i].m_color; + char szTempLine[MAX_MB_CHARS], *pszSearch = nullptr; + [[maybe_unused]] const auto llen = line.m_subLines[i].m_text.size(); + VERIFY(llen < MAX_MB_CHARS); + xr_strcpy(szTempLine, line.m_subLines[i].m_text.c_str()); + pszSearch = szTempLine; + size_t br_off = 0, br_skip = 0; + while (next_line_break_in_scan(pszSearch, br_off, br_skip)) { - char* pszTemp; - const u32 tcolor = line.m_subLines[i].m_color; - char szTempLine[MAX_MB_CHARS], *pszSearch = nullptr; - [[maybe_unused]] const auto llen = line.m_subLines[i].m_text.size(); - VERIFY(llen < MAX_MB_CHARS); - xr_strcpy(szTempLine, line.m_subLines[i].m_text.c_str()); - pszSearch = szTempLine; - while ((pszTemp = strstr(pszSearch, "\\n")) != nullptr) - { - bNewLines = true; - *pszTemp = '\0'; - tmp_line.AddSubLine({ pszSearch, tcolor, true }); - pszSearch = pszTemp + 2; - } - tmp_line.AddSubLine(pszSearch, tcolor); + bNewLines = true; + char* seg_end = const_cast(pszSearch) + br_off; + *seg_end = '\0'; + tmp_line.AddSubLine({ pszSearch, tcolor, true }); + pszSearch = seg_end + br_skip; } - line = std::move(tmp_line); - } - else - { - line.ProcessNewLines(); + tmp_line.AddSubLine(pszSearch, tcolor); } + line = std::move(tmp_line); } - if (m_pFont->IsMultibyte()) - { -#define UBUFFER_SIZE 100 - u16 aMarkers[UBUFFER_SIZE]; - char szTempLine[MAX_MB_CHARS]; - float fTargetWidth = 1.0f; - UI().ClientToScreenScaledWidth(fTargetWidth); - VERIFY((m_wndSize.x > 0) && (fTargetWidth > 0)); - fTargetWidth = m_wndSize.x / fTargetWidth; - size_t vsz = line.m_subLines.size(); - VERIFY(vsz); - if ((vsz > 1) && (!bNewLines)) - { // only colored line - for (auto& subLine : line.m_subLines) - { - VERIFY(subLine.m_text.data()); - subLine.m_last_in_line = false; - } - m_lines.emplace_back(std::move(line)); - } - else + + // All TTF/OTF fonts are multibyte; use the UTF-8 aware SplitByWidth path. + constexpr size_t UBUFFER_SIZE = 256; + u16 aMarkers[UBUFFER_SIZE]; + char szTempLine[MAX_MB_CHARS]; + float fTargetWidth = 1.0f; + UI().ClientToScreenScaledWidth(fTargetWidth); + VERIFY((m_wndSize.x > 0) && (fTargetWidth > 0)); + fTargetWidth = m_wndSize.x / fTargetWidth; + size_t vsz = line.m_subLines.size(); + VERIFY(vsz); + if ((vsz > 1) && (!bNewLines)) + { // only colored line + for (auto& subLine : line.m_subLines) { - CUILine tmp_line; - for (size_t i = 0; i < vsz; i++) - { - CUISubLine subLine = line.m_subLines[i]; - const char* pszText = subLine.m_text.c_str(); - const u32 tcolor = subLine.m_color; - u16 uFrom = 0; - VERIFY(pszText); - u16 nMarkers = m_pFont->SplitByWidth(aMarkers, UBUFFER_SIZE, fTargetWidth, pszText); - for (u16 j = 0; j < nMarkers; j++) - { - const u16 uPartLen = aMarkers[j] - uFrom; - VERIFY((uPartLen > 0) && (uPartLen < MAX_MB_CHARS)); - strncpy_s(szTempLine, pszText + uFrom, uPartLen); - szTempLine[uPartLen] = '\0'; - tmp_line.AddSubLine(szTempLine, tcolor); - m_lines.emplace_back(tmp_line); - tmp_line.Clear(); - uFrom += uPartLen; - } - strncpy_s(szTempLine, pszText + uFrom, MAX_MB_CHARS); - tmp_line.AddSubLine(szTempLine, tcolor); - if (subLine.m_last_in_line || i == (vsz -1)) - { - m_lines.emplace_back(tmp_line); - tmp_line.Clear(); - } - } + VERIFY(subLine.m_text.data()); + subLine.m_last_in_line = false; } + m_lines.emplace_back(std::move(line)); } else { - float max_width = m_wndSize.x; - size_t sbl_cnt = line.m_subLines.size(); CUILine tmp_line; - string4096 buff; - float curr_width = 0.0f; - - float eps = get_str_width(m_pFont, 'o'); // hack -( - for (size_t sbl_idx = 0; sbl_idx < sbl_cnt; ++sbl_idx) + for (size_t i = 0; i < vsz; i++) { - bool b_last_subl = (sbl_idx == sbl_cnt - 1); - CUISubLine& sbl = line.m_subLines[sbl_idx]; - size_t sub_len = sbl.m_text.length(); - size_t curr_w_pos = 0; - - size_t last_space_idx = 0; - for (size_t idx = 0; idx < sub_len; ++idx) + CUISubLine subLine = line.m_subLines[i]; + const char* pszText = subLine.m_text.c_str(); + const u32 tcolor = subLine.m_color; + u16 uFrom = 0; + VERIFY(pszText); + u16 nMarkers = m_pFont->SplitByWidth(aMarkers, UBUFFER_SIZE, fTargetWidth, pszText); + for (u16 j = 0; j < nMarkers; j++) { - bool b_last_ch = (idx == sub_len - 1); - - if (isspace((unsigned char)sbl.m_text[idx])) - last_space_idx = idx; - - float w1 = get_str_width(m_pFont, sbl.m_text[idx]); - bool bOver = (curr_width + w1 + eps > max_width); - - if (bOver || b_last_ch) - { - if (last_space_idx && !b_last_ch) - { - idx = last_space_idx; - last_space_idx = 0; - } - - strncpy_s(buff, sizeof(buff), sbl.m_text.c_str() + curr_w_pos, idx - curr_w_pos + 1); - tmp_line.AddSubLine(buff, sbl.m_color); - curr_w_pos = idx + 1; - } - else - curr_width += w1; - - if (bOver || (b_last_ch && sbl.m_last_in_line)) - { - m_lines.emplace_back(tmp_line); - tmp_line.Clear(); - curr_width = 0.0f; - } + const u16 uPartLen = aMarkers[j] - uFrom; + VERIFY((uPartLen > 0) && (uPartLen < MAX_MB_CHARS)); + copy_valid_utf8(szTempLine, MAX_MB_CHARS, pszText + uFrom, uPartLen); + tmp_line.AddSubLine(szTempLine, tcolor); + m_lines.emplace_back(tmp_line); + tmp_line.Clear(); + uFrom += uPartLen; } - if (b_last_subl && !tmp_line.IsEmpty()) + copy_valid_utf8(szTempLine, MAX_MB_CHARS, pszText + uFrom, MAX_MB_CHARS - 1); + tmp_line.AddSubLine(szTempLine, tcolor); + if (subLine.m_last_in_line || i == (vsz -1)) { m_lines.emplace_back(tmp_line); tmp_line.Clear(); - curr_width = 0.0f; } } } @@ -245,7 +247,10 @@ float CUILines::GetVisibleHeight() } else { - float _curr_h = m_pFont->GetHeight(); + // Use CurrentHeight_() (which applies g_text_scale) instead of GetHeight() + // so that vertical centering/bottom alignment matches the actually drawn + // text height when the global text scale differs from 1.0. + float _curr_h = m_pFont->CurrentHeight_(); UI().ClientToScreenScaledHeight(_curr_h); return _curr_h; } @@ -281,21 +286,30 @@ LPCSTR GetElipsisText(CGameFont* pFont, float width, LPCSTR source_text, pstr bu buff[0] = 0; float el_len = pFont->SizeOf_(".."); UI().ClientToScreenScaledWidth(el_len); - float total = 0.0f; - u16 pos = 0; - while (total + el_len < width) + size_t buff_pos = 0; + for (pcstr p = source_text; *p; p = XRay::Utf8::Next(p)) { - const char c = *(source_text + pos); - float ch_len = pFont->SizeOf_(c); - UI().ClientToScreenScaledWidth(ch_len); + pcstr next = XRay::Utf8::Next(p); + const size_t cp_bytes = next - p; - if (total + ch_len + el_len < width) - buff[pos] = c; + xr_string slice(source_text, next - source_text); + float slice_len = pFont->SizeOf_(slice.c_str()); + UI().ClientToScreenScaledWidth(slice_len); + + if (slice_len + el_len < width) + { + if (buff_pos + cp_bytes >= static_cast(buff_len)) + break; - total += ch_len; - ++pos; - buff[pos] = 0; + strncpy_s(buff + buff_pos, static_cast(buff_len) - buff_pos, p, cp_bytes); + buff_pos += cp_bytes; + buff[buff_pos] = 0; + } + else + { + break; + } } xr_strcat(buff, buff_len, ".."); @@ -308,8 +322,6 @@ void CUILines::Draw(float x, float y) x += m_TextOffset.x; y += m_TextOffset.y; - static string256 passText; - if (m_text.empty()) return; @@ -327,12 +339,10 @@ void CUILines::Draw(float x, float y) if (uFlags.test(flPasswordMode)) { - const size_t sz = m_text.size(); - for (size_t i = 0; i < sz; i++) - passText[i] = '*'; - passText[sz] = 0; + const size_t sz = XRay::Utf8::LengthCodepoints(m_text.c_str()); + xr_string passText(sz, '*'); m_pFont->SetAligment((CGameFont::EAligment)m_eTextAlign); - m_pFont->Out(text_pos.x, text_pos.y, "%s", passText); + m_pFont->Out(text_pos.x, text_pos.y, "%s", passText.c_str()); } else { @@ -372,6 +382,261 @@ void CUILines::Draw(float x, float y) } void CUILines::OnDeviceReset() { uFlags.set(flNeedReparse, true); } + +bool CUILines::BuildVisualLineRanges(xr_vector>& out_ranges) +{ + out_ranges.clear(); + ParseText(true); + const size_t text_len = m_text.size(); + if (text_len == 0) + return true; + + if (m_lines.empty()) + return false; + + size_t accum = 0; + for (auto& l : m_lines) + { + size_t line_len = 0; + for (auto& sl : l.m_subLines) + line_len += sl.m_text.length(); + out_ranges.emplace_back(accum, line_len); + accum += line_len; + + // Parsed visual lines don't keep newline separators; remap to source text indices. + if (accum < text_len) + { + if (m_text[accum] == '\r') + { + ++accum; + if (accum < text_len && m_text[accum] == '\n') + ++accum; + } + else if (m_text[accum] == '\n') + { + ++accum; + } + else if (m_text[accum] == '\\' && (accum + 1) < text_len && m_text[accum + 1] == 'n') + { + accum += 2; + } + } + } + return true; +} + +bool CUILines::ComputeCursorPlacement(size_t cursor_pos, float& out_x_in_line, size_t& out_visual_line_idx) +{ + out_x_in_line = 0.f; + out_visual_line_idx = 0; + + if (!m_pFont) + return false; + + if (!uFlags.test(flComplexMode)) + { + const size_t c = std::min(cursor_pos, m_text.size()); + xr_string slice; + slice.assign(m_text.c_str(), c); + // Same units as CUICustomEdit::Draw: raw SizeOf_, then parent scales with ClientToScreenScaled once. + out_x_in_line = m_pFont->SizeOf_(slice.c_str()); + return true; + } + + if (m_text.empty()) + return true; + + xr_vector> line_ranges; + if (!BuildVisualLineRanges(line_ranges) || line_ranges.empty()) + return false; + + const size_t text_len = m_text.size(); + const size_t c = std::min(cursor_pos, text_len); + + size_t vi = 0; + bool found = false; + for (size_t i = 0; i < line_ranges.size(); ++i) + { + const size_t start = line_ranges[i].first; + const size_t len = line_ranges[i].second; + if (c >= start && c <= start + len) + { + vi = i; + found = true; + break; + } + } + if (!found) + return false; + + const size_t line_start = line_ranges[vi].first; + const size_t span = c - line_start; + + xr_string slice; + slice.assign(m_text.c_str() + line_start, span); + out_x_in_line = m_pFont->SizeOf_(slice.c_str()); + out_visual_line_idx = vi; + return true; +} + +bool CUILines::CursorPosFromLocalPoint(float local_x, float local_y, size_t& out_pos, float scroll_offset_y, + const float* vindent_override) +{ + out_pos = 0; + if (!m_pFont) + return false; + + auto prefix_width = [&](size_t glyph_start, size_t n) -> float + { + if (n == 0) + return 0.f; + xr_string slice; + slice.assign(m_text.c_str() + glyph_start, n); + float w = m_pFont->SizeOf_(slice.c_str()); + UI().ClientToScreenScaledWidth(w); + return w; + }; + + if (!uFlags.test(flComplexMode)) + { + const size_t len = m_text.size(); + if (len == 0) + return true; + + const float rel_x = local_x - m_TextOffset.x - GetIndentByAlign(); + if (rel_x <= 0.f) + { + out_pos = 0; + return true; + } + + pcstr p = m_text.c_str(); + while (*p) + { + pcstr next = XRay::Utf8::Next(p); + const float w = prefix_width(0, next - m_text.c_str()); + if (rel_x <= w) + { + out_pos = next - m_text.c_str(); + return true; + } + p = next; + } + out_pos = len; + return true; + } + + ParseText(true); + if (m_text.empty()) + return true; + + xr_vector> line_ranges; + if (!BuildVisualLineRanges(line_ranges) || line_ranges.empty()) + return false; + + float line_h = m_pFont->CurrentHeight_(); + UI().ClientToScreenScaledHeight(line_h); + + const float vindent = vindent_override ? *vindent_override : GetVIndentByAlign(); + const float rel_y = local_y - m_TextOffset.y - vindent + scroll_offset_y; + const float rel_x = local_x - m_TextOffset.x - GetIndentByAlign(); + + const float total_h = line_h * static_cast(line_ranges.size()); + int line_idx = 0; + if (line_h > EPS_L) + { + if (rel_y < 0.f) + line_idx = 0; + else if (rel_y >= total_h) + line_idx = static_cast(line_ranges.size()) - 1; + else + line_idx = static_cast(rel_y / line_h); + } + + { + const int max_i = static_cast(line_ranges.size()) - 1; + if (line_idx < 0) + line_idx = 0; + else if (line_idx > max_i) + line_idx = max_i; + } + + const size_t start = line_ranges[line_idx].first; + const size_t line_len = line_ranges[line_idx].second; + const size_t line_end = start + line_len; + + if (rel_x <= 0.f) + { + out_pos = start; + return true; + } + + pcstr line_start = m_text.c_str() + start; + pcstr p = line_start; + while (*p) + { + pcstr next = XRay::Utf8::Next(p); + if (next > m_text.c_str() + line_end) + break; + + const float w = prefix_width(start, next - line_start); + if (rel_x <= w) + { + out_pos = next - m_text.c_str(); + return true; + } + p = next; + } + out_pos = line_end; + return true; +} + +bool CUILines::MoveCursorByVisualLine(size_t& io_cursor, int delta) +{ + if (!uFlags.test(flComplexMode) || delta == 0 || (delta != -1 && delta != 1)) + return false; + + if (m_text.empty()) + return false; + + xr_vector> line_ranges; + if (!BuildVisualLineRanges(line_ranges) || line_ranges.empty()) + return false; + + const size_t text_len = m_text.size(); + + const size_t c = std::min(io_cursor, text_len); + size_t vi = 0; + size_t col = 0; + bool found = false; + for (size_t i = 0; i < line_ranges.size(); ++i) + { + const size_t start = line_ranges[i].first; + const size_t len = line_ranges[i].second; + if (c >= start && c <= start + len) + { + vi = i; + col = XRay::Utf8::DistanceCodepoints(m_text.c_str() + start, m_text.c_str() + c); + found = true; + break; + } + } + if (!found) + return false; + + const int target_vi = static_cast(vi) + delta; + if (target_vi < 0 || static_cast(target_vi) >= line_ranges.size()) + return false; + + const size_t tgt_start = line_ranges[target_vi].first; + const size_t tgt_len = line_ranges[target_vi].second; + const size_t tgt_cp_len = XRay::Utf8::DistanceCodepoints( + m_text.c_str() + tgt_start, m_text.c_str() + tgt_start + tgt_len); + const size_t new_col = std::min(col, tgt_cp_len); + io_cursor = XRay::Utf8::Advance(m_text.c_str() + tgt_start, new_col) - m_text.c_str(); + return true; +} + float CUILines::GetIndentByAlign() const { switch (m_eTextAlign) diff --git a/src/xrUICore/Lines/UILines.h b/src/xrUICore/Lines/UILines.h index 442fa79bdbb..307847f6133 100644 --- a/src/xrUICore/Lines/UILines.h +++ b/src/xrUICore/Lines/UILines.h @@ -49,6 +49,13 @@ class XRUICORE_API CUILines final : public CDeviceResetNotifier float GetVisibleHeight(); float GetIndentByAlign() const; + // UTF-8 text cursor helpers used by CUICustomEdit + bool BuildVisualLineRanges(xr_vector>& out_ranges); + bool ComputeCursorPlacement(size_t cursor_pos, float& out_x_in_line, size_t& out_visual_line_idx); + bool CursorPosFromLocalPoint(float local_x, float local_y, size_t& out_pos, float scroll_offset_y, + const float* vindent_override = nullptr); + bool MoveCursorByVisualLine(size_t& io_cursor, int delta); + Fvector2 m_TextOffset{}; Fvector2 m_wndSize{}; Fvector2 m_wndPos{}; diff --git a/src/xrUICore/ui_base.cpp b/src/xrUICore/ui_base.cpp index 37d3abaf41f..76938a38edc 100644 --- a/src/xrUICore/ui_base.cpp +++ b/src/xrUICore/ui_base.cpp @@ -7,7 +7,6 @@ CUICursor& GetUICursor() { return GEnv.UI->GetUICursor(); } UICore& UI() { return *GEnv.UI; } -extern ENGINE_API Fvector2 g_current_font_scale; void S2DVert::rotate_pt(const Fvector2& pivot, const float cosA, const float sinA, const float kx) { @@ -220,7 +219,6 @@ UICore::UICore() OnUIReset(); m_current_scale = &m_scale_; - g_current_font_scale.set(1.0f, 1.0f); m_currentPointType = IUIRender::pttTL; } @@ -300,16 +298,12 @@ void UICore::pp_start() float(Device.dwHeight))); m_current_scale = &m_pp_scale_; - - g_current_font_scale.set(float(Device.dwWidth) / float(Device.dwWidth), - float(Device.dwHeight) / float(Device.dwHeight)); } void UICore::pp_stop() { m_bPostprocess = false; m_current_scale = &m_scale_; - g_current_font_scale.set(1.0f, 1.0f); } void UICore::RenderFont()