diff --git a/examples/readme_examples.cpp b/examples/readme_examples.cpp index 1493729..2fe9a03 100644 --- a/examples/readme_examples.cpp +++ b/examples/readme_examples.cpp @@ -21,11 +21,13 @@ import std; #include #include #include +#include #include #include #include #include #include +#include #endif namespace beman::utf_view::examples { @@ -267,7 +269,7 @@ constexpr bool is_continuation(T c) { } template -constexpr detail::fake_inplace_vector transcode_trucating_correctly( +constexpr detail::fake_inplace_vector transcode_truncating_correctly( std::basic_string_view input) { detail::fake_inplace_vector output; for (auto code_point_view : input @@ -282,6 +284,50 @@ constexpr detail::fake_inplace_vector transcode_trucating_correctly( } #endif +#if __cpp_lib_ranges_as_input >= 202502L +void print_utf8_code_points_and_code_units(std::ranges::range auto input) { + auto print_code_point{ + [](char32_t code_point, auto code_unit_range) { + std::println( + "{:#x} = {::#x}", static_cast(code_point), + code_unit_range + | std::views::transform([](char8_t c) { return (std::uint8_t)c; })); + }}; + auto code_points = input + | std::ranges::to() + | beman::utf_view::to_utf32; + for (auto it = code_points.begin(); it != code_points.end(); ++it) { + print_code_point(*it, std::ranges::subrange(it.base(), std::ranges::next(it).base())); + } +} + +constexpr bool is_utf8_continuation(char8_t c) { return (c & 0xC0) == 0x80; } + +void print_utf16_and_utf8_code_units_per_code_point(std::ranges::range auto input) { + auto print_code_point{ + [](auto u16_view, auto u8_view) { + std::println( + "{::#x} = {::#x}", + u16_view | std::views::transform([](char16_t c) { return (std::uint16_t)c; }), + u8_view | std::views::transform([](char8_t c) { return (std::uint8_t)c; })); + }}; + auto it = input.begin(); + std::u8string code_point; + while (it != input.end()) { + code_point.clear(); + code_point.push_back(*it); // lead byte + ++it; + it = std::ranges::find_if(std::move(it), input.end(), [&](char8_t c) { + if (!is_utf8_continuation(c)) return true; // next lead: stop, don't consume + code_point.push_back(c); // continuation: keep + return false; + }); + print_code_point(code_point | beman::utf_view::to_utf16, code_point); + } +} + +#endif + bool readme_examples() { using namespace std::string_view_literals; #ifndef _MSC_VER @@ -335,23 +381,27 @@ bool readme_examples() { return false; } { - auto result = transcode_trucating_correctly(u8"πŸ˜€abc"sv); + auto result = transcode_truncating_correctly(u8"πŸ˜€abc"sv); if (result.size() != 5) { return false; } - auto result2 = transcode_trucating_correctly(u8"πŸ˜€abc"sv); + auto result2 = transcode_truncating_correctly(u8"πŸ˜€abc"sv); if (result2.size() != 4) { return false; } - auto result3 = transcode_trucating_correctly(u8"πŸ˜€abc"sv); + auto result3 = transcode_truncating_correctly(u8"πŸ˜€abc"sv); if (result3.size() != 2) { return false; } - auto result4 = transcode_trucating_correctly(u8"πŸ˜€abc"sv); + auto result4 = transcode_truncating_correctly(u8"πŸ˜€abc"sv); if (result4.size() != 0) { return false; } } +#endif +#if __cpp_lib_ranges_as_input >= 202502L + print_utf8_code_points_and_code_units(u8"AΞ©β‚¬πŸ˜€b"sv | std::views::as_input); + print_utf16_and_utf8_code_units_per_code_point(u8"AΞ©β‚¬πŸ˜€b"sv | std::views::as_input); #endif { std::u16string zamin = u"π’€­π’Žπ’„ˆπ’‹’π’ π’Š©"; diff --git a/papers/P2728.md b/papers/P2728.md index 26c41a2..2bcf17b 100644 --- a/papers/P2728.md +++ b/papers/P2728.md @@ -1,6 +1,6 @@ --- title: "Unicode in the Library, Part 1: UTF Transcoding" -document: P2728R14 +document: D2728R14 date: 2026-06-07 audience: - SG-16 Unicode @@ -437,7 +437,7 @@ constexpr bool is_continuation(T c) { } template -constexpr std::inplace_vector transcode_trucating_correctly( +constexpr std::inplace_vector transcode_truncating_correctly( std::basic_string_view input) { std::inplace_vector output; for (auto code_point_view : input @@ -458,12 +458,12 @@ constexpr std::inplace_vector transcode_trucating_correctly( // Adapted from an ICU unit test: // https://github.com/unicode-org/icu/blob/649262a75ecddb15a0e58d71f637a8a32eaabd43/icu4c/source/test/intltest/utfiteratortest.cpp#L1205-L1228 std::u16string zamin = u"π’€­π’Žπ’„ˆπ’‹’π’ π’Š©"; -auto it = zamin | std::views::to_utf32; -auto begin = it.begin(); -++begin; -auto ningirsuBegin = begin.base(); -std::advance(begin, 3); // was 2 + implicit end() -auto ningirsuEnd = begin.base(); +auto view = zamin | std::views::to_utf32; +auto it = view.begin(); +++it; +auto ningirsuBegin = it.base(); +std::advance(it, 3); +auto ningirsuEnd = it.base(); zamin.replace(ningirsuBegin, ningirsuEnd, u"π’ŠΊπ’‰€"); assert(std::ranges::equal(zamin, u"π’€­π’ŠΊπ’‰€π’ π’Š©"sv)); ``` @@ -1202,12 +1202,223 @@ There's precedent for this kind of approach in the `views::reverse` CPO, which s gives back the original underlying view if it detects that it's reversing another `reverse_view`. +## `.base_code_units()` + +### Proposal + +I received feedback that it could be useful to provide a `.base_code_units()` member +function on the transcoding iterator which would give out a range of iterators from the +underlying range delimiting the code units that make up the current code point. + +Since we can't give out iterators to the underlying range if it's a (non-forward) input +range, it's also been suggested that in this case, `.base_code_units()` would still be +available, but would give out iterators to a special cache that's stored in the iterator. + +To quote from a reflector email discussing this suggestion: + +> I think it would be useful to differentiate access to the (complete) underlying range vs +> access to the input code unit sequence for the current character. Obviously, access to the +> complete underlying range isn't possible for input iterators, but access to the current +> input code unit sequence is (with the caching approach described above is). The iterators +> could expose this interface: +> +> ```c++ +> // Forward+ iterators only; returns an iterator into the underlying range. +> constexpr const iterator_t& base() const & noexcept requires forward_range { ... } +> constexpr iterator_t base() && requires forward_range { ... } +> +> // Input+ iterators; returns a subrange containing the input code units for the current character. +> // References the input code unit sequence cache for input iterators. +> // References the underlying range otherwise. +> constexpr subrange<...> base_code_units() const noexcept { ... } +> ``` +> +> Unlike `base()`, `base_code_units()` would not necessarily contain iterators for the +> underlying range (e.g., in the case of a caching input iterator). + +Note that the choice to provide `.base_code_units()` for input ranges affects ABI since +the size of the transcoding iterator depends on whether it contains the cache. + +### Precedent + +[@P0244R2] provides transcoding iterators with a `.base_range()` member function that +provide this range, although its input iterator functionality is implemented using special +caching iterators that have shared ownership of a cache, instead of by storing the cached +range in the iterator itself. + +ICU provides multiple analogous APIs. The most directly comparable one is the +`.stringView()` [member +function](https://github.com/unicode-org/icu/blob/649262a75ecddb15a0e58d71f637a8a32eaabd43/icu4c/source/common/unicode/utfiterator.h#L416) +on the `UnsafeCodeUnits` transcoding iterator, which provides a `std::basic_string_view` +containing the underlying code units for the current code point. `UnsafeCodeUnits` also +provides `.begin()` and `.end()` member functions which give out the same range. Unlike +the proposed `.base_code_units()` member function, neither of these APIs provide support +for input iterators; `.stringView()` is only enabled when the base range is contiguous, +and `.begin()` and `.end()` are only enabled if it's a forward range. + +### Lifetime Issues + +Here's an example of a function where the use of `.base_code_units()` subtly introduces UB +when the function is passed an input range. + +This is a run-length-encoder that prints a count of the number of consecutive times it's +seen a code point, followed by the code units making up that code point: + +```cpp +void print_runs(std::ranges::range auto input) { + auto utf_view = input | std::views::to_utf32; + auto it = utf_view.begin(); + while (it != utf_view.end()) { + auto units = it.base_code_units(); + char32_t code_point = *it; + int count = 1; + ++it; + while (it != utf_view.end() && *it == code_point) { + ++count; + ++it; + } + std::print( + "{}x{::#x} ", count, + units | std::views::transform([](char8_t c) { return (std::uint8_t)c; } )); + } + std::println(""); +} +``` + +When invoked with `u8"β’Άβ’Άβ’Άβ’·β’Έ"sv`, it prints: + +``` +3x[0xe2, 0x92, 0xb6] 1x[0xe2, 0x92, 0xb7] 1x[0xe2, 0x92, 0xb8] +``` + +When invoked with `u8"β’Άβ’Άβ’Άβ’·β’Έ"sv | std::views::as_input`, it invokes library undefined +behavior and prints corrupted output. Worse, the UB here isn't caught by AddressSanitizer +or UndefinedBehaviorSanitizer because the invalidated `auto units` range points into the +same, valid, transcoding iterator, whose cache simply contains the values for the +subsequent code point, so the corrupted output is not automatically diagnosable. + +With ICU's APIs, on the other hand, this would fail to compile, because ICU only provides +them for forward ranges. + +I think this footgun would show up frequently. + +### Alternatives to `.base_code_units()` for Users + +#### Forward Ranges + +For forward ranges, `it.base_code_units()` is equivalent to +`std::ranges::subrange(it.base(), std::ranges::next(it).base())`. + +The expression above raised concerns about the fact that its use in a loop would mean +performing two `operator++` operations on every loop iteration, but that can be mitigated +by simply caching the previous iterator while iterating forwards: + +```cpp +auto prev_base = it.base(); +++it; +auto code_units = std::ranges::subrange(prev_base, it.base()); +``` + +#### Input Ranges + +For input ranges, since the transcoding view doesn't provide `.base()`, the workaround +involves making a copy of the input range in order to get a forward range. + +##### Copying the entire range + +If it's viable to copy the entire range, you can simply insert a +`std::ranges::to()` into the range pipeline. + +```cpp +void print_utf8_code_points_and_code_units(std::ranges::range auto input) { + auto print_code_point{ + [](char32_t code_point, auto code_unit_range) { + std::println( + "{:#x} = {::#x}", static_cast(code_point), + code_unit_range | std::views::transform([](char8_t c) { return (std::uint8_t)c; })); + }}; + auto code_points = input + | std::ranges::to() + | std::views::to_utf32; + for (auto it = code_points.begin(); it != code_points.end(); ++it) { + print_code_point(*it, std::ranges::subrange(it.base(), std::ranges::next(it).base())); + } +} +``` + +When invoked with `u8"AΞ©β‚¬πŸ˜€b"sv | std::views::as_input`, this prints: + +``` +0x41 = [0x41] +0x3a9 = [0xce, 0xa9] +0x20ac = [0xe2, 0x82, 0xac] +0x1f600 = [0xf0, 0x9f, 0x98, 0x80] +0x62 = [0x62] +``` + +##### Avoiding copying + +Although it requires rolling your own segmentation, it is possible to iterate over an +input view's code unit subsequences with additional refactoring: + +```cpp +constexpr bool is_utf8_continuation(char8_t c) { return (c & 0xC0) == 0x80; } + +void print_utf16_and_utf8_code_units_per_code_point(std::ranges::range auto input) { + auto print_code_point{ + [](auto u16_view, auto u8_view) { + std::println( + "{::#x} = {::#x}", + u16_view | std::views::transform([](char16_t c) { return (std::uint16_t)c; }), + u8_view | std::views::transform([](char8_t c) { return (std::uint8_t)c; })); + }}; + auto it = input.begin(); + std::u8string code_point; + while (it != input.end()) { + code_point.clear(); + code_point.push_back(*it); + ++it; + it = std::ranges::find_if(std::move(it), input.end(), [&](char8_t c) { + if (!is_utf8_continuation(c)) return true; + code_point.push_back(c); + return false; + }); + print_code_point(code_point | std::views::to_utf16, code_point); + } +} +``` + +When invoked with `u8"AΞ©β‚¬πŸ˜€b"sv | std::views::as_input`, this prints: + +``` +[0x41] = [0x41] +[0x3a9] = [0xce, 0xa9] +[0x20ac] = [0xe2, 0x82, 0xac] +[0xd83d, 0xde00] = [0xf0, 0x9f, 0x98, 0x80] +[0x62] = [0x62] +``` + +Clearly, this isn't an ideal user experience, but it only applies to users who: + +- Need to access the underlying code unit sequence for a code point, and +- Have a non-forward input range +- That is too large to copy + +In my opinion, preserving the ergonomics of that use case is not worth the tradeoff of +introducing the safety footgun demonstrated by the RLE example above. + +### Implementation + +An experimental implementation of `.base_code_units()` is available on the +`enolan_basecodeunits1` branch of `beman.utf_view`. + # Changelog ## Changes since R13 - Fix typo in Table 3-9 of error handling diagram pointed out during SG16 review (`ED A0` had been misspelled as `E0 A0`). +- Add design discussion about `.base_code_units()`. ## Changes since R12