bemanproject · ednolan · Jun 7, 2026 · Jun 7, 2026
@@ -21,11 +21,13 @@ import std;
 #include <functional>
 #include <iterator>
 #include <optional>
+#include <print>
 #include <ranges>
 #include <stdexcept>
 #include <string>
 #include <string_view>
 #include <vector>
+#include <version>
 #endif
 
 namespace beman::utf_view::examples {
@@ -267,7 +269,7 @@ constexpr bool is_continuation(T c) {
 }
 
 template <typename FromType, typename ToType, std::size_t N>
-constexpr detail::fake_inplace_vector<ToType, N> transcode_trucating_correctly(
+constexpr detail::fake_inplace_vector<ToType, N> transcode_truncating_correctly(
     std::basic_string_view<FromType> input) {
   detail::fake_inplace_vector<ToType, N> output;
   for (auto code_point_view : input
@@ -282,6 +284,50 @@ constexpr detail::fake_inplace_vector<ToType, N> transcode_trucating_correctly(
 }
 #endif
 
+#if __cpp_lib_ranges_as_input >= 202502L
+void print_utf8_code_points_and_code_units(std::ranges::range auto input) {
+  auto print_code_point{
+    [](char32_t code_point, auto code_unit_range) {
+    std::println(
+      "{:#x} = {::#x}", static_cast<std::uint32_t>(code_point),
+      code_unit_range
+      | std::views::transform([](char8_t c) { return (std::uint8_t)c; }));
+    }};
+  auto code_points = input
+                     | std::ranges::to<std::u8string>()
+                     | beman::utf_view::to_utf32;
+  for (auto it = code_points.begin(); it != code_points.end(); ++it) {
+    print_code_point(*it, std::ranges::subrange(it.base(), std::ranges::next(it).base()));
+  }
+}
+
+constexpr bool is_utf8_continuation(char8_t c) { return (c & 0xC0) == 0x80; }
+
+void print_utf16_and_utf8_code_units_per_code_point(std::ranges::range auto input) {
+  auto print_code_point{
+    [](auto u16_view, auto u8_view) {
+      std::println(
+        "{::#x} = {::#x}",
+        u16_view | std::views::transform([](char16_t c) { return (std::uint16_t)c; }),
+        u8_view | std::views::transform([](char8_t c) { return (std::uint8_t)c; }));
+    }};
+  auto it = input.begin();
+  std::u8string code_point;
+  while (it != input.end()) {
+    code_point.clear();
+    code_point.push_back(*it);                     // lead byte
+    ++it;
+    it = std::ranges::find_if(std::move(it), input.end(), [&](char8_t c) {
+      if (!is_utf8_continuation(c)) return true;    // next lead: stop, don't consume
+      code_point.push_back(c);                      // continuation: keep
+      return false;
+    });
+    print_code_point(code_point | beman::utf_view::to_utf16, code_point);
+  }
+}
+
+#endif
+
 bool readme_examples() {
   using namespace std::string_view_literals;
 #ifndef _MSC_VER
@@ -335,23 +381,27 @@ bool readme_examples() {
     return false;
   }
   {
-    auto result = transcode_trucating_correctly<char8_t, char16_t, 5>(u8"😀abc"sv);
+    auto result = transcode_truncating_correctly<char8_t, char16_t, 5>(u8"😀abc"sv);
     if (result.size() != 5) {
       return false;
     }
-    auto result2 = transcode_trucating_correctly<char8_t, char16_t, 4>(u8"😀abc"sv);
+    auto result2 = transcode_truncating_correctly<char8_t, char16_t, 4>(u8"😀abc"sv);
     if (result2.size() != 4) {
       return false;
     }
-    auto result3 = transcode_trucating_correctly<char8_t, char16_t, 2>(u8"😀abc"sv);
+    auto result3 = transcode_truncating_correctly<char8_t, char16_t, 2>(u8"😀abc"sv);
     if (result3.size() != 2) {
       return false;
     }
-    auto result4 = transcode_trucating_correctly<char8_t, char16_t, 1>(u8"😀abc"sv);
+    auto result4 = transcode_truncating_correctly<char8_t, char16_t, 1>(u8"😀abc"sv);
     if (result4.size() != 0) {
       return false;
     }
   }
+#endif
+#if __cpp_lib_ranges_as_input >= 202502L
+  print_utf8_code_points_and_code_units(u8"AΩ€😀b"sv | std::views::as_input);
+  print_utf16_and_utf8_code_units_per_code_point(u8"AΩ€😀b"sv | std::views::as_input);
 #endif
   {
     std::u16string zamin = u"𒀭𒎏𒄈𒋢𒍠𒊩";

@@ -1,6 +1,6 @@
 ---
 title: "Unicode in the Library, Part 1: UTF Transcoding"
-document: P2728R14
+document: D2728R14
 date: 2026-06-07
 audience:
   - SG-16 Unicode
@@ -437,7 +437,7 @@ constexpr bool is_continuation(T c) {
 }
 
 template <typename FromType, typename ToType, std::size_t N>
-constexpr std::inplace_vector<ToType, N> transcode_trucating_correctly(
+constexpr std::inplace_vector<ToType, N> transcode_truncating_correctly(
     std::basic_string_view<FromType> input) {
   std::inplace_vector<ToType, N> output;
   for (auto code_point_view : input
@@ -458,12 +458,12 @@ constexpr std::inplace_vector<ToType, N> transcode_trucating_correctly(
 // Adapted from an ICU unit test:
 // https://github.com/unicode-org/icu/blob/649262a75ecddb15a0e58d71f637a8a32eaabd43/icu4c/source/test/intltest/utfiteratortest.cpp#L1205-L1228
 std::u16string zamin = u"𒀭𒎏𒄈𒋢𒍠𒊩";
-auto it = zamin | std::views::to_utf32;
-auto begin = it.begin();
-++begin;
-auto ningirsuBegin = begin.base();
-std::advance(begin, 3); // was 2 + implicit end()
-auto ningirsuEnd = begin.base();
+auto view = zamin | std::views::to_utf32;
+auto it = view.begin();
+++it;
+auto ningirsuBegin = it.base();
+std::advance(it, 3);
+auto ningirsuEnd = it.base();
 zamin.replace(ningirsuBegin, ningirsuEnd, u"𒊺𒉀");
 assert(std::ranges::equal(zamin, u"𒀭𒊺𒉀𒍠𒊩"sv));
 ```
@@ -1202,12 +1202,223 @@ There's precedent for this kind of approach in the `views::reverse` CPO, which s
 gives back the original underlying view if it detects that it's reversing another
 `reverse_view`.
 
+## `.base_code_units()`
+
+### Proposal
+
+I received feedback that it could be useful to provide a `.base_code_units()` member
+function on the transcoding iterator which would give out a range of iterators from the
+underlying range delimiting the code units that make up the current code point.
+
+Since we can't give out iterators to the underlying range if it's a (non-forward) input
+range, it's also been suggested that in this case, `.base_code_units()` would still be
+available, but would give out iterators to a special cache that's stored in the iterator.
+
+To quote from a reflector email discussing this suggestion:
+
+> I think it would be useful to differentiate access to the (complete) underlying range vs
+> access to the input code unit sequence for the current character. Obviously, access to the
+> complete underlying range isn't possible for input iterators, but access to the current
+> input code unit sequence is (with the caching approach described above is). The iterators
+> could expose this interface:
+>
+> ```c++
+>     // Forward+ iterators only; returns an iterator into the underlying range.
+>     constexpr const iterator_t<Base>& base() const & noexcept requires forward_range<Base> { ... }
+>     constexpr iterator_t<Base> base() && requires forward_range<Base> { ... }
+>
+>     // Input+ iterators; returns a subrange containing the input code units for the current character.
+>     // References the input code unit sequence cache for input iterators.
+>     // References the underlying range otherwise.
+>     constexpr subrange<...> base_code_units() const noexcept { ... }
+> ```
+>
+> Unlike `base()`, `base_code_units()` would not necessarily contain iterators for the
+> underlying range (e.g., in the case of a caching input iterator).
+
+Note that the choice to provide `.base_code_units()` for input ranges affects ABI since
+the size of the transcoding iterator depends on whether it contains the cache.
+
+### Precedent
+
+[@P0244R2] provides transcoding iterators with a `.base_range()` member function that
+provide this range, although its input iterator functionality is implemented using special
+caching iterators that have shared ownership of a cache, instead of by storing the cached
+range in the iterator itself.
+
+ICU provides multiple analogous APIs. The most directly comparable one is the
+`.stringView()` [member
+function](https://github.com/unicode-org/icu/blob/649262a75ecddb15a0e58d71f637a8a32eaabd43/icu4c/source/common/unicode/utfiterator.h#L416)
+on the `UnsafeCodeUnits` transcoding iterator, which provides a `std::basic_string_view`
+containing the underlying code units for the current code point. `UnsafeCodeUnits` also
+provides `.begin()` and `.end()` member functions which give out the same range. Unlike
+the proposed `.base_code_units()` member function, neither of these APIs provide support
+for input iterators; `.stringView()` is only enabled when the base range is contiguous,
+and `.begin()` and `.end()` are only enabled if it's a forward range.
+
+### Lifetime Issues
+
+Here's an example of a function where the use of `.base_code_units()` subtly introduces UB
+when the function is passed an input range.
+
+This is a run-length-encoder that prints a count of the number of consecutive times it's
+seen a code point, followed by the code units making up that code point:
+
+```cpp
+void print_runs(std::ranges::range auto input) {
+  auto utf_view = input | std::views::to_utf32;
+  auto it = utf_view.begin();
+  while (it != utf_view.end()) {
+    auto units = it.base_code_units();
+    char32_t code_point = *it;
+    int count = 1;
+    ++it;
+    while (it != utf_view.end() && *it == code_point) {
+      ++count;
+      ++it;
+    }
+    std::print(
+      "{}x{::#x} ", count,
+      units | std::views::transform([](char8_t c) { return (std::uint8_t)c; } ));
+  }
+  std::println("");
+}
+```
+
+When invoked with `u8"ⒶⒶⒶⒷⒸ"sv`, it prints:
+
+```
+3x[0xe2, 0x92, 0xb6] 1x[0xe2, 0x92, 0xb7] 1x[0xe2, 0x92, 0xb8]
+```
+
+When invoked with `u8"ⒶⒶⒶⒷⒸ"sv | std::views::as_input`, it invokes library undefined
+behavior and prints corrupted output. Worse, the UB here isn't caught by AddressSanitizer
+or UndefinedBehaviorSanitizer because the invalidated `auto units` range points into the
+same, valid, transcoding iterator, whose cache simply contains the values for the
+subsequent code point, so the corrupted output is not automatically diagnosable.
+
+With ICU's APIs, on the other hand, this would fail to compile, because ICU only provides
+them for forward ranges.
+
+I think this footgun would show up frequently.
+
+### Alternatives to `.base_code_units()` for Users
+
+#### Forward Ranges
+
+For forward ranges, `it.base_code_units()` is equivalent to
+`std::ranges::subrange(it.base(), std::ranges::next(it).base())`.
+
+The expression above raised concerns about the fact that its use in a loop would mean
+performing two `operator++` operations on every loop iteration, but that can be mitigated
+by simply caching the previous iterator while iterating forwards:
+
+```cpp
+auto prev_base = it.base();
+++it;
+auto code_units = std::ranges::subrange(prev_base, it.base());
+```
+
+#### Input Ranges
+
+For input ranges, since the transcoding view doesn't provide `.base()`, the workaround
+involves making a copy of the input range in order to get a forward range.
+
+##### Copying the entire range
+
+If it's viable to copy the entire range, you can simply insert a
+`std::ranges::to<std::u8string>()` into the range pipeline.
+
+```cpp
+void print_utf8_code_points_and_code_units(std::ranges::range auto input) {
+  auto print_code_point{
+    [](char32_t code_point, auto code_unit_range) {
+    std::println(
+      "{:#x} = {::#x}", static_cast<std::uint32_t>(code_point),
+      code_unit_range | std::views::transform([](char8_t c) { return (std::uint8_t)c; }));
+    }};
+  auto code_points = input
+                     | std::ranges::to<std::u8string>()
+                     | std::views::to_utf32;
+  for (auto it = code_points.begin(); it != code_points.end(); ++it) {
+    print_code_point(*it, std::ranges::subrange(it.base(), std::ranges::next(it).base()));
+  }
+}
+```
+
+When invoked with `u8"AΩ€😀b"sv | std::views::as_input`, this prints:
+
+```
+0x41 = [0x41]
+0x3a9 = [0xce, 0xa9]
+0x20ac = [0xe2, 0x82, 0xac]
+0x1f600 = [0xf0, 0x9f, 0x98, 0x80]
+0x62 = [0x62]
+```
+
+##### Avoiding copying
+
+Although it requires rolling your own segmentation, it is possible to iterate over an
+input view's code unit subsequences with additional refactoring:
+
+```cpp
+constexpr bool is_utf8_continuation(char8_t c) { return (c & 0xC0) == 0x80; }
+
+void print_utf16_and_utf8_code_units_per_code_point(std::ranges::range auto input) {
+  auto print_code_point{
+    [](auto u16_view, auto u8_view) {
+      std::println(
+        "{::#x} = {::#x}",
+        u16_view | std::views::transform([](char16_t c) { return (std::uint16_t)c; }),
+        u8_view | std::views::transform([](char8_t c) { return (std::uint8_t)c; }));
+    }};
+  auto it = input.begin();
+  std::u8string code_point;
+  while (it != input.end()) {
+    code_point.clear();
+    code_point.push_back(*it);
+    ++it;
+    it = std::ranges::find_if(std::move(it), input.end(), [&](char8_t c) {
+      if (!is_utf8_continuation(c)) return true;
+      code_point.push_back(c);
+      return false;
+    });
+    print_code_point(code_point | std::views::to_utf16, code_point);
+  }
+}
+```
+
+When invoked with `u8"AΩ€😀b"sv | std::views::as_input`, this prints:
+
+```
+[0x41] = [0x41]
+[0x3a9] = [0xce, 0xa9]
+[0x20ac] = [0xe2, 0x82, 0xac]
+[0xd83d, 0xde00] = [0xf0, 0x9f, 0x98, 0x80]
+[0x62] = [0x62]
+```
+
+Clearly, this isn't an ideal user experience, but it only applies to users who:
+
+- Need to access the underlying code unit sequence for a code point, and
+- Have a non-forward input range
+- That is too large to copy
+
+In my opinion, preserving the ergonomics of that use case is not worth the tradeoff of
+introducing the safety footgun demonstrated by the RLE example above.
+
+### Implementation
+
+An experimental implementation of `.base_code_units()` is available on the
+`enolan_basecodeunits1` branch of `beman.utf_view`.
+
 # Changelog
 
 ## Changes since R13
 
 - Fix typo in Table 3-9 of error handling diagram pointed out during SG16 review (`ED A0`
   had been misspelled as `E0 A0`).
+- Add design discussion about `.base_code_units()`.
 
 ## Changes since R12