Add function to check string for invalid UTF-8 characters

Author: joto

include/osmium/io/detail/string_util.hpp

@@ -142,6 +142,55 @@ namespace osmium {
                 return 0;
             }
 
+            /**
+             * Check that a string contains only valid UTF-8. Returns nullptr
+             * if the string is valid, otherwise it returns a pointer to the
+             * first invalid character.
+             *
+             * Based on https://www.cl.cam.ac.uk/~mgk25/ucs/utf8_check.c
+             * License: http://www.cl.cam.ac.uk/~mgk25/short-license.html
+             */
+            inline char const* utf8_check(char const* str) noexcept {
+                auto* s = reinterpret_cast<unsigned char const*>(str);
+                while (*s) {
+                    if (*s < 0x80) {
+                        /* 0xxxxxxx */
+                        s++;
+                    } else if ((s[0] & 0xe0) == 0xc0) {
+                        /* 110XXXXx 10xxxxxx */
+                        if ((s[1] & 0xc0) != 0x80 || (s[0] & 0xfe) == 0xc0) /* overlong? */ {
+                            return reinterpret_cast<char const *>(s);
+                        } else {
+                            s += 2;
+                        }
+                    } else if ((s[0] & 0xf0) == 0xe0) {
+                        /* 1110XXXX 10Xxxxxx 10xxxxxx */
+                        if ((s[1] & 0xc0) != 0x80 || (s[2] & 0xc0) != 0x80 ||
+                            (s[0] == 0xe0 && (s[1] & 0xe0) == 0x80) || /* overlong? */
+                            (s[0] == 0xed && (s[1] & 0xe0) == 0xa0) || /* surrogate? */
+                            (s[0] == 0xef && s[1] == 0xbf &&
+                            (s[2] & 0xfe) == 0xbe)) /* U+FFFE or U+FFFF? */ {
+                            return reinterpret_cast<char const *>(s);
+                        } else {
+                            s += 3;
+                        }
+                    } else if ((s[0] & 0xf8) == 0xf0) {
+                        /* 11110XXX 10XXxxxx 10xxxxxx 10xxxxxx */
+                        if ((s[1] & 0xc0) != 0x80 || (s[2] & 0xc0) != 0x80 ||
+                            (s[3] & 0xc0) != 0x80 ||
+                            (s[0] == 0xf0 && (s[1] & 0xf0) == 0x80) ||    /* overlong? */
+                            (s[0] == 0xf4 && s[1] > 0x8f) || s[0] > 0xf4) /* > U+10FFFF? */ {
+                            return reinterpret_cast<char const *>(s);
+                        } else {
+                            s += 4;
+                        }
+                    } else {
+                        return reinterpret_cast<char const *>(s);
+                    }
+                }
+                return nullptr;
+            }
+
             inline uint32_t next_utf8_codepoint(char const** begin, const char* end) {
                 const auto* it = reinterpret_cast<const uint8_t*>(*begin);
                 uint32_t cp = 0xffU & *it;

test/t/io/test_output_utils.cpp

@@ -231,3 +231,23 @@ TEST_CASE("incomplete Unicode codepoint") {
     }
 }
 
+TEST_CASE("utf-8 check for valid UTF-8") {
+    REQUIRE(osmium::io::detail::utf8_check("") == nullptr);
+    REQUIRE(osmium::io::detail::utf8_check("a") == nullptr);
+    REQUIRE(osmium::io::detail::utf8_check("abc") == nullptr);
+
+    const char* s = u8cast(u8"\n_\u01a2_\u30dc_\U0001d11e_\U0001f680");
+    REQUIRE(osmium::io::detail::utf8_check(s) == nullptr);
+}
+
+TEST_CASE("utf-8 check for illegal value") {
+    const char* s = "abc\xff";
+    REQUIRE(osmium::io::detail::utf8_check(s) != nullptr);
+}
+
+TEST_CASE("utf-8 check for incomplete Unicode codepoint") {
+    std::string s{u8cast(u8"\U0001f680")}; // rocket
+    s.resize(s.size() - 1);
+    REQUIRE(osmium::io::detail::utf8_check(s.c_str()) != nullptr);
+}
+