A UTF-8 lead byte announces how many continuation bytes (10xxxxxx) follow. If a multi-byte character gets split by a byte-offset-based line wrap, one of two things happens: a lead byte ends up with too few/no continuation bytes following it, or a continuation byte appears without a lead byte before it. Either way, this returns false.
| 13 | // bytes following it, or a continuation byte appears without a lead byte |
| 14 | // before it. Either way, this returns false. |
| 15 | bool is_valid_utf8(const std::string &s) { |
| 16 | size_t i = 0; |
| 17 | while (i < s.size()) { |
| 18 | unsigned char c = static_cast<unsigned char>(s[i]); |
| 19 | size_t len; |
| 20 | if ((c & 0x80) == 0x00) |
| 21 | len = 1; |
| 22 | else if ((c & 0xE0) == 0xC0) |
| 23 | len = 2; |
| 24 | else if ((c & 0xF0) == 0xE0) |
| 25 | len = 3; |
| 26 | else if ((c & 0xF8) == 0xF0) |
| 27 | len = 4; |
| 28 | else |
| 29 | return false; |
| 30 | if (i + len > s.size()) |
| 31 | return false; |
| 32 | for (size_t k = 1; k < len; ++k) { |
| 33 | if ((static_cast<unsigned char>(s[i + k]) & 0xC0) != 0x80) |
| 34 | return false; |
| 35 | } |
| 36 | i += len; |
| 37 | } |
| 38 | return true; |
| 39 | } |
| 40 | } // namespace |
| 41 | |
| 42 | // Regression test for https://github.com/p-ranav/tabulate/issues/127: forcing |