MCPcopy Create free account
hub / github.com/apache/arrow / utf8_length

Function utf8_length

cpp/src/gandiva/precompiled/string_ops.cc:176–195  ·  view source on GitHub ↗

Count the number of utf8 characters return 0 for invalid/incomplete input byte sequences

Source from the content-addressed store, hash-verified

174// Count the number of utf8 characters
175// return 0 for invalid/incomplete input byte sequences
176FORCE_INLINE
177gdv_int32 utf8_length(gdv_int64 context, const char* data, gdv_int32 data_len) {
178 int char_len = 0;
179 int count = 0;
180 for (int i = 0; i < data_len; i += char_len) {
181 char_len = utf8_char_length(data[i]);
182 if (char_len == 0 || i + char_len > data_len) { // invalid byte or incomplete glyph
183 set_error_for_invalid_utf(context, data[i]);
184 return 0;
185 }
186 for (int j = 1; j < char_len; ++j) {
187 if ((data[i + j] & 0xC0) != 0x80) { // bytes following head-byte of glyph
188 set_error_for_invalid_utf(context, data[i + j]);
189 return 0;
190 }
191 }
192 ++count;
193 }
194 return count;
195}
196
197// Count the number of utf8 characters, ignoring invalid char, considering size 1
198FORCE_INLINE

Callers 5

substr_utf8_int64_int64Function · 0.85
locate_utf8_utf8_int32Function · 0.85
left_utf8_int32Function · 0.85
right_utf8_int32Function · 0.85
TESTFunction · 0.85

Calls

no outgoing calls

Tested by 1

TESTFunction · 0.68