diff options
| -rw-r--r-- | src/bun.js/bindings/simdutf.cpp | 689 | ||||
| -rw-r--r-- | src/bun.js/bindings/simdutf.h | 511 |
2 files changed, 859 insertions, 341 deletions
diff --git a/src/bun.js/bindings/simdutf.cpp b/src/bun.js/bindings/simdutf.cpp index f9c0a649d..ea0d95f42 100644 --- a/src/bun.js/bindings/simdutf.cpp +++ b/src/bun.js/bindings/simdutf.cpp @@ -1,4 +1,4 @@ -/* auto-generated on 2023-01-02 15:43:33 -0500. Do not edit! */ +/* auto-generated on 2023-02-10 14:42:58 -0500. Do not edit! */ // dofile: invoked with prepath=/Users/dlemire/CVS/github/simdutf/src, filename=simdutf.cpp /* begin file src/simdutf.cpp */ #include "simdutf.h" @@ -509,7 +509,7 @@ simdutf_really_inline int16x8_t make_int16x8_t(int16_t x1, int16_t x2, int16_t simdutf_really_inline void store_ascii_as_utf16(char16_t * p) const { uint16x8_t first = vmovl_u8(vget_low_u8 (vreinterpretq_u8_s8(this->value))); uint16x8_t second = vmovl_high_u8(vreinterpretq_u8_s8(this->value)); - if (big_endian) { + if (!match_system(big_endian)) { #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO const uint8x16_t swap = make_uint8x16_t(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); #else @@ -4386,7 +4386,6 @@ private: const implementation *set_best() const noexcept; }; -const detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton; const std::initializer_list<const implementation *> available_implementation_pointers { #if SIMDUTF_IMPLEMENTATION_ICELAKE @@ -4662,187 +4661,322 @@ const implementation *detect_best_supported_implementation_on_first_use::set_bes SIMDUTF_POP_DISABLE_WARNINGS if (force_implementation_name) { - auto force_implementation = available_implementations[force_implementation_name]; + auto force_implementation = get_available_implementations()[force_implementation_name]; if (force_implementation) { - return active_implementation = force_implementation; + return get_active_implementation() = force_implementation; } else { // Note: abort() and stderr usage within the library is forbidden. - return active_implementation = &unsupported_singleton; + return get_active_implementation() = &unsupported_singleton; } } - return active_implementation = available_implementations.detect_best_supported(); + return get_active_implementation() = get_available_implementations().detect_best_supported(); } } // namespace internal -SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations{}; -SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> active_implementation{&internal::detect_best_supported_implementation_on_first_use_singleton}; + + +/** + * The list of available implementations compiled into simdutf. + */ +SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list& get_available_implementations() { + static const internal::available_implementation_list available_implementations{}; + return available_implementations; +} + +/** + * The active implementation. + */ +SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr<const implementation>& get_active_implementation() { + static const internal::detect_best_supported_implementation_on_first_use detect_best_supported_implementation_on_first_use_singleton; + static internal::atomic_ptr<const implementation> active_implementation{&detect_best_supported_implementation_on_first_use_singleton}; + return active_implementation; +} simdutf_warn_unused bool validate_utf8(const char *buf, size_t len) noexcept { - return active_implementation->validate_utf8(buf, len); + return get_active_implementation()->validate_utf8(buf, len); } simdutf_warn_unused result validate_utf8_with_errors(const char *buf, size_t len) noexcept { - return active_implementation->validate_utf8_with_errors(buf, len); + return get_active_implementation()->validate_utf8_with_errors(buf, len); } simdutf_warn_unused bool validate_ascii(const char *buf, size_t len) noexcept { - return active_implementation->validate_ascii(buf, len); + return get_active_implementation()->validate_ascii(buf, len); } simdutf_warn_unused result validate_ascii_with_errors(const char *buf, size_t len) noexcept { - return active_implementation->validate_ascii_with_errors(buf, len); + return get_active_implementation()->validate_ascii_with_errors(buf, len); +} +simdutf_warn_unused size_t convert_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_output) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return convert_utf8_to_utf16be(input, length, utf16_output); + #else + return convert_utf8_to_utf16le(input, length, utf16_output); + #endif } simdutf_warn_unused size_t convert_utf8_to_utf16le(const char * input, size_t length, char16_t* utf16_output) noexcept { - return active_implementation->convert_utf8_to_utf16le(input, length, utf16_output); + return get_active_implementation()->convert_utf8_to_utf16le(input, length, utf16_output); } simdutf_warn_unused size_t convert_utf8_to_utf16be(const char * input, size_t length, char16_t* utf16_output) noexcept { - return active_implementation->convert_utf8_to_utf16be(input, length, utf16_output); + return get_active_implementation()->convert_utf8_to_utf16be(input, length, utf16_output); +} +simdutf_warn_unused result convert_utf8_to_utf16_with_errors(const char * input, size_t length, char16_t* utf16_output) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return convert_utf8_to_utf16be_with_errors(input, length, utf16_output); + #else + return convert_utf8_to_utf16le_with_errors(input, length, utf16_output); + #endif } simdutf_warn_unused result convert_utf8_to_utf16le_with_errors(const char * input, size_t length, char16_t* utf16_output) noexcept { - return active_implementation->convert_utf8_to_utf16le_with_errors(input, length, utf16_output); + return get_active_implementation()->convert_utf8_to_utf16le_with_errors(input, length, utf16_output); } simdutf_warn_unused result convert_utf8_to_utf16be_with_errors(const char * input, size_t length, char16_t* utf16_output) noexcept { - return active_implementation->convert_utf8_to_utf16be_with_errors(input, length, utf16_output); + return get_active_implementation()->convert_utf8_to_utf16be_with_errors(input, length, utf16_output); } simdutf_warn_unused size_t convert_utf8_to_utf32(const char * input, size_t length, char32_t* utf32_output) noexcept { - return active_implementation->convert_utf8_to_utf32(input, length, utf32_output); + return get_active_implementation()->convert_utf8_to_utf32(input, length, utf32_output); } simdutf_warn_unused result convert_utf8_to_utf32_with_errors(const char * input, size_t length, char32_t* utf32_output) noexcept { - return active_implementation->convert_utf8_to_utf32_with_errors(input, length, utf32_output); + return get_active_implementation()->convert_utf8_to_utf32_with_errors(input, length, utf32_output); +} +simdutf_warn_unused bool validate_utf16(const char16_t * buf, size_t len) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return validate_utf16be(buf, len); + #else + return validate_utf16le(buf, len); + #endif } simdutf_warn_unused bool validate_utf16le(const char16_t * buf, size_t len) noexcept { - return active_implementation->validate_utf16le(buf, len); + return get_active_implementation()->validate_utf16le(buf, len); } simdutf_warn_unused bool validate_utf16be(const char16_t * buf, size_t len) noexcept { - return active_implementation->validate_utf16be(buf, len); + return get_active_implementation()->validate_utf16be(buf, len); +} +simdutf_warn_unused result validate_utf16_with_errors(const char16_t * buf, size_t len) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return validate_utf16be_with_errors(buf, len); + #else + return validate_utf16le_with_errors(buf, len); + #endif } simdutf_warn_unused result validate_utf16le_with_errors(const char16_t * buf, size_t len) noexcept { - return active_implementation->validate_utf16le_with_errors(buf, len); + return get_active_implementation()->validate_utf16le_with_errors(buf, len); } simdutf_warn_unused result validate_utf16be_with_errors(const char16_t * buf, size_t len) noexcept { - return active_implementation->validate_utf16be_with_errors(buf, len); + return get_active_implementation()->validate_utf16be_with_errors(buf, len); } simdutf_warn_unused bool validate_utf32(const char32_t * buf, size_t len) noexcept { - return active_implementation->validate_utf32(buf, len); + return get_active_implementation()->validate_utf32(buf, len); } simdutf_warn_unused result validate_utf32_with_errors(const char32_t * buf, size_t len) noexcept { - return active_implementation->validate_utf32_with_errors(buf, len); + return get_active_implementation()->validate_utf32_with_errors(buf, len); +} +simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_buffer) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return convert_valid_utf8_to_utf16be(input, length, utf16_buffer); + #else + return convert_valid_utf8_to_utf16le(input, length, utf16_buffer); + #endif } simdutf_warn_unused size_t convert_valid_utf8_to_utf16le(const char * input, size_t length, char16_t* utf16_buffer) noexcept { - return active_implementation->convert_valid_utf8_to_utf16le(input, length, utf16_buffer); + return get_active_implementation()->convert_valid_utf8_to_utf16le(input, length, utf16_buffer); } simdutf_warn_unused size_t convert_valid_utf8_to_utf16be(const char * input, size_t length, char16_t* utf16_buffer) noexcept { - return active_implementation->convert_valid_utf8_to_utf16be(input, length, utf16_buffer); + return get_active_implementation()->convert_valid_utf8_to_utf16be(input, length, utf16_buffer); } simdutf_warn_unused size_t convert_valid_utf8_to_utf32(const char * input, size_t length, char32_t* utf32_buffer) noexcept { - return active_implementation->convert_valid_utf8_to_utf32(input, length, utf32_buffer); + return get_active_implementation()->convert_valid_utf8_to_utf32(input, length, utf32_buffer); +} +simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return convert_utf16be_to_utf8(buf, len, utf8_buffer); + #else + return convert_utf16le_to_utf8(buf, len, utf8_buffer); + #endif } simdutf_warn_unused size_t convert_utf16le_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) noexcept { - return active_implementation->convert_utf16le_to_utf8(buf, len, utf8_buffer); + return get_active_implementation()->convert_utf16le_to_utf8(buf, len, utf8_buffer); } simdutf_warn_unused size_t convert_utf16be_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) noexcept { - return active_implementation->convert_utf16be_to_utf8(buf, len, utf8_buffer); + return get_active_implementation()->convert_utf16be_to_utf8(buf, len, utf8_buffer); +} +simdutf_warn_unused result convert_utf16_to_utf8_with_errors(const char16_t * buf, size_t len, char* utf8_buffer) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return convert_utf16be_to_utf8_with_errors(buf, len, utf8_buffer); + #else + return convert_utf16le_to_utf8_with_errors(buf, len, utf8_buffer); + #endif } simdutf_warn_unused result convert_utf16le_to_utf8_with_errors(const char16_t * buf, size_t len, char* utf8_buffer) noexcept { - return active_implementation->convert_utf16le_to_utf8_with_errors(buf, len, utf8_buffer); + return get_active_implementation()->convert_utf16le_to_utf8_with_errors(buf, len, utf8_buffer); } simdutf_warn_unused result convert_utf16be_to_utf8_with_errors(const char16_t * buf, size_t len, char* utf8_buffer) noexcept { - return active_implementation->convert_utf16be_to_utf8_with_errors(buf, len, utf8_buffer); + return get_active_implementation()->convert_utf16be_to_utf8_with_errors(buf, len, utf8_buffer); +} +simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) noexcept { + #if BIG_ENDIAN + return convert_valid_utf16be_to_utf8(buf, len, utf8_buffer); + #else + return convert_valid_utf16le_to_utf8(buf, len, utf8_buffer); + #endif } simdutf_warn_unused size_t convert_valid_utf16le_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) noexcept { - return active_implementation->convert_valid_utf16le_to_utf8(buf, len, utf8_buffer); + return get_active_implementation()->convert_valid_utf16le_to_utf8(buf, len, utf8_buffer); } simdutf_warn_unused size_t convert_valid_utf16be_to_utf8(const char16_t * buf, size_t len, char* utf8_buffer) noexcept { - return active_implementation->convert_valid_utf16be_to_utf8(buf, len, utf8_buffer); + return get_active_implementation()->convert_valid_utf16be_to_utf8(buf, len, utf8_buffer); } simdutf_warn_unused size_t convert_utf32_to_utf8(const char32_t * buf, size_t len, char* utf8_buffer) noexcept { - return active_implementation->convert_utf32_to_utf8(buf, len, utf8_buffer); + return get_active_implementation()->convert_utf32_to_utf8(buf, len, utf8_buffer); } simdutf_warn_unused result convert_utf32_to_utf8_with_errors(const char32_t * buf, size_t len, char* utf8_buffer) noexcept { - return active_implementation->convert_utf32_to_utf8_with_errors(buf, len, utf8_buffer); + return get_active_implementation()->convert_utf32_to_utf8_with_errors(buf, len, utf8_buffer); } simdutf_warn_unused size_t convert_valid_utf32_to_utf8(const char32_t * buf, size_t len, char* utf8_buffer) noexcept { - return active_implementation->convert_valid_utf32_to_utf8(buf, len, utf8_buffer); + return get_active_implementation()->convert_valid_utf32_to_utf8(buf, len, utf8_buffer); +} +simdutf_warn_unused size_t convert_utf32_to_utf16(const char32_t * buf, size_t len, char16_t* utf16_buffer) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return convert_utf32_to_utf16be(buf, len, utf16_buffer); + #else + return convert_utf32_to_utf16le(buf, len, utf16_buffer); + #endif } simdutf_warn_unused size_t convert_utf32_to_utf16le(const char32_t * buf, size_t len, char16_t* utf16_buffer) noexcept { - return active_implementation->convert_utf32_to_utf16le(buf, len, utf16_buffer); + return get_active_implementation()->convert_utf32_to_utf16le(buf, len, utf16_buffer); } simdutf_warn_unused size_t convert_utf32_to_utf16be(const char32_t * buf, size_t len, char16_t* utf16_buffer) noexcept { - return active_implementation->convert_utf32_to_utf16be(buf, len, utf16_buffer); + return get_active_implementation()->convert_utf32_to_utf16be(buf, len, utf16_buffer); +} +simdutf_warn_unused result convert_utf32_to_utf16_with_errors(const char32_t * buf, size_t len, char16_t* utf16_buffer) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return convert_utf32_to_utf16be_with_errors(buf, len, utf16_buffer); + #else + return convert_utf32_to_utf16le_with_errors(buf, len, utf16_buffer); + #endif } simdutf_warn_unused result convert_utf32_to_utf16le_with_errors(const char32_t * buf, size_t len, char16_t* utf16_buffer) noexcept { - return active_implementation->convert_utf32_to_utf16le_with_errors(buf, len, utf16_buffer); + return get_active_implementation()->convert_utf32_to_utf16le_with_errors(buf, len, utf16_buffer); } simdutf_warn_unused result convert_utf32_to_utf16be_with_errors(const char32_t * buf, size_t len, char16_t* utf16_buffer) noexcept { - return active_implementation->convert_utf32_to_utf16be_with_errors(buf, len, utf16_buffer); + return get_active_implementation()->convert_utf32_to_utf16be_with_errors(buf, len, utf16_buffer); +} +simdutf_warn_unused size_t convert_valid_utf32_to_utf16(const char32_t * buf, size_t len, char16_t* utf16_buffer) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return convert_valid_utf32_to_utf16be(buf, len, utf16_buffer); + #else + return convert_valid_utf32_to_utf16le(buf, len, utf16_buffer); + #endif } simdutf_warn_unused size_t convert_valid_utf32_to_utf16le(const char32_t * buf, size_t len, char16_t* utf16_buffer) noexcept { - return active_implementation->convert_valid_utf32_to_utf16le(buf, len, utf16_buffer); + return get_active_implementation()->convert_valid_utf32_to_utf16le(buf, len, utf16_buffer); } simdutf_warn_unused size_t convert_valid_utf32_to_utf16be(const char32_t * buf, size_t len, char16_t* utf16_buffer) noexcept { - return active_implementation->convert_valid_utf32_to_utf16be(buf, len, utf16_buffer); + return get_active_implementation()->convert_valid_utf32_to_utf16be(buf, len, utf16_buffer); +} +simdutf_warn_unused size_t convert_utf16_to_utf32(const char16_t * buf, size_t len, char32_t* utf32_buffer) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return convert_utf16be_to_utf32(buf, len, utf32_buffer); + #else + return convert_utf16le_to_utf32(buf, len, utf32_buffer); + #endif } simdutf_warn_unused size_t convert_utf16le_to_utf32(const char16_t * buf, size_t len, char32_t* utf32_buffer) noexcept { - return active_implementation->convert_utf16le_to_utf32(buf, len, utf32_buffer); + return get_active_implementation()->convert_utf16le_to_utf32(buf, len, utf32_buffer); } simdutf_warn_unused size_t convert_utf16be_to_utf32(const char16_t * buf, size_t len, char32_t* utf32_buffer) noexcept { - return active_implementation->convert_utf16be_to_utf32(buf, len, utf32_buffer); + return get_active_implementation()->convert_utf16be_to_utf32(buf, len, utf32_buffer); +} +simdutf_warn_unused result convert_utf16_to_utf32_with_errors(const char16_t * buf, size_t len, char32_t* utf32_buffer) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return convert_utf16be_to_utf32_with_errors(buf, len, utf32_buffer); + #else + return convert_utf16le_to_utf32_with_errors(buf, len, utf32_buffer); + #endif } simdutf_warn_unused result convert_utf16le_to_utf32_with_errors(const char16_t * buf, size_t len, char32_t* utf32_buffer) noexcept { - return active_implementation->convert_utf16le_to_utf32_with_errors(buf, len, utf32_buffer); + return get_active_implementation()->convert_utf16le_to_utf32_with_errors(buf, len, utf32_buffer); } simdutf_warn_unused result convert_utf16be_to_utf32_with_errors(const char16_t * buf, size_t len, char32_t* utf32_buffer) noexcept { - return active_implementation->convert_utf16be_to_utf32_with_errors(buf, len, utf32_buffer); + return get_active_implementation()->convert_utf16be_to_utf32_with_errors(buf, len, utf32_buffer); +} +simdutf_warn_unused size_t convert_valid_utf16_to_utf32(const char16_t * buf, size_t len, char32_t* utf32_buffer) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return convert_valid_utf16be_to_utf32(buf, len, utf32_buffer); + #else + return convert_valid_utf16le_to_utf32(buf, len, utf32_buffer); + #endif } simdutf_warn_unused size_t convert_valid_utf16le_to_utf32(const char16_t * buf, size_t len, char32_t* utf32_buffer) noexcept { - return active_implementation->convert_valid_utf16le_to_utf32(buf, len, utf32_buffer); + return get_active_implementation()->convert_valid_utf16le_to_utf32(buf, len, utf32_buffer); } simdutf_warn_unused size_t convert_valid_utf16be_to_utf32(const char16_t * buf, size_t len, char32_t* utf32_buffer) noexcept { - return active_implementation->convert_valid_utf16be_to_utf32(buf, len, utf32_buffer); + return get_active_implementation()->convert_valid_utf16be_to_utf32(buf, len, utf32_buffer); } void change_endianness_utf16(const char16_t * input, size_t length, char16_t * output) noexcept { - active_implementation->change_endianness_utf16(input, length, output); + get_active_implementation()->change_endianness_utf16(input, length, output); +} +simdutf_warn_unused size_t count_utf16(const char16_t * input, size_t length) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return count_utf16be(input, length); + #else + return count_utf16le(input, length); + #endif } simdutf_warn_unused size_t count_utf16le(const char16_t * input, size_t length) noexcept { - return active_implementation->count_utf16le(input, length); + return get_active_implementation()->count_utf16le(input, length); } simdutf_warn_unused size_t count_utf16be(const char16_t * input, size_t length) noexcept { - return active_implementation->count_utf16be(input, length); + return get_active_implementation()->count_utf16be(input, length); } simdutf_warn_unused size_t count_utf8(const char * input, size_t length) noexcept { - return active_implementation->count_utf8(input, length); + return get_active_implementation()->count_utf8(input, length); +} +simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t * input, size_t length) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return utf8_length_from_utf16be(input, length); + #else + return utf8_length_from_utf16le(input, length); + #endif } simdutf_warn_unused size_t utf8_length_from_utf16le(const char16_t * input, size_t length) noexcept { - return active_implementation->utf8_length_from_utf16le(input, length); + return get_active_implementation()->utf8_length_from_utf16le(input, length); } simdutf_warn_unused size_t utf8_length_from_utf16be(const char16_t * input, size_t length) noexcept { - return active_implementation->utf8_length_from_utf16be(input, length); + return get_active_implementation()->utf8_length_from_utf16be(input, length); +} +simdutf_warn_unused size_t utf32_length_from_utf16(const char16_t * input, size_t length) noexcept { + #if SIMDUTF_IS_BIG_ENDIAN + return utf32_length_from_utf16be(input, length); + #else + return utf32_length_from_utf16le(input, length); + #endif } simdutf_warn_unused size_t utf32_length_from_utf16le(const char16_t * input, size_t length) noexcept { - return active_implementation->utf32_length_from_utf16le(input, length); + return get_active_implementation()->utf32_length_from_utf16le(input, length); } simdutf_warn_unused size_t utf32_length_from_utf16be(const char16_t * input, size_t length) noexcept { - return active_implementation->utf32_length_from_utf16be(input, length); + return get_active_implementation()->utf32_length_from_utf16be(input, length); } simdutf_warn_unused size_t utf16_length_from_utf8(const char * input, size_t length) noexcept { - return active_implementation->utf16_length_from_utf8(input, length); + return get_active_implementation()->utf16_length_from_utf8(input, length); } simdutf_warn_unused size_t utf8_length_from_utf32(const char32_t * input, size_t length) noexcept { - return active_implementation->utf8_length_from_utf32(input, length); + return get_active_implementation()->utf8_length_from_utf32(input, length); } simdutf_warn_unused size_t utf16_length_from_utf32(const char32_t * input, size_t length) noexcept { - return active_implementation->utf16_length_from_utf32(input, length); + return get_active_implementation()->utf16_length_from_utf32(input, length); } simdutf_warn_unused size_t utf32_length_from_utf8(const char * input, size_t length) noexcept { - return active_implementation->utf32_length_from_utf8(input, length); + return get_active_implementation()->utf32_length_from_utf8(input, length); } simdutf_warn_unused simdutf::encoding_type autodetect_encoding(const char * buf, size_t length) noexcept { - return active_implementation->autodetect_encoding(buf, length); + return get_active_implementation()->autodetect_encoding(buf, length); } simdutf_warn_unused int detect_encodings(const char * buf, size_t length) noexcept { - return active_implementation->detect_encodings(buf, length); + return get_active_implementation()->detect_encodings(buf, length); } const implementation * builtin_implementation() { - static const implementation * builtin_impl = available_implementations[SIMDUTF_STRINGIFY(SIMDUTF_BUILTIN_IMPLEMENTATION)]; + static const implementation * builtin_impl = get_available_implementations()[SIMDUTF_STRINGIFY(SIMDUTF_BUILTIN_IMPLEMENTATION)]; return builtin_impl; } @@ -4854,6 +4988,14 @@ const implementation * builtin_implementation() { /* begin file src/encoding_types.cpp */ namespace simdutf { +bool match_system(endianness e) { +#if SIMDUTF_IS_BIG_ENDIAN + return e == endianness::BIG; +#else + return e == endianness::LITTLE; +#endif +} + std::string to_string(encoding_type bom) { switch (bom) { case UTF16_LE: return "UTF16 little-endian"; @@ -9805,7 +9947,8 @@ namespace simdutf { namespace scalar { namespace { namespace ascii { - +#if SIMDUTF_IMPLEMENTATION_FALLBACK +// Only used by the fallback kernel. inline simdutf_warn_unused bool validate(const char *buf, size_t len) noexcept { const uint8_t *data = reinterpret_cast<const uint8_t *>(buf); uint64_t pos = 0; @@ -9824,6 +9967,7 @@ inline simdutf_warn_unused bool validate(const char *buf, size_t len) noexcept { } return true; } +#endif inline simdutf_warn_unused result validate_with_errors(const char *buf, size_t len) noexcept { const uint8_t *data = reinterpret_cast<const uint8_t *>(buf); @@ -9864,6 +10008,8 @@ namespace simdutf { namespace scalar { namespace { namespace utf8 { +#if SIMDUTF_IMPLEMENTATION_FALLBACK +// only used by the fallback kernel. // credit: based on code from Google Fuchsia (Apache Licensed) inline simdutf_warn_unused bool validate(const char *buf, size_t len) noexcept { const uint8_t *data = reinterpret_cast<const uint8_t *>(buf); @@ -9929,6 +10075,7 @@ inline simdutf_warn_unused bool validate(const char *buf, size_t len) noexcept { } return true; } +#endif inline simdutf_warn_unused result validate_with_errors(const char *buf, size_t len) noexcept { const uint8_t *data = reinterpret_cast<const uint8_t *>(buf); @@ -10035,16 +10182,6 @@ inline size_t utf16_length_from_utf8(const char* buf, size_t len) { return counter; } -inline size_t utf32_length_from_utf8(const char* buf, size_t len) { - const int8_t * p = reinterpret_cast<const int8_t *>(buf); - size_t counter{0}; - for(size_t i = 0; i < len; i++) { - // -65 is 0b10111111, anything larger in two-complement's should start a new code point. - if(p[i] > -65) { counter++; } - } - return counter; -} - } // utf8 namespace } // unnamed namespace } // namespace scalar @@ -10071,12 +10208,12 @@ inline simdutf_warn_unused bool validate(const char16_t *buf, size_t len) noexce const uint16_t *data = reinterpret_cast<const uint16_t *>(buf); uint64_t pos = 0; while (pos < len) { - uint16_t word = big_endian ? swap_bytes(data[pos]) : data[pos]; + uint16_t word = !match_system(big_endian) ? swap_bytes(data[pos]) : data[pos]; if((word &0xF800) == 0xD800) { if(pos + 1 >= len) { return false; } uint16_t diff = uint16_t(word - 0xD800); if(diff > 0x3FF) { return false; } - uint16_t next_word = big_endian ? uint16_t((data[pos + 1] >> 8) | (data[pos + 1] << 8)) : data[pos + 1]; + uint16_t next_word = !match_system(big_endian) ? swap_bytes(data[pos + 1]) : data[pos + 1]; uint16_t diff2 = uint16_t(next_word - 0xDC00); if(diff2 > 0x3FF) { return false; } pos += 2; @@ -10092,12 +10229,12 @@ inline simdutf_warn_unused result validate_with_errors(const char16_t *buf, size const uint16_t *data = reinterpret_cast<const uint16_t *>(buf); size_t pos = 0; while (pos < len) { - uint16_t word = big_endian ? swap_bytes(data[pos]) : data[pos]; + uint16_t word = !match_system(big_endian) ? swap_bytes(data[pos]) : data[pos]; if((word & 0xF800) == 0xD800) { if(pos + 1 >= len) { return result(error_code::SURROGATE, pos); } uint16_t diff = uint16_t(word - 0xD800); if(diff > 0x3FF) { return result(error_code::SURROGATE, pos); } - uint16_t next_word = big_endian ? uint16_t((data[pos + 1] >> 8) | (data[pos + 1] << 8)) : data[pos + 1]; + uint16_t next_word = !match_system(big_endian) ? swap_bytes(data[pos + 1]) : data[pos + 1]; uint16_t diff2 = uint16_t(next_word - 0xDC00); if(diff2 > 0x3FF) { return result(error_code::SURROGATE, pos); } pos += 2; @@ -10114,7 +10251,7 @@ inline size_t count_code_points(const char16_t* buf, size_t len) { const uint16_t * p = reinterpret_cast<const uint16_t *>(buf); size_t counter{0}; for(size_t i = 0; i < len; i++) { - uint16_t word = big_endian ? swap_bytes(p[i]) : p[i]; + uint16_t word = !match_system(big_endian) ? swap_bytes(p[i]) : p[i]; counter += ((word & 0xFC00) != 0xDC00); } return counter; @@ -10126,7 +10263,7 @@ inline size_t utf8_length_from_utf16(const char16_t* buf, size_t len) { const uint16_t * p = reinterpret_cast<const uint16_t *>(buf); size_t counter{0}; for(size_t i = 0; i < len; i++) { - uint16_t word = big_endian ? swap_bytes(p[i]) : p[i]; + uint16_t word = !match_system(big_endian) ? swap_bytes(p[i]) : p[i]; /** ASCII **/ if(word <= 0x7F) { counter++; } /** two-byte **/ @@ -10145,7 +10282,7 @@ inline size_t utf32_length_from_utf16(const char16_t* buf, size_t len) { const uint16_t * p = reinterpret_cast<const uint16_t *>(buf); size_t counter{0}; for(size_t i = 0; i < len; i++) { - uint16_t word = big_endian ? swap_bytes(p[i]) : p[i]; + uint16_t word = !match_system(big_endian) ? swap_bytes(p[i]) : p[i]; counter += ((word & 0xFC00) != 0xDC00); } return counter; @@ -10251,6 +10388,8 @@ namespace scalar { namespace { namespace utf32_to_utf8 { +#if SIMDUTF_IMPLEMENTATION_FALLBACK || SIMDUTF_IMPLEMENTATION_PPC64 +// only used by the fallback and POWER kernel inline size_t convert_valid(const char32_t* buf, size_t len, char* utf8_output) { const uint32_t *data = reinterpret_cast<const uint32_t *>(buf); size_t pos = 0; @@ -10297,6 +10436,7 @@ inline size_t convert_valid(const char32_t* buf, size_t len, char* utf8_output) } return utf8_output - start; } +#endif // SIMDUTF_IMPLEMENTATION_FALLBACK || SIMDUTF_IMPLEMENTATION_PPC64 } // utf32_to_utf8 namespace } // unnamed namespace @@ -10440,14 +10580,14 @@ inline size_t convert_valid(const char32_t* buf, size_t len, char16_t* utf16_out uint32_t word = data[pos]; if((word & 0xFFFF0000)==0) { // will not generate a surrogate pair - *utf16_output++ = big_endian ? char16_t(utf16::swap_bytes(uint16_t(word))) : char16_t(word); + *utf16_output++ = !match_system(big_endian) ? char16_t(utf16::swap_bytes(uint16_t(word))) : char16_t(word); pos++; } else { // will generate a surrogate pair word -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (big_endian) { + if (!match_system(big_endian)) { high_surrogate = utf16::swap_bytes(high_surrogate); low_surrogate = utf16::swap_bytes(low_surrogate); } @@ -10486,14 +10626,14 @@ inline size_t convert(const char32_t* buf, size_t len, char16_t* utf16_output) { if((word & 0xFFFF0000)==0) { if (word >= 0xD800 && word <= 0xDFFF) { return 0; } // will not generate a surrogate pair - *utf16_output++ = big_endian ? char16_t(utf16::swap_bytes(uint16_t(word))) : char16_t(word); + *utf16_output++ = !match_system(big_endian) ? char16_t(utf16::swap_bytes(uint16_t(word))) : char16_t(word); } else { // will generate a surrogate pair if (word > 0x10FFFF) { return 0; } word -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (big_endian) { + if (!match_system(big_endian)) { high_surrogate = utf16::swap_bytes(high_surrogate); low_surrogate = utf16::swap_bytes(low_surrogate); } @@ -10515,14 +10655,14 @@ inline result convert_with_errors(const char32_t* buf, size_t len, char16_t* utf if((word & 0xFFFF0000)==0) { if (word >= 0xD800 && word <= 0xDFFF) { return result(error_code::SURROGATE, pos); } // will not generate a surrogate pair - *utf16_output++ = big_endian ? char16_t(utf16::swap_bytes(uint16_t(word))) : char16_t(word); + *utf16_output++ = !match_system(big_endian) ? char16_t(utf16::swap_bytes(uint16_t(word))) : char16_t(word); } else { // will generate a surrogate pair if (word > 0x10FFFF) { return result(error_code::TOO_LARGE, pos); } word -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (big_endian) { + if (!match_system(big_endian)) { high_surrogate = utf16::swap_bytes(high_surrogate); low_surrogate = utf16::swap_bytes(low_surrogate); } @@ -10562,17 +10702,18 @@ inline size_t convert_valid(const char16_t* buf, size_t len, char* utf8_output) if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that they are ascii uint64_t v; ::memcpy(&v, data + pos, sizeof(uint64_t)); - if (big_endian) v = (v >> 8) | (v << (64 - 8)); + if (!match_system(big_endian)) v = (v >> 8) | (v << (64 - 8)); if ((v & 0xFF80FF80FF80FF80) == 0) { size_t final_pos = pos + 4; while(pos < final_pos) { - *utf8_output++ = big_endian ? char(utf16::swap_bytes(buf[pos])) : char(buf[pos]); + *utf8_output++ = !match_system(big_endian) ? char(utf16::swap_bytes(buf[pos])) : char(buf[pos]); pos++; } continue; } } - uint16_t word = big_endian ? utf16::swap_bytes(data[pos]) : data[pos]; + + uint16_t word = !match_system(big_endian) ? utf16::swap_bytes(data[pos]) : data[pos]; if((word & 0xFF80)==0) { // will generate one UTF-8 bytes *utf8_output++ = char(word); @@ -10594,7 +10735,7 @@ inline size_t convert_valid(const char16_t* buf, size_t len, char* utf8_output) // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); if(pos + 1 >= len) { return 0; } // minimal bound checking - uint16_t next_word = big_endian ? utf16::swap_bytes(data[pos + 1]) : data[pos + 1]; + uint16_t next_word = !match_system(big_endian) ? utf16::swap_bytes(data[pos + 1]) : data[pos + 1]; uint16_t diff2 = uint16_t(next_word - 0xDC00); uint32_t value = (diff << 10) + diff2 + 0x10000; // will generate four UTF-8 bytes @@ -10636,17 +10777,17 @@ inline size_t convert(const char16_t* buf, size_t len, char* utf8_output) { if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that they are ascii uint64_t v; ::memcpy(&v, data + pos, sizeof(uint64_t)); - if (big_endian) v = (v >> 8) | (v << (64 - 8)); + if (!match_system(big_endian)) v = (v >> 8) | (v << (64 - 8)); if ((v & 0xFF80FF80FF80FF80) == 0) { size_t final_pos = pos + 4; while(pos < final_pos) { - *utf8_output++ = big_endian ? char(utf16::swap_bytes(buf[pos])) : char(buf[pos]); + *utf8_output++ = !match_system(big_endian) ? char(utf16::swap_bytes(buf[pos])) : char(buf[pos]); pos++; } continue; } } - uint16_t word = big_endian ? utf16::swap_bytes(data[pos]) : data[pos]; + uint16_t word = !match_system(big_endian) ? utf16::swap_bytes(data[pos]) : data[pos]; if((word & 0xFF80)==0) { // will generate one UTF-8 bytes *utf8_output++ = char(word); @@ -10669,7 +10810,7 @@ inline size_t convert(const char16_t* buf, size_t len, char* utf8_output) { if(pos + 1 >= len) { return 0; } uint16_t diff = uint16_t(word - 0xD800); if(diff > 0x3FF) { return 0; } - uint16_t next_word = big_endian ? utf16::swap_bytes(data[pos + 1]) : data[pos + 1]; + uint16_t next_word = !match_system(big_endian) ? utf16::swap_bytes(data[pos + 1]) : data[pos + 1]; uint16_t diff2 = uint16_t(next_word - 0xDC00); if(diff2 > 0x3FF) { return 0; } uint32_t value = (diff << 10) + diff2 + 0x10000; @@ -10695,17 +10836,17 @@ inline result convert_with_errors(const char16_t* buf, size_t len, char* utf8_ou if (pos + 4 <= len) { // if it is safe to read 8 more bytes, check that they are ascii uint64_t v; ::memcpy(&v, data + pos, sizeof(uint64_t)); - if (big_endian) v = (v >> 8) | (v << (64 - 8)); + if (!match_system(big_endian)) v = (v >> 8) | (v << (64 - 8)); if ((v & 0xFF80FF80FF80FF80) == 0) { size_t final_pos = pos + 4; while(pos < final_pos) { - *utf8_output++ = big_endian ? char(utf16::swap_bytes(buf[pos])) : char(buf[pos]); + *utf8_output++ = !match_system(big_endian) ? char(utf16::swap_bytes(buf[pos])) : char(buf[pos]); pos++; } continue; } } - uint16_t word = big_endian ? utf16::swap_bytes(data[pos]) : data[pos]; + uint16_t word = !match_system(big_endian) ? utf16::swap_bytes(data[pos]) : data[pos]; if((word & 0xFF80)==0) { // will generate one UTF-8 bytes *utf8_output++ = char(word); @@ -10728,7 +10869,7 @@ inline result convert_with_errors(const char16_t* buf, size_t len, char* utf8_ou if(pos + 1 >= len) { return result(error_code::SURROGATE, pos); } uint16_t diff = uint16_t(word - 0xD800); if(diff > 0x3FF) { return result(error_code::SURROGATE, pos); } - uint16_t next_word = big_endian ? utf16::swap_bytes(data[pos + 1]) : data[pos + 1]; + uint16_t next_word = !match_system(big_endian) ? utf16::swap_bytes(data[pos + 1]) : data[pos + 1]; uint16_t diff2 = uint16_t(next_word - 0xDC00); if(diff2 > 0x3FF) { return result(error_code::SURROGATE, pos); } uint32_t value = (diff << 10) + diff2 + 0x10000; @@ -10768,7 +10909,7 @@ inline size_t convert_valid(const char16_t* buf, size_t len, char32_t* utf32_out size_t pos = 0; char32_t* start{utf32_output}; while (pos < len) { - uint16_t word = big_endian ? utf16::swap_bytes(data[pos]) : data[pos]; + uint16_t word = !match_system(big_endian) ? utf16::swap_bytes(data[pos]) : data[pos]; if((word &0xF800 ) != 0xD800) { // No surrogate pair, extend 16-bit word to 32-bit word *utf32_output++ = char32_t(word); @@ -10777,7 +10918,7 @@ inline size_t convert_valid(const char16_t* buf, size_t len, char32_t* utf32_out // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); if(pos + 1 >= len) { return 0; } // minimal bound checking - uint16_t next_word = big_endian ? utf16::swap_bytes(data[pos + 1]) : data[pos + 1]; + uint16_t next_word = !match_system(big_endian) ? utf16::swap_bytes(data[pos + 1]) : data[pos + 1]; uint16_t diff2 = uint16_t(next_word - 0xDC00); uint32_t value = (diff << 10) + diff2 + 0x10000; *utf32_output++ = char32_t(value); @@ -10810,7 +10951,7 @@ inline size_t convert(const char16_t* buf, size_t len, char32_t* utf32_output) { size_t pos = 0; char32_t* start{utf32_output}; while (pos < len) { - uint16_t word = big_endian ? utf16::swap_bytes(data[pos]) : data[pos]; + uint16_t word = !match_system(big_endian) ? utf16::swap_bytes(data[pos]) : data[pos]; if((word &0xF800 ) != 0xD800) { // No surrogate pair, extend 16-bit word to 32-bit word *utf32_output++ = char32_t(word); @@ -10820,7 +10961,7 @@ inline size_t convert(const char16_t* buf, size_t len, char32_t* utf32_output) { uint16_t diff = uint16_t(word - 0xD800); if(diff > 0x3FF) { return 0; } if(pos + 1 >= len) { return 0; } // minimal bound checking - uint16_t next_word = big_endian ? utf16::swap_bytes(data[pos + 1]) : data[pos + 1]; + uint16_t next_word = !match_system(big_endian) ? utf16::swap_bytes(data[pos + 1]) : data[pos + 1]; uint16_t diff2 = uint16_t(next_word - 0xDC00); if(diff2 > 0x3FF) { return 0; } uint32_t value = (diff << 10) + diff2 + 0x10000; @@ -10837,7 +10978,7 @@ inline result convert_with_errors(const char16_t* buf, size_t len, char32_t* utf size_t pos = 0; char32_t* start{utf32_output}; while (pos < len) { - uint16_t word = big_endian ? utf16::swap_bytes(data[pos]) : data[pos]; + uint16_t word = !match_system(big_endian) ? utf16::swap_bytes(data[pos]) : data[pos]; if((word &0xF800 ) != 0xD800) { // No surrogate pair, extend 16-bit word to 32-bit word *utf32_output++ = char32_t(word); @@ -10847,7 +10988,7 @@ inline result convert_with_errors(const char16_t* buf, size_t len, char32_t* utf uint16_t diff = uint16_t(word - 0xD800); if(diff > 0x3FF) { return result(error_code::SURROGATE, pos); } if(pos + 1 >= len) { return result(error_code::SURROGATE, pos); } // minimal bound checking - uint16_t next_word = big_endian ? utf16::swap_bytes(data[pos + 1]) : data[pos + 1]; + uint16_t next_word = !match_system(big_endian) ? utf16::swap_bytes(data[pos + 1]) : data[pos + 1]; uint16_t diff2 = uint16_t(next_word - 0xDC00); if(diff2 > 0x3FF) { return result(error_code::SURROGATE, pos); } uint32_t value = (diff << 10) + diff2 + 0x10000; @@ -10889,7 +11030,7 @@ inline size_t convert_valid(const char* buf, size_t len, char16_t* utf16_output) if ((v & 0x8080808080808080) == 0) { size_t final_pos = pos + 8; while(pos < final_pos) { - *utf16_output++ = big_endian ? char16_t(utf16::swap_bytes(buf[pos])) : char16_t(buf[pos]); + *utf16_output++ = !match_system(big_endian) ? char16_t(utf16::swap_bytes(buf[pos])) : char16_t(buf[pos]); pos++; } continue; @@ -10898,14 +11039,14 @@ inline size_t convert_valid(const char* buf, size_t len, char16_t* utf16_output) uint8_t leading_byte = data[pos]; // leading byte if (leading_byte < 0b10000000) { // converting one ASCII byte !!! - *utf16_output++ = big_endian ? char16_t(utf16::swap_bytes(leading_byte)) : char16_t(leading_byte); + *utf16_output++ = !match_system(big_endian) ? char16_t(utf16::swap_bytes(leading_byte)) : char16_t(leading_byte); pos++; } else if ((leading_byte & 0b11100000) == 0b11000000) { // We have a two-byte UTF-8, it should become // a single UTF-16 word. if(pos + 1 >= len) { break; } // minimal bound checking uint16_t code_point = uint16_t(((leading_byte &0b00011111) << 6) | (data[pos + 1] &0b00111111)); - if (big_endian) { + if (!match_system(big_endian)) { code_point = utf16::swap_bytes(uint16_t(code_point)); } *utf16_output++ = char16_t(code_point); @@ -10915,7 +11056,7 @@ inline size_t convert_valid(const char* buf, size_t len, char16_t* utf16_output) // a single UTF-16 word. if(pos + 2 >= len) { break; } // minimal bound checking uint16_t code_point = uint16_t(((leading_byte &0b00001111) << 12) | ((data[pos + 1] &0b00111111) << 6) | (data[pos + 2] &0b00111111)); - if (big_endian) { + if (!match_system(big_endian)) { code_point = utf16::swap_bytes(uint16_t(code_point)); } *utf16_output++ = char16_t(code_point); @@ -10928,7 +11069,7 @@ inline size_t convert_valid(const char* buf, size_t len, char16_t* utf16_output) code_point -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); - if (big_endian) { + if (!match_system(big_endian)) { high_surrogate = utf16::swap_bytes(high_surrogate); low_surrogate = utf16::swap_bytes(low_surrogate); } @@ -10977,16 +11118,17 @@ inline size_t convert(const char* buf, size_t len, char16_t* utf16_output) { if ((v & 0x8080808080808080) == 0) { size_t final_pos = pos + 16; while(pos < final_pos) { - *utf16_output++ = big_endian ? char16_t(utf16::swap_bytes(buf[pos])) : char16_t(buf[pos]); + *utf16_output++ = !match_system(big_endian) ? char16_t(utf16::swap_bytes(buf[pos])) : char16_t(buf[pos]); pos++; } continue; } } + uint8_t leading_byte = data[pos]; // leading byte if (leading_byte < 0b10000000) { // converting one ASCII byte !!! - *utf16_output++ = big_endian ? char16_t(utf16::swap_bytes(leading_byte)): char16_t(leading_byte); + *utf16_output++ = !match_system(big_endian) ? char16_t(utf16::swap_bytes(leading_byte)): char16_t(leading_byte); pos++; } else if ((leading_byte & 0b11100000) == 0b11000000) { // We have a two-byte UTF-8, it should become @@ -10996,7 +11138,7 @@ inline size_t convert(const char* buf, size_t len, char16_t* utf16_output) { // range check uint32_t code_point = (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); if (code_point < 0x80 || 0x7ff < code_point) { return 0; } - if (big_endian) { + if (!match_system(big_endian)) { code_point = uint32_t(utf16::swap_bytes(uint16_t(code_point))); } *utf16_output++ = char16_t(code_point); @@ -11016,7 +11158,7 @@ inline size_t convert(const char* buf, size_t len, char16_t* utf16_output) { (0xd7ff < code_point && code_point < 0xe000)) { return 0; } - if (big_endian) { + if (!match_system(big_endian)) { code_point = uint32_t(utf16::swap_bytes(uint16_t(code_point))); } *utf16_output++ = char16_t(code_point); @@ -11036,7 +11178,7 @@ inline size_t convert(const char* buf, size_t len, char16_t* utf16_output) { code_point -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); - if (big_endian) { + if (!match_system(big_endian)) { high_surrogate = utf16::swap_bytes(high_surrogate); low_surrogate = utf16::swap_bytes(low_surrogate); } @@ -11066,7 +11208,7 @@ inline result convert_with_errors(const char* buf, size_t len, char16_t* utf16_o if ((v & 0x8080808080808080) == 0) { size_t final_pos = pos + 16; while(pos < final_pos) { - *utf16_output++ = big_endian ? char16_t(utf16::swap_bytes(buf[pos])) : char16_t(buf[pos]); + *utf16_output++ = !match_system(big_endian) ? char16_t(utf16::swap_bytes(buf[pos])) : char16_t(buf[pos]); pos++; } continue; @@ -11075,7 +11217,7 @@ inline result convert_with_errors(const char* buf, size_t len, char16_t* utf16_o uint8_t leading_byte = data[pos]; // leading byte if (leading_byte < 0b10000000) { // converting one ASCII byte !!! - *utf16_output++ = big_endian ? char16_t(utf16::swap_bytes(leading_byte)): char16_t(leading_byte); + *utf16_output++ = !match_system(big_endian) ? char16_t(utf16::swap_bytes(leading_byte)): char16_t(leading_byte); pos++; } else if ((leading_byte & 0b11100000) == 0b11000000) { // We have a two-byte UTF-8, it should become @@ -11085,7 +11227,7 @@ inline result convert_with_errors(const char* buf, size_t len, char16_t* utf16_o // range check uint32_t code_point = (leading_byte & 0b00011111) << 6 | (data[pos + 1] & 0b00111111); if (code_point < 0x80 || 0x7ff < code_point) { return result(error_code::OVERLONG, pos); } - if (big_endian) { + if (!match_system(big_endian)) { code_point = uint32_t(utf16::swap_bytes(uint16_t(code_point))); } *utf16_output++ = char16_t(code_point); @@ -11103,7 +11245,7 @@ inline result convert_with_errors(const char* buf, size_t len, char16_t* utf16_o (data[pos + 2] & 0b00111111); if ((code_point < 0x800) || (0xffff < code_point)) { return result(error_code::OVERLONG, pos);} if (0xd7ff < code_point && code_point < 0xe000) { return result(error_code::SURROGATE, pos); } - if (big_endian) { + if (!match_system(big_endian)) { code_point = uint32_t(utf16::swap_bytes(uint16_t(code_point))); } *utf16_output++ = char16_t(code_point); @@ -11124,7 +11266,7 @@ inline result convert_with_errors(const char* buf, size_t len, char16_t* utf16_o code_point -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (code_point >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (code_point & 0x3FF)); - if (big_endian) { + if (!match_system(big_endian)) { high_surrogate = utf16::swap_bytes(high_surrogate); low_surrogate = utf16::swap_bytes(low_surrogate); } @@ -11140,20 +11282,49 @@ inline result convert_with_errors(const char* buf, size_t len, char16_t* utf16_o return result(error_code::SUCCESS, utf16_output - start); } +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and we have + * up to len input bytes left, and we encountered some error. It is possible that + * the error is at 'buf' exactly, but it could also be in the previous bytes (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the current memory section + * and can be safely accessed. We prior_bytes to access safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occured prior to 'buf', the count value contain in the result + * will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ template <endianness endian> -inline result rewind_and_convert_with_errors(const char* buf, size_t len, char16_t* utf16_output) { +inline result rewind_and_convert_with_errors(size_t prior_bytes, const char* buf, size_t len, char16_t* utf16_output) { size_t extra_len{0}; - // A leading byte cannot be further than 4 bytes away - for(int i = 0; i < 5; i++) { - unsigned char byte = *buf; - if ((byte & 0b11000000) != 0b10000000) { + // We potentially need to go back in time and find a leading byte. + size_t how_far_back = 3; // 3 bytes in the past + current position + if(how_far_back >= prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for(size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-i]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if(found_leading_bytes) { + buf -= i; + extra_len = i; break; - } else { - buf--; - extra_len++; } } - + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in <cstddef> which is described in C Standard as <stddef.h>. + // C Standard Section 4.1.5 defines size_t as an unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if(!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is continuation] + // Or we possibly have a stream that does not start with a leading byte. + return result(error_code::TOO_LONG, -how_far_back); + } result res = convert_with_errors<endian>(buf, len + extra_len, utf16_output); if (res.error) { res.count -= extra_len; @@ -11390,18 +11561,48 @@ inline result convert_with_errors(const char* buf, size_t len, char32_t* utf32_o return result(error_code::SUCCESS, utf32_output - start); } -inline result rewind_and_convert_with_errors(const char* buf, size_t len, char32_t* utf32_output) { +/** + * When rewind_and_convert_with_errors is called, we are pointing at 'buf' and we have + * up to len input bytes left, and we encountered some error. It is possible that + * the error is at 'buf' exactly, but it could also be in the previous bytes location (up to 3 bytes back). + * + * prior_bytes indicates how many bytes, prior to 'buf' may belong to the current memory section + * and can be safely accessed. We prior_bytes to access safely up to three bytes before 'buf'. + * + * The caller is responsible to ensure that len > 0. + * + * If the error is believed to have occured prior to 'buf', the count value contain in the result + * will be SIZE_T - 1, SIZE_T - 2, or SIZE_T - 3. + */ +inline result rewind_and_convert_with_errors(size_t prior_bytes, const char* buf, size_t len, char32_t* utf32_output) { size_t extra_len{0}; - // A leading byte cannot be further than 4 bytes away - for(int i = 0; i < 5; i++) { - unsigned char byte = *buf; - if ((byte & 0b11000000) != 0b10000000) { + // We potentially need to go back in time and find a leading byte. + size_t how_far_back = 3; // 3 bytes in the past + current position + if(how_far_back > prior_bytes) { how_far_back = prior_bytes; } + bool found_leading_bytes{false}; + // important: it is i <= how_far_back and not 'i < how_far_back'. + for(size_t i = 0; i <= how_far_back; i++) { + unsigned char byte = buf[-i]; + found_leading_bytes = ((byte & 0b11000000) != 0b10000000); + if(found_leading_bytes) { + buf -= i; + extra_len = i; break; - } else { - buf--; - extra_len++; } } + // + // It is possible for this function to return a negative count in its result. + // C++ Standard Section 18.1 defines size_t is in <cstddef> which is described in C Standard as <stddef.h>. + // C Standard Section 4.1.5 defines size_t as an unsigned integral type of the result of the sizeof operator + // + // An unsigned type will simply wrap round arithmetically (well defined). + // + if(!found_leading_bytes) { + // If how_far_back == 3, we may have four consecutive continuation bytes!!! + // [....] [continuation] [continuation] [continuation] | [buf is continuation] + // Or we possibly have a stream that does not start with a leading byte. + return result(error_code::TOO_LONG, -how_far_back); + } result res = convert_with_errors(buf, len + extra_len, utf32_output); if (res.error) { @@ -11510,8 +11711,8 @@ int arm_detect_encodings(const char * buf, size_t len) { if (surrogates_wordmask0 != 0 || surrogates_wordmask1 != 0) { // Cannot be UTF8 is_utf8 = false; - // Can still be either UTF-16LE or UTF-32LE depending on the positions of the surrogates - // To be valid UTF-32LE, a surrogate cannot be in the two most significant bytes of any 32-bit word. + // Can still be either UTF-16LE or UTF-32 depending on the positions of the surrogates + // To be valid UTF-32, a surrogate cannot be in the two most significant bytes of any 32-bit word. // On the other hand, to be valid UTF-16LE, at least one surrogate must be in the two most significant // bytes of a 32-bit word since they always come in pairs in UTF-16LE. // Note that we always proceed in multiple of 4 before this point so there is no offset in 32-bit words. @@ -11582,7 +11783,7 @@ int arm_detect_encodings(const char * buf, size_t len) { } } else { is_utf16 = false; - // Check for UTF-32LE + // Check for UTF-32 if (len % 4 == 0) { const char32_t * input = reinterpret_cast<const char32_t*>(buf); const char32_t* end32 = reinterpret_cast<const char32_t*>(start) + len/4; @@ -11626,7 +11827,7 @@ int arm_detect_encodings(const char * buf, size_t len) { } // If no surrogate, validate under other encodings as well - // UTF-32LE validation + // UTF-32 validation currentmax = vmaxq_u32(vreinterpretq_u32_u16(in),currentmax); currentmax = vmaxq_u32(vreinterpretq_u32_u16(secondin),currentmax); currentmax = vmaxq_u32(vreinterpretq_u32_u16(thirdin),currentmax); @@ -11686,7 +11887,7 @@ const char16_t* arm_validate_utf16(const char16_t* input, size_t size) { // consists only the higher bytes. auto in0 = simd16<uint16_t>(input); auto in1 = simd16<uint16_t>(input + simd16<uint16_t>::SIZE / sizeof(char16_t)); - if (big_endian) { + if (!match_system(big_endian)) { #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO const uint8x16_t swap = make_uint8x16_t(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); #else @@ -11762,7 +11963,7 @@ const result arm_validate_utf16_with_errors(const char16_t* input, size_t size) auto in0 = simd16<uint16_t>(input); auto in1 = simd16<uint16_t>(input + simd16<uint16_t>::SIZE / sizeof(char16_t)); - if (big_endian) { + if (!match_system(big_endian)) { #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO const uint8x16_t swap = make_uint8x16_t(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); #else @@ -11918,7 +12119,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, // We process in chunks of 16 bytes uint16x8_t ascii_first = vmovl_u8(vget_low_u8 (in)); uint16x8_t ascii_second = vmovl_high_u8(in); - if (big_endian) { + if (!match_system(big_endian)) { ascii_first = vreinterpretq_u16_u8(vqtbl1q_u8(vreinterpretq_u8_u16(ascii_first), swap)); ascii_second = vreinterpretq_u16_u8(vqtbl1q_u8(vreinterpretq_u8_u16(ascii_second), swap)); } @@ -11934,7 +12135,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, uint8x16_t ascii = vandq_u8(perm, vreinterpretq_u8_u16(vmovq_n_u16(0x7f))); uint8x16_t highbyte = vandq_u8(perm, vreinterpretq_u8_u16(vmovq_n_u16(0x1f00))); uint8x16_t composed = vorrq_u8(ascii, vreinterpretq_u8_u16(vshrq_n_u16(vreinterpretq_u16_u8(highbyte), 2))); - if (big_endian) composed = vqtbl1q_u8(composed, swap); + if (!match_system(big_endian)) composed = vqtbl1q_u8(composed, swap); vst1q_u8(reinterpret_cast<uint8_t*>(utf16_output), composed); utf16_output += 8; // We wrote 16 bytes, 8 code points. return 16; @@ -11959,7 +12160,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, uint32x4_t composed = vorrq_u32(vorrq_u32(vreinterpretq_u32_u8(ascii), vreinterpretq_u32_u8(middlebyte_shifted)), highbyte_shifted); uint16x8_t composed_repacked = vmovn_high_u32(vmovn_u32(composed), composed); - if (big_endian) composed_repacked = vreinterpretq_u16_u8(vqtbl1q_u8(vreinterpretq_u8_u16(composed_repacked), swap)); + if (!match_system(big_endian)) composed_repacked = vreinterpretq_u16_u8(vqtbl1q_u8(vreinterpretq_u8_u16(composed_repacked), swap)); vst1q_u16(reinterpret_cast<uint16_t*>(utf16_output), composed_repacked); utf16_output += 4; return 12; @@ -11982,7 +12183,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, uint8x16_t ascii = vandq_u8(perm, vreinterpretq_u8_u16(vmovq_n_u16(0x7f))); uint8x16_t highbyte = vandq_u8(perm, vreinterpretq_u8_u16(vmovq_n_u16(0x1f00))); uint8x16_t composed = vorrq_u8(ascii, vreinterpretq_u8_u16(vshrq_n_u16(vreinterpretq_u16_u8(highbyte), 2))); - if (big_endian) composed = vqtbl1q_u8(composed, swap); + if (!match_system(big_endian)) composed = vqtbl1q_u8(composed, swap); vst1q_u8(reinterpret_cast<uint8_t*>(utf16_output), composed); utf16_output += 6; // We wrote 12 bytes, 6 code points. } else if (idx < 145) { @@ -12000,7 +12201,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, uint32x4_t composed = vorrq_u32(vorrq_u32(vreinterpretq_u32_u8(ascii), vreinterpretq_u32_u8(middlebyte_shifted)), highbyte_shifted); uint16x8_t composed_repacked = vmovn_high_u32(vmovn_u32(composed), composed); - if (big_endian) composed_repacked = vreinterpretq_u16_u8(vqtbl1q_u8(vreinterpretq_u8_u16(composed_repacked), swap)); + if (!match_system(big_endian)) composed_repacked = vreinterpretq_u16_u8(vqtbl1q_u8(vreinterpretq_u8_u16(composed_repacked), swap)); vst1q_u16(reinterpret_cast<uint16_t*>(utf16_output), composed_repacked); utf16_output += 4; } else if (idx < 209) { @@ -12035,7 +12236,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, vorrq_u32(hightenbitsadd, lowtenbitsaddshifted); uint32_t basic_buffer[4]; uint32_t basic_buffer_swap[4]; - if (big_endian) { + if (!match_system(big_endian)) { vst1q_u32(basic_buffer_swap, vreinterpretq_u32_u8(vqtbl1q_u8(composed, swap))); surrogates = vreinterpretq_u32_u8(vqtbl1q_u8(vreinterpretq_u8_u32(surrogates), swap)); } @@ -12044,7 +12245,7 @@ size_t convert_masked_utf8_to_utf16(const char *input, vst1q_u32(surrogate_buffer, surrogates); for (size_t i = 0; i < 3; i++) { if (basic_buffer[i] < 65536) { - utf16_output[0] = big_endian ? uint16_t(basic_buffer_swap[i]) : uint16_t(basic_buffer[i]); + utf16_output[0] = !match_system(big_endian) ? uint16_t(basic_buffer_swap[i]) : uint16_t(basic_buffer[i]); utf16_output++; } else { utf16_output[0] = uint16_t(surrogate_buffer[i] & 0xffff); @@ -12260,7 +12461,7 @@ std::pair<const char16_t*, char*> arm_convert_utf16_to_utf8(const char16_t* buf, while (buf + 16 <= end) { uint16x8_t in = vld1q_u16(reinterpret_cast<const uint16_t *>(buf)); - if (big_endian) { + if (!match_system(big_endian)) { #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO const uint8x16_t swap = make_uint8x16_t(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); #else @@ -12271,7 +12472,7 @@ std::pair<const char16_t*, char*> arm_convert_utf16_to_utf8(const char16_t* buf, if(vmaxvq_u16(in) <= 0x7F) { // ASCII fast path!!!! // It is common enough that we have sequences of 16 consecutive ASCII characters. uint16x8_t nextin = vld1q_u16(reinterpret_cast<const uint16_t *>(buf) + 8); - if (big_endian) { + if (!match_system(big_endian)) { #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO const uint8x16_t swap = make_uint8x16_t(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); #else @@ -12477,7 +12678,7 @@ std::pair<const char16_t*, char*> arm_convert_utf16_to_utf8(const char16_t* buf, size_t k = 0; if(size_t(end - buf) < forward + 1) { forward = size_t(end - buf - 1);} for(; k < forward; k++) { - uint16_t word = big_endian ? scalar::utf16::swap_bytes(buf[k]) : buf[k]; + uint16_t word = !match_system(big_endian) ? scalar::utf16::swap_bytes(buf[k]) : buf[k]; if((word & 0xFF80)==0) { *utf8_output++ = char(word); } else if((word & 0xF800)==0) { @@ -12490,7 +12691,7 @@ std::pair<const char16_t*, char*> arm_convert_utf16_to_utf8(const char16_t* buf, } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = big_endian ? scalar::utf16::swap_bytes(buf[k + 1]) : buf[k + 1]; + uint16_t next_word = !match_system(big_endian) ? scalar::utf16::swap_bytes(buf[k + 1]) : buf[k + 1]; k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if((diff | diff2) > 0x3FF) { return std::make_pair(nullptr, reinterpret_cast<char*>(utf8_output)); } @@ -12527,7 +12728,7 @@ std::pair<result, char*> arm_convert_utf16_to_utf8_with_errors(const char16_t* b while (buf + 16 <= end) { uint16x8_t in = vld1q_u16(reinterpret_cast<const uint16_t *>(buf)); - if (big_endian) { + if (!match_system(big_endian)) { #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO const uint8x16_t swap = make_uint8x16_t(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); #else @@ -12538,7 +12739,7 @@ std::pair<result, char*> arm_convert_utf16_to_utf8_with_errors(const char16_t* b if(vmaxvq_u16(in) <= 0x7F) { // ASCII fast path!!!! // It is common enough that we have sequences of 16 consecutive ASCII characters. uint16x8_t nextin = vld1q_u16(reinterpret_cast<const uint16_t *>(buf) + 8); - if (big_endian) { + if (!match_system(big_endian)) { #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO const uint8x16_t swap = make_uint8x16_t(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); #else @@ -12744,7 +12945,7 @@ std::pair<result, char*> arm_convert_utf16_to_utf8_with_errors(const char16_t* b size_t k = 0; if(size_t(end - buf) < forward + 1) { forward = size_t(end - buf - 1);} for(; k < forward; k++) { - uint16_t word = big_endian ? scalar::utf16::swap_bytes(buf[k]) : buf[k]; + uint16_t word = !match_system(big_endian) ? scalar::utf16::swap_bytes(buf[k]) : buf[k]; if((word & 0xFF80)==0) { *utf8_output++ = char(word); } else if((word & 0xF800)==0) { @@ -12757,7 +12958,7 @@ std::pair<result, char*> arm_convert_utf16_to_utf8_with_errors(const char16_t* b } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = big_endian ? scalar::utf16::swap_bytes(buf[k + 1]) : buf[k + 1]; + uint16_t next_word = !match_system(big_endian) ? scalar::utf16::swap_bytes(buf[k + 1]) : buf[k + 1]; k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if((diff | diff2) > 0x3FF) { return std::make_pair(result(error_code::SURROGATE, buf - start + k - 1), reinterpret_cast<char*>(utf8_output)); } @@ -12839,7 +13040,7 @@ std::pair<const char16_t*, char32_t*> arm_convert_utf16_to_utf32(const char16_t* while (buf + 16 <= end) { uint16x8_t in = vld1q_u16(reinterpret_cast<const uint16_t *>(buf)); - if (big_endian) { + if (!match_system(big_endian)) { #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO const uint8x16_t swap = make_uint8x16_t(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); #else @@ -12866,13 +13067,13 @@ std::pair<const char16_t*, char32_t*> arm_convert_utf16_to_utf32(const char16_t* size_t k = 0; if(size_t(end - buf) < forward + 1) { forward = size_t(end - buf - 1);} for(; k < forward; k++) { - uint16_t word = big_endian ? scalar::utf16::swap_bytes(buf[k]) : buf[k]; + uint16_t word = !match_system(big_endian) ? scalar::utf16::swap_bytes(buf[k]) : buf[k]; if((word &0xF800 ) != 0xD800) { *utf32_output++ = char32_t(word); } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = big_endian ? scalar::utf16::swap_bytes(buf[k + 1]) : buf[k + 1]; + uint16_t next_word = !match_system(big_endian) ? scalar::utf16::swap_bytes(buf[k + 1]) : buf[k + 1]; k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if((diff | diff2) > 0x3FF) { return std::make_pair(nullptr, reinterpret_cast<char32_t*>(utf32_output)); } @@ -12904,7 +13105,7 @@ std::pair<result, char32_t*> arm_convert_utf16_to_utf32_with_errors(const char16 while (buf + 16 <= end) { uint16x8_t in = vld1q_u16(reinterpret_cast<const uint16_t *>(buf)); - if (big_endian) { + if (!match_system(big_endian)) { #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO const uint8x16_t swap = make_uint8x16_t(1, 0, 3, 2, 5, 4, 7, 6, 9, 8, 11, 10, 13, 12, 15, 14); #else @@ -12931,13 +13132,13 @@ std::pair<result, char32_t*> arm_convert_utf16_to_utf32_with_errors(const char16 size_t k = 0; if(size_t(end - buf) < forward + 1) { forward = size_t(end - buf - 1);} for(; k < forward; k++) { - uint16_t word = big_endian ? scalar::utf16::swap_bytes(buf[k]) : buf[k]; + uint16_t word = !match_system(big_endian) ? scalar::utf16::swap_bytes(buf[k]) : buf[k]; if((word &0xF800 ) != 0xD800) { *utf32_output++ = char32_t(word); } else { // must be a surrogate pair uint16_t diff = uint16_t(word - 0xD800); - uint16_t next_word = big_endian ? scalar::utf16::swap_bytes(buf[k + 1]) : buf[k + 1]; + uint16_t next_word = !match_system(big_endian) ? scalar::utf16::swap_bytes(buf[k + 1]) : buf[k + 1]; k++; uint16_t diff2 = uint16_t(next_word - 0xDC00); if((diff | diff2) > 0x3FF) { return std::make_pair(result(error_code::SURROGATE, buf - start + k - 1), reinterpret_cast<char32_t*>(utf32_output)); } @@ -13445,7 +13646,7 @@ std::pair<const char32_t*, char16_t*> arm_convert_utf32_to_utf16(const char32_t* const uint16x4_t v_dfff = vmov_n_u16((uint16_t)0xdfff); forbidden_bytemask = vorr_u16(vand_u16(vcle_u16(utf16_packed, v_dfff), vcge_u16(utf16_packed, v_d800)), forbidden_bytemask); - if (big_endian) { + if (!match_system(big_endian)) { #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO const uint8x8_t swap = make_uint8x8_t(1, 0, 3, 2, 5, 4, 7, 6); #else @@ -13465,14 +13666,14 @@ std::pair<const char32_t*, char16_t*> arm_convert_utf32_to_utf16(const char32_t* if((word & 0xFFFF0000)==0) { // will not generate a surrogate pair if (word >= 0xD800 && word <= 0xDFFF) { return std::make_pair(nullptr, reinterpret_cast<char16_t*>(utf16_output)); } - *utf16_output++ = big_endian ? char16_t(word >> 8 | word << 8) : char16_t(word); + *utf16_output++ = !match_system(big_endian) ? char16_t(word >> 8 | word << 8) : char16_t(word); } else { // will generate a surrogate pair if (word > 0x10FFFF) { return std::make_pair(nullptr, reinterpret_cast<char16_t*>(utf16_output)); } word -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (big_endian) { + if (!match_system(big_endian)) { high_surrogate = uint16_t(high_surrogate >> 8 | high_surrogate << 8); low_surrogate = uint16_t(low_surrogate << 8 | low_surrogate >> 8); } @@ -13513,7 +13714,7 @@ std::pair<result, char16_t*> arm_convert_utf32_to_utf16_with_errors(const char32 return std::make_pair(result(error_code::SURROGATE, buf - start), reinterpret_cast<char16_t*>(utf16_output)); } - if (big_endian) { + if (!match_system(big_endian)) { #ifdef SIMDUTF_REGULAR_VISUAL_STUDIO const uint8x8_t swap = make_uint8x8_t(1, 0, 3, 2, 5, 4, 7, 6); #else @@ -13533,14 +13734,14 @@ std::pair<result, char16_t*> arm_convert_utf32_to_utf16_with_errors(const char32 if((word & 0xFFFF0000)==0) { // will not generate a surrogate pair if (word >= 0xD800 && word <= 0xDFFF) { return std::make_pair(result(error_code::SURROGATE, buf - start + k), reinterpret_cast<char16_t*>(utf16_output)); } - *utf16_output++ = big_endian ? char16_t(word >> 8 | word << 8) : char16_t(word); + *utf16_output++ = !match_system(big_endian) ? char16_t(word >> 8 | word << 8) : char16_t(word); } else { // will generate a surrogate pair if (word > 0x10FFFF) { return std::make_pair(result(error_code::TOO_LARGE, buf - start + k), reinterpret_cast<char16_t*>(utf16_output)); } word -= 0x10000; uint16_t high_surrogate = uint16_t(0xD800 + (word >> 10)); uint16_t low_surrogate = uint16_t(0xDC00 + (word & 0x3FF)); - if (big_endian) { + if (!match_system(big_endian)) { high_surrogate = uint16_t(high_surrogate >> 8 | high_surrogate << 8); low_surrogate = uint16_t(low_surrogate << 8 | low_surrogate >> 8); } @@ -14261,7 +14462,9 @@ using namespace simd; this->check_utf8_bytes(input.chunks[3], input.chunks[2]); } if (errors()) { - result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(in + pos, size - pos, utf16_output); + // rewind_and_convert_with_errors will seek a potential error from in+pos onward, + // with the ability to go back up to pos bytes, and read size-pos bytes forward. + result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(pos, in + pos, size - pos, utf16_output); res.count += pos; return res; } @@ -14295,12 +14498,16 @@ using namespace simd; } } if(errors()) { - result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(in + pos, size - pos, utf16_output); + // rewind_and_convert_with_errors will seek a potential error from in+pos onward, + // with the ability to go back up to pos bytes, and read size-pos bytes forward. + result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(pos, in + pos, size - pos, utf16_output); res.count += pos; return res; } if(pos < size) { - result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(in + pos, size - pos, utf16_output); + // rewind_and_convert_with_errors will seek a potential error from in+pos onward, + // with the ability to go back up to pos bytes, and read size-pos bytes forward. + result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(pos, in + pos, size - pos, utf16_output); if (res.error) { // In case of error, we want the error position res.count += pos; return res; @@ -14584,7 +14791,7 @@ using namespace simd; this->check_utf8_bytes(input.chunks[3], input.chunks[2]); } if (errors()) { - result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(in + pos, size - pos, utf32_output); + result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(pos, in + pos, size - pos, utf32_output); res.count += pos; return res; } @@ -14618,12 +14825,12 @@ using namespace simd; } } if(errors()) { - result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(in + pos, size - pos, utf32_output); + result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(pos, in + pos, size - pos, utf32_output); res.count += pos; return res; } if(pos < size) { - result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(in + pos, size - pos, utf32_output); + result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(pos, in + pos, size - pos, utf32_output); if (res.error) { // In case of error, we want the error position res.count += pos; return res; @@ -14705,7 +14912,7 @@ simdutf_really_inline size_t count_code_points(const char16_t* in, size_t size) size_t count = 0; for(;pos + 32 <= size; pos += 32) { simd16x32<uint16_t> input(reinterpret_cast<const uint16_t *>(in + pos)); - if (big_endian) input.swap_bytes(); + if (!match_system(big_endian)) input.swap_bytes(); uint64_t not_pair = input.not_in_range(0xDC00, 0xDFFF); count += count_ones(not_pair) / 2; } @@ -14719,7 +14926,7 @@ simdutf_really_inline size_t utf8_length_from_utf16(const char16_t* in, size_t s // This algorithm could no doubt be improved! for(;pos + 32 <= size; pos += 32) { simd16x32<uint16_t> input(reinterpret_cast<const uint16_t *>(in + pos)); - if (big_endian) input.swap_bytes(); + if (!match_system(big_endian)) input.swap_bytes(); uint64_t ascii_mask = input.lteq(0x7F); uint64_t twobyte_mask = input.lteq(0x7FF); uint64_t not_pair_mask = input.not_in_range(0xD800, 0xDFFF); @@ -15473,7 +15680,7 @@ simdutf_warn_unused size_t implementation::utf16_length_from_utf32(const char32_ } simdutf_warn_unused size_t implementation::utf32_length_from_utf8(const char * input, size_t length) const noexcept { - return scalar::utf8::utf32_length_from_utf8(input, length); + return scalar::utf8::count_code_points(input, length); } } // namespace fallback @@ -16622,7 +16829,9 @@ simdutf::result fast_avx512_convert_utf8_to_utf16_with_errors(const char *in, si } else { break; } } if(!result) { - simdutf::result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<big_endian>(in, final_in - in, out); + // rewind_and_convert_with_errors will seek a potential error from in onward, + // with the ability to go back up to in - init_in bytes, and read final_in - in bytes forward. + simdutf::result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<big_endian>(in - init_in, in, final_in - in, out); res.count += (in - init_in); return res; } else { @@ -17618,7 +17827,7 @@ std::pair<result, char16_t*> avx512_convert_utf32_to_utf16_with_errors(const cha /* begin file src/icelake/icelake_ascii_validation.inl.cpp */ // file included directly -const char* validate_ascii(const char* buf, size_t len) { +bool validate_ascii(const char* buf, size_t len) { const char* end = buf + len; const __m512i ascii = _mm512_set1_epi8((uint8_t)0x80); __m512i running_or = _mm512_setzero_si512(); @@ -17626,11 +17835,11 @@ const char* validate_ascii(const char* buf, size_t len) { const __m512i utf8 = _mm512_loadu_si512((const __m512i*)buf); running_or = _mm512_ternarylogic_epi32(running_or, utf8, ascii, 0xf8); // running_or | (utf8 & ascii) } - if (_mm512_test_epi8_mask(running_or, running_or) != 0) { - return nullptr; - } else { - return buf; + if(buf < end) { + const __m512i utf8 = _mm512_maskz_loadu_epi8((uint64_t(1) << (end-buf)) - 1,(const __m512i*)buf); + running_or = _mm512_ternarylogic_epi32(running_or, utf8, ascii, 0xf8); // running_or | (utf8 & ascii) } + return (_mm512_test_epi8_mask(running_or, running_or) == 0); } /* end file src/icelake/icelake_ascii_validation.inl.cpp */ // dofile: invoked with prepath=/Users/dlemire/CVS/github/simdutf/src, filename=icelake/icelake_utf32_validation.inl.cpp @@ -17902,8 +18111,8 @@ implementation::detect_encodings(const char *input, if (surrogates) { is_utf8 = false; - // Can still be either UTF-16LE or UTF-32LE depending on the positions - // of the surrogates To be valid UTF-32LE, a surrogate cannot be in the + // Can still be either UTF-16LE or UTF-32 depending on the positions + // of the surrogates To be valid UTF-32, a surrogate cannot be in the // two most significant bytes of any 32-bit word. On the other hand, to // be valid UTF-16LE, at least one surrogate must be in the two most // significant bytes of a 32-bit word since they always come in pairs in @@ -17940,7 +18149,7 @@ implementation::detect_encodings(const char *input, } else { is_utf16 = false; - // Check for UTF-32LE + // Check for UTF-32 if (length % 4 == 0) { const char32_t *input32 = reinterpret_cast<const char32_t *>(buf); const char32_t *end32 = @@ -17955,7 +18164,7 @@ implementation::detect_encodings(const char *input, } // If no surrogate, validate under other encodings as well - // UTF-32LE validation + // UTF-32 validation currentmax = _mm512_max_epu32(in, currentmax); // UTF-8 validation @@ -18053,12 +18262,7 @@ simdutf_warn_unused result implementation::validate_utf8_with_errors(const char } simdutf_warn_unused bool implementation::validate_ascii(const char *buf, size_t len) const noexcept { - const char* tail = icelake::validate_ascii(buf, len); - if (tail) { - return scalar::ascii::validate(tail, len - (tail - buf)); - } else { - return false; - } + return icelake::validate_ascii(buf, len); } simdutf_warn_unused result implementation::validate_ascii_with_errors(const char *buf, size_t len) const noexcept { @@ -18432,7 +18636,10 @@ simdutf_warn_unused result implementation::convert_utf8_to_utf32_with_errors(con uint32_t * utf32_output = reinterpret_cast<uint32_t *>(utf32); auto ret = icelake::validating_utf8_to_fixed_length_with_constant_checks<endianness::LITTLE, uint32_t>(buf, len, utf32_output); if (!std::get<2>(ret)) { - result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(std::get<0>(ret), len - (std::get<0>(ret) - buf), reinterpret_cast<char32_t *>(std::get<1>(ret))); + auto new_buf = std::get<0>(ret); + // rewind_and_convert_with_errors will seek a potential error from new_buf onward, + // with the ability to go back up to new_buf - buf bytes, and read len - (new_buf - buf) bytes forward. + result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(new_buf - buf, new_buf, len - (new_buf - buf), reinterpret_cast<char32_t *>(std::get<1>(ret))); res.count += (std::get<0>(ret) - buf); return res; } @@ -19081,8 +19288,8 @@ int avx2_detect_encodings(const char * buf, size_t len) { if (surrogates_bitmask0 != 0x0) { // Cannot be UTF8 is_utf8 = false; - // Can still be either UTF-16LE or UTF-32LE depending on the positions of the surrogates - // To be valid UTF-32LE, a surrogate cannot be in the two most significant bytes of any 32-bit word. + // Can still be either UTF-16LE or UTF-32 depending on the positions of the surrogates + // To be valid UTF-32, a surrogate cannot be in the two most significant bytes of any 32-bit word. // On the other hand, to be valid UTF-16LE, at least one surrogate must be in the two most significant // bytes of a 32-bit word since they always come in pairs in UTF-16LE. // Note that we always proceed in multiple of 4 before this point so there is no offset in 32-bit words. @@ -19153,7 +19360,7 @@ int avx2_detect_encodings(const char * buf, size_t len) { } } else { is_utf16 = false; - // Check for UTF-32LE + // Check for UTF-32 if (len % 4 == 0) { const char32_t * input = reinterpret_cast<const char32_t*>(buf); const char32_t* end32 = reinterpret_cast<const char32_t*>(start) + len/4; @@ -19188,7 +19395,7 @@ int avx2_detect_encodings(const char * buf, size_t len) { } // If no surrogate, validate under other encodings as well - // UTF-32LE validation + // UTF-32 validation currentmax = _mm256_max_epu32(in, currentmax); currentmax = _mm256_max_epu32(nextin, currentmax); @@ -21834,7 +22041,9 @@ using namespace simd; this->check_utf8_bytes(input.chunks[3], input.chunks[2]); } if (errors()) { - result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(in + pos, size - pos, utf16_output); + // rewind_and_convert_with_errors will seek a potential error from in+pos onward, + // with the ability to go back up to pos bytes, and read size-pos bytes forward. + result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(pos, in + pos, size - pos, utf16_output); res.count += pos; return res; } @@ -21868,12 +22077,16 @@ using namespace simd; } } if(errors()) { - result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(in + pos, size - pos, utf16_output); + // rewind_and_convert_with_errors will seek a potential error from in+pos onward, + // with the ability to go back up to pos bytes, and read size-pos bytes forward. + result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(pos, in + pos, size - pos, utf16_output); res.count += pos; return res; } if(pos < size) { - result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(in + pos, size - pos, utf16_output); + // rewind_and_convert_with_errors will seek a potential error from in+pos onward, + // with the ability to go back up to pos bytes, and read size-pos bytes forward. + result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(pos, in + pos, size - pos, utf16_output); if (res.error) { // In case of error, we want the error position res.count += pos; return res; @@ -22157,7 +22370,7 @@ using namespace simd; this->check_utf8_bytes(input.chunks[3], input.chunks[2]); } if (errors()) { - result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(in + pos, size - pos, utf32_output); + result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(pos, in + pos, size - pos, utf32_output); res.count += pos; return res; } @@ -22191,12 +22404,12 @@ using namespace simd; } } if(errors()) { - result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(in + pos, size - pos, utf32_output); + result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(pos, in + pos, size - pos, utf32_output); res.count += pos; return res; } if(pos < size) { - result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(in + pos, size - pos, utf32_output); + result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(pos, in + pos, size - pos, utf32_output); if (res.error) { // In case of error, we want the error position res.count += pos; return res; @@ -22278,7 +22491,7 @@ simdutf_really_inline size_t count_code_points(const char16_t* in, size_t size) size_t count = 0; for(;pos + 32 <= size; pos += 32) { simd16x32<uint16_t> input(reinterpret_cast<const uint16_t *>(in + pos)); - if (big_endian) input.swap_bytes(); + if (!match_system(big_endian)) input.swap_bytes(); uint64_t not_pair = input.not_in_range(0xDC00, 0xDFFF); count += count_ones(not_pair) / 2; } @@ -22292,7 +22505,7 @@ simdutf_really_inline size_t utf8_length_from_utf16(const char16_t* in, size_t s // This algorithm could no doubt be improved! for(;pos + 32 <= size; pos += 32) { simd16x32<uint16_t> input(reinterpret_cast<const uint16_t *>(in + pos)); - if (big_endian) input.swap_bytes(); + if (!match_system(big_endian)) input.swap_bytes(); uint64_t ascii_mask = input.lteq(0x7F); uint64_t twobyte_mask = input.lteq(0x7FF); uint64_t not_pair_mask = input.not_in_range(0xD800, 0xDFFF); @@ -22787,7 +23000,7 @@ simdutf_warn_unused size_t implementation::utf16_length_from_utf32(const char32_ } simdutf_warn_unused size_t implementation::utf32_length_from_utf8(const char * input, size_t length) const noexcept { - return utf8::utf32_length_from_utf8(input, length); + return scalar::utf8::count_code_points(input, length); } } // namespace haswell @@ -23557,7 +23770,9 @@ using namespace simd; this->check_utf8_bytes(input.chunks[3], input.chunks[2]); } if (errors()) { - result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(in + pos, size - pos, utf16_output); + // rewind_and_convert_with_errors will seek a potential error from in+pos onward, + // with the ability to go back up to pos bytes, and read size-pos bytes forward. + result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(pos, in + pos, size - pos, utf16_output); res.count += pos; return res; } @@ -23591,12 +23806,16 @@ using namespace simd; } } if(errors()) { - result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(in + pos, size - pos, utf16_output); + // rewind_and_convert_with_errors will seek a potential error from in+pos onward, + // with the ability to go back up to pos bytes, and read size-pos bytes forward. + result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(pos, in + pos, size - pos, utf16_output); res.count += pos; return res; } if(pos < size) { - result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(in + pos, size - pos, utf16_output); + // rewind_and_convert_with_errors will seek a potential error from in+pos onward, + // with the ability to go back up to pos bytes, and read size-pos bytes forward. + result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(pos, in + pos, size - pos, utf16_output); if (res.error) { // In case of error, we want the error position res.count += pos; return res; @@ -23880,7 +24099,7 @@ using namespace simd; this->check_utf8_bytes(input.chunks[3], input.chunks[2]); } if (errors()) { - result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(in + pos, size - pos, utf32_output); + result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(pos, in + pos, size - pos, utf32_output); res.count += pos; return res; } @@ -23914,12 +24133,12 @@ using namespace simd; } } if(errors()) { - result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(in + pos, size - pos, utf32_output); + result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(pos, in + pos, size - pos, utf32_output); res.count += pos; return res; } if(pos < size) { - result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(in + pos, size - pos, utf32_output); + result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(pos, in + pos, size - pos, utf32_output); if (res.error) { // In case of error, we want the error position res.count += pos; return res; @@ -24001,7 +24220,7 @@ simdutf_really_inline size_t count_code_points(const char16_t* in, size_t size) size_t count = 0; for(;pos + 32 <= size; pos += 32) { simd16x32<uint16_t> input(reinterpret_cast<const uint16_t *>(in + pos)); - if (big_endian) input.swap_bytes(); + if (!match_system(big_endian)) input.swap_bytes(); uint64_t not_pair = input.not_in_range(0xDC00, 0xDFFF); count += count_ones(not_pair) / 2; } @@ -24015,7 +24234,7 @@ simdutf_really_inline size_t utf8_length_from_utf16(const char16_t* in, size_t s // This algorithm could no doubt be improved! for(;pos + 32 <= size; pos += 32) { simd16x32<uint16_t> input(reinterpret_cast<const uint16_t *>(in + pos)); - if (big_endian) input.swap_bytes(); + if (!match_system(big_endian)) input.swap_bytes(); uint64_t ascii_mask = input.lteq(0x7F); uint64_t twobyte_mask = input.lteq(0x7FF); uint64_t not_pair_mask = input.not_in_range(0xD800, 0xDFFF); @@ -24281,7 +24500,7 @@ simdutf_warn_unused size_t implementation::utf16_length_from_utf32(const char32_ } simdutf_warn_unused size_t implementation::utf32_length_from_utf8(const char * input, size_t length) const noexcept { - return scalar::utf8::utf32_length_from_utf8(input, length); + return scalar::utf8::count_code_points(input, length); } } // namespace ppc64 @@ -24382,8 +24601,8 @@ int sse_detect_encodings(const char * buf, size_t len) { if (surrogates_bitmask0 != 0x0 || surrogates_bitmask1 != 0x0) { // Cannot be UTF8 is_utf8 = false; - // Can still be either UTF-16LE or UTF-32LE depending on the positions of the surrogates - // To be valid UTF-32LE, a surrogate cannot be in the two most significant bytes of any 32-bit word. + // Can still be either UTF-16LE or UTF-32 depending on the positions of the surrogates + // To be valid UTF-32, a surrogate cannot be in the two most significant bytes of any 32-bit word. // On the other hand, to be valid UTF-16LE, at least one surrogate must be in the two most significant // bytes of a 32-bit word since they always come in pairs in UTF-16LE. // Note that we always proceed in multiple of 4 before this point so there is no offset in 32-bit words. @@ -24459,7 +24678,7 @@ int sse_detect_encodings(const char * buf, size_t len) { } } else { is_utf16 = false; - // Check for UTF-32LE + // Check for UTF-32 if (len % 4 == 0) { const char32_t * input = reinterpret_cast<const char32_t*>(buf); const char32_t* end32 = reinterpret_cast<const char32_t*>(start) + len/4; @@ -24498,7 +24717,7 @@ int sse_detect_encodings(const char * buf, size_t len) { } // If no surrogate, validate under other encodings as well - // UTF-32LE validation + // UTF-32 validation currentmax = _mm_max_epu32(in, currentmax); currentmax = _mm_max_epu32(secondin, currentmax); currentmax = _mm_max_epu32(thirdin, currentmax); @@ -27152,7 +27371,9 @@ using namespace simd; this->check_utf8_bytes(input.chunks[3], input.chunks[2]); } if (errors()) { - result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(in + pos, size - pos, utf16_output); + // rewind_and_convert_with_errors will seek a potential error from in+pos onward, + // with the ability to go back up to pos bytes, and read size-pos bytes forward. + result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(pos, in + pos, size - pos, utf16_output); res.count += pos; return res; } @@ -27186,12 +27407,16 @@ using namespace simd; } } if(errors()) { - result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(in + pos, size - pos, utf16_output); + // rewind_and_convert_with_errors will seek a potential error from in+pos onward, + // with the ability to go back up to pos bytes, and read size-pos bytes forward. + result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(pos, in + pos, size - pos, utf16_output); res.count += pos; return res; } if(pos < size) { - result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(in + pos, size - pos, utf16_output); + // rewind_and_convert_with_errors will seek a potential error from in+pos onward, + // with the ability to go back up to pos bytes, and read size-pos bytes forward. + result res = scalar::utf8_to_utf16::rewind_and_convert_with_errors<endian>(pos, in + pos, size - pos, utf16_output); if (res.error) { // In case of error, we want the error position res.count += pos; return res; @@ -27475,7 +27700,7 @@ using namespace simd; this->check_utf8_bytes(input.chunks[3], input.chunks[2]); } if (errors()) { - result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(in + pos, size - pos, utf32_output); + result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(pos, in + pos, size - pos, utf32_output); res.count += pos; return res; } @@ -27509,12 +27734,12 @@ using namespace simd; } } if(errors()) { - result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(in + pos, size - pos, utf32_output); + result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(pos, in + pos, size - pos, utf32_output); res.count += pos; return res; } if(pos < size) { - result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(in + pos, size - pos, utf32_output); + result res = scalar::utf8_to_utf32::rewind_and_convert_with_errors(pos, in + pos, size - pos, utf32_output); if (res.error) { // In case of error, we want the error position res.count += pos; return res; @@ -27596,7 +27821,7 @@ simdutf_really_inline size_t count_code_points(const char16_t* in, size_t size) size_t count = 0; for(;pos + 32 <= size; pos += 32) { simd16x32<uint16_t> input(reinterpret_cast<const uint16_t *>(in + pos)); - if (big_endian) input.swap_bytes(); + if (!match_system(big_endian)) input.swap_bytes(); uint64_t not_pair = input.not_in_range(0xDC00, 0xDFFF); count += count_ones(not_pair) / 2; } @@ -27610,7 +27835,7 @@ simdutf_really_inline size_t utf8_length_from_utf16(const char16_t* in, size_t s // This algorithm could no doubt be improved! for(;pos + 32 <= size; pos += 32) { simd16x32<uint16_t> input(reinterpret_cast<const uint16_t *>(in + pos)); - if (big_endian) input.swap_bytes(); + if (!match_system(big_endian)) input.swap_bytes(); uint64_t ascii_mask = input.lteq(0x7F); uint64_t twobyte_mask = input.lteq(0x7FF); uint64_t not_pair_mask = input.not_in_range(0xD800, 0xDFFF); @@ -28109,7 +28334,7 @@ simdutf_warn_unused size_t implementation::utf16_length_from_utf32(const char32_ } simdutf_warn_unused size_t implementation::utf32_length_from_utf8(const char * input, size_t length) const noexcept { - return utf8::utf32_length_from_utf8(input, length); + return scalar::utf8::count_code_points(input, length); } } // namespace westmere diff --git a/src/bun.js/bindings/simdutf.h b/src/bun.js/bindings/simdutf.h index 3514f117d..0a57a69f7 100644 --- a/src/bun.js/bindings/simdutf.h +++ b/src/bun.js/bindings/simdutf.h @@ -1,4 +1,4 @@ -/* auto-generated on 2023-01-02 15:43:33 -0500. Do not edit! */ +/* auto-generated on 2023-02-10 14:42:58 -0500. Do not edit! */ // dofile: invoked with prepath=/Users/dlemire/CVS/github/simdutf/include, filename=simdutf.h /* begin file include/simdutf.h */ #ifndef SIMDUTF_H @@ -64,6 +64,48 @@ #include <strings.h> #endif +/** + * We want to check that it is actually a little endian system at + * compile-time. + */ + +#if defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) +#define SIMDUTF_IS_BIG_ENDIAN (__BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) +#elif defined(_WIN32) +#define SIMDUTF_IS_BIG_ENDIAN 0 +#else +#if defined(__APPLE__) || defined(__FreeBSD__) // defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__ +#include <machine/endian.h> +#elif defined(sun) || defined(__sun) // defined(__APPLE__) || defined(__FreeBSD__) +#include <sys/byteorder.h> +#else // defined(__APPLE__) || defined(__FreeBSD__) + +#ifdef __has_include +#if __has_include(<endian.h>) +#include <endian.h> +#endif //__has_include(<endian.h>) +#endif //__has_include + +#endif // defined(__APPLE__) || defined(__FreeBSD__) + + +#ifndef !defined(__BYTE_ORDER__) || !defined(__ORDER_LITTLE_ENDIAN__) +#define SIMDUTF_IS_BIG_ENDIAN 0 +#endif + +#if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define SIMDUTF_IS_BIG_ENDIAN 0 +#else // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ +#define SIMDUTF_IS_BIG_ENDIAN 1 +#endif // __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__ + +#endif // defined __BYTE_ORDER__ && defined __ORDER_BIG_ENDIAN__ + + +/** + * At this point in time, SIMDUTF_IS_BIG_ENDIAN is defined. + */ + #ifdef _MSC_VER #define SIMDUTF_VISUAL_STUDIO 1 /** @@ -98,8 +140,10 @@ #define SIMDUTF_IS_ARM64 1 #elif defined(__PPC64__) || defined(_M_PPC64) //#define SIMDUTF_IS_PPC64 1 -#pragma message("The simdutf library does yet support SIMD acceleration under\ -POWER processors. Please see https://github.com/lemire/simdutf/issues/51") +// The simdutf library does yet support SIMD acceleration under +// POWER processors. Please see https://github.com/lemire/simdutf/issues/51 +#elif defined(__s390__) +// s390 IBM system. Big endian. #else // The simdutf library is designed // for 64-bit processors and it seems that you are not @@ -456,6 +500,8 @@ enum endianness { BIG }; +bool match_system(endianness e); + std::string to_string(encoding_type bom); // Note that BOM for UTF8 is discouraged. @@ -492,7 +538,7 @@ enum error_code { HEADER_BITS, // Any byte must have fewer than 5 header bits. TOO_SHORT, // The leading byte must be followed by N-1 continuation bytes, where N is the UTF-8 character length // This is also the error when the input is truncated. - TOO_LONG, // The leading byte must not be a continuation byte. + TOO_LONG, // We either have too many consecutive continuation bytes or the string starts with a continuation byte. OVERLONG, // The decoded character must be above U+7F for two-byte characters, U+7FF for three-byte characters, // and U+FFFF for four-byte characters. TOO_LARGE, // The decoded character must be less than or equal to U+10FFFF OR less than or equal than U+7F for ASCII. @@ -526,18 +572,18 @@ SIMDUTF_DISABLE_UNDESIRED_WARNINGS #define SIMDUTF_SIMDUTF_VERSION_H /** The version of simdutf being used (major.minor.revision) */ -#define SIMDUTF_VERSION "2.1.0" +#define SIMDUTF_VERSION "3.2.0" namespace simdutf { enum { /** * The major version (MAJOR.minor.revision) of simdutf being used. */ - SIMDUTF_VERSION_MAJOR = 2, + SIMDUTF_VERSION_MAJOR = 3, /** * The minor version (major.MINOR.revision) of simdutf being used. */ - SIMDUTF_VERSION_MINOR = 1, + SIMDUTF_VERSION_MINOR = 2, /** * The revision (major.minor.REVISION) of simdutf being used. */ @@ -873,6 +919,21 @@ simdutf_warn_unused bool validate_ascii(const char *buf, size_t len) noexcept; simdutf_warn_unused result validate_ascii_with_errors(const char *buf, size_t len) noexcept; /** + * Using native endianness; Validate the UTF-16 string. + * This function may be best when you expect the input to be almost always valid. + * Otherwise, consider using validate_utf16_with_errors. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in number of 2-byte words (char16_t). + * @return true if and only if the string is valid UTF-16. + */ +simdutf_warn_unused bool validate_utf16(const char16_t *buf, size_t len) noexcept; + +/** * Validate the UTF-16LE string. This function may be best when you expect * the input to be almost always valid. Otherwise, consider using * validate_utf16le_with_errors. @@ -903,6 +964,20 @@ simdutf_warn_unused bool validate_utf16le(const char16_t *buf, size_t len) noexc simdutf_warn_unused bool validate_utf16be(const char16_t *buf, size_t len) noexcept; /** + * Using native endianness; Validate the UTF-16 string and stop on error. + * It might be faster than validate_utf16 when an error is expected to occur early. + * + * Overridden by each implementation. + * + * This function is not BOM-aware. + * + * @param buf the UTF-16 string to validate. + * @param len the length of the string in number of 2-byte words (char16_t). + * @return a result pair struct with an error code and either the position of the error if any or the number of words validated if successful. + */ +simdutf_warn_unused result validate_utf16_with_errors(const char16_t *buf, size_t len) noexcept; + +/** * Validate the UTF-16LE string and stop on error. It might be faster than * validate_utf16le when an error is expected to occur early. * @@ -931,7 +1006,7 @@ simdutf_warn_unused result validate_utf16le_with_errors(const char16_t *buf, siz simdutf_warn_unused result validate_utf16be_with_errors(const char16_t *buf, size_t len) noexcept; /** - * Validate the UTF-32LE string. This function may be best when you expect + * Validate the UTF-32 string. This function may be best when you expect * the input to be almost always valid. Otherwise, consider using * validate_utf32_with_errors. * @@ -939,27 +1014,40 @@ simdutf_warn_unused result validate_utf16be_with_errors(const char16_t *buf, siz * * This function is not BOM-aware. * - * @param buf the UTF-32LE string to validate. + * @param buf the UTF-32 string to validate. * @param len the length of the string in number of 4-byte words (char32_t). - * @return true if and only if the string is valid UTF-32LE. + * @return true if and only if the string is valid UTF-32. */ simdutf_warn_unused bool validate_utf32(const char32_t *buf, size_t len) noexcept; /** - * Validate the UTF-32LE string and stop on error. It might be faster than + * Validate the UTF-32 string and stop on error. It might be faster than * validate_utf32 when an error is expected to occur early. * * Overridden by each implementation. * * This function is not BOM-aware. * - * @param buf the UTF-32LE string to validate. + * @param buf the UTF-32 string to validate. * @param len the length of the string in number of 4-byte words (char32_t). * @return a result pair struct with an error code and either the position of the error if any or the number of words validated if successful. */ simdutf_warn_unused result validate_utf32_with_errors(const char32_t *buf, size_t len) noexcept; /** + * Using native endianness; Convert possibly broken UTF-8 string into UTF-16 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t; 0 if the input was not valid UTF-8 string + */ +simdutf_warn_unused size_t convert_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_output) noexcept; + +/** * Convert possibly broken UTF-8 string into UTF-16LE string. * * During the conversion also validation of the input string is done. @@ -986,6 +1074,20 @@ simdutf_warn_unused size_t convert_utf8_to_utf16le(const char * input, size_t le simdutf_warn_unused size_t convert_utf8_to_utf16be(const char * input, size_t length, char16_t* utf16_output) noexcept; /** + * Using native endianness; Convert possibly broken UTF-8 string into UTF-16 + * string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct with an error code and either the position of the error if any or the number of char16_t written if successful. + */ +simdutf_warn_unused result convert_utf8_to_utf16_with_errors(const char * input, size_t length, char16_t* utf16_output) noexcept; + +/** * Convert possibly broken UTF-8 string into UTF-16LE string and stop on error. * * During the conversion also validation of the input string is done. @@ -1012,7 +1114,7 @@ simdutf_warn_unused result convert_utf8_to_utf16le_with_errors(const char * inpu simdutf_warn_unused result convert_utf8_to_utf16be_with_errors(const char * input, size_t length, char16_t* utf16_output) noexcept; /** - * Convert possibly broken UTF-8 string into UTF-32LE string. + * Convert possibly broken UTF-8 string into UTF-32 string. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. @@ -1025,7 +1127,7 @@ simdutf_warn_unused result convert_utf8_to_utf16be_with_errors(const char * inpu simdutf_warn_unused size_t convert_utf8_to_utf32(const char * input, size_t length, char32_t* utf32_output) noexcept; /** - * Convert possibly broken UTF-8 string into UTF-32LE string and stop on error. + * Convert possibly broken UTF-8 string into UTF-32 string and stop on error. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. @@ -1038,6 +1140,18 @@ simdutf_warn_unused size_t convert_utf8_to_utf32(const char * input, size_t leng simdutf_warn_unused result convert_utf8_to_utf32_with_errors(const char * input, size_t length, char32_t* utf32_output) noexcept; /** + * Using native endianness; Convert valid UTF-8 string into UTF-16 string. + * + * This function assumes that the input string is valid UTF-8. + * + * @param input the UTF-8 string to convert + * @param length the length of the string in bytes + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return the number of written char16_t + */ +simdutf_warn_unused size_t convert_valid_utf8_to_utf16(const char * input, size_t length, char16_t* utf16_buffer) noexcept; + +/** * Convert valid UTF-8 string into UTF-16LE string. * * This function assumes that the input string is valid UTF-8. @@ -1062,7 +1176,7 @@ simdutf_warn_unused size_t convert_valid_utf8_to_utf16le(const char * input, siz simdutf_warn_unused size_t convert_valid_utf8_to_utf16be(const char * input, size_t length, char16_t* utf16_buffer) noexcept; /** - * Convert valid UTF-8 string into UTF-32LE string. + * Convert valid UTF-8 string into UTF-32 string. * * This function assumes that the input string is valid UTF-8. * @@ -1087,7 +1201,7 @@ simdutf_warn_unused size_t convert_valid_utf8_to_utf32(const char * input, size_ simdutf_warn_unused size_t utf16_length_from_utf8(const char * input, size_t length) noexcept; /** - * Compute the number of 4-byte words that this UTF-8 string would require in UTF-32LE format. + * Compute the number of 4-byte words that this UTF-8 string would require in UTF-32 format. * * This function is equivalent to count_utf8 * @@ -1097,11 +1211,26 @@ simdutf_warn_unused size_t utf16_length_from_utf8(const char * input, size_t len * * @param input the UTF-8 string to process * @param length the length of the string in bytes - * @return the number of char32_t words required to encode the UTF-8 string as UTF-32LE + * @return the number of char32_t words required to encode the UTF-8 string as UTF-32 */ simdutf_warn_unused size_t utf32_length_from_utf8(const char * input, size_t length) noexcept; /** + * Using native endianness; Convert possibly broken UTF-16 string into UTF-8 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return number of written words; 0 if input is not a valid UTF-16LE string + */ +simdutf_warn_unused size_t convert_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept; + +/** * Convert possibly broken UTF-16LE string into UTF-8 string. * * During the conversion also validation of the input string is done. @@ -1132,6 +1261,21 @@ simdutf_warn_unused size_t convert_utf16le_to_utf8(const char16_t * input, size_ simdutf_warn_unused size_t convert_utf16be_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept; /** + * Using native endianness; Convert possibly broken UTF-16 string into UTF-8 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @param utf8_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct with an error code and either the position of the error if any or the number of char written if successful. + */ +simdutf_warn_unused result convert_utf16_to_utf8_with_errors(const char16_t * input, size_t length, char* utf8_buffer) noexcept; + +/** * Convert possibly broken UTF-16LE string into UTF-8 string and stop on error. * * During the conversion also validation of the input string is done. @@ -1162,6 +1306,20 @@ simdutf_warn_unused result convert_utf16le_to_utf8_with_errors(const char16_t * simdutf_warn_unused result convert_utf16be_to_utf8_with_errors(const char16_t * input, size_t length, char* utf8_buffer) noexcept; /** + * Using native endianness; Convert valid UTF-16 string into UTF-8 string. + * + * This function assumes that the input string is valid UTF-16LE. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @param utf8_buffer the pointer to buffer that can hold the conversion result + * @return number of written words; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept; + +/** * Convert valid UTF-16LE string into UTF-8 string. * * This function assumes that the input string is valid UTF-16LE. @@ -1190,7 +1348,22 @@ simdutf_warn_unused size_t convert_valid_utf16le_to_utf8(const char16_t * input, simdutf_warn_unused size_t convert_valid_utf16be_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) noexcept; /** - * Convert possibly broken UTF-16LE string into UTF-32LE string. + * Using native endianness; Convert possibly broken UTF-16 string into UTF-32 string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return number of written words; 0 if input is not a valid UTF-16LE string + */ +simdutf_warn_unused size_t convert_utf16_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept; + +/** + * Convert possibly broken UTF-16LE string into UTF-32 string. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. @@ -1205,7 +1378,7 @@ simdutf_warn_unused size_t convert_valid_utf16be_to_utf8(const char16_t * input, simdutf_warn_unused size_t convert_utf16le_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept; /** - * Convert possibly broken UTF-16BE string into UTF-32LE string. + * Convert possibly broken UTF-16BE string into UTF-32 string. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. @@ -1220,7 +1393,23 @@ simdutf_warn_unused size_t convert_utf16le_to_utf32(const char16_t * input, size simdutf_warn_unused size_t convert_utf16be_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept; /** - * Convert possibly broken UTF-16LE string into UTF-32LE string and stop on error. + * Using native endianness; Convert possibly broken UTF-16 string into + * UTF-32 string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @param utf32_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct with an error code and either the position of the error if any or the number of char32_t written if successful. + */ +simdutf_warn_unused result convert_utf16_to_utf32_with_errors(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept; + +/** + * Convert possibly broken UTF-16LE string into UTF-32 string and stop on error. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. @@ -1235,7 +1424,7 @@ simdutf_warn_unused size_t convert_utf16be_to_utf32(const char16_t * input, size simdutf_warn_unused result convert_utf16le_to_utf32_with_errors(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept; /** - * Convert possibly broken UTF-16BE string into UTF-32LE string and stop on error. + * Convert possibly broken UTF-16BE string into UTF-32 string and stop on error. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. @@ -1250,7 +1439,21 @@ simdutf_warn_unused result convert_utf16le_to_utf32_with_errors(const char16_t * simdutf_warn_unused result convert_utf16be_to_utf32_with_errors(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept; /** - * Convert valid UTF-16LE string into UTF-32LE string. + * Using native endianness; Convert valid UTF-16 string into UTF-32 string. + * + * This function assumes that the input string is valid UTF-16 (native endianness). + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @param utf32_buffer the pointer to buffer that can hold the conversion result + * @return number of written words; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf16_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept; + +/** + * Convert valid UTF-16LE string into UTF-32 string. * * This function assumes that the input string is valid UTF-16LE. * @@ -1264,7 +1467,7 @@ simdutf_warn_unused result convert_utf16be_to_utf32_with_errors(const char16_t * simdutf_warn_unused size_t convert_valid_utf16le_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept; /** - * Convert valid UTF-16BE string into UTF-32LE string. + * Convert valid UTF-16BE string into UTF-32 string. * * This function assumes that the input string is valid UTF-16LE. * @@ -1278,6 +1481,18 @@ simdutf_warn_unused size_t convert_valid_utf16le_to_utf32(const char16_t * input simdutf_warn_unused size_t convert_valid_utf16be_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) noexcept; /** + * Using native endianness; Compute the number of bytes that this UTF-16 + * string would require in UTF-8 format. + * + * This function does not validate the input. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-8 + */ +simdutf_warn_unused size_t utf8_length_from_utf16(const char16_t * input, size_t length) noexcept; + +/** * Compute the number of bytes that this UTF-16LE string would require in UTF-8 format. * * This function does not validate the input. @@ -1300,29 +1515,29 @@ simdutf_warn_unused size_t utf8_length_from_utf16le(const char16_t * input, size simdutf_warn_unused size_t utf8_length_from_utf16be(const char16_t * input, size_t length) noexcept; /** - * Convert possibly broken UTF-32LE string into UTF-8 string. + * Convert possibly broken UTF-32 string into UTF-8 string. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf8_buffer the pointer to buffer that can hold conversion result - * @return number of written words; 0 if input is not a valid UTF-32LE string + * @return number of written words; 0 if input is not a valid UTF-32 string */ simdutf_warn_unused size_t convert_utf32_to_utf8(const char32_t * input, size_t length, char* utf8_buffer) noexcept; /** - * Convert possibly broken UTF-32LE string into UTF-8 string and stop on error. + * Convert possibly broken UTF-32 string into UTF-8 string and stop on error. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf8_buffer the pointer to buffer that can hold conversion result * @return a result pair struct with an error code and either the position of the error if any or the number of char written if successful. @@ -1330,13 +1545,13 @@ simdutf_warn_unused size_t convert_utf32_to_utf8(const char32_t * input, size_t simdutf_warn_unused result convert_utf32_to_utf8_with_errors(const char32_t * input, size_t length, char* utf8_buffer) noexcept; /** - * Convert valid UTF-32LE string into UTF-8 string. + * Convert valid UTF-32 string into UTF-8 string. * - * This function assumes that the input string is valid UTF-32LE. + * This function assumes that the input string is valid UTF-32. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf8_buffer the pointer to buffer that can hold the conversion result * @return number of written words; 0 if conversion is not possible @@ -1344,44 +1559,75 @@ simdutf_warn_unused result convert_utf32_to_utf8_with_errors(const char32_t * in simdutf_warn_unused size_t convert_valid_utf32_to_utf8(const char32_t * input, size_t length, char* utf8_buffer) noexcept; /** - * Convert possibly broken UTF-32LE string into UTF-16LE string. + * Using native endianness; Convert possibly broken UTF-32 string into UTF-16 string. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf16_buffer the pointer to buffer that can hold conversion result - * @return number of written words; 0 if input is not a valid UTF-32LE string + * @return number of written words; 0 if input is not a valid UTF-32 string + */ +simdutf_warn_unused size_t convert_utf32_to_utf16(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept; + +/** + * Convert possibly broken UTF-32 string into UTF-16LE string. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte words (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return number of written words; 0 if input is not a valid UTF-32 string */ simdutf_warn_unused size_t convert_utf32_to_utf16le(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept; /** - * Convert possibly broken UTF-32LE string into UTF-16BE string. + * Convert possibly broken UTF-32 string into UTF-16BE string. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf16_buffer the pointer to buffer that can hold conversion result - * @return number of written words; 0 if input is not a valid UTF-32LE string + * @return number of written words; 0 if input is not a valid UTF-32 string */ simdutf_warn_unused size_t convert_utf32_to_utf16be(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept; /** - * Convert possibly broken UTF-32LE string into UTF-16LE string and stop on error. + * Using native endianness; Convert possibly broken UTF-32 string into UTF-16 + * string and stop on error. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte words (char32_t) + * @param utf16_buffer the pointer to buffer that can hold conversion result + * @return a result pair struct with an error code and either the position of the error if any or the number of char16_t written if successful. + */ +simdutf_warn_unused result convert_utf32_to_utf16_with_errors(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept; + +/** + * Convert possibly broken UTF-32 string into UTF-16LE string and stop on error. + * + * During the conversion also validation of the input string is done. + * This function is suitable to work with inputs from untrusted sources. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf16_buffer the pointer to buffer that can hold conversion result * @return a result pair struct with an error code and either the position of the error if any or the number of char16_t written if successful. @@ -1389,14 +1635,14 @@ simdutf_warn_unused size_t convert_utf32_to_utf16be(const char32_t * input, size simdutf_warn_unused result convert_utf32_to_utf16le_with_errors(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept; /** - * Convert possibly broken UTF-32LE string into UTF-16BE string and stop on error. + * Convert possibly broken UTF-32 string into UTF-16BE string and stop on error. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf16_buffer the pointer to buffer that can hold conversion result * @return a result pair struct with an error code and either the position of the error if any or the number of char16_t written if successful. @@ -1404,13 +1650,27 @@ simdutf_warn_unused result convert_utf32_to_utf16le_with_errors(const char32_t * simdutf_warn_unused result convert_utf32_to_utf16be_with_errors(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept; /** - * Convert valid UTF-32LE string into UTF-16LE string. + * Using native endianness; Convert valid UTF-32 string into UTF-16 string. + * + * This function assumes that the input string is valid UTF-32. + * + * This function is not BOM-aware. + * + * @param input the UTF-32 string to convert + * @param length the length of the string in 4-byte words (char32_t) + * @param utf16_buffer the pointer to buffer that can hold the conversion result + * @return number of written words; 0 if conversion is not possible + */ +simdutf_warn_unused size_t convert_valid_utf32_to_utf16(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept; + +/** + * Convert valid UTF-32 string into UTF-16LE string. * - * This function assumes that the input string is valid UTF-32LE. + * This function assumes that the input string is valid UTF-32. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf16_buffer the pointer to buffer that can hold the conversion result * @return number of written words; 0 if conversion is not possible @@ -1418,13 +1678,13 @@ simdutf_warn_unused result convert_utf32_to_utf16be_with_errors(const char32_t * simdutf_warn_unused size_t convert_valid_utf32_to_utf16le(const char32_t * input, size_t length, char16_t* utf16_buffer) noexcept; /** - * Convert valid UTF-32LE string into UTF-16BE string. + * Convert valid UTF-32 string into UTF-16BE string. * - * This function assumes that the input string is valid UTF-32LE. + * This function assumes that the input string is valid UTF-32. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf16_buffer the pointer to buffer that can hold the conversion result * @return number of written words; 0 if conversion is not possible @@ -1446,29 +1706,45 @@ simdutf_warn_unused size_t convert_valid_utf32_to_utf16be(const char32_t * input void change_endianness_utf16(const char16_t * input, size_t length, char16_t * output) noexcept; /** - * Compute the number of bytes that this UTF-32LE string would require in UTF-8 format. + * Compute the number of bytes that this UTF-32 string would require in UTF-8 format. * * This function does not validate the input. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) - * @return the number of bytes required to encode the UTF-32LE string as UTF-8 + * @return the number of bytes required to encode the UTF-32 string as UTF-8 */ simdutf_warn_unused size_t utf8_length_from_utf32(const char32_t * input, size_t length) noexcept; /** - * Compute the number of two-byte words that this UTF-32LE string would require in UTF-16 format. + * Compute the number of two-byte words that this UTF-32 string would require in UTF-16 format. * * This function does not validate the input. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) - * @return the number of bytes required to encode the UTF-32LE string as UTF-16 + * @return the number of bytes required to encode the UTF-32 string as UTF-16 */ simdutf_warn_unused size_t utf16_length_from_utf32(const char32_t * input, size_t length) noexcept; /** - * Compute the number of bytes that this UTF-16LE string would require in UTF-32LE format. + * Using native endianness; Compute the number of bytes that this UTF-16 + * string would require in UTF-32 format. + * + * This function is equivalent to count_utf16. + * + * This function does not validate the input. + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to convert + * @param length the length of the string in 2-byte words (char16_t) + * @return the number of bytes required to encode the UTF-16LE string as UTF-32 + */ +simdutf_warn_unused size_t utf32_length_from_utf16(const char16_t * input, size_t length) noexcept; + +/** + * Compute the number of bytes that this UTF-16LE string would require in UTF-32 format. * * This function is equivalent to count_utf16le. * @@ -1478,12 +1754,12 @@ simdutf_warn_unused size_t utf16_length_from_utf32(const char32_t * input, size_ * * @param input the UTF-16LE string to convert * @param length the length of the string in 2-byte words (char16_t) - * @return the number of bytes required to encode the UTF-16LE string as UTF-32LE + * @return the number of bytes required to encode the UTF-16LE string as UTF-32 */ simdutf_warn_unused size_t utf32_length_from_utf16le(const char16_t * input, size_t length) noexcept; /** - * Compute the number of bytes that this UTF-16BE string would require in UTF-32LE format. + * Compute the number of bytes that this UTF-16BE string would require in UTF-32 format. * * This function is equivalent to count_utf16be. * @@ -1493,7 +1769,7 @@ simdutf_warn_unused size_t utf32_length_from_utf16le(const char16_t * input, siz * * @param input the UTF-16BE string to convert * @param length the length of the string in 2-byte words (char16_t) - * @return the number of bytes required to encode the UTF-16BE string as UTF-32LE + * @return the number of bytes required to encode the UTF-16BE string as UTF-32 */ simdutf_warn_unused size_t utf32_length_from_utf16be(const char16_t * input, size_t length) noexcept; @@ -1501,6 +1777,20 @@ simdutf_warn_unused size_t utf32_length_from_utf16be(const char16_t * input, siz * Count the number of code points (characters) in the string assuming that * it is valid. * + * This function assumes that the input string is valid UTF-16 (native endianness). + * + * This function is not BOM-aware. + * + * @param input the UTF-16 string to process + * @param length the length of the string in 2-byte words (char16_t) + * @return number of code points + */ +simdutf_warn_unused size_t count_utf16(const char16_t * input, size_t length) noexcept; + +/** + * Count the number of code points (characters) in the string assuming that + * it is valid. + * * This function assumes that the input string is valid UTF-16LE. * * This function is not BOM-aware. @@ -1599,7 +1889,7 @@ public: * * @return a mask of all required `internal::instruction_set::` values */ - virtual uint32_t required_instruction_sets() const { return _required_instruction_sets; }; + virtual uint32_t required_instruction_sets() const { return _required_instruction_sets; } /** @@ -1705,26 +1995,26 @@ public: simdutf_warn_unused virtual result validate_utf16be_with_errors(const char16_t *buf, size_t len) const noexcept = 0; /** - * Validate the UTF-32LE string. + * Validate the UTF-32 string. * * Overridden by each implementation. * * This function is not BOM-aware. * - * @param buf the UTF-32LE string to validate. + * @param buf the UTF-32 string to validate. * @param len the length of the string in number of 4-byte words (char32_t). - * @return true if and only if the string is valid UTF-32LE. + * @return true if and only if the string is valid UTF-32. */ simdutf_warn_unused virtual bool validate_utf32(const char32_t *buf, size_t len) const noexcept = 0; /** - * Validate the UTF-32LE string and stop on error. + * Validate the UTF-32 string and stop on error. * * Overridden by each implementation. * * This function is not BOM-aware. * - * @param buf the UTF-32LE string to validate. + * @param buf the UTF-32 string to validate. * @param len the length of the string in number of 4-byte words (char32_t). * @return a result pair struct with an error code and either the position of the error if any or the number of words validated if successful. */ @@ -1783,7 +2073,7 @@ public: simdutf_warn_unused virtual result convert_utf8_to_utf16be_with_errors(const char * input, size_t length, char16_t* utf16_output) const noexcept = 0; /** - * Convert possibly broken UTF-8 string into UTF-32LE string. + * Convert possibly broken UTF-8 string into UTF-32 string. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. @@ -1796,7 +2086,7 @@ public: simdutf_warn_unused virtual size_t convert_utf8_to_utf32(const char * input, size_t length, char32_t* utf32_output) const noexcept = 0; /** - * Convert possibly broken UTF-8 string into UTF-32LE string and stop on error. + * Convert possibly broken UTF-8 string into UTF-32 string and stop on error. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. @@ -1833,7 +2123,7 @@ public: simdutf_warn_unused virtual size_t convert_valid_utf8_to_utf16be(const char * input, size_t length, char16_t* utf16_buffer) const noexcept = 0; /** - * Convert valid UTF-8 string into UTF-32LE string. + * Convert valid UTF-8 string into UTF-32 string. * * This function assumes that the input string is valid UTF-8. * @@ -1856,7 +2146,7 @@ public: simdutf_warn_unused virtual size_t utf16_length_from_utf8(const char * input, size_t length) const noexcept = 0; /** - * Compute the number of 4-byte words that this UTF-8 string would require in UTF-32LE format. + * Compute the number of 4-byte words that this UTF-8 string would require in UTF-32 format. * * This function is equivalent to count_utf8. * @@ -1864,7 +2154,7 @@ public: * * @param input the UTF-8 string to process * @param length the length of the string in bytes - * @return the number of char32_t words required to encode the UTF-8 string as UTF-32LE + * @return the number of char32_t words required to encode the UTF-8 string as UTF-32 */ simdutf_warn_unused virtual size_t utf32_length_from_utf8(const char * input, size_t length) const noexcept = 0; @@ -1957,7 +2247,7 @@ public: simdutf_warn_unused virtual size_t convert_valid_utf16be_to_utf8(const char16_t * input, size_t length, char* utf8_buffer) const noexcept = 0; /** - * Convert possibly broken UTF-16LE string into UTF-32LE string. + * Convert possibly broken UTF-16LE string into UTF-32 string. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. @@ -1972,7 +2262,7 @@ public: simdutf_warn_unused virtual size_t convert_utf16le_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) const noexcept = 0; /** - * Convert possibly broken UTF-16BE string into UTF-32LE string. + * Convert possibly broken UTF-16BE string into UTF-32 string. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. @@ -1987,7 +2277,7 @@ public: simdutf_warn_unused virtual size_t convert_utf16be_to_utf32(const char16_t * input, size_t length, char32_t* utf32_buffer) const noexcept = 0; /** - * Convert possibly broken UTF-16LE string into UTF-32LE string and stop on error. + * Convert possibly broken UTF-16LE string into UTF-32 string and stop on error. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. @@ -2002,7 +2292,7 @@ public: simdutf_warn_unused virtual result convert_utf16le_to_utf32_with_errors(const char16_t * input, size_t length, char32_t* utf32_buffer) const noexcept = 0; /** - * Convert possibly broken UTF-16BE string into UTF-32LE string and stop on error. + * Convert possibly broken UTF-16BE string into UTF-32 string and stop on error. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. @@ -2017,7 +2307,7 @@ public: simdutf_warn_unused virtual result convert_utf16be_to_utf32_with_errors(const char16_t * input, size_t length, char32_t* utf32_buffer) const noexcept = 0; /** - * Convert valid UTF-16LE string into UTF-32LE string. + * Convert valid UTF-16LE string into UTF-32 string. * * This function assumes that the input string is valid UTF-16LE. * @@ -2071,29 +2361,29 @@ public: simdutf_warn_unused virtual size_t utf8_length_from_utf16be(const char16_t * input, size_t length) const noexcept = 0; /** - * Convert possibly broken UTF-32LE string into UTF-8 string. + * Convert possibly broken UTF-32 string into UTF-8 string. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf8_buffer the pointer to buffer that can hold conversion result - * @return number of written words; 0 if input is not a valid UTF-32LE string + * @return number of written words; 0 if input is not a valid UTF-32 string */ simdutf_warn_unused virtual size_t convert_utf32_to_utf8(const char32_t * input, size_t length, char* utf8_buffer) const noexcept = 0; /** - * Convert possibly broken UTF-32LE string into UTF-8 string and stop on error. + * Convert possibly broken UTF-32 string into UTF-8 string and stop on error. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf8_buffer the pointer to buffer that can hold conversion result * @return a result pair struct with an error code and either the position of the error if any or the number of char written if successful. @@ -2101,13 +2391,13 @@ public: simdutf_warn_unused virtual result convert_utf32_to_utf8_with_errors(const char32_t * input, size_t length, char* utf8_buffer) const noexcept = 0; /** - * Convert valid UTF-32LE string into UTF-8 string. + * Convert valid UTF-32 string into UTF-8 string. * - * This function assumes that the input string is valid UTF-32LE. + * This function assumes that the input string is valid UTF-32. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf8_buffer the pointer to buffer that can hold the conversion result * @return number of written words; 0 if conversion is not possible @@ -2115,44 +2405,44 @@ public: simdutf_warn_unused virtual size_t convert_valid_utf32_to_utf8(const char32_t * input, size_t length, char* utf8_buffer) const noexcept = 0; /** - * Convert possibly broken UTF-32LE string into UTF-16LE string. + * Convert possibly broken UTF-32 string into UTF-16LE string. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf16_buffer the pointer to buffer that can hold conversion result - * @return number of written words; 0 if input is not a valid UTF-32LE string + * @return number of written words; 0 if input is not a valid UTF-32 string */ simdutf_warn_unused virtual size_t convert_utf32_to_utf16le(const char32_t * input, size_t length, char16_t* utf16_buffer) const noexcept = 0; /** - * Convert possibly broken UTF-32LE string into UTF-16BE string. + * Convert possibly broken UTF-32 string into UTF-16BE string. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf16_buffer the pointer to buffer that can hold conversion result - * @return number of written words; 0 if input is not a valid UTF-32LE string + * @return number of written words; 0 if input is not a valid UTF-32 string */ simdutf_warn_unused virtual size_t convert_utf32_to_utf16be(const char32_t * input, size_t length, char16_t* utf16_buffer) const noexcept = 0; /** - * Convert possibly broken UTF-32LE string into UTF-16LE string and stop on error. + * Convert possibly broken UTF-32 string into UTF-16LE string and stop on error. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf16_buffer the pointer to buffer that can hold conversion result * @return a result pair struct with an error code and either the position of the error if any or the number of char16_t written if successful. @@ -2160,14 +2450,14 @@ public: simdutf_warn_unused virtual result convert_utf32_to_utf16le_with_errors(const char32_t * input, size_t length, char16_t* utf16_buffer) const noexcept = 0; /** - * Convert possibly broken UTF-32LE string into UTF-16BE string and stop on error. + * Convert possibly broken UTF-32 string into UTF-16BE string and stop on error. * * During the conversion also validation of the input string is done. * This function is suitable to work with inputs from untrusted sources. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf16_buffer the pointer to buffer that can hold conversion result * @return a result pair struct with an error code and either the position of the error if any or the number of char16_t written if successful. @@ -2175,13 +2465,13 @@ public: simdutf_warn_unused virtual result convert_utf32_to_utf16be_with_errors(const char32_t * input, size_t length, char16_t* utf16_buffer) const noexcept = 0; /** - * Convert valid UTF-32LE string into UTF-16LE string. + * Convert valid UTF-32 string into UTF-16LE string. * - * This function assumes that the input string is valid UTF-32LE. + * This function assumes that the input string is valid UTF-32. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf16_buffer the pointer to buffer that can hold the conversion result * @return number of written words; 0 if conversion is not possible @@ -2189,13 +2479,13 @@ public: simdutf_warn_unused virtual size_t convert_valid_utf32_to_utf16le(const char32_t * input, size_t length, char16_t* utf16_buffer) const noexcept = 0; /** - * Convert valid UTF-32LE string into UTF-16BE string. + * Convert valid UTF-32 string into UTF-16BE string. * - * This function assumes that the input string is valid UTF-32LE. + * This function assumes that the input string is valid UTF-32. * * This function is not BOM-aware. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) * @param utf16_buffer the pointer to buffer that can hold the conversion result * @return number of written words; 0 if conversion is not possible @@ -2217,29 +2507,29 @@ public: virtual void change_endianness_utf16(const char16_t * input, size_t length, char16_t * output) const noexcept = 0; /** - * Compute the number of bytes that this UTF-32LE string would require in UTF-8 format. + * Compute the number of bytes that this UTF-32 string would require in UTF-8 format. * * This function does not validate the input. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) - * @return the number of bytes required to encode the UTF-32LE string as UTF-8 + * @return the number of bytes required to encode the UTF-32 string as UTF-8 */ simdutf_warn_unused virtual size_t utf8_length_from_utf32(const char32_t * input, size_t length) const noexcept = 0; /** - * Compute the number of two-byte words that this UTF-32LE string would require in UTF-16 format. + * Compute the number of two-byte words that this UTF-32 string would require in UTF-16 format. * * This function does not validate the input. * - * @param input the UTF-32LE string to convert + * @param input the UTF-32 string to convert * @param length the length of the string in 4-byte words (char32_t) - * @return the number of bytes required to encode the UTF-32LE string as UTF-16 + * @return the number of bytes required to encode the UTF-32 string as UTF-16 */ simdutf_warn_unused virtual size_t utf16_length_from_utf32(const char32_t * input, size_t length) const noexcept = 0; /* - * Compute the number of bytes that this UTF-16LE string would require in UTF-32LE format. + * Compute the number of bytes that this UTF-16LE string would require in UTF-32 format. * * This function is equivalent to count_utf16le. * @@ -2249,12 +2539,12 @@ public: * * @param input the UTF-16LE string to convert * @param length the length of the string in 2-byte words (char16_t) - * @return the number of bytes required to encode the UTF-16LE string as UTF-32LE + * @return the number of bytes required to encode the UTF-16LE string as UTF-32 */ simdutf_warn_unused virtual size_t utf32_length_from_utf16le(const char16_t * input, size_t length) const noexcept = 0; /* - * Compute the number of bytes that this UTF-16BE string would require in UTF-32LE format. + * Compute the number of bytes that this UTF-16BE string would require in UTF-32 format. * * This function is equivalent to count_utf16be. * @@ -2264,7 +2554,7 @@ public: * * @param input the UTF-16BE string to convert * @param length the length of the string in 2-byte words (char16_t) - * @return the number of bytes required to encode the UTF-16BE string as UTF-32LE + * @return the number of bytes required to encode the UTF-16BE string as UTF-32 */ simdutf_warn_unused virtual size_t utf32_length_from_utf16be(const char16_t * input, size_t length) const noexcept = 0; @@ -2429,19 +2719,22 @@ private: #endif }; +class detect_best_supported_implementation_on_first_use; + } // namespace internal /** * The list of available implementations compiled into simdutf. */ -extern SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list available_implementations; +extern SIMDUTF_DLLIMPORTEXPORT const internal::available_implementation_list& get_available_implementations(); /** * The active implementation. * * Automatically initialized on first use to the most advanced implementation supported by this hardware. */ -extern SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr<const implementation> active_implementation; +extern SIMDUTF_DLLIMPORTEXPORT internal::atomic_ptr<const implementation>& get_active_implementation(); + } // namespace simdutf |