diff --git a/benchmark/string_tr.yml b/benchmark/string_tr.yml index e0f582c240c17b..63805213848d46 100644 --- a/benchmark/string_tr.yml +++ b/benchmark/string_tr.yml @@ -6,9 +6,11 @@ prelude: | STR = ((("a" * 31) + "<") * 1000).freeze ESCAPED_CHARS = { - ">" => '\u003e', - "<" => '\u003c', - "&" => '\u0026', + "\\" => "'", + "\"" => """, + ">" => '>', + "<" => '<', + "&" => '&', }.freeze ESCAPE_PATTERN = Regexp.union(ESCAPED_CHARS.keys) diff --git a/internal/simd.h b/internal/simd.h new file mode 100644 index 00000000000000..44478c03e81f3f --- /dev/null +++ b/internal/simd.h @@ -0,0 +1,18 @@ +#ifndef INTERNAL_SIMD_H +#define INTERNAL_SIMD_H + +#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) +#ifdef HAVE_X86INTRIN_H +#include +#define HAVE_SIMD 1 +#define HAVE_SIMD_SSE2 1 +#endif +#endif + +#if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(__aarch64__) || defined(_M_ARM64) +#define HAVE_SIMD 1 +#define HAVE_SIMD_NEON 1 +#include +#endif + +#endif /* INTERNAL_SIMD_H */ diff --git a/string.c b/string.c index 32c113dc82e99e..9b5667f3a26813 100644 --- a/string.c +++ b/string.c @@ -39,6 +39,7 @@ #include "internal/proc.h" #include "internal/re.h" #include "internal/sanitizers.h" +#include "internal/simd.h" #include "internal/string.h" #include "internal/transcode.h" #include "probes.h" @@ -9211,7 +9212,7 @@ tr_buffer_ensure_capa(struct tr_buffer *buffer, size_t extra_capa) size_t new_capa = buffer->capa ? buffer->capa : buffer->initial_capa; RUBY_ASSERT(new_capa >= 32); // Lower would cause infinite loop while (new_capa < required_capa) { - new_capa *= 1.2; + new_capa = (size_t)(new_capa * 1.2); } SIZED_REALLOC_N(buffer->buf, unsigned char, new_capa, buffer->capa); buffer->ptr = buffer->buf + offset; @@ -9220,7 +9221,7 @@ tr_buffer_ensure_capa(struct tr_buffer *buffer, size_t extra_capa) } static inline void -tr_buffer_append(struct tr_buffer *buffer, unsigned char *ptr, size_t len) +tr_buffer_append(struct tr_buffer *buffer, const unsigned char *ptr, size_t len) { if (len) { tr_buffer_ensure_capa(buffer, len); @@ -9286,6 +9287,164 @@ tr_trans_pairs_coerce_i(st_data_t key, st_data_t value, st_data_t _args) return ST_CONTINUE; } +#define TR_TRANS_PAIRS_SIMD_MAX_NEEDLES 16 + +struct tr_trans_pairs_search { + const unsigned char *s; + const unsigned char *send; + +#ifdef HAVE_SIMD + unsigned char needles[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES]; + int needles_count; +#ifdef HAVE_SIMD_NEON + uint64_t matches_bitmap; +#endif +#ifdef HAVE_SIMD_SSE2 + int matches_bitmap; +#endif +#endif + + VALUE trans_table[256]; +}; + +static inline VALUE +tr_trans_pairs_search_basic(struct tr_trans_pairs_search *search) +{ + while (search->s < search->send) { + VALUE repl = search->trans_table[*search->s]; + if (UNLIKELY(repl)) { + return repl; + } + + search->s++; + } + + return 0; +} + +#ifdef HAVE_SIMD_SSE2 +static inline bool +tr_trans_pairs_next_match_sse2(struct tr_trans_pairs_search *search) +{ + size_t next_match_offset = ntz_int32(search->matches_bitmap); + search->matches_bitmap >>= (next_match_offset + 1); + search->s += next_match_offset; + if (search->s > search->send) { + search->s = search->send; + return false; + } + return true; +} + +static inline VALUE +tr_trans_pairs_search_sse2(struct tr_trans_pairs_search *search) +{ + RBIMPL_ASSERT_OR_ASSUME(search->needles_count > 0); + RBIMPL_ASSERT_OR_ASSUME(search->needles_count < TR_TRANS_PAIRS_SIMD_MAX_NEEDLES); + + if (search->matches_bitmap) { + return tr_trans_pairs_next_match_sse2(search); + } + + if ((size_t)(search->send - search->s) >= sizeof(__m128i)) { + int i; + __m128i masks[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES]; + for (i = 0; i < search->needles_count; i++) { + masks[i] = _mm_set1_epi8(search->needles[i]); + } + + do { + const __m128i bytes = _mm_loadu_si128((__m128i const *)search->s); + + __m128i matches[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES]; + for (i = 0; i < search->needles_count; i++) { + matches[i] = _mm_cmpeq_epi8(bytes, masks[i]); + } + + for (i = i; i < search->needles_count; i++) { + matches[0] = _mm_or_si128(matches[0], matches[i]); + } + + const int bitmap = _mm_movemask_epi8(matches[0]); + + if (bitmap) { + search->matches_bitmap = bitmap; + return tr_trans_pairs_next_match_sse2(search); + } + search->s += sizeof(__m128i); + } while ((size_t)(search->send - search->s) >= sizeof(__m128i)); + } + return tr_trans_pairs_search_basic(search); +} + +#define tr_trans_pairs_search_impl tr_trans_pairs_search_sse2 +#endif + +#ifdef HAVE_SIMD_NEON +static inline bool +tr_trans_pairs_next_match_neon(struct tr_trans_pairs_search *search) +{ + size_t next_match_offset = ntz_int64(search->matches_bitmap) / 4; + search->matches_bitmap >>= (next_match_offset + 1) * 4; + search->s += next_match_offset; + if (search->s > search->send) { + search->s = search->send; + return false; + } + return true; +} + +static inline VALUE +tr_trans_pairs_search_neon(struct tr_trans_pairs_search *search) +{ + if (search->needles_count) { + RBIMPL_ASSERT_OR_ASSUME(search->needles_count > 0); + RBIMPL_ASSERT_OR_ASSUME(search->needles_count <= TR_TRANS_PAIRS_SIMD_MAX_NEEDLES); + + if (search->matches_bitmap) { + return tr_trans_pairs_next_match_neon(search); + } + + if ((size_t)(search->send - search->s) >= sizeof(uint8x16_t)) { + int i; + uint8x16_t masks[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES]; + for (i = 0; i < search->needles_count; i++) { + masks[i] = vdupq_n_u8(search->needles[i]); + } + + do { + const uint8x16_t bytes = vld1q_u8(search->s); + + uint8x16_t matches[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES]; + for (i = 0; i < search->needles_count; i++) { + matches[i] = vceqq_u8(bytes, masks[i]); + } + + for (i = i; i < search->needles_count; i++) { + matches[0] = vorrq_u8(matches[0], matches[i]); + } + + const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(matches[0]), 4); + const uint64_t bitmap = vget_lane_u64(vreinterpret_u64_u8(res), 0) & 0x8888888888888888ull; + + if (bitmap) { + search->matches_bitmap = bitmap; + return tr_trans_pairs_next_match_neon(search); + } + search->s += sizeof(uint8x16_t); + } while ((size_t)(search->send - search->s) >= sizeof(uint8x16_t)); + } + } + return tr_trans_pairs_search_basic(search); +} + +#define tr_trans_pairs_search_impl tr_trans_pairs_search_neon +#endif + +#ifndef tr_trans_pairs_search_impl +#define tr_trans_pairs_search_impl tr_trans_pairs_search_basic +#endif + static VALUE tr_trans_pairs(VALUE str, VALUE pairs_val) { @@ -9311,17 +9470,19 @@ tr_trans_pairs(VALUE str, VALUE pairs_val) VALUE hash = 0; - unsigned char *sstart = (unsigned char *)RSTRING_PTR(str); - unsigned char *s = sstart; - unsigned char *send = sstart + RSTRING_LEN(str); + const unsigned char *sstart = (unsigned char *)RSTRING_PTR(str); + long str_len = RSTRING_LEN(str); int termlen = rb_enc_mbminlen(e1); struct tr_buffer buffer; - tr_buffer_init(&buffer, send - s); + tr_buffer_init(&buffer, str_len); bool modify = false; if (RB_LIKELY(rb_str_encindex_fastpath(rb_enc_to_index(e1)))) { - VALUE trans_table[256] = { 0 }; + struct tr_trans_pairs_search search = { + .s = sstart, + .send = sstart + str_len, + }; for (size_t index = 0; index < pairs_count; index++) { struct tr_pair *pair = &pairs[index]; @@ -9329,11 +9490,20 @@ tr_trans_pairs(VALUE str, VALUE pairs_val) char *ptr = RSTRING_PTR(pair->search); unsigned int codepoint = rb_enc_mbc_to_codepoint(ptr, RSTRING_END(pair->search), e1); + const unsigned char first_byte = (unsigned char)*ptr; + +#ifdef HAVE_SIMD + if (pairs_count <= TR_TRANS_PAIRS_SIMD_MAX_NEEDLES) { + search.needles[index] = first_byte; + search.needles_count++; + } +#endif + if (rb_enc_codelen(codepoint, e1) == 1) { - trans_table[(unsigned char)*ptr] = pair->replace; + search.trans_table[first_byte] = pair->replace; } else { - trans_table[(unsigned char)*ptr] = Qundef; + search.trans_table[first_byte] = Qundef; if (!hash) { hash = rb_obj_hide(rb_hash_new_capa(pairs_count)); } @@ -9341,42 +9511,43 @@ tr_trans_pairs(VALUE str, VALUE pairs_val) } } - unsigned char *checkpoint = s; - while (s < send) { - VALUE repl = trans_table[*s]; - + const unsigned char *checkpoint = search.s; + VALUE repl; + while ((repl = tr_trans_pairs_search_impl(&search))) { int clen = 1; if (UNLIKELY(repl == Qundef)) { - unsigned int c = rb_enc_mbc_to_codepoint((char *)s, (char *)send, e1); + unsigned int c = rb_enc_mbc_to_codepoint((char *)search.s, (char *)search.send, e1); clen = rb_enc_codelen(c, e1); repl = rb_hash_lookup2(hash, UINT2NUM(c), 0); - } - - if (LIKELY(repl == 0)) { - s += clen; - continue; + if (!repl) { + search.s += clen; + continue; + } } modify = true; - if (checkpoint < s) { - tr_buffer_append(&buffer, checkpoint, s - checkpoint); + if (checkpoint < search.s) { + tr_buffer_append(&buffer, checkpoint, search.s - checkpoint); } tr_buffer_append_str(&buffer, repl); - s += clen; - checkpoint = s; + search.s += clen; + checkpoint = search.s; if (cr == ENC_CODERANGE_7BIT && rb_enc_str_coderange(repl) != ENC_CODERANGE_7BIT) { - cr == ENC_CODERANGE_VALID; + cr = ENC_CODERANGE_VALID; } } - if (modify && checkpoint < s) { - tr_buffer_append(&buffer, checkpoint, s - checkpoint); + if (modify && checkpoint < search.s) { + tr_buffer_append(&buffer, checkpoint, search.s - checkpoint); } } else { + const unsigned char *s = sstart; + const unsigned char *send = sstart + str_len; + hash = rb_obj_hide(rb_hash_new_capa(pairs_count)); for (size_t index = 0; index < pairs_count; index++) {