From 6acfd9c2b9b594bbf3e7ccecb61a528b29661106 Mon Sep 17 00:00:00 2001 From: Jean Boussier Date: Sun, 6 Sep 2026 14:19:01 +0200 Subject: [PATCH] string.c: Speedup `String#tr` fastpath with SIMD The fast path is now 4.4x faster, now making it ~2.6x times faster than `CGI.escape_html`, and 26x faster than an equivalent `gsub`. ``` compare-ruby: ruby 4.1.0dev (2026-09-06T11:22:48Z master 13fc4ffd32) +YJIT +PRISM [arm64-darwin25] built-ruby: ruby 4.1.0dev (2026-09-06T14:39:43Z string-tr-simd e2526ca9ba) +YJIT +PRISM [arm64-darwin25] `` | |compare-ruby|built-ruby| |:------------|-----------:|---------:| |tr_escape | 41.595k| 182.732k| | | -| 4.39x| |gsub_escape | 7.135k| 6.977k| | | 1.02x| -| |cgi_escape | 70.117k| 70.179k| | | -| 1.00x| However, contrary to a lookup table, here the performance is linear with the number of characters to match, hence once we're looking for 16 distinct bytes, the gain compared to the lookup table is down to ~2x. --- benchmark/string_tr.yml | 8 +- internal/simd.h | 18 ++++ string.c | 223 +++++++++++++++++++++++++++++++++++----- 3 files changed, 220 insertions(+), 29 deletions(-) create mode 100644 internal/simd.h diff --git a/benchmark/string_tr.yml b/benchmark/string_tr.yml index e0f582c240c17b..63805213848d46 100644 --- a/benchmark/string_tr.yml +++ b/benchmark/string_tr.yml @@ -6,9 +6,11 @@ prelude: | STR = ((("a" * 31) + "<") * 1000).freeze ESCAPED_CHARS = { - ">" => '\u003e', - "<" => '\u003c', - "&" => '\u0026', + "\\" => "'", + "\"" => """, + ">" => '>', + "<" => '<', + "&" => '&', }.freeze ESCAPE_PATTERN = Regexp.union(ESCAPED_CHARS.keys) diff --git a/internal/simd.h b/internal/simd.h new file mode 100644 index 00000000000000..44478c03e81f3f --- /dev/null +++ b/internal/simd.h @@ -0,0 +1,18 @@ +#ifndef INTERNAL_SIMD_H +#define INTERNAL_SIMD_H + +#if defined(__amd64__) || defined(__amd64) || defined(__x86_64__) || defined(__x86_64) || defined(_M_X64) || defined(_M_AMD64) +#ifdef HAVE_X86INTRIN_H +#include +#define HAVE_SIMD 1 +#define HAVE_SIMD_SSE2 1 +#endif +#endif + +#if defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(__aarch64__) || defined(_M_ARM64) +#define HAVE_SIMD 1 +#define HAVE_SIMD_NEON 1 +#include +#endif + +#endif /* INTERNAL_SIMD_H */ diff --git a/string.c b/string.c index 32c113dc82e99e..9b5667f3a26813 100644 --- a/string.c +++ b/string.c @@ -39,6 +39,7 @@ #include "internal/proc.h" #include "internal/re.h" #include "internal/sanitizers.h" +#include "internal/simd.h" #include "internal/string.h" #include "internal/transcode.h" #include "probes.h" @@ -9211,7 +9212,7 @@ tr_buffer_ensure_capa(struct tr_buffer *buffer, size_t extra_capa) size_t new_capa = buffer->capa ? buffer->capa : buffer->initial_capa; RUBY_ASSERT(new_capa >= 32); // Lower would cause infinite loop while (new_capa < required_capa) { - new_capa *= 1.2; + new_capa = (size_t)(new_capa * 1.2); } SIZED_REALLOC_N(buffer->buf, unsigned char, new_capa, buffer->capa); buffer->ptr = buffer->buf + offset; @@ -9220,7 +9221,7 @@ tr_buffer_ensure_capa(struct tr_buffer *buffer, size_t extra_capa) } static inline void -tr_buffer_append(struct tr_buffer *buffer, unsigned char *ptr, size_t len) +tr_buffer_append(struct tr_buffer *buffer, const unsigned char *ptr, size_t len) { if (len) { tr_buffer_ensure_capa(buffer, len); @@ -9286,6 +9287,164 @@ tr_trans_pairs_coerce_i(st_data_t key, st_data_t value, st_data_t _args) return ST_CONTINUE; } +#define TR_TRANS_PAIRS_SIMD_MAX_NEEDLES 16 + +struct tr_trans_pairs_search { + const unsigned char *s; + const unsigned char *send; + +#ifdef HAVE_SIMD + unsigned char needles[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES]; + int needles_count; +#ifdef HAVE_SIMD_NEON + uint64_t matches_bitmap; +#endif +#ifdef HAVE_SIMD_SSE2 + int matches_bitmap; +#endif +#endif + + VALUE trans_table[256]; +}; + +static inline VALUE +tr_trans_pairs_search_basic(struct tr_trans_pairs_search *search) +{ + while (search->s < search->send) { + VALUE repl = search->trans_table[*search->s]; + if (UNLIKELY(repl)) { + return repl; + } + + search->s++; + } + + return 0; +} + +#ifdef HAVE_SIMD_SSE2 +static inline bool +tr_trans_pairs_next_match_sse2(struct tr_trans_pairs_search *search) +{ + size_t next_match_offset = ntz_int32(search->matches_bitmap); + search->matches_bitmap >>= (next_match_offset + 1); + search->s += next_match_offset; + if (search->s > search->send) { + search->s = search->send; + return false; + } + return true; +} + +static inline VALUE +tr_trans_pairs_search_sse2(struct tr_trans_pairs_search *search) +{ + RBIMPL_ASSERT_OR_ASSUME(search->needles_count > 0); + RBIMPL_ASSERT_OR_ASSUME(search->needles_count < TR_TRANS_PAIRS_SIMD_MAX_NEEDLES); + + if (search->matches_bitmap) { + return tr_trans_pairs_next_match_sse2(search); + } + + if ((size_t)(search->send - search->s) >= sizeof(__m128i)) { + int i; + __m128i masks[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES]; + for (i = 0; i < search->needles_count; i++) { + masks[i] = _mm_set1_epi8(search->needles[i]); + } + + do { + const __m128i bytes = _mm_loadu_si128((__m128i const *)search->s); + + __m128i matches[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES]; + for (i = 0; i < search->needles_count; i++) { + matches[i] = _mm_cmpeq_epi8(bytes, masks[i]); + } + + for (i = i; i < search->needles_count; i++) { + matches[0] = _mm_or_si128(matches[0], matches[i]); + } + + const int bitmap = _mm_movemask_epi8(matches[0]); + + if (bitmap) { + search->matches_bitmap = bitmap; + return tr_trans_pairs_next_match_sse2(search); + } + search->s += sizeof(__m128i); + } while ((size_t)(search->send - search->s) >= sizeof(__m128i)); + } + return tr_trans_pairs_search_basic(search); +} + +#define tr_trans_pairs_search_impl tr_trans_pairs_search_sse2 +#endif + +#ifdef HAVE_SIMD_NEON +static inline bool +tr_trans_pairs_next_match_neon(struct tr_trans_pairs_search *search) +{ + size_t next_match_offset = ntz_int64(search->matches_bitmap) / 4; + search->matches_bitmap >>= (next_match_offset + 1) * 4; + search->s += next_match_offset; + if (search->s > search->send) { + search->s = search->send; + return false; + } + return true; +} + +static inline VALUE +tr_trans_pairs_search_neon(struct tr_trans_pairs_search *search) +{ + if (search->needles_count) { + RBIMPL_ASSERT_OR_ASSUME(search->needles_count > 0); + RBIMPL_ASSERT_OR_ASSUME(search->needles_count <= TR_TRANS_PAIRS_SIMD_MAX_NEEDLES); + + if (search->matches_bitmap) { + return tr_trans_pairs_next_match_neon(search); + } + + if ((size_t)(search->send - search->s) >= sizeof(uint8x16_t)) { + int i; + uint8x16_t masks[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES]; + for (i = 0; i < search->needles_count; i++) { + masks[i] = vdupq_n_u8(search->needles[i]); + } + + do { + const uint8x16_t bytes = vld1q_u8(search->s); + + uint8x16_t matches[TR_TRANS_PAIRS_SIMD_MAX_NEEDLES]; + for (i = 0; i < search->needles_count; i++) { + matches[i] = vceqq_u8(bytes, masks[i]); + } + + for (i = i; i < search->needles_count; i++) { + matches[0] = vorrq_u8(matches[0], matches[i]); + } + + const uint8x8_t res = vshrn_n_u16(vreinterpretq_u16_u8(matches[0]), 4); + const uint64_t bitmap = vget_lane_u64(vreinterpret_u64_u8(res), 0) & 0x8888888888888888ull; + + if (bitmap) { + search->matches_bitmap = bitmap; + return tr_trans_pairs_next_match_neon(search); + } + search->s += sizeof(uint8x16_t); + } while ((size_t)(search->send - search->s) >= sizeof(uint8x16_t)); + } + } + return tr_trans_pairs_search_basic(search); +} + +#define tr_trans_pairs_search_impl tr_trans_pairs_search_neon +#endif + +#ifndef tr_trans_pairs_search_impl +#define tr_trans_pairs_search_impl tr_trans_pairs_search_basic +#endif + static VALUE tr_trans_pairs(VALUE str, VALUE pairs_val) { @@ -9311,17 +9470,19 @@ tr_trans_pairs(VALUE str, VALUE pairs_val) VALUE hash = 0; - unsigned char *sstart = (unsigned char *)RSTRING_PTR(str); - unsigned char *s = sstart; - unsigned char *send = sstart + RSTRING_LEN(str); + const unsigned char *sstart = (unsigned char *)RSTRING_PTR(str); + long str_len = RSTRING_LEN(str); int termlen = rb_enc_mbminlen(e1); struct tr_buffer buffer; - tr_buffer_init(&buffer, send - s); + tr_buffer_init(&buffer, str_len); bool modify = false; if (RB_LIKELY(rb_str_encindex_fastpath(rb_enc_to_index(e1)))) { - VALUE trans_table[256] = { 0 }; + struct tr_trans_pairs_search search = { + .s = sstart, + .send = sstart + str_len, + }; for (size_t index = 0; index < pairs_count; index++) { struct tr_pair *pair = &pairs[index]; @@ -9329,11 +9490,20 @@ tr_trans_pairs(VALUE str, VALUE pairs_val) char *ptr = RSTRING_PTR(pair->search); unsigned int codepoint = rb_enc_mbc_to_codepoint(ptr, RSTRING_END(pair->search), e1); + const unsigned char first_byte = (unsigned char)*ptr; + +#ifdef HAVE_SIMD + if (pairs_count <= TR_TRANS_PAIRS_SIMD_MAX_NEEDLES) { + search.needles[index] = first_byte; + search.needles_count++; + } +#endif + if (rb_enc_codelen(codepoint, e1) == 1) { - trans_table[(unsigned char)*ptr] = pair->replace; + search.trans_table[first_byte] = pair->replace; } else { - trans_table[(unsigned char)*ptr] = Qundef; + search.trans_table[first_byte] = Qundef; if (!hash) { hash = rb_obj_hide(rb_hash_new_capa(pairs_count)); } @@ -9341,42 +9511,43 @@ tr_trans_pairs(VALUE str, VALUE pairs_val) } } - unsigned char *checkpoint = s; - while (s < send) { - VALUE repl = trans_table[*s]; - + const unsigned char *checkpoint = search.s; + VALUE repl; + while ((repl = tr_trans_pairs_search_impl(&search))) { int clen = 1; if (UNLIKELY(repl == Qundef)) { - unsigned int c = rb_enc_mbc_to_codepoint((char *)s, (char *)send, e1); + unsigned int c = rb_enc_mbc_to_codepoint((char *)search.s, (char *)search.send, e1); clen = rb_enc_codelen(c, e1); repl = rb_hash_lookup2(hash, UINT2NUM(c), 0); - } - - if (LIKELY(repl == 0)) { - s += clen; - continue; + if (!repl) { + search.s += clen; + continue; + } } modify = true; - if (checkpoint < s) { - tr_buffer_append(&buffer, checkpoint, s - checkpoint); + if (checkpoint < search.s) { + tr_buffer_append(&buffer, checkpoint, search.s - checkpoint); } tr_buffer_append_str(&buffer, repl); - s += clen; - checkpoint = s; + search.s += clen; + checkpoint = search.s; if (cr == ENC_CODERANGE_7BIT && rb_enc_str_coderange(repl) != ENC_CODERANGE_7BIT) { - cr == ENC_CODERANGE_VALID; + cr = ENC_CODERANGE_VALID; } } - if (modify && checkpoint < s) { - tr_buffer_append(&buffer, checkpoint, s - checkpoint); + if (modify && checkpoint < search.s) { + tr_buffer_append(&buffer, checkpoint, search.s - checkpoint); } } else { + const unsigned char *s = sstart; + const unsigned char *send = sstart + str_len; + hash = rb_obj_hide(rb_hash_new_capa(pairs_count)); for (size_t index = 0; index < pairs_count; index++) {