diff --git a/.gdbinit b/.gdbinit index f045b12fb0f913..9281744c137b63 100644 --- a/.gdbinit +++ b/.gdbinit @@ -237,7 +237,7 @@ define rp else if ($flags & RUBY_T_MASK) == RUBY_T_IMEMO printf "%sT_IMEMO%s(", $color_type, $color_end - output (enum imemo_type)(($flags>>RUBY_FL_USHIFT)&RUBY_IMEMO_MASK) + output (enum imemo_type)(($flags&RUBY_IMEMO_MASK)>>RUBY_FL_USHIFT) printf "): " rp_imemo $arg0 else @@ -539,7 +539,7 @@ document rp_class end define rp_imemo - set $flags = (enum imemo_type)((((struct RBasic *)($arg0))->flags >> RUBY_FL_USHIFT) & RUBY_IMEMO_MASK) + set $flags = (enum imemo_type)((((struct RBasic *)($arg0))->flags & RUBY_IMEMO_MASK) >> RUBY_FL_USHIFT) if $flags == imemo_cref printf "(rb_cref_t *) %p\n", (void*)$arg0 print *(rb_cref_t *)$arg0 @@ -1100,7 +1100,7 @@ define rb_ps_thread while $cfp < $cfpend if $cfp->_iseq set $iseq = rb_get_cfp_iseq($cfp) - if !((VALUE)$iseq & RUBY_IMMEDIATE_MASK) && (((imemo_ifunc << RUBY_FL_USHIFT) | RUBY_T_IMEMO)==$iseq->flags & ((RUBY_IMEMO_MASK << RUBY_FL_USHIFT) | RUBY_T_MASK)) + if !((VALUE)$iseq & RUBY_IMMEDIATE_MASK) && (((imemo_ifunc << RUBY_FL_USHIFT) | RUBY_T_IMEMO)==$iseq->flags & (RUBY_IMEMO_MASK | RUBY_T_MASK)) printf "%d:ifunc ", $cfpend-$cfp set print symbol-filename on output/a $iseq.body diff --git a/ext/objspace/objspace.c b/ext/objspace/objspace.c index 753f77a6010904..a05d491e35b57a 100644 --- a/ext/objspace/objspace.c +++ b/ext/objspace/objspace.c @@ -403,7 +403,7 @@ count_tdata_objects(int argc, VALUE *argv, VALUE self) return hash; } -static ID imemo_type_ids[IMEMO_MASK+1]; +static ID imemo_type_ids[(IMEMO_MASK >> FL_USHIFT) + 1]; static void count_imemo_objects_i(VALUE v, void *data) diff --git a/internal/imemo.h b/internal/imemo.h index fee5c07bc33703..02fb0a131e1345 100644 --- a/internal/imemo.h +++ b/internal/imemo.h @@ -15,17 +15,17 @@ #include "ruby/internal/stdbool.h" /* for bool */ #include "ruby/ruby.h" /* for rb_block_call_func_t */ -#define IMEMO_MASK 0x0f - -/* FL_USER0 to FL_USER3 is for type */ -#define IMEMO_FL_USHIFT (FL_USHIFT + 4) -#define IMEMO_FL_USER0 FL_USER4 -#define IMEMO_FL_USER1 FL_USER5 -#define IMEMO_FL_USER2 FL_USER6 -#define IMEMO_FL_USER3 FL_USER7 -#define IMEMO_FL_USER4 FL_USER8 -#define IMEMO_FL_USER5 FL_USER9 -#define IMEMO_FL_USER6 FL_USER10 +#define IMEMO_MASK (FL_USER0 | FL_USER1 | FL_USER2 | FL_USER3 | FL_USER4) + +/* FL_USER0 to FL_USER4 is for type */ +#define IMEMO_FL_USHIFT (FL_USHIFT + 5) +#define IMEMO_FL_USER0 FL_USER5 +#define IMEMO_FL_USER1 FL_USER6 +#define IMEMO_FL_USER2 FL_USER7 +#define IMEMO_FL_USER3 FL_USER8 +#define IMEMO_FL_USER4 FL_USER9 +#define IMEMO_FL_USER5 FL_USER10 +#define IMEMO_FL_USER6 FL_USER11 enum imemo_type { imemo_env = 0, @@ -171,7 +171,7 @@ RUBY_SYMBOL_EXPORT_END static inline enum imemo_type imemo_type(VALUE imemo) { - return (RBASIC(imemo)->flags >> FL_USHIFT) & IMEMO_MASK; + return (RBASIC(imemo)->flags & IMEMO_MASK) >> FL_USHIFT; } static inline int @@ -179,7 +179,7 @@ imemo_type_p(VALUE imemo, enum imemo_type imemo_type) { if (LIKELY(!RB_SPECIAL_CONST_P(imemo))) { /* fixed at compile time if imemo_type is given. */ - const VALUE mask = (IMEMO_MASK << FL_USHIFT) | RUBY_T_MASK; + const VALUE mask = IMEMO_MASK | RUBY_T_MASK; const VALUE expected_type = (imemo_type << FL_USHIFT) | T_IMEMO; /* fixed at runtime. */ return expected_type == (RBASIC(imemo)->flags & mask); diff --git a/jit.c b/jit.c index 21a16a156e488c..484c7a63a9e35c 100644 --- a/jit.c +++ b/jit.c @@ -562,6 +562,19 @@ rb_jit_array_len(VALUE a) return rb_array_len(a); } +// Return non-zero when `obj` is an array and its last item is a +// `ruby2_keywords` hash. The JITs don't support this kind of splat. +size_t +rb_jit_ruby2_keywords_splat_p(VALUE obj) +{ + if (!RB_TYPE_P(obj, T_ARRAY)) return 0; + long len = RARRAY_LEN(obj); + if (len == 0) return 0; + VALUE last = RARRAY_AREF(obj, len - 1); + if (!RB_TYPE_P(last, T_HASH)) return 0; + return FL_TEST_RAW(last, RHASH_PASS_AS_KEYWORDS); +} + void rb_set_cfp_pc(struct rb_control_frame_struct *cfp, const VALUE *pc) { diff --git a/misc/lldb_cruby.py b/misc/lldb_cruby.py index b3d4fb509add14..2eec8cfce75724 100644 --- a/misc/lldb_cruby.py +++ b/misc/lldb_cruby.py @@ -417,7 +417,7 @@ def lldb_inspect(debugger, target, result, val): append_expression(debugger, "*(struct RMatch *) %0#x" % val.GetValueAsUnsigned(), result) elif flType == RUBY_T_IMEMO: # I'm not sure how to get IMEMO_MASK out of lldb. It's not in globals() - imemo_type = (flags >> RUBY_FL_USHIFT) & 0x0F # IMEMO_MASK + imemo_type = (flags >> RUBY_FL_USHIFT) & 0x1F # IMEMO_MASK print("T_IMEMO: ", file=result) append_expression(debugger, "(enum imemo_type) %d" % imemo_type, result) diff --git a/misc/lldb_rb/constants.py b/misc/lldb_rb/constants.py index 9cd56eccb0ebdc..c3132c366b25f8 100644 --- a/misc/lldb_rb/constants.py +++ b/misc/lldb_rb/constants.py @@ -3,4 +3,4 @@ HEAP_PAGE_ALIGN = (1 << HEAP_PAGE_ALIGN_LOG) HEAP_PAGE_SIZE = HEAP_PAGE_ALIGN -IMEMO_MASK = 0x0F +IMEMO_MASK = 0x1F diff --git a/tool/timeline/lib/converter_defs.rb b/tool/timeline/lib/converter_defs.rb index e4cd2bc079bfc1..46a19fe27457b9 100644 --- a/tool/timeline/lib/converter_defs.rb +++ b/tool/timeline/lib/converter_defs.rb @@ -85,7 +85,7 @@ def self.FL_USER_N(n) }) # Keep in sync with `IMEMO_MASK` in `internal/imemo.h`. - IMEMO_MASK = 0x0f + IMEMO_MASK = 0x1f # Keep in sync with both `internal/string.h` and `include/ruby/internal/core/rstring.h`. StringFlags = FlagsConverter.new({ diff --git a/yjit.c b/yjit.c index 2b6f1110275362..d59bfaa38108ee 100644 --- a/yjit.c +++ b/yjit.c @@ -242,19 +242,6 @@ rb_yjit_rb_ary_subseq_length(VALUE ary, long beg) return rb_ary_subseq(ary, beg, len); } -// Return non-zero when `obj` is an array and its last item is a -// `ruby2_keywords` hash. We don't support this kind of splat. -size_t -rb_yjit_ruby2_keywords_splat_p(VALUE obj) -{ - if (!RB_TYPE_P(obj, T_ARRAY)) return 0; - long len = RARRAY_LEN(obj); - if (len == 0) return 0; - VALUE last = RARRAY_AREF(obj, len - 1); - if (!RB_TYPE_P(last, T_HASH)) return 0; - return FL_TEST_RAW(last, RHASH_PASS_AS_KEYWORDS); -} - // Checks to establish preconditions for rb_yjit_splat_varg_cfunc() VALUE rb_yjit_splat_varg_checks(VALUE *sp, VALUE splat_array, rb_control_frame_t *cfp) diff --git a/yjit/bindgen/src/main.rs b/yjit/bindgen/src/main.rs index 28afb79144f7fc..ff9e587484a1e3 100644 --- a/yjit/bindgen/src/main.rs +++ b/yjit/bindgen/src/main.rs @@ -371,7 +371,7 @@ fn main() { .allowlist_function("rb_yarv_str_eql_internal") .allowlist_function("rb_str_neq_internal") .allowlist_function("rb_yarv_ary_entry_internal") - .allowlist_function("rb_yjit_ruby2_keywords_splat_p") + .allowlist_function("rb_jit_ruby2_keywords_splat_p") .allowlist_function("rb_jit_fix_div_fix") .allowlist_function("rb_jit_fix_mod_fix") .allowlist_function("rb_FL_TEST") diff --git a/yjit/src/codegen.rs b/yjit/src/codegen.rs index dffb4593c09202..a1ee87e2f29e9e 100644 --- a/yjit/src/codegen.rs +++ b/yjit/src/codegen.rs @@ -7031,7 +7031,7 @@ fn gen_send_cfunc( if variable_splat { let splat_array_idx = i32::from(kw_splat) + i32::from(block_arg); let comptime_splat_array = jit.peek_at_stack(&asm.ctx, splat_array_idx as isize); - if unsafe { rb_yjit_ruby2_keywords_splat_p(comptime_splat_array) } != 0 { + if unsafe { rb_jit_ruby2_keywords_splat_p(comptime_splat_array) } != 0 { gen_counter_incr(jit, asm, Counter::send_cfunc_splat_varg_ruby2_keywords); return None; } @@ -7932,7 +7932,7 @@ fn gen_send_iseq( // All splats need to guard for ruby2_keywords hash. Check with a function call when // splatting into a rest param since the index for the last item in the array is dynamic. asm_comment!(asm, "guard no ruby2_keywords hash in splat"); - let bad_splat = asm.ccall(rb_yjit_ruby2_keywords_splat_p as _, vec![asm.stack_opnd(splat_pos)]); + let bad_splat = asm.ccall(rb_jit_ruby2_keywords_splat_p as _, vec![asm.stack_opnd(splat_pos)]); asm.cmp(bad_splat, 0.into()); asm.jnz(Target::side_exit(Counter::guard_send_splatarray_last_ruby2_keywords)); } diff --git a/yjit/src/cruby.rs b/yjit/src/cruby.rs index c97e50ac1cc18d..dc8b3200aa9fd5 100644 --- a/yjit/src/cruby.rs +++ b/yjit/src/cruby.rs @@ -754,7 +754,7 @@ mod manual_defs { pub const RSTRUCT_EMBED_LEN_MASK: usize = (RUBY_FL_USER7 | RUBY_FL_USER6 | RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3 |RUBY_FL_USER2 | RUBY_FL_USER1) as usize; // From iseq.h - via a different constant, which seems to confuse bindgen - pub const ISEQ_TRANSLATED: usize = RUBY_FL_USER7 as usize; + pub const ISEQ_TRANSLATED: usize = RUBY_FL_USER8 as usize; // We'll need to encode a lot of Ruby struct/field offsets as constants unless we want to // redeclare all the Ruby C structs and write our own offsetof macro. For now, we use constants. diff --git a/yjit/src/cruby_bindings.inc.rs b/yjit/src/cruby_bindings.inc.rs index 143eef16e28ed8..a5302a9a56b17d 100644 --- a/yjit/src/cruby_bindings.inc.rs +++ b/yjit/src/cruby_bindings.inc.rs @@ -1260,7 +1260,6 @@ extern "C" { pub fn rb_str_neq_internal(str1: VALUE, str2: VALUE) -> VALUE; pub fn rb_ary_unshift_m(argc: ::std::os::raw::c_int, argv: *mut VALUE, ary: VALUE) -> VALUE; pub fn rb_yjit_rb_ary_subseq_length(ary: VALUE, beg: ::std::os::raw::c_long) -> VALUE; - pub fn rb_yjit_ruby2_keywords_splat_p(obj: VALUE) -> usize; pub fn rb_yjit_splat_varg_checks( sp: *mut VALUE, splat_array: VALUE, @@ -1386,6 +1385,7 @@ extern "C" { pub fn rb_assert_cme_handle(handle: VALUE); pub fn rb_yarv_ary_entry_internal(ary: VALUE, offset: ::std::os::raw::c_long) -> VALUE; pub fn rb_jit_array_len(a: VALUE) -> ::std::os::raw::c_long; + pub fn rb_jit_ruby2_keywords_splat_p(obj: VALUE) -> usize; pub fn rb_set_cfp_pc(cfp: *mut rb_control_frame_struct, pc: *const VALUE); pub fn rb_set_cfp_sp(cfp: *mut rb_control_frame_struct, sp: *mut VALUE); pub fn rb_jit_shape_complex_p(shape_id: shape_id_t) -> bool; diff --git a/zjit.rb b/zjit.rb index 3b52211eef3735..f2f2742e5ae234 100644 --- a/zjit.rb +++ b/zjit.rb @@ -137,6 +137,7 @@ def stats_string :empty_inline_frame_count, :non_variadic_cfunc_optimized_send_count, :variadic_cfunc_optimized_send_count, + :caller_splat_optimized, ], buf:, stats:, right_align: true, base: :send_count) print_counters([ :dynamic_setivar_count, diff --git a/zjit/bindgen/src/main.rs b/zjit/bindgen/src/main.rs index fe25d56f081a0a..00d169f924fb4e 100644 --- a/zjit/bindgen/src/main.rs +++ b/zjit/bindgen/src/main.rs @@ -313,6 +313,7 @@ fn main() { .allowlist_function("rb_jit_mark_unused") .allowlist_function("rb_jit_get_page_size") .allowlist_function("rb_jit_array_len") + .allowlist_function("rb_jit_ruby2_keywords_splat_p") .allowlist_function("rb_jit_fix_div_fix") .allowlist_function("rb_jit_iseq_builtin_attrs") .allowlist_function("rb_jit_str_concat_codepoint") diff --git a/zjit/src/codegen_tests.rs b/zjit/src/codegen_tests.rs index 0fd00630c1952a..a7aeb434c18352 100644 --- a/zjit/src/codegen_tests.rs +++ b/zjit/src/codegen_tests.rs @@ -7123,13 +7123,108 @@ fn test_send_on_heap_object_in_spilled_arg() { } #[test] -fn test_send_splat() { - assert_snapshot!(inspect(" +fn test_send_caller_splat_arguments() { + eval(" def test(a, b) = [a, b] - def entry(arr) = test(*arr) + def entry(args) = test(*args) entry([1, 2]) + "); + assert_snapshot!(assert_compiles("entry([1, 2])"), @"[1, 2]"); +} + +#[test] +fn test_send_empty_caller_splat_arguments() { + eval(" + def test(a = 1) = a + def entry(args) = test(*args) + entry([]) + "); + assert_snapshot!(assert_compiles("entry([])"), @"1"); +} + +#[test] +fn test_send_caller_splat_arguments_with_positional_prefix() { + eval(" + def test(a, b, c) = [a, b, c] + def entry(args) = test(1, *args) + entry([2, 3]) + "); + assert_snapshot!(assert_compiles("entry([2, 3])"), @"[1, 2, 3]"); +} + +#[test] +fn test_send_many_caller_splat_arguments_to_rest_parameter() { + eval(" + def test(*args) = args.length + def entry(args) = test(*args) + entry([1, 2, 3, 4, 5, 6, 7]) + "); + assert_snapshot!(assert_compiles("entry([1, 2, 3, 4, 5, 6, 7])"), @"7"); +} + +#[test] +fn test_send_caller_splat_arguments_to_complex_parameters() { + eval(" + def test(a, b = 2, *rest, z, k: 40) = [a, b, rest, z, k] + def entry(args) = test(1, *args) + entry([3, 4, 5]) + "); + assert_snapshot!(assert_compiles("entry([3, 4, 5])"), @"[1, 3, [4], 5, 40]"); +} + +#[test] +fn test_send_caller_splat_arguments_with_required_keyword() { + eval(" + def test(*args, k:) = [args, k] + def entry(args) = test(*args, k: 40) entry([1, 2]) - "), @"[1, 2]"); + "); + assert_snapshot!(assert_compiles("entry([1, 2])"), @"[[1, 2], 40]"); +} + +#[test] +fn test_send_caller_splat_arguments_with_block_literal() { + eval(" + def test(*args) = yield args.length + def entry(args) = test(*args) { |n| n + 4 } + entry([1, 2, 3]) + "); + assert_snapshot!(assert_compiles("entry([1, 2, 3])"), @"7"); +} + +#[test] +fn test_send_caller_splat_length_mismatch_side_exits() { + eval(" + def test(*args) = args + def entry(args) = test(*args) + entry([1, 2]) + "); + assert_snapshot!(assert_compiles_allowing_exits("entry([1, 2, 3])"), @"[1, 2, 3]"); +} + +#[test] +fn test_send_caller_splat_with_ruby2_keywords_hash_side_exits() { + eval(" + def capture(*args) = args + ruby2_keywords(:capture) + def test(arg = :default, k: nil) = [arg, k] + def entry(args) = test(*args) + entry(capture(k: 1)) + "); + assert_snapshot!(assert_compiles_allowing_exits("entry(capture(k: 1))"), @"[:default, 1]"); +} + +#[test] +fn test_send_caller_splat_result_used_by_hash_aset() { + eval(" + def test(value) = value + def entry(args) + hash = {} + hash[:value] = test(*args) + end + entry([1]) + "); + assert_snapshot!(assert_compiles("entry([2])"), @"2"); } #[test] diff --git a/zjit/src/cruby.rs b/zjit/src/cruby.rs index d4e6955bededdd..2111266500df4a 100644 --- a/zjit/src/cruby.rs +++ b/zjit/src/cruby.rs @@ -1224,7 +1224,7 @@ mod manual_defs { pub const RSTRUCT_EMBED_LEN_MASK: usize = (RUBY_FL_USER7 | RUBY_FL_USER6 | RUBY_FL_USER5 | RUBY_FL_USER4 | RUBY_FL_USER3 |RUBY_FL_USER2 | RUBY_FL_USER1) as usize; // From iseq.h - via a different constant, which seems to confuse bindgen - pub const ISEQ_TRANSLATED: usize = RUBY_FL_USER7 as usize; + pub const ISEQ_TRANSLATED: usize = RUBY_FL_USER8 as usize; // We'll need to encode a lot of Ruby struct/field offsets as constants unless we want to // redeclare all the Ruby C structs and write our own offsetof macro. For now, we use constants. @@ -1736,6 +1736,7 @@ pub(crate) mod ids { name: aref content: b"[]" name: rb_obj_is_proc name: rb_ivar_get_at_no_ractor_check + name: rb_jit_ruby2_keywords_splat_p name: RUBY_FL_FREEZE name: RUBY_ELTS_SHARED name: RubyVM diff --git a/zjit/src/cruby_bindings.inc.rs b/zjit/src/cruby_bindings.inc.rs index 4bb6e5bc8a38ab..fd4f085023f10e 100644 --- a/zjit/src/cruby_bindings.inc.rs +++ b/zjit/src/cruby_bindings.inc.rs @@ -2566,6 +2566,7 @@ unsafe extern "C" { pub fn rb_assert_cme_handle(handle: VALUE); pub fn rb_yarv_ary_entry_internal(ary: VALUE, offset: ::std::os::raw::c_long) -> VALUE; pub fn rb_jit_array_len(a: VALUE) -> ::std::os::raw::c_long; + pub fn rb_jit_ruby2_keywords_splat_p(obj: VALUE) -> usize; pub fn rb_set_cfp_pc(cfp: *mut rb_control_frame_struct, pc: *const VALUE); pub fn rb_set_cfp_sp(cfp: *mut rb_control_frame_struct, sp: *mut VALUE); pub fn rb_jit_shape_complex_p(shape_id: shape_id_t) -> bool; diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index 44c5da93f37dab..b567c8d2794c84 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -15,7 +15,7 @@ use std::{ use crate::hir_type::{Type, types}; use crate::hir_effect::{Effect, abstract_heaps, effects}; use crate::bitset::BitSet; -use crate::profile::{TypeDistributionSummary, ProfiledType}; +use crate::profile::{ProfiledType, SplatLength, TypeDistributionSummary}; use crate::stats::{Counter, incr_counter}; use SendFallbackReason::*; @@ -665,6 +665,8 @@ pub enum SideExitReason { SplatKwNotNilOrHash, SplatKwPolymorphic, SplatKwNotProfiled, + CallerSplatLengthMismatch, + CallerSplatRuby2Keywords, DirectiveInduced, SendWhileTracing, NoProfileSend, @@ -1169,6 +1171,8 @@ pub enum Insn { cd: *const rb_call_data, block: Option, args: Vec, + /// Caller-splat length selected by `add_iseq_to_hir`. + caller_splat_length: Option, state: InsnId, reason: SendFallbackReason, }, @@ -2631,13 +2635,13 @@ pub enum ValidationError { } /// Check if we can emit SendDirect to the given ISEQ with the given arguments. -fn can_direct_send(iseq: *const rb_iseq_t, ci: *const rb_callinfo, args: &[InsnId], has_block: bool) -> Result<(), SendDirectFailure> { +fn can_direct_send(iseq: *const rb_iseq_t, caller_args: &CallerArguments, has_block: bool, caller_splat: Option) -> Result<(), SendDirectFailure> { let mut complex_arg_counters = vec![]; let mut count_failure = |counter| complex_arg_counters.push(counter); let params = unsafe { iseq.params() }; let callee_has_block_param = 0 != params.flags.has_block(); - let caller_passes_block_arg = has_block && (unsafe { rb_vm_ci_flag(ci) } & VM_CALL_ARGS_BLOCKARG) != 0; + let caller_passes_block_arg = has_block && (caller_args.flags & VM_CALL_ARGS_BLOCKARG) != 0; use Counter::*; if 0 != params.flags.forwardable() { count_failure(complex_arg_pass_param_forwardable) } @@ -2668,15 +2672,20 @@ fn can_direct_send(iseq: *const rb_iseq_t, ci: *const rb_callinfo, args: &[InsnI let keyword = params.keyword; let kw_req_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).required_num } }; let kw_total_num = if keyword.is_null() { 0 } else { unsafe { (*keyword).num } }; - let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; - let caller_kw_count = if kwarg.is_null() { 0 } else { (unsafe { get_cikw_keyword_len(kwarg) }) as usize }; + let caller_kw_count = caller_args.kwarg_count; let has_rest = 0 != params.flags.has_rest(); - let caller_positional = match args.len().checked_sub(caller_kw_count) { + let caller_positional = match caller_args.original.len().checked_sub(caller_kw_count) { Some(count) => count, None => { return Err(SendDirectFailure::new(ArgcParamMismatch)); } }; + // A caller splat occupies one argument slot before expansion. Replace that + // slot with its profiled length to get the effective positional argument count. + let caller_positional = match caller_splat { + None => caller_positional, + Some(splat) => caller_positional - 1 + splat.length as usize, + }; // Match vm_args.c's setup_parameters_complex via args_kw_argv_to_hash: // keywords passed to a method with no keyword parameters can become one @@ -2848,10 +2857,64 @@ struct SendDirectArgs { jit_entry_idx: u16, } +/// Caller Arguments as they appear on the original Send instruction. +struct CallerArguments<'a> { + /// Argument values in the order stored by the original Send. + original: &'a [InsnId], + /// Call-site flags from the Send's callinfo. + flags: u32, + /// Explicit keyword metadata, or null when the caller has no keywords. + kwarg: *const rb_callinfo_kwarg, + /// Number of explicit keyword values at the end of `original`. + kwarg_count: usize, + /// Index of the caller splat array, when VM_CALL_ARGS_SPLAT is set. + splat_arg_idx: Option, +} + +impl<'a> CallerArguments<'a> { + /// Decode callinfo metadata and locate the splat in the original Send arguments. + /// Do this once per Send so builds for different splat lengths share the same layout. + fn new(original: &'a [InsnId], ci: *const rb_callinfo) -> Self { + let flags = unsafe { rb_vm_ci_flag(ci) }; + let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let kwarg_count = if kwarg.is_null() { + 0 + } else { + (unsafe { get_cikw_keyword_len(kwarg) }) as usize + }; + let splat_arg_idx = if flags & VM_CALL_ARGS_SPLAT != 0 { + // The splat array is the final positional operand, before explicit keyword values. + Some(original.len() - kwarg_count - 1) + } else { + None + }; + + Self { original, flags, kwarg, kwarg_count, splat_arg_idx } + } +} + +/// Caller splat expansion selected for one SendDirect path. +#[derive(Clone, Copy)] +struct CallerSplat { + /// Index of the splat array in the original Send argument vector. + arg_idx: usize, + /// HIR value that produces the splat array at runtime. + array: InsnId, + /// Profiled array length handled by this path. + length: SplatLength, +} + /// One SendDirect argument before its HIR value is materialized. enum SendDirectArg { /// A HIR value already present in the original Send argument vector. Existing(InsnId), + /// An element to load from the caller splat array on the selected path. + SplatElement { + /// HIR value that produces the splat array. + array: InsnId, + /// Zero-based index of the element to load. + index: SplatLength, + }, /// A Ruby value to materialize as a Const instruction on the selected path. Constant(VALUE), /// Explicit caller keywords to materialize as one positional Hash. @@ -3829,10 +3892,10 @@ impl Function { } /// Validate and normalize SendDirect arguments without emitting HIR. - fn build_send_direct_args(&self, args: &[InsnId], ci: *const rb_callinfo, iseq: IseqPtr, has_block: bool) -> Result { - can_direct_send(iseq, ci, args, has_block)?; - let args = args.iter().copied().map(SendDirectArg::Existing).collect(); - let (args, kw_bits) = Self::plan_send_direct_keyword_arguments(args, ci, iseq) + fn build_send_direct_args(&self, caller_args: &CallerArguments, caller_splat: Option, iseq: IseqPtr, has_block: bool) -> Result { + can_direct_send(iseq, caller_args, has_block, caller_splat)?; + let args = Self::expand_caller_splat_args(caller_args, caller_splat); + let (args, kw_bits) = Self::plan_send_direct_keyword_arguments(args, caller_args, iseq) .map_err(SendDirectFailure::new)?; let (args, jit_entry_idx) = Self::plan_send_direct_rest_parameter(args, iseq) .map_err(SendDirectFailure::new)?; @@ -3871,6 +3934,10 @@ impl Function { fn emit_send_direct_arg(&mut self, block: BlockId, arg: SendDirectArg, state: InsnId) -> InsnId { match arg { SendDirectArg::Existing(value) => value, + SendDirectArg::SplatElement { array, index } => { + let index = self.push_insn(block, Insn::Const { val: Const::CInt64(i64::from(index)) }); + self.push_insn(block, Insn::ArrayAref { array, index }) + } SendDirectArg::Constant(value) => { self.push_insn(block, Insn::Const { val: Const::Value(value) }) } @@ -3891,6 +3958,83 @@ impl Function { } } + /// Expand the caller splat for the selected length without emitting ArrayAref. + /// Match vm_args.c's setup_parameters_complex: VM_CALL_ARGS_SPLAT stores the + /// array separately and argument setup consumes its elements as positional args. + fn expand_caller_splat_args(caller_args: &CallerArguments, caller_splat: Option) -> Vec { + let Some(splat) = caller_splat else { + return caller_args.original.iter().copied().map(SendDirectArg::Existing).collect(); + }; + + let mut args = Vec::with_capacity(caller_args.original.len() - 1 + splat.length as usize); + args.extend(caller_args.original[..splat.arg_idx].iter().copied().map(SendDirectArg::Existing)); + args.extend((0..splat.length).map(|index| SendDirectArg::SplatElement { array: splat.array, index })); + args.extend(caller_args.original[splat.arg_idx + 1..].iter().copied().map(SendDirectArg::Existing)); + args + } + + /// Select the monomorphic caller-splat length while translating the Send. + /// The selected length is attached to every receiver dispatch arm so later + /// specialization does not need to read the profile again. + fn monomorphic_caller_splat_length(&self, ci: *const rb_callinfo, state: InsnId) -> Option { + if self.policy.no_side_exits { + return None; + } + if unsafe { rb_vm_ci_flag(ci) } & VM_CALL_ARGS_SPLAT == 0 { + return None; + } + let frame_state = self.frame_state_ref(state); + let summary = get_or_create_iseq_payload(frame_state.iseq).profile.get_splat_length_summary(frame_state.insn_idx)?; + if !summary.is_monomorphic() { + return None; + } + summary.bucket(0) + } + + /// Guard the caller-splat length selected for this runtime path. + fn emit_caller_splat( + &mut self, + block: BlockId, + caller_splat: CallerSplat, + state: InsnId, + ) { + // Recompile after enough side exits have re-profiled the original Send. Any + // second observed length makes the distribution non-monomorphic, so the next + // version keeps the dynamic Send instead of emitting the same guard again. + let length = self.push_insn(block, Insn::ArrayLength { array: caller_splat.array }); + self.push_insn(block, Insn::GuardBitEquals { + val: length, + expected: Const::CInt64(i64::from(caller_splat.length)), + reason: Box::new(SideExitReason::CallerSplatLengthMismatch), + state, + recompile: Some(Recompile), + }); + + // An empty splat cannot end in a ruby2_keywords hash, so skip + // that runtime check when the profiled length is zero. + if caller_splat.length != 0 { + // A ruby2_keywords hash changes how the VM interprets the final splat + // element. Recompilation would produce the same length-based plan, so + // side-exit without recompiling when one is present. + let ruby2_keywords_splat = self.push_insn(block, Insn::CCall { + cfunc: rb_jit_ruby2_keywords_splat_p as *const u8, + recv: caller_splat.array, + args: vec![], + name: ID!(rb_jit_ruby2_keywords_splat_p), + owner: Qnil, + return_type: types::CInt64, + elidable: false, + }); + self.push_insn(block, Insn::GuardBitEquals { + val: ruby2_keywords_splat, + expected: Const::CInt64(0), + reason: Box::new(SideExitReason::CallerSplatRuby2Keywords), + state, + recompile: None, + }); + } + } + /// Reorder keyword arguments to match the callee's expected order, and synthesize /// default values for any optional keywords not provided by the caller. /// @@ -3902,10 +4046,10 @@ impl Function { /// (used by checkkeyword to determine if non-constant defaults need evaluation) fn plan_send_direct_keyword_arguments( args: Vec, - ci: *const rb_callinfo, + caller_args: &CallerArguments, iseq: IseqPtr, ) -> Result<(Vec, u32), SendFallbackReason> { - let kwarg = unsafe { rb_vm_ci_kwarg(ci) }; + let kwarg = caller_args.kwarg; let callee_keyword = unsafe { rb_get_iseq_body_param_keyword(iseq) }; if callee_keyword.is_null() { if kwarg.is_null() { @@ -3914,8 +4058,7 @@ impl Function { } let params = unsafe { iseq.params() }; - let ci_flags = unsafe { rb_vm_ci_flag(ci) }; - if ci_flags & VM_CALL_KW_SPLAT != 0 { + if caller_args.flags & VM_CALL_KW_SPLAT != 0 { // Caller **kw is one runtime Hash, not explicit keyword slots, so // there is no static key/value list to repack here. return Err(SendDirectKeywordMismatch); @@ -4487,7 +4630,7 @@ impl Function { self.try_rewrite_freeze(block, insn_id, recv, state), &Insn::Send { recv, block: None, ref args, state, cd, .. } if ruby_call_method_id(cd) == ID!(minusat) && args.is_empty() => self.try_rewrite_uminus(block, insn_id, recv, state), - &Insn::Send { mut recv, cd, state, block: send_block, .. } => { + &Insn::Send { mut recv, cd, state, block: send_block, caller_splat_length, .. } => { let mut has_block = send_block.is_some(); let (klass, profiled_type) = match self.resolve_receiver_type(recv, self.type_of(recv), state) { ReceiverTypeResolution::StaticallyKnown { class } => (class, None), @@ -4598,7 +4741,12 @@ impl Function { // If the call site info indicates that the `Function` has overly complex arguments, then do not optimize into a `SendDirect`. // Optimized methods(`VM_METHOD_TYPE_OPTIMIZED`) and C methods handle their own argument constraints (e.g., kw_splat for Proc call). // Mask out ARGS_BLOCKARG only if we've already handled the nil block arg case above. - let flags_for_check = if stripped_nil_block { flags & !VM_CALL_ARGS_BLOCKARG } else { flags }; + let mut flags_for_check = if stripped_nil_block { flags & !VM_CALL_ARGS_BLOCKARG } else { flags }; + if def_type == VM_METHOD_TYPE_ISEQ { + // Caller splat specialization currently only supports ISEQ callees, so + // skip the generic splat rejection here and validate its profile below. + flags_for_check &= !VM_CALL_ARGS_SPLAT; + } if def_type != VM_METHOD_TYPE_OPTIMIZED && def_type != VM_METHOD_TYPE_CFUNC && unspecializable_call_type(flags_for_check) { self.count_complex_call_features(block, flags, state); self.set_dynamic_send_reason(insn_id, ComplexArgPass); @@ -4610,7 +4758,27 @@ impl Function { // Only specialize positional-positional calls // TODO(max): Handle other kinds of parameter passing let iseq = unsafe { get_def_iseq_ptr((*cme).def) }; - let Ok(call) = self.build_send_direct_args(&args, ci, iseq, has_block) + let caller_args = CallerArguments::new(&args, ci); + let caller_splat = if let Some(arg_idx) = caller_args.splat_arg_idx { + // Count the profile shape for every caller-splat execution; + // complex_arg_pass_caller_splat separately tracks fallbacks. + self.count_caller_splat_profile(block, state); + // `add_iseq_to_hir` selects caller-splat lengths before building + // receiver dispatch. A Send without a selected length stays dynamic. + let Some(length) = caller_splat_length else { + self.count(block, Counter::complex_arg_pass_caller_splat); + self.set_dynamic_send_reason(insn_id, ComplexArgPass); + self.push_insn_id(block, insn_id); continue; + }; + Some(CallerSplat { + arg_idx, + array: caller_args.original[arg_idx], + length, + }) + } else { + None + }; + let Ok(call) = self.build_send_direct_args(&caller_args, caller_splat, iseq, has_block) .inspect_err(|failure| failure.record(self, block, insn_id, SendDirectFallbackContext::Send)) else { self.push_insn_id(block, insn_id); continue; }; @@ -4621,6 +4789,13 @@ impl Function { self.push_insn_id(block, insn_id); continue; } + if let Some(caller_splat) = caller_splat { + self.emit_caller_splat(block, caller_splat, state); + // Count caller-splat executions that take this optimized path. + // This is a feature-specific counter, not part of optimized_send_count. + self.count(block, Counter::caller_splat_optimized); + } + // Add PatchPoint for method redefinition self.push_insn(block, Insn::PatchPoint { invariant: Invariant::MethodRedefined { klass, method: mid, cme }, state }); @@ -4647,7 +4822,8 @@ impl Function { let capture = unsafe { proc_block.as_.captured.as_ref() }; let iseq = unsafe { *capture.code.iseq.as_ref() }; - let Ok(call) = self.build_send_direct_args(&args, ci, iseq, has_block) + let caller_args = CallerArguments::new(&args, ci); + let Ok(call) = self.build_send_direct_args(&caller_args, None, iseq, has_block) .inspect_err(|failure| failure.record(self, block, insn_id, SendDirectFallbackContext::Send)) else { self.push_insn_id(block, insn_id); continue; }; @@ -5195,7 +5371,8 @@ impl Function { // If not, we can't do direct dispatch. let super_iseq = unsafe { get_def_iseq_ptr((*super_cme).def) }; // TODO: pass Option to build_send_direct_args when we start specializing `super { ... }`. - let Ok(call) = self.build_send_direct_args(&args, ci, super_iseq, false) + let caller_args = CallerArguments::new(&args, ci); + let Ok(call) = self.build_send_direct_args(&caller_args, None, super_iseq, false) .inspect_err(|failure| failure.record(self, block, insn_id, SendDirectFallbackContext::Super)) else { self.push_insn_id(block, insn_id); continue; }; @@ -9783,7 +9960,7 @@ fn add_iseq_to_hir( } let args = state.stack_pop_n(argc as usize)?; let recv = state.stack_pop()?; - let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, state: exit_id, reason: Uncategorized(opcode.into()) }); + let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, caller_splat_length: None, state: exit_id, reason: Uncategorized(opcode.into()) }); state.stack_push(send); } YARVINSN_opt_hash_freeze => { @@ -9910,6 +10087,7 @@ fn add_iseq_to_hir( let args = state.stack_pop_n(argc as usize)?; let recv = state.stack_pop()?; + let caller_splat_length = fun.monomorphic_caller_splat_length(call_info, exit_id); if let Some(summary) = fun.polymorphic_summary(&profiles, recv, exit_id) { let join_block = fun.new_block(insn_idx); @@ -9945,19 +10123,19 @@ fn add_iseq_to_hir( // exact type, and resolve_receiver_type prefers profiles over types. profiles.copy_entries_except(exit_id, snapshot, recv, fun); let refined_recv = fun.push_insn(iftrue_block, Insn::RefineType { val: recv, new_type: expected }); - let send = fun.push_insn(iftrue_block, Insn::Send { recv: refined_recv, cd, block: None, args: args.clone(), state: snapshot, reason: Uncategorized(opcode.into()) }); + let send = fun.push_insn(iftrue_block, Insn::Send { recv: refined_recv, cd, block: None, args: args.clone(), caller_splat_length, state: snapshot, reason: Uncategorized(opcode.into()) }); fun.push_insn(iftrue_block, Insn::Jump(BranchEdge { target: join_block, args: vec![send] })); } // In the fallthrough case, do a generic interpreter send and then join. let reason = SendPolymorphicFallback; - let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, state: exit_id, reason }); + let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, caller_splat_length, state: exit_id, reason }); fun.push_insn(block, Insn::Jump(BranchEdge { target: join_block, args: vec![send] })); state.stack_push(join_param); // Continue compilation from the join block at the next instruction. block = join_block; } else { // Maybe monomorphic; handled in type_specialize - let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, state: exit_id, reason: Uncategorized(opcode.into()) }); + let send = fun.push_insn(block, Insn::Send { recv, cd, block: None, args, caller_splat_length, state: exit_id, reason: Uncategorized(opcode.into()) }); state.stack_push(send); } } @@ -9987,7 +10165,8 @@ fn add_iseq_to_hir( } else { None }; - let send = fun.push_insn(block, Insn::Send { recv, cd, block: block_handler, args, state: exit_id, reason: Uncategorized(opcode.into()) }); + let caller_splat_length = fun.monomorphic_caller_splat_length(call_info, exit_id); + let send = fun.push_insn(block, Insn::Send { recv, cd, block: block_handler, args, caller_splat_length, state: exit_id, reason: Uncategorized(opcode.into()) }); state.stack_push(send); if let Some(BlockHandler::BlockIseq(blockiseq)) = block_handler { @@ -10437,7 +10616,7 @@ fn add_iseq_to_hir( fun.push_insn(block, Insn::GuardType { val: recv, guard_type: types::String, state: exit_id, recompile: None }) } else { let recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state: exit_id, recompile: None }); - fun.push_insn(block, Insn::Send { recv, cd, block: None, args: vec![], state: exit_id, reason: ObjToStringNotString }) + fun.push_insn(block, Insn::Send { recv, cd, block: None, args: vec![], caller_splat_length: None, state: exit_id, reason: ObjToStringNotString }) } } else { let has_type = fun.push_insn(block, Insn::HasType { val: recv, expected: types::String }); @@ -10454,7 +10633,7 @@ fn add_iseq_to_hir( fun.push_insn(iftrue_block, Insn::Jump(BranchEdge { target: join_block, args: vec![refined] })); // false block let refined = fun.push_insn(iffalse_block, Insn::RefineType { val: recv, new_type: types::NotString }); - let send = fun.push_insn(iffalse_block, Insn::Send { recv: refined, cd, block: None, args: vec![], state: exit_id, reason: ObjToStringNotString }); + let send = fun.push_insn(iffalse_block, Insn::Send { recv: refined, cd, block: None, args: vec![], caller_splat_length: None, state: exit_id, reason: ObjToStringNotString }); fun.push_insn(iffalse_block, Insn::Jump(BranchEdge { target: join_block, args: vec![send] })); // join block block = join_block; diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index d2adf0478684c6..f8a8fd6d0f7a14 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -14667,34 +14667,395 @@ mod hir_opt_tests { v5:BasicObject = LoadArg :self@0 Jump bb3(v5) bb3(v8:BasicObject): - v49:NilClass = Const Value(nil) + v70:NilClass = Const Value(nil) v13:ArrayExact = NewArray v19:ArrayExact = ToArray v13 - v21:BasicObject = Send v8, :foo, v19 # SendFallbackReason: Complex argument passing - v25:StringExact[VALUE(0x1000)] = Const Value(VALUE(0x1000)) + v49:CInt64 = ArrayLength v19 + v50:CInt64[0] = GuardBitEquals v49, CInt64(0) recompile + PatchPoint MethodRedefined(Object@0x1000, foo@0x1008, cme:0x1010) + v52:ObjectSubclass[class_exact*:Object@VALUE(0x1000)] = GuardType v8, ObjectSubclass[class_exact*:Object@VALUE(0x1000)] recompile + PushInlineFrame :foo, v52 (0x1038), num_args=0 + PatchPoint MethodRedefined(Object@0x1000, itself@0x1058, cme:0x1060) + CheckInterrupts + PopInlineFrame + v25:StringExact[VALUE(0x1088)] = Const Value(VALUE(0x1088)) v26:StringExact = StringCopy v25 PatchPoint NoEPEscape(test) v31:ArrayExact = ToArray v13 v33:BasicObject = Send v26, :display, v31 # SendFallbackReason: Complex argument passing PatchPoint NoEPEscape(test) v41:ArrayExact = ToArray v13 - v43:BasicObject = Send v8, :itself, v41 # SendFallbackReason: Complex argument passing + v43:BasicObject = Send v52, :itself, v41 # SendFallbackReason: Complex argument passing CheckInterrupts Return v43 "); } #[test] - fn dont_specialize_call_to_iseq_with_monomorphic_caller_splat() { + fn inline_call_to_iseq_with_monomorphic_caller_splat() { enable_zjit_stats(); eval(" - def foo(*args) = args + def foo(a, b) = [a, b] + def test(args) = foo(*args) + test([1, 2]) + test([3, 4]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + IncrCounterPtr + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :args@1 + IncrCounterPtr + Jump bb3(v7, v8) + bb3(v11:BasicObject, v12:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v21:ArrayExact = ToArray v12 + IncrCounter zjit_insn_count + IncrCounter caller_splat_profile_monomorphic + v32:CInt64 = ArrayLength v21 + v33:CInt64[2] = GuardBitEquals v32, CInt64(2) recompile + v34:CInt64 = CCall v21, :rb_jit_ruby2_keywords_splat_p@0x1001 + v35:CInt64[0] = GuardBitEquals v34, CInt64(0) + IncrCounter caller_splat_optimized + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v38:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v11, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v39:CInt64[0] = Const CInt64(0) + v40:BasicObject = ArrayAref v21, v39 + v41:CInt64[1] = Const CInt64(1) + v42:BasicObject = ArrayAref v21, v41 + PushInlineFrame :foo, v38 (0x1040), num_args=2 + IncrCounter inline_iseq_optimized_send_count + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v57:ArrayExact = NewArray v40, v42 + IncrCounter zjit_insn_count + CheckInterrupts + PopInlineFrame + IncrCounter zjit_insn_count + Return v57 + "); + } + + #[test] + fn specialize_call_to_iseq_with_monomorphic_caller_splat() { + eval(" + def foo(arg) = arg + 1 def test(args) = foo(*args) test([1]) test([2]) "); assert_snapshot!(hir_string("test"), @" fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact = ToArray v10 + v24:CInt64 = ArrayLength v16 + v25:CInt64[1] = GuardBitEquals v24, CInt64(1) recompile + v26:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 + v27:CInt64[0] = GuardBitEquals v26, CInt64(0) + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v29:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v30:CInt64[0] = Const CInt64(0) + v31:BasicObject = ArrayAref v16, v30 + PushInlineFrame :foo, v29 (0x1040), num_args=1 + v41:Fixnum[1] = Const Value(1) + PatchPoint MethodRedefined(Integer@0x1060, +@0x1068, cme:0x1070) + v55:Fixnum = GuardType v31, Fixnum recompile + v56:Fixnum = FixnumAdd v55, v41 + CheckInterrupts + PopInlineFrame + Return v56 + "); + } + + #[test] + fn specialize_polymorphic_receiver_with_monomorphic_caller_splat() { + set_call_threshold(4); + eval(" + class CallerSplatA + def target(*args) = args + end + class CallerSplatB + def target(*args) = args + end + def test(recv, args) = recv.target(*args) + test(CallerSplatA.new, [1]) + test(CallerSplatB.new, [2]) + test(CallerSplatA.new, [3]) + test(CallerSplatB.new, [4]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:8: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :recv@0x1000 + v4:BasicObject = LoadField v2, :args@0x1001 + Jump bb3(v1, v3, v4) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :recv@1 + v9:BasicObject = LoadArg :args@2 + Jump bb3(v7, v8, v9) + bb3(v11:BasicObject, v12:BasicObject, v13:BasicObject): + v19:ArrayExact = ToArray v13 + v22:CBool = HasType v12, ObjectSubclass[class_exact:CallerSplatA] + CondBranch v22, bb5(), bb6() + bb5(): + v25:ObjectSubclass[class_exact:CallerSplatA] = RefineType v12, ObjectSubclass[class_exact:CallerSplatA] + PatchPoint NoSingletonClass(CallerSplatA@0x1008) + v42:CInt64 = ArrayLength v19 + v43:CInt64[1] = GuardBitEquals v42, CInt64(1) recompile + v44:CInt64 = CCall v19, :rb_jit_ruby2_keywords_splat_p@0x1010 + v45:CInt64[0] = GuardBitEquals v44, CInt64(0) + PatchPoint MethodRedefined(CallerSplatA@0x1008, target@0x1011, cme:0x1018) + v47:CInt64[0] = Const CInt64(0) + v48:BasicObject = ArrayAref v19, v47 + v49:ArrayExact = NewArray v48 + PushInlineFrame :target, v25 (0x1040), num_args=1 + CheckInterrupts + PopInlineFrame + Jump bb4(v49) + bb6(): + v28:CBool = HasType v12, ObjectSubclass[class_exact:CallerSplatB] + CondBranch v28, bb7(), bb8() + bb7(): + v31:ObjectSubclass[class_exact:CallerSplatB] = RefineType v12, ObjectSubclass[class_exact:CallerSplatB] + PatchPoint NoSingletonClass(CallerSplatB@0x1060) + v53:CInt64 = ArrayLength v19 + v54:CInt64[1] = GuardBitEquals v53, CInt64(1) recompile + v55:CInt64 = CCall v19, :rb_jit_ruby2_keywords_splat_p@0x1010 + v56:CInt64[0] = GuardBitEquals v55, CInt64(0) + PatchPoint MethodRedefined(CallerSplatB@0x1060, target@0x1011, cme:0x1068) + v58:CInt64[0] = Const CInt64(0) + v59:BasicObject = ArrayAref v19, v58 + v60:ArrayExact = NewArray v59 + PushInlineFrame :target, v31 (0x1090), num_args=1 + CheckInterrupts + PopInlineFrame + Jump bb4(v60) + bb8(): + v34:BasicObject = Send v12, :target, v19 # SendFallbackReason: Send: polymorphic call site + Jump bb4(v34) + bb4(v21:BasicObject): + CheckInterrupts + Return v21 + "); + } + + #[test] + fn specialize_call_to_iseq_with_empty_caller_splat() { + eval(" + def foo(arg = 1) = arg + def test(args) = foo(*args) + test([]) + test([]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact = ToArray v10 + v24:CInt64 = ArrayLength v16 + v25:CInt64[0] = GuardBitEquals v24, CInt64(0) recompile + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v27:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + PushInlineFrame :foo, v27 (0x1040), num_args=0 + v35:Fixnum[1] = Const Value(1) + CheckInterrupts + PopInlineFrame + Return v35 + "); + } + + #[test] + fn specialize_call_to_iseq_with_caller_splat_and_positional_prefix() { + eval(" + def foo(a, b, c) = [a, b, c] + def test(args) = foo(1, *args) + test([2, 3]) + test([4, 5]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + v18:ArrayExact = ToArray v10 + v26:CInt64 = ArrayLength v18 + v27:CInt64[2] = GuardBitEquals v26, CInt64(2) recompile + v28:CInt64 = CCall v18, :rb_jit_ruby2_keywords_splat_p@0x1001 + v29:CInt64[0] = GuardBitEquals v28, CInt64(0) + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v31:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v32:CInt64[0] = Const CInt64(0) + v33:BasicObject = ArrayAref v18, v32 + v34:CInt64[1] = Const CInt64(1) + v35:BasicObject = ArrayAref v18, v34 + PushInlineFrame :foo, v31 (0x1040), num_args=3 + v49:ArrayExact = NewArray v15, v33, v35 + CheckInterrupts + PopInlineFrame + Return v49 + "); + } + + #[test] + fn specialize_call_to_iseq_with_many_caller_splat_arguments_and_rest_parameter() { + eval(" + def foo(*args) = args.length + def test(args) = foo(*args) + test([1, 2, 3, 4, 5, 6, 7]) + test([8, 9, 10, 11, 12, 13, 14]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact = ToArray v10 + v24:CInt64 = ArrayLength v16 + v25:CInt64[7] = GuardBitEquals v24, CInt64(7) recompile + v26:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 + v27:CInt64[0] = GuardBitEquals v26, CInt64(0) + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v29:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v30:CInt64[0] = Const CInt64(0) + v31:BasicObject = ArrayAref v16, v30 + v32:CInt64[1] = Const CInt64(1) + v33:BasicObject = ArrayAref v16, v32 + v34:CInt64[2] = Const CInt64(2) + v35:BasicObject = ArrayAref v16, v34 + v36:CInt64[3] = Const CInt64(3) + v37:BasicObject = ArrayAref v16, v36 + v38:CInt64[4] = Const CInt64(4) + v39:BasicObject = ArrayAref v16, v38 + v40:CInt64[5] = Const CInt64(5) + v41:BasicObject = ArrayAref v16, v40 + v42:CInt64[6] = Const CInt64(6) + v43:BasicObject = ArrayAref v16, v42 + v44:ArrayExact = NewArray v31, v33, v35, v37, v39, v41, v43 + PushInlineFrame :foo, v29 (0x1040), num_args=1 + PatchPoint NoSingletonClass(Array@0x1060) + PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + v68:CInt64 = ArrayLength v44 + v69:Fixnum = BoxFixnum v68 + CheckInterrupts + PopInlineFrame + Return v69 + "); + } + + #[test] + fn specialize_call_to_iseq_with_caller_splat_and_complex_parameters() { + eval(" + def foo(a, b = 2, *rest, z, k: 40) = [a, b, rest, z, k] + def test(args) = foo(1, *args) + test([3, 4, 5]) + test([6, 7, 8]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v15:Fixnum[1] = Const Value(1) + v18:ArrayExact = ToArray v10 + v26:CInt64 = ArrayLength v18 + v27:CInt64[3] = GuardBitEquals v26, CInt64(3) recompile + v28:CInt64 = CCall v18, :rb_jit_ruby2_keywords_splat_p@0x1001 + v29:CInt64[0] = GuardBitEquals v28, CInt64(0) + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v31:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v32:CInt64[0] = Const CInt64(0) + v33:BasicObject = ArrayAref v18, v32 + v34:CInt64[1] = Const CInt64(1) + v35:BasicObject = ArrayAref v18, v34 + v36:ArrayExact = NewArray v35 + v37:CInt64[2] = Const CInt64(2) + v38:BasicObject = ArrayAref v18, v37 + v39:Fixnum[40] = Const Value(40) + v63:Fixnum[0] = Const Value(0) + PushInlineFrame :foo, v31 (0x1040), num_args=5 + v58:ArrayExact = NewArray v15, v33, v36, v38, v39 + CheckInterrupts + PopInlineFrame + Return v58 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_caller_splat_and_required_keyword() { + enable_zjit_stats(); + eval(" + def foo(*args, k:) = [args, k] + def test(args) = foo(*args, k: 40) + test([1, 2]) + test([3, 4]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: bb1(): EntryPoint interpreter v1:BasicObject = LoadSelf @@ -14714,9 +15075,104 @@ mod hir_opt_tests { IncrCounter zjit_insn_count v21:ArrayExact = ToArray v12 IncrCounter zjit_insn_count + v24:HashExact[VALUE(0x1008)] = Const Value(VALUE(0x1008)) + IncrCounter zjit_insn_count IncrCounter complex_arg_pass_caller_splat IncrCounter caller_splat_profile_monomorphic - v24:BasicObject = Send v11, :foo, v21 # SendFallbackReason: Complex argument passing + IncrCounter complex_arg_pass_caller_kw_splat + v27:BasicObject = Send v11, :foo, v21, v24 # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v27 + "); + } + + #[test] + fn specialize_call_to_iseq_with_caller_splat_and_block_literal() { + eval(" + def foo(*args) = yield args.length + def test(args) = foo(*args) { |n| n + 4 } + test([1, 2, 3]) + test([4, 5, 6]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v6:BasicObject = LoadArg :self@0 + v7:BasicObject = LoadArg :args@1 + Jump bb3(v6, v7) + bb3(v9:BasicObject, v10:BasicObject): + v16:ArrayExact = ToArray v10 + v26:CInt64 = ArrayLength v16 + v27:CInt64[3] = GuardBitEquals v26, CInt64(3) recompile + v28:CInt64 = CCall v16, :rb_jit_ruby2_keywords_splat_p@0x1001 + v29:CInt64[0] = GuardBitEquals v28, CInt64(0) + PatchPoint MethodRedefined(Object@0x1008, foo@0x1010, cme:0x1018) + v31:ObjectSubclass[class_exact*:Object@VALUE(0x1008)] = GuardType v9, ObjectSubclass[class_exact*:Object@VALUE(0x1008)] recompile + v32:CInt64[0] = Const CInt64(0) + v33:BasicObject = ArrayAref v16, v32 + v34:CInt64[1] = Const CInt64(1) + v35:BasicObject = ArrayAref v16, v34 + v36:CInt64[2] = Const CInt64(2) + v37:BasicObject = ArrayAref v16, v36 + v38:ArrayExact = NewArray v33, v35, v37 + PushInlineFrame :foo, v31 (0x1040), num_args=1 + PatchPoint NoSingletonClass(Array@0x1060) + PatchPoint MethodRedefined(Array@0x1060, length@0x1068, cme:0x1070) + v68:CInt64 = ArrayLength v38 + v69:Fixnum = BoxFixnum v68 + v51:CPtr = GetEP 0 + v52:CInt64 = LoadField v51, :VM_ENV_DATA_INDEX_SPECVAL@0x1098 + v53:CInt64[-4] = Const CInt64(-4) + v54:CInt64 = IntAnd v52, v53 + v55:BasicObject = InvokeBlockIseqDirect (0x10a0), v54, v69 + CheckInterrupts + PopInlineFrame + PatchPoint NoEPEscape(test) + Return v55 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_monomorphic_caller_splat_argc_mismatch() { + enable_zjit_stats(); + eval(" + def foo(a, b) = [a, b] + def test(args) = foo(*args) + test([1]) rescue nil + test([2]) rescue nil + "); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + IncrCounterPtr + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :args@1 + IncrCounterPtr + Jump bb3(v7, v8) + bb3(v11:BasicObject, v12:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v21:ArrayExact = ToArray v12 + IncrCounter zjit_insn_count + IncrCounter caller_splat_profile_monomorphic + IncrCounter send_direct_fallback_context_send + v24:BasicObject = Send v11, :foo, v21 # SendFallbackReason: Argument count does not match parameter count IncrCounter zjit_insn_count CheckInterrupts Return v24 @@ -14755,8 +15211,108 @@ mod hir_opt_tests { IncrCounter zjit_insn_count v21:ArrayExact = ToArray v12 IncrCounter zjit_insn_count + IncrCounter caller_splat_profile_polymorphic IncrCounter complex_arg_pass_caller_splat + v24:BasicObject = Send v11, :foo, v21 # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn dont_repeat_caller_splat_length_guard_for_skewed_polymorphic_profile() { + enable_zjit_stats(); + set_call_threshold(5); + set_max_versions(4); + // Profile length 1 on calls 1-4, then compile its monomorphic guard on call 5. + eval(" + def foo(*args) = args + def capture(*args) = args + ruby2_keywords(:capture) + def test(args) = foo(*args) + 5.times { test([1]) } + "); + + // Record a less frequent second length through the recompiling length guard. + eval("test([1, 2])"); + + // Finish the profile window with the first length. These calls exit through + // the non-recompiling ruby2_keywords guard, so the version remains active. + eval("4.times { test(capture(k: 1)) }"); + + // With the profile window complete, the next length mismatch invalidates + // the monomorphic version for recompilation. + eval("test([1, 2])"); + + // The next version must keep the dynamic Send because the accumulated + // length profile is skewed polymorphic rather than monomorphic. + assert_snapshot!(hir_string("test"), @" + fn test@:5: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + IncrCounterPtr + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :args@1 + IncrCounterPtr + Jump bb3(v7, v8) + bb3(v11:BasicObject, v12:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v21:ArrayExact = ToArray v12 + IncrCounter zjit_insn_count + IncrCounter caller_splat_profile_skewed_polymorphic + IncrCounter complex_arg_pass_caller_splat + v24:BasicObject = Send v11, :foo, v21 # SendFallbackReason: Complex argument passing + IncrCounter zjit_insn_count + CheckInterrupts + Return v24 + "); + } + + #[test] + fn dont_specialize_call_to_iseq_with_caller_splat_on_final_version() { + enable_zjit_stats(); + set_max_versions(2); + eval(" + def foo(*args) = args + def test(args) = foo(*args) + test([1]); test([1]) + "); + + // Trigger the length guard enough times to recompile under the + // no-side-exits policy. + eval("50.times { test([1, 2]) }"); + assert_snapshot!(hir_string("test"), @" + fn test@:3: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + IncrCounterPtr + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :args@1 + IncrCounterPtr + Jump bb3(v7, v8) + bb3(v11:BasicObject, v12:BasicObject): + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + IncrCounter zjit_insn_count + v21:ArrayExact = ToArray v12 + IncrCounter zjit_insn_count IncrCounter caller_splat_profile_polymorphic + IncrCounter complex_arg_pass_caller_splat v24:BasicObject = Send v11, :foo, v21 # SendFallbackReason: Complex argument passing IncrCounter zjit_insn_count CheckInterrupts @@ -14764,6 +15320,58 @@ mod hir_opt_tests { "); } + #[test] + fn specialize_call_to_iseq_with_caller_splat_result_used_by_hash_aset() { + // Hash#[]= returns its value argument from its CFunc inline. Ensure it can + // consume the guarded caller-splat result in the same specialization pass. + eval(" + def target(value) = value + def test(args) + hash = {} + hash[:value] = target(*args) + end + test([1]) + test([2]) + "); + assert_snapshot!(hir_string("test"), @" + fn test@:4: + bb1(): + EntryPoint interpreter + v1:BasicObject = LoadSelf + v2:CPtr = LoadSP + v3:BasicObject = LoadField v2, :args@0x1000 + Jump bb3(v1, v3) + bb2(): + EntryPoint JIT(0) + v7:BasicObject = LoadArg :self@0 + v8:BasicObject = LoadArg :args@1 + Jump bb3(v7, v8) + bb3(v11:BasicObject, v12:BasicObject): + v72:NilClass = Const Value(nil) + v17:HashExact = NewHash + PatchPoint NoEPEscape(test) + v23:NilClass = Const Value(nil) + v26:StaticSymbol[:value] = Const Value(VALUE(0x1008)) + v30:ArrayExact = ToArray v12 + v43:CInt64 = ArrayLength v30 + v44:CInt64[1] = GuardBitEquals v43, CInt64(1) recompile + v45:CInt64 = CCall v30, :rb_jit_ruby2_keywords_splat_p@0x1010 + v46:CInt64[0] = GuardBitEquals v45, CInt64(0) + PatchPoint MethodRedefined(Object@0x1018, target@0x1020, cme:0x1028) + v48:ObjectSubclass[class_exact*:Object@VALUE(0x1018)] = GuardType v11, ObjectSubclass[class_exact*:Object@VALUE(0x1018)] recompile + v49:CInt64[0] = Const CInt64(0) + v50:BasicObject = ArrayAref v30, v49 + PushInlineFrame :target, v48 (0x1050), num_args=1 + CheckInterrupts + PopInlineFrame + PatchPoint NoSingletonClass(Hash@0x1070) + PatchPoint MethodRedefined(Hash@0x1070, []=@0x1078, cme:0x1080) + HashAset v17, v26, v50 + CheckInterrupts + Return v50 + "); + } + #[test] fn test_inline_symbol_to_sym() { eval(r#" diff --git a/zjit/src/stats.rs b/zjit/src/stats.rs index c4476db2d9cc40..7574893b7119cf 100644 --- a/zjit/src/stats.rs +++ b/zjit/src/stats.rs @@ -246,6 +246,8 @@ make_counters! { exit_splatkw_not_nil_or_hash, exit_splatkw_polymorphic, exit_splatkw_not_profiled, + exit_caller_splat_length_mismatch, + exit_caller_splat_ruby2_keywords, exit_directive_induced, exit_send_while_tracing, exit_invokeblock_not_ifunc, @@ -442,6 +444,9 @@ make_counters! { caller_splat_profile_megamorphic, caller_splat_profile_skewed_megamorphic, + // Caller splat specialization + caller_splat_optimized, + // Contexts in which SendDirect argument planning failed. These are kept // outside dynamic_send because the detailed fallback reason is also counted. send_direct_fallback_context_send, @@ -642,6 +647,8 @@ pub fn side_exit_counter(reason: crate::hir::SideExitReason) -> Counter { SplatKwNotNilOrHash => exit_splatkw_not_nil_or_hash, SplatKwPolymorphic => exit_splatkw_polymorphic, SplatKwNotProfiled => exit_splatkw_not_profiled, + CallerSplatLengthMismatch => exit_caller_splat_length_mismatch, + CallerSplatRuby2Keywords => exit_caller_splat_ruby2_keywords, DirectiveInduced => exit_directive_induced, PatchPoint(Invariant::BOPRedefined { .. }) => exit_patchpoint_bop_redefined,