From 892991bdc1a5068d74a8597cd0ccf3092afffabf Mon Sep 17 00:00:00 2001 From: Takashi Kokubun Date: Fri, 3 Apr 2026 09:13:39 -0700 Subject: [PATCH] ZJIT: Fallback ifunc invokeblock when tag check fails (#16608) --- zjit/src/hir.rs | 35 +++++++++++++++++++++++------------ zjit/src/hir/opt_tests.rs | 34 ++++++++++++++++++++++++---------- 2 files changed, 47 insertions(+), 22 deletions(-) diff --git a/zjit/src/hir.rs b/zjit/src/hir.rs index bc7be7498a3c81..fa911fc41da470 100644 --- a/zjit/src/hir.rs +++ b/zjit/src/hir.rs @@ -8051,7 +8051,7 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { }); let result = if is_ifunc { - // Get the local EP to load the block handler + // Load the block handler from LEP let level = get_lvar_level(fun.iseq); let lep = fun.push_insn(block, Insn::GetEP { level }); let block_handler = fun.push_insn(block, Insn::LoadField { @@ -8061,23 +8061,34 @@ pub fn iseq_to_hir(iseq: *const rb_iseq_t) -> Result { return_type: types::CInt64, }); - // Guard that the block handler is an IFUNC (tag bits & 0x3 == 0x3), - // matching VM_BH_IFUNC_P() in the interpreter. + // Check IFUNC tag: (block_handler & 0x3) == 0x3 let tag_mask = fun.push_insn(block, Insn::Const { val: Const::CInt64(0x3) }); let tag_bits = fun.push_insn(block, Insn::IntAnd { left: block_handler, right: tag_mask }); - fun.push_insn(block, Insn::GuardBitEquals { - val: tag_bits, - expected: Const::CInt64(0x3), - reason: SideExitReason::InvokeBlockNotIfunc, - state: exit_id, - }); + let ifunc_tag = fun.push_insn(block, Insn::Const { val: Const::CInt64(0x3) }); + let is_ifunc_match = fun.push_insn(block, Insn::IsBitEqual { left: tag_bits, right: ifunc_tag }); - fun.push_insn(block, Insn::InvokeBlockIfunc { + // Branch: on match, call InvokeBlockIfunc directly + let join_block = fun.new_block(insn_idx); + let join_param = fun.push_insn(join_block, Insn::Param); + let ifunc_block = fun.new_block(insn_idx); + fun.push_insn(block, Insn::IfTrue { val: is_ifunc_match, target: BranchEdge { target: ifunc_block, args: vec![] } }); + let ifunc_result = fun.push_insn(ifunc_block, Insn::InvokeBlockIfunc { cd, block_handler, - args, + args: args.clone(), state: exit_id, - }) + }); + fun.push_insn(ifunc_block, Insn::Jump(BranchEdge { target: join_block, args: vec![ifunc_result] })); + + // In the fallthrough case, use generic rb_vm_invokeblock and join + let fallback_result = fun.push_insn(block, Insn::InvokeBlock { + cd, args, state: exit_id, reason: InvokeBlockNotSpecialized, + }); + fun.push_insn(block, Insn::Jump(BranchEdge { target: join_block, args: vec![fallback_result] })); + + // Continue compilation from the join block + block = join_block; + join_param } else { fun.push_insn(block, Insn::InvokeBlock { cd, args, state: exit_id, reason: InvokeBlockNotSpecialized }) }; diff --git a/zjit/src/hir/opt_tests.rs b/zjit/src/hir/opt_tests.rs index fde0ab0583d279..5bef039ed123f1 100644 --- a/zjit/src/hir/opt_tests.rs +++ b/zjit/src/hir/opt_tests.rs @@ -15268,17 +15268,31 @@ mod hir_opt_tests { v13:CInt64 = LoadField v12, :_env_data_index_specval@0x1000 v14:CInt64[3] = Const CInt64(3) v15:CInt64 = IntAnd v13, v14 - v16:CInt64[3] = GuardBitEquals v15, CInt64(3) - v17:BasicObject = InvokeBlockIfunc v13, v10 - v21:Fixnum[2] = Const Value(2) - v23:CPtr = GetEP 0 - v24:CInt64 = LoadField v23, :_env_data_index_specval@0x1000 - v25:CInt64[3] = Const CInt64(3) - v26:CInt64 = IntAnd v24, v25 - v27:CInt64[3] = GuardBitEquals v26, CInt64(3) - v28:BasicObject = InvokeBlockIfunc v24, v21 + v16:CInt64[3] = Const CInt64(3) + v17:CBool = IsBitEqual v15, v16 + IfTrue v17, bb5() + v22:BasicObject = InvokeBlock, v10 # SendFallbackReason: InvokeBlock: not yet specialized + Jump bb4(v22) + bb5(): + v20:BasicObject = InvokeBlockIfunc v13, v10 + Jump bb4(v20) + bb4(v18:BasicObject): + v27:Fixnum[2] = Const Value(2) + v29:CPtr = GetEP 0 + v30:CInt64 = LoadField v29, :_env_data_index_specval@0x1000 + v31:CInt64[3] = Const CInt64(3) + v32:CInt64 = IntAnd v30, v31 + v33:CInt64[3] = Const CInt64(3) + v34:CBool = IsBitEqual v32, v33 + IfTrue v34, bb7() + v39:BasicObject = InvokeBlock, v27 # SendFallbackReason: InvokeBlock: not yet specialized + Jump bb6(v39) + bb7(): + v37:BasicObject = InvokeBlockIfunc v30, v27 + Jump bb6(v37) + bb6(v35:BasicObject): CheckInterrupts - Return v28 + Return v35 "); } }