Skip to content

Commit 6d6f927

Browse files
tekknolagitekknolagi
authored
ZJIT: Merge reduce_send_without_block_to_ccall and reduce_send_to_ccall (ruby#16675)
Thread through special handling of the potential block argument.
1 parent d3f7d2c commit 6d6f927

2 files changed

Lines changed: 81 additions & 249 deletions

File tree

‎zjit/src/hir.rs‎

Lines changed: 79 additions & 247 deletions
Original file line numberDiff line numberDiff line change
@@ -4799,14 +4799,28 @@ impl Function {
47994799
}
48004800

48014801
let blockiseq = match send_block {
4802-
None | Some(BlockHandler::BlockArg) => unreachable!("went to reduce_send_without_block_to_ccall"),
4802+
Some(BlockHandler::BlockArg) => unreachable!("unsupported &block should have been filtered out"),
48034803
Some(BlockHandler::BlockIseq(blockiseq)) => Some(blockiseq),
4804+
None => None,
48044805
};
48054806

48064807
let cfunc = unsafe { get_cme_def_body_cfunc(cme) };
48074808
// Find the `argc` (arity) of the C method, which describes the parameters it expects
48084809
let cfunc_argc = unsafe { get_mct_argc(cfunc) };
48094810
let cfunc_ptr = unsafe { get_mct_func(cfunc) }.cast();
4811+
let name = unsafe { (*cme).called_id };
4812+
4813+
// Look up annotations
4814+
let props = ZJITState::get_method_annotations().get_cfunc_properties(cme);
4815+
if props.is_none() && get_option!(stats) {
4816+
fun.count_not_annotated_cfunc(block, cme);
4817+
}
4818+
let props = props.unwrap_or_default();
4819+
let return_type = props.return_type;
4820+
let elidable = match blockiseq {
4821+
Some(_) => false, // Don't consider cfuncs with block arguments as elidable for now
4822+
None => props.elidable,
4823+
};
48104824

48114825
match cfunc_argc {
48124826
0.. => {
@@ -4832,20 +4846,48 @@ impl Function {
48324846
fun.insn_types[recv.0] = fun.infer_type(recv);
48334847
}
48344848

4835-
// Emit a call
4836-
let cfunc = unsafe { get_mct_func(cfunc) }.cast();
4849+
// Try inlining the cfunc into HIR. Only inline if we don't have a block argument
4850+
if blockiseq.is_none() {
4851+
let tmp_block = fun.new_block(u32::MAX);
4852+
if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) {
4853+
// Copy contents of tmp_block to block
4854+
assert_ne!(block, tmp_block);
4855+
let insns = std::mem::take(&mut fun.blocks[tmp_block.0].insns);
4856+
fun.blocks[block.0].insns.extend(insns);
4857+
fun.count(block, Counter::inline_cfunc_optimized_send_count);
4858+
fun.make_equal_to(send_insn_id, replacement);
4859+
if fun.type_of(replacement).bit_equal(types::Any) {
4860+
// Not set yet; infer type
4861+
fun.insn_types[replacement.0] = fun.infer_type(replacement);
4862+
}
4863+
fun.remove_block(tmp_block);
4864+
return Ok(());
4865+
}
4866+
4867+
// Only allow leaf calls if we don't have a block argument
4868+
if props.leaf && props.no_gc {
4869+
fun.count(block, Counter::inline_cfunc_optimized_send_count);
4870+
let owner = unsafe { (*cme).owner };
4871+
let ccall = fun.push_insn(block, Insn::CCall { cfunc: cfunc_ptr, recv, args, name, owner, return_type, elidable });
4872+
fun.make_equal_to(send_insn_id, ccall);
4873+
return Ok(());
4874+
}
4875+
}
48374876

4838-
let name = unsafe { (*cme).called_id };
4877+
// Emit a call
4878+
if get_option!(stats) {
4879+
fun.count_not_inlined_cfunc(block, cme);
4880+
}
48394881
let ccall = fun.push_insn(block, Insn::CCallWithFrame {
48404882
cd,
4841-
cfunc,
4883+
cfunc: cfunc_ptr,
48424884
recv,
48434885
args,
48444886
cme,
48454887
name,
48464888
state,
4847-
return_type: types::BasicObject,
4848-
elidable: false,
4889+
return_type,
4890+
elidable,
48494891
block: blockiseq.map(BlockHandler::BlockIseq),
48504892
});
48514893
fun.make_equal_to(send_insn_id, ccall);
@@ -4870,215 +4912,8 @@ impl Function {
48704912
fun.insn_types[recv.0] = fun.infer_type(recv);
48714913
}
48724914

4873-
if get_option!(stats) {
4874-
fun.count_not_inlined_cfunc(block, cme);
4875-
}
4876-
4877-
let ccall = fun.push_insn(block, Insn::CCallVariadic {
4878-
cfunc: cfunc_ptr,
4879-
recv,
4880-
args,
4881-
cme,
4882-
name: method_id,
4883-
state,
4884-
return_type: types::BasicObject,
4885-
elidable: false,
4886-
block: blockiseq.map(BlockHandler::BlockIseq),
4887-
});
4888-
4889-
fun.make_equal_to(send_insn_id, ccall);
4890-
Ok(())
4891-
}
4892-
-2 => {
4893-
// (self, args_ruby_array)
4894-
fun.set_dynamic_send_reason(send_insn_id, SendCfuncArrayVariadic);
4895-
Err(())
4896-
}
4897-
_ => unreachable!("unknown cfunc kind: argc={argc}")
4898-
}
4899-
}
4900-
4901-
// Try to reduce a Send insn with blockiseq: None to a CCall/CCallWithFrame
4902-
fn reduce_send_without_block_to_ccall(
4903-
fun: &mut Function,
4904-
block: BlockId,
4905-
self_type: Type,
4906-
send: Insn,
4907-
send_insn_id: InsnId,
4908-
) -> Result<(), ()> {
4909-
let Insn::Send { mut recv, cd, args, state, .. } = send else {
4910-
return Err(());
4911-
};
4912-
4913-
let call_info = unsafe { (*cd).ci };
4914-
let argc = unsafe { vm_ci_argc(call_info) };
4915-
let method_id = unsafe { rb_vm_ci_mid(call_info) };
4916-
4917-
// If we have info about the class of the receiver
4918-
let iseq_insn_idx = fun.frame_state(state).insn_idx;
4919-
let (recv_class, profiled_type) = match fun.resolve_receiver_type(recv, self_type, iseq_insn_idx) {
4920-
ReceiverTypeResolution::StaticallyKnown { class } => (class, None),
4921-
ReceiverTypeResolution::Monomorphic { profiled_type }
4922-
| ReceiverTypeResolution::SkewedPolymorphic { profiled_type } => (profiled_type.class(), Some(profiled_type)),
4923-
ReceiverTypeResolution::SkewedMegamorphic { .. } | ReceiverTypeResolution::Polymorphic | ReceiverTypeResolution::Megamorphic | ReceiverTypeResolution::NoProfile => return Err(()),
4924-
};
4925-
4926-
// Do method lookup
4927-
let mut cme: *const rb_callable_method_entry_struct = unsafe { rb_callable_method_entry(recv_class, method_id) };
4928-
if cme.is_null() {
4929-
fun.set_dynamic_send_reason(send_insn_id, SendWithoutBlockNotOptimizedMethodType(MethodType::Null));
4930-
return Err(());
4931-
}
4932-
4933-
// Filter for C methods
4934-
let mut def_type = unsafe { get_cme_def_type(cme) };
4935-
while def_type == VM_METHOD_TYPE_ALIAS {
4936-
cme = unsafe { rb_aliased_callable_method_entry(cme) };
4937-
def_type = unsafe { get_cme_def_type(cme) };
4938-
}
4939-
if def_type != VM_METHOD_TYPE_CFUNC {
4940-
return Err(());
4941-
}
4942-
4943-
let ci_flags = unsafe { vm_ci_flag(call_info) };
4944-
let visibility = unsafe { METHOD_ENTRY_VISI(cme) };
4945-
match (visibility, ci_flags & VM_CALL_FCALL != 0) {
4946-
(METHOD_VISI_PUBLIC, _) => {}
4947-
(METHOD_VISI_PRIVATE, true) => {}
4948-
(METHOD_VISI_PROTECTED, true) => {}
4949-
_ => {
4950-
fun.set_dynamic_send_reason(send_insn_id, SendWithoutBlockNotOptimizedNeedPermission);
4951-
return Err(());
4952-
}
4953-
}
4954-
4955-
// Find the `argc` (arity) of the C method, which describes the parameters it expects
4956-
let cfunc = unsafe { get_cme_def_body_cfunc(cme) };
4957-
let cfunc_argc = unsafe { get_mct_argc(cfunc) };
4958-
match cfunc_argc {
4959-
0.. => {
4960-
// (self, arg0, arg1, ..., argc) form
4961-
//
4962-
// Bail on argc mismatch
4963-
if argc != cfunc_argc as u32 {
4964-
return Err(());
4965-
}
4966-
4967-
// Filter for simple call sites (i.e. no splats etc.)
4968-
if ci_flags & VM_CALL_ARGS_SIMPLE == 0 {
4969-
// Only count features NOT already counted in type_specialize.
4970-
if !unspecializable_call_type(ci_flags) {
4971-
fun.count_complex_call_features(block, ci_flags);
4972-
}
4973-
fun.set_dynamic_send_reason(send_insn_id, ComplexArgPass);
4974-
return Err(());
4975-
}
4976-
4977-
// Check singleton class assumption first, before emitting other patchpoints
4978-
if !fun.assume_no_singleton_classes(block, recv_class, state) {
4979-
fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen);
4980-
return Err(());
4981-
}
4982-
4983-
// Commit to the replacement. Put PatchPoint.
4984-
fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state);
4985-
4986-
let props = ZJITState::get_method_annotations().get_cfunc_properties(cme);
4987-
if props.is_none() && get_option!(stats) {
4988-
fun.count_not_annotated_cfunc(block, cme);
4989-
}
4990-
let props = props.unwrap_or_default();
4991-
4992-
if let Some(profiled_type) = profiled_type {
4993-
// Guard receiver class
4994-
recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state });
4995-
fun.insn_types[recv.0] = fun.infer_type(recv);
4996-
}
4997-
4998-
// Try inlining the cfunc into HIR
4999-
let tmp_block = fun.new_block(u32::MAX);
5000-
if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) {
5001-
// Copy contents of tmp_block to block
5002-
assert_ne!(block, tmp_block);
5003-
let insns = std::mem::take(&mut fun.blocks[tmp_block.0].insns);
5004-
fun.blocks[block.0].insns.extend(insns);
5005-
fun.count(block, Counter::inline_cfunc_optimized_send_count);
5006-
fun.make_equal_to(send_insn_id, replacement);
5007-
if fun.type_of(replacement).bit_equal(types::Any) {
5008-
// Not set yet; infer type
5009-
fun.insn_types[replacement.0] = fun.infer_type(replacement);
5010-
}
5011-
fun.remove_block(tmp_block);
5012-
return Ok(());
5013-
}
5014-
5015-
// No inlining; emit a call
5016-
let cfunc = unsafe { get_mct_func(cfunc) }.cast();
5017-
let name = unsafe { (*cme).called_id };
5018-
let owner = unsafe { (*cme).owner };
5019-
let return_type = props.return_type;
5020-
let elidable = props.elidable;
5021-
// Filter for a leaf and GC free function
5022-
if props.leaf && props.no_gc {
5023-
fun.count(block, Counter::inline_cfunc_optimized_send_count);
5024-
let ccall = fun.push_insn(block, Insn::CCall { cfunc, recv, args, name, owner, return_type, elidable });
5025-
fun.make_equal_to(send_insn_id, ccall);
5026-
} else {
5027-
if get_option!(stats) {
5028-
fun.count_not_inlined_cfunc(block, cme);
5029-
}
5030-
let ccall = fun.push_insn(block, Insn::CCallWithFrame {
5031-
cd,
5032-
cfunc,
5033-
recv,
5034-
args,
5035-
cme,
5036-
name,
5037-
state,
5038-
return_type,
5039-
elidable,
5040-
block: None,
5041-
});
5042-
fun.make_equal_to(send_insn_id, ccall);
5043-
}
5044-
5045-
return Ok(());
5046-
}
5047-
// Variadic method
5048-
-1 => {
5049-
// The method gets a pointer to the first argument
5050-
// func(int argc, VALUE *argv, VALUE recv)
5051-
let ci_flags = unsafe { vm_ci_flag(call_info) };
5052-
if ci_flags & VM_CALL_ARGS_SIMPLE == 0 {
5053-
// Only count features NOT already counted in type_specialize.
5054-
if !unspecializable_call_type(ci_flags) {
5055-
fun.count_complex_call_features(block, ci_flags);
5056-
}
5057-
fun.set_dynamic_send_reason(send_insn_id, ComplexArgPass);
5058-
return Err(());
5059-
} else {
5060-
// Check singleton class assumption first, before emitting other patchpoints
5061-
if !fun.assume_no_singleton_classes(block, recv_class, state) {
5062-
fun.set_dynamic_send_reason(send_insn_id, SingletonClassSeen);
5063-
return Err(());
5064-
}
5065-
5066-
fun.gen_patch_points_for_optimized_ccall(block, recv_class, method_id, cme, state);
5067-
5068-
if let Some(profiled_type) = profiled_type {
5069-
// Guard receiver class
5070-
recv = fun.push_insn(block, Insn::GuardType { val: recv, guard_type: Type::from_profiled_type(profiled_type), state });
5071-
fun.insn_types[recv.0] = fun.infer_type(recv);
5072-
}
5073-
5074-
let cfunc = unsafe { get_mct_func(cfunc) }.cast();
5075-
let props = ZJITState::get_method_annotations().get_cfunc_properties(cme);
5076-
if props.is_none() && get_option!(stats) {
5077-
fun.count_not_annotated_cfunc(block, cme);
5078-
}
5079-
let props = props.unwrap_or_default();
5080-
5081-
// Try inlining the cfunc into HIR
4915+
// Try inlining the cfunc into HIR. Only inline if we don't have a block argument
4916+
if blockiseq.is_none() {
50824917
let tmp_block = fun.new_block(u32::MAX);
50834918
if let Some(replacement) = (props.inline)(fun, tmp_block, recv, &args, state) {
50844919
// Copy contents of tmp_block to block
@@ -5095,40 +4930,43 @@ impl Function {
50954930
return Ok(());
50964931
}
50974932

5098-
// No inlining; emit a call
5099-
if get_option!(stats) {
5100-
fun.count_not_inlined_cfunc(block, cme);
4933+
// Only allow leaf calls if we don't have a block argument
4934+
if props.leaf && props.no_gc {
4935+
fun.count(block, Counter::inline_cfunc_optimized_send_count);
4936+
let owner = unsafe { (*cme).owner };
4937+
let ccall = fun.push_insn(block, Insn::CCall { cfunc: cfunc_ptr, recv, args, name, owner, return_type, elidable });
4938+
fun.make_equal_to(send_insn_id, ccall);
4939+
return Ok(());
51014940
}
5102-
let return_type = props.return_type;
5103-
let elidable = props.elidable;
5104-
let name = unsafe { (*cme).called_id };
5105-
let ccall = fun.push_insn(block, Insn::CCallVariadic {
5106-
cfunc,
5107-
recv,
5108-
args,
5109-
cme,
5110-
name,
5111-
state,
5112-
return_type,
5113-
elidable,
5114-
block: None,
5115-
});
4941+
}
51164942

5117-
fun.make_equal_to(send_insn_id, ccall);
5118-
return Ok(())
4943+
// No inlining; emit a call
4944+
if get_option!(stats) {
4945+
fun.count_not_inlined_cfunc(block, cme);
51194946
}
51204947

5121-
// Fall through for complex cases (splat, kwargs, etc.)
4948+
let ccall = fun.push_insn(block, Insn::CCallVariadic {
4949+
cfunc: cfunc_ptr,
4950+
recv,
4951+
args,
4952+
cme,
4953+
name: method_id,
4954+
state,
4955+
return_type,
4956+
elidable,
4957+
block: blockiseq.map(BlockHandler::BlockIseq),
4958+
});
4959+
4960+
fun.make_equal_to(send_insn_id, ccall);
4961+
Ok(())
51224962
}
51234963
-2 => {
5124-
// (self, args_ruby_array) parameter form
5125-
// Falling through for now
5126-
fun.set_dynamic_send_reason(send_insn_id, SendWithoutBlockCfuncArrayVariadic);
4964+
// (self, args_ruby_array)
4965+
fun.set_dynamic_send_reason(send_insn_id, SendCfuncArrayVariadic);
4966+
Err(())
51274967
}
51284968
_ => unreachable!("unknown cfunc kind: argc={argc}")
51294969
}
5130-
5131-
Err(())
51324970
}
51334971

51344972
for block in self.rpo() {
@@ -5137,12 +4975,6 @@ impl Function {
51374975
for insn_id in old_insns {
51384976
let send = self.find(insn_id);
51394977
match send {
5140-
send @ Insn::Send { recv, block: None, .. } => {
5141-
let recv_type = self.type_of(recv);
5142-
if reduce_send_without_block_to_ccall(self, block, recv_type, send, insn_id).is_ok() {
5143-
continue;
5144-
}
5145-
}
51464978
send @ Insn::Send { recv, .. } => {
51474979
let recv_type = self.type_of(recv);
51484980
if reduce_send_to_ccall(self, block, recv_type, send, insn_id).is_ok() {

0 commit comments

Comments
 (0)