Skip to content

Commit d43e735

Browse files
committed
[VPlan] Split up attachCheckBlock in distinct helpers for re-use (NFC).
Split up attachCheckBlock into its distinct operations: * inserting the check block in the CFG + updating phis, and * adding the branch VPInstruction. Those helpers can be re-used in follow-up changes.
1 parent 1731c59 commit d43e735

1 file changed

Lines changed: 33 additions & 25 deletions

File tree

llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp

Lines changed: 33 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -966,38 +966,37 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan) {
966966
TopRegion->getEntryBasicBlock()->setName("vector.body");
967967
}
968968

969-
// Likelyhood of bypassing the vectorized loop due to a runtime check block,
970-
// including memory overlap checks block and wrapping/unit-stride checks block.
971-
static constexpr uint32_t CheckBypassWeights[] = {1, 127};
972-
973-
void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
974-
BasicBlock *CheckBlock,
975-
bool AddBranchWeights) {
976-
VPValue *CondVPV = Plan.getOrAddLiveIn(Cond);
977-
VPBasicBlock *CheckBlockVPBB = Plan.createVPIRBasicBlock(CheckBlock);
969+
/// Insert \p CheckBlockVPBB on the edge leading to the vector preheader,
970+
/// connecting it to both vector and scalar preheaders. Updates scalar
971+
/// preheader phis to account for the new predecessor.
972+
static void insertCheckBlockBeforeVectorLoop(VPlan &Plan,
973+
VPBasicBlock *CheckBlockVPBB) {
978974
VPBlockBase *VectorPH = Plan.getVectorPreheader();
979-
VPBlockBase *ScalarPH = Plan.getScalarPreheader();
975+
auto *ScalarPH = cast<VPBasicBlock>(Plan.getScalarPreheader());
980976
VPBlockBase *PreVectorPH = VectorPH->getSinglePredecessor();
981977
VPBlockUtils::insertOnEdge(PreVectorPH, VectorPH, CheckBlockVPBB);
982978
VPBlockUtils::connectBlocks(CheckBlockVPBB, ScalarPH);
983979
CheckBlockVPBB->swapSuccessors();
984-
985-
// We just connected a new block to the scalar preheader. Update all
986-
// VPPhis by adding an incoming value for it, replicating the last value.
987-
unsigned NumPredecessors = ScalarPH->getNumPredecessors();
988-
for (VPRecipeBase &R : cast<VPBasicBlock>(ScalarPH)->phis()) {
989-
assert(isa<VPPhi>(&R) && "Phi expected to be VPPhi");
990-
assert(cast<VPPhi>(&R)->getNumIncoming() == NumPredecessors - 1 &&
991-
"must have incoming values for all operands");
992-
R.addOperand(R.getOperand(NumPredecessors - 2));
980+
unsigned NumPreds = ScalarPH->getNumPredecessors();
981+
for (VPRecipeBase &R : ScalarPH->phis()) {
982+
auto *Phi = cast<VPPhi>(&R);
983+
assert(Phi->getNumIncoming() == NumPreds - 1 &&
984+
"must have incoming values for all predecessors");
985+
Phi->addOperand(Phi->getOperand(NumPreds - 2));
993986
}
987+
}
994988

995-
VPIRMetadata VPBranchWeights;
996-
auto *Term =
997-
VPBuilder(CheckBlockVPBB)
998-
.createNaryOp(
999-
VPInstruction::BranchOnCond, {CondVPV},
1000-
Plan.getVectorLoopRegion()->getCanonicalIV()->getDebugLoc());
989+
// Likelyhood of bypassing the vectorized loop due to a runtime check block,
990+
// including memory overlap checks block and wrapping/unit-stride checks block.
991+
static constexpr uint32_t CheckBypassWeights[] = {1, 127};
992+
993+
/// Create a BranchOnCond terminator in \p CheckBlockVPBB. Optionally adds
994+
/// branch weights.
995+
static void addBypassBranch(VPlan &Plan, VPBasicBlock *CheckBlockVPBB,
996+
VPValue *Cond, bool AddBranchWeights) {
997+
DebugLoc DL = Plan.getVectorLoopRegion()->getCanonicalIV()->getDebugLoc();
998+
auto *Term = VPBuilder(CheckBlockVPBB)
999+
.createNaryOp(VPInstruction::BranchOnCond, {Cond}, DL);
10011000
if (AddBranchWeights) {
10021001
MDBuilder MDB(Plan.getContext());
10031002
MDNode *BranchWeights =
@@ -1006,6 +1005,15 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
10061005
}
10071006
}
10081007

1008+
void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
1009+
BasicBlock *CheckBlock,
1010+
bool AddBranchWeights) {
1011+
VPValue *CondVPV = Plan.getOrAddLiveIn(Cond);
1012+
VPBasicBlock *CheckBlockVPBB = Plan.createVPIRBasicBlock(CheckBlock);
1013+
insertCheckBlockBeforeVectorLoop(Plan, CheckBlockVPBB);
1014+
addBypassBranch(Plan, CheckBlockVPBB, CondVPV, AddBranchWeights);
1015+
}
1016+
10091017
void VPlanTransforms::addMinimumIterationCheck(
10101018
VPlan &Plan, ElementCount VF, unsigned UF,
10111019
ElementCount MinProfitableTripCount, bool RequiresScalarEpilogue,

0 commit comments

Comments
 (0)