@@ -966,38 +966,37 @@ void VPlanTransforms::createLoopRegions(VPlan &Plan) {
966966 TopRegion->getEntryBasicBlock ()->setName (" vector.body" );
967967}
968968
969- // Likelyhood of bypassing the vectorized loop due to a runtime check block,
970- // including memory overlap checks block and wrapping/unit-stride checks block.
971- static constexpr uint32_t CheckBypassWeights[] = {1 , 127 };
972-
973- void VPlanTransforms::attachCheckBlock (VPlan &Plan, Value *Cond,
974- BasicBlock *CheckBlock,
975- bool AddBranchWeights) {
976- VPValue *CondVPV = Plan.getOrAddLiveIn (Cond);
977- VPBasicBlock *CheckBlockVPBB = Plan.createVPIRBasicBlock (CheckBlock);
969+ // / Insert \p CheckBlockVPBB on the edge leading to the vector preheader,
970+ // / connecting it to both vector and scalar preheaders. Updates scalar
971+ // / preheader phis to account for the new predecessor.
972+ static void insertCheckBlockBeforeVectorLoop (VPlan &Plan,
973+ VPBasicBlock *CheckBlockVPBB) {
978974 VPBlockBase *VectorPH = Plan.getVectorPreheader ();
979- VPBlockBase *ScalarPH = Plan.getScalarPreheader ();
975+ auto *ScalarPH = cast<VPBasicBlock>( Plan.getScalarPreheader () );
980976 VPBlockBase *PreVectorPH = VectorPH->getSinglePredecessor ();
981977 VPBlockUtils::insertOnEdge (PreVectorPH, VectorPH, CheckBlockVPBB);
982978 VPBlockUtils::connectBlocks (CheckBlockVPBB, ScalarPH);
983979 CheckBlockVPBB->swapSuccessors ();
984-
985- // We just connected a new block to the scalar preheader. Update all
986- // VPPhis by adding an incoming value for it, replicating the last value.
987- unsigned NumPredecessors = ScalarPH->getNumPredecessors ();
988- for (VPRecipeBase &R : cast<VPBasicBlock>(ScalarPH)->phis ()) {
989- assert (isa<VPPhi>(&R) && " Phi expected to be VPPhi" );
990- assert (cast<VPPhi>(&R)->getNumIncoming () == NumPredecessors - 1 &&
991- " must have incoming values for all operands" );
992- R.addOperand (R.getOperand (NumPredecessors - 2 ));
980+ unsigned NumPreds = ScalarPH->getNumPredecessors ();
981+ for (VPRecipeBase &R : ScalarPH->phis ()) {
982+ auto *Phi = cast<VPPhi>(&R);
983+ assert (Phi->getNumIncoming () == NumPreds - 1 &&
984+ " must have incoming values for all predecessors" );
985+ Phi->addOperand (Phi->getOperand (NumPreds - 2 ));
993986 }
987+ }
994988
995- VPIRMetadata VPBranchWeights;
996- auto *Term =
997- VPBuilder (CheckBlockVPBB)
998- .createNaryOp (
999- VPInstruction::BranchOnCond, {CondVPV},
1000- Plan.getVectorLoopRegion ()->getCanonicalIV ()->getDebugLoc ());
989+ // Likelyhood of bypassing the vectorized loop due to a runtime check block,
990+ // including memory overlap checks block and wrapping/unit-stride checks block.
991+ static constexpr uint32_t CheckBypassWeights[] = {1 , 127 };
992+
993+ // / Create a BranchOnCond terminator in \p CheckBlockVPBB. Optionally adds
994+ // / branch weights.
995+ static void addBypassBranch (VPlan &Plan, VPBasicBlock *CheckBlockVPBB,
996+ VPValue *Cond, bool AddBranchWeights) {
997+ DebugLoc DL = Plan.getVectorLoopRegion ()->getCanonicalIV ()->getDebugLoc ();
998+ auto *Term = VPBuilder (CheckBlockVPBB)
999+ .createNaryOp (VPInstruction::BranchOnCond, {Cond}, DL);
10011000 if (AddBranchWeights) {
10021001 MDBuilder MDB (Plan.getContext ());
10031002 MDNode *BranchWeights =
@@ -1006,6 +1005,15 @@ void VPlanTransforms::attachCheckBlock(VPlan &Plan, Value *Cond,
10061005 }
10071006}
10081007
1008+ void VPlanTransforms::attachCheckBlock (VPlan &Plan, Value *Cond,
1009+ BasicBlock *CheckBlock,
1010+ bool AddBranchWeights) {
1011+ VPValue *CondVPV = Plan.getOrAddLiveIn (Cond);
1012+ VPBasicBlock *CheckBlockVPBB = Plan.createVPIRBasicBlock (CheckBlock);
1013+ insertCheckBlockBeforeVectorLoop (Plan, CheckBlockVPBB);
1014+ addBypassBranch (Plan, CheckBlockVPBB, CondVPV, AddBranchWeights);
1015+ }
1016+
10091017void VPlanTransforms::addMinimumIterationCheck (
10101018 VPlan &Plan, ElementCount VF, unsigned UF,
10111019 ElementCount MinProfitableTripCount, bool RequiresScalarEpilogue,
0 commit comments