Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 4acf4d4

Browse filesBrowse files
fhahnfhahn
committed
[VPlan] Connect Entry to scalar preheader during initial construction.
Update initial construction to connect the Plan's entry to the scalar preheader during initial construction. This moves a small part of the skeleton creation out of ILV and will also enable replacing VPInstruction::ResumePhi with regular VPPhi recipes. Resume phis need 2 incoming values to start with, the second being the bypass value from the scalar ph (and used to replicate the incoming value for other bypass blocks). Adding the extra edge ensures we incoming values for resume phis match the incoming blocks. Still needs various VPlan printing tests to be updated. vplan-printing.ll shows the changes.
1 parent e81fab6 commit 4acf4d4
Copy full SHA for 4acf4d4

File tree

Expand file treeCollapse file tree

5 files changed

+72
-71
lines changed
Filter options
Expand file treeCollapse file tree

5 files changed

+72
-71
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+10-16Lines changed: 10 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -490,7 +490,7 @@ class InnerLoopVectorizer {
490490
MinProfitableTripCount(MinProfitableTripCount), UF(UnrollFactor),
491491
Builder(PSE.getSE()->getContext()), Cost(CM), BFI(BFI), PSI(PSI),
492492
RTChecks(RTChecks), Plan(Plan),
493-
VectorPHVPB(Plan.getEntry()->getSingleSuccessor()) {}
493+
VectorPHVPB(Plan.getEntry()->getSuccessors()[1]) {}
494494

495495
virtual ~InnerLoopVectorizer() = default;
496496

@@ -2365,14 +2365,11 @@ InnerLoopVectorizer::getOrCreateVectorTripCount(BasicBlock *InsertBlock) {
23652365
void InnerLoopVectorizer::introduceCheckBlockInVPlan(BasicBlock *CheckIRBB) {
23662366
VPBlockBase *ScalarPH = Plan.getScalarPreheader();
23672367
VPBlockBase *PreVectorPH = VectorPHVPB->getSinglePredecessor();
2368-
if (PreVectorPH->getNumSuccessors() != 1) {
2369-
assert(PreVectorPH->getNumSuccessors() == 2 && "Expected 2 successors");
2370-
assert(PreVectorPH->getSuccessors()[0] == ScalarPH &&
2371-
"Unexpected successor");
2372-
VPIRBasicBlock *CheckVPIRBB = Plan.createVPIRBasicBlock(CheckIRBB);
2373-
VPBlockUtils::insertOnEdge(PreVectorPH, VectorPHVPB, CheckVPIRBB);
2374-
PreVectorPH = CheckVPIRBB;
2375-
}
2368+
assert(PreVectorPH->getNumSuccessors() == 2 && "Expected 2 successors");
2369+
assert(PreVectorPH->getSuccessors()[0] == ScalarPH && "Unexpected successor");
2370+
VPIRBasicBlock *CheckVPIRBB = Plan.createVPIRBasicBlock(CheckIRBB);
2371+
VPBlockUtils::insertOnEdge(PreVectorPH, VectorPHVPB, CheckVPIRBB);
2372+
PreVectorPH = CheckVPIRBB;
23762373
VPBlockUtils::connectBlocks(PreVectorPH, ScalarPH);
23772374
PreVectorPH->swapSuccessors();
23782375

@@ -2463,9 +2460,6 @@ void InnerLoopVectorizer::emitIterationCountCheck(BasicBlock *Bypass) {
24632460
setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false);
24642461
ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI);
24652462
LoopBypassBlocks.push_back(TCCheckBlock);
2466-
2467-
// TODO: Wrap LoopVectorPreHeader in VPIRBasicBlock here.
2468-
introduceCheckBlockInVPlan(TCCheckBlock);
24692463
}
24702464

24712465
BasicBlock *InnerLoopVectorizer::emitSCEVChecks(BasicBlock *Bypass) {
@@ -7837,7 +7831,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
78377831

78387832
// 1. Set up the skeleton for vectorization, including vector pre-header and
78397833
// middle block. The vector loop is created during VPlan execution.
7840-
VPBasicBlock *VectorPH = cast<VPBasicBlock>(Entry->getSingleSuccessor());
7834+
VPBasicBlock *VectorPH = cast<VPBasicBlock>(Entry->getSuccessors()[1]);
78417835
State.CFG.PrevBB = ILV.createVectorizedLoopSkeleton();
78427836
if (VectorizingEpilogue)
78437837
VPlanTransforms::removeDeadRecipes(BestVPlan);
@@ -8070,7 +8064,8 @@ EpilogueVectorizerMainLoop::emitIterationCountCheck(BasicBlock *Bypass,
80708064
setBranchWeights(BI, MinItersBypassWeights, /*IsExpected=*/false);
80718065
ReplaceInstWithInst(TCCheckBlock->getTerminator(), &BI);
80728066

8073-
introduceCheckBlockInVPlan(TCCheckBlock);
8067+
if (!ForEpilogue)
8068+
introduceCheckBlockInVPlan(TCCheckBlock);
80748069
return TCCheckBlock;
80758070
}
80768071

@@ -8200,7 +8195,6 @@ EpilogueVectorizerEpilogueLoop::emitMinimumVectorEpilogueIterCountCheck(
82008195
Plan.setEntry(NewEntry);
82018196
// OldEntry is now dead and will be cleaned up when the plan gets destroyed.
82028197

8203-
introduceCheckBlockInVPlan(Insert);
82048198
return Insert;
82058199
}
82068200

@@ -9160,7 +9154,7 @@ static void addScalarResumePhis(VPRecipeBuilder &Builder, VPlan &Plan,
91609154
DenseMap<VPValue *, VPValue *> &IVEndValues) {
91619155
VPTypeAnalysis TypeInfo(Plan.getCanonicalIV()->getScalarType());
91629156
auto *ScalarPH = Plan.getScalarPreheader();
9163-
auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getSinglePredecessor());
9157+
auto *MiddleVPBB = cast<VPBasicBlock>(ScalarPH->getPredecessors()[0]);
91649158
VPRegionBlock *VectorRegion = Plan.getVectorLoopRegion();
91659159
VPBuilder VectorPHBuilder(
91669160
cast<VPBasicBlock>(VectorRegion->getSinglePredecessor()));

‎llvm/lib/Transforms/Vectorize/VPlan.h

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlan.h
+2-1Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4246,7 +4246,8 @@ class VPlan {
42464246
/// that this relies on unneeded branches to the scalar tail loop being
42474247
/// removed.
42484248
bool hasScalarTail() const {
4249-
return getScalarPreheader()->getNumPredecessors() != 0;
4249+
return getScalarPreheader()->getNumPredecessors() != 0 &&
4250+
getScalarPreheader()->getSinglePredecessor() != getEntry();
42504251
}
42514252
};
42524253

‎llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlanConstruction.cpp
+6Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -546,6 +546,9 @@ void VPlanTransforms::prepareForVectorization(
546546
if (auto *LatchExitVPB = MiddleVPBB->getSingleSuccessor())
547547
VPBlockUtils::disconnectBlocks(MiddleVPBB, LatchExitVPB);
548548
VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
549+
VPBlockUtils::connectBlocks(Plan.getEntry(), ScalarPH);
550+
Plan.getEntry()->swapSuccessors();
551+
549552
// The exit blocks are unreachable, remove their recipes to make sure no
550553
// users remain that may pessimize transforms.
551554
for (auto *EB : Plan.getExitBlocks()) {
@@ -558,6 +561,9 @@ void VPlanTransforms::prepareForVectorization(
558561
// The connection order corresponds to the operands of the conditional branch,
559562
// with the middle block already connected to the exit block.
560563
VPBlockUtils::connectBlocks(MiddleVPBB, ScalarPH);
564+
// Also connect the entry block to the scalar preheader.
565+
VPBlockUtils::connectBlocks(Plan.getEntry(), ScalarPH);
566+
Plan.getEntry()->swapSuccessors();
561567

562568
auto *ScalarLatchTerm = TheLoop->getLoopLatch()->getTerminator();
563569
// Here we use the same DebugLoc as the scalar loop latch terminator instead

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1732,7 +1732,7 @@ static void removeBranchOnCondTrue(VPlan &Plan) {
17321732
using namespace llvm::VPlanPatternMatch;
17331733
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
17341734
vp_depth_first_shallow(Plan.getEntry()))) {
1735-
if (VPBB->getNumSuccessors() != 2 ||
1735+
if (VPBB->getNumSuccessors() != 2 || isa<VPIRBasicBlock>(VPBB) ||
17361736
!match(&VPBB->back(), m_BranchOnCond(m_True())))
17371737
continue;
17381738

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.