Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit a7fda0e

Browse filesBrowse files
fhahnfhahn
authored
[VPlan] Introduce VPScalarPHIRecipe, use for can & EVL IV codegen (NFC). (#114305)
Introduce a general recipe to generate a scalar phi. Lower VPCanonicalIVPHIRecipe and VPEVLBasedIVRecipe to VPScalarIVPHIrecipe before plan execution, avoiding the need for duplicated ::execute implementations. There are other cases that could benefit, including in-loop reduction phis and pointer induction phis. Builds on a similar idea as #82270. PR: #114305
1 parent c7e1468 commit a7fda0e
Copy full SHA for a7fda0e

File tree

Expand file treeCollapse file tree

8 files changed

+106
-36
lines changed
Filter options
Expand file treeCollapse file tree

8 files changed

+106
-36
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7721,6 +7721,7 @@ DenseMap<const SCEV *, Value *> LoopVectorizationPlanner::executePlan(
77217721
BestVPlan.prepareToExecute(ILV.getTripCount(),
77227722
ILV.getOrCreateVectorTripCount(nullptr),
77237723
CanonicalIVStartValue, State);
7724+
VPlanTransforms::prepareToExecute(BestVPlan);
77247725

77257726
BestVPlan.execute(&State);
77267727

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlan.cpp
+3-4Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1070,10 +1070,9 @@ void VPlan::execute(VPTransformState *State) {
10701070
}
10711071

10721072
auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
1073-
bool NeedsScalar =
1074-
isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe>(PhiR) ||
1075-
(isa<VPReductionPHIRecipe>(PhiR) &&
1076-
cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
1073+
bool NeedsScalar = isa<VPScalarPHIRecipe>(PhiR) ||
1074+
(isa<VPReductionPHIRecipe>(PhiR) &&
1075+
cast<VPReductionPHIRecipe>(PhiR)->isInLoop());
10771076
Value *Phi = State->get(PhiR, NeedsScalar);
10781077
Value *Val = State->get(PhiR->getBackedgeValue(), NeedsScalar);
10791078
cast<PHINode>(Phi)->addIncoming(Val, VectorLatchBB);

‎llvm/lib/Transforms/Vectorize/VPlan.h

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlan.h
+47-5Lines changed: 47 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -2239,6 +2239,45 @@ class VPWidenPointerInductionRecipe : public VPHeaderPHIRecipe,
22392239
#endif
22402240
};
22412241

2242+
/// Recipe to generate a scalar PHI. Used to generate code for recipes that
2243+
/// produce scalar header phis, including VPCanonicalIVPHIRecipe and
2244+
/// VPEVLBasedIVPHIRecipe.
2245+
class VPScalarPHIRecipe : public VPHeaderPHIRecipe {
2246+
std::string Name;
2247+
2248+
public:
2249+
VPScalarPHIRecipe(VPValue *Start, VPValue *BackedgeValue, DebugLoc DL,
2250+
StringRef Name)
2251+
: VPHeaderPHIRecipe(VPDef::VPScalarPHISC, nullptr, Start, DL),
2252+
Name(Name.str()) {
2253+
addOperand(BackedgeValue);
2254+
}
2255+
2256+
~VPScalarPHIRecipe() override = default;
2257+
2258+
VPScalarPHIRecipe *clone() override {
2259+
llvm_unreachable("cloning not implemented yet");
2260+
}
2261+
2262+
VP_CLASSOF_IMPL(VPDef::VPScalarPHISC)
2263+
2264+
/// Generate the phi/select nodes.
2265+
void execute(VPTransformState &State) override;
2266+
2267+
/// Returns true if the recipe only uses the first lane of operand \p Op.
2268+
bool onlyFirstLaneUsed(const VPValue *Op) const override {
2269+
assert(is_contained(operands(), Op) &&
2270+
"Op must be an operand of the recipe");
2271+
return true;
2272+
}
2273+
2274+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
2275+
/// Print the recipe.
2276+
void print(raw_ostream &O, const Twine &Indent,
2277+
VPSlotTracker &SlotTracker) const override;
2278+
#endif
2279+
};
2280+
22422281
/// A recipe for handling phis that are widened in the vector loop.
22432282
/// In the VPlan native path, all incoming VPValues & VPBasicBlock pairs are
22442283
/// managed in the recipe directly.
@@ -3134,8 +3173,10 @@ class VPCanonicalIVPHIRecipe : public VPHeaderPHIRecipe {
31343173
return D->getVPDefID() == VPDef::VPCanonicalIVPHISC;
31353174
}
31363175

3137-
/// Generate the canonical scalar induction phi of the vector loop.
3138-
void execute(VPTransformState &State) override;
3176+
void execute(VPTransformState &State) override {
3177+
llvm_unreachable(
3178+
"cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
3179+
}
31393180

31403181
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
31413182
/// Print the recipe.
@@ -3231,9 +3272,10 @@ class VPEVLBasedIVPHIRecipe : public VPHeaderPHIRecipe {
32313272
return D->getVPDefID() == VPDef::VPEVLBasedIVPHISC;
32323273
}
32333274

3234-
/// Generate phi for handling IV based on EVL over iterations correctly.
3235-
/// TODO: investigate if it can share the code with VPCanonicalIVPHIRecipe.
3236-
void execute(VPTransformState &State) override;
3275+
void execute(VPTransformState &State) override {
3276+
llvm_unreachable(
3277+
"cannot execute this recipe, should be replaced by VPScalarPHIRecipe");
3278+
}
32373279

32383280
/// Return the cost of this VPEVLBasedIVPHIRecipe.
32393281
InstructionCost computeCost(ElementCount VF,

‎llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlanAnalysis.cpp
+8-8Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -213,14 +213,14 @@ Type *VPTypeAnalysis::inferScalarType(const VPValue *V) {
213213
TypeSwitch<const VPRecipeBase *, Type *>(V->getDefiningRecipe())
214214
.Case<VPActiveLaneMaskPHIRecipe, VPCanonicalIVPHIRecipe,
215215
VPFirstOrderRecurrencePHIRecipe, VPReductionPHIRecipe,
216-
VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe>(
217-
[this](const auto *R) {
218-
// Handle header phi recipes, except VPWidenIntOrFpInduction
219-
// which needs special handling due it being possibly truncated.
220-
// TODO: consider inferring/caching type of siblings, e.g.,
221-
// backedge value, here and in cases below.
222-
return inferScalarType(R->getStartValue());
223-
})
216+
VPWidenPointerInductionRecipe, VPEVLBasedIVPHIRecipe,
217+
VPScalarPHIRecipe>([this](const auto *R) {
218+
// Handle header phi recipes, except VPWidenIntOrFpInduction
219+
// which needs special handling due it being possibly truncated.
220+
// TODO: consider inferring/caching type of siblings, e.g.,
221+
// backedge value, here and in cases below.
222+
return inferScalarType(R->getStartValue());
223+
})
224224
.Case<VPWidenIntOrFpInductionRecipe, VPDerivedIVRecipe>(
225225
[](const auto *R) { return R->getScalarType(); })
226226
.Case<VPReductionRecipe, VPPredInstPHIRecipe, VPWidenPHIRecipe,

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+22-19Lines changed: 22 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -3101,17 +3101,6 @@ InstructionCost VPInterleaveRecipe::computeCost(ElementCount VF,
31013101
VectorTy, std::nullopt, CostKind, 0);
31023102
}
31033103

3104-
void VPCanonicalIVPHIRecipe::execute(VPTransformState &State) {
3105-
Value *Start = getStartValue()->getLiveInIRValue();
3106-
PHINode *Phi = PHINode::Create(Start->getType(), 2, "index");
3107-
Phi->insertBefore(State.CFG.PrevBB->getFirstInsertionPt());
3108-
3109-
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
3110-
Phi->addIncoming(Start, VectorPH);
3111-
Phi->setDebugLoc(getDebugLoc());
3112-
State.set(this, Phi, /*IsScalar*/ true);
3113-
}
3114-
31153104
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
31163105
void VPCanonicalIVPHIRecipe::print(raw_ostream &O, const Twine &Indent,
31173106
VPSlotTracker &SlotTracker) const {
@@ -3153,8 +3142,6 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
31533142
assert(!onlyScalarsGenerated(State.VF.isScalable()) &&
31543143
"Recipe should have been replaced");
31553144

3156-
auto *IVR = getParent()->getPlan()->getCanonicalIV();
3157-
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, /*IsScalar*/ true));
31583145
unsigned CurrentPart = getUnrollPart(*this);
31593146

31603147
// Build a pointer phi
@@ -3164,6 +3151,12 @@ void VPWidenPointerInductionRecipe::execute(VPTransformState &State) {
31643151
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
31653152
PHINode *NewPointerPhi = nullptr;
31663153
if (CurrentPart == 0) {
3154+
auto *IVR = cast<VPHeaderPHIRecipe>(&getParent()
3155+
->getPlan()
3156+
->getVectorLoopRegion()
3157+
->getEntryBasicBlock()
3158+
->front());
3159+
PHINode *CanonicalIV = cast<PHINode>(State.get(IVR, /*IsScalar*/ true));
31673160
NewPointerPhi = PHINode::Create(ScStValueType, 2, "pointer.phi",
31683161
CanonicalIV->getIterator());
31693162
NewPointerPhi->addIncoming(ScalarStartValue, VectorPH);
@@ -3477,20 +3470,30 @@ void VPActiveLaneMaskPHIRecipe::print(raw_ostream &O, const Twine &Indent,
34773470
}
34783471
#endif
34793472

3480-
void VPEVLBasedIVPHIRecipe::execute(VPTransformState &State) {
3473+
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3474+
void VPEVLBasedIVPHIRecipe::print(raw_ostream &O, const Twine &Indent,
3475+
VPSlotTracker &SlotTracker) const {
3476+
O << Indent << "EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";
3477+
3478+
printAsOperand(O, SlotTracker);
3479+
O << " = phi ";
3480+
printOperands(O, SlotTracker);
3481+
}
3482+
#endif
3483+
3484+
void VPScalarPHIRecipe::execute(VPTransformState &State) {
34813485
BasicBlock *VectorPH = State.CFG.getPreheaderBBFor(this);
34823486
Value *Start = State.get(getOperand(0), VPLane(0));
3483-
PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, "evl.based.iv");
3487+
PHINode *Phi = State.Builder.CreatePHI(Start->getType(), 2, Name);
34843488
Phi->addIncoming(Start, VectorPH);
34853489
Phi->setDebugLoc(getDebugLoc());
34863490
State.set(this, Phi, /*IsScalar=*/true);
34873491
}
34883492

34893493
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
3490-
void VPEVLBasedIVPHIRecipe::print(raw_ostream &O, const Twine &Indent,
3491-
VPSlotTracker &SlotTracker) const {
3492-
O << Indent << "EXPLICIT-VECTOR-LENGTH-BASED-IV-PHI ";
3493-
3494+
void VPScalarPHIRecipe::print(raw_ostream &O, const Twine &Indent,
3495+
VPSlotTracker &SlotTracker) const {
3496+
O << Indent << "SCALAR-PHI";
34943497
printAsOperand(O, SlotTracker);
34953498
O << " = phi ";
34963499
printOperands(O, SlotTracker);

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+21Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1819,3 +1819,24 @@ void VPlanTransforms::createInterleaveGroups(
18191819
}
18201820
}
18211821
}
1822+
1823+
void VPlanTransforms::prepareToExecute(VPlan &Plan) {
1824+
ReversePostOrderTraversal<VPBlockDeepTraversalWrapper<VPBlockBase *>> RPOT(
1825+
Plan.getVectorLoopRegion());
1826+
for (VPBasicBlock *VPBB : VPBlockUtils::blocksOnly<VPBasicBlock>(
1827+
vp_depth_first_deep(Plan.getEntry()))) {
1828+
for (VPRecipeBase &R : make_early_inc_range(VPBB->phis())) {
1829+
if (!isa<VPCanonicalIVPHIRecipe, VPEVLBasedIVPHIRecipe>(&R))
1830+
continue;
1831+
auto *PhiR = cast<VPHeaderPHIRecipe>(&R);
1832+
StringRef Name =
1833+
isa<VPCanonicalIVPHIRecipe>(PhiR) ? "index" : "evl.based.iv";
1834+
auto *ScalarR =
1835+
new VPScalarPHIRecipe(PhiR->getStartValue(), PhiR->getBackedgeValue(),
1836+
PhiR->getDebugLoc(), Name);
1837+
ScalarR->insertBefore(PhiR);
1838+
PhiR->replaceAllUsesWith(ScalarR);
1839+
PhiR->eraseFromParent();
1840+
}
1841+
}
1842+
}

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.h

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlanTransforms.h
+3Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,9 @@ struct VPlanTransforms {
123123

124124
/// Remove dead recipes from \p Plan.
125125
static void removeDeadRecipes(VPlan &Plan);
126+
127+
/// Lower abstract recipes to concrete ones, that can be codegen'd.
128+
static void prepareToExecute(VPlan &Plan);
126129
};
127130

128131
} // namespace llvm

‎llvm/lib/Transforms/Vectorize/VPlanValue.h

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlanValue.h
+1Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -359,6 +359,7 @@ class VPDef {
359359
VPFirstOrderRecurrencePHISC,
360360
VPWidenIntOrFpInductionSC,
361361
VPWidenPointerInductionSC,
362+
VPScalarPHISC,
362363
VPReductionPHISC,
363364
// END: SubclassID for recipes that inherit VPHeaderPHIRecipe
364365
// END: Phi-like recipes

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.