Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit d300637

Browse filesBrowse files
fhahnfhahn
committed
[VPlan] Rename isUniform(AfterVectorization) to isSingleScalar (NFC).
Update the naming in VPReplicateRecipe and vputils to the more accurate isSingleScalar, as the functions check for cases where only a single scalar is needed, either because it produces the same value for all lanes or has only their first lane used. Discussed in llvm#139150.
1 parent f9dbfb1 commit d300637
Copy full SHA for d300637

File tree

Expand file treeCollapse file tree

6 files changed

+34
-36
lines changed
Filter options
Expand file treeCollapse file tree

6 files changed

+34
-36
lines changed

‎llvm/lib/Transforms/Vectorize/VPlan.cpp

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlan.cpp
+6-6Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -230,7 +230,7 @@ Value *VPTransformState::get(const VPValue *Def, const VPLane &Lane) {
230230
if (hasScalarValue(Def, Lane))
231231
return Data.VPV2Scalars[Def][Lane.mapToCacheIndex(VF)];
232232

233-
if (!Lane.isFirstLane() && vputils::isUniformAfterVectorization(Def) &&
233+
if (!Lane.isFirstLane() && vputils::isSingleScalar(Def) &&
234234
hasScalarValue(Def, VPLane::getFirstLane())) {
235235
return Data.VPV2Scalars[Def][0];
236236
}
@@ -303,17 +303,17 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
303303
return ScalarValue;
304304
}
305305

306-
bool IsUniform = vputils::isUniformAfterVectorization(Def);
306+
bool IsSingleScalar = vputils::isSingleScalar(Def);
307307

308-
VPLane LastLane(IsUniform ? 0 : VF.getKnownMinValue() - 1);
308+
VPLane LastLane(IsSingleScalar ? 0 : VF.getKnownMinValue() - 1);
309309
// Check if there is a scalar value for the selected lane.
310310
if (!hasScalarValue(Def, LastLane)) {
311311
// At the moment, VPWidenIntOrFpInductionRecipes, VPScalarIVStepsRecipes and
312-
// VPExpandSCEVRecipes can also be uniform.
312+
// VPExpandSCEVRecipes can also be a single scalar.
313313
assert((isa<VPWidenIntOrFpInductionRecipe, VPScalarIVStepsRecipe,
314314
VPExpandSCEVRecipe>(Def->getDefiningRecipe())) &&
315315
"unexpected recipe found to be invariant");
316-
IsUniform = true;
316+
IsSingleScalar = true;
317317
LastLane = 0;
318318
}
319319

@@ -334,7 +334,7 @@ Value *VPTransformState::get(const VPValue *Def, bool NeedsScalar) {
334334
// resulting vectors are stored in State, we will only generate the
335335
// insertelements once.
336336
Value *VectorValue = nullptr;
337-
if (IsUniform) {
337+
if (IsSingleScalar) {
338338
VectorValue = GetBroadcastInstrs(ScalarValue);
339339
set(Def, VectorValue);
340340
} else {

‎llvm/lib/Transforms/Vectorize/VPlan.h

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlan.h
+8-7Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2553,20 +2553,21 @@ class VPReductionEVLRecipe : public VPReductionRecipe {
25532553
/// VPReplicateRecipe replicates a given instruction producing multiple scalar
25542554
/// copies of the original scalar type, one per lane, instead of producing a
25552555
/// single copy of widened type for all lanes. If the instruction is known to be
2556-
/// uniform only one copy, per lane zero, will be generated.
2556+
/// a single scalar, only one copy, per lane zero, will be generated.
25572557
class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
25582558
/// Indicator if only a single replica per lane is needed.
2559-
bool IsUniform;
2559+
bool IsSingleScalar;
25602560

25612561
/// Indicator if the replicas are also predicated.
25622562
bool IsPredicated;
25632563

25642564
public:
25652565
VPReplicateRecipe(Instruction *I, ArrayRef<VPValue *> Operands,
2566-
bool IsUniform, VPValue *Mask = nullptr,
2566+
bool IsSingleScalar, VPValue *Mask = nullptr,
25672567
VPIRMetadata Metadata = {})
25682568
: VPRecipeWithIRFlags(VPDef::VPReplicateSC, Operands, *I),
2569-
VPIRMetadata(Metadata), IsUniform(IsUniform), IsPredicated(Mask) {
2569+
VPIRMetadata(Metadata), IsSingleScalar(IsSingleScalar),
2570+
IsPredicated(Mask) {
25702571
if (Mask)
25712572
addOperand(Mask);
25722573
}
@@ -2575,7 +2576,7 @@ class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
25752576

25762577
VPReplicateRecipe *clone() override {
25772578
auto *Copy =
2578-
new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsUniform,
2579+
new VPReplicateRecipe(getUnderlyingInstr(), operands(), IsSingleScalar,
25792580
isPredicated() ? getMask() : nullptr, *this);
25802581
Copy->transferFlags(*this);
25812582
return Copy;
@@ -2598,15 +2599,15 @@ class VPReplicateRecipe : public VPRecipeWithIRFlags, public VPIRMetadata {
25982599
VPSlotTracker &SlotTracker) const override;
25992600
#endif
26002601

2601-
bool isUniform() const { return IsUniform; }
2602+
bool isSingleScalar() const { return IsSingleScalar; }
26022603

26032604
bool isPredicated() const { return IsPredicated; }
26042605

26052606
/// Returns true if the recipe only uses the first lane of operand \p Op.
26062607
bool onlyFirstLaneUsed(const VPValue *Op) const override {
26072608
assert(is_contained(operands(), Op) &&
26082609
"Op must be an operand of the recipe");
2609-
return isUniform();
2610+
return isSingleScalar();
26102611
}
26112612

26122613
/// Returns true if the recipe uses scalars of operand \p Op.

‎llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlanRecipes.cpp
+7-8Lines changed: 7 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1190,7 +1190,7 @@ void VPIRPhi::execute(VPTransformState &State) {
11901190
PHINode *Phi = &getIRPhi();
11911191
for (const auto &[Idx, Op] : enumerate(operands())) {
11921192
VPValue *ExitValue = Op;
1193-
auto Lane = vputils::isUniformAfterVectorization(ExitValue)
1193+
auto Lane = vputils::isSingleScalar(ExitValue)
11941194
? VPLane::getFirstLane()
11951195
: VPLane::getLastLaneForVF(State.VF);
11961196
VPBlockBase *Pred = getParent()->getPredecessors()[Idx];
@@ -2624,7 +2624,7 @@ static void scalarizeInstruction(const Instruction *Instr,
26242624
for (const auto &I : enumerate(RepRecipe->operands())) {
26252625
auto InputLane = Lane;
26262626
VPValue *Operand = I.value();
2627-
if (vputils::isUniformAfterVectorization(Operand))
2627+
if (vputils::isSingleScalar(Operand))
26282628
InputLane = VPLane::getFirstLane();
26292629
Cloned->setOperand(I.index(), State.get(Operand, InputLane));
26302630
}
@@ -2650,7 +2650,7 @@ static void scalarizeInstruction(const Instruction *Instr,
26502650
void VPReplicateRecipe::execute(VPTransformState &State) {
26512651
Instruction *UI = getUnderlyingInstr();
26522652
if (State.Lane) { // Generate a single instance.
2653-
assert((State.VF.isScalar() || !isUniform()) &&
2653+
assert((State.VF.isScalar() || !isSingleScalar()) &&
26542654
"uniform recipe shouldn't be predicated");
26552655
assert(!State.VF.isScalable() && "Can't scalarize a scalable vector");
26562656
scalarizeInstruction(UI, this, *State.Lane, State);
@@ -2668,16 +2668,15 @@ void VPReplicateRecipe::execute(VPTransformState &State) {
26682668
return;
26692669
}
26702670

2671-
if (IsUniform) {
2671+
if (IsSingleScalar) {
26722672
// Uniform within VL means we need to generate lane 0.
26732673
scalarizeInstruction(UI, this, VPLane(0), State);
26742674
return;
26752675
}
26762676

26772677
// A store of a loop varying value to a uniform address only needs the last
26782678
// copy of the store.
2679-
if (isa<StoreInst>(UI) &&
2680-
vputils::isUniformAfterVectorization(getOperand(1))) {
2679+
if (isa<StoreInst>(UI) && vputils::isSingleScalar(getOperand(1))) {
26812680
auto Lane = VPLane::getLastLaneForVF(State.VF);
26822681
scalarizeInstruction(UI, this, VPLane(Lane), State);
26832682
return;
@@ -2738,7 +2737,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
27382737
UI->getOpcode(), ResultTy, CostKind,
27392738
{TargetTransformInfo::OK_AnyValue, TargetTransformInfo::OP_None},
27402739
Op2Info, Operands, UI, &Ctx.TLI) *
2741-
(isUniform() ? 1 : VF.getKnownMinValue());
2740+
(isSingleScalar() ? 1 : VF.getKnownMinValue());
27422741
}
27432742
}
27442743

@@ -2748,7 +2747,7 @@ InstructionCost VPReplicateRecipe::computeCost(ElementCount VF,
27482747
#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
27492748
void VPReplicateRecipe::print(raw_ostream &O, const Twine &Indent,
27502749
VPSlotTracker &SlotTracker) const {
2751-
O << Indent << (IsUniform ? "CLONE " : "REPLICATE ");
2750+
O << Indent << (IsSingleScalar ? "CLONE " : "REPLICATE ");
27522751

27532752
if (!getUnderlyingInstr()->getType()->isVoidTy()) {
27542753
printAsOperand(O, SlotTracker);

‎llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp
+4-5Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,7 @@ static bool sinkScalarOperands(VPlan &Plan) {
151151
SinkCandidate->mayReadOrWriteMemory())
152152
continue;
153153
if (auto *RepR = dyn_cast<VPReplicateRecipe>(SinkCandidate)) {
154-
if (!ScalarVFOnly && RepR->isUniform())
154+
if (!ScalarVFOnly && RepR->isSingleScalar())
155155
continue;
156156
} else if (!isa<VPScalarIVStepsRecipe>(SinkCandidate))
157157
continue;
@@ -347,7 +347,7 @@ static VPRegionBlock *createReplicateRegion(VPReplicateRecipe *PredRecipe,
347347
auto *RecipeWithoutMask = new VPReplicateRecipe(
348348
PredRecipe->getUnderlyingInstr(),
349349
make_range(PredRecipe->op_begin(), std::prev(PredRecipe->op_end())),
350-
PredRecipe->isUniform(), nullptr /*Mask*/, *PredRecipe);
350+
PredRecipe->isSingleScalar(), nullptr /*Mask*/, *PredRecipe);
351351
auto *Pred =
352352
Plan.createVPBasicBlock(Twine(RegionName) + ".if", RecipeWithoutMask);
353353

@@ -643,12 +643,11 @@ static void legalizeAndOptimizeInductions(VPlan &Plan) {
643643
// Skip recipes that shouldn't be narrowed.
644644
if (!Def || !isa<VPReplicateRecipe, VPWidenRecipe>(Def) ||
645645
Def->getNumUsers() == 0 || !Def->getUnderlyingValue() ||
646-
(RepR && (RepR->isUniform() || RepR->isPredicated())))
646+
(RepR && (RepR->isSingleScalar() || RepR->isPredicated())))
647647
continue;
648648

649649
// Skip recipes that may have other lanes than their first used.
650-
if (!vputils::isUniformAfterVectorization(Def) &&
651-
!vputils::onlyFirstLaneUsed(Def))
650+
if (!vputils::isSingleScalar(Def) && !vputils::onlyFirstLaneUsed(Def))
652651
continue;
653652

654653
auto *Clone = new VPReplicateRecipe(Def->getUnderlyingInstr(),

‎llvm/lib/Transforms/Vectorize/VPlanUtils.cpp

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlanUtils.cpp
+1-1Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,7 +109,7 @@ bool vputils::isUniformAcrossVFsAndUFs(VPValue *V) {
109109
// VPReplicateRecipe.IsUniform. They are also uniform across UF parts if
110110
// all their operands are invariant.
111111
// TODO: Further relax the restrictions.
112-
return R->isUniform() &&
112+
return R->isSingleScalar() &&
113113
(isa<LoadInst, StoreInst>(R->getUnderlyingValue())) &&
114114
all_of(R->operands(), isUniformAcrossVFsAndUFs);
115115
})

‎llvm/lib/Transforms/Vectorize/VPlanUtils.h

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/VPlanUtils.h
+8-9Lines changed: 8 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,8 +37,9 @@ VPValue *getOrCreateVPValueForSCEVExpr(VPlan &Plan, const SCEV *Expr,
3737
/// SCEV expression could be constructed.
3838
const SCEV *getSCEVExprForVPValue(VPValue *V, ScalarEvolution &SE);
3939

40-
/// Returns true if \p VPV is uniform after vectorization.
41-
inline bool isUniformAfterVectorization(const VPValue *VPV) {
40+
/// Returns true if \p VPV is a single scalar, either because it produces the
41+
/// same value for all lanes or only has its first lane used.
42+
inline bool isSingleScalar(const VPValue *VPV) {
4243
auto PreservesUniformity = [](unsigned Opcode) -> bool {
4344
if (Instruction::isBinaryOp(Opcode) || Instruction::isCast(Opcode))
4445
return true;
@@ -65,21 +66,19 @@ inline bool isUniformAfterVectorization(const VPValue *VPV) {
6566
// lanes.
6667
if (RegionOfR && RegionOfR->isReplicator())
6768
return false;
68-
return Rep->isUniform() ||
69-
(PreservesUniformity(Rep->getOpcode()) &&
70-
all_of(Rep->operands(), isUniformAfterVectorization));
69+
return Rep->isSingleScalar() || (PreservesUniformity(Rep->getOpcode()) &&
70+
all_of(Rep->operands(), isSingleScalar));
7171
}
7272
if (isa<VPWidenGEPRecipe, VPDerivedIVRecipe, VPBlendRecipe>(VPV))
73-
return all_of(VPV->getDefiningRecipe()->operands(),
74-
isUniformAfterVectorization);
73+
return all_of(VPV->getDefiningRecipe()->operands(), isSingleScalar);
7574
if (auto *WidenR = dyn_cast<VPWidenRecipe>(VPV)) {
7675
return PreservesUniformity(WidenR->getOpcode()) &&
77-
all_of(WidenR->operands(), isUniformAfterVectorization);
76+
all_of(WidenR->operands(), isSingleScalar);
7877
}
7978
if (auto *VPI = dyn_cast<VPInstruction>(VPV))
8079
return VPI->isSingleScalar() || VPI->isVectorToScalar() ||
8180
(PreservesUniformity(VPI->getOpcode()) &&
82-
all_of(VPI->operands(), isUniformAfterVectorization));
81+
all_of(VPI->operands(), isSingleScalar));
8382

8483
// VPExpandSCEVRecipes must be placed in the entry and are alway uniform.
8584
return isa<VPExpandSCEVRecipe>(VPV);

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.