Skip to content

Navigation Menu

Sign in
Appearance settings

Search code, repositories, users, issues, pull requests...

Provide feedback

We read every piece of feedback, and take your input very seriously.

Saved searches

Use saved searches to filter your results more quickly
Appearance settings

Commit 7a87bf4

Browse filesBrowse files
SamTebbs33SamTebbs33
committed
Use VPExpandSCEVRecipe
1 parent a1b0669 commit 7a87bf4
Copy full SHA for 7a87bf4

File tree

Expand file treeCollapse file tree

4 files changed

+29
-39
lines changed
Filter options
Expand file treeCollapse file tree

4 files changed

+29
-39
lines changed

‎llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/LoopVectorizationPlanner.h
+5-6Lines changed: 5 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -363,7 +363,7 @@ class LoopVectorizationPlanner {
363363
/// loop iteration.
364364
std::optional<VectorizationFactor>
365365
plan(ElementCount UserVF, unsigned UserIC,
366-
std::optional<ArrayRef<PointerDiffInfo>> DiffChecks, std::function<Value*(const SCEV *)> Expander, bool &HasAliasMask);
366+
std::optional<ArrayRef<PointerDiffInfo>> DiffChecks, bool &HasAliasMask);
367367

368368
/// Use the VPlan-native path to plan how to best vectorize, return the best
369369
/// VF and its cost.
@@ -440,18 +440,17 @@ class LoopVectorizationPlanner {
440440
/// setting HasAliasMask to true in the case that an alias mask is generated
441441
/// and the vector loop should be entered even if the pointers alias across a
442442
/// loop iteration.
443-
VPlanPtr
444-
tryToBuildVPlanWithVPRecipes(VFRange &Range,
445-
SmallVector<PointerDiffInfoValues> RTChecks,
446-
bool &HasAliasMask);
443+
VPlanPtr tryToBuildVPlanWithVPRecipes(VFRange &Range,
444+
ArrayRef<PointerDiffInfo> RTChecks,
445+
bool &HasAliasMask);
447446

448447
/// Build VPlans for power-of-2 VF's between \p MinVF and \p MaxVF inclusive,
449448
/// according to the information gathered by Legal when it checked if it is
450449
/// legal to vectorize the loop. This method creates VPlans using VPRecipes.
451450
/// RTChecks contains a list of pointer pairs that an alias mask should be
452451
/// generated for.
453452
void buildVPlansWithVPRecipes(ElementCount MinVF, ElementCount MaxVF,
454-
SmallVector<PointerDiffInfoValues> RTChecks,
453+
ArrayRef<PointerDiffInfo> RTChecks,
455454
bool &HasAliasMask);
456455

457456
// Adjust the recipes for reductions. For in-loop reductions the chain of

‎llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

Copy file name to clipboardExpand all lines: llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+19-30Lines changed: 19 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1936,10 +1936,6 @@ class GeneratedRTChecks {
19361936
OuterLoop = L->getParentLoop();
19371937
}
19381938

1939-
Value *expandCodeForMemCheck(const SCEV *Scev, Instruction *Loc) {
1940-
return MemCheckExp.expandCodeFor(Scev, Scev->getType(), Loc);
1941-
}
1942-
19431939
InstructionCost getCost() {
19441940
if (SCEVCheckBlock || MemCheckBlock)
19451941
LLVM_DEBUG(dbgs() << "Calculating cost of runtime checks:\n");
@@ -6905,10 +6901,9 @@ LoopVectorizationPlanner::planInVPlanNativePath(ElementCount UserVF) {
69056901
return VectorizationFactor::Disabled();
69066902
}
69076903

6908-
std::optional<VectorizationFactor>
6909-
LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
6910-
std::optional<ArrayRef<PointerDiffInfo>> RTChecks,
6911-
std::function<Value*(const SCEV*)> Expander, bool &HasAliasMask) {
6904+
std::optional<VectorizationFactor> LoopVectorizationPlanner::plan(
6905+
ElementCount UserVF, unsigned UserIC,
6906+
std::optional<ArrayRef<PointerDiffInfo>> RTChecks, bool &HasAliasMask) {
69126907
assert(OrigLoop->isInnermost() && "Inner loop expected.");
69136908
CM.collectValuesToIgnore();
69146909
CM.collectElementTypesForWidening();
@@ -6919,15 +6914,9 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
69196914

69206915
// VPlan needs the aliasing pointers as Values and not SCEVs, so expand them
69216916
// here and put them into a list.
6922-
SmallVector<PointerDiffInfoValues> DiffChecksValues;
6923-
if (RTChecks.has_value()
6924-
&& useActiveLaneMask(CM.getTailFoldingStyle(true))) {
6925-
for (auto Check : *RTChecks) {
6926-
Value *Sink = Expander(Check.SinkStart);
6927-
Value *Src = Expander(Check.SrcStart);
6928-
DiffChecksValues.push_back(PointerDiffInfoValues(Src, Sink));
6929-
}
6930-
}
6917+
ArrayRef<PointerDiffInfo> DiffChecks;
6918+
if (RTChecks.has_value() && useActiveLaneMask(CM.getTailFoldingStyle(true)))
6919+
DiffChecks = *RTChecks;
69316920

69326921
// Invalidate interleave groups if all blocks of loop will be predicated.
69336922
if (CM.blockNeedsPredicationForAnyReason(OrigLoop->getHeader()) &&
@@ -6957,7 +6946,7 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
69576946
CM.collectInLoopReductions();
69586947
if (CM.selectUserVectorizationFactor(UserVF)) {
69596948
LLVM_DEBUG(dbgs() << "LV: Using user VF " << UserVF << ".\n");
6960-
buildVPlansWithVPRecipes(UserVF, UserVF, DiffChecksValues, HasAliasMask);
6949+
buildVPlansWithVPRecipes(UserVF, UserVF, DiffChecks, HasAliasMask);
69616950
if (!hasPlanWithVF(UserVF)) {
69626951
LLVM_DEBUG(dbgs() << "LV: No VPlan could be built for " << UserVF
69636952
<< ".\n");
@@ -6992,9 +6981,9 @@ LoopVectorizationPlanner::plan(ElementCount UserVF, unsigned UserIC,
69926981
}
69936982

69946983
buildVPlansWithVPRecipes(ElementCount::getFixed(1), MaxFactors.FixedVF,
6995-
DiffChecksValues, HasAliasMask);
6984+
DiffChecks, HasAliasMask);
69966985
buildVPlansWithVPRecipes(ElementCount::getScalable(1), MaxFactors.ScalableVF,
6997-
DiffChecksValues, HasAliasMask);
6986+
DiffChecks, HasAliasMask);
69986987

69996988
LLVM_DEBUG(printPlans(dbgs()));
70006989
if (VPlans.empty())
@@ -8387,8 +8376,8 @@ VPRecipeBuilder::tryToCreateWidenRecipe(Instruction *Instr,
83878376
}
83888377

83898378
void LoopVectorizationPlanner::buildVPlansWithVPRecipes(
8390-
ElementCount MinVF, ElementCount MaxVF,
8391-
SmallVector<PointerDiffInfoValues> RTChecks, bool &HasAliasMask) {
8379+
ElementCount MinVF, ElementCount MaxVF, ArrayRef<PointerDiffInfo> RTChecks,
8380+
bool &HasAliasMask) {
83928381
assert(OrigLoop->isInnermost() && "Inner loop expected.");
83938382

83948383
auto MaxVFTimes2 = MaxVF * 2;
@@ -8534,8 +8523,7 @@ static void addLiveOutsForFirstOrderRecurrences(VPlan &Plan) {
85348523
}
85358524

85368525
VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
8537-
VFRange &Range, SmallVector<PointerDiffInfoValues> RTChecks,
8538-
bool &HasAliasMask) {
8526+
VFRange &Range, ArrayRef<PointerDiffInfo> RTChecks, bool &HasAliasMask) {
85398527

85408528
SmallPtrSet<const InterleaveGroup<Instruction> *, 1> InterleaveGroups;
85418529

@@ -8584,8 +8572,10 @@ VPlanPtr LoopVectorizationPlanner::tryToBuildVPlanWithVPRecipes(
85848572
VPBuilder Builder(VecPreheader);
85858573
for (auto C : RTChecks) {
85868574
HasAliasMask = true;
8587-
VPValue *Sink = Plan->getOrAddLiveIn(C.Sink);
8588-
VPValue *Src = Plan->getOrAddLiveIn(C.Src);
8575+
VPValue *Sink = vputils::getOrCreateVPValueForSCEVExpr(*Plan, C.SinkStart,
8576+
*PSE.getSE());
8577+
VPValue *Src = vputils::getOrCreateVPValueForSCEVExpr(*Plan, C.SrcStart,
8578+
*PSE.getSE());
85898579
VPValue *M =
85908580
Builder.createNaryOp(VPInstruction::AliasLaneMask, {Sink, Src}, DL,
85918581
"active.lane.mask.alias");
@@ -9921,11 +9911,10 @@ bool LoopVectorizePass::processLoop(Loop *L) {
99219911
AddBranchWeights);
99229912

99239913
// Plan how to best vectorize, return the best VF and its cost.
9924-
auto Expand = [&Checks, &L](const SCEV *S) {
9925-
return Checks.expandCodeForMemCheck(S, L->getLoopPreheader()->getTerminator());
9926-
};
99279914
std::optional<VectorizationFactor> MaybeVF =
9928-
LVP.plan(UserVF, UserIC, LVL.getLAI()->getRuntimePointerChecking()->getDiffChecks(), Expand, Checks.HasAliasMask);
9915+
LVP.plan(UserVF, UserIC,
9916+
LVL.getLAI()->getRuntimePointerChecking()->getDiffChecks(),
9917+
Checks.HasAliasMask);
99299918
if (Checks.HasAliasMask)
99309919
LoopsAliasMasked++;
99319920

‎llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll

Copy file name to clipboardExpand all lines: llvm/test/Transforms/LoopVectorize/AArch64/induction-costs-sve.ll
+3-1Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,12 +146,14 @@ define void @iv_casts(ptr %dst, ptr %src, i32 %x, i64 %N) #0 {
146146
; PRED-NEXT: entry:
147147
; PRED-NEXT: [[SRC2:%.*]] = ptrtoint ptr [[SRC]] to i64
148148
; PRED-NEXT: [[DST1:%.*]] = ptrtoint ptr [[DST]] to i64
149+
; PRED-NEXT: [[SRC3:%.*]] = ptrtoint ptr [[SRC]] to i64
150+
; PRED-NEXT: [[DST2:%.*]] = ptrtoint ptr [[DST]] to i64
149151
; PRED-NEXT: [[TMP0:%.*]] = add i64 [[N]], 1
150152
; PRED-NEXT: br i1 false, label [[SCALAR_PH:%.*]], label [[VECTOR_MEMCHECK:%.*]]
151153
; PRED: vector.memcheck:
152154
; PRED-NEXT: [[TMP1:%.*]] = call i64 @llvm.vscale.i64()
153155
; PRED-NEXT: [[TMP2:%.*]] = mul i64 [[TMP1]], 8
154-
; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[DST1]], [[SRC2]]
156+
; PRED-NEXT: [[TMP3:%.*]] = sub i64 [[DST2]], [[SRC3]]
155157
; PRED-NEXT: [[DIFF_CHECK:%.*]] = icmp ult i64 [[TMP3]], [[TMP2]]
156158
; PRED-NEXT: br label [[VECTOR_PH:%.*]]
157159
; PRED: vector.ph:

‎llvm/test/Transforms/LoopVectorize/ARM/scalar-block-cost.ll

Copy file name to clipboardExpand all lines: llvm/test/Transforms/LoopVectorize/ARM/scalar-block-cost.ll
+2-2Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ define void @pred_loop(ptr %off, ptr %data, ptr %dst, i32 %n) #0 {
99
; CHECK-COST: LV: Found an estimated cost of 0 for VF 1 For instruction: %i.09 = phi i32 [ %add, %for.body ], [ 0, %for.body.preheader ]
1010
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: %add = add nuw nsw i32 %i.09, 1
1111
; CHECK-COST-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction: %arrayidx = getelementptr inbounds i32, ptr %data, i32 %add
12-
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: %1 = load i32, ptr %arrayidx, align 4
13-
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: %add1 = add nsw i32 %1, 5
12+
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: %0 = load i32, ptr %arrayidx, align 4
13+
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: %add1 = add nsw i32 %0, 5
1414
; CHECK-COST-NEXT: LV: Found an estimated cost of 0 for VF 1 For instruction: %arrayidx2 = getelementptr inbounds i32, ptr %dst, i32 %i.09
1515
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: store i32 %add1, ptr %arrayidx2, align 4
1616
; CHECK-COST-NEXT: LV: Found an estimated cost of 1 for VF 1 For instruction: %exitcond.not = icmp eq i32 %add, %n

0 commit comments

Comments
0 (0)
Morty Proxy This is a proxified and sanitized view of the page, visit original site.