From 7b11b965dd88e9b6fc70e3569a250d11274c3ef4 Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Tue, 1 Apr 2025 08:03:16 -0500 Subject: [PATCH 01/18] [IR] Add CallBr intrinsics support This commit adds support for using intrinsics with callbr. The uses of this will most of the time look like this example: ```llvm callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill] kill: unreachable cont: ... ``` --- llvm/include/llvm/Analysis/RegionInfoImpl.h | 12 ++++ .../llvm/Transforms/Utils/BasicBlockUtils.h | 6 +- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 59 +++++++++++++++- .../SelectionDAG/SelectionDAGBuilder.cpp | 53 ++++++++++---- llvm/lib/IR/Verifier.cpp | 29 ++++++-- llvm/lib/Transforms/Scalar/StructurizeCFG.cpp | 11 ++- llvm/lib/Transforms/Utils/BasicBlockUtils.cpp | 4 +- llvm/test/CodeGen/AMDGPU/callbr.ll | 70 +++++++++++++++++++ 8 files changed, 211 insertions(+), 33 deletions(-) create mode 100644 llvm/test/CodeGen/AMDGPU/callbr.ll diff --git a/llvm/include/llvm/Analysis/RegionInfoImpl.h b/llvm/include/llvm/Analysis/RegionInfoImpl.h index eb99d8bc6fb23..759e9c47bebb8 100644 --- a/llvm/include/llvm/Analysis/RegionInfoImpl.h +++ b/llvm/include/llvm/Analysis/RegionInfoImpl.h @@ -553,6 +553,18 @@ bool RegionInfoBase::isRegion(BlockT *entry, BlockT *exit) const { using DST = typename DomFrontierT::DomSetType; + // TODO? post domination frontier? + if constexpr (std::is_same_v) { + if (DomTreeNodeT *PDTNode = PDT->getNode(exit); PDTNode) { + for (DomTreeNodeT *PredNode : *PDTNode) { + for (BasicBlock *Pred : predecessors(PredNode->getBlock())) { + if (isa(Pred->getTerminator())) + return false; + } + } + } + } + DST *entrySuccs = &DF->find(entry)->second; // Exit is the header of a loop that contains the entry. In this case, diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h index 979f3b3eb72ff..98115c109dfd6 100644 --- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h +++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h @@ -607,9 +607,9 @@ LLVM_ABI bool SplitIndirectBrCriticalEdges(Function &F, // successors LLVM_ABI void InvertBranch(BranchInst *PBI, IRBuilderBase &Builder); -// Check whether the function only has simple terminator: -// br/brcond/unreachable/ret -LLVM_ABI bool hasOnlySimpleTerminator(const Function &F); +// Check whether the function only has blocks with simple terminators: +// br/brcond/unreachable/ret (or callbr if AllowCallBr) +LLVM_ABI bool hasOnlySimpleTerminator(const Function &F, bool AllowCallBr = true); } // end namespace llvm diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index fe5dcd14d8804..65e36de76307d 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -3008,8 +3008,63 @@ bool IRTranslator::translateInvoke(const User &U, bool IRTranslator::translateCallBr(const User &U, MachineIRBuilder &MIRBuilder) { - // FIXME: Implement this. - return false; + const CallBrInst &I = cast(U); + MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB(); + + // TODO: operand bundles (see SelDAG implementation of callbr)? + assert(!I.hasOperandBundles() && + "Cannot lower callbrs with operand bundles yet"); + + if (I.isInlineAsm()) { + // FIXME: inline asm not yet supported + if (!translateInlineAsm(I, MIRBuilder)) + return false; + } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) { + switch (I.getIntrinsicID()) { + default: + report_fatal_error("Unsupported intrinsic for callbr"); + case Intrinsic::amdgcn_kill: + if (I.getNumIndirectDests() != 1) + report_fatal_error( + "amdgcn.kill supportes exactly one indirect destination"); + CallInst *CI = + CallInst::Create(I.getFunctionType(), I.getCalledFunction(), + SmallVector(I.args())); + bool Success = translateCall(*CI, MIRBuilder); + CI->deleteValue(); + if (!Success) + return false; + break; + } + } else { + report_fatal_error("Only know how to handle inlineasm/intrinsic callbr"); + } + + // Retrieve successors. + SmallPtrSet Dests; + Dests.insert(I.getDefaultDest()); + MachineBasicBlock *Return = &getMBB(*I.getDefaultDest()); + + // Update successor info. + addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne()); + // TODO: For most of the cases where there is an intrinsic callbr, we're + // having exactly one indirect target, which will be unreachable. As soon as + // this changes, we might need to enhance Target->setIsInlineAsmBrIndirectTarget + // or add something similar for intrinsic indirect branches. + if (I.isInlineAsm()) { + for (BasicBlock *Dest : I.getIndirectDests()) { + MachineBasicBlock *Target = &getMBB(*Dest); + Target->setIsInlineAsmBrIndirectTarget(); + Target->setMachineBlockAddressTaken(); + Target->setLabelMustBeEmitted(); + // Don't add duplicate machine successors. + if (Dests.insert(Dest).second) + addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero()); + } + } + CallBrMBB->normalizeSuccProbs(); + + return true; } bool IRTranslator::translateLandingPad(const User &U, diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index c63eb7fc6b374..d4a97d2dfc196 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3447,8 +3447,26 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { reportFatalUsageError( "cannot lower callbrs with arbitrary operand bundles!"); - assert(I.isInlineAsm() && "Only know how to handle inlineasm callbr"); - visitInlineAsm(I); + if (I.isInlineAsm()) { + visitInlineAsm(I); + } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) { + switch (I.getIntrinsicID()) { + default: + report_fatal_error("Unsupported intrinsic for callbr"); + case Intrinsic::amdgcn_kill: + if (I.getNumIndirectDests() != 1) + report_fatal_error( + "amdgcn.kill supportes exactly one indirect destination"); + CallInst *CI = + CallInst::Create(I.getFunctionType(), I.getCalledFunction(), + SmallVector(I.args())); + visitCall(*CI); + CI->deleteValue(); + break; + } + } else { + report_fatal_error("Only know how to handle inlineasm/intrinsic callbr"); + } CopyToExportRegsIfNeeded(&I); // Retrieve successors. @@ -3458,19 +3476,24 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { // Update successor info. addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne()); - for (unsigned i = 0, e = I.getNumIndirectDests(); i < e; ++i) { - BasicBlock *Dest = I.getIndirectDest(i); - MachineBasicBlock *Target = FuncInfo.getMBB(Dest); - Target->setIsInlineAsmBrIndirectTarget(); - // If we introduce a type of asm goto statement that is permitted to use an - // indirect call instruction to jump to its labels, then we should add a - // call to Target->setMachineBlockAddressTaken() here, to mark the target - // block as requiring a BTI. - - Target->setLabelMustBeEmitted(); - // Don't add duplicate machine successors. - if (Dests.insert(Dest).second) - addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero()); + // TODO: For most of the cases where there is an intrinsic callbr, we're + // having exactly one indirect target, which will be unreachable. As soon as + // this changes, we might need to enhance Target->setIsInlineAsmBrIndirectTarget + // or add something similar for intrinsic indirect branches. + if (I.isInlineAsm()) { + for (BasicBlock *Dest : I.getIndirectDests()) { + MachineBasicBlock *Target = FuncInfo.getMBB(Dest); + Target->setIsInlineAsmBrIndirectTarget(); + // If we introduce a type of asm goto statement that is permitted to use an + // indirect call instruction to jump to its labels, then we should add a + // call to Target->setMachineBlockAddressTaken() here, to mark the target + // block as requiring a BTI. + + Target->setLabelMustBeEmitted(); + // Don't add duplicate machine successors. + if (Dests.insert(Dest).second) + addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero()); + } } CallBrMBB->normalizeSuccProbs(); diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 1f1041b259736..6404cfc780125 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -3296,11 +3296,30 @@ void Verifier::visitIndirectBrInst(IndirectBrInst &BI) { } void Verifier::visitCallBrInst(CallBrInst &CBI) { - Check(CBI.isInlineAsm(), "Callbr is currently only used for asm-goto!", &CBI); - const InlineAsm *IA = cast(CBI.getCalledOperand()); - Check(!IA->canThrow(), "Unwinding from Callbr is not allowed"); + if (!CBI.isInlineAsm()) { + switch (CBI.getIntrinsicID()) { + case Intrinsic::amdgcn_kill: { + Check(CBI.getNumIndirectDests() == 1, + "Callbr amdgcn_kill only supports one indirect dest"); + bool Unreachable = isa(CBI.getIndirectDest(0)->begin()); + CallInst *Call = dyn_cast(CBI.getIndirectDest(0)->begin()); + Check(Unreachable || + (Call && Call->getIntrinsicID() == Intrinsic::amdgcn_unreachable), + "Callbr amdgcn_kill indirect dest needs to be unreachable"); + visitIntrinsicCall(Intrinsic::amdgcn_kill, CBI); + break; + } + default: + CheckFailed( + "Callbr currently only supports asm-goto and selected intrinsics"); + } + visitIntrinsicCall(CBI.getIntrinsicID(), CBI); + } else { + const InlineAsm *IA = cast(CBI.getCalledOperand()); + Check(!IA->canThrow(), "Unwinding from Callbr is not allowed"); - verifyInlineAsmCall(CBI); + verifyInlineAsmCall(CBI); + } visitTerminator(CBI); } @@ -5272,7 +5291,7 @@ void Verifier::visitInstruction(Instruction &I) { (CBI && &CBI->getCalledOperandUse() == &I.getOperandUse(i)) || IsAttachedCallOperand(F, CBI, i)), "Cannot take the address of an intrinsic!", &I); - Check(!F->isIntrinsic() || isa(I) || + Check(!F->isIntrinsic() || isa(I) || isa(I) || F->getIntrinsicID() == Intrinsic::donothing || F->getIntrinsicID() == Intrinsic::seh_try_begin || F->getIntrinsicID() == Intrinsic::seh_try_end || diff --git a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp index a69d64956d6d9..18d3381b420bd 100644 --- a/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp +++ b/llvm/lib/Transforms/Scalar/StructurizeCFG.cpp @@ -480,11 +480,10 @@ void StructurizeCFG::analyzeLoops(RegionNode *N) { } else { // Test for successors as back edge BasicBlock *BB = N->getNodeAs(); - BranchInst *Term = cast(BB->getTerminator()); - - for (BasicBlock *Succ : Term->successors()) - if (Visited.count(Succ)) - Loops[Succ] = BB; + if (BranchInst *Term = dyn_cast(BB->getTerminator()); Term) + for (BasicBlock *Succ : Term->successors()) + if (Visited.count(Succ)) + Loops[Succ] = BB; } } @@ -516,7 +515,7 @@ void StructurizeCFG::gatherPredicates(RegionNode *N) { for (BasicBlock *P : predecessors(BB)) { // Ignore it if it's a branch from outside into our region entry - if (!ParentRegion->contains(P)) + if (!ParentRegion->contains(P) || !dyn_cast(P->getTerminator())) continue; Region *R = RI->getRegionFor(P); diff --git a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp index 1feed14b4fed8..3cf6cc7f7c946 100644 --- a/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp +++ b/llvm/lib/Transforms/Utils/BasicBlockUtils.cpp @@ -1786,11 +1786,11 @@ void llvm::InvertBranch(BranchInst *PBI, IRBuilderBase &Builder) { PBI->swapSuccessors(); } -bool llvm::hasOnlySimpleTerminator(const Function &F) { +bool llvm::hasOnlySimpleTerminator(const Function &F, bool AllowCallBr) { for (auto &BB : F) { auto *Term = BB.getTerminator(); if (!(isa(Term) || isa(Term) || - isa(Term))) + isa(Term) || (AllowCallBr && isa(Term)))) return false; } return true; diff --git a/llvm/test/CodeGen/AMDGPU/callbr.ll b/llvm/test/CodeGen/AMDGPU/callbr.ll new file mode 100644 index 0000000000000..e2e84dca96cbf --- /dev/null +++ b/llvm/test/CodeGen/AMDGPU/callbr.ll @@ -0,0 +1,70 @@ +; RUN: rm -rf %t && split-file %s %t +; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/with-callbr-seldag.s < %t/with-callbr.ll +; RUN: FileCheck --check-prefix=SELDAG %s < %t/with-callbr-seldag.s +; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/with-callbr-gisel.s -global-isel < %t/with-callbr.ll +; RUN: FileCheck --check-prefix=GISEL %s < %t/with-callbr-gisel.s +; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/without-callbr-seldag.s < %t/without-callbr.ll +; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/without-callbr-gisel.s -global-isel < %t/without-callbr.ll +; RUN: diff %t/with-callbr-seldag.s %t/without-callbr-seldag.s +; RUN: diff %t/with-callbr-gisel.s %t/without-callbr-gisel.s + +;--- with-callbr.ll + +; SELDAG-LABEL: test_kill: +; SELDAG-NEXT: ; %bb.0: +; SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SELDAG-NEXT: flat_load_dword v0, v[0:1] +; SELDAG-NEXT: v_and_b32_e32 v1, 1, v4 +; SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1 +; SELDAG-NEXT: s_mov_b64 s[4:5], exec +; SELDAG-NEXT: s_andn2_b64 s[6:7], exec, vcc +; SELDAG-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7] +; SELDAG-NEXT: s_cbranch_scc0 .LBB0_2 +; SELDAG-NEXT: ; %bb.1: +; SELDAG-NEXT: s_and_b64 exec, exec, s[4:5] +; SELDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; SELDAG-NEXT: flat_store_dword v[2:3], v0 +; SELDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; SELDAG-NEXT: s_setpc_b64 s[30:31] +; SELDAG-NEXT: .LBB0_2: +; SELDAG-NEXT: s_mov_b64 exec, 0 +; SELDAG-NEXT: s_endpgm + +; GISEL-LABEL: test_kill: +; GISEL-NEXT: ; %bb.0: +; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GISEL-NEXT: flat_load_dword v0, v[0:1] +; GISEL-NEXT: v_and_b32_e32 v1, 1, v4 +; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1 +; GISEL-NEXT: s_mov_b64 s[4:5], exec +; GISEL-NEXT: s_andn2_b64 s[6:7], exec, vcc +; GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7] +; GISEL-NEXT: s_cbranch_scc0 .LBB0_2 +; GISEL-NEXT: ; %bb.1: +; GISEL-NEXT: s_and_b64 exec, exec, s[4:5] +; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GISEL-NEXT: flat_store_dword v[2:3], v0 +; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GISEL-NEXT: s_setpc_b64 s[30:31] +; GISEL-NEXT: .LBB0_2: +; GISEL-NEXT: s_mov_b64 exec, 0 +; GISEL-NEXT: s_endpgm + +define void @test_kill(ptr %src, ptr %dst, i1 %c) { + %a = load i32, ptr %src, align 4 + callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill] +kill: + unreachable +cont: + store i32 %a, ptr %dst, align 4 + ret void +} + +;--- without-callbr.ll + +define void @test_kill(ptr %src, ptr %dst, i1 %c) { + %a = load i32, ptr %src, align 4 + call void @llvm.amdgcn.kill(i1 %c) + store i32 %a, ptr %dst, align 4 + ret void +} From 38f0441d7351d5fd17e9d4fe6ba6d5dc61947d7b Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Tue, 1 Apr 2025 08:07:44 -0500 Subject: [PATCH 02/18] fix formatting --- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 5 +++-- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 9 +++++---- llvm/lib/IR/Verifier.cpp | 4 ++-- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 65e36de76307d..3b33d3784c1f9 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -3049,8 +3049,9 @@ bool IRTranslator::translateCallBr(const User &U, addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne()); // TODO: For most of the cases where there is an intrinsic callbr, we're // having exactly one indirect target, which will be unreachable. As soon as - // this changes, we might need to enhance Target->setIsInlineAsmBrIndirectTarget - // or add something similar for intrinsic indirect branches. + // this changes, we might need to enhance + // Target->setIsInlineAsmBrIndirectTarget or add something similar for + // intrinsic indirect branches. if (I.isInlineAsm()) { for (BasicBlock *Dest : I.getIndirectDests()) { MachineBasicBlock *Target = &getMBB(*Dest); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index d4a97d2dfc196..66288b266f400 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3455,8 +3455,8 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { report_fatal_error("Unsupported intrinsic for callbr"); case Intrinsic::amdgcn_kill: if (I.getNumIndirectDests() != 1) - report_fatal_error( - "amdgcn.kill supportes exactly one indirect destination"); + report_fatal_error( + "amdgcn.kill supportes exactly one indirect destination"); CallInst *CI = CallInst::Create(I.getFunctionType(), I.getCalledFunction(), SmallVector(I.args())); @@ -3478,8 +3478,9 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne()); // TODO: For most of the cases where there is an intrinsic callbr, we're // having exactly one indirect target, which will be unreachable. As soon as - // this changes, we might need to enhance Target->setIsInlineAsmBrIndirectTarget - // or add something similar for intrinsic indirect branches. + // this changes, we might need to enhance + // Target->setIsInlineAsmBrIndirectTarget or add something similar for + // intrinsic indirect branches. if (I.isInlineAsm()) { for (BasicBlock *Dest : I.getIndirectDests()) { MachineBasicBlock *Target = FuncInfo.getMBB(Dest); diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 6404cfc780125..c0bf78da4b427 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -3303,8 +3303,8 @@ void Verifier::visitCallBrInst(CallBrInst &CBI) { "Callbr amdgcn_kill only supports one indirect dest"); bool Unreachable = isa(CBI.getIndirectDest(0)->begin()); CallInst *Call = dyn_cast(CBI.getIndirectDest(0)->begin()); - Check(Unreachable || - (Call && Call->getIntrinsicID() == Intrinsic::amdgcn_unreachable), + Check(Unreachable || (Call && Call->getIntrinsicID() == + Intrinsic::amdgcn_unreachable), "Callbr amdgcn_kill indirect dest needs to be unreachable"); visitIntrinsicCall(Intrinsic::amdgcn_kill, CBI); break; From 74a03138e9b1c746eaf9b74b4d22e2f1ba2c37c1 Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Wed, 2 Apr 2025 04:26:22 -0500 Subject: [PATCH 03/18] implement feedback --- llvm/include/llvm/CodeGen/Analysis.h | 2 +- .../llvm/CodeGen/GlobalISel/IRTranslator.h | 18 ++--- llvm/include/llvm/CodeGen/TargetLowering.h | 6 +- llvm/include/llvm/IR/DiagnosticInfo.h | 2 +- llvm/include/llvm/IR/IntrinsicInst.h | 8 ++- llvm/lib/CodeGen/Analysis.cpp | 2 +- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 34 ++++----- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 6 +- .../SelectionDAG/SelectionDAGBuilder.cpp | 71 +++++++++---------- .../SelectionDAG/SelectionDAGBuilder.h | 61 ++++++++-------- .../SelectionDAG/StatepointLowering.cpp | 2 +- .../CodeGen/SelectionDAG/TargetLowering.cpp | 2 +- llvm/lib/IR/DiagnosticInfo.cpp | 2 +- .../Target/AArch64/AArch64ISelLowering.cpp | 4 +- llvm/lib/Target/AArch64/AArch64ISelLowering.h | 2 +- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +- llvm/lib/Target/AMDGPU/SIISelLowering.h | 4 +- llvm/lib/Target/ARM/ARMISelLowering.cpp | 2 +- llvm/lib/Target/ARM/ARMISelLowering.h | 3 +- .../Target/Hexagon/HexagonISelLowering.cpp | 2 +- llvm/lib/Target/Hexagon/HexagonISelLowering.h | 2 +- .../LoongArch/LoongArchISelLowering.cpp | 2 +- .../Target/LoongArch/LoongArchISelLowering.h | 2 +- llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 7 +- llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 2 +- llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 7 +- llvm/lib/Target/PowerPC/PPCISelLowering.h | 9 ++- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +- llvm/lib/Target/RISCV/RISCVISelLowering.h | 2 +- llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 2 +- llvm/lib/Target/SPIRV/SPIRVISelLowering.h | 2 +- .../WebAssembly/WebAssemblyISelLowering.cpp | 2 +- .../WebAssembly/WebAssemblyISelLowering.h | 2 +- llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +- llvm/lib/Target/X86/X86ISelLowering.h | 2 +- 35 files changed, 138 insertions(+), 146 deletions(-) diff --git a/llvm/include/llvm/CodeGen/Analysis.h b/llvm/include/llvm/CodeGen/Analysis.h index 362cc30bbd06a..a4a604dd2c608 100644 --- a/llvm/include/llvm/CodeGen/Analysis.h +++ b/llvm/include/llvm/CodeGen/Analysis.h @@ -150,7 +150,7 @@ bool returnTypeIsEligibleForTailCall(const Function *F, const Instruction *I, /// Returns true if the parent of \p CI returns CI's first argument after /// calling \p CI. -bool funcReturnsFirstArgOfCall(const CallInst &CI); +bool funcReturnsFirstArgOfCall(const CallBase &CI); DenseMap getEHScopeMembership(const MachineFunction &MF); diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h index 6fd05c8fddd5f..d3f731cf02be9 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -235,26 +235,26 @@ class IRTranslator : public MachineFunctionPass { bool translateStore(const User &U, MachineIRBuilder &MIRBuilder); /// Translate an LLVM string intrinsic (memcpy, memset, ...). - bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, + bool translateMemFunc(const CallBase &CI, MachineIRBuilder &MIRBuilder, unsigned Opcode); /// Translate an LLVM trap intrinsic (trap, debugtrap, ubsantrap). - bool translateTrap(const CallInst &U, MachineIRBuilder &MIRBuilder, + bool translateTrap(const CallBase &U, MachineIRBuilder &MIRBuilder, unsigned Opcode); // Translate @llvm.vector.interleave2 and // @llvm.vector.deinterleave2 intrinsics for fixed-width vector // types into vector shuffles. - bool translateVectorInterleave2Intrinsic(const CallInst &CI, + bool translateVectorInterleave2Intrinsic(const CallBase &CI, MachineIRBuilder &MIRBuilder); - bool translateVectorDeinterleave2Intrinsic(const CallInst &CI, + bool translateVectorDeinterleave2Intrinsic(const CallBase &CI, MachineIRBuilder &MIRBuilder); void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder); - bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op, + bool translateOverflowIntrinsic(const CallBase &CI, unsigned Op, MachineIRBuilder &MIRBuilder); - bool translateFixedPointIntrinsic(unsigned Op, const CallInst &CI, + bool translateFixedPointIntrinsic(unsigned Op, const CallBase &CI, MachineIRBuilder &MIRBuilder); /// Helper function for translateSimpleIntrinsic. @@ -265,13 +265,13 @@ class IRTranslator : public MachineFunctionPass { /// Translates the intrinsics defined in getSimpleIntrinsicOpcode. /// \return true if the translation succeeded. - bool translateSimpleIntrinsic(const CallInst &CI, Intrinsic::ID ID, + bool translateSimpleIntrinsic(const CallBase &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder); bool translateConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder); - bool translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, + bool translateKnownIntrinsic(const CallBase &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder); /// Returns the single livein physical register Arg was lowered to, if @@ -588,7 +588,7 @@ class IRTranslator : public MachineFunctionPass { return false; } - bool translateConvergenceControlIntrinsic(const CallInst &CI, + bool translateConvergenceControlIntrinsic(const CallBase &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder); diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 4ed81d25e8e22..11e24cdb94047 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -1240,7 +1240,7 @@ class LLVM_ABI TargetLoweringBase { /// to a MemIntrinsicNode (touches memory). If this is the case, it returns /// true and store the intrinsic information into the IntrinsicInfo that was /// passed to the function. - virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, + virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallBase &, MachineFunction &, unsigned /*Intrinsic*/) const { return false; @@ -5247,9 +5247,9 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase { const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const; - // Targets may override this function to collect operands from the CallInst + // Targets may override this function to collect operands from the CallBase // and for example, lower them into the SelectionDAG operands. - virtual void CollectTargetIntrinsicOperands(const CallInst &I, + virtual void CollectTargetIntrinsicOperands(const CallBase &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const; diff --git a/llvm/include/llvm/IR/DiagnosticInfo.h b/llvm/include/llvm/IR/DiagnosticInfo.h index 862be04e0fc00..983605dec9197 100644 --- a/llvm/include/llvm/IR/DiagnosticInfo.h +++ b/llvm/include/llvm/IR/DiagnosticInfo.h @@ -39,7 +39,7 @@ namespace llvm { class DiagnosticPrinter; class DIFile; class DISubprogram; -class CallInst; +class CallBase; class Function; class Instruction; class InstructionCost; diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index 20e667524fa0d..09b35e586c9bd 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -135,8 +135,14 @@ class IntrinsicInst : public CallInst { return CF->isIntrinsic(); return false; } + static bool classof(const CallBase *I) { + if (const Function *CF = I->getCalledFunction()) + return CF->isIntrinsic(); + return false; + } static bool classof(const Value *V) { - return isa(V) && classof(cast(V)); + return (isa(V) && classof(cast(V))) || + (isa(V) && classof(cast(V))); } }; diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp index e7b9417de8c9f..f950bb756fb2b 100644 --- a/llvm/lib/CodeGen/Analysis.cpp +++ b/llvm/lib/CodeGen/Analysis.cpp @@ -712,7 +712,7 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F, return true; } -bool llvm::funcReturnsFirstArgOfCall(const CallInst &CI) { +bool llvm::funcReturnsFirstArgOfCall(const CallBase &CI) { const ReturnInst *Ret = dyn_cast(CI.getParent()->getTerminator()); Value *RetVal = Ret ? Ret->getReturnValue() : nullptr; bool ReturnsFirstArg = false; diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 3b33d3784c1f9..1b6efe1c91fc4 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1693,7 +1693,7 @@ bool IRTranslator::translateGetElementPtr(const User &U, return true; } -bool IRTranslator::translateMemFunc(const CallInst &CI, +bool IRTranslator::translateMemFunc(const CallBase &CI, MachineIRBuilder &MIRBuilder, unsigned Opcode) { const Value *SrcPtr = CI.getArgOperand(1); @@ -1780,7 +1780,7 @@ bool IRTranslator::translateMemFunc(const CallInst &CI, return true; } -bool IRTranslator::translateTrap(const CallInst &CI, +bool IRTranslator::translateTrap(const CallBase &CI, MachineIRBuilder &MIRBuilder, unsigned Opcode) { StringRef TrapFuncName = @@ -1807,7 +1807,7 @@ bool IRTranslator::translateTrap(const CallInst &CI, } bool IRTranslator::translateVectorInterleave2Intrinsic( - const CallInst &CI, MachineIRBuilder &MIRBuilder) { + const CallBase &CI, MachineIRBuilder &MIRBuilder) { assert(CI.getIntrinsicID() == Intrinsic::vector_interleave2 && "This function can only be called on the interleave2 intrinsic!"); // Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG). @@ -1823,7 +1823,7 @@ bool IRTranslator::translateVectorInterleave2Intrinsic( } bool IRTranslator::translateVectorDeinterleave2Intrinsic( - const CallInst &CI, MachineIRBuilder &MIRBuilder) { + const CallBase &CI, MachineIRBuilder &MIRBuilder) { assert(CI.getIntrinsicID() == Intrinsic::vector_deinterleave2 && "This function can only be called on the deinterleave2 intrinsic!"); // Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to @@ -1863,7 +1863,7 @@ void IRTranslator::getStackGuard(Register DstReg, MIB.setMemRefs({MemRef}); } -bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op, +bool IRTranslator::translateOverflowIntrinsic(const CallBase &CI, unsigned Op, MachineIRBuilder &MIRBuilder) { ArrayRef ResRegs = getOrCreateVRegs(CI); MIRBuilder.buildInstr( @@ -1873,7 +1873,7 @@ bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op, return true; } -bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI, +bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallBase &CI, MachineIRBuilder &MIRBuilder) { Register Dst = getOrCreateVReg(CI); Register Src0 = getOrCreateVReg(*CI.getOperand(0)); @@ -2022,7 +2022,7 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) { return Intrinsic::not_intrinsic; } -bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI, +bool IRTranslator::translateSimpleIntrinsic(const CallBase &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) { @@ -2144,7 +2144,7 @@ static unsigned getConvOpcode(Intrinsic::ID ID) { } bool IRTranslator::translateConvergenceControlIntrinsic( - const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) { + const CallBase &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) { MachineInstrBuilder MIB = MIRBuilder.buildInstr(getConvOpcode(ID)); Register OutputReg = getOrCreateConvergenceTokenVReg(CI); MIB.addDef(OutputReg); @@ -2160,7 +2160,7 @@ bool IRTranslator::translateConvergenceControlIntrinsic( return true; } -bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, +bool IRTranslator::translateKnownIntrinsic(const CallBase &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) { if (auto *MI = dyn_cast(&CI)) { if (ORE->enabled()) { @@ -2754,7 +2754,7 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { if (containsBF16Type(U)) return false; - const CallInst &CI = cast(U); + const CallBase &CI = cast(U); const Function *F = CI.getCalledFunction(); // FIXME: support Windows dllimport function calls and calls through @@ -3022,22 +3022,14 @@ bool IRTranslator::translateCallBr(const User &U, } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) { switch (I.getIntrinsicID()) { default: - report_fatal_error("Unsupported intrinsic for callbr"); + return false; case Intrinsic::amdgcn_kill: - if (I.getNumIndirectDests() != 1) - report_fatal_error( - "amdgcn.kill supportes exactly one indirect destination"); - CallInst *CI = - CallInst::Create(I.getFunctionType(), I.getCalledFunction(), - SmallVector(I.args())); - bool Success = translateCall(*CI, MIRBuilder); - CI->deleteValue(); - if (!Success) + if (!translateCall(I, MIRBuilder)) return false; break; } } else { - report_fatal_error("Only know how to handle inlineasm/intrinsic callbr"); + return false; } // Retrieve successors. diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 45a37622a531b..e759a4a4d96c6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -8706,7 +8706,7 @@ static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, SDValue SelectionDAG::getMemcpy( SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, - Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, + Align Alignment, bool isVol, bool AlwaysInline, const CallBase *CI, std::optional OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, BatchAAResults *BatchAA) { @@ -8832,7 +8832,7 @@ SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, - bool isVol, const CallInst *CI, + bool isVol, const CallBase *CI, std::optional OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, @@ -8950,7 +8950,7 @@ SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, - const CallInst *CI, + const CallBase *CI, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo) { // Check to see if we should lower the memset to stores first. diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 66288b266f400..7bba7d0954f08 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3454,14 +3454,7 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { default: report_fatal_error("Unsupported intrinsic for callbr"); case Intrinsic::amdgcn_kill: - if (I.getNumIndirectDests() != 1) - report_fatal_error( - "amdgcn.kill supportes exactly one indirect destination"); - CallInst *CI = - CallInst::Create(I.getFunctionType(), I.getCalledFunction(), - SmallVector(I.args())); - visitCall(*CI); - CI->deleteValue(); + visitCallBase(I); break; } } else { @@ -4814,7 +4807,7 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) { DAG.setRoot(StoreNode); } -void SelectionDAGBuilder::visitMaskedStore(const CallInst &I, +void SelectionDAGBuilder::visitMaskedStore(const CallBase &I, bool IsCompressing) { SDLoc sdl = getCurSDLoc(); @@ -4945,7 +4938,7 @@ static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, return true; } -void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) { +void SelectionDAGBuilder::visitMaskedScatter(const CallBase &I) { SDLoc sdl = getCurSDLoc(); // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask) @@ -4990,7 +4983,7 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) { setValue(&I, Scatter); } -void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) { +void SelectionDAGBuilder::visitMaskedLoad(const CallBase &I, bool IsExpanding) { SDLoc sdl = getCurSDLoc(); auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, @@ -5059,7 +5052,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) { setValue(&I, Res); } -void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) { +void SelectionDAGBuilder::visitMaskedGather(const CallBase &I) { SDLoc sdl = getCurSDLoc(); // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0) @@ -5293,7 +5286,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC /// node. -void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, +void SelectionDAGBuilder::visitTargetIntrinsic(const CallBase &I, unsigned Intrinsic) { // Ignore the callsite's attributes. A specific call site may be marked with // readnone, but the lowering code will expect the chain based on the @@ -6376,7 +6369,7 @@ bool SelectionDAGBuilder::visitEntryValueDbgValue( } /// Lower the call to the specified intrinsic function. -void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I, +void SelectionDAGBuilder::visitConvergenceControl(const CallBase &I, unsigned Intrinsic) { SDLoc sdl = getCurSDLoc(); switch (Intrinsic) { @@ -6396,7 +6389,7 @@ void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I, } } -void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I, +void SelectionDAGBuilder::visitVectorHistogram(const CallBase &I, unsigned IntrinsicID) { // For now, we're only lowering an 'add' histogram. // We can add others later, e.g. saturating adds, min/max. @@ -6454,7 +6447,7 @@ void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I, DAG.setRoot(Histogram); } -void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I, +void SelectionDAGBuilder::visitVectorExtractLastActive(const CallBase &I, unsigned Intrinsic) { assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active && "Tried lowering invalid vector extract last"); @@ -6482,7 +6475,7 @@ void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I, } /// Lower the call to the specified intrinsic function. -void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, +void SelectionDAGBuilder::visitIntrinsicCall(const CallBase &I, unsigned Intrinsic) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDLoc sdl = getCurSDLoc(); @@ -9115,7 +9108,7 @@ void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I, /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) { +bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallBase &I) { const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1); const Value *Size = I.getArgOperand(2); const ConstantSDNode *CSize = dyn_cast(getValue(Size)); @@ -9207,7 +9200,7 @@ bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) { /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) { +bool SelectionDAGBuilder::visitMemChrCall(const CallBase &I) { const Value *Src = I.getArgOperand(0); const Value *Char = I.getArgOperand(1); const Value *Length = I.getArgOperand(2); @@ -9231,7 +9224,7 @@ bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) { /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) { +bool SelectionDAGBuilder::visitMemPCpyCall(const CallBase &I) { SDValue Dst = getValue(I.getArgOperand(0)); SDValue Src = getValue(I.getArgOperand(1)); SDValue Size = getValue(I.getArgOperand(2)); @@ -9269,7 +9262,7 @@ bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) { /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) { +bool SelectionDAGBuilder::visitStrCpyCall(const CallBase &I, bool isStpcpy) { const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); @@ -9292,7 +9285,7 @@ bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) { /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) { +bool SelectionDAGBuilder::visitStrCmpCall(const CallBase &I) { const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); @@ -9315,7 +9308,7 @@ bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) { /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) { +bool SelectionDAGBuilder::visitStrLenCall(const CallBase &I) { const Value *Arg0 = I.getArgOperand(0); const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); @@ -9336,7 +9329,7 @@ bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) { /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) { +bool SelectionDAGBuilder::visitStrNLenCall(const CallBase &I) { const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); @@ -9358,7 +9351,7 @@ bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) { /// false and it will be lowered like a normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I, +bool SelectionDAGBuilder::visitUnaryFloatCall(const CallBase &I, unsigned Opcode) { // We already checked this call's prototype; verify it doesn't modify errno. if (!I.onlyReadsMemory()) @@ -9378,7 +9371,7 @@ bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I, /// false, and it will be lowered like a normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I, +bool SelectionDAGBuilder::visitBinaryFloatCall(const CallBase &I, unsigned Opcode) { // We already checked this call's prototype; verify it doesn't modify errno. if (!I.onlyReadsMemory()) @@ -9394,7 +9387,9 @@ bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I, return true; } -void SelectionDAGBuilder::visitCall(const CallInst &I) { +void SelectionDAGBuilder::visitCall(const CallInst &I) { visitCallBase(I); } + +void SelectionDAGBuilder::visitCallBase(const CallBase &I) { // Handle inline assembly differently. if (I.isInlineAsm()) { visitInlineAsm(I); @@ -10557,7 +10552,7 @@ void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call, setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc())); } -void SelectionDAGBuilder::visitVAStart(const CallInst &I) { +void SelectionDAGBuilder::visitVAStart(const CallBase &I) { DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(), MVT::Other, getRoot(), getValue(I.getArgOperand(0)), @@ -10579,14 +10574,14 @@ void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) { setValue(&I, V); } -void SelectionDAGBuilder::visitVAEnd(const CallInst &I) { +void SelectionDAGBuilder::visitVAEnd(const CallBase &I) { DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(), MVT::Other, getRoot(), getValue(I.getArgOperand(0)), DAG.getSrcValue(I.getArgOperand(0)))); } -void SelectionDAGBuilder::visitVACopy(const CallInst &I) { +void SelectionDAGBuilder::visitVACopy(const CallBase &I) { DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(), MVT::Other, getRoot(), getValue(I.getArgOperand(0)), @@ -10704,7 +10699,7 @@ static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, } /// Lower llvm.experimental.stackmap. -void SelectionDAGBuilder::visitStackmap(const CallInst &CI) { +void SelectionDAGBuilder::visitStackmap(const CallBase &CI) { // void @llvm.experimental.stackmap(i64 , i32 , // [live variables...]) @@ -10916,7 +10911,7 @@ void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB, FuncInfo.MF->getFrameInfo().setHasPatchPoint(); } -void SelectionDAGBuilder::visitVectorReduce(const CallInst &I, +void SelectionDAGBuilder::visitVectorReduce(const CallBase &I, unsigned Intrinsic) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue Op1 = getValue(I.getArgOperand(0)); @@ -12599,14 +12594,14 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { } } -void SelectionDAGBuilder::visitStepVector(const CallInst &I) { +void SelectionDAGBuilder::visitStepVector(const CallBase &I) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); auto DL = getCurSDLoc(); EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); setValue(&I, DAG.getStepVector(DL, ResultVT)); } -void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) { +void SelectionDAGBuilder::visitVectorReverse(const CallBase &I) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); @@ -12629,7 +12624,7 @@ void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) { setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask)); } -void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I, +void SelectionDAGBuilder::visitVectorDeinterleave(const CallBase &I, unsigned Factor) { auto DL = getCurSDLoc(); SDValue InVec = getValue(I.getOperand(0)); @@ -12665,7 +12660,7 @@ void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I, setValue(&I, Res); } -void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I, +void SelectionDAGBuilder::visitVectorInterleave(const CallBase &I, unsigned Factor) { auto DL = getCurSDLoc(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); @@ -12719,7 +12714,7 @@ void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) { DAG.getVTList(ValueVTs), Values)); } -void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) { +void SelectionDAGBuilder::visitVectorSplice(const CallBase &I) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); @@ -12793,7 +12788,7 @@ static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) { // setValue(&I, getCopyFromRegs(CBR, CBR->getType())); // otherwise we will end up with copies of virtregs only valid along direct // edges. -void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) { +void SelectionDAGBuilder::visitCallBrLandingPad(const CallBase &I) { SmallVector ResultVTs; SmallVector ResultValues; const auto *CBR = diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 35c15bc269d4b..483e0b9d2db6c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -483,7 +483,7 @@ class SelectionDAGBuilder { void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB); - void LowerDeoptimizeCall(const CallInst *CI); + void LowerDeoptimizeCall(const CallBase *CI); void LowerDeoptimizingReturn(); void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, @@ -537,7 +537,7 @@ class SelectionDAGBuilder { // These all get lowered before this pass. void visitInvoke(const InvokeInst &I); void visitCallBr(const CallBrInst &I); - void visitCallBrLandingPad(const CallInst &I); + void visitCallBrLandingPad(const CallBase &I); void visitResume(const ResumeInst &I); void visitUnary(const User &I, unsigned Opcode); @@ -594,24 +594,25 @@ class SelectionDAGBuilder { void visitAlloca(const AllocaInst &I); void visitLoad(const LoadInst &I); void visitStore(const StoreInst &I); - void visitMaskedLoad(const CallInst &I, bool IsExpanding = false); - void visitMaskedStore(const CallInst &I, bool IsCompressing = false); - void visitMaskedGather(const CallInst &I); - void visitMaskedScatter(const CallInst &I); + void visitMaskedLoad(const CallBase &I, bool IsExpanding = false); + void visitMaskedStore(const CallBase &I, bool IsCompressing = false); + void visitMaskedGather(const CallBase &I); + void visitMaskedScatter(const CallBase &I); void visitAtomicCmpXchg(const AtomicCmpXchgInst &I); void visitAtomicRMW(const AtomicRMWInst &I); void visitFence(const FenceInst &I); void visitPHI(const PHINode &I); void visitCall(const CallInst &I); - bool visitMemCmpBCmpCall(const CallInst &I); - bool visitMemPCpyCall(const CallInst &I); - bool visitMemChrCall(const CallInst &I); - bool visitStrCpyCall(const CallInst &I, bool isStpcpy); - bool visitStrCmpCall(const CallInst &I); - bool visitStrLenCall(const CallInst &I); - bool visitStrNLenCall(const CallInst &I); - bool visitUnaryFloatCall(const CallInst &I, unsigned Opcode); - bool visitBinaryFloatCall(const CallInst &I, unsigned Opcode); + void visitCallBase(const CallBase &I); + bool visitMemCmpBCmpCall(const CallBase &I); + bool visitMemPCpyCall(const CallBase &I); + bool visitMemChrCall(const CallBase &I); + bool visitStrCpyCall(const CallBase &I, bool isStpcpy); + bool visitStrCmpCall(const CallBase &I); + bool visitStrLenCall(const CallBase &I); + bool visitStrNLenCall(const CallBase &I); + bool visitUnaryFloatCall(const CallBase &I, unsigned Opcode); + bool visitBinaryFloatCall(const CallBase &I, unsigned Opcode); void visitAtomicLoad(const LoadInst &I); void visitAtomicStore(const StoreInst &I); void visitLoadFromSwiftError(const LoadInst &I); @@ -624,12 +625,12 @@ class SelectionDAGBuilder { bool visitEntryValueDbgValue(ArrayRef Values, DILocalVariable *Variable, DIExpression *Expr, DebugLoc DbgLoc); - void visitIntrinsicCall(const CallInst &I, unsigned Intrinsic); - void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic); + void visitIntrinsicCall(const CallBase &I, unsigned Intrinsic); + void visitTargetIntrinsic(const CallBase &I, unsigned Intrinsic); void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI); - void visitConvergenceControl(const CallInst &I, unsigned Intrinsic); - void visitVectorHistogram(const CallInst &I, unsigned IntrinsicID); - void visitVectorExtractLastActive(const CallInst &I, unsigned Intrinsic); + void visitConvergenceControl(const CallBase &I, unsigned Intrinsic); + void visitVectorHistogram(const CallBase &I, unsigned IntrinsicID); + void visitVectorExtractLastActive(const CallBase &I, unsigned Intrinsic); void visitVPLoad(const VPIntrinsic &VPIntrin, EVT VT, const SmallVectorImpl &OpValues); void visitVPStore(const VPIntrinsic &VPIntrin, @@ -645,23 +646,23 @@ class SelectionDAGBuilder { void visitVPCmp(const VPCmpIntrinsic &VPIntrin); void visitVectorPredicationIntrinsic(const VPIntrinsic &VPIntrin); - void visitVAStart(const CallInst &I); + void visitVAStart(const CallBase &I); void visitVAArg(const VAArgInst &I); - void visitVAEnd(const CallInst &I); - void visitVACopy(const CallInst &I); - void visitStackmap(const CallInst &I); + void visitVAEnd(const CallBase &I); + void visitVACopy(const CallBase &I); + void visitStackmap(const CallBase &I); void visitPatchpoint(const CallBase &CB, const BasicBlock *EHPadBB = nullptr); // These two are implemented in StatepointLowering.cpp void visitGCRelocate(const GCRelocateInst &Relocate); void visitGCResult(const GCResultInst &I); - void visitVectorReduce(const CallInst &I, unsigned Intrinsic); - void visitVectorReverse(const CallInst &I); - void visitVectorSplice(const CallInst &I); - void visitVectorInterleave(const CallInst &I, unsigned Factor); - void visitVectorDeinterleave(const CallInst &I, unsigned Factor); - void visitStepVector(const CallInst &I); + void visitVectorReduce(const CallBase &I, unsigned Intrinsic); + void visitVectorReverse(const CallBase &I); + void visitVectorSplice(const CallBase &I); + void visitVectorInterleave(const CallBase &I, unsigned Factor); + void visitVectorDeinterleave(const CallBase &I, unsigned Factor); + void visitStepVector(const CallBase &I); void visitUserOp1(const Instruction &I) { llvm_unreachable("UserOp1 should not exist at instruction selection time!"); diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index 80aeefe8e068a..5227fd6a60e7f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -1303,7 +1303,7 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) { setValue(&Relocate, SD); } -void SelectionDAGBuilder::LowerDeoptimizeCall(const CallInst *CI) { +void SelectionDAGBuilder::LowerDeoptimizeCall(const CallBase *CI) { const auto &TLI = DAG.getTargetLoweringInfo(); SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE), TLI.getPointerTy(DAG.getDataLayout())); diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index a0ffb4b6d5a4c..349c52d1a7f7f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -5691,7 +5691,7 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op, } void TargetLowering::CollectTargetIntrinsicOperands( - const CallInst &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const { + const CallBase &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const { } std::pair diff --git a/llvm/lib/IR/DiagnosticInfo.cpp b/llvm/lib/IR/DiagnosticInfo.cpp index 0f1291b8bd8be..6a3c8e831bcd7 100644 --- a/llvm/lib/IR/DiagnosticInfo.cpp +++ b/llvm/lib/IR/DiagnosticInfo.cpp @@ -455,7 +455,7 @@ void DiagnosticInfoMisExpect::print(DiagnosticPrinter &DP) const { void OptimizationRemarkAnalysisFPCommute::anchor() {} void OptimizationRemarkAnalysisAliasing::anchor() {} -void llvm::diagnoseDontCall(const CallInst &CI) { +void llvm::diagnoseDontCall(const CallBase &CI) { const auto *F = dyn_cast(CI.getCalledOperand()->stripPointerCasts()); diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 7519ac5260a64..2439745f7da7e 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -16266,7 +16266,7 @@ SDValue AArch64TargetLowering::LowerVSCALE(SDValue Op, template static bool setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL, - AArch64TargetLowering::IntrinsicInfo &Info, const CallInst &CI) { + AArch64TargetLowering::IntrinsicInfo &Info, const CallBase &CI) { Info.opc = ISD::INTRINSIC_VOID; // Retrieve EC from first vector argument. const EVT VT = TLI.getMemValueType(DL, CI.getArgOperand(0)->getType()); @@ -16291,7 +16291,7 @@ setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL, /// MemIntrinsicNodes. The associated MachineMemOperands record the alignment /// specified in the intrinsic calls. bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallInst &I, + const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const { auto &DL = I.getDataLayout(); diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h index e0b6c1b8c0baf..4d0a0cb2330ea 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -186,7 +186,7 @@ class AArch64TargetLowering : public TargetLowering { EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 30535ae88f7ba..9970f20ef9cfb 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -1232,7 +1232,7 @@ MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const { } bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallInst &CI, + const CallBase &CI, MachineFunction &MF, unsigned IntrID) const { Info.flags = MachineMemOperand::MONone; @@ -1519,7 +1519,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, } void SITargetLowering::CollectTargetIntrinsicOperands( - const CallInst &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const { + const CallBase &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const { switch (cast(I).getIntrinsicID()) { case Intrinsic::amdgcn_addrspacecast_nonnull: { // The DAG's ValueType loses the addrspaces. diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index d71a22722129e..1c8f3e465eea2 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -314,11 +314,11 @@ class SITargetLowering final : public AMDGPUTargetLowering { MVT getPointerTy(const DataLayout &DL, unsigned AS) const override; MVT getPointerMemTy(const DataLayout &DL, unsigned AS) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, + bool getTgtMemIntrinsic(IntrinsicInfo &, const CallBase &, MachineFunction &MF, unsigned IntrinsicID) const override; - void CollectTargetIntrinsicOperands(const CallInst &I, + void CollectTargetIntrinsicOperands(const CallBase &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const override; diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 5b3664c4e961f..dc6a4f227061b 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -20879,7 +20879,7 @@ bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, /// MemIntrinsicNodes. The associated MachineMemOperands record the alignment /// specified in the intrinsic calls. bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallInst &I, + const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h index 87710ee29a249..a4ea597b95699 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.h +++ b/llvm/lib/Target/ARM/ARMISelLowering.h @@ -616,8 +616,7 @@ class VectorType; bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize = false) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallInst &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp index 078eccaa706a2..078d38ce12c0a 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp @@ -2040,7 +2040,7 @@ static Value *getUnderLyingObjectForBrevLdIntr(Value *V) { /// true and store the intrinsic information into the IntrinsicInfo that was /// passed to the function. bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallInst &I, + const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h index 1321bee44a295..9f315681ab95f 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h +++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h @@ -142,7 +142,7 @@ class HexagonTargetLowering : public TargetLowering { const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, SelectionDAG& DAG) const; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index 99dae6ec3eb08..8a0d3045b9770 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -7424,7 +7424,7 @@ bool LoongArchTargetLowering::hasAndNot(SDValue Y) const { } bool LoongArchTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallInst &I, + const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h index 79aa89726191b..b31648cdc7c7c 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h @@ -236,7 +236,7 @@ class LoongArchTargetLowering : public TargetLowering { Value *NewVal, Value *Mask, AtomicOrdering Ord) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp index 492f4ab76fdbb..bd82195836c16 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -3664,9 +3664,10 @@ void NVPTXTargetLowering::LowerAsmOperandForConstraint( // because we need the information that is only available in the "Value" type // of destination // pointer. In particular, the address space information. -bool NVPTXTargetLowering::getTgtMemIntrinsic( - IntrinsicInfo &Info, const CallInst &I, - MachineFunction &MF, unsigned Intrinsic) const { +bool NVPTXTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, + const CallBase &I, + MachineFunction &MF, + unsigned Intrinsic) const { switch (Intrinsic) { default: return false; diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h index 0a54a8fd71f32..85be05fbc6462 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h @@ -124,7 +124,7 @@ class NVPTXTargetLowering : public TargetLowering { const char *getTargetNodeName(unsigned Opcode) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index 0f8e5e57c58b7..a338383ebda91 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -17841,9 +17841,8 @@ void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op, TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); } -void PPCTargetLowering::CollectTargetIntrinsicOperands(const CallInst &I, - SmallVectorImpl &Ops, - SelectionDAG &DAG) const { +void PPCTargetLowering::CollectTargetIntrinsicOperands( + const CallBase &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const { if (I.getNumOperands() <= 1) return; if (!isa(Ops[1].getNode())) @@ -18031,7 +18030,7 @@ PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { } bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallInst &I, + const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h index 4c88bd372b106..39731001e766e 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.h +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h @@ -1020,9 +1020,9 @@ namespace llvm { return TargetLowering::getInlineAsmMemConstraint(ConstraintCode); } - void CollectTargetIntrinsicOperands(const CallInst &I, - SmallVectorImpl &Ops, - SelectionDAG &DAG) const override; + void CollectTargetIntrinsicOperands(const CallBase &I, + SmallVectorImpl &Ops, + SelectionDAG &DAG) const override; /// isLegalAddressingMode - Return true if the addressing mode represented /// by AM is legal for this target, for a load/store of the specified type. @@ -1081,8 +1081,7 @@ namespace llvm { bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallInst &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 7cfada6c0601c..1743ea327de14 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -1715,7 +1715,7 @@ bool RISCVTargetLowering::shouldExpandCttzElements(EVT VT) const { } bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallInst &I, + const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const { auto &DL = I.getDataLayout(); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h index 417d684a62382..3b52f5e50d769 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.h +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h @@ -35,7 +35,7 @@ class RISCVTargetLowering : public TargetLowering { const RISCVSubtarget &getSubtarget() const { return Subtarget; } - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override; bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp index e742a9811984b..ee90f9799645a 100644 --- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp @@ -90,7 +90,7 @@ MVT SPIRVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context, } bool SPIRVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallInst &I, + const Callase &I, MachineFunction &MF, unsigned Intrinsic) const { unsigned AlignIdx = 3; diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h index 9025e6eb0842e..d6e10d611704a 100644 --- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h +++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h @@ -49,7 +49,7 @@ class SPIRVTargetLowering : public TargetLowering { EVT VT) const override; MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp index aac3473311192..c77a9375fbce8 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp @@ -1034,7 +1034,7 @@ EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL, } bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallInst &I, + const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h index 72401a7a259c0..f8f23c932177b 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h @@ -72,7 +72,7 @@ class WebAssemblyTargetLowering final : public TargetLowering { bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override; EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index b4670e270141f..695cd95f0ad5f 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -3089,7 +3089,7 @@ static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) { } bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallInst &I, + const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const { Info.flags = MachineMemOperand::MONone; diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index 5cb6b3e493a32..ee24f0f91b0a0 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -1475,7 +1475,7 @@ namespace llvm { /// to a MemIntrinsicNode (touches memory). If this is the case, it returns /// true and stores the intrinsic information into the IntrinsicInfo that was /// passed to the function. - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, MachineFunction &MF, unsigned Intrinsic) const override; From 99390adbba32641b51d1a0641ab0040a6f3b4300 Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Wed, 2 Apr 2025 05:04:29 -0500 Subject: [PATCH 04/18] adapt and more tests --- llvm/test/Assembler/callbr.ll | 22 +++++++++++++++++ llvm/test/CodeGen/AMDGPU/callbr.ll | 22 ++--------------- llvm/test/Verifier/callbr.ll | 39 ++++++++++++++++++++++++++++++ 3 files changed, 63 insertions(+), 20 deletions(-) create mode 100644 llvm/test/Assembler/callbr.ll diff --git a/llvm/test/Assembler/callbr.ll b/llvm/test/Assembler/callbr.ll new file mode 100644 index 0000000000000..0084e9763c62c --- /dev/null +++ b/llvm/test/Assembler/callbr.ll @@ -0,0 +1,22 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt -S < %s | FileCheck %s +; RUN: llvm-as < %s | llvm-dis | FileCheck %s + +declare void @llvm.amdgcn.kill(i1) + +define void @test_kill(i1 %c) { +; CHECK-LABEL: define void @test_kill( +; CHECK-SAME: i1 [[C:%.*]]) { +; CHECK-NEXT: callbr void @llvm.amdgcn.kill(i1 [[C]]) +; CHECK-NEXT: to label %[[CONT:.*]] [label %kill] +; CHECK: [[KILL:.*:]] +; CHECK-NEXT: unreachable +; CHECK: [[CONT]]: +; CHECK-NEXT: ret void +; + callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill] +kill: + unreachable +cont: + ret void +} diff --git a/llvm/test/CodeGen/AMDGPU/callbr.ll b/llvm/test/CodeGen/AMDGPU/callbr.ll index e2e84dca96cbf..33c75f6cf5aab 100644 --- a/llvm/test/CodeGen/AMDGPU/callbr.ll +++ b/llvm/test/CodeGen/AMDGPU/callbr.ll @@ -1,14 +1,5 @@ -; RUN: rm -rf %t && split-file %s %t -; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/with-callbr-seldag.s < %t/with-callbr.ll -; RUN: FileCheck --check-prefix=SELDAG %s < %t/with-callbr-seldag.s -; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/with-callbr-gisel.s -global-isel < %t/with-callbr.ll -; RUN: FileCheck --check-prefix=GISEL %s < %t/with-callbr-gisel.s -; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/without-callbr-seldag.s < %t/without-callbr.ll -; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o %t/without-callbr-gisel.s -global-isel < %t/without-callbr.ll -; RUN: diff %t/with-callbr-seldag.s %t/without-callbr-seldag.s -; RUN: diff %t/with-callbr-gisel.s %t/without-callbr-gisel.s - -;--- with-callbr.ll +; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o - < %s | FileCheck --check-prefix=SELDAG %s +; RUN: llc -mtriple=amdgcn -mcpu=gfx90a -o - -global-isel < %s | FileCheck --check-prefix=GISEL %s ; SELDAG-LABEL: test_kill: ; SELDAG-NEXT: ; %bb.0: @@ -59,12 +50,3 @@ cont: store i32 %a, ptr %dst, align 4 ret void } - -;--- without-callbr.ll - -define void @test_kill(ptr %src, ptr %dst, i1 %c) { - %a = load i32, ptr %src, align 4 - call void @llvm.amdgcn.kill(i1 %c) - store i32 %a, ptr %dst, align 4 - ret void -} diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll index 9b819c5fed48b..8e125e723e6fc 100644 --- a/llvm/test/Verifier/callbr.ll +++ b/llvm/test/Verifier/callbr.ll @@ -120,3 +120,42 @@ landingpad: %out = call i32 @llvm.callbr.landingpad.i32(i32 %0) ret i32 %out } + +declare void @llvm.amdgcn.kill(i1) + +; CHECK-NEXT: Callbr amdgcn_kill only supports one indirect dest +define void @test_callbr_intrinsic_indirect0(i1 %c) { + callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [] +kill: + unreachable +cont: + ret void +} + +; CHECK-NEXT: Callbr amdgcn_kill only supports one indirect dest +define void @test_callbr_intrinsic_indirect2(i1 %c) { + callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill1, label %kill2] +kill1: + unreachable +kill2: + unreachable +cont: + ret void +} + +; CHECK-NEXT: Callbr amdgcn_kill indirect dest needs to be unreachable +define void @test_callbr_intrinsic_no_unreachable(i1 %c) { + callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill] +kill: + ret void +cont: + ret void +} + +; CHECK-NEXT: Callbr currently only supports asm-goto and selected intrinsics +declare i32 @llvm.amdgcn.workitem.id.x() +define void @test_callbr_intrinsic_unsupported() { + callbr i32 @llvm.amdgcn.workitem.id.x() to label %cont [] +cont: + ret void +} From 2c719149382e564c1a82ba4612dd0214f23cf000 Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Thu, 3 Apr 2025 02:44:58 -0500 Subject: [PATCH 05/18] Revert "implement feedback" This reverts commit db57ce26f1780f285db04481f22c56d8262d60ba. --- llvm/include/llvm/CodeGen/Analysis.h | 2 +- .../llvm/CodeGen/GlobalISel/IRTranslator.h | 18 ++--- llvm/include/llvm/CodeGen/TargetLowering.h | 6 +- llvm/include/llvm/IR/DiagnosticInfo.h | 2 +- llvm/include/llvm/IR/IntrinsicInst.h | 8 +-- llvm/lib/CodeGen/Analysis.cpp | 2 +- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 34 +++++---- .../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 6 +- .../SelectionDAG/SelectionDAGBuilder.cpp | 71 ++++++++++--------- .../SelectionDAG/SelectionDAGBuilder.h | 61 ++++++++-------- .../SelectionDAG/StatepointLowering.cpp | 2 +- .../CodeGen/SelectionDAG/TargetLowering.cpp | 2 +- llvm/lib/IR/DiagnosticInfo.cpp | 2 +- .../Target/AArch64/AArch64ISelLowering.cpp | 4 +- llvm/lib/Target/AArch64/AArch64ISelLowering.h | 2 +- llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 4 +- llvm/lib/Target/AMDGPU/SIISelLowering.h | 4 +- llvm/lib/Target/ARM/ARMISelLowering.cpp | 2 +- llvm/lib/Target/ARM/ARMISelLowering.h | 3 +- .../Target/Hexagon/HexagonISelLowering.cpp | 2 +- llvm/lib/Target/Hexagon/HexagonISelLowering.h | 2 +- .../LoongArch/LoongArchISelLowering.cpp | 2 +- .../Target/LoongArch/LoongArchISelLowering.h | 2 +- llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 7 +- llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 2 +- llvm/lib/Target/PowerPC/PPCISelLowering.cpp | 7 +- llvm/lib/Target/PowerPC/PPCISelLowering.h | 9 +-- llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 2 +- llvm/lib/Target/RISCV/RISCVISelLowering.h | 2 +- llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp | 2 +- llvm/lib/Target/SPIRV/SPIRVISelLowering.h | 2 +- .../WebAssembly/WebAssemblyISelLowering.cpp | 2 +- .../WebAssembly/WebAssemblyISelLowering.h | 2 +- llvm/lib/Target/X86/X86ISelLowering.cpp | 2 +- llvm/lib/Target/X86/X86ISelLowering.h | 2 +- 35 files changed, 146 insertions(+), 138 deletions(-) diff --git a/llvm/include/llvm/CodeGen/Analysis.h b/llvm/include/llvm/CodeGen/Analysis.h index a4a604dd2c608..362cc30bbd06a 100644 --- a/llvm/include/llvm/CodeGen/Analysis.h +++ b/llvm/include/llvm/CodeGen/Analysis.h @@ -150,7 +150,7 @@ bool returnTypeIsEligibleForTailCall(const Function *F, const Instruction *I, /// Returns true if the parent of \p CI returns CI's first argument after /// calling \p CI. -bool funcReturnsFirstArgOfCall(const CallBase &CI); +bool funcReturnsFirstArgOfCall(const CallInst &CI); DenseMap getEHScopeMembership(const MachineFunction &MF); diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h index d3f731cf02be9..6fd05c8fddd5f 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -235,26 +235,26 @@ class IRTranslator : public MachineFunctionPass { bool translateStore(const User &U, MachineIRBuilder &MIRBuilder); /// Translate an LLVM string intrinsic (memcpy, memset, ...). - bool translateMemFunc(const CallBase &CI, MachineIRBuilder &MIRBuilder, + bool translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, unsigned Opcode); /// Translate an LLVM trap intrinsic (trap, debugtrap, ubsantrap). - bool translateTrap(const CallBase &U, MachineIRBuilder &MIRBuilder, + bool translateTrap(const CallInst &U, MachineIRBuilder &MIRBuilder, unsigned Opcode); // Translate @llvm.vector.interleave2 and // @llvm.vector.deinterleave2 intrinsics for fixed-width vector // types into vector shuffles. - bool translateVectorInterleave2Intrinsic(const CallBase &CI, + bool translateVectorInterleave2Intrinsic(const CallInst &CI, MachineIRBuilder &MIRBuilder); - bool translateVectorDeinterleave2Intrinsic(const CallBase &CI, + bool translateVectorDeinterleave2Intrinsic(const CallInst &CI, MachineIRBuilder &MIRBuilder); void getStackGuard(Register DstReg, MachineIRBuilder &MIRBuilder); - bool translateOverflowIntrinsic(const CallBase &CI, unsigned Op, + bool translateOverflowIntrinsic(const CallInst &CI, unsigned Op, MachineIRBuilder &MIRBuilder); - bool translateFixedPointIntrinsic(unsigned Op, const CallBase &CI, + bool translateFixedPointIntrinsic(unsigned Op, const CallInst &CI, MachineIRBuilder &MIRBuilder); /// Helper function for translateSimpleIntrinsic. @@ -265,13 +265,13 @@ class IRTranslator : public MachineFunctionPass { /// Translates the intrinsics defined in getSimpleIntrinsicOpcode. /// \return true if the translation succeeded. - bool translateSimpleIntrinsic(const CallBase &CI, Intrinsic::ID ID, + bool translateSimpleIntrinsic(const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder); bool translateConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI, MachineIRBuilder &MIRBuilder); - bool translateKnownIntrinsic(const CallBase &CI, Intrinsic::ID ID, + bool translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder); /// Returns the single livein physical register Arg was lowered to, if @@ -588,7 +588,7 @@ class IRTranslator : public MachineFunctionPass { return false; } - bool translateConvergenceControlIntrinsic(const CallBase &CI, + bool translateConvergenceControlIntrinsic(const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder); diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 11e24cdb94047..4ed81d25e8e22 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -1240,7 +1240,7 @@ class LLVM_ABI TargetLoweringBase { /// to a MemIntrinsicNode (touches memory). If this is the case, it returns /// true and store the intrinsic information into the IntrinsicInfo that was /// passed to the function. - virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallBase &, + virtual bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &, unsigned /*Intrinsic*/) const { return false; @@ -5247,9 +5247,9 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase { const AsmOperandInfo &OpInfo, SelectionDAG &DAG) const; - // Targets may override this function to collect operands from the CallBase + // Targets may override this function to collect operands from the CallInst // and for example, lower them into the SelectionDAG operands. - virtual void CollectTargetIntrinsicOperands(const CallBase &I, + virtual void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const; diff --git a/llvm/include/llvm/IR/DiagnosticInfo.h b/llvm/include/llvm/IR/DiagnosticInfo.h index 983605dec9197..862be04e0fc00 100644 --- a/llvm/include/llvm/IR/DiagnosticInfo.h +++ b/llvm/include/llvm/IR/DiagnosticInfo.h @@ -39,7 +39,7 @@ namespace llvm { class DiagnosticPrinter; class DIFile; class DISubprogram; -class CallBase; +class CallInst; class Function; class Instruction; class InstructionCost; diff --git a/llvm/include/llvm/IR/IntrinsicInst.h b/llvm/include/llvm/IR/IntrinsicInst.h index 09b35e586c9bd..20e667524fa0d 100644 --- a/llvm/include/llvm/IR/IntrinsicInst.h +++ b/llvm/include/llvm/IR/IntrinsicInst.h @@ -135,14 +135,8 @@ class IntrinsicInst : public CallInst { return CF->isIntrinsic(); return false; } - static bool classof(const CallBase *I) { - if (const Function *CF = I->getCalledFunction()) - return CF->isIntrinsic(); - return false; - } static bool classof(const Value *V) { - return (isa(V) && classof(cast(V))) || - (isa(V) && classof(cast(V))); + return isa(V) && classof(cast(V)); } }; diff --git a/llvm/lib/CodeGen/Analysis.cpp b/llvm/lib/CodeGen/Analysis.cpp index f950bb756fb2b..e7b9417de8c9f 100644 --- a/llvm/lib/CodeGen/Analysis.cpp +++ b/llvm/lib/CodeGen/Analysis.cpp @@ -712,7 +712,7 @@ bool llvm::returnTypeIsEligibleForTailCall(const Function *F, return true; } -bool llvm::funcReturnsFirstArgOfCall(const CallBase &CI) { +bool llvm::funcReturnsFirstArgOfCall(const CallInst &CI) { const ReturnInst *Ret = dyn_cast(CI.getParent()->getTerminator()); Value *RetVal = Ret ? Ret->getReturnValue() : nullptr; bool ReturnsFirstArg = false; diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 1b6efe1c91fc4..3b33d3784c1f9 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -1693,7 +1693,7 @@ bool IRTranslator::translateGetElementPtr(const User &U, return true; } -bool IRTranslator::translateMemFunc(const CallBase &CI, +bool IRTranslator::translateMemFunc(const CallInst &CI, MachineIRBuilder &MIRBuilder, unsigned Opcode) { const Value *SrcPtr = CI.getArgOperand(1); @@ -1780,7 +1780,7 @@ bool IRTranslator::translateMemFunc(const CallBase &CI, return true; } -bool IRTranslator::translateTrap(const CallBase &CI, +bool IRTranslator::translateTrap(const CallInst &CI, MachineIRBuilder &MIRBuilder, unsigned Opcode) { StringRef TrapFuncName = @@ -1807,7 +1807,7 @@ bool IRTranslator::translateTrap(const CallBase &CI, } bool IRTranslator::translateVectorInterleave2Intrinsic( - const CallBase &CI, MachineIRBuilder &MIRBuilder) { + const CallInst &CI, MachineIRBuilder &MIRBuilder) { assert(CI.getIntrinsicID() == Intrinsic::vector_interleave2 && "This function can only be called on the interleave2 intrinsic!"); // Canonicalize interleave2 to G_SHUFFLE_VECTOR (similar to SelectionDAG). @@ -1823,7 +1823,7 @@ bool IRTranslator::translateVectorInterleave2Intrinsic( } bool IRTranslator::translateVectorDeinterleave2Intrinsic( - const CallBase &CI, MachineIRBuilder &MIRBuilder) { + const CallInst &CI, MachineIRBuilder &MIRBuilder) { assert(CI.getIntrinsicID() == Intrinsic::vector_deinterleave2 && "This function can only be called on the deinterleave2 intrinsic!"); // Canonicalize deinterleave2 to shuffles that extract sub-vectors (similar to @@ -1863,7 +1863,7 @@ void IRTranslator::getStackGuard(Register DstReg, MIB.setMemRefs({MemRef}); } -bool IRTranslator::translateOverflowIntrinsic(const CallBase &CI, unsigned Op, +bool IRTranslator::translateOverflowIntrinsic(const CallInst &CI, unsigned Op, MachineIRBuilder &MIRBuilder) { ArrayRef ResRegs = getOrCreateVRegs(CI); MIRBuilder.buildInstr( @@ -1873,7 +1873,7 @@ bool IRTranslator::translateOverflowIntrinsic(const CallBase &CI, unsigned Op, return true; } -bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallBase &CI, +bool IRTranslator::translateFixedPointIntrinsic(unsigned Op, const CallInst &CI, MachineIRBuilder &MIRBuilder) { Register Dst = getOrCreateVReg(CI); Register Src0 = getOrCreateVReg(*CI.getOperand(0)); @@ -2022,7 +2022,7 @@ unsigned IRTranslator::getSimpleIntrinsicOpcode(Intrinsic::ID ID) { return Intrinsic::not_intrinsic; } -bool IRTranslator::translateSimpleIntrinsic(const CallBase &CI, +bool IRTranslator::translateSimpleIntrinsic(const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) { @@ -2144,7 +2144,7 @@ static unsigned getConvOpcode(Intrinsic::ID ID) { } bool IRTranslator::translateConvergenceControlIntrinsic( - const CallBase &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) { + const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) { MachineInstrBuilder MIB = MIRBuilder.buildInstr(getConvOpcode(ID)); Register OutputReg = getOrCreateConvergenceTokenVReg(CI); MIB.addDef(OutputReg); @@ -2160,7 +2160,7 @@ bool IRTranslator::translateConvergenceControlIntrinsic( return true; } -bool IRTranslator::translateKnownIntrinsic(const CallBase &CI, Intrinsic::ID ID, +bool IRTranslator::translateKnownIntrinsic(const CallInst &CI, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder) { if (auto *MI = dyn_cast(&CI)) { if (ORE->enabled()) { @@ -2754,7 +2754,7 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { if (containsBF16Type(U)) return false; - const CallBase &CI = cast(U); + const CallInst &CI = cast(U); const Function *F = CI.getCalledFunction(); // FIXME: support Windows dllimport function calls and calls through @@ -3022,14 +3022,22 @@ bool IRTranslator::translateCallBr(const User &U, } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) { switch (I.getIntrinsicID()) { default: - return false; + report_fatal_error("Unsupported intrinsic for callbr"); case Intrinsic::amdgcn_kill: - if (!translateCall(I, MIRBuilder)) + if (I.getNumIndirectDests() != 1) + report_fatal_error( + "amdgcn.kill supportes exactly one indirect destination"); + CallInst *CI = + CallInst::Create(I.getFunctionType(), I.getCalledFunction(), + SmallVector(I.args())); + bool Success = translateCall(*CI, MIRBuilder); + CI->deleteValue(); + if (!Success) return false; break; } } else { - return false; + report_fatal_error("Only know how to handle inlineasm/intrinsic callbr"); } // Retrieve successors. diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index e759a4a4d96c6..45a37622a531b 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -8706,7 +8706,7 @@ static void checkAddrSpaceIsValidForLibcall(const TargetLowering *TLI, SDValue SelectionDAG::getMemcpy( SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, - Align Alignment, bool isVol, bool AlwaysInline, const CallBase *CI, + Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, BatchAAResults *BatchAA) { @@ -8832,7 +8832,7 @@ SDValue SelectionDAG::getAtomicMemcpy(SDValue Chain, const SDLoc &dl, SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, - bool isVol, const CallBase *CI, + bool isVol, const CallInst *CI, std::optional OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, @@ -8950,7 +8950,7 @@ SDValue SelectionDAG::getAtomicMemmove(SDValue Chain, const SDLoc &dl, SDValue SelectionDAG::getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, - const CallBase *CI, + const CallInst *CI, MachinePointerInfo DstPtrInfo, const AAMDNodes &AAInfo) { // Check to see if we should lower the memset to stores first. diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 7bba7d0954f08..66288b266f400 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3454,7 +3454,14 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { default: report_fatal_error("Unsupported intrinsic for callbr"); case Intrinsic::amdgcn_kill: - visitCallBase(I); + if (I.getNumIndirectDests() != 1) + report_fatal_error( + "amdgcn.kill supportes exactly one indirect destination"); + CallInst *CI = + CallInst::Create(I.getFunctionType(), I.getCalledFunction(), + SmallVector(I.args())); + visitCall(*CI); + CI->deleteValue(); break; } } else { @@ -4807,7 +4814,7 @@ void SelectionDAGBuilder::visitStore(const StoreInst &I) { DAG.setRoot(StoreNode); } -void SelectionDAGBuilder::visitMaskedStore(const CallBase &I, +void SelectionDAGBuilder::visitMaskedStore(const CallInst &I, bool IsCompressing) { SDLoc sdl = getCurSDLoc(); @@ -4938,7 +4945,7 @@ static bool getUniformBase(const Value *Ptr, SDValue &Base, SDValue &Index, return true; } -void SelectionDAGBuilder::visitMaskedScatter(const CallBase &I) { +void SelectionDAGBuilder::visitMaskedScatter(const CallInst &I) { SDLoc sdl = getCurSDLoc(); // llvm.masked.scatter.*(Src0, Ptrs, alignment, Mask) @@ -4983,7 +4990,7 @@ void SelectionDAGBuilder::visitMaskedScatter(const CallBase &I) { setValue(&I, Scatter); } -void SelectionDAGBuilder::visitMaskedLoad(const CallBase &I, bool IsExpanding) { +void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) { SDLoc sdl = getCurSDLoc(); auto getMaskedLoadOps = [&](Value *&Ptr, Value *&Mask, Value *&Src0, @@ -5052,7 +5059,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallBase &I, bool IsExpanding) { setValue(&I, Res); } -void SelectionDAGBuilder::visitMaskedGather(const CallBase &I) { +void SelectionDAGBuilder::visitMaskedGather(const CallInst &I) { SDLoc sdl = getCurSDLoc(); // @llvm.masked.gather.*(Ptrs, alignment, Mask, Src0) @@ -5286,7 +5293,7 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { /// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC /// node. -void SelectionDAGBuilder::visitTargetIntrinsic(const CallBase &I, +void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic) { // Ignore the callsite's attributes. A specific call site may be marked with // readnone, but the lowering code will expect the chain based on the @@ -6369,7 +6376,7 @@ bool SelectionDAGBuilder::visitEntryValueDbgValue( } /// Lower the call to the specified intrinsic function. -void SelectionDAGBuilder::visitConvergenceControl(const CallBase &I, +void SelectionDAGBuilder::visitConvergenceControl(const CallInst &I, unsigned Intrinsic) { SDLoc sdl = getCurSDLoc(); switch (Intrinsic) { @@ -6389,7 +6396,7 @@ void SelectionDAGBuilder::visitConvergenceControl(const CallBase &I, } } -void SelectionDAGBuilder::visitVectorHistogram(const CallBase &I, +void SelectionDAGBuilder::visitVectorHistogram(const CallInst &I, unsigned IntrinsicID) { // For now, we're only lowering an 'add' histogram. // We can add others later, e.g. saturating adds, min/max. @@ -6447,7 +6454,7 @@ void SelectionDAGBuilder::visitVectorHistogram(const CallBase &I, DAG.setRoot(Histogram); } -void SelectionDAGBuilder::visitVectorExtractLastActive(const CallBase &I, +void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I, unsigned Intrinsic) { assert(Intrinsic == Intrinsic::experimental_vector_extract_last_active && "Tried lowering invalid vector extract last"); @@ -6475,7 +6482,7 @@ void SelectionDAGBuilder::visitVectorExtractLastActive(const CallBase &I, } /// Lower the call to the specified intrinsic function. -void SelectionDAGBuilder::visitIntrinsicCall(const CallBase &I, +void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDLoc sdl = getCurSDLoc(); @@ -9108,7 +9115,7 @@ void SelectionDAGBuilder::processIntegerCallValue(const Instruction &I, /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallBase &I) { +bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallInst &I) { const Value *LHS = I.getArgOperand(0), *RHS = I.getArgOperand(1); const Value *Size = I.getArgOperand(2); const ConstantSDNode *CSize = dyn_cast(getValue(Size)); @@ -9200,7 +9207,7 @@ bool SelectionDAGBuilder::visitMemCmpBCmpCall(const CallBase &I) { /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitMemChrCall(const CallBase &I) { +bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) { const Value *Src = I.getArgOperand(0); const Value *Char = I.getArgOperand(1); const Value *Length = I.getArgOperand(2); @@ -9224,7 +9231,7 @@ bool SelectionDAGBuilder::visitMemChrCall(const CallBase &I) { /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitMemPCpyCall(const CallBase &I) { +bool SelectionDAGBuilder::visitMemPCpyCall(const CallInst &I) { SDValue Dst = getValue(I.getArgOperand(0)); SDValue Src = getValue(I.getArgOperand(1)); SDValue Size = getValue(I.getArgOperand(2)); @@ -9262,7 +9269,7 @@ bool SelectionDAGBuilder::visitMemPCpyCall(const CallBase &I) { /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitStrCpyCall(const CallBase &I, bool isStpcpy) { +bool SelectionDAGBuilder::visitStrCpyCall(const CallInst &I, bool isStpcpy) { const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); @@ -9285,7 +9292,7 @@ bool SelectionDAGBuilder::visitStrCpyCall(const CallBase &I, bool isStpcpy) { /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitStrCmpCall(const CallBase &I) { +bool SelectionDAGBuilder::visitStrCmpCall(const CallInst &I) { const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); @@ -9308,7 +9315,7 @@ bool SelectionDAGBuilder::visitStrCmpCall(const CallBase &I) { /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitStrLenCall(const CallBase &I) { +bool SelectionDAGBuilder::visitStrLenCall(const CallInst &I) { const Value *Arg0 = I.getArgOperand(0); const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); @@ -9329,7 +9336,7 @@ bool SelectionDAGBuilder::visitStrLenCall(const CallBase &I) { /// normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitStrNLenCall(const CallBase &I) { +bool SelectionDAGBuilder::visitStrNLenCall(const CallInst &I) { const Value *Arg0 = I.getArgOperand(0), *Arg1 = I.getArgOperand(1); const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); @@ -9351,7 +9358,7 @@ bool SelectionDAGBuilder::visitStrNLenCall(const CallBase &I) { /// false and it will be lowered like a normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitUnaryFloatCall(const CallBase &I, +bool SelectionDAGBuilder::visitUnaryFloatCall(const CallInst &I, unsigned Opcode) { // We already checked this call's prototype; verify it doesn't modify errno. if (!I.onlyReadsMemory()) @@ -9371,7 +9378,7 @@ bool SelectionDAGBuilder::visitUnaryFloatCall(const CallBase &I, /// false, and it will be lowered like a normal call. /// The caller already checked that \p I calls the appropriate LibFunc with a /// correct prototype. -bool SelectionDAGBuilder::visitBinaryFloatCall(const CallBase &I, +bool SelectionDAGBuilder::visitBinaryFloatCall(const CallInst &I, unsigned Opcode) { // We already checked this call's prototype; verify it doesn't modify errno. if (!I.onlyReadsMemory()) @@ -9387,9 +9394,7 @@ bool SelectionDAGBuilder::visitBinaryFloatCall(const CallBase &I, return true; } -void SelectionDAGBuilder::visitCall(const CallInst &I) { visitCallBase(I); } - -void SelectionDAGBuilder::visitCallBase(const CallBase &I) { +void SelectionDAGBuilder::visitCall(const CallInst &I) { // Handle inline assembly differently. if (I.isInlineAsm()) { visitInlineAsm(I); @@ -10552,7 +10557,7 @@ void SelectionDAGBuilder::emitInlineAsmError(const CallBase &Call, setValue(&Call, DAG.getMergeValues(Ops, getCurSDLoc())); } -void SelectionDAGBuilder::visitVAStart(const CallBase &I) { +void SelectionDAGBuilder::visitVAStart(const CallInst &I) { DAG.setRoot(DAG.getNode(ISD::VASTART, getCurSDLoc(), MVT::Other, getRoot(), getValue(I.getArgOperand(0)), @@ -10574,14 +10579,14 @@ void SelectionDAGBuilder::visitVAArg(const VAArgInst &I) { setValue(&I, V); } -void SelectionDAGBuilder::visitVAEnd(const CallBase &I) { +void SelectionDAGBuilder::visitVAEnd(const CallInst &I) { DAG.setRoot(DAG.getNode(ISD::VAEND, getCurSDLoc(), MVT::Other, getRoot(), getValue(I.getArgOperand(0)), DAG.getSrcValue(I.getArgOperand(0)))); } -void SelectionDAGBuilder::visitVACopy(const CallBase &I) { +void SelectionDAGBuilder::visitVACopy(const CallInst &I) { DAG.setRoot(DAG.getNode(ISD::VACOPY, getCurSDLoc(), MVT::Other, getRoot(), getValue(I.getArgOperand(0)), @@ -10699,7 +10704,7 @@ static void addStackMapLiveVars(const CallBase &Call, unsigned StartIdx, } /// Lower llvm.experimental.stackmap. -void SelectionDAGBuilder::visitStackmap(const CallBase &CI) { +void SelectionDAGBuilder::visitStackmap(const CallInst &CI) { // void @llvm.experimental.stackmap(i64 , i32 , // [live variables...]) @@ -10911,7 +10916,7 @@ void SelectionDAGBuilder::visitPatchpoint(const CallBase &CB, FuncInfo.MF->getFrameInfo().setHasPatchPoint(); } -void SelectionDAGBuilder::visitVectorReduce(const CallBase &I, +void SelectionDAGBuilder::visitVectorReduce(const CallInst &I, unsigned Intrinsic) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); SDValue Op1 = getValue(I.getArgOperand(0)); @@ -12594,14 +12599,14 @@ void SelectionDAGBuilder::visitSwitch(const SwitchInst &SI) { } } -void SelectionDAGBuilder::visitStepVector(const CallBase &I) { +void SelectionDAGBuilder::visitStepVector(const CallInst &I) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); auto DL = getCurSDLoc(); EVT ResultVT = TLI.getValueType(DAG.getDataLayout(), I.getType()); setValue(&I, DAG.getStepVector(DL, ResultVT)); } -void SelectionDAGBuilder::visitVectorReverse(const CallBase &I) { +void SelectionDAGBuilder::visitVectorReverse(const CallInst &I) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); @@ -12624,7 +12629,7 @@ void SelectionDAGBuilder::visitVectorReverse(const CallBase &I) { setValue(&I, DAG.getVectorShuffle(VT, DL, V, DAG.getUNDEF(VT), Mask)); } -void SelectionDAGBuilder::visitVectorDeinterleave(const CallBase &I, +void SelectionDAGBuilder::visitVectorDeinterleave(const CallInst &I, unsigned Factor) { auto DL = getCurSDLoc(); SDValue InVec = getValue(I.getOperand(0)); @@ -12660,7 +12665,7 @@ void SelectionDAGBuilder::visitVectorDeinterleave(const CallBase &I, setValue(&I, Res); } -void SelectionDAGBuilder::visitVectorInterleave(const CallBase &I, +void SelectionDAGBuilder::visitVectorInterleave(const CallInst &I, unsigned Factor) { auto DL = getCurSDLoc(); const TargetLowering &TLI = DAG.getTargetLoweringInfo(); @@ -12714,7 +12719,7 @@ void SelectionDAGBuilder::visitFreeze(const FreezeInst &I) { DAG.getVTList(ValueVTs), Values)); } -void SelectionDAGBuilder::visitVectorSplice(const CallBase &I) { +void SelectionDAGBuilder::visitVectorSplice(const CallInst &I) { const TargetLowering &TLI = DAG.getTargetLoweringInfo(); EVT VT = TLI.getValueType(DAG.getDataLayout(), I.getType()); @@ -12788,7 +12793,7 @@ static Register FollowCopyChain(MachineRegisterInfo &MRI, Register Reg) { // setValue(&I, getCopyFromRegs(CBR, CBR->getType())); // otherwise we will end up with copies of virtregs only valid along direct // edges. -void SelectionDAGBuilder::visitCallBrLandingPad(const CallBase &I) { +void SelectionDAGBuilder::visitCallBrLandingPad(const CallInst &I) { SmallVector ResultVTs; SmallVector ResultValues; const auto *CBR = diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 483e0b9d2db6c..35c15bc269d4b 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -483,7 +483,7 @@ class SelectionDAGBuilder { void LowerCallSiteWithDeoptBundle(const CallBase *Call, SDValue Callee, const BasicBlock *EHPadBB); - void LowerDeoptimizeCall(const CallBase *CI); + void LowerDeoptimizeCall(const CallInst *CI); void LowerDeoptimizingReturn(); void LowerCallSiteWithDeoptBundleImpl(const CallBase *Call, SDValue Callee, @@ -537,7 +537,7 @@ class SelectionDAGBuilder { // These all get lowered before this pass. void visitInvoke(const InvokeInst &I); void visitCallBr(const CallBrInst &I); - void visitCallBrLandingPad(const CallBase &I); + void visitCallBrLandingPad(const CallInst &I); void visitResume(const ResumeInst &I); void visitUnary(const User &I, unsigned Opcode); @@ -594,25 +594,24 @@ class SelectionDAGBuilder { void visitAlloca(const AllocaInst &I); void visitLoad(const LoadInst &I); void visitStore(const StoreInst &I); - void visitMaskedLoad(const CallBase &I, bool IsExpanding = false); - void visitMaskedStore(const CallBase &I, bool IsCompressing = false); - void visitMaskedGather(const CallBase &I); - void visitMaskedScatter(const CallBase &I); + void visitMaskedLoad(const CallInst &I, bool IsExpanding = false); + void visitMaskedStore(const CallInst &I, bool IsCompressing = false); + void visitMaskedGather(const CallInst &I); + void visitMaskedScatter(const CallInst &I); void visitAtomicCmpXchg(const AtomicCmpXchgInst &I); void visitAtomicRMW(const AtomicRMWInst &I); void visitFence(const FenceInst &I); void visitPHI(const PHINode &I); void visitCall(const CallInst &I); - void visitCallBase(const CallBase &I); - bool visitMemCmpBCmpCall(const CallBase &I); - bool visitMemPCpyCall(const CallBase &I); - bool visitMemChrCall(const CallBase &I); - bool visitStrCpyCall(const CallBase &I, bool isStpcpy); - bool visitStrCmpCall(const CallBase &I); - bool visitStrLenCall(const CallBase &I); - bool visitStrNLenCall(const CallBase &I); - bool visitUnaryFloatCall(const CallBase &I, unsigned Opcode); - bool visitBinaryFloatCall(const CallBase &I, unsigned Opcode); + bool visitMemCmpBCmpCall(const CallInst &I); + bool visitMemPCpyCall(const CallInst &I); + bool visitMemChrCall(const CallInst &I); + bool visitStrCpyCall(const CallInst &I, bool isStpcpy); + bool visitStrCmpCall(const CallInst &I); + bool visitStrLenCall(const CallInst &I); + bool visitStrNLenCall(const CallInst &I); + bool visitUnaryFloatCall(const CallInst &I, unsigned Opcode); + bool visitBinaryFloatCall(const CallInst &I, unsigned Opcode); void visitAtomicLoad(const LoadInst &I); void visitAtomicStore(const StoreInst &I); void visitLoadFromSwiftError(const LoadInst &I); @@ -625,12 +624,12 @@ class SelectionDAGBuilder { bool visitEntryValueDbgValue(ArrayRef Values, DILocalVariable *Variable, DIExpression *Expr, DebugLoc DbgLoc); - void visitIntrinsicCall(const CallBase &I, unsigned Intrinsic); - void visitTargetIntrinsic(const CallBase &I, unsigned Intrinsic); + void visitIntrinsicCall(const CallInst &I, unsigned Intrinsic); + void visitTargetIntrinsic(const CallInst &I, unsigned Intrinsic); void visitConstrainedFPIntrinsic(const ConstrainedFPIntrinsic &FPI); - void visitConvergenceControl(const CallBase &I, unsigned Intrinsic); - void visitVectorHistogram(const CallBase &I, unsigned IntrinsicID); - void visitVectorExtractLastActive(const CallBase &I, unsigned Intrinsic); + void visitConvergenceControl(const CallInst &I, unsigned Intrinsic); + void visitVectorHistogram(const CallInst &I, unsigned IntrinsicID); + void visitVectorExtractLastActive(const CallInst &I, unsigned Intrinsic); void visitVPLoad(const VPIntrinsic &VPIntrin, EVT VT, const SmallVectorImpl &OpValues); void visitVPStore(const VPIntrinsic &VPIntrin, @@ -646,23 +645,23 @@ class SelectionDAGBuilder { void visitVPCmp(const VPCmpIntrinsic &VPIntrin); void visitVectorPredicationIntrinsic(const VPIntrinsic &VPIntrin); - void visitVAStart(const CallBase &I); + void visitVAStart(const CallInst &I); void visitVAArg(const VAArgInst &I); - void visitVAEnd(const CallBase &I); - void visitVACopy(const CallBase &I); - void visitStackmap(const CallBase &I); + void visitVAEnd(const CallInst &I); + void visitVACopy(const CallInst &I); + void visitStackmap(const CallInst &I); void visitPatchpoint(const CallBase &CB, const BasicBlock *EHPadBB = nullptr); // These two are implemented in StatepointLowering.cpp void visitGCRelocate(const GCRelocateInst &Relocate); void visitGCResult(const GCResultInst &I); - void visitVectorReduce(const CallBase &I, unsigned Intrinsic); - void visitVectorReverse(const CallBase &I); - void visitVectorSplice(const CallBase &I); - void visitVectorInterleave(const CallBase &I, unsigned Factor); - void visitVectorDeinterleave(const CallBase &I, unsigned Factor); - void visitStepVector(const CallBase &I); + void visitVectorReduce(const CallInst &I, unsigned Intrinsic); + void visitVectorReverse(const CallInst &I); + void visitVectorSplice(const CallInst &I); + void visitVectorInterleave(const CallInst &I, unsigned Factor); + void visitVectorDeinterleave(const CallInst &I, unsigned Factor); + void visitStepVector(const CallInst &I); void visitUserOp1(const Instruction &I) { llvm_unreachable("UserOp1 should not exist at instruction selection time!"); diff --git a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp index 5227fd6a60e7f..80aeefe8e068a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/StatepointLowering.cpp @@ -1303,7 +1303,7 @@ void SelectionDAGBuilder::visitGCRelocate(const GCRelocateInst &Relocate) { setValue(&Relocate, SD); } -void SelectionDAGBuilder::LowerDeoptimizeCall(const CallBase *CI) { +void SelectionDAGBuilder::LowerDeoptimizeCall(const CallInst *CI) { const auto &TLI = DAG.getTargetLoweringInfo(); SDValue Callee = DAG.getExternalSymbol(TLI.getLibcallName(RTLIB::DEOPTIMIZE), TLI.getPointerTy(DAG.getDataLayout())); diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp index 349c52d1a7f7f..a0ffb4b6d5a4c 100644 --- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp @@ -5691,7 +5691,7 @@ void TargetLowering::LowerAsmOperandForConstraint(SDValue Op, } void TargetLowering::CollectTargetIntrinsicOperands( - const CallBase &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const { + const CallInst &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const { } std::pair diff --git a/llvm/lib/IR/DiagnosticInfo.cpp b/llvm/lib/IR/DiagnosticInfo.cpp index 6a3c8e831bcd7..0f1291b8bd8be 100644 --- a/llvm/lib/IR/DiagnosticInfo.cpp +++ b/llvm/lib/IR/DiagnosticInfo.cpp @@ -455,7 +455,7 @@ void DiagnosticInfoMisExpect::print(DiagnosticPrinter &DP) const { void OptimizationRemarkAnalysisFPCommute::anchor() {} void OptimizationRemarkAnalysisAliasing::anchor() {} -void llvm::diagnoseDontCall(const CallBase &CI) { +void llvm::diagnoseDontCall(const CallInst &CI) { const auto *F = dyn_cast(CI.getCalledOperand()->stripPointerCasts()); diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index 2439745f7da7e..7519ac5260a64 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -16266,7 +16266,7 @@ SDValue AArch64TargetLowering::LowerVSCALE(SDValue Op, template static bool setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL, - AArch64TargetLowering::IntrinsicInfo &Info, const CallBase &CI) { + AArch64TargetLowering::IntrinsicInfo &Info, const CallInst &CI) { Info.opc = ISD::INTRINSIC_VOID; // Retrieve EC from first vector argument. const EVT VT = TLI.getMemValueType(DL, CI.getArgOperand(0)->getType()); @@ -16291,7 +16291,7 @@ setInfoSVEStN(const AArch64TargetLowering &TLI, const DataLayout &DL, /// MemIntrinsicNodes. The associated MachineMemOperands record the alignment /// specified in the intrinsic calls. bool AArch64TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallBase &I, + const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const { auto &DL = I.getDataLayout(); diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h index 4d0a0cb2330ea..e0b6c1b8c0baf 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h @@ -186,7 +186,7 @@ class AArch64TargetLowering : public TargetLowering { EmitInstrWithCustomInserter(MachineInstr &MI, MachineBasicBlock *MBB) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index 9970f20ef9cfb..30535ae88f7ba 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -1232,7 +1232,7 @@ MVT SITargetLowering::getPointerMemTy(const DataLayout &DL, unsigned AS) const { } bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallBase &CI, + const CallInst &CI, MachineFunction &MF, unsigned IntrID) const { Info.flags = MachineMemOperand::MONone; @@ -1519,7 +1519,7 @@ bool SITargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, } void SITargetLowering::CollectTargetIntrinsicOperands( - const CallBase &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const { + const CallInst &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const { switch (cast(I).getIntrinsicID()) { case Intrinsic::amdgcn_addrspacecast_nonnull: { // The DAG's ValueType loses the addrspaces. diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h index 1c8f3e465eea2..d71a22722129e 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.h +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h @@ -314,11 +314,11 @@ class SITargetLowering final : public AMDGPUTargetLowering { MVT getPointerTy(const DataLayout &DL, unsigned AS) const override; MVT getPointerMemTy(const DataLayout &DL, unsigned AS) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &, const CallBase &, + bool getTgtMemIntrinsic(IntrinsicInfo &, const CallInst &, MachineFunction &MF, unsigned IntrinsicID) const override; - void CollectTargetIntrinsicOperands(const CallBase &I, + void CollectTargetIntrinsicOperands(const CallInst &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const override; diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index dc6a4f227061b..5b3664c4e961f 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -20879,7 +20879,7 @@ bool ARMTargetLowering::isFPImmLegal(const APFloat &Imm, EVT VT, /// MemIntrinsicNodes. The associated MachineMemOperands record the alignment /// specified in the intrinsic calls. bool ARMTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallBase &I, + const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h index a4ea597b95699..87710ee29a249 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.h +++ b/llvm/lib/Target/ARM/ARMISelLowering.h @@ -616,7 +616,8 @@ class VectorType; bool isFPImmLegal(const APFloat &Imm, EVT VT, bool ForCodeSize = false) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, + const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp index 078d38ce12c0a..078eccaa706a2 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.cpp @@ -2040,7 +2040,7 @@ static Value *getUnderLyingObjectForBrevLdIntr(Value *V) { /// true and store the intrinsic information into the IntrinsicInfo that was /// passed to the function. bool HexagonTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallBase &I, + const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { diff --git a/llvm/lib/Target/Hexagon/HexagonISelLowering.h b/llvm/lib/Target/Hexagon/HexagonISelLowering.h index 9f315681ab95f..1321bee44a295 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelLowering.h +++ b/llvm/lib/Target/Hexagon/HexagonISelLowering.h @@ -142,7 +142,7 @@ class HexagonTargetLowering : public TargetLowering { const SmallVectorImpl &OutVals, const SmallVectorImpl &Ins, SelectionDAG& DAG) const; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp index 8a0d3045b9770..99dae6ec3eb08 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.cpp @@ -7424,7 +7424,7 @@ bool LoongArchTargetLowering::hasAndNot(SDValue Y) const { } bool LoongArchTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallBase &I, + const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { diff --git a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h index b31648cdc7c7c..79aa89726191b 100644 --- a/llvm/lib/Target/LoongArch/LoongArchISelLowering.h +++ b/llvm/lib/Target/LoongArch/LoongArchISelLowering.h @@ -236,7 +236,7 @@ class LoongArchTargetLowering : public TargetLowering { Value *NewVal, Value *Mask, AtomicOrdering Ord) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp index bd82195836c16..492f4ab76fdbb 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -3664,10 +3664,9 @@ void NVPTXTargetLowering::LowerAsmOperandForConstraint( // because we need the information that is only available in the "Value" type // of destination // pointer. In particular, the address space information. -bool NVPTXTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallBase &I, - MachineFunction &MF, - unsigned Intrinsic) const { +bool NVPTXTargetLowering::getTgtMemIntrinsic( + IntrinsicInfo &Info, const CallInst &I, + MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { default: return false; diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h index 85be05fbc6462..0a54a8fd71f32 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h @@ -124,7 +124,7 @@ class NVPTXTargetLowering : public TargetLowering { const char *getTargetNodeName(unsigned Opcode) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index a338383ebda91..0f8e5e57c58b7 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -17841,8 +17841,9 @@ void PPCTargetLowering::LowerAsmOperandForConstraint(SDValue Op, TargetLowering::LowerAsmOperandForConstraint(Op, Constraint, Ops, DAG); } -void PPCTargetLowering::CollectTargetIntrinsicOperands( - const CallBase &I, SmallVectorImpl &Ops, SelectionDAG &DAG) const { +void PPCTargetLowering::CollectTargetIntrinsicOperands(const CallInst &I, + SmallVectorImpl &Ops, + SelectionDAG &DAG) const { if (I.getNumOperands() <= 1) return; if (!isa(Ops[1].getNode())) @@ -18030,7 +18031,7 @@ PPCTargetLowering::isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const { } bool PPCTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallBase &I, + const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h index 39731001e766e..4c88bd372b106 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.h +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h @@ -1020,9 +1020,9 @@ namespace llvm { return TargetLowering::getInlineAsmMemConstraint(ConstraintCode); } - void CollectTargetIntrinsicOperands(const CallBase &I, - SmallVectorImpl &Ops, - SelectionDAG &DAG) const override; + void CollectTargetIntrinsicOperands(const CallInst &I, + SmallVectorImpl &Ops, + SelectionDAG &DAG) const override; /// isLegalAddressingMode - Return true if the addressing mode represented /// by AM is legal for this target, for a load/store of the specified type. @@ -1081,7 +1081,8 @@ namespace llvm { bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, + const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp index 1743ea327de14..7cfada6c0601c 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp @@ -1715,7 +1715,7 @@ bool RISCVTargetLowering::shouldExpandCttzElements(EVT VT) const { } bool RISCVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallBase &I, + const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const { auto &DL = I.getDataLayout(); diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h index 3b52f5e50d769..417d684a62382 100644 --- a/llvm/lib/Target/RISCV/RISCVISelLowering.h +++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h @@ -35,7 +35,7 @@ class RISCVTargetLowering : public TargetLowering { const RISCVSubtarget &getSubtarget() const { return Subtarget; } - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override; bool isLegalAddressingMode(const DataLayout &DL, const AddrMode &AM, Type *Ty, diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp index ee90f9799645a..e742a9811984b 100644 --- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp +++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.cpp @@ -90,7 +90,7 @@ MVT SPIRVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context, } bool SPIRVTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const Callase &I, + const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const { unsigned AlignIdx = 3; diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h index d6e10d611704a..9025e6eb0842e 100644 --- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h +++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h @@ -49,7 +49,7 @@ class SPIRVTargetLowering : public TargetLowering { EVT VT) const override; MVT getRegisterTypeForCallingConv(LLVMContext &Context, CallingConv::ID CC, EVT VT) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp index c77a9375fbce8..aac3473311192 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp @@ -1034,7 +1034,7 @@ EVT WebAssemblyTargetLowering::getSetCCResultType(const DataLayout &DL, } bool WebAssemblyTargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallBase &I, + const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const { switch (Intrinsic) { diff --git a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h index f8f23c932177b..72401a7a259c0 100644 --- a/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h +++ b/llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h @@ -72,7 +72,7 @@ class WebAssemblyTargetLowering final : public TargetLowering { bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override; EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context, EVT VT) const override; - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override; diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index 695cd95f0ad5f..b4670e270141f 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -3089,7 +3089,7 @@ static bool useVPTERNLOG(const X86Subtarget &Subtarget, MVT VT) { } bool X86TargetLowering::getTgtMemIntrinsic(IntrinsicInfo &Info, - const CallBase &I, + const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const { Info.flags = MachineMemOperand::MONone; diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h index ee24f0f91b0a0..5cb6b3e493a32 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.h +++ b/llvm/lib/Target/X86/X86ISelLowering.h @@ -1475,7 +1475,7 @@ namespace llvm { /// to a MemIntrinsicNode (touches memory). If this is the case, it returns /// true and stores the intrinsic information into the IntrinsicInfo that was /// passed to the function. - bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallBase &I, + bool getTgtMemIntrinsic(IntrinsicInfo &Info, const CallInst &I, MachineFunction &MF, unsigned Intrinsic) const override; From 2c9515c330a67a5f257673b173a7c37294d6353c Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Sun, 6 Apr 2025 14:58:41 -0500 Subject: [PATCH 06/18] abstract parts of call code; disallow operand bundles for callbr in verifier --- .../llvm/CodeGen/GlobalISel/IRTranslator.h | 5 + llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 158 +++++++------- .../SelectionDAG/SelectionDAGBuilder.cpp | 193 +++++++++++------- .../SelectionDAG/SelectionDAGBuilder.h | 14 +- llvm/lib/IR/Verifier.cpp | 4 +- llvm/test/Verifier/callbr.ll | 8 + 6 files changed, 237 insertions(+), 145 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h index 6fd05c8fddd5f..ba2aeac8dffe1 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -297,6 +297,10 @@ class IRTranslator : public MachineFunctionPass { /// \pre \p U is a call instruction. bool translateCall(const User &U, MachineIRBuilder &MIRBuilder); + bool translateTargetIntrinsic( + const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder, + TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo = nullptr); + /// When an invoke or a cleanupret unwinds to the next EH pad, there are /// many places it could ultimately go. In the IR, we have a single unwind /// destination, but in the machine CFG, we enumerate all the possible blocks. @@ -313,6 +317,7 @@ class IRTranslator : public MachineFunctionPass { bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder); bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder); + bool translateCallBrIntrinsic(const CallBrInst &I, MachineIRBuilder &MIRBuilder); bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder); diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 3b33d3784c1f9..144203edc0ad7 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -2750,59 +2750,27 @@ bool IRTranslator::translateCallBase(const CallBase &CB, return Success; } -bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { - if (containsBF16Type(U)) - return false; - - const CallInst &CI = cast(U); - const Function *F = CI.getCalledFunction(); - - // FIXME: support Windows dllimport function calls and calls through - // weak symbols. - if (F && (F->hasDLLImportStorageClass() || - (MF->getTarget().getTargetTriple().isOSWindows() && - F->hasExternalWeakLinkage()))) - return false; - - // FIXME: support control flow guard targets. - if (CI.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget)) - return false; - - // FIXME: support statepoints and related. - if (isa(U)) - return false; - - if (CI.isInlineAsm()) - return translateInlineAsm(CI, MIRBuilder); - - diagnoseDontCall(CI); - - Intrinsic::ID ID = Intrinsic::not_intrinsic; - if (F && F->isIntrinsic()) - ID = F->getIntrinsicID(); - - if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic) - return translateCallBase(CI, MIRBuilder); - - assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic"); - - if (translateKnownIntrinsic(CI, ID, MIRBuilder)) - return true; - +/// Translate a call or callbr to a target intrinsic. +/// Depending on whether TLI->getTgtMemIntrinsic() is true, TgtMemIntrinsicInfo +/// is a pointer to the correspondingly populated IntrinsicInfo object. +/// Otherwise, this pointer is null. +bool IRTranslator::translateTargetIntrinsic( + const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder, + TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) { ArrayRef ResultRegs; - if (!CI.getType()->isVoidTy()) - ResultRegs = getOrCreateVRegs(CI); + if (!CB.getType()->isVoidTy()) + ResultRegs = getOrCreateVRegs(CB); // Ignore the callsite attributes. Backend code is most likely not expecting // an intrinsic to sometimes have side effects and sometimes not. MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, ResultRegs); - if (isa(CI)) - MIB->copyIRFlags(CI); + if (isa(CB)) + MIB->copyIRFlags(CB); - for (const auto &Arg : enumerate(CI.args())) { + for (const auto &Arg : enumerate(CB.args())) { // If this is required to be an immediate, don't materialize it in a // register. - if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) { + if (CB.paramHasAttr(Arg.index(), Attribute::ImmArg)) { if (ConstantInt *CI = dyn_cast(Arg.value())) { // imm arguments are more convenient than cimm (and realistically // probably sufficient), so use them. @@ -2831,29 +2799,31 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { } // Add a MachineMemOperand if it is a target mem intrinsic. - TargetLowering::IntrinsicInfo Info; - // TODO: Add a GlobalISel version of getTgtMemIntrinsic. - if (TLI->getTgtMemIntrinsic(Info, CI, *MF, ID)) { - Align Alignment = Info.align.value_or( - DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext()))); - LLT MemTy = Info.memVT.isSimple() - ? getLLTForMVT(Info.memVT.getSimpleVT()) - : LLT::scalar(Info.memVT.getStoreSizeInBits()); + if (TgtMemIntrinsicInfo) { + const Function *F = CB.getCalledFunction(); + + Align Alignment = TgtMemIntrinsicInfo->align.value_or(DL->getABITypeAlign( + TgtMemIntrinsicInfo->memVT.getTypeForEVT(F->getContext()))); + LLT MemTy = + TgtMemIntrinsicInfo->memVT.isSimple() + ? getLLTForMVT(TgtMemIntrinsicInfo->memVT.getSimpleVT()) + : LLT::scalar(TgtMemIntrinsicInfo->memVT.getStoreSizeInBits()); // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic // didn't yield anything useful. MachinePointerInfo MPI; - if (Info.ptrVal) - MPI = MachinePointerInfo(Info.ptrVal, Info.offset); - else if (Info.fallbackAddressSpace) - MPI = MachinePointerInfo(*Info.fallbackAddressSpace); + if (TgtMemIntrinsicInfo->ptrVal) + MPI = MachinePointerInfo(TgtMemIntrinsicInfo->ptrVal, + TgtMemIntrinsicInfo->offset); + else if (TgtMemIntrinsicInfo->fallbackAddressSpace) + MPI = MachinePointerInfo(*TgtMemIntrinsicInfo->fallbackAddressSpace); MIB.addMemOperand(MF->getMachineMemOperand( - MPI, Info.flags, MemTy, Alignment, CI.getAAMetadata(), - /*Ranges=*/nullptr, Info.ssid, Info.order, Info.failureOrder)); + MPI, TgtMemIntrinsicInfo->flags, MemTy, Alignment, CI.getAAMetadata(), + /*Ranges=*/nullptr, TgtMemIntrinsicInfo->ssid, TgtMemIntrinsicInfo->order, TgtMemIntrinsicInfo->failureOrder)); } - if (CI.isConvergent()) { - if (auto Bundle = CI.getOperandBundle(LLVMContext::OB_convergencectrl)) { + if (CB.isConvergent()) { + if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) { auto *Token = Bundle->Inputs[0].get(); Register TokenReg = getOrCreateVReg(*Token); MIB.addUse(TokenReg, RegState::Implicit); @@ -2863,6 +2833,53 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { return true; } +bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) { + if (containsBF16Type(U)) + return false; + + const CallInst &CI = cast(U); + const Function *F = CI.getCalledFunction(); + + // FIXME: support Windows dllimport function calls and calls through + // weak symbols. + if (F && (F->hasDLLImportStorageClass() || + (MF->getTarget().getTargetTriple().isOSWindows() && + F->hasExternalWeakLinkage()))) + return false; + + // FIXME: support control flow guard targets. + if (CI.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget)) + return false; + + // FIXME: support statepoints and related. + if (isa(U)) + return false; + + if (CI.isInlineAsm()) + return translateInlineAsm(CI, MIRBuilder); + + diagnoseDontCall(CI); + + Intrinsic::ID ID = Intrinsic::not_intrinsic; + if (F && F->isIntrinsic()) + ID = F->getIntrinsicID(); + + if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic) + return translateCallBase(CI, MIRBuilder); + + assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic"); + + if (translateKnownIntrinsic(CI, ID, MIRBuilder)) + return true; + + TargetLowering::IntrinsicInfo Info; + // TODO: Add a GlobalISel version of getTgtMemIntrinsic. + bool IsTgtMemIntrinsic = TLI->getTgtMemIntrinsic(Info, CI, *MF, ID); + + return translateTargetIntrinsic(CI, ID, MIRBuilder, + IsTgtMemIntrinsic ? &Info : nullptr); +} + bool IRTranslator::findUnwindDestinations( const BasicBlock *EHPadBB, BranchProbability Prob, @@ -3006,15 +3023,16 @@ bool IRTranslator::translateInvoke(const User &U, return true; } +/// The intrinsics currently supported by callbr are implicit control flow +/// intrinsics such as amdgcn.kill. bool IRTranslator::translateCallBr(const User &U, MachineIRBuilder &MIRBuilder) { + if (containsBF16Type(U)) + return false; // see translateCall + const CallBrInst &I = cast(U); MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB(); - // TODO: operand bundles (see SelDAG implementation of callbr)? - assert(!I.hasOperandBundles() && - "Cannot lower callbrs with operand bundles yet"); - if (I.isInlineAsm()) { // FIXME: inline asm not yet supported if (!translateInlineAsm(I, MIRBuilder)) @@ -3024,15 +3042,7 @@ bool IRTranslator::translateCallBr(const User &U, default: report_fatal_error("Unsupported intrinsic for callbr"); case Intrinsic::amdgcn_kill: - if (I.getNumIndirectDests() != 1) - report_fatal_error( - "amdgcn.kill supportes exactly one indirect destination"); - CallInst *CI = - CallInst::Create(I.getFunctionType(), I.getCalledFunction(), - SmallVector(I.args())); - bool Success = translateCall(*CI, MIRBuilder); - CI->deleteValue(); - if (!Success) + if (!translateTargetIntrinsic(I, Intrinsic::amdgcn_kill, MIRBuilder)) return false; break; } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 66288b266f400..83c61da11b44b 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3437,16 +3437,28 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { DAG.getBasicBlock(Return))); } +/// The intrinsics currently supported by callbr are implicit control flow +/// intrinsics such as amdgcn.kill. +/// - they should be called (no "dontcall-" attributes) +/// - they do not touch memory on the target (= !TLI.getTgtMemIntrinsic()) +/// - they do not need custom argument handling (no TLI.CollectTargetIntrinsicOperands()) +void SelectionDAGBuilder::visitCallBrIntrinsic(const CallBrInst &I) { + auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I); + + SmallVector Ops = + getTargetIntrinsicOperands(I, HasChain, OnlyLoad); + SDVTList VTs = getTargetIntrinsicVTList(I, HasChain); + + // Create the node. + SDValue Result = getTargetNonMemIntrinsicNode(I, HasChain, Ops, VTs); + Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result); + + setValue(&I, Result); +} + void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { MachineBasicBlock *CallBrMBB = FuncInfo.MBB; - // Deopt bundles are lowered in LowerCallSiteWithDeoptBundle, and we don't - // have to do anything here to lower funclet bundles. - if (I.hasOperandBundlesOtherThan( - {LLVMContext::OB_deopt, LLVMContext::OB_funclet})) - reportFatalUsageError( - "cannot lower callbrs with arbitrary operand bundles!"); - if (I.isInlineAsm()) { visitInlineAsm(I); } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) { @@ -3454,14 +3466,7 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { default: report_fatal_error("Unsupported intrinsic for callbr"); case Intrinsic::amdgcn_kill: - if (I.getNumIndirectDests() != 1) - report_fatal_error( - "amdgcn.kill supportes exactly one indirect destination"); - CallInst *CI = - CallInst::Create(I.getFunctionType(), I.getCalledFunction(), - SmallVector(I.args())); - visitCall(*CI); - CI->deleteValue(); + visitCallBrIntrinsic(I); break; } } else { @@ -5291,18 +5296,25 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { DAG.setRoot(OutChain); } -/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC -/// node. -void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, - unsigned Intrinsic) { - // Ignore the callsite's attributes. A specific call site may be marked with - // readnone, but the lowering code will expect the chain based on the - // definition. +/// Check if this intrinsic call depends on the chain (1st return value) +/// and if it only *loads* memory. +/// Ignore the callsite's attributes. A specific call site may be marked with +/// readnone, but the lowering code will expect the chain based on the +/// definition. +std::pair SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase& I) { const Function *F = I.getCalledFunction(); bool HasChain = !F->doesNotAccessMemory(); bool OnlyLoad = HasChain && F->onlyReadsMemory() && F->willReturn() && F->doesNotThrow(); + return {HasChain, OnlyLoad}; +} + +SmallVector SelectionDAGBuilder::getTargetIntrinsicOperands( + const CallBase &I, bool HasChain, bool OnlyLoad, + TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) { + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + // Build the operand list. SmallVector Ops; if (HasChain) { // If this intrinsic has side-effects, chainify it. @@ -5314,17 +5326,10 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, } } - // Info is set by getTgtMemIntrinsic - TargetLowering::IntrinsicInfo Info; - const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - bool IsTgtIntrinsic = TLI.getTgtMemIntrinsic(Info, I, - DAG.getMachineFunction(), - Intrinsic); - // Add the intrinsic ID as an integer operand if it's not a target intrinsic. - if (!IsTgtIntrinsic || Info.opc == ISD::INTRINSIC_VOID || - Info.opc == ISD::INTRINSIC_W_CHAIN) - Ops.push_back(DAG.getTargetConstant(Intrinsic, getCurSDLoc(), + if (!TgtMemIntrinsicInfo || TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_VOID || + TgtMemIntrinsicInfo->opc == ISD::INTRINSIC_W_CHAIN) + Ops.push_back(DAG.getTargetConstant(I.getIntrinsicID(), getCurSDLoc(), TLI.getPointerTy(DAG.getDataLayout()))); // Add all operands of the call to the operand list. @@ -5347,13 +5352,96 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, } } + if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) { + auto *Token = Bundle->Inputs[0].get(); + SDValue ConvControlToken = getValue(Token); + assert(Ops.back().getValueType() != MVT::Glue && + "Did not expected another glue node here."); + ConvControlToken = + DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken); + Ops.push_back(ConvControlToken); + } + + return Ops; +} + +SDVTList SelectionDAGBuilder::getTargetIntrinsicVTList(const CallBase &I, + bool HasChain) { + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + SmallVector ValueVTs; ComputeValueVTs(TLI, DAG.getDataLayout(), I.getType(), ValueVTs); if (HasChain) ValueVTs.push_back(MVT::Other); - SDVTList VTs = DAG.getVTList(ValueVTs); + return DAG.getVTList(ValueVTs); +} + +/// Get an INTRINSIC node for a target intrinsic which does not touch touch memory. +SDValue +SelectionDAGBuilder::getTargetNonMemIntrinsicNode(const CallBase &I, bool HasChain, + SmallVector &Ops, + SDVTList &VTs) { + SDValue Result; + + if (!HasChain) { + Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops); + } else if (!I.getType()->isVoidTy()) { + Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops); + } else { + Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops); + } + + return Result; +} + +/// Set root, convert return type if necessaey and check alignment. +SDValue SelectionDAGBuilder::handleTargetIntrinsicRet(const CallBase &I, + bool HasChain, + bool OnlyLoad, + SDValue Result) { + if (HasChain) { + SDValue Chain = Result.getValue(Result.getNode()->getNumValues() - 1); + if (OnlyLoad) + PendingLoads.push_back(Chain); + else + DAG.setRoot(Chain); + } + + if (I.getType()->isVoidTy()) + return Result; + + if (!isa(I.getType())) + Result = lowerRangeToAssertZExt(DAG, I, Result); + + MaybeAlign Alignment = I.getRetAlign(); + + // Insert `assertalign` node if there's an alignment. + if (InsertAssertAlign && Alignment) { + Result = + DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne()); + } + + return Result; +} + +/// visitTargetIntrinsic - Lower a call of a target intrinsic to an INTRINSIC +/// node. +void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, + unsigned Intrinsic) { + auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I); + + // Info is set by getTgtMemIntrinsic + TargetLowering::IntrinsicInfo Info; + const TargetLowering &TLI = DAG.getTargetLoweringInfo(); + bool IsTgtMemIntrinsic = TLI.getTgtMemIntrinsic(Info, I, + DAG.getMachineFunction(), + Intrinsic); + + SmallVector Ops = getTargetIntrinsicOperands( + I, HasChain, OnlyLoad, IsTgtMemIntrinsic ? &Info : nullptr); + SDVTList VTs = getTargetIntrinsicVTList(I, HasChain); // Propagate fast-math-flags from IR to node(s). SDNodeFlags Flags; @@ -5364,19 +5452,9 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, // Create the node. SDValue Result; - if (auto Bundle = I.getOperandBundle(LLVMContext::OB_convergencectrl)) { - auto *Token = Bundle->Inputs[0].get(); - SDValue ConvControlToken = getValue(Token); - assert(Ops.back().getValueType() != MVT::Glue && - "Did not expected another glue node here."); - ConvControlToken = - DAG.getNode(ISD::CONVERGENCECTRL_GLUE, {}, MVT::Glue, ConvControlToken); - Ops.push_back(ConvControlToken); - } - // In some cases, custom collection of operands from CallInst I may be needed. TLI.CollectTargetIntrinsicOperands(I, Ops, DAG); - if (IsTgtIntrinsic) { + if (IsTgtMemIntrinsic) { // This is target intrinsic that touches memory // // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic @@ -5396,34 +5474,11 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, Info.ssid, Info.order, Info.failureOrder); Result = DAG.getMemIntrinsicNode(Info.opc, getCurSDLoc(), VTs, Ops, MemVT, MMO); - } else if (!HasChain) { - Result = DAG.getNode(ISD::INTRINSIC_WO_CHAIN, getCurSDLoc(), VTs, Ops); - } else if (!I.getType()->isVoidTy()) { - Result = DAG.getNode(ISD::INTRINSIC_W_CHAIN, getCurSDLoc(), VTs, Ops); } else { - Result = DAG.getNode(ISD::INTRINSIC_VOID, getCurSDLoc(), VTs, Ops); + Result = getTargetNonMemIntrinsicNode(I, HasChain, Ops, VTs); } - if (HasChain) { - SDValue Chain = Result.getValue(Result.getNode()->getNumValues()-1); - if (OnlyLoad) - PendingLoads.push_back(Chain); - else - DAG.setRoot(Chain); - } - - if (!I.getType()->isVoidTy()) { - if (!isa(I.getType())) - Result = lowerRangeToAssertZExt(DAG, I, Result); - - MaybeAlign Alignment = I.getRetAlign(); - - // Insert `assertalign` node if there's an alignment. - if (InsertAssertAlign && Alignment) { - Result = - DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne()); - } - } + Result = handleTargetIntrinsicRet(I, HasChain, OnlyLoad, Result); setValue(&I, Result); } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 35c15bc269d4b..71f67a7f1eff6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -536,10 +536,12 @@ class SelectionDAGBuilder { private: // These all get lowered before this pass. void visitInvoke(const InvokeInst &I); - void visitCallBr(const CallBrInst &I); void visitCallBrLandingPad(const CallInst &I); void visitResume(const ResumeInst &I); + void visitCallBr(const CallBrInst &I); + void visitCallBrIntrinsic(const CallBrInst &I); + void visitUnary(const User &I, unsigned Opcode); void visitFNeg(const User &I) { visitUnary(I, ISD::FNEG); } @@ -709,6 +711,16 @@ class SelectionDAGBuilder { MCSymbol *&BeginLabel); SDValue lowerEndEH(SDValue Chain, const InvokeInst *II, const BasicBlock *EHPadBB, MCSymbol *BeginLabel); + + std::pair getTargetIntrinsicCallProperties(const CallBase& I); + SmallVector getTargetIntrinsicOperands( + const CallBase &I, bool HasChain, bool OnlyLoad, + TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo = nullptr); + SDVTList getTargetIntrinsicVTList(const CallBase &I, bool HasChain); + SDValue getTargetNonMemIntrinsicNode(const CallBase &I, bool HasChain, + SmallVector &Ops, SDVTList &VTs); + SDValue handleTargetIntrinsicRet(const CallBase &I, bool HasChain, + bool OnlyLoad, SDValue Result); }; /// This struct represents the registers (physical or virtual) diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index c0bf78da4b427..874288b2aad67 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -3297,6 +3297,9 @@ void Verifier::visitIndirectBrInst(IndirectBrInst &BI) { void Verifier::visitCallBrInst(CallBrInst &CBI) { if (!CBI.isInlineAsm()) { + Check(CBI.getCalledFunction(), "Callbr: indirect function / invalid signature"); + Check(!CBI.hasOperandBundles(), "Callbr currently doesn't support operand bundles"); + switch (CBI.getIntrinsicID()) { case Intrinsic::amdgcn_kill: { Check(CBI.getNumIndirectDests() == 1, @@ -3306,7 +3309,6 @@ void Verifier::visitCallBrInst(CallBrInst &CBI) { Check(Unreachable || (Call && Call->getIntrinsicID() == Intrinsic::amdgcn_unreachable), "Callbr amdgcn_kill indirect dest needs to be unreachable"); - visitIntrinsicCall(Intrinsic::amdgcn_kill, CBI); break; } default: diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll index 8e125e723e6fc..29bd3397b8980 100644 --- a/llvm/test/Verifier/callbr.ll +++ b/llvm/test/Verifier/callbr.ll @@ -159,3 +159,11 @@ define void @test_callbr_intrinsic_unsupported() { cont: ret void } + +; CHECK-NEXT: Callbr: indirect function / invalid signature +define void @test_callbr_intrinsic_wrong_signature(ptr %ptr) { + %func = load ptr, ptr %ptr, align 8 + callbr void %func() to label %cont [] +cont: + ret void +} From d8f540383200ce3b629e3f130c340052bd386b7f Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Sun, 6 Apr 2025 14:59:29 -0500 Subject: [PATCH 07/18] fix formatting --- .../llvm/CodeGen/GlobalISel/IRTranslator.h | 3 ++- .../SelectionDAG/SelectionDAGBuilder.cpp | 24 +++++++++---------- .../SelectionDAG/SelectionDAGBuilder.h | 5 ++-- llvm/lib/IR/Verifier.cpp | 6 +++-- 4 files changed, 21 insertions(+), 17 deletions(-) diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h index ba2aeac8dffe1..941750510f1e1 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h @@ -317,7 +317,8 @@ class IRTranslator : public MachineFunctionPass { bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder); bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder); - bool translateCallBrIntrinsic(const CallBrInst &I, MachineIRBuilder &MIRBuilder); + bool translateCallBrIntrinsic(const CallBrInst &I, + MachineIRBuilder &MIRBuilder); bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 83c61da11b44b..00ad03b0c00ca 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3441,7 +3441,8 @@ void SelectionDAGBuilder::visitInvoke(const InvokeInst &I) { /// intrinsics such as amdgcn.kill. /// - they should be called (no "dontcall-" attributes) /// - they do not touch memory on the target (= !TLI.getTgtMemIntrinsic()) -/// - they do not need custom argument handling (no TLI.CollectTargetIntrinsicOperands()) +/// - they do not need custom argument handling (no +/// TLI.CollectTargetIntrinsicOperands()) void SelectionDAGBuilder::visitCallBrIntrinsic(const CallBrInst &I) { auto [HasChain, OnlyLoad] = getTargetIntrinsicCallProperties(I); @@ -5301,7 +5302,8 @@ void SelectionDAGBuilder::visitAtomicStore(const StoreInst &I) { /// Ignore the callsite's attributes. A specific call site may be marked with /// readnone, but the lowering code will expect the chain based on the /// definition. -std::pair SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase& I) { +std::pair +SelectionDAGBuilder::getTargetIntrinsicCallProperties(const CallBase &I) { const Function *F = I.getCalledFunction(); bool HasChain = !F->doesNotAccessMemory(); bool OnlyLoad = @@ -5378,11 +5380,11 @@ SDVTList SelectionDAGBuilder::getTargetIntrinsicVTList(const CallBase &I, return DAG.getVTList(ValueVTs); } -/// Get an INTRINSIC node for a target intrinsic which does not touch touch memory. -SDValue -SelectionDAGBuilder::getTargetNonMemIntrinsicNode(const CallBase &I, bool HasChain, - SmallVector &Ops, - SDVTList &VTs) { +/// Get an INTRINSIC node for a target intrinsic which does not touch touch +/// memory. +SDValue SelectionDAGBuilder::getTargetNonMemIntrinsicNode( + const CallBase &I, bool HasChain, SmallVector &Ops, + SDVTList &VTs) { SDValue Result; if (!HasChain) { @@ -5419,8 +5421,7 @@ SDValue SelectionDAGBuilder::handleTargetIntrinsicRet(const CallBase &I, // Insert `assertalign` node if there's an alignment. if (InsertAssertAlign && Alignment) { - Result = - DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne()); + Result = DAG.getAssertAlign(getCurSDLoc(), Result, Alignment.valueOrOne()); } return Result; @@ -5435,9 +5436,8 @@ void SelectionDAGBuilder::visitTargetIntrinsic(const CallInst &I, // Info is set by getTgtMemIntrinsic TargetLowering::IntrinsicInfo Info; const TargetLowering &TLI = DAG.getTargetLoweringInfo(); - bool IsTgtMemIntrinsic = TLI.getTgtMemIntrinsic(Info, I, - DAG.getMachineFunction(), - Intrinsic); + bool IsTgtMemIntrinsic = + TLI.getTgtMemIntrinsic(Info, I, DAG.getMachineFunction(), Intrinsic); SmallVector Ops = getTargetIntrinsicOperands( I, HasChain, OnlyLoad, IsTgtMemIntrinsic ? &Info : nullptr); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 71f67a7f1eff6..c1cf2c4035103 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -712,13 +712,14 @@ class SelectionDAGBuilder { SDValue lowerEndEH(SDValue Chain, const InvokeInst *II, const BasicBlock *EHPadBB, MCSymbol *BeginLabel); - std::pair getTargetIntrinsicCallProperties(const CallBase& I); + std::pair getTargetIntrinsicCallProperties(const CallBase &I); SmallVector getTargetIntrinsicOperands( const CallBase &I, bool HasChain, bool OnlyLoad, TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo = nullptr); SDVTList getTargetIntrinsicVTList(const CallBase &I, bool HasChain); SDValue getTargetNonMemIntrinsicNode(const CallBase &I, bool HasChain, - SmallVector &Ops, SDVTList &VTs); + SmallVector &Ops, + SDVTList &VTs); SDValue handleTargetIntrinsicRet(const CallBase &I, bool HasChain, bool OnlyLoad, SDValue Result); }; diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 874288b2aad67..00a03bbb27e14 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -3297,8 +3297,10 @@ void Verifier::visitIndirectBrInst(IndirectBrInst &BI) { void Verifier::visitCallBrInst(CallBrInst &CBI) { if (!CBI.isInlineAsm()) { - Check(CBI.getCalledFunction(), "Callbr: indirect function / invalid signature"); - Check(!CBI.hasOperandBundles(), "Callbr currently doesn't support operand bundles"); + Check(CBI.getCalledFunction(), + "Callbr: indirect function / invalid signature"); + Check(!CBI.hasOperandBundles(), + "Callbr currently doesn't support operand bundles"); switch (CBI.getIntrinsicID()) { case Intrinsic::amdgcn_kill: { From 9219953840a9a2bf82ff2c84f470e2c3d80c96af Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Wed, 9 Apr 2025 09:53:45 -0500 Subject: [PATCH 08/18] add callbr test with different succ order --- llvm/test/CodeGen/AMDGPU/callbr.ll | 50 ++++++++++++++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/llvm/test/CodeGen/AMDGPU/callbr.ll b/llvm/test/CodeGen/AMDGPU/callbr.ll index 33c75f6cf5aab..52856b872ee2a 100644 --- a/llvm/test/CodeGen/AMDGPU/callbr.ll +++ b/llvm/test/CodeGen/AMDGPU/callbr.ll @@ -50,3 +50,53 @@ cont: store i32 %a, ptr %dst, align 4 ret void } + +; SELDAG-LABEL: test_kill_block_order: +; SELDAG-NEXT: ; %bb.0: +; SELDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; SELDAG-NEXT: flat_load_dword v0, v[0:1] +; SELDAG-NEXT: v_and_b32_e32 v1, 1, v4 +; SELDAG-NEXT: v_cmp_eq_u32_e32 vcc, 1, v1 +; SELDAG-NEXT: s_mov_b64 s[4:5], exec +; SELDAG-NEXT: s_andn2_b64 s[6:7], exec, vcc +; SELDAG-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7] +; SELDAG-NEXT: s_cbranch_scc0 .LBB1_2 +; SELDAG-NEXT: ; %bb.1: +; SELDAG-NEXT: s_and_b64 exec, exec, s[4:5] +; SELDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; SELDAG-NEXT: flat_store_dword v[2:3], v0 +; SELDAG-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; SELDAG-NEXT: s_setpc_b64 s[30:31] +; SELDAG-NEXT: .LBB1_2: +; SELDAG-NEXT: s_mov_b64 exec, 0 +; SELDAG-NEXT: s_endpgm + +; GISEL-LABEL: test_kill_block_order: +; GISEL-NEXT: ; %bb.0: +; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0) +; GISEL-NEXT: flat_load_dword v0, v[0:1] +; GISEL-NEXT: v_and_b32_e32 v1, 1, v4 +; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v1 +; GISEL-NEXT: s_mov_b64 s[4:5], exec +; GISEL-NEXT: s_andn2_b64 s[6:7], exec, vcc +; GISEL-NEXT: s_andn2_b64 s[4:5], s[4:5], s[6:7] +; GISEL-NEXT: s_cbranch_scc0 .LBB1_2 +; GISEL-NEXT: ; %bb.1: +; GISEL-NEXT: s_and_b64 exec, exec, s[4:5] +; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GISEL-NEXT: flat_store_dword v[2:3], v0 +; GISEL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GISEL-NEXT: s_setpc_b64 s[30:31] +; GISEL-NEXT: .LBB1_2: +; GISEL-NEXT: s_mov_b64 exec, 0 +; GISEL-NEXT: s_endpgm + +define void @test_kill_block_order(ptr %src, ptr %dst, i1 %c) { + %a = load i32, ptr %src, align 4 + callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill] +cont: + store i32 %a, ptr %dst, align 4 + ret void +kill: + unreachable +} From fec2908bc9bfdf70216154a559c885535b9f22b8 Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Mon, 14 Apr 2025 03:32:41 -0500 Subject: [PATCH 09/18] implement feedback --- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 12 +++--------- .../lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 10 ++-------- 2 files changed, 5 insertions(+), 17 deletions(-) diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 144203edc0ad7..248d9287efdf4 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -3038,16 +3038,10 @@ bool IRTranslator::translateCallBr(const User &U, if (!translateInlineAsm(I, MIRBuilder)) return false; } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) { - switch (I.getIntrinsicID()) { - default: - report_fatal_error("Unsupported intrinsic for callbr"); - case Intrinsic::amdgcn_kill: - if (!translateTargetIntrinsic(I, Intrinsic::amdgcn_kill, MIRBuilder)) - return false; - break; - } + if (!translateTargetIntrinsic(I, I.getIntrinsicID(), MIRBuilder)) + return false; } else { - report_fatal_error("Only know how to handle inlineasm/intrinsic callbr"); + return false; } // Retrieve successors. diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 00ad03b0c00ca..755cd30ab3bc6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3463,15 +3463,9 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { if (I.isInlineAsm()) { visitInlineAsm(I); } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) { - switch (I.getIntrinsicID()) { - default: - report_fatal_error("Unsupported intrinsic for callbr"); - case Intrinsic::amdgcn_kill: - visitCallBrIntrinsic(I); - break; - } + visitCallBrIntrinsic(I); } else { - report_fatal_error("Only know how to handle inlineasm/intrinsic callbr"); + report_fatal_error("only know how to handle inlineasm/intrinsic callbr"); } CopyToExportRegsIfNeeded(&I); From 2370c4b7498d779dead60ac23fc22a3dd2b7caea Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Mon, 12 May 2025 03:57:49 -0500 Subject: [PATCH 10/18] update LangRef --- llvm/docs/LangRef.rst | 28 ++++++++++++++++++++++------ 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst index cc72a37f68599..4fbcb57c30146 100644 --- a/llvm/docs/LangRef.rst +++ b/llvm/docs/LangRef.rst @@ -9590,8 +9590,12 @@ The '``callbr``' instruction causes control to transfer to a specified function, with the possibility of control flow transfer to either the '``fallthrough``' label or one of the '``indirect``' labels. -This instruction should only be used to implement the "goto" feature of gcc -style inline assembly. Any other usage is an error in the IR verifier. +This instruction can currently only be used + +#. to implement the "goto" feature of gcc style inline assembly or +#. to call selected intrinsics. + +Any other usage is an error in the IR verifier. Note that in order to support outputs along indirect edges, LLVM may need to split critical edges, which may require synthesizing a replacement block for @@ -9640,7 +9644,8 @@ This instruction requires several arguments: indicates the function accepts a variable number of arguments, the extra arguments can be specified. #. '``fallthrough label``': the label reached when the inline assembly's - execution exits the bottom. + execution exits the bottom. In case of an intrinsic call, the semantic + depends on the semantic of the intrinsic. #. '``indirect labels``': the labels reached when a callee transfers control to a location other than the '``fallthrough label``'. Label constraints refer to these destinations. @@ -9658,9 +9663,12 @@ flow goes after the call. The output values of a '``callbr``' instruction are available both in the the '``fallthrough``' block, and any '``indirect``' blocks(s). -The only use of this today is to implement the "goto" feature of gcc inline -assembly where additional labels can be provided as locations for the inline -assembly to jump to. +The only uses of this today are: + +#. implement the "goto" feature of gcc inline assembly where additional + labels can be provided as locations for the inline assembly to jump to. +#. support selected intrinsics which manipulate control flow and should + be chained to specific terminators, such as '``unreachable``'. Example: """""""" @@ -9675,6 +9683,14 @@ Example: = callbr i32 asm "", "=r,r,!i"(i32 %x) to label %fallthrough [label %indirect] + ; intrinsic which should be followed by unreachable (the order of the + ; blocks after the callbr instruction doesn't matter) + callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill] + cont: + ... + kill: + unreachable + .. _i_resume: '``resume``' Instruction From 0483dc0a5ec3bd3ecabf75d0e13cc7c93ccc3fea Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Mon, 12 May 2025 04:46:22 -0500 Subject: [PATCH 11/18] don't start supporting inline-asm for callbr in GlobalISel with this patch --- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 15 ++++++--------- 1 file changed, 6 insertions(+), 9 deletions(-) diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 248d9287efdf4..ec0919c4046d7 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -3033,16 +3033,13 @@ bool IRTranslator::translateCallBr(const User &U, const CallBrInst &I = cast(U); MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB(); - if (I.isInlineAsm()) { - // FIXME: inline asm not yet supported - if (!translateInlineAsm(I, MIRBuilder)) - return false; - } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) { - if (!translateTargetIntrinsic(I, I.getIntrinsicID(), MIRBuilder)) - return false; - } else { + // FIXME: inline asm not yet supported + if (I.isInlineAsm()) + return false; + if (I.getIntrinsicID() == Intrinsic::not_intrinsic) + return false; + if (!translateTargetIntrinsic(I, I.getIntrinsicID(), MIRBuilder)) return false; - } // Retrieve successors. SmallPtrSet Dests; From 7ecf00aaca072d83a2780c0577d8e62799ffb782 Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Mon, 12 May 2025 10:22:01 -0500 Subject: [PATCH 12/18] add comments on callbr region handling and fix polly test --- llvm/include/llvm/Analysis/RegionInfoImpl.h | 3 +++ polly/test/ScopDetect/callbr.ll | 9 +++------ 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/llvm/include/llvm/Analysis/RegionInfoImpl.h b/llvm/include/llvm/Analysis/RegionInfoImpl.h index 759e9c47bebb8..3d507f5467037 100644 --- a/llvm/include/llvm/Analysis/RegionInfoImpl.h +++ b/llvm/include/llvm/Analysis/RegionInfoImpl.h @@ -553,6 +553,9 @@ bool RegionInfoBase::isRegion(BlockT *entry, BlockT *exit) const { using DST = typename DomFrontierT::DomSetType; + // Make sure that a region involving a callbr contains every successor + // blocks up to the ones that postdominate the callbr block. Otherwise, + // StructurizeCFG will tear the callbr apart. // TODO? post domination frontier? if constexpr (std::is_same_v) { if (DomTreeNodeT *PDTNode = PDT->getNode(exit); PDTNode) { diff --git a/polly/test/ScopDetect/callbr.ll b/polly/test/ScopDetect/callbr.ll index 4182974693678..75f676afd79c4 100644 --- a/polly/test/ScopDetect/callbr.ll +++ b/polly/test/ScopDetect/callbr.ll @@ -1,10 +1,7 @@ -; RUN: opt %loadNPMPolly '-passes=print' -polly-detect-track-failures -disable-output -pass-remarks-missed=polly-detect < %s 2>&1 | FileCheck %s --check-prefix=REMARK -; RUN: opt %loadNPMPolly '-passes=print' -polly-detect-track-failures -disable-output -stats < %s 2>&1 | FileCheck %s --check-prefix=STAT -; REQUIRES: asserts +; RUN: opt %loadNPMPolly '-passes=print' -disable-output < %s 2>&1 | FileCheck %s -; REMARK: Branch from indirect terminator. - -; STAT: 1 polly-detect - Number of rejected regions: Branch from indirect terminator +; CHECK-LABEL: func +; CHECK-NOT: Valid target datalayout = "e-m:e-p270:32:32-p271:32:32-p272:64:64-i64:64-f80:128-n8:16:32:64-S128" From a4f9c54073dafd8269370e40c5c3557cd6ec8ae0 Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Mon, 26 May 2025 04:40:37 -0500 Subject: [PATCH 13/18] fix typo and formatting --- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index ec0919c4046d7..60b55303ad490 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -2818,8 +2818,9 @@ bool IRTranslator::translateTargetIntrinsic( else if (TgtMemIntrinsicInfo->fallbackAddressSpace) MPI = MachinePointerInfo(*TgtMemIntrinsicInfo->fallbackAddressSpace); MIB.addMemOperand(MF->getMachineMemOperand( - MPI, TgtMemIntrinsicInfo->flags, MemTy, Alignment, CI.getAAMetadata(), - /*Ranges=*/nullptr, TgtMemIntrinsicInfo->ssid, TgtMemIntrinsicInfo->order, TgtMemIntrinsicInfo->failureOrder)); + MPI, TgtMemIntrinsicInfo->flags, MemTy, Alignment, CB.getAAMetadata(), + /*Ranges=*/nullptr, TgtMemIntrinsicInfo->ssid, + TgtMemIntrinsicInfo->order, TgtMemIntrinsicInfo->failureOrder)); } if (CB.isConvergent()) { From 0c9ad3090c6d35edaa4d745e3db1d80f8038c613 Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Tue, 10 Jun 2025 07:45:58 -0500 Subject: [PATCH 14/18] remove trailing space --- llvm/test/Verifier/callbr.ll | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll index 29bd3397b8980..8996938dcf85d 100644 --- a/llvm/test/Verifier/callbr.ll +++ b/llvm/test/Verifier/callbr.ll @@ -153,7 +153,7 @@ cont: } ; CHECK-NEXT: Callbr currently only supports asm-goto and selected intrinsics -declare i32 @llvm.amdgcn.workitem.id.x() +declare i32 @llvm.amdgcn.workitem.id.x() define void @test_callbr_intrinsic_unsupported() { callbr i32 @llvm.amdgcn.workitem.id.x() to label %cont [] cont: From 472767da4c01f71d3f457a491bf284a9d6a8fa04 Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Tue, 10 Jun 2025 07:52:59 -0500 Subject: [PATCH 15/18] reflect #141562 also in IRTranslator.cpp --- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 60b55303ad490..48f2dfe0146c7 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -3058,7 +3058,11 @@ bool IRTranslator::translateCallBr(const User &U, for (BasicBlock *Dest : I.getIndirectDests()) { MachineBasicBlock *Target = &getMBB(*Dest); Target->setIsInlineAsmBrIndirectTarget(); - Target->setMachineBlockAddressTaken(); + // If we introduce a type of asm goto statement that is permitted to use + // an indirect call instruction to jump to its labels, then we should add + // a call to Target->setMachineBlockAddressTaken() here, to mark the + // target block as requiring a BTI. + Target->setLabelMustBeEmitted(); // Don't add duplicate machine successors. if (Dests.insert(Dest).second) From b26115b1b9bf1512ca705d87690d32bad85f19e2 Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Thu, 12 Jun 2025 06:40:23 -0500 Subject: [PATCH 16/18] reflect #142613 in IRTranslator and SelectionDAGBuilder --- llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 7 ++++++- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 4 ++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp index 48f2dfe0146c7..2d9a86492ed8d 100644 --- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp +++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp @@ -3035,8 +3035,13 @@ bool IRTranslator::translateCallBr(const User &U, MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB(); // FIXME: inline asm not yet supported - if (I.isInlineAsm()) + if (I.isInlineAsm()) { + if (I.hasOperandBundlesOtherThan( + {LLVMContext::OB_deopt, LLVMContext::OB_funclet})) + reportFatalUsageError( + "cannot lower callbrs with arbitrary operand bundles!"); return false; + } if (I.getIntrinsicID() == Intrinsic::not_intrinsic) return false; if (!translateTargetIntrinsic(I, I.getIntrinsicID(), MIRBuilder)) diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 755cd30ab3bc6..a65381112af92 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3461,6 +3461,10 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { MachineBasicBlock *CallBrMBB = FuncInfo.MBB; if (I.isInlineAsm()) { + if (I.hasOperandBundlesOtherThan( + {LLVMContext::OB_deopt, LLVMContext::OB_funclet})) + reportFatalUsageError( + "cannot lower callbrs with arbitrary operand bundles!"); visitInlineAsm(I); } else if (I.getIntrinsicID() != Intrinsic::not_intrinsic) { visitCallBrIntrinsic(I); From 315f26ef2a1d4f1c22e57d4316ed6a1af9fd898c Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Sat, 14 Jun 2025 11:35:57 -0500 Subject: [PATCH 17/18] improve error message for callbr intrinsic verifier --- llvm/lib/IR/Verifier.cpp | 2 +- llvm/test/Verifier/callbr.ll | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp index 00a03bbb27e14..c9f112d260d63 100644 --- a/llvm/lib/IR/Verifier.cpp +++ b/llvm/lib/IR/Verifier.cpp @@ -3300,7 +3300,7 @@ void Verifier::visitCallBrInst(CallBrInst &CBI) { Check(CBI.getCalledFunction(), "Callbr: indirect function / invalid signature"); Check(!CBI.hasOperandBundles(), - "Callbr currently doesn't support operand bundles"); + "Callbr for intrinsics currently doesn't support operand bundles"); switch (CBI.getIntrinsicID()) { case Intrinsic::amdgcn_kill: { diff --git a/llvm/test/Verifier/callbr.ll b/llvm/test/Verifier/callbr.ll index 8996938dcf85d..b32c99a412d15 100644 --- a/llvm/test/Verifier/callbr.ll +++ b/llvm/test/Verifier/callbr.ll @@ -167,3 +167,12 @@ define void @test_callbr_intrinsic_wrong_signature(ptr %ptr) { cont: ret void } + +; CHECK-NEXT: Callbr for intrinsics currently doesn't support operand bundles +define void @test_callbr_intrinsic_no_operand_bundles(i1 %c) { + callbr void @llvm.amdgcn.kill(i1 %c) [ "foo"(i1 %c) ] to label %cont [label %kill] +kill: + unreachable +cont: + ret void +} From 0e162f0bcb609aea6f1dd8bfd8881587f03d2357 Mon Sep 17 00:00:00 2001 From: Robert Imschweiler Date: Sat, 14 Jun 2025 11:45:38 -0500 Subject: [PATCH 18/18] fix formatting --- llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h | 3 ++- llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 8 ++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h index 98115c109dfd6..81e8877e55a66 100644 --- a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h +++ b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h @@ -609,7 +609,8 @@ LLVM_ABI void InvertBranch(BranchInst *PBI, IRBuilderBase &Builder); // Check whether the function only has blocks with simple terminators: // br/brcond/unreachable/ret (or callbr if AllowCallBr) -LLVM_ABI bool hasOnlySimpleTerminator(const Function &F, bool AllowCallBr = true); +LLVM_ABI bool hasOnlySimpleTerminator(const Function &F, + bool AllowCallBr = true); } // end namespace llvm diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index a65381112af92..87e79e1158b51 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -3489,10 +3489,10 @@ void SelectionDAGBuilder::visitCallBr(const CallBrInst &I) { for (BasicBlock *Dest : I.getIndirectDests()) { MachineBasicBlock *Target = FuncInfo.getMBB(Dest); Target->setIsInlineAsmBrIndirectTarget(); - // If we introduce a type of asm goto statement that is permitted to use an - // indirect call instruction to jump to its labels, then we should add a - // call to Target->setMachineBlockAddressTaken() here, to mark the target - // block as requiring a BTI. + // If we introduce a type of asm goto statement that is permitted to use + // an indirect call instruction to jump to its labels, then we should add + // a call to Target->setMachineBlockAddressTaken() here, to mark the + // target block as requiring a BTI. Target->setLabelMustBeEmitted(); // Don't add duplicate machine successors.