llvm · ro-i · Apr 1, 2025 · Apr 1, 2025 · Apr 2, 2025 · Apr 2, 2025
diff --git a/llvm/docs/LangRef.rst b/llvm/docs/LangRef.rst
@@ -9585,8 +9585,12 @@ The '``callbr``' instruction causes control to transfer to a specified
 function, with the possibility of control flow transfer to either the
 '``fallthrough``' label or one of the '``indirect``' labels.

-This instruction should only be used to implement the "goto" feature of gcc
-style inline assembly. Any other usage is an error in the IR verifier.
+This instruction can currently only be used
+
+#. to implement the "goto" feature of gcc style inline assembly or
+#. to call selected intrinsics.
+
+Any other usage is an error in the IR verifier.

 Note that in order to support outputs along indirect edges, LLVM may need to
 split critical edges, which may require synthesizing a replacement block for
@@ -9627,7 +9631,8 @@ This instruction requires several arguments:
   indicates the function accepts a variable number of arguments, the
   extra arguments can be specified.
 #. '``fallthrough label``': the label reached when the inline assembly's
-   execution exits the bottom.
+   execution exits the bottom. In case of an intrinsic call, the semantic
+   depends on the semantic of the intrinsic.
 #. '``indirect labels``': the labels reached when a callee transfers control
   to a location other than the '``fallthrough label``'. Label constraints
   refer to these destinations.
@@ -9645,9 +9650,12 @@ flow goes after the call.
 The output values of a '``callbr``' instruction are available both in the
 the '``fallthrough``' block, and any '``indirect``' blocks(s).

-The only use of this today is to implement the "goto" feature of gcc inline
-assembly where additional labels can be provided as locations for the inline
-assembly to jump to.
+The only uses of this today are:
+
+#. implement the "goto" feature of gcc inline assembly where additional
+   labels can be provided as locations for the inline assembly to jump to.
+#. support selected intrinsics which manipulate control flow and should
+   be chained to specific terminators, such as '``unreachable``'.

 Example:
 """"""""
@@ -9662,6 +9670,14 @@ Example:
      <result> = callbr i32 asm "", "=r,r,!i"(i32 %x)
                  to label %fallthrough [label %indirect]

+      ; intrinsic which should be followed by unreachable (the order of the
+      ; blocks after the callbr instruction doesn't matter)
+        callbr void @llvm.amdgcn.kill(i1 %c) to label %cont [label %kill]
+      cont:
+        ...
+      kill:
+        unreachable
+
 .. _i_resume:

 '``resume``' Instruction

diff --git a/llvm/include/llvm/Analysis/RegionInfoImpl.h b/llvm/include/llvm/Analysis/RegionInfoImpl.h
@@ -553,6 +553,21 @@ bool RegionInfoBase<Tr>::isRegion(BlockT *entry, BlockT *exit) const {

  using DST = typename DomFrontierT::DomSetType;

+  // Make sure that a region involving a callbr contains every successor
+  // blocks up to the ones that postdominate the callbr block. Otherwise,
+  // StructurizeCFG will tear the callbr apart.
+  // TODO? post domination frontier?
+  if constexpr (std::is_same_v<BlockT, BasicBlock>) {
+    if (DomTreeNodeT *PDTNode = PDT->getNode(exit); PDTNode) {
+      for (DomTreeNodeT *PredNode : *PDTNode) {
+        for (BasicBlock *Pred : predecessors(PredNode->getBlock())) {
+          if (isa<CallBrInst>(Pred->getTerminator()))
+            return false;
+        }
+      }
+    }
+  }
+
  DST *entrySuccs = &DF->find(entry)->second;

  // Exit is the header of a loop that contains the entry. In this case,

diff --git a/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h b/llvm/include/llvm/CodeGen/GlobalISel/IRTranslator.h
@@ -297,6 +297,10 @@ class IRTranslator : public MachineFunctionPass {
  /// \pre \p U is a call instruction.
  bool translateCall(const User &U, MachineIRBuilder &MIRBuilder);

+  bool translateTargetIntrinsic(
+      const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
+      TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo = nullptr);
+
  /// When an invoke or a cleanupret unwinds to the next EH pad, there are
  /// many places it could ultimately go. In the IR, we have a single unwind
  /// destination, but in the machine CFG, we enumerate all the possible blocks.
@@ -313,6 +317,8 @@ class IRTranslator : public MachineFunctionPass {
  bool translateInvoke(const User &U, MachineIRBuilder &MIRBuilder);

  bool translateCallBr(const User &U, MachineIRBuilder &MIRBuilder);
+  bool translateCallBrIntrinsic(const CallBrInst &I,
+                                MachineIRBuilder &MIRBuilder);

  bool translateLandingPad(const User &U, MachineIRBuilder &MIRBuilder);


diff --git a/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h b/llvm/include/llvm/Transforms/Utils/BasicBlockUtils.h
@@ -606,9 +606,9 @@ bool SplitIndirectBrCriticalEdges(Function &F, bool IgnoreBlocksWithoutPHI,
 // successors
 void InvertBranch(BranchInst *PBI, IRBuilderBase &Builder);

-// Check whether the function only has simple terminator:
-// br/brcond/unreachable/ret
-bool hasOnlySimpleTerminator(const Function &F);
+// Check whether the function only has blocks with simple terminators:
+// br/brcond/unreachable/ret (or callbr if AllowCallBr)
+bool hasOnlySimpleTerminator(const Function &F, bool AllowCallBr = true);

 } // end namespace llvm


diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -2750,59 +2750,27 @@ bool IRTranslator::translateCallBase(const CallBase &CB,
  return Success;
 }

-bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
-  if (containsBF16Type(U))
-    return false;
-
-  const CallInst &CI = cast<CallInst>(U);
-  const Function *F = CI.getCalledFunction();
-
-  // FIXME: support Windows dllimport function calls and calls through
-  // weak symbols.
-  if (F && (F->hasDLLImportStorageClass() ||
-            (MF->getTarget().getTargetTriple().isOSWindows() &&
-             F->hasExternalWeakLinkage())))
-    return false;
-
-  // FIXME: support control flow guard targets.
-  if (CI.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
-    return false;
-
-  // FIXME: support statepoints and related.
-  if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U))
-    return false;
-
-  if (CI.isInlineAsm())
-    return translateInlineAsm(CI, MIRBuilder);
-
-  diagnoseDontCall(CI);
-
-  Intrinsic::ID ID = Intrinsic::not_intrinsic;
-  if (F && F->isIntrinsic())
-    ID = F->getIntrinsicID();
-
-  if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
-    return translateCallBase(CI, MIRBuilder);
-
-  assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
-
-  if (translateKnownIntrinsic(CI, ID, MIRBuilder))
-    return true;
-
+/// Translate a call or callbr to a target intrinsic.
+/// Depending on whether TLI->getTgtMemIntrinsic() is true, TgtMemIntrinsicInfo
+/// is a pointer to the correspondingly populated IntrinsicInfo object.
+/// Otherwise, this pointer is null.
+bool IRTranslator::translateTargetIntrinsic(
+    const CallBase &CB, Intrinsic::ID ID, MachineIRBuilder &MIRBuilder,
+    TargetLowering::IntrinsicInfo *TgtMemIntrinsicInfo) {
  ArrayRef<Register> ResultRegs;
-  if (!CI.getType()->isVoidTy())
-    ResultRegs = getOrCreateVRegs(CI);
+  if (!CB.getType()->isVoidTy())
+    ResultRegs = getOrCreateVRegs(CB);

  // Ignore the callsite attributes. Backend code is most likely not expecting
  // an intrinsic to sometimes have side effects and sometimes not.
  MachineInstrBuilder MIB = MIRBuilder.buildIntrinsic(ID, ResultRegs);
-  if (isa<FPMathOperator>(CI))
-    MIB->copyIRFlags(CI);
+  if (isa<FPMathOperator>(CB))
+    MIB->copyIRFlags(CB);

-  for (const auto &Arg : enumerate(CI.args())) {
+  for (const auto &Arg : enumerate(CB.args())) {
    // If this is required to be an immediate, don't materialize it in a
    // register.
-    if (CI.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
+    if (CB.paramHasAttr(Arg.index(), Attribute::ImmArg)) {
      if (ConstantInt *CI = dyn_cast<ConstantInt>(Arg.value())) {
        // imm arguments are more convenient than cimm (and realistically
        // probably sufficient), so use them.
@@ -2831,29 +2799,32 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
  }

  // Add a MachineMemOperand if it is a target mem intrinsic.
-  TargetLowering::IntrinsicInfo Info;
-  // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
-  if (TLI->getTgtMemIntrinsic(Info, CI, *MF, ID)) {
-    Align Alignment = Info.align.value_or(
-        DL->getABITypeAlign(Info.memVT.getTypeForEVT(F->getContext())));
-    LLT MemTy = Info.memVT.isSimple()
-                    ? getLLTForMVT(Info.memVT.getSimpleVT())
-                    : LLT::scalar(Info.memVT.getStoreSizeInBits());
+  if (TgtMemIntrinsicInfo) {
+    const Function *F = CB.getCalledFunction();
+
+    Align Alignment = TgtMemIntrinsicInfo->align.value_or(DL->getABITypeAlign(
+        TgtMemIntrinsicInfo->memVT.getTypeForEVT(F->getContext())));
+    LLT MemTy =
+        TgtMemIntrinsicInfo->memVT.isSimple()
+            ? getLLTForMVT(TgtMemIntrinsicInfo->memVT.getSimpleVT())
+            : LLT::scalar(TgtMemIntrinsicInfo->memVT.getStoreSizeInBits());

    // TODO: We currently just fallback to address space 0 if getTgtMemIntrinsic
    //       didn't yield anything useful.
    MachinePointerInfo MPI;
-    if (Info.ptrVal)
-      MPI = MachinePointerInfo(Info.ptrVal, Info.offset);
-    else if (Info.fallbackAddressSpace)
-      MPI = MachinePointerInfo(*Info.fallbackAddressSpace);
+    if (TgtMemIntrinsicInfo->ptrVal)
+      MPI = MachinePointerInfo(TgtMemIntrinsicInfo->ptrVal,
+                               TgtMemIntrinsicInfo->offset);
+    else if (TgtMemIntrinsicInfo->fallbackAddressSpace)
+      MPI = MachinePointerInfo(*TgtMemIntrinsicInfo->fallbackAddressSpace);
    MIB.addMemOperand(MF->getMachineMemOperand(
-        MPI, Info.flags, MemTy, Alignment, CI.getAAMetadata(),
-        /*Ranges=*/nullptr, Info.ssid, Info.order, Info.failureOrder));
+        MPI, TgtMemIntrinsicInfo->flags, MemTy, Alignment, CB.getAAMetadata(),
+        /*Ranges=*/nullptr, TgtMemIntrinsicInfo->ssid,
+        TgtMemIntrinsicInfo->order, TgtMemIntrinsicInfo->failureOrder));
  }

-  if (CI.isConvergent()) {
-    if (auto Bundle = CI.getOperandBundle(LLVMContext::OB_convergencectrl)) {
+  if (CB.isConvergent()) {
+    if (auto Bundle = CB.getOperandBundle(LLVMContext::OB_convergencectrl)) {
      auto *Token = Bundle->Inputs[0].get();
      Register TokenReg = getOrCreateVReg(*Token);
      MIB.addUse(TokenReg, RegState::Implicit);
@@ -2863,6 +2834,53 @@ bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
  return true;
 }

+bool IRTranslator::translateCall(const User &U, MachineIRBuilder &MIRBuilder) {
+  if (containsBF16Type(U))
+    return false;
+
+  const CallInst &CI = cast<CallInst>(U);
+  const Function *F = CI.getCalledFunction();
+
+  // FIXME: support Windows dllimport function calls and calls through
+  // weak symbols.
+  if (F && (F->hasDLLImportStorageClass() ||
+            (MF->getTarget().getTargetTriple().isOSWindows() &&
+             F->hasExternalWeakLinkage())))
+    return false;
+
+  // FIXME: support control flow guard targets.
+  if (CI.countOperandBundlesOfType(LLVMContext::OB_cfguardtarget))
+    return false;
+
+  // FIXME: support statepoints and related.
+  if (isa<GCStatepointInst, GCRelocateInst, GCResultInst>(U))
+    return false;
+
+  if (CI.isInlineAsm())
+    return translateInlineAsm(CI, MIRBuilder);
+
+  diagnoseDontCall(CI);
+
+  Intrinsic::ID ID = Intrinsic::not_intrinsic;
+  if (F && F->isIntrinsic())
+    ID = F->getIntrinsicID();
+
+  if (!F || !F->isIntrinsic() || ID == Intrinsic::not_intrinsic)
+    return translateCallBase(CI, MIRBuilder);
+
+  assert(ID != Intrinsic::not_intrinsic && "unknown intrinsic");
+
+  if (translateKnownIntrinsic(CI, ID, MIRBuilder))
+    return true;
+
+  TargetLowering::IntrinsicInfo Info;
+  // TODO: Add a GlobalISel version of getTgtMemIntrinsic.
+  bool IsTgtMemIntrinsic = TLI->getTgtMemIntrinsic(Info, CI, *MF, ID);
+
+  return translateTargetIntrinsic(CI, ID, MIRBuilder,
+                                  IsTgtMemIntrinsic ? &Info : nullptr);
+}
+
 bool IRTranslator::findUnwindDestinations(
    const BasicBlock *EHPadBB,
    BranchProbability Prob,
@@ -3006,10 +3024,50 @@ bool IRTranslator::translateInvoke(const User &U,
  return true;
 }

+/// The intrinsics currently supported by callbr are implicit control flow
+/// intrinsics such as amdgcn.kill.
 bool IRTranslator::translateCallBr(const User &U,
                                   MachineIRBuilder &MIRBuilder) {
-  // FIXME: Implement this.
-  return false;
+  if (containsBF16Type(U))
+    return false; // see translateCall
+
+  const CallBrInst &I = cast<CallBrInst>(U);
+  MachineBasicBlock *CallBrMBB = &MIRBuilder.getMBB();
+
+  // FIXME: inline asm not yet supported
+  if (I.isInlineAsm())
+    return false;
+  if (I.getIntrinsicID() == Intrinsic::not_intrinsic)
+    return false;
+  if (!translateTargetIntrinsic(I, I.getIntrinsicID(), MIRBuilder))
+    return false;
+
+  // Retrieve successors.
+  SmallPtrSet<BasicBlock *, 8> Dests;
+  Dests.insert(I.getDefaultDest());
+  MachineBasicBlock *Return = &getMBB(*I.getDefaultDest());
+
+  // Update successor info.
+  addSuccessorWithProb(CallBrMBB, Return, BranchProbability::getOne());
+  // TODO: For most of the cases where there is an intrinsic callbr, we're
+  // having exactly one indirect target, which will be unreachable. As soon as
+  // this changes, we might need to enhance
+  // Target->setIsInlineAsmBrIndirectTarget or add something similar for
+  // intrinsic indirect branches.
+  if (I.isInlineAsm()) {
+    for (BasicBlock *Dest : I.getIndirectDests()) {
+      MachineBasicBlock *Target = &getMBB(*Dest);
+      Target->setIsInlineAsmBrIndirectTarget();
+      Target->setMachineBlockAddressTaken();
+      Target->setLabelMustBeEmitted();
+      // Don't add duplicate machine successors.
+      if (Dests.insert(Dest).second)
+        addSuccessorWithProb(CallBrMBB, Target, BranchProbability::getZero());
+    }
+  }
+  CallBrMBB->normalizeSuccProbs();
+
+  return true;
 }

 bool IRTranslator::translateLandingPad(const User &U,