[llvm] [LLVM] Change Intrinsic::ID to encode target and intrinsic index (PR #113576)
Rahul Joshi via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 25 16:08:40 PDT 2024
https://github.com/jurahul updated https://github.com/llvm/llvm-project/pull/113576
>From f32b1e42933974fd88aff9800f8ed3a9614b65a3 Mon Sep 17 00:00:00 2001
From: Rahul Joshi <rjoshi at nvidia.com>
Date: Thu, 24 Oct 2024 06:56:59 -0700
Subject: [PATCH] [LLVM] Change Intrinsic::ID to encode target and intrinsic
index
Change `Intrinsic::ID` enum values to encode an 8-bit target index
in upper 16-bits and a 16-bit intrinsic index (within the target) in
lower 16-bits.
This change is in preparation for being able to disable intrinsics for
targets that are not enabled.
---
.../CodeGen/GlobalISel/GIMatchTableExecutor.h | 4 +-
.../GlobalISel/GIMatchTableExecutorImpl.h | 4 +-
llvm/include/llvm/IR/Intrinsics.h | 9 +
llvm/include/llvm/Support/IntrinsicID.h | 53 +++++
llvm/lib/CodeGen/MachineOperand.cpp | 2 +-
llvm/lib/CodeGen/MachineVerifier.cpp | 4 +-
.../SelectionDAG/SelectionDAGDumper.cpp | 2 +-
.../CodeGen/SelectionDAG/SelectionDAGISel.cpp | 2 +-
llvm/lib/IR/Core.cpp | 3 +-
llvm/lib/IR/Intrinsics.cpp | 117 ++++++++---
.../Target/AArch64/AArch64ISelLowering.cpp | 2 +-
llvm/lib/Target/X86/X86IntrinsicsInfo.h | 3 +-
.../match-table-intrinsics.td | 14 +-
.../GlobalISelEmitter-SDNodeXForm-timm.td | 4 +-
...lobalISelEmitter-immarg-literal-pattern.td | 4 +-
.../GlobalISelEmitter-input-discard.td | 2 +-
llvm/test/TableGen/GlobalISelEmitter.td | 8 +-
.../GlobalISelEmitterOverloadedPtr.td | 2 +-
llvm/test/TableGen/immarg-predicated.td | 2 +-
llvm/test/TableGen/immarg.td | 2 +-
.../TableGen/intrinsic-overload-conflict.td | 2 +-
llvm/test/TableGen/intrinsic-struct.td | 2 +-
llvm/test/TableGen/predicate-patfags.td | 4 +-
llvm/unittests/IR/VPIntrinsicTest.cpp | 15 +-
.../TableGen/Basic/CodeGenIntrinsics.cpp | 159 ++++++++++-----
llvm/utils/TableGen/Basic/CodeGenIntrinsics.h | 33 ++--
.../TableGen/Common/CodeGenDAGPatterns.cpp | 2 +-
.../TableGen/Common/CodeGenDAGPatterns.h | 14 +-
.../GlobalISel/GlobalISelMatchTable.cpp | 4 +-
llvm/utils/TableGen/IntrinsicEmitter.cpp | 182 ++++++++++--------
30 files changed, 440 insertions(+), 220 deletions(-)
create mode 100644 llvm/include/llvm/Support/IntrinsicID.h
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
index 7b42722ca8d4f1..383ce96813d84d 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutor.h
@@ -275,7 +275,7 @@ enum {
/// Check the operand is a specific intrinsic ID
/// - InsnID(ULEB128) - Instruction ID
/// - OpIdx(ULEB128) - Operand index
- /// - IID(2) - Expected Intrinsic ID
+ /// - IID(4) - Expected Intrinsic ID
GIM_CheckIntrinsicID,
/// Check the operand is a specific predicate
@@ -411,7 +411,7 @@ enum {
/// Adds an intrinsic ID to the specified instruction.
/// - InsnID(ULEB128) - Instruction ID to modify
- /// - IID(2) - Intrinsic ID
+ /// - IID(4) - Intrinsic ID
GIR_AddIntrinsicID,
/// Marks the implicit def of a register as dead.
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
index 9f8eb030a96c64..775998fe53a7be 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GIMatchTableExecutorImpl.h
@@ -889,7 +889,7 @@ bool GIMatchTableExecutor::executeMatchTable(
case GIM_CheckIntrinsicID: {
uint64_t InsnID = readULEB();
uint64_t OpIdx = readULEB();
- uint16_t Value = readU16();
+ uint32_t Value = readU32();
DEBUG_WITH_TYPE(TgtExecutor::getName(),
dbgs() << CurrentIdx << ": GIM_CheckIntrinsicID(MIs["
<< InsnID << "]->getOperand(" << OpIdx
@@ -1185,7 +1185,7 @@ bool GIMatchTableExecutor::executeMatchTable(
}
case GIR_AddIntrinsicID: {
uint64_t InsnID = readULEB();
- uint16_t Value = readU16();
+ uint32_t Value = readU32();
assert(OutMIs[InsnID] && "Attempted to add to undefined instruction");
OutMIs[InsnID].addIntrinsicID((Intrinsic::ID)Value);
DEBUG_WITH_TYPE(TgtExecutor::getName(),
diff --git a/llvm/include/llvm/IR/Intrinsics.h b/llvm/include/llvm/IR/Intrinsics.h
index e893295e3272b9..e04711c45de803 100644
--- a/llvm/include/llvm/IR/Intrinsics.h
+++ b/llvm/include/llvm/IR/Intrinsics.h
@@ -47,8 +47,17 @@ namespace Intrinsic {
#define GET_INTRINSIC_ENUM_VALUES
#include "llvm/IR/IntrinsicEnums.inc"
#undef GET_INTRINSIC_ENUM_VALUES
+ end_id = ~0U,
};
+ // Returns if `id` is a valid intrinsic ID. Validity means that it value is
+ // one of the values defined for the Intrinsic::ID enum.
+ bool IsIntrinsicIDValid(ID id);
+
+ // Get the next valid ID. This is used in test cases that iterate over valid
+ // intrinsic ID enums.
+ ID GetNextValidIntrinsicID(ID id);
+
/// Return the LLVM name for an intrinsic, such as "llvm.ppc.altivec.lvx".
/// Note, this version is for intrinsics with no overloads. Use the other
/// version of getName if overloads are required.
diff --git a/llvm/include/llvm/Support/IntrinsicID.h b/llvm/include/llvm/Support/IntrinsicID.h
new file mode 100644
index 00000000000000..c1e546c6d4f352
--- /dev/null
+++ b/llvm/include/llvm/Support/IntrinsicID.h
@@ -0,0 +1,53 @@
+//===- llvm/Support/IntrinsicID.h - Intrinsic ID encoding -------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+//
+// This file contains functions to support intrinsic ID encoding. The
+// Intrinsic::ID enum value is constructed using a target prefix index in bits
+// 23-16 (8-bit) and an intrinsic index (index within the list of intrinsics for
+// tha target) in lower 16 bits. To support Intrinsic::ID 0 being not used, the
+// intrinsic index is encoded as Index + 1 for all targets.
+//
+// This file defines functions that encapsulate this encoding.
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_SUPPORT_INTRINSIC_ID_H
+#define LLVM_SUPPORT_INTRINSIC_ID_H
+
+#include <limits>
+#include <optional>
+#include <utility>
+
+namespace llvm::Intrinsic {
+typedef unsigned ID;
+
+inline ID encodeIntrinsicID(unsigned TargetIndex, unsigned IntrinsicIndex) {
+ assert(IntrinsicIndex < std::numeric_limits<uint16_t>::max());
+ assert(TargetIndex <= std::numeric_limits<uint8_t>::max());
+ return (TargetIndex << 16) | (IntrinsicIndex + 1);
+}
+
+inline std::pair<unsigned, unsigned> decodeIntrinsicID(ID id) {
+ unsigned IntrinsicIndex = id & 0xFFFF;
+ unsigned TargetIndex = id >> 16;
+ assert(IntrinsicIndex != 0);
+ return {TargetIndex, IntrinsicIndex - 1};
+}
+
+inline std::optional<std::pair<unsigned, unsigned>>
+decodeIntrinsicIDNoFail(ID id) {
+ unsigned IntrinsicIndex = id & 0xFFFF;
+ unsigned TargetIndex = id >> 16;
+ if (IntrinsicIndex == 0)
+ return std::nullopt;
+ return std::make_pair(TargetIndex, IntrinsicIndex - 1);
+}
+
+} // end namespace llvm::Intrinsic
+
+#endif // LLVM_SUPPORT_INTRINSIC_ID_H
diff --git a/llvm/lib/CodeGen/MachineOperand.cpp b/llvm/lib/CodeGen/MachineOperand.cpp
index c0e004555de959..8261ac9401d6ae 100644
--- a/llvm/lib/CodeGen/MachineOperand.cpp
+++ b/llvm/lib/CodeGen/MachineOperand.cpp
@@ -992,7 +992,7 @@ void MachineOperand::print(raw_ostream &OS, ModuleSlotTracker &MST,
}
case MachineOperand::MO_IntrinsicID: {
Intrinsic::ID ID = getIntrinsicID();
- if (ID < Intrinsic::num_intrinsics)
+ if (Intrinsic::IsIntrinsicIDValid(ID))
OS << "intrinsic(@" << Intrinsic::getBaseName(ID) << ')';
else if (IntrinsicInfo)
OS << "intrinsic(@" << IntrinsicInfo->getName(ID) << ')';
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index e2c09fe25d55cd..77440e5e7275f7 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -1055,7 +1055,7 @@ bool MachineVerifier::verifyGIntrinsicSideEffects(const MachineInstr *MI) {
bool NoSideEffects = Opcode == TargetOpcode::G_INTRINSIC ||
Opcode == TargetOpcode::G_INTRINSIC_CONVERGENT;
unsigned IntrID = cast<GIntrinsic>(MI)->getIntrinsicID();
- if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
+ if (IntrID != 0 && Intrinsic::IsIntrinsicIDValid(IntrID)) {
AttributeList Attrs = Intrinsic::getAttributes(
MF->getFunction().getContext(), static_cast<Intrinsic::ID>(IntrID));
bool DeclHasSideEffects = !Attrs.getMemoryEffects().doesNotAccessMemory();
@@ -1079,7 +1079,7 @@ bool MachineVerifier::verifyGIntrinsicConvergence(const MachineInstr *MI) {
bool NotConvergent = Opcode == TargetOpcode::G_INTRINSIC ||
Opcode == TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS;
unsigned IntrID = cast<GIntrinsic>(MI)->getIntrinsicID();
- if (IntrID != 0 && IntrID < Intrinsic::num_intrinsics) {
+ if (IntrID != 0 && Intrinsic::IsIntrinsicIDValid(IntrID)) {
AttributeList Attrs = Intrinsic::getAttributes(
MF->getFunction().getContext(), static_cast<Intrinsic::ID>(IntrID));
bool DeclIsConvergent = Attrs.hasFnAttr(Attribute::Convergent);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 703efb70089742..d9e3324c6a4981 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -160,7 +160,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::INTRINSIC_W_CHAIN: {
unsigned OpNo = getOpcode() == ISD::INTRINSIC_WO_CHAIN ? 0 : 1;
unsigned IID = getOperand(OpNo)->getAsZExtVal();
- if (IID < Intrinsic::num_intrinsics)
+ if (Intrinsic::IsIntrinsicIDValid(IID))
return Intrinsic::getBaseName((Intrinsic::ID)IID).str();
if (!G)
return "Unknown intrinsic";
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
index 981ab18b59c1c1..391f6facdc901e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp
@@ -4431,7 +4431,7 @@ void SelectionDAGISel::CannotYetSelect(SDNode *N) {
} else {
bool HasInputChain = N->getOperand(0).getValueType() == MVT::Other;
unsigned iid = N->getConstantOperandVal(HasInputChain);
- if (iid < Intrinsic::num_intrinsics)
+ if (Intrinsic::IsIntrinsicIDValid(iid))
Msg << "intrinsic %" << Intrinsic::getBaseName((Intrinsic::ID)iid);
else if (const TargetIntrinsicInfo *TII = TM.getIntrinsicInfo())
Msg << "target intrinsic %" << TII->getName(iid);
diff --git a/llvm/lib/IR/Core.cpp b/llvm/lib/IR/Core.cpp
index 1cf998c6850068..32f31e9900880d 100644
--- a/llvm/lib/IR/Core.cpp
+++ b/llvm/lib/IR/Core.cpp
@@ -2458,7 +2458,8 @@ unsigned LLVMGetIntrinsicID(LLVMValueRef Fn) {
}
static Intrinsic::ID llvm_map_to_intrinsic_id(unsigned ID) {
- assert(ID < llvm::Intrinsic::num_intrinsics && "Intrinsic ID out of range");
+ assert(llvm::Intrinsic::IsIntrinsicIDValid(ID) &&
+ "Intrinsic ID out of range");
return llvm::Intrinsic::ID(ID);
}
diff --git a/llvm/lib/IR/Intrinsics.cpp b/llvm/lib/IR/Intrinsics.cpp
index 1b92daf15b463e..f2e27a0c20b708 100644
--- a/llvm/lib/IR/Intrinsics.cpp
+++ b/llvm/lib/IR/Intrinsics.cpp
@@ -33,8 +33,54 @@
#include "llvm/IR/IntrinsicsXCore.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/Type.h"
+#include "llvm/Support/IntrinsicID.h"
using namespace llvm;
+using namespace Intrinsic;
+
+/// Table of per-target intrinsic name tables.
+#define GET_INTRINSIC_TARGET_DATA
+#include "llvm/IR/IntrinsicImpl.inc"
+#undef GET_INTRINSIC_TARGET_DATA
+size_t constexpr NumTargets = sizeof(TargetInfos) / sizeof(TargetInfos[0]);
+
+// Returns true if the given intrinsic ID is valid, that is, its value is one
+// of the enum values defined for this intrinsic (including not_intrinsic).
+bool Intrinsic::IsIntrinsicIDValid(ID ID) {
+ if (ID == Intrinsic::not_intrinsic)
+ return true;
+ auto Decoded = decodeIntrinsicIDNoFail(ID);
+ if (!Decoded)
+ return false;
+ unsigned TargetIdx = Decoded->first;
+ unsigned IntrinsicIdx = Decoded->second;
+ return TargetIdx < NumTargets && IntrinsicIdx < TargetInfos[TargetIdx].Count;
+}
+
+// Returns linear index of ID if its valid, else returns 0.
+unsigned getLinearIndex(Intrinsic::ID ID) {
+ auto Decoded = decodeIntrinsicIDNoFail(ID);
+ if (!Decoded)
+ return 0;
+ unsigned TargetIdx = Decoded->first;
+ unsigned IntrinsicIdx = Decoded->second;
+ return TargetInfos[TargetIdx].FirstLinearIndex + IntrinsicIdx;
+}
+
+ID Intrinsic::GetNextValidIntrinsicID(Intrinsic::ID ID) {
+ if (ID == Intrinsic::not_intrinsic)
+ return encodeIntrinsicID(0, 0);
+ if (ID == Intrinsic::last_valid_intrinsic_id)
+ return Intrinsic::end_id;
+ if (ID == Intrinsic::end_id)
+ llvm_unreachable("Cannot find the next valid intrisnic");
+ auto [TargetIndex, IntrinsicIndex] = decodeIntrinsicID(ID);
+ if (IntrinsicIndex + 1 < TargetInfos[TargetIndex].Count)
+ return encodeIntrinsicID(TargetIndex, IntrinsicIndex + 1);
+ if (TargetIndex + 1 < NumTargets)
+ return encodeIntrinsicID(TargetIndex + 1, 0);
+ llvm_unreachable("Cannot find the next valid intrisnic");
+}
/// Table of string intrinsic names indexed by enum value.
static constexpr const char *const IntrinsicNameTable[] = {
@@ -45,12 +91,12 @@ static constexpr const char *const IntrinsicNameTable[] = {
};
StringRef Intrinsic::getBaseName(ID id) {
- assert(id < num_intrinsics && "Invalid intrinsic ID!");
- return IntrinsicNameTable[id];
+ assert(IsIntrinsicIDValid(id) && "Invalid intrinsic ID!");
+ return ArrayRef(IntrinsicNameTable)[getLinearIndex(id)];
}
StringRef Intrinsic::getName(ID id) {
- assert(id < num_intrinsics && "Invalid intrinsic ID!");
+ assert(IsIntrinsicIDValid(id) && "Invalid intrinsic ID!");
assert(!Intrinsic::isOverloaded(id) &&
"This version of getName does not support overloading");
return getBaseName(id);
@@ -157,8 +203,7 @@ static std::string getMangledTypeStr(Type *Ty, bool &HasUnnamedType) {
static std::string getIntrinsicNameImpl(Intrinsic::ID Id, ArrayRef<Type *> Tys,
Module *M, FunctionType *FT,
bool EarlyModuleCheck) {
-
- assert(Id < Intrinsic::num_intrinsics && "Invalid intrinsic ID!");
+ assert(IsIntrinsicIDValid(Id) && "Invalid intrinsic ID!");
assert((Tys.empty() || Intrinsic::isOverloaded(Id)) &&
"This version of getName is for overloaded intrinsics only");
(void)EarlyModuleCheck;
@@ -450,11 +495,15 @@ DecodeIITType(unsigned &NextElt, ArrayRef<unsigned char> Infos,
#undef GET_INTRINSIC_GENERATOR_GLOBAL
void Intrinsic::getIntrinsicInfoTableEntries(
- ID id, SmallVectorImpl<IITDescriptor> &T) {
+ ID IntrinsicID, SmallVectorImpl<IITDescriptor> &T) {
static_assert(sizeof(IIT_Table[0]) == 2,
"Expect 16-bit entries in IIT_Table");
+ assert(IsIntrinsicIDValid(IntrinsicID));
+ unsigned Idx = getLinearIndex(IntrinsicID);
+ if (Idx == 0)
+ return;
// Check to see if the intrinsic's type was expressible by the table.
- uint16_t TableVal = IIT_Table[id - 1];
+ uint16_t TableVal = IIT_Table[Idx - 1];
// Decode the TableVal into an array of IITValues.
SmallVector<unsigned char> IITValues;
@@ -609,19 +658,20 @@ FunctionType *Intrinsic::getType(LLVMContext &Context, ID id,
return FunctionType::get(ResultTy, ArgTys, false);
}
-bool Intrinsic::isOverloaded(ID id) {
+// Check if an intrinsic is overloaded or not using its linear index.
+static bool isOverloadedUsingLinearIndex(unsigned Idx) {
#define GET_INTRINSIC_OVERLOAD_TABLE
#include "llvm/IR/IntrinsicImpl.inc"
#undef GET_INTRINSIC_OVERLOAD_TABLE
}
-/// Table of per-target intrinsic name tables.
-#define GET_INTRINSIC_TARGET_DATA
-#include "llvm/IR/IntrinsicImpl.inc"
-#undef GET_INTRINSIC_TARGET_DATA
+bool Intrinsic::isOverloaded(ID id) {
+ assert(IsIntrinsicIDValid(id));
+ return isOverloadedUsingLinearIndex(getLinearIndex(id));
+}
bool Intrinsic::isTargetIntrinsic(Intrinsic::ID IID) {
- return IID > TargetInfos[0].Count;
+ return IID != Intrinsic::not_intrinsic && decodeIntrinsicID(IID).first != 0;
}
int llvm::Intrinsic::lookupLLVMIntrinsicByName(ArrayRef<const char *> NameTable,
@@ -683,7 +733,29 @@ findTargetSubtable(StringRef Name) {
// We've either found the target or just fall back to the generic set, which
// is always first.
const auto &TI = It != Targets.end() && It->Name == Target ? *It : Targets[0];
- return {ArrayRef(&IntrinsicNameTable[1] + TI.Offset, TI.Count), TI.Name};
+ unsigned LinearIndex = TI.FirstLinearIndex;
+ return {ArrayRef(IntrinsicNameTable + LinearIndex, TI.Count), TI.Name};
+}
+
+static Intrinsic::ID getIntrinsicIDFromIndex(unsigned Idx) {
+ if (Idx == 0)
+ return Intrinsic::not_intrinsic;
+
+ auto It =
+ partition_point(TargetInfos, [Idx](const IntrinsicTargetInfo &Info) {
+ return Info.FirstLinearIndex + Info.Count < Idx;
+ });
+ // Idx, if present, will be in the entry at It or It + 1.
+ if (It == std::end(TargetInfos))
+ return Intrinsic::not_intrinsic;
+ unsigned TargetIndex = std::distance(std::begin(TargetInfos), It);
+ if (It->FirstLinearIndex <= Idx && Idx < It->FirstLinearIndex + It->Count)
+ return encodeIntrinsicID(TargetIndex, Idx - It->FirstLinearIndex);
+ ++It;
+ ++TargetIndex;
+ if (It->FirstLinearIndex <= Idx && Idx < It->FirstLinearIndex + It->Count)
+ return encodeIntrinsicID(TargetIndex, Idx - It->FirstLinearIndex);
+ return Intrinsic::not_intrinsic;
}
/// This does the actual lookup of an intrinsic ID which matches the given
@@ -693,19 +765,19 @@ Intrinsic::ID Intrinsic::lookupIntrinsicID(StringRef Name) {
int Idx = Intrinsic::lookupLLVMIntrinsicByName(NameTable, Name, Target);
if (Idx == -1)
return Intrinsic::not_intrinsic;
-
- // Intrinsic IDs correspond to the location in IntrinsicNameTable, but we have
- // an index into a sub-table.
+ const auto MatchSize = strlen(NameTable[Idx]);
+ // Adjust the index from sub-table index to index into the global table.
int Adjust = NameTable.data() - IntrinsicNameTable;
- Intrinsic::ID ID = static_cast<Intrinsic::ID>(Idx + Adjust);
+ Idx += Adjust;
// If the intrinsic is not overloaded, require an exact match. If it is
// overloaded, require either exact or prefix match.
- const auto MatchSize = strlen(NameTable[Idx]);
assert(Name.size() >= MatchSize && "Expected either exact or prefix match");
bool IsExactMatch = Name.size() == MatchSize;
- return IsExactMatch || Intrinsic::isOverloaded(ID) ? ID
- : Intrinsic::not_intrinsic;
+ Intrinsic::ID r = IsExactMatch || isOverloadedUsingLinearIndex(Idx)
+ ? getIntrinsicIDFromIndex(Idx)
+ : Intrinsic::not_intrinsic;
+ return r;
}
/// This defines the "Intrinsic::getAttributes(ID id)" method.
@@ -1043,8 +1115,9 @@ bool Intrinsic::matchIntrinsicVarArg(
bool Intrinsic::getIntrinsicSignature(Intrinsic::ID ID, FunctionType *FT,
SmallVectorImpl<Type *> &ArgTys) {
- if (!ID)
+ if (ID == Intrinsic::not_intrinsic)
return false;
+ assert(IsIntrinsicIDValid(ID));
SmallVector<Intrinsic::IITDescriptor, 8> Table;
getIntrinsicInfoTableEntries(ID, Table);
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 4c0cd1ac3d4512..e4d8a24623d1b7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -7545,7 +7545,7 @@ static unsigned getIntrinsicID(const SDNode *N) {
return Intrinsic::not_intrinsic;
case ISD::INTRINSIC_WO_CHAIN: {
unsigned IID = N->getConstantOperandVal(0);
- if (IID < Intrinsic::num_intrinsics)
+ if (Intrinsic::IsIntrinsicIDValid(IID))
return IID;
return Intrinsic::not_intrinsic;
}
diff --git a/llvm/lib/Target/X86/X86IntrinsicsInfo.h b/llvm/lib/Target/X86/X86IntrinsicsInfo.h
index 86fd04046d16a0..f5477ec2f81315 100644
--- a/llvm/lib/Target/X86/X86IntrinsicsInfo.h
+++ b/llvm/lib/Target/X86/X86IntrinsicsInfo.h
@@ -79,8 +79,7 @@ enum IntrinsicType : uint16_t {
};
struct IntrinsicData {
-
- uint16_t Id;
+ unsigned Id;
IntrinsicType Type;
uint16_t Opc0;
uint16_t Opc1;
diff --git a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td
index 365d0c9fbff494..8d96f703f97892 100644
--- a/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td
+++ b/llvm/test/TableGen/GlobalISelCombinerEmitter/match-table-intrinsics.td
@@ -36,7 +36,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 3*/ GIMT_Encode4([[L72:[0-9]+]]), // Rule ID 0 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule0Enabled),
// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::1in_1out),
+// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::1in_1out),
// CHECK-NEXT: // MIs[0] a
// CHECK-NEXT: // No operand predicates
// CHECK-NEXT: // MIs[0] Operand 2
@@ -45,10 +45,10 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: // Combiner Rule #0: IntrinTest0
// CHECK-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC),
// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
-// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::0in_1out),
+// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode4(Intrinsic::0in_1out),
// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC),
// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a
-// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::1in_1out),
+// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode4(Intrinsic::1in_1out),
// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0,
// CHECK-NEXT: GIR_EraseRootFromParent_Done,
// CHECK-NEXT: // Label 3: @[[L72]]
@@ -57,7 +57,7 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: GIM_Try, /*On fail goto*//*Label 4*/ GIMT_Encode4([[L131:[0-9]+]]), // Rule ID 1 //
// CHECK-NEXT: GIM_CheckSimplePredicate, GIMT_Encode2(GICXXPred_Simple_IsRule1Enabled),
// CHECK-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
-// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::sideeffects_1in_1out),
+// CHECK-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::sideeffects_1in_1out),
// CHECK-NEXT: // MIs[0] a
// CHECK-NEXT: // No operand predicates
// CHECK-NEXT: // MIs[0] b
@@ -66,11 +66,11 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: // Combiner Rule #1: SpecialIntrins
// CHECK-NEXT: GIR_BuildRootMI, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT),
// CHECK-NEXT: GIR_AddTempRegister, /*InsnID*/0, /*TempRegID*/0, /*TempRegFlags*/GIMT_Encode2(RegState::Define),
-// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode2(Intrinsic::convergent_1in_1out),
+// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/0, GIMT_Encode4(Intrinsic::convergent_1in_1out),
// CHECK-NEXT: GIR_RootToRootCopy, /*OpIdx*/2, // b
// CHECK-NEXT: GIR_BuildMI, /*InsnID*/1, /*Opcode*/GIMT_Encode2(TargetOpcode::G_INTRINSIC_CONVERGENT_W_SIDE_EFFECTS),
// CHECK-NEXT: GIR_Copy, /*NewInsnID*/1, /*OldInsnID*/0, /*OpIdx*/0, // a
-// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode2(Intrinsic::convergent_sideeffects_1in_1out),
+// CHECK-NEXT: GIR_AddIntrinsicID, /*MI*/1, GIMT_Encode4(Intrinsic::convergent_sideeffects_1in_1out),
// CHECK-NEXT: GIR_AddSimpleTempRegister, /*InsnID*/1, /*TempRegID*/0,
// CHECK-NEXT: GIR_MergeMemOperands, /*InsnID*/1, /*NumInsns*/1, /*MergeInsnID's*/0,
// CHECK-NEXT: GIR_EraseRootFromParent_Done,
@@ -78,6 +78,6 @@ def MyCombiner: GICombiner<"GenMyCombiner", [
// CHECK-NEXT: GIM_Reject,
// CHECK-NEXT: // Label 2: @[[L132]]
// CHECK-NEXT: GIM_Reject,
-// CHECK-NEXT: }; // Size: 125 bytes
+// CHECK-NEXT: }; // Size: 137 bytes
// CHECK-NEXT: return MatchTable0;
// CHECK-NEXT: }
diff --git a/llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td b/llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td
index 8d6dedf2f920ce..e4f0a3bc5f6cd9 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-SDNodeXForm-timm.td
@@ -19,7 +19,7 @@ def SLEEP : I<(outs), (ins i32imm:$src0), []>;
def FOO : I<(outs GPR32:$dst), (ins GPR32:$src0, i32imm:$src1), []>;
// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC),
-// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::mytarget_foo),
+// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::mytarget_foo),
// GISEL: GIM_CheckIsImm, /*MI*/0, /*Op*/3,
// GISEL: GIR_CustomOperandRenderer, /*InsnID*/0, /*OldInsnID*/0, /*OpIdx*/3, /*OperandRenderer*/GIMT_Encode2(GICR_renderShiftImml1), // src1
def : Pat<
@@ -28,7 +28,7 @@ def : Pat<
>;
// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
-// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep),
+// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode4(Intrinsic::mytarget_sleep),
// GISEL: GIM_CheckIsImm, /*MI*/0, /*Op*/1,
// GISEL: GIR_CustomOperandRenderer, /*InsnID*/0, /*OldInsnID*/0, /*OpIdx*/1, /*OperandRenderer*/GIMT_Encode2(GICR_renderShiftImml1), // src0
def : Pat<
diff --git a/llvm/test/TableGen/GlobalISelEmitter-immarg-literal-pattern.td b/llvm/test/TableGen/GlobalISelEmitter-immarg-literal-pattern.td
index 6b4012eb736cb3..2c9bd7ae3a0497 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-immarg-literal-pattern.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-immarg-literal-pattern.td
@@ -23,7 +23,7 @@ def CAT1 : I<(outs GPR32:$dst), (ins GPR32:$src0), []>;
// Make sure there is no type check.
// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
-// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep),
+// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode4(Intrinsic::mytarget_sleep),
// GISEL-NEXT: // MIs[0] Operand 1
// GISEL-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, GIMT_Encode8(0),
def : Pat<
@@ -32,7 +32,7 @@ def : Pat<
>;
// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
-// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep),
+// GISEL: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode4(Intrinsic::mytarget_sleep),
// GISEL-NEXT: // MIs[0] Operand 1
// GISEL-NEXT: GIM_CheckLiteralInt, /*MI*/0, /*Op*/1, GIMT_Encode8(1),
def : Pat<
diff --git a/llvm/test/TableGen/GlobalISelEmitter-input-discard.td b/llvm/test/TableGen/GlobalISelEmitter-input-discard.td
index 202ff4a5758d7f..fbfa6707d1e7bc 100644
--- a/llvm/test/TableGen/GlobalISelEmitter-input-discard.td
+++ b/llvm/test/TableGen/GlobalISelEmitter-input-discard.td
@@ -10,7 +10,7 @@ def FOO : I<(outs GPR32:$dst), (ins GPR32Op:$src0, GPR32Op:$src1), []>;
// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
// GISEL-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/4,
-// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::tgt_foo),
+// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::tgt_foo),
// GISEL-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32,
// GISEL-NEXT: GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
// GISEL-NEXT: GIM_RootCheckType, /*Op*/3, /*Type*/GILLT_s32,
diff --git a/llvm/test/TableGen/GlobalISelEmitter.td b/llvm/test/TableGen/GlobalISelEmitter.td
index 7dbaf4390c0f70..fb6c0a3f5006e9 100644
--- a/llvm/test/TableGen/GlobalISelEmitter.td
+++ b/llvm/test/TableGen/GlobalISelEmitter.td
@@ -513,7 +513,7 @@ def : Pat<(frag GPR32:$src1, complex:$src2, complex:$src3),
// R00O-NEXT: GIM_Reject,
// R00O: // Label [[DEFAULT_NUM]]: @[[DEFAULT]]
// R00O-NEXT: GIM_Reject,
-// R00O-NEXT: }; // Size: 1832 bytes
+// R00O-NEXT: }; // Size: 1834 bytes
def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, GPR32:$src4),
[(set GPR32:$dst,
@@ -534,7 +534,7 @@ def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, G
// R01C-NEXT: GIM_Try, /*On fail goto*//*Label [[LABEL_NUM:[0-9]+]]*/ GIMT_Encode4([[LABEL:[0-9]+]]), // Rule ID 1 //
// R01C-NEXT: GIM_CheckNumOperands, /*MI*/0, /*Expected*/3,
//
-// R01O-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::mytarget_nop),
+// R01O-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::mytarget_nop),
// R01O-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32,
// R01O-NEXT: GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
// R01O-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
@@ -544,7 +544,7 @@ def INSNBOB : I<(outs GPR32:$dst), (ins GPR32:$src1, GPR32:$src2, GPR32:$src3, G
// R01N-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32,
// R01N-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
// R01N-NEXT: // MIs[0] Operand 1
-// R01N-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::mytarget_nop),
+// R01N-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::mytarget_nop),
// R01N-NEXT: // MIs[0] src1
// R01N-NEXT: GIM_RootCheckType, /*Op*/2, /*Type*/GILLT_s32,
//
@@ -1206,5 +1206,5 @@ def BR : I<(outs), (ins unknown:$target),
[(br bb:$target)]>;
// NOOPT-NEXT: GIM_Reject,
-// NOOPT-NEXT: }; // Size: 1459 bytes
+// NOOPT-NEXT: }; // Size: 1461 bytes
// NOOPT-NEXT: return MatchTable0;
diff --git a/llvm/test/TableGen/GlobalISelEmitterOverloadedPtr.td b/llvm/test/TableGen/GlobalISelEmitterOverloadedPtr.td
index 422edbba0e7a0f..c57da4ebe0dbdb 100644
--- a/llvm/test/TableGen/GlobalISelEmitterOverloadedPtr.td
+++ b/llvm/test/TableGen/GlobalISelEmitterOverloadedPtr.td
@@ -11,7 +11,7 @@ let TargetPrefix = "mytarget" in {
// Ensure that llvm_anyptr_ty on an intrinsic results in a
// GIM_CheckPointerToAny rather than a GIM_CheckType.
//
-// CHECK: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode2(Intrinsic::mytarget_anyptr),
+// CHECK: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/1, GIMT_Encode4(Intrinsic::mytarget_anyptr),
// CHECK-NEXT: GIM_RootCheckType, /*Op*/0, /*Type*/GILLT_s32,
// CHECK-NEXT: GIM_RootCheckRegBankForClass, /*Op*/0, /*RC*/GIMT_Encode2(MyTarget::GPR32RegClassID),
// CHECK-NEXT: // MIs[0] src
diff --git a/llvm/test/TableGen/immarg-predicated.td b/llvm/test/TableGen/immarg-predicated.td
index dcacb2f8f1de35..31f5bf49c7d3d9 100644
--- a/llvm/test/TableGen/immarg-predicated.td
+++ b/llvm/test/TableGen/immarg-predicated.td
@@ -9,7 +9,7 @@ def int_mytarget_sleep0 : Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>;
// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
// GISEL-NEXT: // MIs[0] Operand 0
-// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep0),
+// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode4(Intrinsic::mytarget_sleep0),
// GISEL-NEXT: // MIs[0] src
// GISEL-NEXT: GIM_CheckIsImm, /*MI*/0, /*Op*/1,
// GISEL-NEXT: GIM_CheckImmOperandPredicate, /*MI*/0, /*MO*/1, /*Predicate*/GIMT_Encode2(GICXXPred_I64_Predicate_tuimm9),
diff --git a/llvm/test/TableGen/immarg.td b/llvm/test/TableGen/immarg.td
index e5fd06ce6c083f..625b5d46a4ef61 100644
--- a/llvm/test/TableGen/immarg.td
+++ b/llvm/test/TableGen/immarg.td
@@ -10,7 +10,7 @@ def int_mytarget_sleep1 : Intrinsic<[], [llvm_i32_ty], [ImmArg<ArgIndex<0>>]>;
// GISEL: GIM_CheckOpcode, /*MI*/0, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
// GISEL-NEXT: // MIs[0] Operand 0
-// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode2(Intrinsic::mytarget_sleep0),
+// GISEL-NEXT: GIM_CheckIntrinsicID, /*MI*/0, /*Op*/0, GIMT_Encode4(Intrinsic::mytarget_sleep0),
// GISEL-NEXT: // MIs[0] src
// GISEL-NEXT: GIM_CheckIsImm, /*MI*/0, /*Op*/1,
// GISEL-NEXT: // (intrinsic_void {{[0-9]+}}:{ *:[iPTR] }, (timm:{ *:[i32] }):$src) => (SLEEP0 (timm:{ *:[i32] }):$src)
diff --git a/llvm/test/TableGen/intrinsic-overload-conflict.td b/llvm/test/TableGen/intrinsic-overload-conflict.td
index 84333119d41f53..788c49acde8abf 100644
--- a/llvm/test/TableGen/intrinsic-overload-conflict.td
+++ b/llvm/test/TableGen/intrinsic-overload-conflict.td
@@ -3,7 +3,7 @@
include "llvm/IR/Intrinsics.td"
-// CHECK: foo = 1,
+// CHECK: foo = {{.+}},
def int_foo : Intrinsic<[llvm_any_ty]>;
// No conflicts, since .bar is not a vaid mangled type.
diff --git a/llvm/test/TableGen/intrinsic-struct.td b/llvm/test/TableGen/intrinsic-struct.td
index 467fd9057c1833..10e98e589b8e43 100644
--- a/llvm/test/TableGen/intrinsic-struct.td
+++ b/llvm/test/TableGen/intrinsic-struct.td
@@ -7,7 +7,7 @@
include "llvm/IR/Intrinsics.td"
// Make sure we can return up to 9 values.
-// CHECK-ENUM: returns_9_results = {{[0-9]+}}, // llvm.returns.9.results
+// CHECK-ENUM: returns_9_results = {{.+}}, // llvm.returns.9.results
def int_returns_9_results : Intrinsic<
!listsplat(llvm_anyint_ty, 9),
[], [], "llvm.returns.9.results">;
diff --git a/llvm/test/TableGen/predicate-patfags.td b/llvm/test/TableGen/predicate-patfags.td
index 39133f324f305d..42dfd700f18a3a 100644
--- a/llvm/test/TableGen/predicate-patfags.td
+++ b/llvm/test/TableGen/predicate-patfags.td
@@ -55,12 +55,12 @@ def TGTmul24_oneuse : PatFrag<
// SBUILTIN: if (!SDValue(N, 0).hasOneUse()) return false;
// GISEL: GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
-// GISEL: GIM_CheckIntrinsicID, /*MI*/1, /*Op*/1, GIMT_Encode2(Intrinsic::tgt_mul24),
+// GISEL: GIM_CheckIntrinsicID, /*MI*/1, /*Op*/1, GIMT_Encode4(Intrinsic::tgt_mul24),
// GBUILTIN: GIM_CheckHasOneUse, /*MI*/1,
// GCUSTOM: GIM_CheckCxxInsnPredicate, /*MI*/1, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_TGTmul24_oneuse),
// GISEL: GIM_CheckOpcode, /*MI*/1, GIMT_Encode2(TargetOpcode::G_INTRINSIC_W_SIDE_EFFECTS),
-// GISEL: GIM_CheckIntrinsicID, /*MI*/1, /*Op*/1, GIMT_Encode2(Intrinsic::tgt_mul24),
+// GISEL: GIM_CheckIntrinsicID, /*MI*/1, /*Op*/1, GIMT_Encode4(Intrinsic::tgt_mul24),
// GBUILTIN: GIM_CheckHasOneUse, /*MI*/1,
// GCUSTOM: GIM_CheckCxxInsnPredicate, /*MI*/1, /*FnId*/GIMT_Encode2(GICXXPred_MI_Predicate_TGTmul24_oneuse),
diff --git a/llvm/unittests/IR/VPIntrinsicTest.cpp b/llvm/unittests/IR/VPIntrinsicTest.cpp
index d6ad7599ce4610..48a0dc86724d41 100644
--- a/llvm/unittests/IR/VPIntrinsicTest.cpp
+++ b/llvm/unittests/IR/VPIntrinsicTest.cpp
@@ -370,27 +370,28 @@ TEST_F(VPIntrinsicTest, IntrinsicIDRoundTrip) {
/// Check that going from intrinsic to VP intrinsic and back results in the same
/// intrinsic.
TEST_F(VPIntrinsicTest, IntrinsicToVPRoundTrip) {
+ using namespace Intrinsic;
bool IsFullTrip = false;
- Intrinsic::ID IntrinsicID = Intrinsic::not_intrinsic + 1;
- for (; IntrinsicID < Intrinsic::num_intrinsics; IntrinsicID++) {
- Intrinsic::ID VPID = VPIntrinsic::getForIntrinsic(IntrinsicID);
+ for (ID IID = GetNextValidIntrinsicID(not_intrinsic); IID != end_id;
+ IID = GetNextValidIntrinsicID(IID)) {
+ ID VPID = VPIntrinsic::getForIntrinsic(IID);
// No equivalent VP intrinsic available.
if (VPID == Intrinsic::not_intrinsic)
continue;
// Return itself if passed intrinsic ID is VP intrinsic.
- if (VPIntrinsic::isVPIntrinsic(IntrinsicID)) {
- ASSERT_EQ(IntrinsicID, VPID);
+ if (VPIntrinsic::isVPIntrinsic(IID)) {
+ ASSERT_EQ(IID, VPID);
continue;
}
- std::optional<Intrinsic::ID> RoundTripIntrinsicID =
+ std::optional<ID> RoundTripIntrinsicID =
VPIntrinsic::getFunctionalIntrinsicIDForVP(VPID);
// No equivalent non-predicated intrinsic available.
if (!RoundTripIntrinsicID)
continue;
- ASSERT_EQ(*RoundTripIntrinsicID, IntrinsicID);
+ ASSERT_EQ(*RoundTripIntrinsicID, IID);
IsFullTrip = true;
}
ASSERT_TRUE(IsFullTrip);
diff --git a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
index 18e0b8fd135bb0..1ecf46edf79090 100644
--- a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
+++ b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.cpp
@@ -13,8 +13,12 @@
#include "CodeGenIntrinsics.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/StringSet.h"
#include "llvm/ADT/Twine.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/IntrinsicID.h"
#include "llvm/TableGen/Error.h"
#include "llvm/TableGen/Record.h"
#include <algorithm>
@@ -44,35 +48,45 @@ CodeGenIntrinsicTable::CodeGenIntrinsicTable(const RecordKeeper &RC) {
CodeGenIntrinsicContext Ctx(RC);
ArrayRef<const Record *> Defs = RC.getAllDerivedDefinitions("Intrinsic");
- Intrinsics.reserve(Defs.size());
-
- for (const Record *Def : Defs)
- Intrinsics.emplace_back(CodeGenIntrinsic(Def, Ctx));
-
- llvm::sort(Intrinsics,
- [](const CodeGenIntrinsic &LHS, const CodeGenIntrinsic &RHS) {
- // Order target independent intrinsics before target dependent
- // ones.
- bool LHSHasTarget = !LHS.TargetPrefix.empty();
- bool RHSHasTarget = !RHS.TargetPrefix.empty();
-
- // To ensure deterministic sorted order when duplicates are
- // present, use record ID as a tie-breaker similar to
- // sortAndReportDuplicates in Utils.cpp.
- unsigned LhsID = LHS.TheDef->getID();
- unsigned RhsID = RHS.TheDef->getID();
-
- return std::tie(LHSHasTarget, LHS.Name, LhsID) <
- std::tie(RHSHasTarget, RHS.Name, RhsID);
- });
-
- Targets.push_back({"", 0, 0});
- for (size_t I = 0, E = Intrinsics.size(); I < E; ++I)
- if (Intrinsics[I].TargetPrefix != Targets.back().Name) {
- Targets.back().Count = I - Targets.back().Offset;
- Targets.push_back({Intrinsics[I].TargetPrefix, I, 0});
- }
- Targets.back().Count = Intrinsics.size() - Targets.back().Offset;
+
+ // Bucket each intrinsic into a per-target list of intrinsics. Use std::map
+ // so that the targets sorted by name when we iterate over the map.
+ std::map<StringRef, TargetSet> TargetMap;
+
+ // Always create entry for target independent intrinsics.
+ TargetMap[""].Name = "";
+
+ for (const Record *Def : Defs) {
+ CodeGenIntrinsic Int(Def, Ctx);
+ TargetMap[Int.TargetPrefix].Intrinsics.push_back(Int);
+ }
+
+ auto IntrinsicCmp = [](const CodeGenIntrinsic &LHS,
+ const CodeGenIntrinsic &RHS) {
+ // To ensure deterministic sorted order when duplicates are present, use
+ // record ID as a tie-breaker similar to sortAndReportDuplicates in
+ // Utils.cpp.
+ unsigned LhsID = LHS.TheDef->getID();
+ unsigned RhsID = RHS.TheDef->getID();
+
+ return std::tie(LHS.Name, LhsID) < std::tie(RHS.Name, RhsID);
+ };
+
+ // Sort intrinsics by name within each target, and collect all targets
+ // (alreay sorted by target name). Also assign linear index to all of them.
+ // Linear index 0 is reserved for not_intrinsic.
+ unsigned LinearIndex = 1;
+ unsigned TargetIndex = 0;
+ for (auto &[TargetName, TSet] : TargetMap) {
+ Targets.push_back(std::move(TSet));
+ TargetSet &T = Targets.back();
+ T.Name = TargetName;
+ T.TargetIndex = TargetIndex++;
+ llvm::sort(T.Intrinsics, IntrinsicCmp);
+ T.FirstLinearIndex = LinearIndex;
+ LinearIndex += T.Intrinsics.size();
+ }
+ NumIntrinsics = LinearIndex - 1;
CheckDuplicateIntrinsics();
CheckTargetIndependentIntrinsics();
@@ -87,20 +101,23 @@ void CodeGenIntrinsicTable::CheckDuplicateIntrinsics() const {
// there cannot be be duplicate as TableGen parser would have flagged that.
// However, if the name was specified in the intrinsic definition, then its
// possible to have duplicate names.
- auto I = std::adjacent_find(
- Intrinsics.begin(), Intrinsics.end(),
- [](const CodeGenIntrinsic &Int1, const CodeGenIntrinsic &Int2) {
- return Int1.Name == Int2.Name;
- });
- if (I == Intrinsics.end())
- return;
-
- // Found a duplicate intrinsics.
- const CodeGenIntrinsic &First = *I;
- const CodeGenIntrinsic &Second = *(I + 1);
- PrintError(Second.TheDef,
- Twine("Intrinsic `") + First.Name + "` is already defined");
- PrintFatalNote(First.TheDef, "Previous definition here");
+ for (const TargetSet &T : getAllTargets()) {
+ ArrayRef<CodeGenIntrinsic> Intrinsics = T.getIntrinsics();
+ auto I = std::adjacent_find(
+ Intrinsics.begin(), Intrinsics.end(),
+ [](const CodeGenIntrinsic &Int1, const CodeGenIntrinsic &Int2) {
+ return Int1.Name == Int2.Name;
+ });
+ if (I == Intrinsics.end())
+ return;
+
+ // Found a duplicate intrinsics.
+ const CodeGenIntrinsic &First = *I;
+ const CodeGenIntrinsic &Second = *(I + 1);
+ PrintError(Second.TheDef,
+ Twine("Intrinsic `") + First.Name + "` is already defined");
+ PrintFatalNote(First.TheDef, "Previous definition here");
+ }
}
// For target independent intrinsics, check that their second dotted component
@@ -111,8 +128,7 @@ void CodeGenIntrinsicTable::CheckTargetIndependentIntrinsics() const {
TargetNames.insert(Target.Name);
// Set of target independent intrinsics.
- const auto &Set = Targets[0];
- for (const auto &Int : ArrayRef(&Intrinsics[Set.Offset], Set.Count)) {
+ for (const auto &Int : Targets[0].getIntrinsics()) {
StringRef Name = Int.Name;
StringRef Prefix = Name.drop_front(5).split('.').first;
if (!TargetNames.contains(Prefix))
@@ -204,7 +220,7 @@ static bool doesSuffixLookLikeMangledType(StringRef Suffix) {
void CodeGenIntrinsicTable::CheckOverloadSuffixConflicts() const {
for (const TargetSet &Set : Targets) {
const CodeGenIntrinsic *Overloaded = nullptr;
- for (const CodeGenIntrinsic &Int : (*this)[Set]) {
+ for (const CodeGenIntrinsic &Int : Set.getIntrinsics()) {
// If we do not have an overloaded intrinsic to check against, nothing
// to do except potentially identifying this as a candidate for checking
// against in future iteration.
@@ -248,6 +264,57 @@ void CodeGenIntrinsicTable::CheckOverloadSuffixConflicts() const {
}
}
+// Enumerate all intrinsics and call back the visitor function with the
+// intrinsic and its index.
+void CodeGenIntrinsicTable::enumerateIntrinsics(
+ CodeGenIntrinsicTable::IntrinsicVisitor Visitor) const {
+ for (const TargetSet &T : getAllTargets()) {
+ unsigned Idx = T.FirstLinearIndex - 1;
+ for (const CodeGenIntrinsic &Int : T.getIntrinsics())
+ Visitor(Idx++, Int);
+ }
+}
+
+unsigned CodeGenIntrinsicTable::getIntrinsicID(const Record *Def) const {
+ for (const auto &T : getAllTargets())
+ for (const auto &[IntrinsicIndex, Int] : enumerate(T.getIntrinsics()))
+ if (Int.TheDef == Def)
+ return Intrinsic::encodeIntrinsicID(T.TargetIndex, IntrinsicIndex);
+ errs() << "Cannot find intrinsic for record: " << Def->getName() << '\n';
+ llvm_unreachable("Unknown intrinsic!");
+}
+
+const CodeGenIntrinsic &
+CodeGenIntrinsicTable::getIntrinsic(const Record *Def) const {
+ for (const TargetSet &T : getAllTargets())
+ for (const CodeGenIntrinsic &Int : T.getIntrinsics())
+ if (Int.TheDef == Def)
+ return Int;
+ errs() << "Cannot find intrinsic for record: " << Def->getName() << '\n';
+ llvm_unreachable("Unknown intrinsic!");
+}
+
+const CodeGenIntrinsic &CodeGenIntrinsicTable::getIntrinsic(unsigned ID) const {
+ auto getIntrinsicImpl = [&](unsigned ID) -> const CodeGenIntrinsic * {
+ auto Decoded = Intrinsic::decodeIntrinsicIDNoFail(ID);
+ if (!Decoded)
+ return nullptr;
+ unsigned TargetIndex = Decoded->first;
+ unsigned IntrinsicIndex = Decoded->second;
+ if (TargetIndex >= getAllTargets().size())
+ return nullptr;
+ const TargetSet &T = getAllTargets()[TargetIndex];
+ ArrayRef<CodeGenIntrinsic> Ints = T.getIntrinsics();
+ if (IntrinsicIndex >= Ints.size())
+ return nullptr;
+ return &Ints[IntrinsicIndex];
+ };
+ if (const CodeGenIntrinsic *Result = getIntrinsicImpl(ID))
+ return *Result;
+ errs() << "Cannot find intrinsic for ID: 0x" << Twine::utohexstr(ID) << '\n';
+ llvm_unreachable("Unknown intrinsic!");
+}
+
const CodeGenIntrinsic &CodeGenIntrinsicMap::operator[](const Record *Record) {
if (!Record->isSubClassOf("Intrinsic"))
PrintFatalError("Intrinsic defs should be subclass of 'Intrinsic' class");
diff --git a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
index 8428d09a94009e..bd77d666ab45a4 100644
--- a/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
+++ b/llvm/utils/TableGen/Basic/CodeGenIntrinsics.h
@@ -16,6 +16,7 @@
#include "SDNodeProperties.h"
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/DenseMap.h"
+#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Support/ModRef.h"
#include <string>
@@ -172,31 +173,35 @@ class CodeGenIntrinsicTable {
public:
struct TargetSet {
StringRef Name;
- size_t Offset;
- size_t Count;
+ unsigned TargetIndex;
+ std::vector<CodeGenIntrinsic> Intrinsics;
+ size_t FirstLinearIndex;
+
+ ArrayRef<CodeGenIntrinsic> getIntrinsics() const { return Intrinsics; }
};
explicit CodeGenIntrinsicTable(const RecordKeeper &RC);
- bool empty() const { return Intrinsics.empty(); }
- size_t size() const { return Intrinsics.size(); }
- auto begin() const { return Intrinsics.begin(); }
- auto end() const { return Intrinsics.end(); }
- const CodeGenIntrinsic &operator[](size_t Pos) const {
- return Intrinsics[Pos];
- }
- ArrayRef<CodeGenIntrinsic> operator[](const TargetSet &Set) const {
- return ArrayRef(&Intrinsics[Set.Offset], Set.Count);
- }
- ArrayRef<TargetSet> getTargets() const { return Targets; }
+ ArrayRef<TargetSet> getAllTargets() const { return Targets; }
+
+ using IntrinsicVisitor =
+ function_ref<void(unsigned Idx, const CodeGenIntrinsic &Int)>;
+ void enumerateIntrinsics(IntrinsicVisitor Visitor) const;
+
+ unsigned getNumIntrinsics() const { return NumIntrinsics; }
+
+ // Find the intrinsic corresponding to the given record.
+ unsigned getIntrinsicID(const Record *Def) const;
+ const CodeGenIntrinsic &getIntrinsic(const Record *Def) const;
+ const CodeGenIntrinsic &getIntrinsic(unsigned ID) const;
private:
void CheckDuplicateIntrinsics() const;
void CheckTargetIndependentIntrinsics() const;
void CheckOverloadSuffixConflicts() const;
- std::vector<CodeGenIntrinsic> Intrinsics;
std::vector<TargetSet> Targets;
+ unsigned NumIntrinsics;
};
// This class builds `CodeGenIntrinsic` on demand for a given Def.
diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
index d2228c902a56b4..7b75600dcb3d02 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.cpp
@@ -2985,7 +2985,7 @@ TreePatternNodePtr TreePattern::ParseTreePattern(const Init *TheInit,
// convert the intrinsic name to a number.
if (Operator->isSubClassOf("Intrinsic")) {
const CodeGenIntrinsic &Int = getDAGPatterns().getIntrinsic(Operator);
- unsigned IID = getDAGPatterns().getIntrinsicID(Operator) + 1;
+ unsigned IID = getDAGPatterns().getIntrinsicID(Operator);
// If this intrinsic returns void, it must have side-effects and thus a
// chain.
diff --git a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
index f85753ff5ac80b..0cfb07a07f54c4 100644
--- a/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
+++ b/llvm/utils/TableGen/Common/CodeGenDAGPatterns.h
@@ -1158,23 +1158,15 @@ class CodeGenDAGPatterns {
}
const CodeGenIntrinsic &getIntrinsic(const Record *R) const {
- for (unsigned i = 0, e = Intrinsics.size(); i != e; ++i)
- if (Intrinsics[i].TheDef == R)
- return Intrinsics[i];
- llvm_unreachable("Unknown intrinsic!");
+ return Intrinsics.getIntrinsic(R);
}
const CodeGenIntrinsic &getIntrinsicInfo(unsigned IID) const {
- if (IID - 1 < Intrinsics.size())
- return Intrinsics[IID - 1];
- llvm_unreachable("Bad intrinsic ID!");
+ return Intrinsics.getIntrinsic(IID);
}
unsigned getIntrinsicID(const Record *R) const {
- for (unsigned i = 0, e = Intrinsics.size(); i != e; ++i)
- if (Intrinsics[i].TheDef == R)
- return i;
- llvm_unreachable("Unknown intrinsic!");
+ return Intrinsics.getIntrinsicID(R);
}
const DAGDefaultOperand &getDefaultOperand(const Record *R) const {
diff --git a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
index 5de5dd894f84ec..4bb8ab2043389f 100644
--- a/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
+++ b/llvm/utils/TableGen/Common/GlobalISel/GlobalISelMatchTable.cpp
@@ -1305,7 +1305,7 @@ void IntrinsicIDOperandMatcher::emitPredicateOpcodes(MatchTable &Table,
Table << MatchTable::Opcode("GIM_CheckIntrinsicID")
<< MatchTable::Comment("MI") << MatchTable::ULEB128Value(InsnVarID)
<< MatchTable::Comment("Op") << MatchTable::ULEB128Value(OpIdx)
- << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName.str())
+ << MatchTable::NamedValue(4, "Intrinsic::" + II->EnumName.str())
<< MatchTable::LineBreak;
}
@@ -2095,7 +2095,7 @@ void IntrinsicIDRenderer::emitRenderOpcodes(MatchTable &Table,
RuleMatcher &Rule) const {
Table << MatchTable::Opcode("GIR_AddIntrinsicID") << MatchTable::Comment("MI")
<< MatchTable::ULEB128Value(InsnID)
- << MatchTable::NamedValue(2, "Intrinsic::" + II->EnumName.str())
+ << MatchTable::NamedValue(4, "Intrinsic::" + II->EnumName.str())
<< MatchTable::LineBreak;
}
diff --git a/llvm/utils/TableGen/IntrinsicEmitter.cpp b/llvm/utils/TableGen/IntrinsicEmitter.cpp
index 1968e7eac21e33..82a9492a62069a 100644
--- a/llvm/utils/TableGen/IntrinsicEmitter.cpp
+++ b/llvm/utils/TableGen/IntrinsicEmitter.cpp
@@ -19,6 +19,7 @@
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/FormatVariadic.h"
+#include "llvm/Support/IntrinsicID.h"
#include "llvm/Support/ModRef.h"
#include "llvm/Support/raw_ostream.h"
#include "llvm/TableGen/Error.h"
@@ -36,6 +37,7 @@
#include <utility>
#include <vector>
using namespace llvm;
+using TargetSet = CodeGenIntrinsicTable::TargetSet;
static cl::OptionCategory GenIntrinsicCat("Options for -gen-intrinsic-enums");
static cl::opt<std::string>
@@ -88,8 +90,10 @@ void IntrinsicEmitter::run(raw_ostream &OS, bool Enums) {
// Emit the enum information.
EmitEnumInfo(Ints, OS);
- // Emit ArgKind for Intrinsics.h.
- EmitArgKind(OS);
+ if (IntrinsicPrefix.empty()) {
+ // Emit ArgKind for Intrinsics.h.
+ EmitArgKind(OS);
+ }
} else {
// Emit IIT_Info constants.
EmitIITInfo(OS);
@@ -122,9 +126,8 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
// Find the TargetSet for which to generate enums. There will be an initial
// set with an empty target prefix which will include target independent
// intrinsics like dbg.value.
- using TargetSet = CodeGenIntrinsicTable::TargetSet;
const TargetSet *Set = nullptr;
- for (const auto &Target : Ints.getTargets()) {
+ for (const auto &[Idx, Target] : enumerate(Ints.getAllTargets())) {
if (Target.Name == IntrinsicPrefix) {
Set = &Target;
break;
@@ -132,7 +135,7 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
}
if (!Set) {
// The first entry is for target independent intrinsics, so drop it.
- auto KnowTargets = Ints.getTargets().drop_front();
+ auto KnowTargets = Ints.getAllTargets().drop_front();
PrintFatalError([KnowTargets](raw_ostream &OS) {
OS << "tried to generate intrinsics for unknown target "
<< IntrinsicPrefix << "\nKnown targets are: ";
@@ -143,10 +146,10 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
}
// Generate a complete header for target specific intrinsics.
+ std::string UpperPrefix = StringRef(IntrinsicPrefix).upper();
if (IntrinsicPrefix.empty()) {
OS << "#ifdef GET_INTRINSIC_ENUM_VALUES\n";
} else {
- std::string UpperPrefix = StringRef(IntrinsicPrefix).upper();
OS << formatv("#ifndef LLVM_IR_INTRINSIC_{}_ENUMS_H\n", UpperPrefix);
OS << formatv("#define LLVM_IR_INTRINSIC_{}_ENUMS_H\n", UpperPrefix);
OS << "namespace llvm::Intrinsic {\n";
@@ -155,13 +158,13 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
OS << "// Enum values for intrinsics.\n";
bool First = true;
- for (const auto &Int : Ints[*Set]) {
+ for (const auto &Int : Set->getIntrinsics()) {
OS << " " << Int.EnumName;
// Assign a value to the first intrinsic in this target set so that all
// intrinsic ids are distinct.
if (First) {
- OS << " = " << Set->Offset + 1;
+ OS << formatv(" = ({} << 16) + 1", Set->TargetIndex);
First = false;
}
@@ -171,22 +174,23 @@ void IntrinsicEmitter::EmitEnumInfo(const CodeGenIntrinsicTable &Ints,
OS << formatv(" // {}\n", Int.Name);
}
- // Emit num_intrinsics into the target neutral enum.
if (IntrinsicPrefix.empty()) {
- OS << formatv(" num_intrinsics = {}\n", Ints.size() + 1);
+ const TargetSet &LastTarget = Ints.getAllTargets().back();
+ const size_t LastCount = LastTarget.getIntrinsics().size();
+ OS << formatv(" last_valid_intrinsic_id = ({} << 16) + {} + 1,\n",
+ LastTarget.TargetIndex, LastCount - 1);
OS << "#endif\n\n";
} else {
- OS << R"(}; // enum
+ OS << formatv(R"(}; // enum {}Intrinsics
} // namespace llvm::Intrinsic
#endif
-)";
+)",
+ UpperPrefix);
}
}
void IntrinsicEmitter::EmitArgKind(raw_ostream &OS) {
- if (!IntrinsicPrefix.empty())
- return;
OS << "// llvm::Intrinsic::IITDescriptor::ArgKind.\n";
OS << "#ifdef GET_INTRINSIC_ARGKIND\n";
if (const auto RecArgKind = Records.getDef("ArgKind")) {
@@ -225,14 +229,15 @@ void IntrinsicEmitter::EmitTargetInfo(const CodeGenIntrinsicTable &Ints,
#ifdef GET_INTRINSIC_TARGET_DATA
struct IntrinsicTargetInfo {
StringLiteral Name;
- size_t Offset;
- size_t Count;
+ unsigned Count;
+ unsigned FirstLinearIndex;
};
static constexpr IntrinsicTargetInfo TargetInfos[] = {
)";
- for (const auto [Name, Offset, Count] : Ints.getTargets())
- OS << formatv(" {{\"{}\", {}, {}},\n", Name, Offset, Count);
- OS << R"(};
+ for (const TargetSet &T : Ints.getAllTargets())
+ OS << formatv(" {{\"{}\", {}, {}},\n", T.Name, T.Intrinsics.size(),
+ T.FirstLinearIndex);
+ OS << R"(};
#endif
)";
@@ -244,30 +249,30 @@ void IntrinsicEmitter::EmitIntrinsicToNameTable(
#ifdef GET_INTRINSIC_NAME_TABLE
// Note that entry #0 is the invalid intrinsic!
)";
- for (const auto &Int : Ints)
+ Ints.enumerateIntrinsics([&OS](unsigned, const CodeGenIntrinsic &Int) {
OS << " \"" << Int.Name << "\",\n";
+ });
OS << "#endif\n\n";
}
void IntrinsicEmitter::EmitIntrinsicToOverloadTable(
const CodeGenIntrinsicTable &Ints, raw_ostream &OS) {
- OS << R"(// Intrinsic ID to overload bitset.
+ OS << formatv(R"(// Intrinsic ID to overload bitset.
#ifdef GET_INTRINSIC_OVERLOAD_TABLE
-static constexpr uint8_t OTable[] = {
- 0
- )";
- for (auto [I, Int] : enumerate(Ints)) {
- // Add one to the index so we emit a null bit for the invalid #0 intrinsic.
- size_t Idx = I + 1;
-
- if (Idx % 8 == 0)
+static constexpr uint8_t OTable[] = {{
+ 0)");
+ Ints.enumerateIntrinsics([&OS](unsigned Idx, const CodeGenIntrinsic &Int) {
+ // Add one to the index so we emit a null bit for the invalid #0
+ // intrinsic.
+ unsigned LinearIndex = Idx + 1;
+ if (LinearIndex % 8 == 0)
OS << ",\n 0";
if (Int.isOverloaded)
- OS << " | (1<<" << Idx % 8 << ')';
- }
+ OS << " | (1<<" << LinearIndex % 8 << ')';
+ });
OS << "\n};\n\n";
// OTable contains a true bit at the position if the intrinsic is overloaded.
- OS << "return (OTable[id/8] & (1 << (id%8))) != 0;\n";
+ OS << "return (ArrayRef(OTable)[Idx/8] & (1 << (Idx%8))) != 0;\n";
OS << "#endif\n\n";
}
@@ -317,10 +322,10 @@ void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,
std::vector<FixedEncodingTy> FixedEncodings;
SequenceToOffsetTable<TypeSigTy> LongEncodingTable;
- FixedEncodings.reserve(Ints.size());
+ FixedEncodings.reserve(Ints.getNumIntrinsics());
// Compute the unique argument type info.
- for (const CodeGenIntrinsic &Int : Ints) {
+ Ints.enumerateIntrinsics([&](unsigned, const CodeGenIntrinsic &Int) {
// Get the signature for the intrinsic.
TypeSigTy TypeSig = ComputeTypeSignature(Int);
@@ -328,14 +333,14 @@ void IntrinsicEmitter::EmitGenerator(const CodeGenIntrinsicTable &Ints,
std::optional<uint32_t> Result = encodePacked(TypeSig);
if (Result && (*Result & Mask) == Result) {
FixedEncodings.push_back(static_cast<FixedEncodingTy>(*Result));
- continue;
+ return;
}
LongEncodingTable.add(TypeSig);
// This is a placehold that we'll replace after the table is laid out.
FixedEncodings.push_back(static_cast<FixedEncodingTy>(~0U));
- }
+ });
LongEncodingTable.layout();
@@ -346,24 +351,25 @@ static constexpr {} IIT_Table[] = {{
FixedEncodingTypeName);
unsigned MaxOffset = 0;
- for (auto [Idx, FixedEncoding, Int] : enumerate(FixedEncodings, Ints)) {
+ Ints.enumerateIntrinsics([&](unsigned Idx, const CodeGenIntrinsic &Int) {
+ const FixedEncodingTy &FixedEncoding = FixedEncodings[Idx];
if ((Idx & 7) == 7)
OS << "\n ";
// If the entry fit in the table, just emit it.
if ((FixedEncoding & Mask) == FixedEncoding) {
OS << "0x" << Twine::utohexstr(FixedEncoding) << ", ";
- continue;
+ return;
}
TypeSigTy TypeSig = ComputeTypeSignature(Int);
unsigned Offset = LongEncodingTable.get(TypeSig);
MaxOffset = std::max(MaxOffset, Offset);
- // Otherwise, emit the offset into the long encoding table. We emit it this
- // way so that it is easier to read the offset in the .def file.
+ // Otherwise, emit the offset into the long encoding table. We emit it
+ // this way so that it is easier to read the offset in the .def file.
OS << formatv("(1U<<{}) | {}, ", MSBPostion, Offset);
- }
+ });
OS << "0\n};\n\n";
@@ -486,7 +492,7 @@ static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID) {
// Compute unique argument attribute sets.
std::map<SmallVector<CodeGenIntrinsic::ArgAttribute, 0>, unsigned>
UniqArgAttributes;
- for (const CodeGenIntrinsic &Int : Ints) {
+ Ints.enumerateIntrinsics([&](unsigned, const CodeGenIntrinsic &Int) {
for (auto &Attrs : Int.ArgumentAttributes) {
if (Attrs.empty())
continue;
@@ -513,7 +519,7 @@ static AttributeSet getIntrinsicArgAttributeSet(LLVMContext &C, unsigned ID) {
}
OS << " });";
}
- }
+ });
OS << R"(
}
} // getIntrinsicArgAttributeSet
@@ -527,12 +533,12 @@ static AttributeSet getIntrinsicFnAttributeSet(LLVMContext &C, unsigned ID) {
switch (ID) {
default: llvm_unreachable("Invalid attribute set number");)";
- for (const CodeGenIntrinsic &Int : Ints) {
+ Ints.enumerateIntrinsics([&](unsigned, const CodeGenIntrinsic &Int) {
if (!hasFnAttributes(Int))
- continue;
+ return;
unsigned ID = UniqFnAttributes.size();
if (!UniqFnAttributes.try_emplace(&Int, ID).second)
- continue;
+ return;
OS << formatv(R"(
case {}:
return AttributeSet::get(C, {{
@@ -574,7 +580,7 @@ static AttributeSet getIntrinsicFnAttributeSet(LLVMContext &C, unsigned ID) {
ME.toIntValue());
}
OS << " });";
- }
+ });
OS << R"(
}
} // getIntrinsicFnAttributeSet
@@ -587,10 +593,10 @@ AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) {
// arguments or not.
std::map<const CodeGenIntrinsic *, unsigned, AttributeComparator>
UniqAttributes;
- for (const CodeGenIntrinsic &Int : Ints) {
+ Ints.enumerateIntrinsics([&](unsigned, const CodeGenIntrinsic &Int) {
unsigned ID = UniqAttributes.size();
UniqAttributes.try_emplace(&Int, ID);
- }
+ });
// Assign a 16-bit packed ID for each intrinsic. The lower 8-bits will be its
// "argument attribute ID" (index in UniqAttributes) and upper 8 bits will be
@@ -603,18 +609,20 @@ AttributeList Intrinsic::getAttributes(LLVMContext &C, ID id) {
// Emit an array of AttributeList. Most intrinsics will have at least one
// entry, for the function itself (index ~1), which is usually nounwind.
OS << " static constexpr uint16_t IntrinsicsToAttributesMap[] = {";
- for (const CodeGenIntrinsic &Int : Ints) {
+ Ints.enumerateIntrinsics([&](unsigned, const CodeGenIntrinsic &Int) {
uint16_t FnAttrIndex = hasFnAttributes(Int) ? UniqFnAttributes[&Int] : 0;
OS << formatv("\n {} << 8 | {}, // {}", FnAttrIndex,
UniqAttributes[&Int], Int.Name);
- }
+ });
OS << formatv(R"(
};
- if (id == 0)
+ // First remap the Intrinsic::ID to intrinsic index.
+ unsigned Idx = getLinearIndex(id);
+ if (Idx == 0)
return AttributeList();
- uint16_t PackedID = IntrinsicsToAttributesMap[id - 1];
+ uint16_t PackedID = IntrinsicsToAttributesMap[Idx - 1];
uint8_t FnAttrID = PackedID >> 8;
switch(PackedID & 0xFF) {{
default: llvm_unreachable("Invalid attribute number");
@@ -677,10 +685,10 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
std::pair<std::map<StringRef, StringRef>, std::optional<StringRef>>;
std::map<StringRef, BIMEntryTy> BuiltinMap;
- for (const CodeGenIntrinsic &Int : Ints) {
+ Ints.enumerateIntrinsics([&](unsigned, const CodeGenIntrinsic &Int) {
StringRef BuiltinName = IsClang ? Int.ClangBuiltinName : Int.MSBuiltinName;
if (BuiltinName.empty())
- continue;
+ return;
// Get the map for this target prefix.
auto &[Map, CommonPrefix] = BuiltinMap[Int.TargetPrefix];
@@ -693,14 +701,14 @@ void IntrinsicEmitter::EmitIntrinsicToBuiltinMap(
if (!CommonPrefix) {
// For the first builtin for this target, initialize the common prefix.
CommonPrefix = BuiltinName;
- continue;
+ return;
}
// Update the common prefix. Note that this assumes that `take_front` will
// never set the `Data` pointer in CommonPrefix to nullptr.
const char *Mismatch = mismatch(*CommonPrefix, BuiltinName).first;
*CommonPrefix = CommonPrefix->take_front(Mismatch - CommonPrefix->begin());
- }
+ });
// Populate the string table with the names of all the builtins after
// removing this common prefix.
@@ -728,7 +736,7 @@ Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix,
if (BuiltinMap.empty()) {
OS << formatv(R"(
return not_intrinsic;
- }
+}
#endif // GET_LLVM_INTRINSIC_FOR_{}_BUILTIN
)",
UpperCompilerName);
@@ -753,11 +761,13 @@ Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix,
// Emit a per target table of bultin names.
bool HasTargetIndependentBuiltins = false;
+ bool HasTargetDependentBuiltins = false;
StringRef TargetIndepndentCommonPrefix;
for (const auto &[TargetPrefix, Entry] : BuiltinMap) {
const auto &[Map, CommonPrefix] = Entry;
if (!TargetPrefix.empty()) {
OS << formatv(" // Builtins for {0}.\n", TargetPrefix);
+ HasTargetDependentBuiltins = true;
} else {
OS << " // Target independent builtins.\n";
HasTargetIndependentBuiltins = true;
@@ -775,30 +785,32 @@ Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix,
OS << formatv(" }; // {}Names\n\n", TargetPrefix);
}
- // After emitting the builtin tables for all targets, emit a lookup table for
- // all targets. We will use binary search, similar to the table for builtin
- // names to lookup into this table.
- OS << R"(
- struct TargetEntry {
- StringLiteral TargetPrefix;
- ArrayRef<BuiltinEntry> Names;
- StringLiteral CommonPrefix;
- bool operator<(StringRef RHS) const {
- return TargetPrefix < RHS;
+ if (HasTargetDependentBuiltins) {
+ // After emitting the builtin tables for all targets, emit a lookup table
+ // for all targets. We will use binary search, similar to the table for
+ // builtin names to lookup into this table.
+ OS << R"(
+ struct TargetEntry {
+ StringLiteral TargetPrefix;
+ ArrayRef<BuiltinEntry> Names;
+ StringLiteral CommonPrefix;
+ bool operator<(StringRef RHS) const {
+ return TargetPrefix < RHS;
+ };
};
- };
- static constexpr TargetEntry TargetTable[] = {
-)";
+ static constexpr TargetEntry TargetTable[] = {
+ )";
- for (const auto &[TargetPrefix, Entry] : BuiltinMap) {
- const auto &[Map, CommonPrefix] = Entry;
- if (TargetPrefix.empty())
- continue;
- OS << formatv(R"( {{"{0}", {0}Names, "{1}"},)", TargetPrefix,
- CommonPrefix)
- << "\n";
+ for (const auto &[TargetPrefix, Entry] : BuiltinMap) {
+ const auto &[Map, CommonPrefix] = Entry;
+ if (TargetPrefix.empty())
+ continue;
+ OS << formatv(R"( {{"{0}", {0}Names, "{1}"},)", TargetPrefix,
+ CommonPrefix)
+ << "\n";
+ }
+ OS << " };\n";
}
- OS << " };\n";
// Now for the actual lookup, first check the target independent table if
// we emitted one.
@@ -817,8 +829,10 @@ Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix,
TargetIndepndentCommonPrefix);
}
- // If a target independent builtin was not found, lookup the target specific.
- OS << formatv(R"(
+ if (HasTargetDependentBuiltins) {
+ // If a target independent builtin was not found, lookup the target
+ // specific.
+ OS << R"(
auto TI = lower_bound(TargetTable, TargetPrefix);
if (TI == std::end(TargetTable) || TI->TargetPrefix != TargetPrefix)
return not_intrinsic;
@@ -830,6 +844,12 @@ Intrinsic::getIntrinsicFor{}Builtin(StringRef TargetPrefix,
if (II == std::end(TI->Names) || II->getName() != BuiltinName)
return not_intrinsic;
return II->IntrinsicID;
+)";
+ } else {
+ OS << " return not_intrinsic;";
+ }
+
+ OS << formatv(R"(
}
#endif // GET_LLVM_INTRINSIC_FOR_{}_BUILTIN
More information about the llvm-commits
mailing list