[llvm] [RISCV][GlobalISel] Legalize and select G_PREFETCH (PR #215466)
Kane Wang via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 00:46:08 PDT 2026
https://github.com/ReVe1uv updated https://github.com/llvm/llvm-project/pull/215466
>From 3891bcf5a73c72301b873cc25f4a60524b92914b Mon Sep 17 00:00:00 2001
From: Kane Wang <wangqiang1 at kylinos.cn>
Date: Thu, 13 Aug 2026 10:55:08 +0800
Subject: [PATCH 1/4] [GlobalISel] Use untyped imm operands for G_PREFETCH
hints
G_PREFETCH's rw/locality/cachetype are inline immediates emitted via
addImm. Declaring them i32imm makes the importer emit GIM_CheckType,
which rejects inline-imm operands (no LLT). Use distinct untyped_imm
types so it emits GIM_CheckLiteralInt instead, allowing targets to
select G_PREFETCH via imported patterns.
Patch authored by Craig Topper <craig.topper at sifive.com>.
---
llvm/include/llvm/MC/MCInstrDesc.h | 6 ++++--
llvm/include/llvm/Target/GenericOpcodes.td | 2 +-
llvm/include/llvm/Target/Target.td | 6 ++++--
.../AArch64/GlobalISel/legalizer-info-validation.mir | 2 +-
4 files changed, 10 insertions(+), 6 deletions(-)
diff --git a/llvm/include/llvm/MC/MCInstrDesc.h b/llvm/include/llvm/MC/MCInstrDesc.h
index da65487ce9c65..59325414f8022 100644
--- a/llvm/include/llvm/MC/MCInstrDesc.h
+++ b/llvm/include/llvm/MC/MCInstrDesc.h
@@ -74,9 +74,11 @@ enum OperandType {
OPERAND_FIRST_GENERIC_IMM = 12,
OPERAND_GENERIC_IMM_0 = 12,
- OPERAND_LAST_GENERIC_IMM = 12,
+ OPERAND_GENERIC_IMM_1 = 13,
+ OPERAND_GENERIC_IMM_2 = 14,
+ OPERAND_LAST_GENERIC_IMM = 14,
- OPERAND_FIRST_TARGET = 13,
+ OPERAND_FIRST_TARGET = 15,
};
} // namespace MCOI
diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index b7b216b552a04..55572f230b964 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -1506,7 +1506,7 @@ def G_FENCE : GenericInstruction {
// Generic opcode equivalent to the llvm.prefetch intrinsic.
def G_PREFETCH : GenericInstruction {
let OutOperandList = (outs);
- let InOperandList = (ins ptype0:$address, i32imm:$rw, i32imm:$locality, i32imm:$cachetype);
+ let InOperandList = (ins ptype0:$address, untyped_imm_0:$rw, untyped_imm_1:$locality, untyped_imm_2:$cachetype);
let hasSideEffects = true;
let mayLoad = true;
let mayStore = true;
diff --git a/llvm/include/llvm/Target/Target.td b/llvm/include/llvm/Target/Target.td
index 7d1e7a28b7b6e..3acb2531a5466 100644
--- a/llvm/include/llvm/Target/Target.td
+++ b/llvm/include/llvm/Target/Target.td
@@ -1243,8 +1243,10 @@ let IsPointer = true in {
// untyped_imm is for operands where isImm() will be true. It currently has no
// special behaviour and is only used for clarity.
-def untyped_imm_0 : TypedOperand<"OPERAND_GENERIC_IMM_0"> {
- let IsImmediate = true;
+let IsImmediate = true in {
+ def untyped_imm_0 : TypedOperand<"OPERAND_GENERIC_IMM_0">;
+ def untyped_imm_1 : TypedOperand<"OPERAND_GENERIC_IMM_1">;
+ def untyped_imm_2 : TypedOperand<"OPERAND_GENERIC_IMM_2">;
}
/// zero_reg definition - Special node to stand for the zero register.
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 4a88a30d56e01..3353f0e036cda 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -321,7 +321,7 @@
# DEBUG-NEXT: G_FENCE (opcode {{[0-9]+}}): 0 type indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
-# DEBUG-NEXT: G_PREFETCH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
+# DEBUG-NEXT: G_PREFETCH (opcode {{[0-9]+}}): 1 type index, 3 imm indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_BRCOND (opcode {{[0-9]+}}): 1 type index, 0 imm indices
>From 6b5c94b23aaf9cde428813c2bdb35fd0c5d45f46 Mon Sep 17 00:00:00 2001
From: Kane Wang <wangqiang1 at kylinos.cn>
Date: Thu, 13 Aug 2026 11:27:58 +0800
Subject: [PATCH 2/4] [RISCV][GlobalISel] Legalize and select G_PREFETCH
GlobalISel aborted on llvm.prefetch ("unable to legalize G_PREFETCH")
while SDAG lowers it to prefetch.r/w/i. Add a legalizer rule marking
G_PREFETCH legal for p0 addresses and select it via the existing SDAG
patterns: add a GIComplexPatternEquiv for AddrRegImmLsb00000 so the
patterns import and call a selectAddrRegImmLsb00000 matcher that folds
a simm12_lsb00000 offset.
---
.../RISCV/GISel/RISCVInstructionSelector.cpp | 37 ++++++
.../Target/RISCV/GISel/RISCVLegalizerInfo.cpp | 2 +
llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td | 3 +
.../GlobalISel/legalizer-info-validation.mir | 6 +-
.../test/CodeGen/RISCV/GlobalISel/prefetch.ll | 116 ++++++++++++++++++
5 files changed, 161 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/CodeGen/RISCV/GlobalISel/prefetch.ll
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index 16af89c715861..921feb7393374 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -107,6 +107,7 @@ class RISCVInstructionSelector : public InstructionSelector {
return selectShiftMask(Root, 32);
}
ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
+ ComplexRendererFns selectAddrRegImmLsb00000(MachineOperand &Root) const;
ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
template <unsigned Bits>
@@ -593,6 +594,42 @@ RISCVInstructionSelector::selectAddrRegImm(MachineOperand &Root) const {
[=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
}
+InstructionSelector::ComplexRendererFns
+RISCVInstructionSelector::selectAddrRegImmLsb00000(MachineOperand &Root) const {
+ if (!Root.isReg())
+ return std::nullopt;
+
+ MachineInstr *RootDef = MRI->getVRegDef(Root.getReg());
+ if (RootDef->getOpcode() == TargetOpcode::G_FRAME_INDEX) {
+ return {{
+ [=](MachineInstrBuilder &MIB) { MIB.add(RootDef->getOperand(1)); },
+ [=](MachineInstrBuilder &MIB) { MIB.addImm(0); },
+ }};
+ }
+
+ if (isBaseWithConstantOffset(Root, *MRI)) {
+ MachineOperand &LHS = RootDef->getOperand(1);
+ MachineOperand &RHS = RootDef->getOperand(2);
+ MachineInstr *LHSDef = MRI->getVRegDef(LHS.getReg());
+ MachineInstr *RHSDef = MRI->getVRegDef(RHS.getReg());
+ int64_t RHSC = RHSDef->getOperand(1).getCImm()->getSExtValue();
+ if (isInt<12>(RHSC) && (RHSC % 32 == 0)) {
+ if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
+ return {{
+ [=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
+ [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); },
+ }};
+ return {{[=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
+ [=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
+ }
+ }
+
+ return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
+ [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
+ // TODO: fold large-constant offsets (ADDI adjustment / Hi-Lo12 split) like
+ // SDAG's SelectAddrRegImmLsb00000; load/store share this gap.
+}
+
/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
/// CC Must be an ICMP Predicate.
static RISCVCC::CondCode getRISCVCCFromICmp(CmpInst::Predicate CC) {
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
index c5459d862b03f..ca4c09952fe7f 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVLegalizerInfo.cpp
@@ -777,6 +777,8 @@ RISCVLegalizerInfo::RISCVLegalizerInfo(const RISCVSubtarget &ST)
.clampScalar(0, sXLen, sXLen)
.unsupported();
+ getActionDefinitionsBuilder(G_PREFETCH).legalIf(typeIs(0, p0));
+
LegalityPredicate InsertVectorEltPred = [=](const LegalityQuery &Query) {
LLT VecTy = Query.Types[0];
LLT EltTy = Query.Types[1];
diff --git a/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td b/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
index b7e844d5f77b1..45ad0ddb0b801 100644
--- a/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
+++ b/llvm/lib/Target/RISCV/RISCVInstrInfoZicbo.td
@@ -70,6 +70,9 @@ def PREFETCH_W : Prefetch_ri<0b00011, "prefetch.w">, Sched<[]>;
//===----------------------------------------------------------------------===//
def AddrRegImmLsb00000 : ComplexPattern<iPTR, 2, "SelectAddrRegImmLsb00000">;
+def gi_addr_regimm_lsb00000
+ : GIComplexOperandMatcher<s64, "selectAddrRegImmLsb00000">,
+ GIComplexPatternEquiv<AddrRegImmLsb00000>;
let Predicates = [NoVendorXMIPSCBOP] in {
def : Pat<(prefetch (AddrRegImmLsb00000 (XLenVT GPR:$rs1), simm12_lsb00000:$imm12),
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
index 1770bc6430eb9..71c7fc26f40cb 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/legalizer-info-validation.mir
@@ -320,9 +320,9 @@
# DEBUG-NEXT: G_FENCE (opcode {{[0-9]+}}): 0 type indices
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
-# DEBUG-NEXT: G_PREFETCH (opcode {{[0-9]+}}): 1 type index, 0 imm indices
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: G_PREFETCH (opcode {{[0-9]+}}): 1 type index, 3 imm indices
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_BRCOND (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. the first uncovered type index: 1, OK
# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/prefetch.ll b/llvm/test/CodeGen/RISCV/GlobalISel/prefetch.ll
new file mode 100644
index 0000000000000..c725b37c0812f
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/prefetch.ll
@@ -0,0 +1,116 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=riscv32 -mattr=+zicbop -global-isel -global-isel-abort=1 \
+; RUN: < %s | FileCheck %s --check-prefix=RV32ZICBOP
+; RUN: llc -mtriple=riscv64 -mattr=+zicbop -global-isel -global-isel-abort=1 \
+; RUN: < %s | FileCheck %s --check-prefix=RV64ZICBOP
+; RUN: llc -mtriple=riscv64 -global-isel -global-isel-abort=1 \
+; RUN: < %s | FileCheck %s --check-prefix=RV64
+
+; prefetch hints are available even without Zicbop per the psABI, so the
+; RV64 run (no +zicbop) still emits prefetch.r/w/i.
+
+define void @prefetch_r(ptr %a) nounwind {
+; RV32ZICBOP-LABEL: prefetch_r:
+; RV32ZICBOP: # %bb.0:
+; RV32ZICBOP-NEXT: prefetch.r 0(a0)
+; RV32ZICBOP-NEXT: ret
+;
+; RV64ZICBOP-LABEL: prefetch_r:
+; RV64ZICBOP: # %bb.0:
+; RV64ZICBOP-NEXT: prefetch.r 0(a0)
+; RV64ZICBOP-NEXT: ret
+;
+; RV64-LABEL: prefetch_r:
+; RV64: # %bb.0:
+; RV64-NEXT: prefetch.r 0(a0)
+; RV64-NEXT: ret
+ call void @llvm.prefetch(ptr %a, i32 0, i32 3, i32 1)
+ ret void
+}
+
+define void @prefetch_w(ptr %a) nounwind {
+; RV32ZICBOP-LABEL: prefetch_w:
+; RV32ZICBOP: # %bb.0:
+; RV32ZICBOP-NEXT: prefetch.w 0(a0)
+; RV32ZICBOP-NEXT: ret
+;
+; RV64ZICBOP-LABEL: prefetch_w:
+; RV64ZICBOP: # %bb.0:
+; RV64ZICBOP-NEXT: prefetch.w 0(a0)
+; RV64ZICBOP-NEXT: ret
+;
+; RV64-LABEL: prefetch_w:
+; RV64: # %bb.0:
+; RV64-NEXT: prefetch.w 0(a0)
+; RV64-NEXT: ret
+ call void @llvm.prefetch(ptr %a, i32 1, i32 3, i32 1)
+ ret void
+}
+
+define void @prefetch_i(ptr %a) nounwind {
+; RV32ZICBOP-LABEL: prefetch_i:
+; RV32ZICBOP: # %bb.0:
+; RV32ZICBOP-NEXT: prefetch.i 0(a0)
+; RV32ZICBOP-NEXT: ret
+;
+; RV64ZICBOP-LABEL: prefetch_i:
+; RV64ZICBOP: # %bb.0:
+; RV64ZICBOP-NEXT: prefetch.i 0(a0)
+; RV64ZICBOP-NEXT: ret
+;
+; RV64-LABEL: prefetch_i:
+; RV64: # %bb.0:
+; RV64-NEXT: prefetch.i 0(a0)
+; RV64-NEXT: ret
+ call void @llvm.prefetch(ptr %a, i32 0, i32 3, i32 0)
+ ret void
+}
+
+; A constant offset that is a multiple of 32 (simm12_lsb00000) folds into the
+; prefetch immediate.
+define void @prefetch_r_aligned_offset(ptr %a) nounwind {
+; RV32ZICBOP-LABEL: prefetch_r_aligned_offset:
+; RV32ZICBOP: # %bb.0:
+; RV32ZICBOP-NEXT: prefetch.r 64(a0)
+; RV32ZICBOP-NEXT: ret
+;
+; RV64ZICBOP-LABEL: prefetch_r_aligned_offset:
+; RV64ZICBOP: # %bb.0:
+; RV64ZICBOP-NEXT: prefetch.r 64(a0)
+; RV64ZICBOP-NEXT: ret
+;
+; RV64-LABEL: prefetch_r_aligned_offset:
+; RV64: # %bb.0:
+; RV64-NEXT: prefetch.r 64(a0)
+; RV64-NEXT: ret
+ %q = getelementptr i8, ptr %a, i32 64
+ call void @llvm.prefetch(ptr %q, i32 0, i32 3, i32 1)
+ ret void
+}
+
+; An offset that is not a multiple of 32 cannot be encoded in simm12_lsb00000;
+; the address is materialized with an ADDI and a zero offset is used.
+define void @prefetch_r_nonaligned_offset(ptr %a) nounwind {
+; RV32ZICBOP-LABEL: prefetch_r_nonaligned_offset:
+; RV32ZICBOP: # %bb.0:
+; RV32ZICBOP-NEXT: addi a0, a0, 4
+; RV32ZICBOP-NEXT: prefetch.r 0(a0)
+; RV32ZICBOP-NEXT: ret
+;
+; RV64ZICBOP-LABEL: prefetch_r_nonaligned_offset:
+; RV64ZICBOP: # %bb.0:
+; RV64ZICBOP-NEXT: addi a0, a0, 4
+; RV64ZICBOP-NEXT: prefetch.r 0(a0)
+; RV64ZICBOP-NEXT: ret
+;
+; RV64-LABEL: prefetch_r_nonaligned_offset:
+; RV64: # %bb.0:
+; RV64-NEXT: addi a0, a0, 4
+; RV64-NEXT: prefetch.r 0(a0)
+; RV64-NEXT: ret
+ %q = getelementptr i8, ptr %a, i32 4
+ call void @llvm.prefetch(ptr %q, i32 0, i32 3, i32 1)
+ ret void
+}
+
+declare void @llvm.prefetch(ptr, i32, i32, i32)
>From 686435f9064353159d625df430bf6d51a8e991bc Mon Sep 17 00:00:00 2001
From: Kane Wang <wangqiang1 at kylinos.cn>
Date: Fri, 14 Aug 2026 10:20:39 +0800
Subject: [PATCH 3/4] [RISCV][GlobalISel] Fold large prefetch offsets like SDAG
Port the remaining offset-folding cases of SDAG's
SelectAddrRegImmLsb00000 (ADDI adjustment, Hi/Lo12 split, bare constant
addresses) into the GISel matcher, and add
materializeInstSeq/computeConstAddrPlan/materializeConstBase helpers
to support them.
---
.../RISCV/GISel/RISCVInstructionSelector.cpp | 201 +++++-
.../test/CodeGen/RISCV/GlobalISel/prefetch.ll | 646 +++++++++++++++---
2 files changed, 741 insertions(+), 106 deletions(-)
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index 921feb7393374..96adff6028dc3 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -82,6 +82,9 @@ class RISCVInstructionSelector : public InstructionSelector {
bool selectCopy(MachineInstr &MI) const;
bool selectImplicitDef(MachineInstr &MI) const;
bool materializeImm(Register Reg, int64_t Imm, MachineInstr &MI) const;
+ // Emit a constant-materialization instruction sequence.
+ bool materializeInstSeq(Register DstReg, const RISCVMatInt::InstSeq &Seq,
+ MachineInstr &MI) const;
bool selectAddr(MachineInstr &MI, bool IsLocal = true,
bool IsExternWeak = false) const;
bool selectSelect(MachineInstr &MI) const;
@@ -109,6 +112,23 @@ class RISCVInstructionSelector : public InstructionSelector {
ComplexRendererFns selectAddrRegImm(MachineOperand &Root) const;
ComplexRendererFns selectAddrRegImmLsb00000(MachineOperand &Root) const;
+ // Plan for materializing a constant address as (Hi materialization, Lo12
+ // offset). Lo12 is a simm12 that, for prefetch (IsPrefetch), must be
+ // a multiple of 32.
+ struct ConstAddrPlan {
+ enum { X0, LUI, InstSeq } Kind = X0;
+ int64_t Hi20 = 0;
+ RISCVMatInt::InstSeq Seq;
+ int64_t Lo12 = 0;
+ };
+ std::optional<ConstAddrPlan> computeConstAddrPlan(int64_t CVal,
+ bool IsPrefetch) const;
+ // Materialize the high part of Plan into a register. If OrigBase is valid,
+ // ADD it to the materialized high part (for G_PTR_ADD + large constant).
+ Register materializeConstBase(MachineInstrBuilder &MIB,
+ const ConstAddrPlan &Plan,
+ Register OrigBase) const;
+
ComplexRendererFns selectSExtBits(MachineOperand &Root, unsigned Bits) const;
template <unsigned Bits>
ComplexRendererFns selectSExtBits(MachineOperand &Root) const {
@@ -613,7 +633,14 @@ RISCVInstructionSelector::selectAddrRegImmLsb00000(MachineOperand &Root) const {
MachineInstr *LHSDef = MRI->getVRegDef(LHS.getReg());
MachineInstr *RHSDef = MRI->getVRegDef(RHS.getReg());
int64_t RHSC = RHSDef->getOperand(1).getCImm()->getSExtValue();
- if (isInt<12>(RHSC) && (RHSC % 32 == 0)) {
+
+ if (isInt<12>(RHSC)) {
+ // Not a multiple of 32: can't encode, use the address as-is.
+ if ((RHSC & 0b11111) != 0) {
+ return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
+ [=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
+ }
+ // Fold the offset.
if (LHSDef->getOpcode() == TargetOpcode::G_FRAME_INDEX)
return {{
[=](MachineInstrBuilder &MIB) { MIB.add(LHSDef->getOperand(1)); },
@@ -622,12 +649,57 @@ RISCVInstructionSelector::selectAddrRegImmLsb00000(MachineOperand &Root) const {
return {{[=](MachineInstrBuilder &MIB) { MIB.add(LHS); },
[=](MachineInstrBuilder &MIB) { MIB.addImm(RHSC); }}};
}
+
+ // Large constant: fold a -2048/2016 adjustment to save an instruction.
+ if ((-2049 >= RHSC && RHSC >= -4096) || (4063 >= RHSC && RHSC >= 2017)) {
+ int64_t Adj = RHSC < 0 ? -2048 : 2016;
+ int64_t AdjustedOffset = RHSC - Adj;
+ Register BaseReg = LHS.getReg();
+ return {{[=](MachineInstrBuilder &MIB) {
+ Register Tmp = MRI->createVirtualRegister(&RISCV::GPRRegClass);
+ MachineInstr *Addi =
+ BuildMI(*MIB->getParent(), *MIB.getInstr(),
+ MIB->getDebugLoc(), TII.get(RISCV::ADDI), Tmp)
+ .addReg(BaseReg)
+ .addImm(AdjustedOffset);
+ constrainSelectedInstRegOperands(*Addi, TII, TRI, RBI);
+ MIB.addReg(Tmp);
+ },
+ [=](MachineInstrBuilder &MIB) { MIB.addImm(Adj); }}};
+ }
+
+ // Otherwise split the constant into Hi (materialized + added to the base)
+ // and Lo12 (folded offset).
+ if (auto Plan = computeConstAddrPlan(RHSC, /*IsPrefetch=*/true)) {
+ ConstAddrPlan PlanVal = *Plan;
+ Register BaseReg = LHS.getReg();
+ return {{[=](MachineInstrBuilder &MIB) {
+ MIB.addReg(materializeConstBase(MIB, PlanVal, BaseReg));
+ },
+ [=](MachineInstrBuilder &MIB) { MIB.addImm(PlanVal.Lo12); }}};
+ }
+ }
+
+ // Bare constant address. IRTranslator emits inttoptr(C) as
+ // G_INTTOPTR(G_CONSTANT); look through the G_INTTOPTR to reach the constant.
+ if (RootDef->getOpcode() == TargetOpcode::G_INTTOPTR) {
+ MachineInstr *SrcDef = MRI->getVRegDef(RootDef->getOperand(1).getReg());
+ if (SrcDef->getOpcode() == TargetOpcode::G_CONSTANT)
+ RootDef = SrcDef;
+ }
+ if (RootDef->getOpcode() == TargetOpcode::G_CONSTANT) {
+ int64_t CVal = RootDef->getOperand(1).getCImm()->getSExtValue();
+ if (auto Plan = computeConstAddrPlan(CVal, /*IsPrefetch=*/true)) {
+ ConstAddrPlan PlanVal = *Plan;
+ return {{[=](MachineInstrBuilder &MIB) {
+ MIB.addReg(materializeConstBase(MIB, PlanVal, Register()));
+ },
+ [=](MachineInstrBuilder &MIB) { MIB.addImm(PlanVal.Lo12); }}};
+ }
}
return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
[=](MachineInstrBuilder &MIB) { MIB.addImm(0); }}};
- // TODO: fold large-constant offsets (ADDI adjustment / Hi-Lo12 split) like
- // SDAG's SelectAddrRegImmLsb00000; load/store share this gap.
}
/// Returns the RISCVCC::CondCode that corresponds to the CmpInst::Predicate CC.
@@ -1731,6 +1803,129 @@ bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
return true;
}
+bool RISCVInstructionSelector::materializeInstSeq(
+ Register DstReg, const RISCVMatInt::InstSeq &Seq, MachineInstr &MI) const {
+ assert(!Seq.empty() && "materializeInstSeq requires a non-empty sequence");
+
+ MachineBasicBlock &MBB = *MI.getParent();
+ DebugLoc DL = MI.getDebugLoc();
+ unsigned NumInsts = Seq.size();
+ Register SrcReg = RISCV::X0;
+
+ for (unsigned i = 0; i < NumInsts; i++) {
+ Register TmpReg = i < NumInsts - 1
+ ? MRI->createVirtualRegister(&RISCV::GPRRegClass)
+ : DstReg;
+ const RISCVMatInt::Inst &I = Seq[i];
+ MachineInstr *Result;
+
+ switch (I.getOpndKind()) {
+ case RISCVMatInt::Imm:
+ Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
+ .addImm(I.getImm());
+ break;
+ case RISCVMatInt::RegX0:
+ Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
+ .addReg(SrcReg)
+ .addReg(RISCV::X0);
+ break;
+ case RISCVMatInt::RegReg:
+ Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
+ .addReg(SrcReg)
+ .addReg(SrcReg);
+ break;
+ case RISCVMatInt::RegImm:
+ Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
+ .addReg(SrcReg)
+ .addImm(I.getImm());
+ break;
+ }
+
+ constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
+
+ SrcReg = TmpReg;
+ }
+
+ return true;
+}
+
+std::optional<RISCVInstructionSelector::ConstAddrPlan>
+RISCVInstructionSelector::computeConstAddrPlan(int64_t CVal,
+ bool IsPrefetch) const {
+ // Split the constant into a materialized high part (the base) and
+ // a simm12 low part (the offset). For prefetch the low part
+ // must additionally be a multiple of 32 (simm12_lsb00000).
+ int64_t Lo12 = SignExtend64<12>(CVal);
+ int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
+ if (!Subtarget->is64Bit() || isInt<32>(Hi)) {
+ if (IsPrefetch && (Lo12 & 0b11111) != 0)
+ return std::nullopt;
+ ConstAddrPlan Plan;
+ Plan.Lo12 = Lo12;
+ if (Hi) {
+ Plan.Kind = ConstAddrPlan::LUI;
+ Plan.Hi20 = (Hi >> 12) & 0xfffff;
+ }
+ return Plan;
+ }
+
+ // Otherwise ask constant materialization how it would handle the constant
+ // and fold the trailing ADDI into the offset.
+ RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(CVal, *Subtarget);
+ if (Seq.back().getOpcode() != RISCV::ADDI)
+ return std::nullopt;
+ Lo12 = Seq.back().getImm();
+ if (IsPrefetch && (Lo12 & 0b11111) != 0)
+ return std::nullopt;
+ Seq.pop_back();
+ if (Seq.empty())
+ return std::nullopt;
+ ConstAddrPlan Plan;
+ Plan.Kind = ConstAddrPlan::InstSeq;
+ Plan.Seq = std::move(Seq);
+ Plan.Lo12 = Lo12;
+ return Plan;
+}
+
+Register
+RISCVInstructionSelector::materializeConstBase(MachineInstrBuilder &MIB,
+ const ConstAddrPlan &Plan,
+ Register OrigBase) const {
+ MachineBasicBlock &MBB = *MIB->getParent();
+ DebugLoc DL = MIB->getDebugLoc();
+ MachineInstr &InsertPt = *MIB.getInstr();
+
+ Register HiReg = RISCV::X0;
+ switch (Plan.Kind) {
+ case ConstAddrPlan::X0:
+ break;
+ case ConstAddrPlan::LUI: {
+ HiReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
+ MachineInstr *LUI = BuildMI(MBB, InsertPt, DL, TII.get(RISCV::LUI), HiReg)
+ .addImm(Plan.Hi20);
+ constrainSelectedInstRegOperands(*LUI, TII, TRI, RBI);
+ break;
+ }
+ case ConstAddrPlan::InstSeq: {
+ HiReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
+ materializeInstSeq(HiReg, Plan.Seq, InsertPt);
+ break;
+ }
+ }
+
+ // For G_PTR_ADD + large constant, add the original base to the materialized
+ // high part.
+ if (OrigBase.isValid() && HiReg != RISCV::X0) {
+ Register BaseReg = MRI->createVirtualRegister(&RISCV::GPRRegClass);
+ MachineInstr *Add = BuildMI(MBB, InsertPt, DL, TII.get(RISCV::ADD), BaseReg)
+ .addReg(OrigBase)
+ .addReg(HiReg);
+ constrainSelectedInstRegOperands(*Add, TII, TRI, RBI);
+ return BaseReg;
+ }
+ return OrigBase.isValid() ? OrigBase : HiReg;
+}
+
bool RISCVInstructionSelector::selectAddr(MachineInstr &MI, bool IsLocal,
bool IsExternWeak) const {
assert((MI.getOpcode() == TargetOpcode::G_GLOBAL_VALUE ||
diff --git a/llvm/test/CodeGen/RISCV/GlobalISel/prefetch.ll b/llvm/test/CodeGen/RISCV/GlobalISel/prefetch.ll
index c725b37c0812f..7b9de0947d202 100644
--- a/llvm/test/CodeGen/RISCV/GlobalISel/prefetch.ll
+++ b/llvm/test/CodeGen/RISCV/GlobalISel/prefetch.ll
@@ -1,116 +1,556 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=riscv32 -mattr=+zicbop -global-isel -global-isel-abort=1 \
-; RUN: < %s | FileCheck %s --check-prefix=RV32ZICBOP
-; RUN: llc -mtriple=riscv64 -mattr=+zicbop -global-isel -global-isel-abort=1 \
-; RUN: < %s | FileCheck %s --check-prefix=RV64ZICBOP
-; RUN: llc -mtriple=riscv64 -global-isel -global-isel-abort=1 \
-; RUN: < %s | FileCheck %s --check-prefix=RV64
-
-; prefetch hints are available even without Zicbop per the psABI, so the
-; RV64 run (no +zicbop) still emits prefetch.r/w/i.
-
-define void @prefetch_r(ptr %a) nounwind {
-; RV32ZICBOP-LABEL: prefetch_r:
-; RV32ZICBOP: # %bb.0:
-; RV32ZICBOP-NEXT: prefetch.r 0(a0)
-; RV32ZICBOP-NEXT: ret
-;
-; RV64ZICBOP-LABEL: prefetch_r:
-; RV64ZICBOP: # %bb.0:
-; RV64ZICBOP-NEXT: prefetch.r 0(a0)
-; RV64ZICBOP-NEXT: ret
-;
-; RV64-LABEL: prefetch_r:
-; RV64: # %bb.0:
-; RV64-NEXT: prefetch.r 0(a0)
-; RV64-NEXT: ret
+; RUN: llc -global-isel -global-isel-abort=1 -mtriple=riscv32 < %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -global-isel -global-isel-abort=1 -mtriple=riscv64 < %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -global-isel -global-isel-abort=1 -mtriple=riscv32 -mattr=+zicbop < %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -global-isel -global-isel-abort=1 -mtriple=riscv64 -mattr=+zicbop < %s | FileCheck -check-prefixes=CHECK %s
+; RUN: llc -global-isel -global-isel-abort=1 -mtriple=riscv64 -mattr=+zicbop,+zihintntl < %s | FileCheck -check-prefixes=CHECK %s
+
+define void @test_prefetch_read_locality_0(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_read_locality_0:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: ret
+ call void @llvm.prefetch(ptr %a, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_write_locality_0(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_write_locality_0:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.w 0(a0)
+; CHECK-NEXT: ret
+ call void @llvm.prefetch(ptr %a, i32 1, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_instruction_locality_0(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_instruction_locality_0:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.i 0(a0)
+; CHECK-NEXT: ret
+ call void @llvm.prefetch(ptr %a, i32 0, i32 0, i32 0)
+ ret void
+}
+
+define void @test_prefetch_read_locality_1(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_read_locality_1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: ret
+ call void @llvm.prefetch(ptr %a, i32 0, i32 1, i32 1)
+ ret void
+}
+
+define void @test_prefetch_write_locality_1(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_write_locality_1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.w 0(a0)
+; CHECK-NEXT: ret
+ call void @llvm.prefetch(ptr %a, i32 1, i32 1, i32 1)
+ ret void
+}
+
+define void @test_prefetch_instruction_locality_1(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_instruction_locality_1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.i 0(a0)
+; CHECK-NEXT: ret
+ call void @llvm.prefetch(ptr %a, i32 0, i32 1, i32 0)
+ ret void
+}
+
+define void @test_prefetch_read_locality_2(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_read_locality_2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: ret
+ call void @llvm.prefetch(ptr %a, i32 0, i32 2, i32 1)
+ ret void
+}
+
+define void @test_prefetch_write_locality_2(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_write_locality_2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.w 0(a0)
+; CHECK-NEXT: ret
+ call void @llvm.prefetch(ptr %a, i32 1, i32 2, i32 1)
+ ret void
+}
+
+define void @test_prefetch_instruction_locality_2(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_instruction_locality_2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.i 0(a0)
+; CHECK-NEXT: ret
+ call void @llvm.prefetch(ptr %a, i32 0, i32 2, i32 0)
+ ret void
+}
+
+define void @test_prefetch_read_locality_3(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_read_locality_3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: ret
call void @llvm.prefetch(ptr %a, i32 0, i32 3, i32 1)
ret void
}
-define void @prefetch_w(ptr %a) nounwind {
-; RV32ZICBOP-LABEL: prefetch_w:
-; RV32ZICBOP: # %bb.0:
-; RV32ZICBOP-NEXT: prefetch.w 0(a0)
-; RV32ZICBOP-NEXT: ret
-;
-; RV64ZICBOP-LABEL: prefetch_w:
-; RV64ZICBOP: # %bb.0:
-; RV64ZICBOP-NEXT: prefetch.w 0(a0)
-; RV64ZICBOP-NEXT: ret
-;
-; RV64-LABEL: prefetch_w:
-; RV64: # %bb.0:
-; RV64-NEXT: prefetch.w 0(a0)
-; RV64-NEXT: ret
+define void @test_prefetch_write_locality_3(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_write_locality_3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.w 0(a0)
+; CHECK-NEXT: ret
call void @llvm.prefetch(ptr %a, i32 1, i32 3, i32 1)
ret void
}
-define void @prefetch_i(ptr %a) nounwind {
-; RV32ZICBOP-LABEL: prefetch_i:
-; RV32ZICBOP: # %bb.0:
-; RV32ZICBOP-NEXT: prefetch.i 0(a0)
-; RV32ZICBOP-NEXT: ret
-;
-; RV64ZICBOP-LABEL: prefetch_i:
-; RV64ZICBOP: # %bb.0:
-; RV64ZICBOP-NEXT: prefetch.i 0(a0)
-; RV64ZICBOP-NEXT: ret
-;
-; RV64-LABEL: prefetch_i:
-; RV64: # %bb.0:
-; RV64-NEXT: prefetch.i 0(a0)
-; RV64-NEXT: ret
+define void @test_prefetch_instruction_locality_3(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_instruction_locality_3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.i 0(a0)
+; CHECK-NEXT: ret
call void @llvm.prefetch(ptr %a, i32 0, i32 3, i32 0)
ret void
}
-; A constant offset that is a multiple of 32 (simm12_lsb00000) folds into the
-; prefetch immediate.
-define void @prefetch_r_aligned_offset(ptr %a) nounwind {
-; RV32ZICBOP-LABEL: prefetch_r_aligned_offset:
-; RV32ZICBOP: # %bb.0:
-; RV32ZICBOP-NEXT: prefetch.r 64(a0)
-; RV32ZICBOP-NEXT: ret
-;
-; RV64ZICBOP-LABEL: prefetch_r_aligned_offset:
-; RV64ZICBOP: # %bb.0:
-; RV64ZICBOP-NEXT: prefetch.r 64(a0)
-; RV64ZICBOP-NEXT: ret
-;
-; RV64-LABEL: prefetch_r_aligned_offset:
-; RV64: # %bb.0:
-; RV64-NEXT: prefetch.r 64(a0)
-; RV64-NEXT: ret
- %q = getelementptr i8, ptr %a, i32 64
- call void @llvm.prefetch(ptr %q, i32 0, i32 3, i32 1)
- ret void
-}
-
-; An offset that is not a multiple of 32 cannot be encoded in simm12_lsb00000;
-; the address is materialized with an ADDI and a zero offset is used.
-define void @prefetch_r_nonaligned_offset(ptr %a) nounwind {
-; RV32ZICBOP-LABEL: prefetch_r_nonaligned_offset:
-; RV32ZICBOP: # %bb.0:
-; RV32ZICBOP-NEXT: addi a0, a0, 4
-; RV32ZICBOP-NEXT: prefetch.r 0(a0)
-; RV32ZICBOP-NEXT: ret
-;
-; RV64ZICBOP-LABEL: prefetch_r_nonaligned_offset:
-; RV64ZICBOP: # %bb.0:
-; RV64ZICBOP-NEXT: addi a0, a0, 4
-; RV64ZICBOP-NEXT: prefetch.r 0(a0)
-; RV64ZICBOP-NEXT: ret
-;
-; RV64-LABEL: prefetch_r_nonaligned_offset:
-; RV64: # %bb.0:
-; RV64-NEXT: addi a0, a0, 4
-; RV64-NEXT: prefetch.r 0(a0)
-; RV64-NEXT: ret
- %q = getelementptr i8, ptr %a, i32 4
- call void @llvm.prefetch(ptr %q, i32 0, i32 3, i32 1)
- ret void
-}
-
-declare void @llvm.prefetch(ptr, i32, i32, i32)
+define void @test_prefetch_offsetable_0(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_offsetable_0:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.r 2016(a0)
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 2016
+ call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_offsetable_1(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_offsetable_1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.r -2048(a0)
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 -2048
+ call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_offsetable_2(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_offsetable_2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.r 32(a0)
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 32
+ call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_offsetable_3(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_offsetable_3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: prefetch.r -32(a0)
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 -32
+ call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_offsetable_4(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_offsetable_4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi a0, a0, 32
+; CHECK-NEXT: prefetch.r 2016(a0)
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 2048
+ call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_offsetable_5(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_offsetable_5:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi a0, a0, -1
+; CHECK-NEXT: prefetch.r -2048(a0)
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 -2049
+ call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_offsetable_6(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_offsetable_6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi a0, a0, 16
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 16
+ call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_offsetable_7(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_offsetable_7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi a0, a0, -16
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 -16
+ call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_offsetable_8(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_offsetable_8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a1, 1048575
+; CHECK-NEXT: add a0, a0, a1
+; CHECK-NEXT: prefetch.r -64(a0)
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 -4160
+ call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_offsetable_9(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_offsetable_9:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a1, 1
+; CHECK-NEXT: add a0, a0, a1
+; CHECK-NEXT: prefetch.r 64(a0)
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 4160
+ call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_offsetable_10(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_offsetable_10:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi a0, a0, 2047
+; CHECK-NEXT: prefetch.r 2016(a0)
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 4063
+ call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+; The upper bound of the ADDI-adjustment range. 4064 = 2016 + 2048 would
+; overflow simm12 (max 2047) in the folded ADDI, so it must instead be split
+; into LUI + simm12 by selectConstantAddr.
+define void @test_prefetch_offsetable_11(ptr %a) nounwind {
+; CHECK-LABEL: test_prefetch_offsetable_11:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a1, 1
+; CHECK-NEXT: add a0, a0, a1
+; CHECK-NEXT: prefetch.r -32(a0)
+; CHECK-NEXT: ret
+ %addr = getelementptr i8, ptr %a, i64 4064
+ call void @llvm.prefetch(ptr %addr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_frameindex_0() nounwind {
+; CHECK-LABEL: test_prefetch_frameindex_0:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi sp, sp, -512
+; CHECK-NEXT: prefetch.r 0(sp)
+; CHECK-NEXT: addi sp, sp, 512
+; CHECK-NEXT: ret
+ %data = alloca [128 x i32], align 4
+ %base = bitcast ptr %data to ptr
+ %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 0
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_frameindex_1() nounwind {
+; CHECK-LABEL: test_prefetch_frameindex_1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, 1
+; CHECK-NEXT: addi a0, a0, 16
+; CHECK-NEXT: sub sp, sp, a0
+; CHECK-NEXT: addi a0, sp, 16
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: lui a0, 1
+; CHECK-NEXT: addi a0, a0, 16
+; CHECK-NEXT: add sp, sp, a0
+; CHECK-NEXT: ret
+ %data = alloca [1024 x i32], align 4
+ %ptr = bitcast ptr %data to ptr
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_frameindex_2() nounwind {
+; CHECK-LABEL: test_prefetch_frameindex_2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi sp, sp, -512
+; CHECK-NEXT: mv a0, sp
+; CHECK-NEXT: addi a0, a0, 16
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: addi sp, sp, 512
+; CHECK-NEXT: ret
+ %data = alloca [128 x i32], align 4
+ %base = bitcast ptr %data to ptr
+ %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 4
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_frameindex_3() nounwind {
+; CHECK-LABEL: test_prefetch_frameindex_3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi sp, sp, -512
+; CHECK-NEXT: mv a0, sp
+; CHECK-NEXT: addi a0, a0, -16
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: addi sp, sp, 512
+; CHECK-NEXT: ret
+ %data = alloca [128 x i32], align 4
+ %base = bitcast ptr %data to ptr
+ %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 -4
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_frameindex_4() nounwind {
+; CHECK-LABEL: test_prefetch_frameindex_4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi sp, sp, -512
+; CHECK-NEXT: prefetch.r 32(sp)
+; CHECK-NEXT: addi sp, sp, 512
+; CHECK-NEXT: ret
+ %data = alloca [128 x i32], align 4
+ %base = bitcast ptr %data to ptr
+ %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 8
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_frameindex_5() nounwind {
+; CHECK-LABEL: test_prefetch_frameindex_5:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi sp, sp, -512
+; CHECK-NEXT: prefetch.r -32(sp)
+; CHECK-NEXT: addi sp, sp, 512
+; CHECK-NEXT: ret
+ %data = alloca [128 x i32], align 4
+ %base = bitcast ptr %data to ptr
+ %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 -8
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_frameindex_6() nounwind {
+; CHECK-LABEL: test_prefetch_frameindex_6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi sp, sp, -512
+; CHECK-NEXT: prefetch.r 2016(sp)
+; CHECK-NEXT: addi sp, sp, 512
+; CHECK-NEXT: ret
+ %data = alloca [128 x i32], align 4
+ %base = bitcast ptr %data to ptr
+ %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 504
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_frameindex_7() nounwind {
+; CHECK-LABEL: test_prefetch_frameindex_7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi sp, sp, -512
+; CHECK-NEXT: prefetch.r -2048(sp)
+; CHECK-NEXT: addi sp, sp, 512
+; CHECK-NEXT: ret
+ %data = alloca [128 x i32], align 4
+ %base = bitcast ptr %data to ptr
+ %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 -512
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_frameindex_8() nounwind {
+; CHECK-LABEL: test_prefetch_frameindex_8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi sp, sp, -512
+; CHECK-NEXT: mv a0, sp
+; CHECK-NEXT: addi a0, a0, 2020
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: addi sp, sp, 512
+; CHECK-NEXT: ret
+ %data = alloca [128 x i32], align 4
+ %base = bitcast ptr %data to ptr
+ %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 505
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_frameindex_9() nounwind {
+; CHECK-LABEL: test_prefetch_frameindex_9:
+; CHECK: # %bb.0:
+; CHECK-NEXT: addi sp, sp, -512
+; CHECK-NEXT: mv a0, sp
+; CHECK-NEXT: addi a0, a0, -4
+; CHECK-NEXT: prefetch.r -2048(a0)
+; CHECK-NEXT: addi sp, sp, 512
+; CHECK-NEXT: ret
+ %data = alloca [128 x i32], align 4
+ %base = bitcast ptr %data to ptr
+ %ptr = getelementptr [128 x i32], ptr %base, i32 0, i32 -513
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_constant_address_0() nounwind {
+; CHECK-LABEL: test_prefetch_constant_address_0:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, 1
+; CHECK-NEXT: prefetch.r 32(a0)
+; CHECK-NEXT: ret
+ %ptr = inttoptr i64 4128 to ptr
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_constant_address_1() nounwind {
+; CHECK-LABEL: test_prefetch_constant_address_1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, 1
+; CHECK-NEXT: addi a0, a0, 31
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: ret
+ %ptr = inttoptr i64 4127 to ptr
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_constant_address_2() nounwind {
+; CHECK-LABEL: test_prefetch_constant_address_2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, 1048561
+; CHECK-NEXT: prefetch.r 32(a0)
+; CHECK-NEXT: ret
+ %ptr = inttoptr i64 18446744073709490208 to ptr
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_constant_address_3() nounwind {
+; CHECK-LABEL: test_prefetch_constant_address_3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, 1048561
+; CHECK-NEXT: addi a0, a0, 31
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: ret
+ %ptr = inttoptr i64 18446744073709490207 to ptr
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+ at g = external global [1024 x i32], align 4
+
+define void @test_prefetch_global_0() nounwind {
+; CHECK-LABEL: test_prefetch_global_0:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, %hi(g)
+; CHECK-NEXT: addi a0, a0, %lo(g)
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: ret
+ %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 0
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_global_1() nounwind {
+; CHECK-LABEL: test_prefetch_global_1:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, %hi(g+16)
+; CHECK-NEXT: addi a0, a0, %lo(g+16)
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: ret
+ %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 4
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_global_2() nounwind {
+; CHECK-LABEL: test_prefetch_global_2:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, %hi(g-16)
+; CHECK-NEXT: addi a0, a0, %lo(g-16)
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: ret
+ %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 -4
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_global_3() nounwind {
+; CHECK-LABEL: test_prefetch_global_3:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, %hi(g)
+; CHECK-NEXT: addi a0, a0, %lo(g)
+; CHECK-NEXT: prefetch.r 32(a0)
+; CHECK-NEXT: ret
+ %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 8
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_global_4() nounwind {
+; CHECK-LABEL: test_prefetch_global_4:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, %hi(g)
+; CHECK-NEXT: addi a0, a0, %lo(g)
+; CHECK-NEXT: prefetch.r -32(a0)
+; CHECK-NEXT: ret
+ %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 -8
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_global_5() nounwind {
+; CHECK-LABEL: test_prefetch_global_5:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, %hi(g)
+; CHECK-NEXT: addi a0, a0, %lo(g)
+; CHECK-NEXT: prefetch.r 2016(a0)
+; CHECK-NEXT: ret
+ %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 504
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_global_6() nounwind {
+; CHECK-LABEL: test_prefetch_global_6:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, %hi(g)
+; CHECK-NEXT: addi a0, a0, %lo(g)
+; CHECK-NEXT: prefetch.r -2048(a0)
+; CHECK-NEXT: ret
+ %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 -512
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_global_7() nounwind {
+; CHECK-LABEL: test_prefetch_global_7:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, %hi(g+2020)
+; CHECK-NEXT: addi a0, a0, %lo(g+2020)
+; CHECK-NEXT: prefetch.r 0(a0)
+; CHECK-NEXT: ret
+ %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 505
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
+
+define void @test_prefetch_global_8() nounwind {
+; CHECK-LABEL: test_prefetch_global_8:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lui a0, %hi(g-4)
+; CHECK-NEXT: addi a0, a0, %lo(g-4)
+; CHECK-NEXT: prefetch.r -2048(a0)
+; CHECK-NEXT: ret
+ %ptr = getelementptr [1024 x i32], ptr @g, i32 0, i32 -513
+ call void @llvm.prefetch(ptr %ptr, i32 0, i32 0, i32 1)
+ ret void
+}
>From 9146595b4c694f14eee64a1fb6c14c2718909b39 Mon Sep 17 00:00:00 2001
From: Kane Wang <wangqiang1 at kylinos.cn>
Date: Fri, 14 Aug 2026 15:45:41 +0800
Subject: [PATCH 4/4] [RISCV][GlobalISel] NFC: deduplicate prefetch
constant-materialization
Have materializeImm call materializeInstSeq for its sequence loop, and
make computeConstAddrPlan return ComplexRendererFns directly so the two
call sites no longer duplicate the renderer construction.
---
.../RISCV/GISel/RISCVInstructionSelector.cpp | 85 +++++--------------
1 file changed, 20 insertions(+), 65 deletions(-)
diff --git a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
index 96adff6028dc3..6555d02d81195 100644
--- a/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
+++ b/llvm/lib/Target/RISCV/GISel/RISCVInstructionSelector.cpp
@@ -121,8 +121,8 @@ class RISCVInstructionSelector : public InstructionSelector {
RISCVMatInt::InstSeq Seq;
int64_t Lo12 = 0;
};
- std::optional<ConstAddrPlan> computeConstAddrPlan(int64_t CVal,
- bool IsPrefetch) const;
+ ComplexRendererFns computeConstAddrPlan(int64_t CVal, bool IsPrefetch,
+ Register OrigBase) const;
// Materialize the high part of Plan into a register. If OrigBase is valid,
// ADD it to the materialized high part (for G_PTR_ADD + large constant).
Register materializeConstBase(MachineInstrBuilder &MIB,
@@ -670,14 +670,8 @@ RISCVInstructionSelector::selectAddrRegImmLsb00000(MachineOperand &Root) const {
// Otherwise split the constant into Hi (materialized + added to the base)
// and Lo12 (folded offset).
- if (auto Plan = computeConstAddrPlan(RHSC, /*IsPrefetch=*/true)) {
- ConstAddrPlan PlanVal = *Plan;
- Register BaseReg = LHS.getReg();
- return {{[=](MachineInstrBuilder &MIB) {
- MIB.addReg(materializeConstBase(MIB, PlanVal, BaseReg));
- },
- [=](MachineInstrBuilder &MIB) { MIB.addImm(PlanVal.Lo12); }}};
- }
+ if (auto Fns = computeConstAddrPlan(RHSC, /*IsPrefetch=*/true, LHS.getReg()))
+ return Fns;
}
// Bare constant address. IRTranslator emits inttoptr(C) as
@@ -689,13 +683,8 @@ RISCVInstructionSelector::selectAddrRegImmLsb00000(MachineOperand &Root) const {
}
if (RootDef->getOpcode() == TargetOpcode::G_CONSTANT) {
int64_t CVal = RootDef->getOperand(1).getCImm()->getSExtValue();
- if (auto Plan = computeConstAddrPlan(CVal, /*IsPrefetch=*/true)) {
- ConstAddrPlan PlanVal = *Plan;
- return {{[=](MachineInstrBuilder &MIB) {
- MIB.addReg(materializeConstBase(MIB, PlanVal, Register()));
- },
- [=](MachineInstrBuilder &MIB) { MIB.addImm(PlanVal.Lo12); }}};
- }
+ if (auto Fns = computeConstAddrPlan(CVal, /*IsPrefetch=*/true, Register()))
+ return Fns;
}
return {{[=](MachineInstrBuilder &MIB) { MIB.addReg(Root.getReg()); },
@@ -1751,56 +1740,16 @@ bool RISCVInstructionSelector::selectImplicitDef(MachineInstr &MI) const {
bool RISCVInstructionSelector::materializeImm(Register DstReg, int64_t Imm,
MachineInstr &MI) const {
- MachineBasicBlock &MBB = *MI.getParent();
- DebugLoc DL = MI.getDebugLoc();
-
if (Imm == 0) {
+ MachineBasicBlock &MBB = *MI.getParent();
+ DebugLoc DL = MI.getDebugLoc();
BuildMI(MBB, MI, DL, TII.get(TargetOpcode::COPY), DstReg).addReg(RISCV::X0);
RBI.constrainGenericRegister(DstReg, RISCV::GPRRegClass, *MRI);
return true;
}
RISCVMatInt::InstSeq Seq = RISCVMatInt::generateInstSeq(Imm, *Subtarget);
- unsigned NumInsts = Seq.size();
- Register SrcReg = RISCV::X0;
-
- for (unsigned i = 0; i < NumInsts; i++) {
- Register TmpReg = i < NumInsts - 1
- ? MRI->createVirtualRegister(&RISCV::GPRRegClass)
- : DstReg;
- const RISCVMatInt::Inst &I = Seq[i];
- MachineInstr *Result;
-
- switch (I.getOpndKind()) {
- case RISCVMatInt::Imm:
- // clang-format off
- Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
- .addImm(I.getImm());
- // clang-format on
- break;
- case RISCVMatInt::RegX0:
- Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
- .addReg(SrcReg)
- .addReg(RISCV::X0);
- break;
- case RISCVMatInt::RegReg:
- Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
- .addReg(SrcReg)
- .addReg(SrcReg);
- break;
- case RISCVMatInt::RegImm:
- Result = BuildMI(MBB, MI, DL, TII.get(I.getOpcode()), TmpReg)
- .addReg(SrcReg)
- .addImm(I.getImm());
- break;
- }
-
- constrainSelectedInstRegOperands(*Result, TII, TRI, RBI);
-
- SrcReg = TmpReg;
- }
-
- return true;
+ return materializeInstSeq(DstReg, Seq, MI);
}
bool RISCVInstructionSelector::materializeInstSeq(
@@ -1849,14 +1798,20 @@ bool RISCVInstructionSelector::materializeInstSeq(
return true;
}
-std::optional<RISCVInstructionSelector::ConstAddrPlan>
-RISCVInstructionSelector::computeConstAddrPlan(int64_t CVal,
- bool IsPrefetch) const {
+InstructionSelector::ComplexRendererFns
+RISCVInstructionSelector::computeConstAddrPlan(int64_t CVal, bool IsPrefetch,
+ Register OrigBase) const {
// Split the constant into a materialized high part (the base) and
// a simm12 low part (the offset). For prefetch the low part
// must additionally be a multiple of 32 (simm12_lsb00000).
int64_t Lo12 = SignExtend64<12>(CVal);
int64_t Hi = (uint64_t)CVal - (uint64_t)Lo12;
+ auto emit = [&](ConstAddrPlan Plan) -> ComplexRendererFns {
+ return {{[=](MachineInstrBuilder &MIB) {
+ MIB.addReg(materializeConstBase(MIB, Plan, OrigBase));
+ },
+ [=](MachineInstrBuilder &MIB) { MIB.addImm(Plan.Lo12); }}};
+ };
if (!Subtarget->is64Bit() || isInt<32>(Hi)) {
if (IsPrefetch && (Lo12 & 0b11111) != 0)
return std::nullopt;
@@ -1866,7 +1821,7 @@ RISCVInstructionSelector::computeConstAddrPlan(int64_t CVal,
Plan.Kind = ConstAddrPlan::LUI;
Plan.Hi20 = (Hi >> 12) & 0xfffff;
}
- return Plan;
+ return emit(std::move(Plan));
}
// Otherwise ask constant materialization how it would handle the constant
@@ -1884,7 +1839,7 @@ RISCVInstructionSelector::computeConstAddrPlan(int64_t CVal,
Plan.Kind = ConstAddrPlan::InstSeq;
Plan.Seq = std::move(Seq);
Plan.Lo12 = Lo12;
- return Plan;
+ return emit(std::move(Plan));
}
Register
More information about the llvm-commits
mailing list