[llvm] [GlobalISel] Fast-path single reg loads and stores in IRTranslator (PR #195245)
Cullen Rhodes via llvm-commits
llvm-commits at lists.llvm.org
Fri May 1 03:04:25 PDT 2026
https://github.com/c-rhodes updated https://github.com/llvm/llvm-project/pull/195245
>From e535e96fd0952cfa646d48ef84cf5948c0a8a2ab Mon Sep 17 00:00:00 2001
From: Cullen Rhodes <cullen.rhodes at arm.com>
Date: Thu, 30 Apr 2026 14:04:59 +0000
Subject: [PATCH 1/2] [GlobalISel] Add a fast-path for single-register loads
and stores
IRTranslator::translateLoad is hot in compile-time profiles of sqlite on
aarch64-O0-g. Aggregates are flattened into multiple vregs with offsets.
Most loads and stores lower to a single register so this offset
materialization can be avoided.
---
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 43 ++++++++++++++------
1 file changed, 30 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index b5b73332bac62..e80b786a13ec5 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1387,13 +1387,10 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
return true;
ArrayRef<Register> Regs = getOrCreateVRegs(LI);
- ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
Register Base = getOrCreateVReg(*LI.getPointerOperand());
AAMDNodes AAInfo = LI.getAAMetadata();
const Value *Ptr = LI.getPointerOperand();
- Type *OffsetIRTy = DL->getIndexType(Ptr->getType());
- LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
if (CLI->supportSwiftError() && isSwiftError(Ptr)) {
assert(Regs.size() == 1 && "swifterror should be single pointer");
@@ -1412,18 +1409,30 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
}
}
- const MDNode *Ranges =
- Regs.size() == 1 ? LI.getMetadata(LLVMContext::MD_range) : nullptr;
+ // Fast-path the common single-register load.
+ if (Regs.size() == 1) {
+ auto MMO = MF->getMachineMemOperand(
+ MachinePointerInfo(LI.getPointerOperand()), Flags,
+ MRI->getType(Regs[0]), getMemOpAlign(LI), AAInfo,
+ LI.getMetadata(LLVMContext::MD_range), LI.getSyncScopeID(),
+ LI.getOrdering());
+ MIRBuilder.buildLoad(Regs[0], Base, *MMO);
+ return true;
+ }
+
+ ArrayRef<uint64_t> Offsets = *VMap.getOffsets(LI);
+ Type *OffsetIRTy = DL->getIndexType(Ptr->getType());
+ LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
for (unsigned i = 0; i < Regs.size(); ++i) {
Register Addr;
MIRBuilder.materializeObjectPtrOffset(Addr, Base, OffsetTy, Offsets[i]);
MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i]);
Align BaseAlign = getMemOpAlign(LI);
- auto MMO =
- MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Regs[i]),
- commonAlignment(BaseAlign, Offsets[i]), AAInfo,
- Ranges, LI.getSyncScopeID(), LI.getOrdering());
+ auto MMO = MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Regs[i]),
+ commonAlignment(BaseAlign, Offsets[i]),
+ AAInfo, nullptr, LI.getSyncScopeID(),
+ LI.getOrdering());
MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
}
@@ -1436,12 +1445,8 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
return true;
ArrayRef<Register> Vals = getOrCreateVRegs(*SI.getValueOperand());
- ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
Register Base = getOrCreateVReg(*SI.getPointerOperand());
- Type *OffsetIRTy = DL->getIndexType(SI.getPointerOperandType());
- LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
-
if (CLI->supportSwiftError() && isSwiftError(SI.getPointerOperand())) {
assert(Vals.size() == 1 && "swifterror should be single pointer");
@@ -1452,7 +1457,19 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
}
MachineMemOperand::Flags Flags = TLI->getStoreMemOperandFlags(SI, *DL);
+ // Fast-path the common single-register store.
+ if (Vals.size() == 1) {
+ auto MMO = MF->getMachineMemOperand(
+ MachinePointerInfo(SI.getPointerOperand()), Flags,
+ MRI->getType(Vals[0]), getMemOpAlign(SI), SI.getAAMetadata(), nullptr,
+ SI.getSyncScopeID(), SI.getOrdering());
+ MIRBuilder.buildStore(Vals[0], Base, *MMO);
+ return true;
+ }
+ ArrayRef<uint64_t> Offsets = *VMap.getOffsets(*SI.getValueOperand());
+ Type *OffsetIRTy = DL->getIndexType(SI.getPointerOperandType());
+ LLT OffsetTy = getLLTForType(*OffsetIRTy, *DL);
for (unsigned i = 0; i < Vals.size(); ++i) {
Register Addr;
MIRBuilder.materializeObjectPtrOffset(Addr, Base, OffsetTy, Offsets[i]);
>From 5da7d17fcd3e91a1a5eae76bcef9d1db21e305b1 Mon Sep 17 00:00:00 2001
From: Cullen Rhodes <cullen.rhodes at arm.com>
Date: Fri, 1 May 2026 10:01:28 +0000
Subject: [PATCH 2/2] address comments
---
llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp | 20 ++++++++++----------
1 file changed, 10 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
index e80b786a13ec5..01142f9dd7bf6 100644
--- a/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/IRTranslator.cpp
@@ -1411,7 +1411,7 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
// Fast-path the common single-register load.
if (Regs.size() == 1) {
- auto MMO = MF->getMachineMemOperand(
+ auto *MMO = MF->getMachineMemOperand(
MachinePointerInfo(LI.getPointerOperand()), Flags,
MRI->getType(Regs[0]), getMemOpAlign(LI), AAInfo,
LI.getMetadata(LLVMContext::MD_range), LI.getSyncScopeID(),
@@ -1429,10 +1429,10 @@ bool IRTranslator::translateLoad(const User &U, MachineIRBuilder &MIRBuilder) {
MachinePointerInfo Ptr(LI.getPointerOperand(), Offsets[i]);
Align BaseAlign = getMemOpAlign(LI);
- auto MMO = MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Regs[i]),
- commonAlignment(BaseAlign, Offsets[i]),
- AAInfo, nullptr, LI.getSyncScopeID(),
- LI.getOrdering());
+ auto *MMO = MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Regs[i]),
+ commonAlignment(BaseAlign, Offsets[i]),
+ AAInfo, nullptr, LI.getSyncScopeID(),
+ LI.getOrdering());
MIRBuilder.buildLoad(Regs[i], Addr, *MMO);
}
@@ -1459,7 +1459,7 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
MachineMemOperand::Flags Flags = TLI->getStoreMemOperandFlags(SI, *DL);
// Fast-path the common single-register store.
if (Vals.size() == 1) {
- auto MMO = MF->getMachineMemOperand(
+ auto *MMO = MF->getMachineMemOperand(
MachinePointerInfo(SI.getPointerOperand()), Flags,
MRI->getType(Vals[0]), getMemOpAlign(SI), SI.getAAMetadata(), nullptr,
SI.getSyncScopeID(), SI.getOrdering());
@@ -1476,10 +1476,10 @@ bool IRTranslator::translateStore(const User &U, MachineIRBuilder &MIRBuilder) {
MachinePointerInfo Ptr(SI.getPointerOperand(), Offsets[i]);
Align BaseAlign = getMemOpAlign(SI);
- auto MMO = MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Vals[i]),
- commonAlignment(BaseAlign, Offsets[i]),
- SI.getAAMetadata(), nullptr,
- SI.getSyncScopeID(), SI.getOrdering());
+ auto *MMO = MF->getMachineMemOperand(Ptr, Flags, MRI->getType(Vals[i]),
+ commonAlignment(BaseAlign, Offsets[i]),
+ SI.getAAMetadata(), nullptr,
+ SI.getSyncScopeID(), SI.getOrdering());
MIRBuilder.buildStore(Vals[i], Addr, *MMO);
}
return true;
More information about the llvm-commits
mailing list