[llvm] 1a499d7 - [RISCV] Port Gather/Scatter Lowering to NewPM (#215669)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 11 19:56:50 PDT 2026
Author: Sam Elliott
Date: 2026-08-11T19:56:43-07:00
New Revision: 1a499d7487829c3c0d95af174dd8230fdecf6412
URL: https://github.com/llvm/llvm-project/commit/1a499d7487829c3c0d95af174dd8230fdecf6412
DIFF: https://github.com/llvm/llvm-project/commit/1a499d7487829c3c0d95af174dd8230fdecf6412.diff
LOG: [RISCV] Port Gather/Scatter Lowering to NewPM (#215669)
This change also adds some missing pass dependencies to the legacy
version of the pass, to reflect some analyses that are already being
used.
Assisted-by: AI
Added:
llvm/lib/Target/RISCV/RISCVGatherScatterLowering.h
Modified:
llvm/lib/Target/RISCV/RISCV.h
llvm/lib/Target/RISCV/RISCVCodeGenPassBuilder.cpp
llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
llvm/lib/Target/RISCV/RISCVPassRegistry.def
llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
llvm/test/CodeGen/RISCV/O1-newpm-pipeline.ll
llvm/test/CodeGen/RISCV/O3-newpm-pipeline.ll
llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-negative.ll
llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store.ll
llvm/test/CodeGen/RISCV/rvv/strided-load-store.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/RISCV/RISCV.h b/llvm/lib/Target/RISCV/RISCV.h
index f66291d7ae755..18887f3a9d120 100644
--- a/llvm/lib/Target/RISCV/RISCV.h
+++ b/llvm/lib/Target/RISCV/RISCV.h
@@ -62,9 +62,6 @@ void initializeRISCVLateBranchOptPass(PassRegistry &);
FunctionPass *createRISCVMakeCompressibleOptPass();
void initializeRISCVMakeCompressibleOptPass(PassRegistry &);
-FunctionPass *createRISCVGatherScatterLoweringPass();
-void initializeRISCVGatherScatterLoweringPass(PassRegistry &);
-
FunctionPass *createRISCVVectorPeepholePass();
void initializeRISCVVectorPeepholePass(PassRegistry &);
diff --git a/llvm/lib/Target/RISCV/RISCVCodeGenPassBuilder.cpp b/llvm/lib/Target/RISCV/RISCVCodeGenPassBuilder.cpp
index 7ccaa74cbb208..652d545bfee60 100644
--- a/llvm/lib/Target/RISCV/RISCVCodeGenPassBuilder.cpp
+++ b/llvm/lib/Target/RISCV/RISCVCodeGenPassBuilder.cpp
@@ -11,6 +11,7 @@
#include "RISCV.h"
#include "RISCVAsmPrinter.h"
+#include "RISCVGatherScatterLowering.h"
#include "RISCVTargetMachine.h"
#include "llvm/CodeGen/AtomicExpand.h"
#include "llvm/CodeGen/BranchRelaxation.h"
@@ -70,7 +71,7 @@ void RISCVCodeGenPassBuilder::addIRPasses(PassManagerWrapper &PMW) const {
if (getOptLevel() != CodeGenOptLevel::None) {
addFunctionPass(LoopDataPrefetchPass(), PMW);
- // TODO: RISCVGatherScatterLoweringPass
+ addFunctionPass(RISCVGatherScatterLoweringPass(&TM), PMW);
addFunctionPass(InterleavedAccessPass(TM), PMW);
addFunctionPass(RISCVCodeGenPreparePass(&TM), PMW);
}
diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
index ab089c4ab0198..c1dff38e3d0a1 100644
--- a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.cpp
@@ -11,7 +11,7 @@
//
//===----------------------------------------------------------------------===//
-#include "RISCV.h"
+#include "RISCVGatherScatterLowering.h"
#include "RISCVTargetMachine.h"
#include "llvm/Analysis/InstSimplifyFolder.h"
#include "llvm/Analysis/LoopInfo.h"
@@ -22,6 +22,8 @@
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/IntrinsicInst.h"
#include "llvm/IR/PatternMatch.h"
+#include "llvm/InitializePasses.h"
+#include "llvm/Pass.h"
#include "llvm/Transforms/Utils/Local.h"
#include <optional>
@@ -32,11 +34,11 @@ using namespace PatternMatch;
namespace {
-class RISCVGatherScatterLowering : public FunctionPass {
- const RISCVSubtarget *ST = nullptr;
- const RISCVTargetLowering *TLI = nullptr;
- LoopInfo *LI = nullptr;
- const DataLayout *DL = nullptr;
+class RISCVGatherScatterLoweringImpl {
+ const RISCVSubtarget *ST;
+ const RISCVTargetLowering *TLI;
+ LoopInfo *LI;
+ const DataLayout *DL;
SmallVector<WeakTrackingVH> MaybeDeadPHIs;
@@ -46,21 +48,11 @@ class RISCVGatherScatterLowering : public FunctionPass {
DenseMap<GetElementPtrInst *, std::pair<Value *, Value *>> StridedAddrs;
public:
- static char ID; // Pass identification, replacement for typeid
+ RISCVGatherScatterLoweringImpl(const RISCVSubtarget *ST, LoopInfo *LI,
+ const DataLayout *DL)
+ : ST(ST), TLI(ST->getTargetLowering()), LI(LI), DL(DL) {}
- RISCVGatherScatterLowering() : FunctionPass(ID) {}
-
- bool runOnFunction(Function &F) override;
-
- void getAnalysisUsage(AnalysisUsage &AU) const override {
- AU.setPreservesCFG();
- AU.addRequired<TargetPassConfig>();
- AU.addRequired<LoopInfoWrapperPass>();
- }
-
- StringRef getPassName() const override {
- return "RISC-V gather/scatter lowering";
- }
+ bool run(Function &F);
private:
bool tryCreateStridedLoadStore(IntrinsicInst *II);
@@ -75,13 +67,38 @@ class RISCVGatherScatterLowering : public FunctionPass {
} // end anonymous namespace
-char RISCVGatherScatterLowering::ID = 0;
+namespace {
+class RISCVGatherScatterLoweringLegacy : public FunctionPass {
+public:
+ static char ID;
+
+ RISCVGatherScatterLoweringLegacy() : FunctionPass(ID) {}
+
+ bool runOnFunction(Function &F) override;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.setPreservesCFG();
+ AU.addRequired<TargetPassConfig>();
+ AU.addRequired<LoopInfoWrapperPass>();
+ }
+
+ StringRef getPassName() const override {
+ return "RISC-V gather/scatter lowering";
+ }
+};
+} // namespace
+
+char RISCVGatherScatterLoweringLegacy::ID = 0;
-INITIALIZE_PASS(RISCVGatherScatterLowering, DEBUG_TYPE,
- "RISC-V gather/scatter lowering pass", false, false)
+INITIALIZE_PASS_BEGIN(RISCVGatherScatterLoweringLegacy, DEBUG_TYPE,
+ "RISC-V gather/scatter lowering pass", false, false)
+INITIALIZE_PASS_DEPENDENCY(LoopInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(TargetPassConfig)
+INITIALIZE_PASS_END(RISCVGatherScatterLoweringLegacy, DEBUG_TYPE,
+ "RISC-V gather/scatter lowering pass", false, false)
FunctionPass *llvm::createRISCVGatherScatterLoweringPass() {
- return new RISCVGatherScatterLowering();
+ return new RISCVGatherScatterLoweringLegacy();
}
// TODO: Should we consider the mask when looking for a stride?
@@ -189,11 +206,9 @@ static std::pair<Value *, Value *> matchStridedStart(Value *Start,
// start value. Build and update a scalar recurrence as we unwind the recursion.
// We also update the Stride as we unwind. Our goal is to move all of the
// arithmetic out of the loop.
-bool RISCVGatherScatterLowering::matchStridedRecurrence(Value *Index, Loop *L,
- Value *&Stride,
- PHINode *&BasePtr,
- BinaryOperator *&Inc,
- IRBuilderBase &Builder) {
+bool RISCVGatherScatterLoweringImpl::matchStridedRecurrence(
+ Value *Index, Loop *L, Value *&Stride, PHINode *&BasePtr,
+ BinaryOperator *&Inc, IRBuilderBase &Builder) {
// Our base case is a Phi.
if (auto *Phi = dyn_cast<PHINode>(Index)) {
// A phi node we want to perform this function on should be from the
@@ -338,8 +353,8 @@ bool RISCVGatherScatterLowering::matchStridedRecurrence(Value *Index, Loop *L,
}
std::pair<Value *, Value *>
-RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr,
- IRBuilderBase &Builder) {
+RISCVGatherScatterLoweringImpl::determineBaseAndStride(Instruction *Ptr,
+ IRBuilderBase &Builder) {
// A gather/scatter of a splat is a zero strided load/store.
if (auto *BasePtr = getSplatValue(Ptr)) {
@@ -492,7 +507,8 @@ RISCVGatherScatterLowering::determineBaseAndStride(Instruction *Ptr,
return P;
}
-bool RISCVGatherScatterLowering::tryCreateStridedLoadStore(IntrinsicInst *II) {
+bool RISCVGatherScatterLoweringImpl::tryCreateStridedLoadStore(
+ IntrinsicInst *II) {
VectorType *DataType;
Value *StoreVal = nullptr, *Ptr, *Mask, *EVL = nullptr;
Align Alignment;
@@ -590,22 +606,10 @@ bool RISCVGatherScatterLowering::tryCreateStridedLoadStore(IntrinsicInst *II) {
return true;
}
-bool RISCVGatherScatterLowering::runOnFunction(Function &F) {
- if (skipFunction(F))
- return false;
-
- auto &TPC = getAnalysis<TargetPassConfig>();
- auto &TM = TPC.getTM<RISCVTargetMachine>();
- ST = &TM.getSubtarget<RISCVSubtarget>(F);
+bool RISCVGatherScatterLoweringImpl::run(Function &F) {
if (!ST->hasVInstructions() || !ST->useRVVForFixedLengthVectors())
return false;
- TLI = ST->getTargetLowering();
- DL = &F.getDataLayout();
- LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
-
- StridedAddrs.clear();
-
SmallVector<IntrinsicInst *, 4> Worklist;
bool Changed = false;
@@ -640,3 +644,26 @@ bool RISCVGatherScatterLowering::runOnFunction(Function &F) {
return Changed;
}
+
+bool RISCVGatherScatterLoweringLegacy::runOnFunction(Function &F) {
+ if (skipFunction(F))
+ return false;
+
+ auto &TPC = getAnalysis<TargetPassConfig>();
+ auto &TM = TPC.getTM<RISCVTargetMachine>();
+ auto *ST = &TM.getSubtarget<RISCVSubtarget>(F);
+ auto *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+ return RISCVGatherScatterLoweringImpl(ST, LI, &F.getDataLayout()).run(F);
+}
+
+PreservedAnalyses
+RISCVGatherScatterLoweringPass::run(Function &F, FunctionAnalysisManager &FAM) {
+ auto *ST = &TM->getSubtarget<RISCVSubtarget>(F);
+ auto *LI = &FAM.getResult<LoopAnalysis>(F);
+ bool Changed =
+ RISCVGatherScatterLoweringImpl(ST, LI, &F.getDataLayout()).run(F);
+ if (!Changed)
+ return PreservedAnalyses::all();
+
+ return PreservedAnalyses::allInSet<CFGAnalyses>();
+}
diff --git a/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.h b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.h
new file mode 100644
index 0000000000000..886f3b8217457
--- /dev/null
+++ b/llvm/lib/Target/RISCV/RISCVGatherScatterLowering.h
@@ -0,0 +1,40 @@
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+///
+/// \file
+/// This file declares the RISC-V gather/scatter lowering passes.
+///
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_LIB_TARGET_RISCV_RISCVGATHERSCATTERLOWERING_H
+#define LLVM_LIB_TARGET_RISCV_RISCVGATHERSCATTERLOWERING_H
+
+#include "llvm/IR/PassManager.h"
+
+namespace llvm {
+
+class FunctionPass;
+class PassRegistry;
+class RISCVTargetMachine;
+
+class RISCVGatherScatterLoweringPass
+ : public OptionalPassInfoMixin<RISCVGatherScatterLoweringPass> {
+private:
+ const RISCVTargetMachine *TM;
+
+public:
+ RISCVGatherScatterLoweringPass(const RISCVTargetMachine *TM) : TM(TM) {}
+ PreservedAnalyses run(Function &F, FunctionAnalysisManager &FAM);
+};
+
+FunctionPass *createRISCVGatherScatterLoweringPass();
+void initializeRISCVGatherScatterLoweringLegacyPass(PassRegistry &);
+
+} // namespace llvm
+
+#endif // LLVM_LIB_TARGET_RISCV_RISCVGATHERSCATTERLOWERING_H
diff --git a/llvm/lib/Target/RISCV/RISCVPassRegistry.def b/llvm/lib/Target/RISCV/RISCVPassRegistry.def
index d954cf36e85e1..cb8e31724f82b 100644
--- a/llvm/lib/Target/RISCV/RISCVPassRegistry.def
+++ b/llvm/lib/Target/RISCV/RISCVPassRegistry.def
@@ -17,6 +17,8 @@
#define FUNCTION_PASS(NAME, CREATE_PASS)
#endif
FUNCTION_PASS("riscv-codegenprepare", RISCVCodeGenPreparePass(this))
+FUNCTION_PASS("riscv-gather-scatter-lowering",
+ RISCVGatherScatterLoweringPass(this))
FUNCTION_PASS("riscv-zacas-abi-fix", RISCVZacasABIFixPass(this))
#undef FUNCTION_PASS
diff --git a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
index e5ebd2a071d05..9e12dbbe752a1 100644
--- a/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
+++ b/llvm/lib/Target/RISCV/RISCVTargetMachine.cpp
@@ -13,6 +13,7 @@
#include "RISCVTargetMachine.h"
#include "MCTargetDesc/RISCVBaseInfo.h"
#include "RISCV.h"
+#include "RISCVGatherScatterLowering.h"
#include "RISCVMachineFunctionInfo.h"
#include "RISCVMachineScheduler.h"
#include "RISCVTargetObjectFile.h"
@@ -127,7 +128,7 @@ extern "C" LLVM_ABI LLVM_EXTERNAL_VISIBILITY void LLVMInitializeRISCVTarget() {
initializeRISCVLateBranchOptPass(*PR);
initializeRISCVMakeCompressibleOptPass(*PR);
initializeRISCVQCRelaxMarkingPass(*PR);
- initializeRISCVGatherScatterLoweringPass(*PR);
+ initializeRISCVGatherScatterLoweringLegacyPass(*PR);
initializeRISCVCodeGenPrepareLegacyPassPass(*PR);
initializeRISCVZacasABIFixLegacyPass(*PR);
initializeRISCVPostRAExpandPseudoPass(*PR);
diff --git a/llvm/test/CodeGen/RISCV/O1-newpm-pipeline.ll b/llvm/test/CodeGen/RISCV/O1-newpm-pipeline.ll
index 291a95830dfd3..57fc4ce5bd9c5 100644
--- a/llvm/test/CodeGen/RISCV/O1-newpm-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O1-newpm-pipeline.ll
@@ -16,6 +16,7 @@
; CHECK-NEXT: atomic-expand
; CHECK-NEXT: riscv-zacas-abi-fix
; CHECK-NEXT: loop-data-prefetch
+; CHECK-NEXT: riscv-gather-scatter-lowering
; CHECK-NEXT: interleaved-access
; CHECK-NEXT: riscv-codegenprepare
; CHECK-NEXT: verify
diff --git a/llvm/test/CodeGen/RISCV/O3-newpm-pipeline.ll b/llvm/test/CodeGen/RISCV/O3-newpm-pipeline.ll
index 69db5f46c3268..6f6edc02ea7bd 100644
--- a/llvm/test/CodeGen/RISCV/O3-newpm-pipeline.ll
+++ b/llvm/test/CodeGen/RISCV/O3-newpm-pipeline.ll
@@ -16,6 +16,7 @@
; CHECK-NEXT: atomic-expand
; CHECK-NEXT: riscv-zacas-abi-fix
; CHECK-NEXT: loop-data-prefetch
+; CHECK-NEXT: riscv-gather-scatter-lowering
; CHECK-NEXT: interleaved-access
; CHECK-NEXT: riscv-codegenprepare
; CHECK-NEXT: verify
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-negative.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-negative.ll
index ef09a3fb6d5fa..6e63f948a7e80 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-negative.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store-negative.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt %s -S -riscv-gather-scatter-lowering -mtriple=riscv64 -mattr=+m,+v,+zvl256b | FileCheck %s
+; RUN: opt %s -S -passes=riscv-gather-scatter-lowering -mtriple=riscv64 -mattr=+m,+v,+zvl256b | FileCheck %s
; This contains negative tests for the strided load/store recognition in
; RISCVGatherScatterLowering.cpp
diff --git a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store.ll b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store.ll
index 108c75c8c4abc..d52c1c1edf7e3 100644
--- a/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/fixed-vectors-strided-load-store.ll
@@ -1,6 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt %s -S -riscv-gather-scatter-lowering -mtriple=riscv64 -mattr=+m,+v,+zvl256b | FileCheck %s --check-prefixes=CHECK,V
+; RUN: opt %s -S -passes=riscv-gather-scatter-lowering -mtriple=riscv64 -mattr=+m,+v,+zvl256b | FileCheck %s --check-prefixes=CHECK,V
; RUN: opt %s -S -riscv-gather-scatter-lowering -mtriple=riscv64 -mattr=+m,+f,+zve32f,+zvl256b | FileCheck %s --check-prefixes=CHECK,ZVE32F
+; RUN: opt %s -S -passes=riscv-gather-scatter-lowering -mtriple=riscv64 -mattr=+m,+f,+zve32f,+zvl256b | FileCheck %s --check-prefixes=CHECK,ZVE32F
%struct.foo = type { i32, i32, i32, i32 }
diff --git a/llvm/test/CodeGen/RISCV/rvv/strided-load-store.ll b/llvm/test/CodeGen/RISCV/rvv/strided-load-store.ll
index 6c51848d9080d..312310f228b82 100644
--- a/llvm/test/CodeGen/RISCV/rvv/strided-load-store.ll
+++ b/llvm/test/CodeGen/RISCV/rvv/strided-load-store.ll
@@ -1,5 +1,6 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
; RUN: opt %s -S -riscv-gather-scatter-lowering -mtriple=riscv64 -mattr=+m,+v | FileCheck %s --check-prefixes=CHECK
+; RUN: opt %s -S -passes=riscv-gather-scatter-lowering -mtriple=riscv64 -mattr=+m,+v | FileCheck %s --check-prefixes=CHECK
%struct.foo = type { i32, i32, i32, i32 }
More information about the llvm-commits
mailing list