[llvm] [AMDGPU] Invalidate stale UniformityInfo entries in 64-bit div/rem expansion (PR #215460)
Arseniy Obolenskiy via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 10 22:59:36 PDT 2026
https://github.com/aobolensk updated https://github.com/llvm/llvm-project/pull/215460
>From 5e7c67bfe2598f974392efe6bcc034eb06e0b747 Mon Sep 17 00:00:00 2001
From: Arseniy Obolenskiy <arseniy.obolenskiy at amd.com>
Date: Tue, 11 Aug 2026 07:36:20 +0200
Subject: [PATCH] [AMDGPU] Invalidate stale UniformityInfo entries on
instruction erase
expandDivRem64 erases instructions outside DeadVals worklist, so a freed instruction address could be reused and misclassified by UniformityInfo
Found as a result of an audit after finding the same class of bug fixed for Reassociate in #214719
---
.../llvm/Transforms/Utils/IntegerDivision.h | 44 +++++++++----
.../Target/AMDGPU/AMDGPUCodeGenPrepare.cpp | 20 +++---
llvm/lib/Transforms/Utils/IntegerDivision.cpp | 66 ++++++++++---------
3 files changed, 78 insertions(+), 52 deletions(-)
diff --git a/llvm/include/llvm/Transforms/Utils/IntegerDivision.h b/llvm/include/llvm/Transforms/Utils/IntegerDivision.h
index 020db95bd1e64..f853aa59e86ee 100644
--- a/llvm/include/llvm/Transforms/Utils/IntegerDivision.h
+++ b/llvm/include/llvm/Transforms/Utils/IntegerDivision.h
@@ -17,21 +17,29 @@
#define LLVM_TRANSFORMS_UTILS_INTEGERDIVISION_H
#include "llvm/Support/Compiler.h"
+#include <functional>
namespace llvm {
class BinaryOperator;
+ class Value;
}
namespace llvm {
- /// Generate code to calculate the remainder of two integers, replacing Rem
- /// with the generated code. This currently generates code using the udiv
- /// expansion, but future work includes generating more specialized code,
- /// e.g. when more information about the operands are known. Implements both
- /// 32bit and 64bit scalar division.
- ///
- /// Replace Rem with generated code.
-LLVM_ABI bool expandRemainder(BinaryOperator *Rem);
+/// Mirrors the AboutToDeleteCallback convention in
+/// llvm/Transforms/Utils/Local.h.
+using AboutToDeleteCallback = std::function<void(Value *)>;
+
+/// Generate code to calculate the remainder of two integers, replacing Rem
+/// with the generated code. This currently generates code using the udiv
+/// expansion, but future work includes generating more specialized code,
+/// e.g. when more information about the operands are known. Implements both
+/// 32bit and 64bit scalar division.
+///
+/// Replace Rem with generated code.
+LLVM_ABI bool
+expandRemainder(BinaryOperator *Rem,
+ AboutToDeleteCallback OnErased = AboutToDeleteCallback());
/// Generate code to divide two integers, replacing Div with the generated
/// code. This currently generates code similarly to compiler-rt's
@@ -40,7 +48,9 @@ LLVM_ABI bool expandRemainder(BinaryOperator *Rem);
/// 32bit and 64bit scalar division.
///
/// Replace Div with generated code.
-LLVM_ABI bool expandDivision(BinaryOperator *Div);
+LLVM_ABI bool
+expandDivision(BinaryOperator *Div,
+ AboutToDeleteCallback OnErased = AboutToDeleteCallback());
/// Generate code to calculate the remainder of two integers, replacing Rem
/// with the generated code. Uses ExpandReminder with a 32bit Rem which
@@ -48,26 +58,34 @@ LLVM_ABI bool expandDivision(BinaryOperator *Div);
/// 32 bit arithmetic.
///
/// Replace Rem with generated code.
-LLVM_ABI bool expandRemainderUpTo32Bits(BinaryOperator *Rem);
+LLVM_ABI bool expandRemainderUpTo32Bits(
+ BinaryOperator *Rem,
+ AboutToDeleteCallback OnErased = AboutToDeleteCallback());
/// Generate code to calculate the remainder of two integers, replacing Rem
/// with the generated code. Uses ExpandReminder with a 64bit Rem.
///
/// Replace Rem with generated code.
-LLVM_ABI bool expandRemainderUpTo64Bits(BinaryOperator *Rem);
+LLVM_ABI bool expandRemainderUpTo64Bits(
+ BinaryOperator *Rem,
+ AboutToDeleteCallback OnErased = AboutToDeleteCallback());
/// Generate code to divide two integers, replacing Div with the generated
/// code. Uses ExpandDivision with a 32bit Div which makes it useful for
/// targets with little or no support for less than 32 bit arithmetic.
///
/// Replace Rem with generated code.
-LLVM_ABI bool expandDivisionUpTo32Bits(BinaryOperator *Div);
+LLVM_ABI bool expandDivisionUpTo32Bits(
+ BinaryOperator *Div,
+ AboutToDeleteCallback OnErased = AboutToDeleteCallback());
/// Generate code to divide two integers, replacing Div with the generated
/// code. Uses ExpandDivision with a 64bit Div.
///
/// Replace Rem with generated code.
-LLVM_ABI bool expandDivisionUpTo64Bits(BinaryOperator *Div);
+LLVM_ABI bool expandDivisionUpTo64Bits(
+ BinaryOperator *Div,
+ AboutToDeleteCallback OnErased = AboutToDeleteCallback());
} // End llvm namespace
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
index fd0eb6d7a12e7..8bd075460a2fc 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUCodeGenPrepare.cpp
@@ -103,7 +103,7 @@ class AMDGPUCodeGenPrepareImpl
const AMDGPUTargetMachine &TM;
const TargetTransformInfo &TTI;
const TargetLibraryInfo *TLI;
- const UniformityInfo &UA;
+ UniformityInfo &UA;
const DataLayout &DL;
SimplifyQuery SQ;
const bool HasFP32DenormalFlush;
@@ -117,7 +117,7 @@ class AMDGPUCodeGenPrepareImpl
AMDGPUCodeGenPrepareImpl(Function &F, const AMDGPUTargetMachine &TM,
const TargetTransformInfo &TTI,
const TargetLibraryInfo *TLI, AssumptionCache *AC,
- const DominatorTree *DT, const UniformityInfo &UA)
+ const DominatorTree *DT, UniformityInfo &UA)
: F(F), ST(TM.getSubtarget<GCNSubtarget>(F)), TM(TM), TTI(TTI), TLI(TLI),
UA(UA), DL(F.getDataLayout()), SQ(DL, TLI, DT, AC),
HasFP32DenormalFlush(SIModeRegisterDefaults(F, ST).FP32Denormals ==
@@ -200,7 +200,7 @@ class AMDGPUCodeGenPrepareImpl
Value *shrinkDivRem64(IRBuilder<> &Builder, BinaryOperator &I,
Value *Num, Value *Den) const;
- void expandDivRem64(BinaryOperator &I) const;
+ void expandDivRem64(BinaryOperator &I);
/// Widen a scalar load.
///
@@ -1356,16 +1356,20 @@ Value *AMDGPUCodeGenPrepareImpl::shrinkDivRem64(IRBuilder<> &Builder,
return nullptr;
}
-void AMDGPUCodeGenPrepareImpl::expandDivRem64(BinaryOperator &I) const {
+void AMDGPUCodeGenPrepareImpl::expandDivRem64(BinaryOperator &I) {
Instruction::BinaryOps Opc = I.getOpcode();
+ // Bypasses the DeadVals worklist, so forget erased values from UA now to
+ // avoid a stale entry for a reused instruction address.
+ auto OnErased = [&](Value *Erased) { UA.forgetValue(Erased); };
+
// Do the general expansion.
if (Opc == Instruction::UDiv || Opc == Instruction::SDiv) {
- expandDivisionUpTo64Bits(&I);
+ expandDivisionUpTo64Bits(&I, OnErased);
return;
}
if (Opc == Instruction::URem || Opc == Instruction::SRem) {
- expandRemainderUpTo64Bits(&I);
+ expandRemainderUpTo64Bits(&I, OnErased);
return;
}
@@ -2264,7 +2268,7 @@ bool AMDGPUCodeGenPrepare::runOnFunction(Function &F) {
&getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
auto *DTWP = getAnalysisIfAvailable<DominatorTreeWrapperPass>();
const DominatorTree *DT = DTWP ? &DTWP->getDomTree() : nullptr;
- const UniformityInfo &UA =
+ UniformityInfo &UA =
getAnalysis<UniformityInfoWrapperPass>().getUniformityInfo();
return AMDGPUCodeGenPrepareImpl(F, TM, TTI, TLI, AC, DT, UA).run();
}
@@ -2276,7 +2280,7 @@ PreservedAnalyses AMDGPUCodeGenPreparePass::run(Function &F,
const TargetLibraryInfo *TLI = &FAM.getResult<TargetLibraryAnalysis>(F);
AssumptionCache *AC = &FAM.getResult<AssumptionAnalysis>(F);
const DominatorTree *DT = FAM.getCachedResult<DominatorTreeAnalysis>(F);
- const UniformityInfo &UA = FAM.getResult<UniformityInfoAnalysis>(F);
+ UniformityInfo &UA = FAM.getResult<UniformityInfoAnalysis>(F);
AMDGPUCodeGenPrepareImpl Impl(F, ATM, TTI, TLI, AC, DT, UA);
if (!Impl.run())
return PreservedAnalyses::all();
diff --git a/llvm/lib/Transforms/Utils/IntegerDivision.cpp b/llvm/lib/Transforms/Utils/IntegerDivision.cpp
index 875ebecd493a5..c09eeda8179e7 100644
--- a/llvm/lib/Transforms/Utils/IntegerDivision.cpp
+++ b/llvm/lib/Transforms/Utils/IntegerDivision.cpp
@@ -29,6 +29,13 @@ using namespace llvm;
#define DEBUG_TYPE "integer-division"
+static void eraseInstruction(Instruction *I, AboutToDeleteCallback OnErased) {
+ if (OnErased)
+ OnErased(I);
+ I->dropAllReferences();
+ I->eraseFromParent();
+}
+
/// Generate code to compute the remainder of two signed integers. Returns the
/// remainder, which will have the sign of the dividend. Builder's insert point
/// should be pointing where the caller wants code generated, e.g. at the srem
@@ -400,7 +407,8 @@ static Value *generateUnsignedDivisionCode(Value *Dividend, Value *Divisor,
/// information about the operands are known.
///
/// Replace Rem with generated code.
-bool llvm::expandRemainder(BinaryOperator *Rem) {
+bool llvm::expandRemainder(BinaryOperator *Rem,
+ AboutToDeleteCallback OnErased) {
assert((Rem->getOpcode() == Instruction::SRem ||
Rem->getOpcode() == Instruction::URem) &&
"Trying to expand remainder from a non-remainder function");
@@ -417,8 +425,7 @@ bool llvm::expandRemainder(BinaryOperator *Rem) {
// Check whether this is the insert point while Rem is still valid.
bool IsInsertPoint = Rem->getIterator() == Builder.GetInsertPoint();
Rem->replaceAllUsesWith(Remainder);
- Rem->dropAllReferences();
- Rem->eraseFromParent();
+ eraseInstruction(Rem, OnErased);
// If we didn't actually generate an urem instruction, we're done
// This happens for example if the input were constant. In this case the
@@ -434,13 +441,12 @@ bool llvm::expandRemainder(BinaryOperator *Rem) {
Rem->getOperand(1), Builder);
Rem->replaceAllUsesWith(Remainder);
- Rem->dropAllReferences();
- Rem->eraseFromParent();
+ eraseInstruction(Rem, OnErased);
// Expand the udiv
if (BinaryOperator *UDiv = dyn_cast<BinaryOperator>(Builder.GetInsertPoint())) {
assert(UDiv->getOpcode() == Instruction::UDiv && "Non-udiv in expansion?");
- expandDivision(UDiv);
+ expandDivision(UDiv, OnErased);
}
return true;
@@ -452,7 +458,7 @@ bool llvm::expandRemainder(BinaryOperator *Rem) {
/// when more information about the operands are known.
///
/// Replace Div with generated code.
-bool llvm::expandDivision(BinaryOperator *Div) {
+bool llvm::expandDivision(BinaryOperator *Div, AboutToDeleteCallback OnErased) {
assert((Div->getOpcode() == Instruction::SDiv ||
Div->getOpcode() == Instruction::UDiv) &&
"Trying to expand division from a non-division function");
@@ -470,8 +476,7 @@ bool llvm::expandDivision(BinaryOperator *Div) {
// Check whether this is the insert point while Div is still valid.
bool IsInsertPoint = Div->getIterator() == Builder.GetInsertPoint();
Div->replaceAllUsesWith(Quotient);
- Div->dropAllReferences();
- Div->eraseFromParent();
+ eraseInstruction(Div, OnErased);
// If we didn't actually generate an udiv instruction, we're done
// This happens for example if the input were constant. In this case the
@@ -488,8 +493,7 @@ bool llvm::expandDivision(BinaryOperator *Div) {
Div->getOperand(1),
Builder);
Div->replaceAllUsesWith(Quotient);
- Div->dropAllReferences();
- Div->eraseFromParent();
+ eraseInstruction(Div, OnErased);
return true;
}
@@ -501,7 +505,8 @@ bool llvm::expandDivision(BinaryOperator *Div) {
/// arithmetic.
///
/// Replace Rem with emulation code.
-bool llvm::expandRemainderUpTo32Bits(BinaryOperator *Rem) {
+bool llvm::expandRemainderUpTo32Bits(BinaryOperator *Rem,
+ AboutToDeleteCallback OnErased) {
assert((Rem->getOpcode() == Instruction::SRem ||
Rem->getOpcode() == Instruction::URem) &&
"Trying to expand remainder from a non-remainder function");
@@ -515,7 +520,7 @@ bool llvm::expandRemainderUpTo32Bits(BinaryOperator *Rem) {
"Div of bitwidth greater than 32 not supported");
if (RemTyBitWidth == 32)
- return expandRemainder(Rem);
+ return expandRemainder(Rem, OnErased);
// If bitwidth smaller than 32 extend inputs, extend output and proceed
// with 32 bit division.
@@ -539,10 +544,9 @@ bool llvm::expandRemainderUpTo32Bits(BinaryOperator *Rem) {
Trunc = Builder.CreateTrunc(ExtRem, RemTy);
Rem->replaceAllUsesWith(Trunc);
- Rem->dropAllReferences();
- Rem->eraseFromParent();
+ eraseInstruction(Rem, OnErased);
- return expandRemainder(cast<BinaryOperator>(ExtRem));
+ return expandRemainder(cast<BinaryOperator>(ExtRem), OnErased);
}
/// Generate code to compute the remainder of two integers of bitwidth up to
@@ -550,7 +554,8 @@ bool llvm::expandRemainderUpTo32Bits(BinaryOperator *Rem) {
/// outputs to operate in 64 bits.
///
/// Replace Rem with emulation code.
-bool llvm::expandRemainderUpTo64Bits(BinaryOperator *Rem) {
+bool llvm::expandRemainderUpTo64Bits(BinaryOperator *Rem,
+ AboutToDeleteCallback OnErased) {
assert((Rem->getOpcode() == Instruction::SRem ||
Rem->getOpcode() == Instruction::URem) &&
"Trying to expand remainder from a non-remainder function");
@@ -561,7 +566,7 @@ bool llvm::expandRemainderUpTo64Bits(BinaryOperator *Rem) {
unsigned RemTyBitWidth = RemTy->getIntegerBitWidth();
if (RemTyBitWidth >= 64)
- return expandRemainder(Rem);
+ return expandRemainder(Rem, OnErased);
// If bitwidth smaller than 64 extend inputs, extend output and proceed
// with 64 bit division.
@@ -585,10 +590,9 @@ bool llvm::expandRemainderUpTo64Bits(BinaryOperator *Rem) {
Trunc = Builder.CreateTrunc(ExtRem, RemTy);
Rem->replaceAllUsesWith(Trunc);
- Rem->dropAllReferences();
- Rem->eraseFromParent();
+ eraseInstruction(Rem, OnErased);
- return expandRemainder(cast<BinaryOperator>(ExtRem));
+ return expandRemainder(cast<BinaryOperator>(ExtRem), OnErased);
}
/// Generate code to divide two integers of bitwidth up to 32 bits. Uses the
@@ -597,7 +601,8 @@ bool llvm::expandRemainderUpTo64Bits(BinaryOperator *Rem) {
/// or very little support for smaller than 32 bit integer arithmetic.
///
/// Replace Div with emulation code.
-bool llvm::expandDivisionUpTo32Bits(BinaryOperator *Div) {
+bool llvm::expandDivisionUpTo32Bits(BinaryOperator *Div,
+ AboutToDeleteCallback OnErased) {
assert((Div->getOpcode() == Instruction::SDiv ||
Div->getOpcode() == Instruction::UDiv) &&
"Trying to expand division from a non-division function");
@@ -610,7 +615,7 @@ bool llvm::expandDivisionUpTo32Bits(BinaryOperator *Div) {
assert(DivTyBitWidth <= 32 && "Div of bitwidth greater than 32 not supported");
if (DivTyBitWidth == 32)
- return expandDivision(Div);
+ return expandDivision(Div, OnErased);
// If bitwidth smaller than 32 extend inputs, extend output and proceed
// with 32 bit division.
@@ -634,10 +639,9 @@ bool llvm::expandDivisionUpTo32Bits(BinaryOperator *Div) {
Trunc = Builder.CreateTrunc(ExtDiv, DivTy);
Div->replaceAllUsesWith(Trunc);
- Div->dropAllReferences();
- Div->eraseFromParent();
+ eraseInstruction(Div, OnErased);
- return expandDivision(cast<BinaryOperator>(ExtDiv));
+ return expandDivision(cast<BinaryOperator>(ExtDiv), OnErased);
}
/// Generate code to divide two integers of bitwidth up to 64 bits. Uses the
@@ -645,7 +649,8 @@ bool llvm::expandDivisionUpTo32Bits(BinaryOperator *Div) {
/// in 64 bits.
///
/// Replace Div with emulation code.
-bool llvm::expandDivisionUpTo64Bits(BinaryOperator *Div) {
+bool llvm::expandDivisionUpTo64Bits(BinaryOperator *Div,
+ AboutToDeleteCallback OnErased) {
assert((Div->getOpcode() == Instruction::SDiv ||
Div->getOpcode() == Instruction::UDiv) &&
"Trying to expand division from a non-division function");
@@ -656,7 +661,7 @@ bool llvm::expandDivisionUpTo64Bits(BinaryOperator *Div) {
unsigned DivTyBitWidth = DivTy->getIntegerBitWidth();
if (DivTyBitWidth >= 64)
- return expandDivision(Div);
+ return expandDivision(Div, OnErased);
// If bitwidth smaller than 64 extend inputs, extend output and proceed
// with 64 bit division.
@@ -680,8 +685,7 @@ bool llvm::expandDivisionUpTo64Bits(BinaryOperator *Div) {
Trunc = Builder.CreateTrunc(ExtDiv, DivTy);
Div->replaceAllUsesWith(Trunc);
- Div->dropAllReferences();
- Div->eraseFromParent();
+ eraseInstruction(Div, OnErased);
- return expandDivision(cast<BinaryOperator>(ExtDiv));
+ return expandDivision(cast<BinaryOperator>(ExtDiv), OnErased);
}
More information about the llvm-commits
mailing list