[llvm] [GlobalISel] Add `isKnownNeverZero` to `GISelValueTracking` (PR #198438)
Alan Li via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 20 17:54:57 PDT 2026
https://github.com/lialan updated https://github.com/llvm/llvm-project/pull/198438
>From edd694b88ad3a536602437b0981a5df46e6e8195 Mon Sep 17 00:00:00 2001
From: Alan Li <me at alanli.org>
Date: Sat, 16 May 2026 18:22:10 -0700
Subject: [PATCH 01/10] [GlobalISel] Add `isKnownNeverZero` to
`GISelValueTracking`
Port `SelectionDAG::isKnownNeverZero` to `GISelValueTracking` matching the SDAG API.
Falls back to `getKnownBits(R, DemandedElts, Depth).isNonZero()`
for opcodes not handled directly. Target-specific opcodes flow through
the existing computeKnownBitsForTargetInstr hook.
Adds 12 unit tests in `KnownBitsTest.cpp` covering each handled opcode
group plus the KnownBits fallback path and the `MaxDepth` cutoff.
---
.../CodeGen/GlobalISel/GISelValueTracking.h | 6 +
.../CodeGen/GlobalISel/GISelValueTracking.cpp | 197 ++++++++++++++
.../CodeGen/GlobalISel/KnownBitsTest.cpp | 254 ++++++++++++++++++
3 files changed, 457 insertions(+)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h b/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h
index 722a45eeb07e5..d1d173547c034 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h
@@ -92,6 +92,12 @@ class LLVM_ABI GISelValueTracking : public GISelChangeObserver {
/// predicate to simplify operations downstream.
bool signBitIsZero(Register Op);
+ /// Return true if the value defined by \p R is provably never zero. This is
+ /// the GlobalISel analog of \c SelectionDAG::isKnownNeverZero.
+ bool isKnownNeverZero(Register R, unsigned Depth = 0);
+ bool isKnownNeverZero(Register R, const APInt &DemandedElts,
+ unsigned Depth = 0);
+
static void computeKnownBitsForAlignment(KnownBits &Known, Align Alignment) {
// The low bits are known zero if the pointer is aligned.
Known.Zero.setLowBits(Log2(Alignment));
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
index 4d4f0bc3a190a..8940a5ab91a95 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
@@ -105,6 +105,203 @@ bool GISelValueTracking::signBitIsZero(Register R) {
return maskedValueIsZero(R, APInt::getSignMask(BitWidth));
}
+bool GISelValueTracking::isKnownNeverZero(Register R, unsigned Depth) {
+ LLT Ty = MRI.getType(R);
+ APInt DemandedElts =
+ Ty.isFixedVector() ? APInt::getAllOnes(Ty.getNumElements()) : APInt(1, 1);
+ return isKnownNeverZero(R, DemandedElts, Depth);
+}
+
+bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
+ unsigned Depth) {
+ if (Depth >= getMaxDepth())
+ return false;
+
+ MachineInstr *MI = MRI.getVRegDef(R);
+ if (!MI)
+ return false;
+
+ auto Recurse = [&](Register Reg, const APInt &Elts) {
+ return isKnownNeverZero(Reg, Elts, Depth + 1);
+ };
+ auto KnownOf = [&](Register Reg) {
+ return getKnownBits(Reg, DemandedElts, Depth + 1);
+ };
+
+ switch (MI->getOpcode()) {
+ default:
+ break;
+
+ case TargetOpcode::G_CONSTANT:
+ return !MI->getOperand(1).getCImm()->isZero();
+
+ case TargetOpcode::G_BUILD_VECTOR: {
+ for (const auto &[I, MO] : enumerate(drop_begin(MI->operands()))) {
+ if (!DemandedElts[I])
+ continue;
+ if (!Recurse(MO.getReg(), APInt(1, 1)))
+ return false;
+ }
+ return true;
+ }
+
+ case TargetOpcode::G_SPLAT_VECTOR: {
+ // G_SPLAT_VECTOR allows its source scalar to be wider than the destination
+ // element type, with the high bits implicitly truncated. Structural
+ // recursion ignores width, so only recurse when the widths already match;
+ // otherwise drop to KnownBits and truncate explicitly.
+ Register Src = MI->getOperand(1).getReg();
+ unsigned EltBW = MRI.getType(R).getScalarSizeInBits();
+ if (MRI.getType(Src).getSizeInBits() == EltBW)
+ return Recurse(Src, APInt(1, 1));
+ return getKnownBits(Src, APInt(1, 1), Depth + 1).trunc(EltBW).isNonZero();
+ }
+
+ case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
+ GExtractVectorElement &Extract = cast<GExtractVectorElement>(*MI);
+ Register InVec = Extract.getVectorReg();
+ LLT VecTy = MRI.getType(InVec);
+ if (VecTy.isScalableVector())
+ break;
+ unsigned NumSrcElts = VecTy.getNumElements();
+ APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
+ if (auto Idx = getIConstantVRegVal(Extract.getIndexReg(), MRI))
+ if (Idx->ult(NumSrcElts))
+ DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, Idx->getZExtValue());
+ return Recurse(InVec, DemandedSrcElts);
+ }
+
+ case TargetOpcode::G_SHUFFLE_VECTOR: {
+ GShuffleVector &Shuf = cast<GShuffleVector>(*MI);
+ LLT SrcTy = MRI.getType(Shuf.getSrc1Reg());
+ if (SrcTy.isScalableVector())
+ break;
+ APInt DemandedLHS, DemandedRHS;
+ if (!getShuffleDemandedElts(SrcTy.getNumElements(), Shuf.getMask(),
+ DemandedElts, DemandedLHS, DemandedRHS))
+ break;
+ if (!!DemandedLHS && !Recurse(Shuf.getSrc1Reg(), DemandedLHS))
+ return false;
+ if (!!DemandedRHS && !Recurse(Shuf.getSrc2Reg(), DemandedRHS))
+ return false;
+ return true;
+ }
+
+ case TargetOpcode::G_OR:
+ return Recurse(MI->getOperand(1).getReg(), DemandedElts) ||
+ Recurse(MI->getOperand(2).getReg(), DemandedElts);
+
+ case TargetOpcode::G_SELECT:
+ return Recurse(MI->getOperand(2).getReg(), DemandedElts) &&
+ Recurse(MI->getOperand(3).getReg(), DemandedElts);
+
+ case TargetOpcode::G_SHL: {
+ Register LHSReg = MI->getOperand(1).getReg();
+ if (MI->getFlag(MachineInstr::NoSWrap) ||
+ MI->getFlag(MachineInstr::NoUWrap))
+ return Recurse(LHSReg, DemandedElts);
+ KnownBits ValKnown = KnownOf(LHSReg);
+ if (ValKnown.One[0])
+ return true;
+ APInt MaxCnt = KnownOf(MI->getOperand(2).getReg()).getMaxValue();
+ if (MaxCnt.ult(ValKnown.getBitWidth()) &&
+ !ValKnown.One.shl(MaxCnt).isZero())
+ return true;
+ break;
+ }
+
+ case TargetOpcode::G_LSHR:
+ case TargetOpcode::G_ASHR: {
+ Register LHSReg = MI->getOperand(1).getReg();
+ if (MI->getFlag(MachineInstr::IsExact))
+ return Recurse(LHSReg, DemandedElts);
+ KnownBits ValKnown = KnownOf(LHSReg);
+ if (MI->getOpcode() == TargetOpcode::G_ASHR && ValKnown.isNegative())
+ return true;
+ // If max shift cnt of known ones is non-zero, result is non-zero.
+ APInt MaxCnt = KnownOf(MI->getOperand(2).getReg()).getMaxValue();
+ if (MaxCnt.ult(ValKnown.getBitWidth()) &&
+ !ValKnown.One.lshr(MaxCnt).isZero())
+ return true;
+ break;
+ }
+
+ case TargetOpcode::G_UDIV:
+ case TargetOpcode::G_SDIV:
+ if (MI->getFlag(MachineInstr::IsExact))
+ return Recurse(MI->getOperand(1).getReg(), DemandedElts);
+ break;
+
+ case TargetOpcode::G_ADD:
+ if (MI->getFlag(MachineInstr::NoUWrap))
+ return Recurse(MI->getOperand(1).getReg(), DemandedElts) ||
+ Recurse(MI->getOperand(2).getReg(), DemandedElts);
+ break;
+
+ case TargetOpcode::G_SUB: {
+ Register LHSReg = MI->getOperand(1).getReg();
+ Register RHSReg = MI->getOperand(2).getReg();
+ if (auto LHSC = getIConstantVRegVal(LHSReg, MRI))
+ if (LHSC->isZero())
+ return Recurse(RHSReg, DemandedElts);
+ return KnownBits::ne(KnownOf(LHSReg), KnownOf(RHSReg)).value_or(false);
+ }
+
+ case TargetOpcode::G_MUL:
+ if (MI->getFlag(MachineInstr::NoUWrap) ||
+ MI->getFlag(MachineInstr::NoSWrap))
+ return Recurse(MI->getOperand(1).getReg(), DemandedElts) &&
+ Recurse(MI->getOperand(2).getReg(), DemandedElts);
+ break;
+
+ case TargetOpcode::G_UADDSAT:
+ case TargetOpcode::G_UMAX:
+ return Recurse(MI->getOperand(1).getReg(), DemandedElts) ||
+ Recurse(MI->getOperand(2).getReg(), DemandedElts);
+
+ case TargetOpcode::G_UMIN:
+ return Recurse(MI->getOperand(1).getReg(), DemandedElts) &&
+ Recurse(MI->getOperand(2).getReg(), DemandedElts);
+
+ case TargetOpcode::G_SMAX: {
+ KnownBits L = KnownOf(MI->getOperand(1).getReg());
+ KnownBits R2 = KnownOf(MI->getOperand(2).getReg());
+ if (L.isStrictlyPositive() || R2.isStrictlyPositive())
+ return true;
+ if (L.isNonZero() && R2.isNonZero())
+ return true;
+ return Recurse(MI->getOperand(1).getReg(), DemandedElts) &&
+ Recurse(MI->getOperand(2).getReg(), DemandedElts);
+ }
+
+ case TargetOpcode::G_SMIN: {
+ KnownBits L = KnownOf(MI->getOperand(1).getReg());
+ KnownBits R2 = KnownOf(MI->getOperand(2).getReg());
+ if (L.isNegative() || R2.isNegative())
+ return true;
+ if (L.isNonZero() && R2.isNonZero())
+ return true;
+ return Recurse(MI->getOperand(1).getReg(), DemandedElts) &&
+ Recurse(MI->getOperand(2).getReg(), DemandedElts);
+ }
+
+ case TargetOpcode::G_ROTL:
+ case TargetOpcode::G_ROTR:
+ case TargetOpcode::G_BSWAP:
+ case TargetOpcode::G_BITREVERSE:
+ case TargetOpcode::G_CTPOP:
+ case TargetOpcode::G_ABS:
+ case TargetOpcode::G_ZEXT:
+ case TargetOpcode::G_SEXT:
+ case TargetOpcode::G_FREEZE:
+ return Recurse(MI->getOperand(1).getReg(), DemandedElts);
+ }
+
+ // Pass through this frame's Depth (not Depth+1) because we have not recursed
+ // into a child MI here: the fallback queries KnownBits for the same R.
+ return getKnownBits(R, DemandedElts, Depth).isNonZero();
+}
+
APInt GISelValueTracking::getKnownZeroes(Register R) {
return getKnownBits(R).Zero;
}
diff --git a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
index 8563d7f1f15c9..0ede4e9e2f518 100644
--- a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
@@ -2150,3 +2150,257 @@ TEST_F(AArch64GISelMITest, TestKnownBitsUADDO) {
EXPECT_EQ(0u, Res.One.getZExtValue());
EXPECT_EQ(31u, Res.Zero.countl_one());
}
+
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroConstant) {
+ StringRef MIRString = R"(
+ %zero:_(s32) = G_CONSTANT i32 0
+ %one:_(s32) = G_CONSTANT i32 1
+ %copy_zero:_(s32) = COPY %zero
+ %copy_one:_(s32) = COPY %one
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopyOne = Copies[Copies.size() - 1];
+ Register CopyZero = Copies[Copies.size() - 2];
+ GISelValueTracking Info(*MF);
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopyOne)->getOperand(1).getReg()));
+ EXPECT_FALSE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopyZero)->getOperand(1).getReg()));
+}
+
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroOr) {
+ StringRef MIRString = R"(
+ %one:_(s32) = G_CONSTANT i32 1
+ %x:_(s32) = COPY $w0
+ %or:_(s32) = G_OR %x, %one
+ %copy_or:_(s32) = COPY %or
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopyOr = Copies[Copies.size() - 1];
+ GISelValueTracking Info(*MF);
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopyOr)->getOperand(1).getReg()));
+}
+
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroSelect) {
+ StringRef MIRString = R"(
+ %one:_(s32) = G_CONSTANT i32 1
+ %two:_(s32) = G_CONSTANT i32 2
+ %cond:_(s1) = G_IMPLICIT_DEF
+ %sel:_(s32) = G_SELECT %cond, %one, %two
+ %copy_sel:_(s32) = COPY %sel
+
+ %zero:_(s32) = G_CONSTANT i32 0
+ %selz:_(s32) = G_SELECT %cond, %one, %zero
+ %copy_selz:_(s32) = COPY %selz
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopySelZ = Copies[Copies.size() - 1];
+ Register CopySel = Copies[Copies.size() - 2];
+ GISelValueTracking Info(*MF);
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopySel)->getOperand(1).getReg()));
+ EXPECT_FALSE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopySelZ)->getOperand(1).getReg()));
+}
+
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroShlNUW) {
+ StringRef MIRString = R"(
+ %one:_(s32) = G_CONSTANT i32 1
+ %x:_(s32) = COPY $w0
+ %shl:_(s32) = nuw G_SHL %one, %x
+ %copy_shl:_(s32) = COPY %shl
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopyShl = Copies[Copies.size() - 1];
+ GISelValueTracking Info(*MF);
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopyShl)->getOperand(1).getReg()));
+}
+
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroShlOneByVar) {
+ // 1 << X is always nonzero in the i32 case for any X < 32; KnownBits-based
+ // path proves it from One[0] of the LHS without needing the nuw flag.
+ StringRef MIRString = R"(
+ %one:_(s32) = G_CONSTANT i32 1
+ %x:_(s32) = COPY $w0
+ %shl:_(s32) = G_SHL %one, %x
+ %copy_shl:_(s32) = COPY %shl
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopyShl = Copies[Copies.size() - 1];
+ GISelValueTracking Info(*MF);
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopyShl)->getOperand(1).getReg()));
+}
+
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroAshrExact) {
+ StringRef MIRString = R"(
+ %one:_(s32) = G_CONSTANT i32 1
+ %x:_(s32) = COPY $w0
+ %ashr:_(s32) = exact G_ASHR %one, %x
+ %copy_ashr:_(s32) = COPY %ashr
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopyAshr = Copies[Copies.size() - 1];
+ GISelValueTracking Info(*MF);
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopyAshr)->getOperand(1).getReg()));
+}
+
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroUMaxUMin) {
+ StringRef MIRString = R"(
+ %one:_(s32) = G_CONSTANT i32 1
+ %x:_(s32) = COPY $w0
+ %umax:_(s32) = G_UMAX %one, %x
+ %copy_umax:_(s32) = COPY %umax
+
+ %two:_(s32) = G_CONSTANT i32 2
+ %umin:_(s32) = G_UMIN %one, %two
+ %copy_umin:_(s32) = COPY %umin
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopyUMin = Copies[Copies.size() - 1];
+ Register CopyUMax = Copies[Copies.size() - 2];
+ GISelValueTracking Info(*MF);
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopyUMax)->getOperand(1).getReg()));
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopyUMin)->getOperand(1).getReg()));
+}
+
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroZextSext) {
+ StringRef MIRString = R"(
+ %one:_(s8) = G_CONSTANT i8 1
+ %zext:_(s32) = G_ZEXT %one
+ %sext:_(s32) = G_SEXT %one
+ %copy_zext:_(s32) = COPY %zext
+ %copy_sext:_(s32) = COPY %sext
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopySext = Copies[Copies.size() - 1];
+ Register CopyZext = Copies[Copies.size() - 2];
+ GISelValueTracking Info(*MF);
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopyZext)->getOperand(1).getReg()));
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopySext)->getOperand(1).getReg()));
+}
+
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroBuildVector) {
+ StringRef MIRString = R"(
+ %one:_(s32) = G_CONSTANT i32 1
+ %two:_(s32) = G_CONSTANT i32 2
+ %bv:_(<2 x s32>) = G_BUILD_VECTOR %one, %two
+ %copy_bv:_(<2 x s32>) = COPY %bv
+
+ %zero:_(s32) = G_CONSTANT i32 0
+ %bvz:_(<2 x s32>) = G_BUILD_VECTOR %one, %zero
+ %copy_bvz:_(<2 x s32>) = COPY %bvz
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopyBVZ = Copies[Copies.size() - 1];
+ Register CopyBV = Copies[Copies.size() - 2];
+ GISelValueTracking Info(*MF);
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopyBV)->getOperand(1).getReg()));
+ EXPECT_FALSE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopyBVZ)->getOperand(1).getReg()));
+}
+
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroSplatVector) {
+ // Same-width splat: recurses structurally on the source.
+ // Wide-source splat: 0x100 (i64) splatted into i32 lanes is still nonzero;
+ // 0x100000000 (i64) splatted into i32 lanes is zero after truncation.
+ StringRef MIRString = R"(
+ %one:_(s32) = G_CONSTANT i32 1
+ %sv:_(<vscale x 2 x s32>) = G_SPLAT_VECTOR %one
+ %copy_sv:_(<vscale x 2 x s32>) = COPY %sv
+
+ %hi32:_(s64) = G_CONSTANT i64 256
+ %sv_hi:_(<vscale x 2 x s32>) = G_SPLAT_VECTOR %hi32
+ %copy_sv_hi:_(<vscale x 2 x s32>) = COPY %sv_hi
+
+ %hi33:_(s64) = G_CONSTANT i64 4294967296
+ %sv_trunc_zero:_(<vscale x 2 x s32>) = G_SPLAT_VECTOR %hi33
+ %copy_sv_trunc_zero:_(<vscale x 2 x s32>) = COPY %sv_trunc_zero
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopySVTruncZero = Copies[Copies.size() - 1];
+ Register CopySVHi = Copies[Copies.size() - 2];
+ Register CopySV = Copies[Copies.size() - 3];
+ GISelValueTracking Info(*MF);
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopySV)->getOperand(1).getReg()));
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopySVHi)->getOperand(1).getReg()));
+ EXPECT_FALSE(Info.isKnownNeverZero(
+ MRI->getVRegDef(CopySVTruncZero)->getOperand(1).getReg()));
+}
+
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroFallbackKnownBits) {
+ // G_AND has no dedicated case in the switch, so the proof must come from the
+ // KnownBits fallback (which sees the bit-0 set by the inner G_OR).
+ StringRef MIRString = R"(
+ %one:_(s32) = G_CONSTANT i32 1
+ %allones:_(s32) = G_CONSTANT i32 -1
+ %x:_(s32) = COPY $w0
+ %or:_(s32) = G_OR %x, %one
+ %and:_(s32) = G_AND %or, %allones
+ %copy_and:_(s32) = COPY %and
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopyAnd = Copies[Copies.size() - 1];
+ GISelValueTracking Info(*MF);
+ EXPECT_TRUE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopyAnd)->getOperand(1).getReg()));
+}
+
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroDepthCutoff) {
+ // A deep chain of G_OR with one constant-one operand at the very bottom
+ // should be proved nonzero — but only if depth allows. Use a small depth
+ // limit to verify cutoff returns false.
+ StringRef MIRString = R"(
+ %one:_(s32) = G_CONSTANT i32 1
+ %x:_(s32) = COPY $w0
+ %or0:_(s32) = G_OR %x, %one
+ %or1:_(s32) = G_OR %x, %or0
+ %or2:_(s32) = G_OR %x, %or1
+ %or3:_(s32) = G_OR %x, %or2
+ %or4:_(s32) = G_OR %x, %or3
+ %or5:_(s32) = G_OR %x, %or4
+ %or6:_(s32) = G_OR %x, %or5
+ %copy_or:_(s32) = COPY %or6
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopyOr = Copies[Copies.size() - 1];
+ // Default MaxDepth (6) is shallower than the chain (7 nested ORs above the
+ // copy), so the recursive proof should give up rather than crash.
+ GISelValueTracking Info(*MF);
+ EXPECT_FALSE(
+ Info.isKnownNeverZero(MRI->getVRegDef(CopyOr)->getOperand(1).getReg()));
+}
>From d139bfdec286acd239258bcba6407f6fef4afa00 Mon Sep 17 00:00:00 2001
From: Alan Li <me at alanli.org>
Date: Sat, 16 May 2026 19:06:42 -0700
Subject: [PATCH 02/10] [GlobalISel][test] Add CTLZ/CTTZ -> *_ZERO_POISON MIR
test cases
---
.../combine-ctlz-cttz-zero-poison.mir | 112 ++++++++++++++++++
1 file changed, 112 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/GlobalISel/combine-ctlz-cttz-zero-poison.mir
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-ctlz-cttz-zero-poison.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-ctlz-cttz-zero-poison.mir
new file mode 100644
index 0000000000000..9bf32d36b1905
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-ctlz-cttz-zero-poison.mir
@@ -0,0 +1,112 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
+# RUN: llc -mtriple=aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
+
+# The constant-input cases below are reduced by `constant_fold_unary_int_op`
+# before the new `count_zero_poison_combines` rule runs. They guard that
+# behavior against accidental regression but do not exercise the new combine.
+# The combine itself is exercised by `ctlz_or_with_one`, `cttz_shl_one_nuw`,
+# and `cttz_select_both_nonzero` below.
+---
+name: ctlz_constant_nonzero
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: ctlz_constant_nonzero
+ ; CHECK: %r:_(s32) = G_CONSTANT i32 29
+ ; CHECK-NEXT: $w0 = COPY %r(s32)
+ %c:_(s32) = G_CONSTANT i32 5
+ %r:_(s32) = G_CTLZ %c
+ $w0 = COPY %r(s32)
+...
+---
+name: ctlz_constant_zero_no_combine
+body: |
+ bb.0:
+ ; CHECK-LABEL: name: ctlz_constant_zero_no_combine
+ ; CHECK: %r:_(s32) = G_CONSTANT i32 32
+ ; CHECK-NEXT: $w0 = COPY %r(s32)
+ %z:_(s32) = G_CONSTANT i32 0
+ %r:_(s32) = G_CTLZ %z
+ $w0 = COPY %r(s32)
+...
+---
+name: ctlz_unknown_no_combine
+body: |
+ bb.0:
+ liveins: $w0
+ ; CHECK-LABEL: name: ctlz_unknown_no_combine
+ ; CHECK: liveins: $w0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %x:_(s32) = COPY $w0
+ ; CHECK-NEXT: %r:_(s32) = G_CTLZ %x(s32)
+ ; CHECK-NEXT: $w0 = COPY %r(s32)
+ %x:_(s32) = COPY $w0
+ %r:_(s32) = G_CTLZ %x
+ $w0 = COPY %r(s32)
+...
+---
+name: ctlz_or_with_one
+body: |
+ bb.0:
+ liveins: $w0
+ ; CHECK-LABEL: name: ctlz_or_with_one
+ ; CHECK: liveins: $w0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %x:_(s32) = COPY $w0
+ ; CHECK-NEXT: %one:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %or:_(s32) = G_OR %x, %one
+ ; CHECK-NEXT: %r:_(s32) = G_CTLZ_ZERO_POISON %or(s32)
+ ; CHECK-NEXT: $w0 = COPY %r(s32)
+ %x:_(s32) = COPY $w0
+ %one:_(s32) = G_CONSTANT i32 1
+ %or:_(s32) = G_OR %x, %one
+ %r:_(s32) = G_CTLZ %or
+ $w0 = COPY %r(s32)
+...
+---
+name: cttz_shl_one_nuw
+body: |
+ bb.0:
+ liveins: $w0
+ ; CHECK-LABEL: name: cttz_shl_one_nuw
+ ; CHECK: liveins: $w0
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %x:_(s32) = COPY $w0
+ ; CHECK-NEXT: %one:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %shl:_(s32) = nuw G_SHL %one, %x(s32)
+ ; CHECK-NEXT: %r:_(s32) = G_CTTZ_ZERO_POISON %shl(s32)
+ ; CHECK-NEXT: $w0 = COPY %r(s32)
+ %x:_(s32) = COPY $w0
+ %one:_(s32) = G_CONSTANT i32 1
+ %shl:_(s32) = nuw G_SHL %one, %x
+ %r:_(s32) = G_CTTZ %shl
+ $w0 = COPY %r(s32)
+...
+---
+name: cttz_select_both_nonzero
+body: |
+ bb.0:
+ liveins: $w0, $w1, $w2
+ ; CHECK-LABEL: name: cttz_select_both_nonzero
+ ; CHECK: liveins: $w0, $w1, $w2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %x:_(s32) = COPY $w0
+ ; CHECK-NEXT: %y:_(s32) = COPY $w1
+ ; CHECK-NEXT: %c:_(s32) = COPY $w2
+ ; CHECK-NEXT: %cond:_(s1) = G_TRUNC %c(s32)
+ ; CHECK-NEXT: %one:_(s32) = G_CONSTANT i32 1
+ ; CHECK-NEXT: %xo:_(s32) = G_OR %x, %one
+ ; CHECK-NEXT: %yo:_(s32) = G_OR %y, %one
+ ; CHECK-NEXT: %sel:_(s32) = G_SELECT %cond(s1), %xo, %yo
+ ; CHECK-NEXT: %r:_(s32) = G_CTTZ_ZERO_POISON %sel(s32)
+ ; CHECK-NEXT: $w0 = COPY %r(s32)
+ %x:_(s32) = COPY $w0
+ %y:_(s32) = COPY $w1
+ %c:_(s32) = COPY $w2
+ %cond:_(s1) = G_TRUNC %c
+ %one:_(s32) = G_CONSTANT i32 1
+ %xo:_(s32) = G_OR %x, %one
+ %yo:_(s32) = G_OR %y, %one
+ %sel:_(s32) = G_SELECT %cond(s1), %xo, %yo
+ %r:_(s32) = G_CTTZ %sel
+ $w0 = COPY %r(s32)
+...
>From cd3e63eb2c38b591526486044515a74625f9d8f0 Mon Sep 17 00:00:00 2001
From: Alan Li <me at alanli.org>
Date: Sat, 16 May 2026 19:06:42 -0700
Subject: [PATCH 03/10] [GlobalISel] Combine G_CTLZ/G_CTTZ -> *_ZERO_POISON
when input nonzero
Mirror DAGCombiner.cpp:12394-12397/12431-12433: when the input to a
G_CTLZ/G_CTTZ is known never to be zero and the target supports the
G_CTLZ_ZERO_POISON / G_CTTZ_ZERO_POISON variant, switch to the poison
variant. First in-tree consumer of GISelValueTracking::isKnownNeverZero.
Implementation notes:
- applyCountZeroToZeroPoison: `replaceOpcodeWith` preserves Dst,
debug-loc, flags, and notifies the observer.
- matchCountZeroToZeroPoison: `getCountZeroPoisonOpcode` helper is the
single source of truth for the opcode map; `isKnownNeverZero` runs
before the legality query so cheap failure exits first.
- combine-ctlz-cttz-zero-poison.mir: the `ctlz_constant_nonzero` case
is omitted because `constant_fold_unary_int_op` already folds it
before the new rule runs.
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply at anthropic.com>
---
.../llvm/CodeGen/GlobalISel/CombinerHelper.h | 3 ++
.../include/llvm/Target/GlobalISel/Combine.td | 19 ++++++++++-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 32 +++++++++++++++++++
.../CodeGen/GlobalISel/GISelValueTracking.cpp | 1 -
.../combine-ctlz-cttz-zero-poison.mir | 17 +---------
5 files changed, 54 insertions(+), 18 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index a04ff991b2cf8..6e3a2c8157b7a 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -1177,6 +1177,9 @@ class CombinerHelper {
LLVM_ABI bool matchAVG(MachineInstr &MI, MachineRegisterInfo &MRI, Register X,
Register Y, unsigned TargetOpc) const;
+ bool matchCountZeroToZeroPoison(MachineInstr &MI) const;
+ void applyCountZeroToZeroPoison(MachineInstr &MI) const;
+
private:
/// Checks for legality of an indexed variant of \p LdSt.
bool isIndexedLoadStoreLegal(GLoadStore &LdSt) const;
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index c448a00068006..925244714e1d3 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -2337,6 +2337,22 @@ def ctls_combines : GICombineGroup<[
ctlz_zero_poison_to_ctls,
]>;
+// Switch G_CTLZ / G_CTTZ to *_ZERO_POISON when the input is provably nonzero
+// and the target supports the variant. Mirrors DAGCombiner.cpp:12394-12397 /
+// 12431-12433.
+class count_zero_to_zero_poison<Instruction opc> : GICombineRule<
+ (defs root:$root),
+ (match (opc $dst, $src):$root,
+ [{ return Helper.matchCountZeroToZeroPoison(*${root}); }]),
+ (apply [{ Helper.applyCountZeroToZeroPoison(*${root}); }])>;
+
+def ctlz_to_ctlz_zero_poison : count_zero_to_zero_poison<G_CTLZ>;
+def cttz_to_cttz_zero_poison : count_zero_to_zero_poison<G_CTTZ>;
+
+def count_zero_poison_combines : GICombineGroup<[
+ ctlz_to_ctlz_zero_poison, cttz_to_cttz_zero_poison
+]>;
+
def narrow_binop_add : narrow_binop_opcode<G_ADD>;
def narrow_binop_sub : narrow_binop_opcode<G_SUB>;
def narrow_binop_mul : narrow_binop_opcode<G_MUL>;
@@ -2548,7 +2564,8 @@ def all_combines : GICombineGroup<[integer_reassoc_combines, trivial_combines,
simplify_neg_minmax, combine_concat_vector,
sext_trunc, zext_trunc, prefer_sign_combines, shuffle_combines,
combine_use_vector_truncate, merge_combines, overflow_combines,
- truncsat_combines, lshr_of_trunc_of_lshr, ctls_combines, add_shift, sub_one_from_sub,
+ truncsat_combines, lshr_of_trunc_of_lshr, ctls_combines,
+ count_zero_poison_combines, add_shift, sub_one_from_sub,
binop_with_neg, sub_minus_one, sub_of_mul_const, max_min_combines,
avgfloor_u_match, avgfloor_s_match,
avgceil_u_match, avgceil_s_match, combine_or_of_and]>;
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index 3cc00d816c0e6..e2af6201f4243 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -8816,3 +8816,35 @@ bool CombinerHelper::matchAVG(MachineInstr &MI, MachineRegisterInfo &MRI,
LLT XTy = MRI.getType(X);
return XTy == MRI.getType(Y) && isLegal({TargetOpc, {XTy}});
}
+
+static unsigned getCountZeroPoisonOpcode(unsigned Opc) {
+ switch (Opc) {
+ case TargetOpcode::G_CTLZ:
+ return TargetOpcode::G_CTLZ_ZERO_POISON;
+ case TargetOpcode::G_CTTZ:
+ return TargetOpcode::G_CTTZ_ZERO_POISON;
+ default:
+ return 0;
+ }
+}
+
+bool CombinerHelper::matchCountZeroToZeroPoison(MachineInstr &MI) const {
+ if (!VT)
+ return false;
+
+ unsigned ZPOpc = getCountZeroPoisonOpcode(MI.getOpcode());
+ if (!ZPOpc)
+ return false;
+
+ Register Src = MI.getOperand(1).getReg();
+ if (!VT->isKnownNeverZero(Src))
+ return false;
+
+ LLT DstTy = MRI.getType(MI.getOperand(0).getReg());
+ LLT SrcTy = MRI.getType(Src);
+ return isLegalOrBeforeLegalizer({ZPOpc, {DstTy, SrcTy}});
+}
+
+void CombinerHelper::applyCountZeroToZeroPoison(MachineInstr &MI) const {
+ replaceOpcodeWith(MI, getCountZeroPoisonOpcode(MI.getOpcode()));
+}
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
index 8940a5ab91a95..ed32abd94b3ec 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
@@ -218,7 +218,6 @@ bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
KnownBits ValKnown = KnownOf(LHSReg);
if (MI->getOpcode() == TargetOpcode::G_ASHR && ValKnown.isNegative())
return true;
- // If max shift cnt of known ones is non-zero, result is non-zero.
APInt MaxCnt = KnownOf(MI->getOperand(2).getReg()).getMaxValue();
if (MaxCnt.ult(ValKnown.getBitWidth()) &&
!ValKnown.One.lshr(MaxCnt).isZero())
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/combine-ctlz-cttz-zero-poison.mir b/llvm/test/CodeGen/AArch64/GlobalISel/combine-ctlz-cttz-zero-poison.mir
index 9bf32d36b1905..329261c98a188 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/combine-ctlz-cttz-zero-poison.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/combine-ctlz-cttz-zero-poison.mir
@@ -1,22 +1,7 @@
# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py
# RUN: llc -mtriple=aarch64 -run-pass=aarch64-prelegalizer-combiner -verify-machineinstrs %s -o - | FileCheck %s
-# The constant-input cases below are reduced by `constant_fold_unary_int_op`
-# before the new `count_zero_poison_combines` rule runs. They guard that
-# behavior against accidental regression but do not exercise the new combine.
-# The combine itself is exercised by `ctlz_or_with_one`, `cttz_shl_one_nuw`,
-# and `cttz_select_both_nonzero` below.
----
-name: ctlz_constant_nonzero
-body: |
- bb.0:
- ; CHECK-LABEL: name: ctlz_constant_nonzero
- ; CHECK: %r:_(s32) = G_CONSTANT i32 29
- ; CHECK-NEXT: $w0 = COPY %r(s32)
- %c:_(s32) = G_CONSTANT i32 5
- %r:_(s32) = G_CTLZ %c
- $w0 = COPY %r(s32)
-...
+# Negative: ctlz(0) is defined as bitwidth and must not become poison.
---
name: ctlz_constant_zero_no_combine
body: |
>From 86eed6ee7b54c994509bba54b08da2ded1611f2f Mon Sep 17 00:00:00 2001
From: Alan Li <me at alanli.org>
Date: Mon, 18 May 2026 19:14:15 -0700
Subject: [PATCH 04/10] [GlobalISel][AMDGPU][test] Add end-to-end CTLZ/CTTZ
ZERO_POISON combine test
Adds llc-driven coverage that the new G_CTLZ/G_CTTZ -> *_ZERO_POISON
combine fires on AMDGPU when the input is provably nonzero. Without the
combine, AMDGPU custom-lowers G_CTLZ/G_CTTZ as v_ffbh_u32 / v_ffbl_b32
followed by v_min_u32 32 to clamp the hardware -1-on-zero result up to
the bitwidth; with the combine, the v_min disappears.
Test covers i32 ctlz/cttz of (or x, 1), i32 ctlz of (shl 1, n), and the
i64 ctlz/cttz of (or x, 1) where the legalizer-driven half-merge still
needs a min but no extra bitwidth clamp is added.
---
.../combine-ctlz-cttz-zero-poison.ll | 136 ++++++++++++++++++
1 file changed, 136 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/GlobalISel/combine-ctlz-cttz-zero-poison.ll
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-ctlz-cttz-zero-poison.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-ctlz-cttz-zero-poison.ll
new file mode 100644
index 0000000000000..82f45c9a2104b
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-ctlz-cttz-zero-poison.ll
@@ -0,0 +1,136 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -global-isel %s -o - | FileCheck -check-prefix=GCN %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -global-isel \
+; RUN: --amdgpuprelegalizercombiner-disable-rule=ctlz_to_ctlz_zero_poison,cttz_to_cttz_zero_poison \
+; RUN: --amdgpupostlegalizercombiner-disable-rule=ctlz_to_ctlz_zero_poison,cttz_to_cttz_zero_poison \
+; RUN: %s -o - | FileCheck -check-prefix=GCN-NOCOMBINE %s
+
+; Demonstrate the CTLZ/CTTZ -> *_ZERO_POISON combine on AMDGPU. AMDGPU
+; custom-lowers G_CTLZ/G_CTTZ as v_ffbh_u32 / v_ffbl_b32 followed by
+; `v_min_u32 32, ...` to clamp the hardware -1-on-zero result up to the
+; bitwidth. When the input is provably nonzero, the combine switches to
+; G_CTLZ_ZERO_POISON / G_CTTZ_ZERO_POISON and the clamp disappears.
+;
+; GCN = combine enabled (this PR) -> no v_min_u32 32 clamp
+; GCN-NOCOMBINE = combine disabled (baseline) -> v_min_u32 32 clamp present
+
+declare i32 @llvm.ctlz.i32(i32, i1)
+declare i32 @llvm.cttz.i32(i32, i1)
+declare i64 @llvm.ctlz.i64(i64, i1)
+declare i64 @llvm.cttz.i64(i64, i1)
+
+define i32 @ctlz_or1(i32 %x) {
+; GCN-LABEL: ctlz_or1:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_or_b32_e32 v0, 1, v0
+; GCN-NEXT: v_ffbh_u32_e32 v0, v0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-NOCOMBINE-LABEL: ctlz_or1:
+; GCN-NOCOMBINE: ; %bb.0:
+; GCN-NOCOMBINE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NOCOMBINE-NEXT: v_or_b32_e32 v0, 1, v0
+; GCN-NOCOMBINE-NEXT: v_ffbh_u32_e32 v0, v0
+; GCN-NOCOMBINE-NEXT: v_min_u32_e32 v0, 32, v0
+; GCN-NOCOMBINE-NEXT: s_setpc_b64 s[30:31]
+ %or = or i32 %x, 1
+ %r = call i32 @llvm.ctlz.i32(i32 %or, i1 false)
+ ret i32 %r
+}
+
+define i32 @cttz_or1(i32 %x) {
+; GCN-LABEL: cttz_or1:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_or_b32_e32 v0, 1, v0
+; GCN-NEXT: v_ffbl_b32_e32 v0, v0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-NOCOMBINE-LABEL: cttz_or1:
+; GCN-NOCOMBINE: ; %bb.0:
+; GCN-NOCOMBINE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NOCOMBINE-NEXT: v_or_b32_e32 v0, 1, v0
+; GCN-NOCOMBINE-NEXT: v_ffbl_b32_e32 v0, v0
+; GCN-NOCOMBINE-NEXT: v_min_u32_e32 v0, 32, v0
+; GCN-NOCOMBINE-NEXT: s_setpc_b64 s[30:31]
+ %or = or i32 %x, 1
+ %r = call i32 @llvm.cttz.i32(i32 %or, i1 false)
+ ret i32 %r
+}
+
+define i32 @ctlz_shl1(i32 %n) {
+; GCN-LABEL: ctlz_shl1:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_lshlrev_b32_e64 v0, v0, 1
+; GCN-NEXT: v_ffbh_u32_e32 v0, v0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-NOCOMBINE-LABEL: ctlz_shl1:
+; GCN-NOCOMBINE: ; %bb.0:
+; GCN-NOCOMBINE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NOCOMBINE-NEXT: v_lshlrev_b32_e64 v0, v0, 1
+; GCN-NOCOMBINE-NEXT: v_ffbh_u32_e32 v0, v0
+; GCN-NOCOMBINE-NEXT: v_min_u32_e32 v0, 32, v0
+; GCN-NOCOMBINE-NEXT: s_setpc_b64 s[30:31]
+ %s = shl i32 1, %n
+ %r = call i32 @llvm.ctlz.i32(i32 %s, i1 false)
+ ret i32 %r
+}
+
+define i64 @ctlz_or1_i64(i64 %x) {
+; GCN-LABEL: ctlz_or1_i64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_or_b32_e32 v0, 1, v0
+; GCN-NEXT: v_ffbh_u32_e32 v0, v0
+; GCN-NEXT: v_ffbh_u32_e32 v1, v1
+; GCN-NEXT: v_add_u32_e32 v0, 32, v0
+; GCN-NEXT: v_min_u32_e32 v0, v1, v0
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-NOCOMBINE-LABEL: ctlz_or1_i64:
+; GCN-NOCOMBINE: ; %bb.0:
+; GCN-NOCOMBINE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NOCOMBINE-NEXT: v_or_b32_e32 v0, 1, v0
+; GCN-NOCOMBINE-NEXT: v_ffbh_u32_e32 v0, v0
+; GCN-NOCOMBINE-NEXT: v_ffbh_u32_e32 v1, v1
+; GCN-NOCOMBINE-NEXT: v_add_u32_e64 v0, v0, 32 clamp
+; GCN-NOCOMBINE-NEXT: v_min_u32_e32 v0, v1, v0
+; GCN-NOCOMBINE-NEXT: v_min_u32_e32 v0, 64, v0
+; GCN-NOCOMBINE-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NOCOMBINE-NEXT: s_setpc_b64 s[30:31]
+ %or = or i64 %x, 1
+ %r = call i64 @llvm.ctlz.i64(i64 %or, i1 false)
+ ret i64 %r
+}
+
+define i64 @cttz_or1_i64(i64 %x) {
+; GCN-LABEL: cttz_or1_i64:
+; GCN: ; %bb.0:
+; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NEXT: v_or_b32_e32 v0, 1, v0
+; GCN-NEXT: v_ffbl_b32_e32 v1, v1
+; GCN-NEXT: v_ffbl_b32_e32 v0, v0
+; GCN-NEXT: v_add_u32_e32 v1, 32, v1
+; GCN-NEXT: v_min_u32_e32 v0, v0, v1
+; GCN-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NEXT: s_setpc_b64 s[30:31]
+;
+; GCN-NOCOMBINE-LABEL: cttz_or1_i64:
+; GCN-NOCOMBINE: ; %bb.0:
+; GCN-NOCOMBINE-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GCN-NOCOMBINE-NEXT: v_or_b32_e32 v0, 1, v0
+; GCN-NOCOMBINE-NEXT: v_ffbl_b32_e32 v1, v1
+; GCN-NOCOMBINE-NEXT: v_ffbl_b32_e32 v0, v0
+; GCN-NOCOMBINE-NEXT: v_add_u32_e64 v1, v1, 32 clamp
+; GCN-NOCOMBINE-NEXT: v_min_u32_e32 v0, v0, v1
+; GCN-NOCOMBINE-NEXT: v_min_u32_e32 v0, 64, v0
+; GCN-NOCOMBINE-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NOCOMBINE-NEXT: s_setpc_b64 s[30:31]
+ %or = or i64 %x, 1
+ %r = call i64 @llvm.cttz.i64(i64 %or, i1 false)
+ ret i64 %r
+}
>From 66df423e8f258d1969f7dba05a27619e2f13b8c3 Mon Sep 17 00:00:00 2001
From: Alan Li <me at alanli.org>
Date: Tue, 19 May 2026 08:14:16 -0700
Subject: [PATCH 05/10] [GlobalISel] Tighten isKnownNeverZero proofs
Do not look through G_FREEZE, since freezing poison can produce zero. Also avoid structural G_EXTRACT_VECTOR_ELT proofs across scalar-size changes and remove unreachable count-zero combine checks.
---
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 5 +-
.../CodeGen/GlobalISel/GISelValueTracking.cpp | 141 ++++++++++--------
.../CodeGen/GlobalISel/KnownBitsTest.cpp | 40 +++++
3 files changed, 118 insertions(+), 68 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index e2af6201f4243..a00a8a04db1b4 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -8824,7 +8824,7 @@ static unsigned getCountZeroPoisonOpcode(unsigned Opc) {
case TargetOpcode::G_CTTZ:
return TargetOpcode::G_CTTZ_ZERO_POISON;
default:
- return 0;
+ llvm_unreachable("Expected count-zero opcode");
}
}
@@ -8833,9 +8833,6 @@ bool CombinerHelper::matchCountZeroToZeroPoison(MachineInstr &MI) const {
return false;
unsigned ZPOpc = getCountZeroPoisonOpcode(MI.getOpcode());
- if (!ZPOpc)
- return false;
-
Register Src = MI.getOperand(1).getReg();
if (!VT->isKnownNeverZero(Src))
return false;
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
index ed32abd94b3ec..8dc0b21bace15 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
@@ -117,29 +117,20 @@ bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
if (Depth >= getMaxDepth())
return false;
- MachineInstr *MI = MRI.getVRegDef(R);
- if (!MI)
- return false;
-
- auto Recurse = [&](Register Reg, const APInt &Elts) {
- return isKnownNeverZero(Reg, Elts, Depth + 1);
- };
- auto KnownOf = [&](Register Reg) {
- return getKnownBits(Reg, DemandedElts, Depth + 1);
- };
+ MachineInstr &MI = *MRI.getVRegDef(R);
- switch (MI->getOpcode()) {
+ switch (MI.getOpcode()) {
default:
break;
case TargetOpcode::G_CONSTANT:
- return !MI->getOperand(1).getCImm()->isZero();
+ return !MI.getOperand(1).getCImm()->isZero();
case TargetOpcode::G_BUILD_VECTOR: {
- for (const auto &[I, MO] : enumerate(drop_begin(MI->operands()))) {
+ for (const auto &[I, MO] : enumerate(drop_begin(MI.operands()))) {
if (!DemandedElts[I])
continue;
- if (!Recurse(MO.getReg(), APInt(1, 1)))
+ if (!isKnownNeverZero(MO.getReg(), APInt(1, 1), Depth + 1))
return false;
}
return true;
@@ -150,29 +141,31 @@ bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
// element type, with the high bits implicitly truncated. Structural
// recursion ignores width, so only recurse when the widths already match;
// otherwise drop to KnownBits and truncate explicitly.
- Register Src = MI->getOperand(1).getReg();
+ Register Src = MI.getOperand(1).getReg();
unsigned EltBW = MRI.getType(R).getScalarSizeInBits();
if (MRI.getType(Src).getSizeInBits() == EltBW)
- return Recurse(Src, APInt(1, 1));
+ return isKnownNeverZero(Src, APInt(1, 1), Depth + 1);
return getKnownBits(Src, APInt(1, 1), Depth + 1).trunc(EltBW).isNonZero();
}
case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
- GExtractVectorElement &Extract = cast<GExtractVectorElement>(*MI);
+ GExtractVectorElement &Extract = cast<GExtractVectorElement>(MI);
Register InVec = Extract.getVectorReg();
LLT VecTy = MRI.getType(InVec);
if (VecTy.isScalableVector())
break;
+ if (MRI.getType(R).getScalarSizeInBits() != VecTy.getScalarSizeInBits())
+ return false;
unsigned NumSrcElts = VecTy.getNumElements();
APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
if (auto Idx = getIConstantVRegVal(Extract.getIndexReg(), MRI))
if (Idx->ult(NumSrcElts))
DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, Idx->getZExtValue());
- return Recurse(InVec, DemandedSrcElts);
+ return isKnownNeverZero(InVec, DemandedSrcElts, Depth + 1);
}
case TargetOpcode::G_SHUFFLE_VECTOR: {
- GShuffleVector &Shuf = cast<GShuffleVector>(*MI);
+ GShuffleVector &Shuf = cast<GShuffleVector>(MI);
LLT SrcTy = MRI.getType(Shuf.getSrc1Reg());
if (SrcTy.isScalableVector())
break;
@@ -180,30 +173,35 @@ bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
if (!getShuffleDemandedElts(SrcTy.getNumElements(), Shuf.getMask(),
DemandedElts, DemandedLHS, DemandedRHS))
break;
- if (!!DemandedLHS && !Recurse(Shuf.getSrc1Reg(), DemandedLHS))
+ if (!!DemandedLHS &&
+ !isKnownNeverZero(Shuf.getSrc1Reg(), DemandedLHS, Depth + 1))
return false;
- if (!!DemandedRHS && !Recurse(Shuf.getSrc2Reg(), DemandedRHS))
+ if (!!DemandedRHS &&
+ !isKnownNeverZero(Shuf.getSrc2Reg(), DemandedRHS, Depth + 1))
return false;
return true;
}
case TargetOpcode::G_OR:
- return Recurse(MI->getOperand(1).getReg(), DemandedElts) ||
- Recurse(MI->getOperand(2).getReg(), DemandedElts);
+ return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
+ Depth + 1) ||
+ isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts, Depth + 1);
case TargetOpcode::G_SELECT:
- return Recurse(MI->getOperand(2).getReg(), DemandedElts) &&
- Recurse(MI->getOperand(3).getReg(), DemandedElts);
+ return isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts,
+ Depth + 1) &&
+ isKnownNeverZero(MI.getOperand(3).getReg(), DemandedElts, Depth + 1);
case TargetOpcode::G_SHL: {
- Register LHSReg = MI->getOperand(1).getReg();
- if (MI->getFlag(MachineInstr::NoSWrap) ||
- MI->getFlag(MachineInstr::NoUWrap))
- return Recurse(LHSReg, DemandedElts);
- KnownBits ValKnown = KnownOf(LHSReg);
+ Register LHSReg = MI.getOperand(1).getReg();
+ if (MI.getFlag(MachineInstr::NoSWrap) || MI.getFlag(MachineInstr::NoUWrap))
+ return isKnownNeverZero(LHSReg, DemandedElts, Depth + 1);
+ KnownBits ValKnown = getKnownBits(LHSReg, DemandedElts, Depth + 1);
if (ValKnown.One[0])
return true;
- APInt MaxCnt = KnownOf(MI->getOperand(2).getReg()).getMaxValue();
+ APInt MaxCnt =
+ getKnownBits(MI.getOperand(2).getReg(), DemandedElts, Depth + 1)
+ .getMaxValue();
if (MaxCnt.ult(ValKnown.getBitWidth()) &&
!ValKnown.One.shl(MaxCnt).isZero())
return true;
@@ -212,13 +210,15 @@ bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
case TargetOpcode::G_LSHR:
case TargetOpcode::G_ASHR: {
- Register LHSReg = MI->getOperand(1).getReg();
- if (MI->getFlag(MachineInstr::IsExact))
- return Recurse(LHSReg, DemandedElts);
- KnownBits ValKnown = KnownOf(LHSReg);
- if (MI->getOpcode() == TargetOpcode::G_ASHR && ValKnown.isNegative())
+ Register LHSReg = MI.getOperand(1).getReg();
+ if (MI.getFlag(MachineInstr::IsExact))
+ return isKnownNeverZero(LHSReg, DemandedElts, Depth + 1);
+ KnownBits ValKnown = getKnownBits(LHSReg, DemandedElts, Depth + 1);
+ if (MI.getOpcode() == TargetOpcode::G_ASHR && ValKnown.isNegative())
return true;
- APInt MaxCnt = KnownOf(MI->getOperand(2).getReg()).getMaxValue();
+ APInt MaxCnt =
+ getKnownBits(MI.getOperand(2).getReg(), DemandedElts, Depth + 1)
+ .getMaxValue();
if (MaxCnt.ult(ValKnown.getBitWidth()) &&
!ValKnown.One.lshr(MaxCnt).isZero())
return true;
@@ -227,61 +227,75 @@ bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
case TargetOpcode::G_UDIV:
case TargetOpcode::G_SDIV:
- if (MI->getFlag(MachineInstr::IsExact))
- return Recurse(MI->getOperand(1).getReg(), DemandedElts);
+ if (MI.getFlag(MachineInstr::IsExact))
+ return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
+ Depth + 1);
break;
case TargetOpcode::G_ADD:
- if (MI->getFlag(MachineInstr::NoUWrap))
- return Recurse(MI->getOperand(1).getReg(), DemandedElts) ||
- Recurse(MI->getOperand(2).getReg(), DemandedElts);
+ if (MI.getFlag(MachineInstr::NoUWrap))
+ return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
+ Depth + 1) ||
+ isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts,
+ Depth + 1);
break;
case TargetOpcode::G_SUB: {
- Register LHSReg = MI->getOperand(1).getReg();
- Register RHSReg = MI->getOperand(2).getReg();
+ Register LHSReg = MI.getOperand(1).getReg();
+ Register RHSReg = MI.getOperand(2).getReg();
if (auto LHSC = getIConstantVRegVal(LHSReg, MRI))
if (LHSC->isZero())
- return Recurse(RHSReg, DemandedElts);
- return KnownBits::ne(KnownOf(LHSReg), KnownOf(RHSReg)).value_or(false);
+ return isKnownNeverZero(RHSReg, DemandedElts, Depth + 1);
+ return KnownBits::ne(getKnownBits(LHSReg, DemandedElts, Depth + 1),
+ getKnownBits(RHSReg, DemandedElts, Depth + 1))
+ .value_or(false);
}
case TargetOpcode::G_MUL:
- if (MI->getFlag(MachineInstr::NoUWrap) ||
- MI->getFlag(MachineInstr::NoSWrap))
- return Recurse(MI->getOperand(1).getReg(), DemandedElts) &&
- Recurse(MI->getOperand(2).getReg(), DemandedElts);
+ if (MI.getFlag(MachineInstr::NoUWrap) || MI.getFlag(MachineInstr::NoSWrap))
+ return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
+ Depth + 1) &&
+ isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts,
+ Depth + 1);
break;
case TargetOpcode::G_UADDSAT:
case TargetOpcode::G_UMAX:
- return Recurse(MI->getOperand(1).getReg(), DemandedElts) ||
- Recurse(MI->getOperand(2).getReg(), DemandedElts);
+ return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
+ Depth + 1) ||
+ isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts, Depth + 1);
case TargetOpcode::G_UMIN:
- return Recurse(MI->getOperand(1).getReg(), DemandedElts) &&
- Recurse(MI->getOperand(2).getReg(), DemandedElts);
+ return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
+ Depth + 1) &&
+ isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts, Depth + 1);
case TargetOpcode::G_SMAX: {
- KnownBits L = KnownOf(MI->getOperand(1).getReg());
- KnownBits R2 = KnownOf(MI->getOperand(2).getReg());
+ KnownBits L =
+ getKnownBits(MI.getOperand(1).getReg(), DemandedElts, Depth + 1);
+ KnownBits R2 =
+ getKnownBits(MI.getOperand(2).getReg(), DemandedElts, Depth + 1);
if (L.isStrictlyPositive() || R2.isStrictlyPositive())
return true;
if (L.isNonZero() && R2.isNonZero())
return true;
- return Recurse(MI->getOperand(1).getReg(), DemandedElts) &&
- Recurse(MI->getOperand(2).getReg(), DemandedElts);
+ return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
+ Depth + 1) &&
+ isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts, Depth + 1);
}
case TargetOpcode::G_SMIN: {
- KnownBits L = KnownOf(MI->getOperand(1).getReg());
- KnownBits R2 = KnownOf(MI->getOperand(2).getReg());
+ KnownBits L =
+ getKnownBits(MI.getOperand(1).getReg(), DemandedElts, Depth + 1);
+ KnownBits R2 =
+ getKnownBits(MI.getOperand(2).getReg(), DemandedElts, Depth + 1);
if (L.isNegative() || R2.isNegative())
return true;
if (L.isNonZero() && R2.isNonZero())
return true;
- return Recurse(MI->getOperand(1).getReg(), DemandedElts) &&
- Recurse(MI->getOperand(2).getReg(), DemandedElts);
+ return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
+ Depth + 1) &&
+ isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts, Depth + 1);
}
case TargetOpcode::G_ROTL:
@@ -292,8 +306,7 @@ bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
case TargetOpcode::G_ABS:
case TargetOpcode::G_ZEXT:
case TargetOpcode::G_SEXT:
- case TargetOpcode::G_FREEZE:
- return Recurse(MI->getOperand(1).getReg(), DemandedElts);
+ return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts, Depth + 1);
}
// Pass through this frame's Depth (not Depth+1) because we have not recursed
diff --git a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
index 0ede4e9e2f518..eda97b5bca420 100644
--- a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
@@ -2244,6 +2244,23 @@ TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroShlOneByVar) {
Info.isKnownNeverZero(MRI->getVRegDef(CopyShl)->getOperand(1).getReg()));
}
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroFreezeShlOneByVar) {
+ StringRef MIRString = R"(
+ %one:_(s32) = G_CONSTANT i32 1
+ %x:_(s32) = COPY $w0
+ %shl:_(s32) = G_SHL %one, %x
+ %freeze:_(s32) = G_FREEZE %shl
+ %copy_freeze:_(s32) = COPY %freeze
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopyFreeze = Copies[Copies.size() - 1];
+ GISelValueTracking Info(*MF);
+ EXPECT_FALSE(Info.isKnownNeverZero(
+ MRI->getVRegDef(CopyFreeze)->getOperand(1).getReg()));
+}
+
TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroAshrExact) {
StringRef MIRString = R"(
%one:_(s32) = G_CONSTANT i32 1
@@ -2326,6 +2343,29 @@ TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroBuildVector) {
Info.isKnownNeverZero(MRI->getVRegDef(CopyBVZ)->getOperand(1).getReg()));
}
+TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroExtractVectorElt) {
+ StringRef MIRString = R"(
+ %hi33:_(s64) = G_CONSTANT i64 4294967296
+ %vec:_(<2 x s64>) = G_BUILD_VECTOR %hi33, %hi33
+ %idx:_(s64) = G_CONSTANT i64 0
+ %extract64:_(s64) = G_EXTRACT_VECTOR_ELT %vec, %idx
+ %copy_extract64:_(s64) = COPY %extract64
+
+ %extract32:_(s32) = G_EXTRACT_VECTOR_ELT %vec, %idx
+ %copy_extract32:_(s32) = COPY %extract32
+)";
+ setUp(MIRString);
+ if (!TM)
+ GTEST_SKIP();
+ Register CopyExtract32 = Copies[Copies.size() - 1];
+ Register CopyExtract64 = Copies[Copies.size() - 2];
+ GISelValueTracking Info(*MF);
+ EXPECT_TRUE(Info.isKnownNeverZero(
+ MRI->getVRegDef(CopyExtract64)->getOperand(1).getReg()));
+ EXPECT_FALSE(Info.isKnownNeverZero(
+ MRI->getVRegDef(CopyExtract32)->getOperand(1).getReg()));
+}
+
TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroSplatVector) {
// Same-width splat: recurses structurally on the source.
// Wide-source splat: 0x100 (i64) splatted into i32 lanes is still nonzero;
>From 56174131e4a172a403190669f84a56f43950402c Mon Sep 17 00:00:00 2001
From: Alan Li <me at alanli.org>
Date: Wed, 17 Jun 2026 08:47:04 -0700
Subject: [PATCH 06/10] [GlobalISel] Address isKnownNeverZero review comments
---
.../CodeGen/GlobalISel/GISelValueTracking.h | 7 +-
.../include/llvm/Target/GlobalISel/Combine.td | 4 +-
.../lib/CodeGen/GlobalISel/CombinerHelper.cpp | 13 +-
.../CodeGen/GlobalISel/GISelValueTracking.cpp | 115 ++----------------
.../CodeGen/GlobalISel/KnownBitsTest.cpp | 59 ---------
5 files changed, 25 insertions(+), 173 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h b/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h
index d1d173547c034..b86d832bb3b65 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h
@@ -92,8 +92,11 @@ class LLVM_ABI GISelValueTracking : public GISelChangeObserver {
/// predicate to simplify operations downstream.
bool signBitIsZero(Register Op);
- /// Return true if the value defined by \p R is provably never zero. This is
- /// the GlobalISel analog of \c SelectionDAG::isKnownNeverZero.
+ /// Return true if the value defined by \p R is provably never zero.
+ ///
+ /// \p DemandedElts selects the vector elements that must be proven nonzero.
+ /// For scalar values this is a one-bit mask. The overload without
+ /// \p DemandedElts demands every fixed-vector element, or the scalar value.
bool isKnownNeverZero(Register R, unsigned Depth = 0);
bool isKnownNeverZero(Register R, const APInt &DemandedElts,
unsigned Depth = 0);
diff --git a/llvm/include/llvm/Target/GlobalISel/Combine.td b/llvm/include/llvm/Target/GlobalISel/Combine.td
index 925244714e1d3..9d0f30f67c8f1 100644
--- a/llvm/include/llvm/Target/GlobalISel/Combine.td
+++ b/llvm/include/llvm/Target/GlobalISel/Combine.td
@@ -2338,8 +2338,8 @@ def ctls_combines : GICombineGroup<[
]>;
// Switch G_CTLZ / G_CTTZ to *_ZERO_POISON when the input is provably nonzero
-// and the target supports the variant. Mirrors DAGCombiner.cpp:12394-12397 /
-// 12431-12433.
+// and the target supports the variant. The zero-poison variants compute the
+// same result for nonzero inputs and can let targets skip zero-input fixups.
class count_zero_to_zero_poison<Instruction opc> : GICombineRule<
(defs root:$root),
(match (opc $dst, $src):$root,
diff --git a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
index a00a8a04db1b4..56af7c472f024 100644
--- a/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/CombinerHelper.cpp
@@ -8817,14 +8817,17 @@ bool CombinerHelper::matchAVG(MachineInstr &MI, MachineRegisterInfo &MRI,
return XTy == MRI.getType(Y) && isLegal({TargetOpc, {XTy}});
}
-static unsigned getCountZeroPoisonOpcode(unsigned Opc) {
- switch (Opc) {
+static unsigned getCountZeroPoisonOpcode(const MachineInstr &MI) {
+ assert((MI.getOpcode() == TargetOpcode::G_CTLZ ||
+ MI.getOpcode() == TargetOpcode::G_CTTZ) &&
+ "Expected count-zero opcode");
+ switch (MI.getOpcode()) {
case TargetOpcode::G_CTLZ:
return TargetOpcode::G_CTLZ_ZERO_POISON;
case TargetOpcode::G_CTTZ:
return TargetOpcode::G_CTTZ_ZERO_POISON;
default:
- llvm_unreachable("Expected count-zero opcode");
+ llvm_unreachable("Unexpected count-zero opcode");
}
}
@@ -8832,7 +8835,7 @@ bool CombinerHelper::matchCountZeroToZeroPoison(MachineInstr &MI) const {
if (!VT)
return false;
- unsigned ZPOpc = getCountZeroPoisonOpcode(MI.getOpcode());
+ unsigned ZPOpc = getCountZeroPoisonOpcode(MI);
Register Src = MI.getOperand(1).getReg();
if (!VT->isKnownNeverZero(Src))
return false;
@@ -8843,5 +8846,5 @@ bool CombinerHelper::matchCountZeroToZeroPoison(MachineInstr &MI) const {
}
void CombinerHelper::applyCountZeroToZeroPoison(MachineInstr &MI) const {
- replaceOpcodeWith(MI, getCountZeroPoisonOpcode(MI.getOpcode()));
+ replaceOpcodeWith(MI, getCountZeroPoisonOpcode(MI));
}
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
index 8dc0b21bace15..c621441dc2086 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
@@ -107,8 +107,10 @@ bool GISelValueTracking::signBitIsZero(Register R) {
bool GISelValueTracking::isKnownNeverZero(Register R, unsigned Depth) {
LLT Ty = MRI.getType(R);
- APInt DemandedElts =
- Ty.isFixedVector() ? APInt::getAllOnes(Ty.getNumElements()) : APInt(1, 1);
+ const APInt ScalarDemandedElts(1, 1);
+ APInt DemandedElts = Ty.isFixedVector()
+ ? APInt::getAllOnes(Ty.getNumElements())
+ : ScalarDemandedElts;
return isKnownNeverZero(R, DemandedElts, Depth);
}
@@ -117,6 +119,7 @@ bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
if (Depth >= getMaxDepth())
return false;
+ const APInt ScalarDemandedElts(1, 1);
MachineInstr &MI = *MRI.getVRegDef(R);
switch (MI.getOpcode()) {
@@ -130,7 +133,7 @@ bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
for (const auto &[I, MO] : enumerate(drop_begin(MI.operands()))) {
if (!DemandedElts[I])
continue;
- if (!isKnownNeverZero(MO.getReg(), APInt(1, 1), Depth + 1))
+ if (!isKnownNeverZero(MO.getReg(), ScalarDemandedElts, Depth + 1))
return false;
}
return true;
@@ -144,8 +147,10 @@ bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
Register Src = MI.getOperand(1).getReg();
unsigned EltBW = MRI.getType(R).getScalarSizeInBits();
if (MRI.getType(Src).getSizeInBits() == EltBW)
- return isKnownNeverZero(Src, APInt(1, 1), Depth + 1);
- return getKnownBits(Src, APInt(1, 1), Depth + 1).trunc(EltBW).isNonZero();
+ return isKnownNeverZero(Src, ScalarDemandedElts, Depth + 1);
+ return getKnownBits(Src, ScalarDemandedElts, Depth + 1)
+ .trunc(EltBW)
+ .isNonZero();
}
case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
@@ -207,106 +212,6 @@ bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
return true;
break;
}
-
- case TargetOpcode::G_LSHR:
- case TargetOpcode::G_ASHR: {
- Register LHSReg = MI.getOperand(1).getReg();
- if (MI.getFlag(MachineInstr::IsExact))
- return isKnownNeverZero(LHSReg, DemandedElts, Depth + 1);
- KnownBits ValKnown = getKnownBits(LHSReg, DemandedElts, Depth + 1);
- if (MI.getOpcode() == TargetOpcode::G_ASHR && ValKnown.isNegative())
- return true;
- APInt MaxCnt =
- getKnownBits(MI.getOperand(2).getReg(), DemandedElts, Depth + 1)
- .getMaxValue();
- if (MaxCnt.ult(ValKnown.getBitWidth()) &&
- !ValKnown.One.lshr(MaxCnt).isZero())
- return true;
- break;
- }
-
- case TargetOpcode::G_UDIV:
- case TargetOpcode::G_SDIV:
- if (MI.getFlag(MachineInstr::IsExact))
- return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
- Depth + 1);
- break;
-
- case TargetOpcode::G_ADD:
- if (MI.getFlag(MachineInstr::NoUWrap))
- return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
- Depth + 1) ||
- isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts,
- Depth + 1);
- break;
-
- case TargetOpcode::G_SUB: {
- Register LHSReg = MI.getOperand(1).getReg();
- Register RHSReg = MI.getOperand(2).getReg();
- if (auto LHSC = getIConstantVRegVal(LHSReg, MRI))
- if (LHSC->isZero())
- return isKnownNeverZero(RHSReg, DemandedElts, Depth + 1);
- return KnownBits::ne(getKnownBits(LHSReg, DemandedElts, Depth + 1),
- getKnownBits(RHSReg, DemandedElts, Depth + 1))
- .value_or(false);
- }
-
- case TargetOpcode::G_MUL:
- if (MI.getFlag(MachineInstr::NoUWrap) || MI.getFlag(MachineInstr::NoSWrap))
- return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
- Depth + 1) &&
- isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts,
- Depth + 1);
- break;
-
- case TargetOpcode::G_UADDSAT:
- case TargetOpcode::G_UMAX:
- return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
- Depth + 1) ||
- isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts, Depth + 1);
-
- case TargetOpcode::G_UMIN:
- return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
- Depth + 1) &&
- isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts, Depth + 1);
-
- case TargetOpcode::G_SMAX: {
- KnownBits L =
- getKnownBits(MI.getOperand(1).getReg(), DemandedElts, Depth + 1);
- KnownBits R2 =
- getKnownBits(MI.getOperand(2).getReg(), DemandedElts, Depth + 1);
- if (L.isStrictlyPositive() || R2.isStrictlyPositive())
- return true;
- if (L.isNonZero() && R2.isNonZero())
- return true;
- return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
- Depth + 1) &&
- isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts, Depth + 1);
- }
-
- case TargetOpcode::G_SMIN: {
- KnownBits L =
- getKnownBits(MI.getOperand(1).getReg(), DemandedElts, Depth + 1);
- KnownBits R2 =
- getKnownBits(MI.getOperand(2).getReg(), DemandedElts, Depth + 1);
- if (L.isNegative() || R2.isNegative())
- return true;
- if (L.isNonZero() && R2.isNonZero())
- return true;
- return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
- Depth + 1) &&
- isKnownNeverZero(MI.getOperand(2).getReg(), DemandedElts, Depth + 1);
- }
-
- case TargetOpcode::G_ROTL:
- case TargetOpcode::G_ROTR:
- case TargetOpcode::G_BSWAP:
- case TargetOpcode::G_BITREVERSE:
- case TargetOpcode::G_CTPOP:
- case TargetOpcode::G_ABS:
- case TargetOpcode::G_ZEXT:
- case TargetOpcode::G_SEXT:
- return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts, Depth + 1);
}
// Pass through this frame's Depth (not Depth+1) because we have not recursed
diff --git a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
index eda97b5bca420..ecd0ec8f6234b 100644
--- a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
@@ -2261,65 +2261,6 @@ TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroFreezeShlOneByVar) {
MRI->getVRegDef(CopyFreeze)->getOperand(1).getReg()));
}
-TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroAshrExact) {
- StringRef MIRString = R"(
- %one:_(s32) = G_CONSTANT i32 1
- %x:_(s32) = COPY $w0
- %ashr:_(s32) = exact G_ASHR %one, %x
- %copy_ashr:_(s32) = COPY %ashr
-)";
- setUp(MIRString);
- if (!TM)
- GTEST_SKIP();
- Register CopyAshr = Copies[Copies.size() - 1];
- GISelValueTracking Info(*MF);
- EXPECT_TRUE(
- Info.isKnownNeverZero(MRI->getVRegDef(CopyAshr)->getOperand(1).getReg()));
-}
-
-TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroUMaxUMin) {
- StringRef MIRString = R"(
- %one:_(s32) = G_CONSTANT i32 1
- %x:_(s32) = COPY $w0
- %umax:_(s32) = G_UMAX %one, %x
- %copy_umax:_(s32) = COPY %umax
-
- %two:_(s32) = G_CONSTANT i32 2
- %umin:_(s32) = G_UMIN %one, %two
- %copy_umin:_(s32) = COPY %umin
-)";
- setUp(MIRString);
- if (!TM)
- GTEST_SKIP();
- Register CopyUMin = Copies[Copies.size() - 1];
- Register CopyUMax = Copies[Copies.size() - 2];
- GISelValueTracking Info(*MF);
- EXPECT_TRUE(
- Info.isKnownNeverZero(MRI->getVRegDef(CopyUMax)->getOperand(1).getReg()));
- EXPECT_TRUE(
- Info.isKnownNeverZero(MRI->getVRegDef(CopyUMin)->getOperand(1).getReg()));
-}
-
-TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroZextSext) {
- StringRef MIRString = R"(
- %one:_(s8) = G_CONSTANT i8 1
- %zext:_(s32) = G_ZEXT %one
- %sext:_(s32) = G_SEXT %one
- %copy_zext:_(s32) = COPY %zext
- %copy_sext:_(s32) = COPY %sext
-)";
- setUp(MIRString);
- if (!TM)
- GTEST_SKIP();
- Register CopySext = Copies[Copies.size() - 1];
- Register CopyZext = Copies[Copies.size() - 2];
- GISelValueTracking Info(*MF);
- EXPECT_TRUE(
- Info.isKnownNeverZero(MRI->getVRegDef(CopyZext)->getOperand(1).getReg()));
- EXPECT_TRUE(
- Info.isKnownNeverZero(MRI->getVRegDef(CopySext)->getOperand(1).getReg()));
-}
-
TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroBuildVector) {
StringRef MIRString = R"(
%one:_(s32) = G_CONSTANT i32 1
>From 8c5e451f98ca67cca5a54b7e9feb53bcd2ac25ef Mon Sep 17 00:00:00 2001
From: Alan Li <me at alanli.org>
Date: Wed, 17 Jun 2026 09:01:05 -0700
Subject: [PATCH 07/10] [GlobalISel] Remove G_CONSTANT nonzero special case
---
llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp | 3 ---
.../AMDGPU/GlobalISel/combine-ctlz-cttz-zero-poison.ll | 6 ++----
2 files changed, 2 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
index c621441dc2086..c10cb3dbaf195 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
@@ -126,9 +126,6 @@ bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
default:
break;
- case TargetOpcode::G_CONSTANT:
- return !MI.getOperand(1).getCImm()->isZero();
-
case TargetOpcode::G_BUILD_VECTOR: {
for (const auto &[I, MO] : enumerate(drop_begin(MI.operands()))) {
if (!DemandedElts[I])
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-ctlz-cttz-zero-poison.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-ctlz-cttz-zero-poison.ll
index 82f45c9a2104b..c292117f371f3 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-ctlz-cttz-zero-poison.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/combine-ctlz-cttz-zero-poison.ll
@@ -98,8 +98,7 @@ define i64 @ctlz_or1_i64(i64 %x) {
; GCN-NOCOMBINE-NEXT: v_ffbh_u32_e32 v0, v0
; GCN-NOCOMBINE-NEXT: v_ffbh_u32_e32 v1, v1
; GCN-NOCOMBINE-NEXT: v_add_u32_e64 v0, v0, 32 clamp
-; GCN-NOCOMBINE-NEXT: v_min_u32_e32 v0, v1, v0
-; GCN-NOCOMBINE-NEXT: v_min_u32_e32 v0, 64, v0
+; GCN-NOCOMBINE-NEXT: v_min3_u32 v0, v1, v0, 64
; GCN-NOCOMBINE-NEXT: v_mov_b32_e32 v1, 0
; GCN-NOCOMBINE-NEXT: s_setpc_b64 s[30:31]
%or = or i64 %x, 1
@@ -126,8 +125,7 @@ define i64 @cttz_or1_i64(i64 %x) {
; GCN-NOCOMBINE-NEXT: v_ffbl_b32_e32 v1, v1
; GCN-NOCOMBINE-NEXT: v_ffbl_b32_e32 v0, v0
; GCN-NOCOMBINE-NEXT: v_add_u32_e64 v1, v1, 32 clamp
-; GCN-NOCOMBINE-NEXT: v_min_u32_e32 v0, v0, v1
-; GCN-NOCOMBINE-NEXT: v_min_u32_e32 v0, 64, v0
+; GCN-NOCOMBINE-NEXT: v_min3_u32 v0, v0, v1, 64
; GCN-NOCOMBINE-NEXT: v_mov_b32_e32 v1, 0
; GCN-NOCOMBINE-NEXT: s_setpc_b64 s[30:31]
%or = or i64 %x, 1
>From cda060ffdfbba28bfd68c33348d552f5be2da3b1 Mon Sep 17 00:00:00 2001
From: Alan Li <me at alanli.org>
Date: Wed, 17 Jun 2026 09:07:06 -0700
Subject: [PATCH 08/10] [GlobalISel] Limit initial isKnownNeverZero cases
---
.../CodeGen/GlobalISel/GISelValueTracking.cpp | 59 --------------
.../CodeGen/GlobalISel/KnownBitsTest.cpp | 78 -------------------
2 files changed, 137 deletions(-)
diff --git a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
index c10cb3dbaf195..50266b8cc8623 100644
--- a/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/GISelValueTracking.cpp
@@ -119,71 +119,12 @@ bool GISelValueTracking::isKnownNeverZero(Register R, const APInt &DemandedElts,
if (Depth >= getMaxDepth())
return false;
- const APInt ScalarDemandedElts(1, 1);
MachineInstr &MI = *MRI.getVRegDef(R);
switch (MI.getOpcode()) {
default:
break;
- case TargetOpcode::G_BUILD_VECTOR: {
- for (const auto &[I, MO] : enumerate(drop_begin(MI.operands()))) {
- if (!DemandedElts[I])
- continue;
- if (!isKnownNeverZero(MO.getReg(), ScalarDemandedElts, Depth + 1))
- return false;
- }
- return true;
- }
-
- case TargetOpcode::G_SPLAT_VECTOR: {
- // G_SPLAT_VECTOR allows its source scalar to be wider than the destination
- // element type, with the high bits implicitly truncated. Structural
- // recursion ignores width, so only recurse when the widths already match;
- // otherwise drop to KnownBits and truncate explicitly.
- Register Src = MI.getOperand(1).getReg();
- unsigned EltBW = MRI.getType(R).getScalarSizeInBits();
- if (MRI.getType(Src).getSizeInBits() == EltBW)
- return isKnownNeverZero(Src, ScalarDemandedElts, Depth + 1);
- return getKnownBits(Src, ScalarDemandedElts, Depth + 1)
- .trunc(EltBW)
- .isNonZero();
- }
-
- case TargetOpcode::G_EXTRACT_VECTOR_ELT: {
- GExtractVectorElement &Extract = cast<GExtractVectorElement>(MI);
- Register InVec = Extract.getVectorReg();
- LLT VecTy = MRI.getType(InVec);
- if (VecTy.isScalableVector())
- break;
- if (MRI.getType(R).getScalarSizeInBits() != VecTy.getScalarSizeInBits())
- return false;
- unsigned NumSrcElts = VecTy.getNumElements();
- APInt DemandedSrcElts = APInt::getAllOnes(NumSrcElts);
- if (auto Idx = getIConstantVRegVal(Extract.getIndexReg(), MRI))
- if (Idx->ult(NumSrcElts))
- DemandedSrcElts = APInt::getOneBitSet(NumSrcElts, Idx->getZExtValue());
- return isKnownNeverZero(InVec, DemandedSrcElts, Depth + 1);
- }
-
- case TargetOpcode::G_SHUFFLE_VECTOR: {
- GShuffleVector &Shuf = cast<GShuffleVector>(MI);
- LLT SrcTy = MRI.getType(Shuf.getSrc1Reg());
- if (SrcTy.isScalableVector())
- break;
- APInt DemandedLHS, DemandedRHS;
- if (!getShuffleDemandedElts(SrcTy.getNumElements(), Shuf.getMask(),
- DemandedElts, DemandedLHS, DemandedRHS))
- break;
- if (!!DemandedLHS &&
- !isKnownNeverZero(Shuf.getSrc1Reg(), DemandedLHS, Depth + 1))
- return false;
- if (!!DemandedRHS &&
- !isKnownNeverZero(Shuf.getSrc2Reg(), DemandedRHS, Depth + 1))
- return false;
- return true;
- }
-
case TargetOpcode::G_OR:
return isKnownNeverZero(MI.getOperand(1).getReg(), DemandedElts,
Depth + 1) ||
diff --git a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
index ecd0ec8f6234b..349db4dbabfb1 100644
--- a/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
+++ b/llvm/unittests/CodeGen/GlobalISel/KnownBitsTest.cpp
@@ -2261,84 +2261,6 @@ TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroFreezeShlOneByVar) {
MRI->getVRegDef(CopyFreeze)->getOperand(1).getReg()));
}
-TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroBuildVector) {
- StringRef MIRString = R"(
- %one:_(s32) = G_CONSTANT i32 1
- %two:_(s32) = G_CONSTANT i32 2
- %bv:_(<2 x s32>) = G_BUILD_VECTOR %one, %two
- %copy_bv:_(<2 x s32>) = COPY %bv
-
- %zero:_(s32) = G_CONSTANT i32 0
- %bvz:_(<2 x s32>) = G_BUILD_VECTOR %one, %zero
- %copy_bvz:_(<2 x s32>) = COPY %bvz
-)";
- setUp(MIRString);
- if (!TM)
- GTEST_SKIP();
- Register CopyBVZ = Copies[Copies.size() - 1];
- Register CopyBV = Copies[Copies.size() - 2];
- GISelValueTracking Info(*MF);
- EXPECT_TRUE(
- Info.isKnownNeverZero(MRI->getVRegDef(CopyBV)->getOperand(1).getReg()));
- EXPECT_FALSE(
- Info.isKnownNeverZero(MRI->getVRegDef(CopyBVZ)->getOperand(1).getReg()));
-}
-
-TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroExtractVectorElt) {
- StringRef MIRString = R"(
- %hi33:_(s64) = G_CONSTANT i64 4294967296
- %vec:_(<2 x s64>) = G_BUILD_VECTOR %hi33, %hi33
- %idx:_(s64) = G_CONSTANT i64 0
- %extract64:_(s64) = G_EXTRACT_VECTOR_ELT %vec, %idx
- %copy_extract64:_(s64) = COPY %extract64
-
- %extract32:_(s32) = G_EXTRACT_VECTOR_ELT %vec, %idx
- %copy_extract32:_(s32) = COPY %extract32
-)";
- setUp(MIRString);
- if (!TM)
- GTEST_SKIP();
- Register CopyExtract32 = Copies[Copies.size() - 1];
- Register CopyExtract64 = Copies[Copies.size() - 2];
- GISelValueTracking Info(*MF);
- EXPECT_TRUE(Info.isKnownNeverZero(
- MRI->getVRegDef(CopyExtract64)->getOperand(1).getReg()));
- EXPECT_FALSE(Info.isKnownNeverZero(
- MRI->getVRegDef(CopyExtract32)->getOperand(1).getReg()));
-}
-
-TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroSplatVector) {
- // Same-width splat: recurses structurally on the source.
- // Wide-source splat: 0x100 (i64) splatted into i32 lanes is still nonzero;
- // 0x100000000 (i64) splatted into i32 lanes is zero after truncation.
- StringRef MIRString = R"(
- %one:_(s32) = G_CONSTANT i32 1
- %sv:_(<vscale x 2 x s32>) = G_SPLAT_VECTOR %one
- %copy_sv:_(<vscale x 2 x s32>) = COPY %sv
-
- %hi32:_(s64) = G_CONSTANT i64 256
- %sv_hi:_(<vscale x 2 x s32>) = G_SPLAT_VECTOR %hi32
- %copy_sv_hi:_(<vscale x 2 x s32>) = COPY %sv_hi
-
- %hi33:_(s64) = G_CONSTANT i64 4294967296
- %sv_trunc_zero:_(<vscale x 2 x s32>) = G_SPLAT_VECTOR %hi33
- %copy_sv_trunc_zero:_(<vscale x 2 x s32>) = COPY %sv_trunc_zero
-)";
- setUp(MIRString);
- if (!TM)
- GTEST_SKIP();
- Register CopySVTruncZero = Copies[Copies.size() - 1];
- Register CopySVHi = Copies[Copies.size() - 2];
- Register CopySV = Copies[Copies.size() - 3];
- GISelValueTracking Info(*MF);
- EXPECT_TRUE(
- Info.isKnownNeverZero(MRI->getVRegDef(CopySV)->getOperand(1).getReg()));
- EXPECT_TRUE(
- Info.isKnownNeverZero(MRI->getVRegDef(CopySVHi)->getOperand(1).getReg()));
- EXPECT_FALSE(Info.isKnownNeverZero(
- MRI->getVRegDef(CopySVTruncZero)->getOperand(1).getReg()));
-}
-
TEST_F(AArch64GISelMITest, TestIsKnownNeverZeroFallbackKnownBits) {
// G_AND has no dedicated case in the switch, so the proof must come from the
// KnownBits fallback (which sees the bit-0 set by the inner G_OR).
>From 61239914afdd9039bc3db619ab41256a2fe59ce7 Mon Sep 17 00:00:00 2001
From: Alan Li <me at alanli.org>
Date: Wed, 17 Jun 2026 11:45:47 -0700
Subject: [PATCH 09/10] [GlobalISel] Export count-zero combine helpers
---
llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
index 6e3a2c8157b7a..374111d2b3986 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/CombinerHelper.h
@@ -1177,8 +1177,8 @@ class CombinerHelper {
LLVM_ABI bool matchAVG(MachineInstr &MI, MachineRegisterInfo &MRI, Register X,
Register Y, unsigned TargetOpc) const;
- bool matchCountZeroToZeroPoison(MachineInstr &MI) const;
- void applyCountZeroToZeroPoison(MachineInstr &MI) const;
+ LLVM_ABI bool matchCountZeroToZeroPoison(MachineInstr &MI) const;
+ LLVM_ABI void applyCountZeroToZeroPoison(MachineInstr &MI) const;
private:
/// Checks for legality of an indexed variant of \p LdSt.
>From 06852629c82ef4e835ad8c738500d7512cf8212c Mon Sep 17 00:00:00 2001
From: Alan Li <alan.li at me.com>
Date: Mon, 20 Jul 2026 20:54:42 -0400
Subject: [PATCH 10/10] Update
llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h
Co-authored-by: Quentin Colombet <quentin.colombet at gmail.com>
---
llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h b/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h
index b86d832bb3b65..a2e1b4535aad7 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/GISelValueTracking.h
@@ -96,7 +96,7 @@ class LLVM_ABI GISelValueTracking : public GISelChangeObserver {
///
/// \p DemandedElts selects the vector elements that must be proven nonzero.
/// For scalar values this is a one-bit mask. The overload without
- /// \p DemandedElts demands every fixed-vector element, or the scalar value.
+ /// \p DemandedElts demands every fixed-vector element, or the scalar value to be non-zero.
bool isKnownNeverZero(Register R, unsigned Depth = 0);
bool isKnownNeverZero(Register R, const APInt &DemandedElts,
unsigned Depth = 0);
More information about the llvm-commits
mailing list