[llvm] [CodeGen] Expand power-of-2 div/rem at IR level in ExpandIRInsts. (PR #180654)
Daniil Fukalov via llvm-commits
llvm-commits at lists.llvm.org
Mon Feb 16 09:47:56 PST 2026
https://github.com/dfukalov updated https://github.com/llvm/llvm-project/pull/180654
>From 046ee44ee6417481a9afdca3683185570fb306a6 Mon Sep 17 00:00:00 2001
From: Daniil Fukalov <dfukalov at gmail.com>
Date: Mon, 9 Feb 2026 21:28:37 +0100
Subject: [PATCH 1/3] [CodeGen] Expand power-of-2 div/rem at IR level in
ExpandIRInsts.
Previously, power-of-2 div/rem operations wider than
MaxLegalDivRemBitWidth were excluded from IR expansion and left for
backend peephole optimizations. Some backends can fail to process such
instructions in case we switch off DAGCombiner.
Now ExpandIRInsts expands them into shift/mask sequences:
- udiv X, 2^C -> lshr X, C
- urem X, 2^C -> and X, (2^C - 1)
- sdiv X, 2^C -> bias adjustment + ashr X, C
- srem X, 2^C -> X - (((X + Bias) >> C) << C)
Special cases handled:
- Division/remainder by 1 or -1 (identity, negation, or zero)
- Exact division (sdiv exact skips bias, produces ashr exact)
- Negative power-of-2 divisors (result is negated)
- INT_MIN divisor (correct via countr_zero on bit pattern)
---
llvm/lib/CodeGen/ExpandIRInsts.cpp | 134 +-
llvm/test/CodeGen/AMDGPU/div_i128.ll | 67 +-
llvm/test/CodeGen/AMDGPU/div_v2i128.ll | 1352 +----------------
llvm/test/CodeGen/AMDGPU/rem_i128.ll | 61 +-
llvm/test/CodeGen/RISCV/div_minsize.ll | 6 +-
llvm/test/CodeGen/X86/div_i129_v_pow2k.ll | 148 +-
.../Transforms/ExpandIRInsts/X86/sdiv129.ll | 148 ++
.../Transforms/ExpandIRInsts/X86/srem129.ll | 115 ++
.../Transforms/ExpandIRInsts/X86/udiv129.ll | 51 +
.../Transforms/ExpandIRInsts/X86/urem129.ll | 25 +
10 files changed, 644 insertions(+), 1463 deletions(-)
diff --git a/llvm/lib/CodeGen/ExpandIRInsts.cpp b/llvm/lib/CodeGen/ExpandIRInsts.cpp
index c701d10c66528..5eed9c904edef 100644
--- a/llvm/lib/CodeGen/ExpandIRInsts.cpp
+++ b/llvm/lib/CodeGen/ExpandIRInsts.cpp
@@ -85,6 +85,111 @@ bool isSigned(unsigned int Opcode) {
return Opcode == Instruction::SDiv || Opcode == Instruction::SRem;
}
+/// For signed div/rem by a power of 2, compute the bias-adjusted dividend:
+/// Sign = ashr X, (BitWidth - 1) -- 0 or -1
+/// Bias = lshr Sign, (BitWidth - ShiftAmt) -- 0 or 2^ShiftAmt - 1
+/// Adjusted = add X, Bias
+/// The bias adds (2^ShiftAmt - 1) for negative X, correcting rounding towards
+/// zero (instead of towards -inf that a plain ashr would give).
+/// The lshr form is used instead of 'and' to avoid large immediate constants.
+static Value *addSignedBias(IRBuilder<> &Builder, Value *X, unsigned BitWidth,
+ unsigned ShiftAmt) {
+ assert(ShiftAmt > 0 && ShiftAmt < BitWidth &&
+ "ShiftAmt out of range; callers should handle ShiftAmt == 0");
+ Value *Sign = Builder.CreateAShr(X, BitWidth - 1, "sign");
+ Value *Bias = Builder.CreateLShr(Sign, BitWidth - ShiftAmt, "bias");
+ return Builder.CreateAdd(X, Bias, "adjusted");
+}
+
+/// Expand division by a power-of-2 constant.
+/// udiv X, 2^C -> lshr X, C
+/// sdiv X, 2^C -> ashr (add X, Bias), C (Bias corrects rounding)
+/// sdiv exact X, 2^C -> ashr exact X, C (no bias needed)
+/// For negative power-of-2 divisors, the result is negated.
+static void expandPow2Division(BinaryOperator *Div) {
+ bool IsSigned = isSigned(Div->getOpcode());
+ bool IsExact = Div->isExact();
+ Value *X = Div->getOperand(0);
+ auto *C = cast<ConstantInt>(Div->getOperand(1));
+ Type *Ty = Div->getType();
+ unsigned BitWidth = Ty->getIntegerBitWidth();
+
+ APInt DivisorVal = C->getValue();
+ bool IsNegativeDivisor = IsSigned && DivisorVal.isNegative();
+ // Use countr_zero() to get the shift amount directly from the bit pattern.
+ // This works correctly for both positive and negative powers of 2, including
+ // INT_MIN, without needing to negate the value first.
+ unsigned ShiftAmt = DivisorVal.countr_zero();
+
+ IRBuilder<> Builder(Div);
+ Value *Result;
+
+ if (ShiftAmt == 0) {
+ // Division by 1 or -1.
+ // X / 1 = X, X / -1 = -X.
+ Result = IsNegativeDivisor ? Builder.CreateNeg(X) : X;
+ } else if (IsSigned) {
+ // For exact division, no bias is needed since there's no rounding.
+ Value *Dividend =
+ IsExact ? X : addSignedBias(Builder, X, BitWidth, ShiftAmt);
+ Result = Builder.CreateAShr(Dividend, ShiftAmt,
+ IsNegativeDivisor ? "pre.neg" : "", IsExact);
+ if (IsNegativeDivisor)
+ Result = Builder.CreateNeg(Result);
+ } else {
+ Result = Builder.CreateLShr(X, ShiftAmt, "", IsExact);
+ }
+
+ Div->replaceAllUsesWith(Result);
+ // Transfer the name of the original instruction to its replacement,
+ // unless the result is the original dividend itself (div by 1).
+ if (Result != X)
+ if (auto *RI = dyn_cast<Instruction>(Result))
+ RI->takeName(Div);
+ Div->dropAllReferences();
+ Div->eraseFromParent();
+}
+
+/// Expand remainder by a power-of-2 constant (let C = log2(|divisor|)).
+/// urem X, 2^C -> and X, (2^C - 1)
+/// srem X, 2^C -> sub X, (shl (ashr (add X, Bias), C), C)
+static void expandPow2Remainder(BinaryOperator *Rem) {
+ bool IsSigned = isSigned(Rem->getOpcode());
+ Value *X = Rem->getOperand(0);
+ auto *C = cast<ConstantInt>(Rem->getOperand(1));
+ Type *Ty = Rem->getType();
+ unsigned BitWidth = Ty->getIntegerBitWidth();
+
+ // Use countr_zero() to get the shift amount directly from the bit pattern.
+ // This works for both positive and negative powers of 2, including INT_MIN.
+ unsigned ShiftAmt = C->getValue().countr_zero();
+
+ IRBuilder<> Builder(Rem);
+ Value *Result;
+
+ if (ShiftAmt == 0) {
+ // Remainder by 1 or -1 is always 0.
+ Result = ConstantInt::get(Ty, 0);
+ } else if (IsSigned) {
+ Value *Adjusted = addSignedBias(Builder, X, BitWidth, ShiftAmt);
+ // Clear lower ShiftAmt bits via round-trip shift:
+ // Truncated = (Adjusted >> ShiftAmt) << ShiftAmt
+ Value *Shifted = Builder.CreateAShr(Adjusted, ShiftAmt, "shifted");
+ Value *Truncated = Builder.CreateShl(Shifted, ShiftAmt, "truncated");
+ Result = Builder.CreateSub(X, Truncated);
+ } else {
+ APInt Mask = APInt::getLowBitsSet(BitWidth, ShiftAmt);
+ Value *MaskVal = ConstantInt::get(Ty, Mask);
+ Result = Builder.CreateAnd(X, MaskVal);
+ }
+
+ Rem->replaceAllUsesWith(Result);
+ if (auto *RI = dyn_cast<Instruction>(Result))
+ RI->takeName(Rem);
+ Rem->dropAllReferences();
+ Rem->eraseFromParent();
+}
+
/// This class implements a precise expansion of the frem instruction.
/// The generated code is based on the fmod implementation in the AMD device
/// libs.
@@ -1106,12 +1211,14 @@ static bool runImpl(Function &F, const TargetLowering &TLI,
case Instruction::SDiv:
case Instruction::URem:
case Instruction::SRem:
+ // TODO: We don't consider vectors here.
+ // Power-of-2 divisors are handled inside the expansion (via efficient
+ // shift/mask sequences) rather than being excluded here, so that
+ // backends that cannot lower wide div/rem even for powers of two
+ // (e.g. when DAGCombiner is disabled) still get valid lowered code.
return !DisableExpandLargeDivRem &&
cast<IntegerType>(Ty->getScalarType())->getIntegerBitWidth() >
- MaxLegalDivRemBitWidth
- // The backend has peephole optimizations for powers of two.
- // TODO: We don't consider vectors here.
- && !isConstantPowerOfTwo(I.getOperand(1), isSigned(I.getOpcode()));
+ MaxLegalDivRemBitWidth;
case Instruction::Call: {
auto *II = dyn_cast<IntrinsicInst>(&I);
if (II && (II->getIntrinsicID() == Intrinsic::fptoui_sat ||
@@ -1169,14 +1276,23 @@ static bool runImpl(Function &F, const TargetLowering &TLI,
break;
case Instruction::UDiv:
- case Instruction::SDiv:
- expandDivision(cast<BinaryOperator>(I));
+ case Instruction::SDiv: {
+ auto *BO = cast<BinaryOperator>(I);
+ if (isConstantPowerOfTwo(BO->getOperand(1), isSigned(BO->getOpcode())))
+ expandPow2Division(BO);
+ else
+ expandDivision(BO);
break;
+ }
case Instruction::URem:
- case Instruction::SRem:
- expandRemainder(cast<BinaryOperator>(I));
+ case Instruction::SRem: {
+ auto *BO = cast<BinaryOperator>(I);
+ if (isConstantPowerOfTwo(BO->getOperand(1), isSigned(BO->getOpcode())))
+ expandPow2Remainder(BO);
+ else
+ expandRemainder(BO);
break;
-
+ }
case Instruction::Call: {
auto *II = cast<IntrinsicInst>(I);
assert(II->getIntrinsicID() == Intrinsic::fptoui_sat ||
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index 167ef82a8d94e..033149b17fe09 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -4268,19 +4268,21 @@ define i128 @v_sdiv_i128_v_pow2k(i128 %lhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v5, v3
; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v0
+; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-O0-NEXT: s_mov_b32 s4, 63
+; GFX9-O0-NEXT: v_mov_b32_e32 v7, v5
+; GFX9-O0-NEXT: v_mov_b32_e32 v6, v4
+; GFX9-O0-NEXT: v_ashrrev_i64 v[6:7], s4, v[6:7]
+; GFX9-O0-NEXT: s_mov_b32 s5, 31
+; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], s5, v[6:7]
; GFX9-O0-NEXT: v_mov_b32_e32 v0, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v4
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-O0-NEXT: s_mov_b32 s4, 63
-; GFX9-O0-NEXT: v_ashrrev_i64 v[4:5], s4, v[4:5]
-; GFX9-O0-NEXT: s_mov_b32 s5, 31
-; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], s5, v[4:5]
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v7
; GFX9-O0-NEXT: s_mov_b64 s[8:9], 0
@@ -4291,12 +4293,12 @@ define i128 @v_sdiv_i128_v_pow2k(i128 %lhs) {
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s6
; GFX9-O0-NEXT: v_addc_co_u32_e32 v5, vcc, v2, v4, vcc
; GFX9-O0-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-O0-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
-; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v6, v1
-; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5
+; GFX9-O0-NEXT: v_addc_co_u32_e32 v2, vcc, v1, v2, vcc
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3
+; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2
+; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5
; GFX9-O0-NEXT: s_mov_b32 s4, 33
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
@@ -4377,13 +4379,7 @@ define i128 @v_sdiv_exact_i128_v_pow2k(i128 %lhs) {
; GFX9-LABEL: v_sdiv_exact_i128_v_pow2k:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; GFX9-NEXT: v_mov_b32_e32 v5, v4
-; GFX9-NEXT: v_lshrrev_b64 v[4:5], 31, v[4:5]
-; GFX9-NEXT: v_add_co_u32_e32 v0, vcc, v0, v4
-; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v1, v5, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v2, vcc, 0, v2, vcc
-; GFX9-NEXT: v_addc_co_u32_e32 v3, vcc, 0, v3, vcc
+; GFX9-NEXT: v_mov_b32_e32 v4, v1
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 31, v[2:3]
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 1, v4
; GFX9-NEXT: v_or_b32_e32 v0, v2, v0
@@ -4394,42 +4390,17 @@ define i128 @v_sdiv_exact_i128_v_pow2k(i128 %lhs) {
; GFX9-O0-LABEL: v_sdiv_exact_i128_v_pow2k:
; GFX9-O0: ; %bb.0:
; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2
-; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 ; 4-byte Folded Spill
-; GFX9-O0-NEXT: v_mov_b32_e32 v1, v0
-; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
-; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v5, v3
-; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
-; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
-; GFX9-O0-NEXT: v_mov_b32_e32 v2, v0
-; GFX9-O0-NEXT: v_mov_b32_e32 v0, v1
-; GFX9-O0-NEXT: v_mov_b32_e32 v3, v2
-; GFX9-O0-NEXT: v_mov_b32_e32 v2, v4
-; GFX9-O0-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-O0-NEXT: s_mov_b32 s4, 63
-; GFX9-O0-NEXT: v_ashrrev_i64 v[4:5], s4, v[4:5]
-; GFX9-O0-NEXT: s_mov_b32 s5, 31
-; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], s5, v[4:5]
-; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
-; GFX9-O0-NEXT: v_mov_b32_e32 v4, v7
-; GFX9-O0-NEXT: s_mov_b64 s[8:9], 0
-; GFX9-O0-NEXT: s_mov_b32 s6, s8
-; GFX9-O0-NEXT: s_mov_b32 s4, s9
-; GFX9-O0-NEXT: v_add_co_u32_e32 v0, vcc, v0, v5
-; GFX9-O0-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
-; GFX9-O0-NEXT: v_mov_b32_e32 v4, s6
-; GFX9-O0-NEXT: v_addc_co_u32_e32 v5, vcc, v2, v4, vcc
-; GFX9-O0-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-O0-NEXT: v_addc_co_u32_e32 v1, vcc, v1, v2, vcc
+; GFX9-O0-NEXT: v_mov_b32_e32 v5, v2
+; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1
+; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2
; GFX9-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v6, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v6, v3
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v5
-; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v1, v3
; GFX9-O0-NEXT: s_mov_b32 s4, 33
; GFX9-O0-NEXT: v_lshrrev_b64 v[0:1], s4, v[0:1]
; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 killed $vgpr0_vgpr1 killed $exec
+; GFX9-O0-NEXT: s_mov_b32 s5, 31
; GFX9-O0-NEXT: v_lshl_or_b32 v0, v2, s5, v0
; GFX9-O0-NEXT: v_mov_b32_e32 v3, v5
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v6
diff --git a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
index 52410c6d3698e..0eebada5c1c9e 100644
--- a/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_v2i128.ll
@@ -824,681 +824,59 @@ define <2 x i128> @v_sdiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
define <2 x i128> @v_sdiv_v2i128_v_pow2k(<2 x i128> %lhs) {
; SDAG-LABEL: v_sdiv_v2i128_v_pow2k:
-; SDAG: ; %bb.0: ; %_udiv-special-cases_udiv-special-cases
+; SDAG: ; %bb.0:
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT: v_ashrrev_i32_e32 v18, 31, v3
-; SDAG-NEXT: v_sub_i32_e32 v8, vcc, 0, v0
-; SDAG-NEXT: v_mov_b32_e32 v9, 0
-; SDAG-NEXT: s_mov_b64 s[6:7], 0x7f
-; SDAG-NEXT: v_mov_b32_e32 v19, v18
-; SDAG-NEXT: v_subb_u32_e32 v10, vcc, 0, v1, vcc
-; SDAG-NEXT: v_subb_u32_e32 v11, vcc, 0, v2, vcc
-; SDAG-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v3
-; SDAG-NEXT: v_cndmask_b32_e64 v13, v1, v10, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v12, v0, v8, s[4:5]
-; SDAG-NEXT: v_subb_u32_e32 v0, vcc, 0, v3, vcc
-; SDAG-NEXT: v_cndmask_b32_e64 v10, v2, v11, s[4:5]
-; SDAG-NEXT: v_ffbh_u32_e32 v1, v12
-; SDAG-NEXT: v_ffbh_u32_e32 v2, v13
-; SDAG-NEXT: v_cndmask_b32_e64 v11, v3, v0, s[4:5]
-; SDAG-NEXT: v_or_b32_e32 v0, v12, v10
-; SDAG-NEXT: v_ffbh_u32_e32 v3, v10
-; SDAG-NEXT: v_add_i32_e32 v8, vcc, 32, v1
-; SDAG-NEXT: v_or_b32_e32 v1, v13, v11
-; SDAG-NEXT: v_add_i32_e32 v3, vcc, 32, v3
-; SDAG-NEXT: v_ffbh_u32_e32 v14, v11
-; SDAG-NEXT: v_min_u32_e32 v2, v8, v2
-; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
-; SDAG-NEXT: v_min_u32_e32 v0, v3, v14
-; SDAG-NEXT: v_add_i32_e32 v1, vcc, 64, v2
-; SDAG-NEXT: v_addc_u32_e64 v2, s[8:9], 0, 0, vcc
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; SDAG-NEXT: v_cndmask_b32_e64 v3, v2, 0, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; SDAG-NEXT: v_sub_i32_e32 v2, vcc, 0x5e, v0
-; SDAG-NEXT: v_subb_u32_e32 v3, vcc, 0, v3, vcc
-; SDAG-NEXT: v_xor_b32_e32 v0, 0x7f, v2
-; SDAG-NEXT: v_subb_u32_e32 v8, vcc, 0, v9, vcc
-; SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], s[6:7], v[2:3]
-; SDAG-NEXT: v_cndmask_b32_e64 v14, 0, 1, s[6:7]
-; SDAG-NEXT: v_subb_u32_e32 v9, vcc, 0, v9, vcc
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v8
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; SDAG-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; SDAG-NEXT: v_or_b32_e32 v1, v3, v9
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
-; SDAG-NEXT: v_cndmask_b32_e32 v14, v15, v14, vcc
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
-; SDAG-NEXT: v_and_b32_e32 v0, 1, v14
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v0
-; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
-; SDAG-NEXT: v_cndmask_b32_e64 v1, v11, 0, s[4:5]
-; SDAG-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; SDAG-NEXT: v_cndmask_b32_e64 v0, v10, 0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v16, v13, 0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v17, v12, 0, s[4:5]
-; SDAG-NEXT: s_and_b64 s[4:5], s[6:7], vcc
-; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
-; SDAG-NEXT: s_cbranch_execz .LBB1_6
-; SDAG-NEXT: ; %bb.1: ; %udiv-bb15
-; SDAG-NEXT: v_add_i32_e32 v20, vcc, 1, v2
-; SDAG-NEXT: v_sub_i32_e64 v0, s[4:5], 63, v2
-; SDAG-NEXT: v_addc_u32_e32 v21, vcc, 0, v3, vcc
-; SDAG-NEXT: v_lshl_b64 v[0:1], v[12:13], v0
-; SDAG-NEXT: v_addc_u32_e32 v22, vcc, 0, v8, vcc
-; SDAG-NEXT: v_addc_u32_e32 v23, vcc, 0, v9, vcc
-; SDAG-NEXT: v_or_b32_e32 v8, v20, v22
-; SDAG-NEXT: v_sub_i32_e32 v16, vcc, 0x7f, v2
-; SDAG-NEXT: v_or_b32_e32 v9, v21, v23
-; SDAG-NEXT: v_lshl_b64 v[2:3], v[10:11], v16
-; SDAG-NEXT: v_sub_i32_e32 v17, vcc, 64, v16
-; SDAG-NEXT: v_lshl_b64 v[14:15], v[12:13], v16
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; SDAG-NEXT: v_lshr_b64 v[8:9], v[12:13], v17
-; SDAG-NEXT: v_or_b32_e32 v3, v3, v9
-; SDAG-NEXT: v_or_b32_e32 v2, v2, v8
-; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v16
-; SDAG-NEXT: v_cndmask_b32_e64 v1, v1, v3, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v0, v0, v2, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v9, 0, v15, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v8, 0, v14, s[4:5]
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; SDAG-NEXT: v_cndmask_b32_e64 v1, v1, v11, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v0, v0, v10, s[4:5]
-; SDAG-NEXT: v_mov_b32_e32 v2, 0
-; SDAG-NEXT: v_mov_b32_e32 v3, 0
-; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; SDAG-NEXT: s_cbranch_execz .LBB1_5
-; SDAG-NEXT: ; %bb.2: ; %udiv-preheader4
-; SDAG-NEXT: v_lshr_b64 v[2:3], v[12:13], v20
-; SDAG-NEXT: v_sub_i32_e32 v14, vcc, 64, v20
-; SDAG-NEXT: v_lshl_b64 v[14:15], v[10:11], v14
-; SDAG-NEXT: v_or_b32_e32 v15, v3, v15
-; SDAG-NEXT: v_or_b32_e32 v14, v2, v14
-; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v20
-; SDAG-NEXT: v_subrev_i32_e64 v2, s[4:5], 64, v20
-; SDAG-NEXT: v_lshr_b64 v[2:3], v[10:11], v2
-; SDAG-NEXT: v_cndmask_b32_e32 v3, v3, v15, vcc
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v20
-; SDAG-NEXT: v_cndmask_b32_e64 v13, v3, v13, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v14, vcc
-; SDAG-NEXT: v_cndmask_b32_e64 v12, v2, v12, s[4:5]
-; SDAG-NEXT: v_lshr_b64 v[2:3], v[10:11], v20
-; SDAG-NEXT: v_cndmask_b32_e32 v15, 0, v3, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v14, 0, v2, vcc
-; SDAG-NEXT: v_mov_b32_e32 v10, 0
-; SDAG-NEXT: v_mov_b32_e32 v11, 0
-; SDAG-NEXT: s_mov_b64 s[4:5], 0
-; SDAG-NEXT: v_mov_b32_e32 v3, 0
-; SDAG-NEXT: v_mov_b32_e32 v16, 0
-; SDAG-NEXT: v_mov_b32_e32 v17, 0
-; SDAG-NEXT: .LBB1_3: ; %udiv-do-while3
-; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: v_lshrrev_b32_e32 v2, 31, v9
-; SDAG-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
-; SDAG-NEXT: v_lshl_b64 v[14:15], v[14:15], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v24, 31, v13
-; SDAG-NEXT: v_lshl_b64 v[12:13], v[12:13], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v25, 31, v1
-; SDAG-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; SDAG-NEXT: v_or_b32_e32 v9, v17, v9
-; SDAG-NEXT: v_or_b32_e32 v8, v16, v8
-; SDAG-NEXT: v_or_b32_e32 v14, v14, v24
-; SDAG-NEXT: v_or_b32_e32 v12, v12, v25
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v2
-; SDAG-NEXT: v_sub_i32_e32 v2, vcc, -1, v12
-; SDAG-NEXT: v_subb_u32_e32 v2, vcc, 1, v13, vcc
-; SDAG-NEXT: v_subb_u32_e32 v2, vcc, 0, v14, vcc
-; SDAG-NEXT: v_subb_u32_e32 v2, vcc, 0, v15, vcc
-; SDAG-NEXT: v_ashrrev_i32_e32 v2, 31, v2
-; SDAG-NEXT: v_subrev_i32_e32 v12, vcc, 0, v12
-; SDAG-NEXT: v_and_b32_e32 v16, 2, v2
-; SDAG-NEXT: v_and_b32_e32 v2, 1, v2
-; SDAG-NEXT: v_subb_u32_e32 v13, vcc, v13, v16, vcc
-; SDAG-NEXT: v_subbrev_u32_e32 v14, vcc, 0, v14, vcc
-; SDAG-NEXT: v_subbrev_u32_e32 v15, vcc, 0, v15, vcc
-; SDAG-NEXT: v_add_i32_e32 v20, vcc, -1, v20
-; SDAG-NEXT: v_addc_u32_e32 v21, vcc, -1, v21, vcc
-; SDAG-NEXT: v_addc_u32_e32 v22, vcc, -1, v22, vcc
-; SDAG-NEXT: v_addc_u32_e32 v23, vcc, -1, v23, vcc
-; SDAG-NEXT: v_or_b32_e32 v16, v20, v22
-; SDAG-NEXT: v_or_b32_e32 v17, v21, v23
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[16:17]
-; SDAG-NEXT: v_or_b32_e32 v1, v11, v1
-; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; SDAG-NEXT: v_or_b32_e32 v0, v10, v0
-; SDAG-NEXT: v_mov_b32_e32 v17, v3
-; SDAG-NEXT: v_mov_b32_e32 v16, v2
-; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; SDAG-NEXT: s_cbranch_execnz .LBB1_3
-; SDAG-NEXT: ; %bb.4: ; %Flow13
-; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
-; SDAG-NEXT: .LBB1_5: ; %Flow14
-; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
-; SDAG-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v10, 31, v9
-; SDAG-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v10
-; SDAG-NEXT: v_or_b32_e32 v16, v3, v9
-; SDAG-NEXT: v_or_b32_e32 v17, v2, v8
-; SDAG-NEXT: .LBB1_6: ; %Flow16
-; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
-; SDAG-NEXT: v_ashrrev_i32_e32 v20, 31, v7
-; SDAG-NEXT: v_sub_i32_e32 v2, vcc, 0, v4
-; SDAG-NEXT: v_mov_b32_e32 v12, 0
-; SDAG-NEXT: s_mov_b64 s[6:7], 0x7f
-; SDAG-NEXT: v_mov_b32_e32 v21, v20
-; SDAG-NEXT: v_subb_u32_e32 v3, vcc, 0, v5, vcc
-; SDAG-NEXT: v_subb_u32_e32 v8, vcc, 0, v6, vcc
-; SDAG-NEXT: v_cmp_gt_i32_e64 s[4:5], 0, v7
-; SDAG-NEXT: v_cndmask_b32_e64 v11, v5, v3, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v10, v4, v2, s[4:5]
-; SDAG-NEXT: v_subb_u32_e32 v2, vcc, 0, v7, vcc
-; SDAG-NEXT: v_cndmask_b32_e64 v8, v6, v8, s[4:5]
-; SDAG-NEXT: v_ffbh_u32_e32 v3, v10
-; SDAG-NEXT: v_ffbh_u32_e32 v4, v11
-; SDAG-NEXT: v_cndmask_b32_e64 v9, v7, v2, s[4:5]
-; SDAG-NEXT: v_or_b32_e32 v2, v10, v8
-; SDAG-NEXT: v_ffbh_u32_e32 v5, v8
-; SDAG-NEXT: v_add_i32_e32 v6, vcc, 32, v3
-; SDAG-NEXT: v_or_b32_e32 v3, v11, v9
-; SDAG-NEXT: v_add_i32_e32 v5, vcc, 32, v5
-; SDAG-NEXT: v_ffbh_u32_e32 v7, v9
-; SDAG-NEXT: v_min_u32_e32 v4, v6, v4
-; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[2:3]
-; SDAG-NEXT: v_min_u32_e32 v2, v5, v7
-; SDAG-NEXT: v_add_i32_e32 v3, vcc, 64, v4
-; SDAG-NEXT: v_addc_u32_e64 v4, s[8:9], 0, 0, vcc
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; SDAG-NEXT: v_cndmask_b32_e64 v4, v4, 0, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
-; SDAG-NEXT: v_sub_i32_e32 v2, vcc, 0x5e, v2
-; SDAG-NEXT: v_subb_u32_e32 v3, vcc, 0, v4, vcc
-; SDAG-NEXT: v_xor_b32_e32 v6, 0x7f, v2
-; SDAG-NEXT: v_subb_u32_e32 v4, vcc, 0, v12, vcc
-; SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], s[6:7], v[2:3]
-; SDAG-NEXT: v_cndmask_b32_e64 v13, 0, 1, s[6:7]
-; SDAG-NEXT: v_subb_u32_e32 v5, vcc, 0, v12, vcc
-; SDAG-NEXT: v_or_b32_e32 v6, v6, v4
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; SDAG-NEXT: v_or_b32_e32 v7, v3, v5
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v12, v12, v13, vcc
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; SDAG-NEXT: v_and_b32_e32 v6, 1, v12
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v6
-; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
-; SDAG-NEXT: v_cndmask_b32_e64 v13, v9, 0, s[4:5]
-; SDAG-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; SDAG-NEXT: v_cndmask_b32_e64 v12, v8, 0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v7, v11, 0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v6, v10, 0, s[4:5]
-; SDAG-NEXT: s_and_b64 s[4:5], s[6:7], vcc
-; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
-; SDAG-NEXT: s_cbranch_execz .LBB1_12
-; SDAG-NEXT: ; %bb.7: ; %udiv-bb1
-; SDAG-NEXT: v_add_i32_e32 v22, vcc, 1, v2
-; SDAG-NEXT: v_sub_i32_e64 v6, s[4:5], 63, v2
-; SDAG-NEXT: v_addc_u32_e32 v23, vcc, 0, v3, vcc
-; SDAG-NEXT: v_lshl_b64 v[6:7], v[10:11], v6
-; SDAG-NEXT: v_addc_u32_e32 v24, vcc, 0, v4, vcc
-; SDAG-NEXT: v_addc_u32_e32 v25, vcc, 0, v5, vcc
-; SDAG-NEXT: v_or_b32_e32 v3, v22, v24
-; SDAG-NEXT: v_sub_i32_e32 v5, vcc, 0x7f, v2
-; SDAG-NEXT: v_or_b32_e32 v4, v23, v25
-; SDAG-NEXT: v_lshl_b64 v[12:13], v[8:9], v5
-; SDAG-NEXT: v_sub_i32_e32 v2, vcc, 64, v5
-; SDAG-NEXT: v_lshl_b64 v[14:15], v[10:11], v5
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[3:4]
-; SDAG-NEXT: v_lshr_b64 v[2:3], v[10:11], v2
-; SDAG-NEXT: v_or_b32_e32 v3, v13, v3
-; SDAG-NEXT: v_or_b32_e32 v2, v12, v2
-; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v5
-; SDAG-NEXT: v_cndmask_b32_e64 v4, v7, v3, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v6, v6, v2, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v15, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v14, s[4:5]
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v5
-; SDAG-NEXT: v_cndmask_b32_e64 v5, v4, v9, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v4, v6, v8, s[4:5]
-; SDAG-NEXT: v_mov_b32_e32 v6, 0
-; SDAG-NEXT: v_mov_b32_e32 v7, 0
-; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; SDAG-NEXT: s_cbranch_execz .LBB1_11
-; SDAG-NEXT: ; %bb.8: ; %udiv-preheader
-; SDAG-NEXT: v_lshr_b64 v[6:7], v[10:11], v22
-; SDAG-NEXT: v_sub_i32_e32 v12, vcc, 64, v22
-; SDAG-NEXT: v_lshl_b64 v[12:13], v[8:9], v12
-; SDAG-NEXT: v_or_b32_e32 v13, v7, v13
-; SDAG-NEXT: v_or_b32_e32 v12, v6, v12
-; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v22
-; SDAG-NEXT: v_subrev_i32_e64 v6, s[4:5], 64, v22
-; SDAG-NEXT: v_lshr_b64 v[6:7], v[8:9], v6
-; SDAG-NEXT: v_cndmask_b32_e32 v7, v7, v13, vcc
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v22
-; SDAG-NEXT: v_cndmask_b32_e64 v11, v7, v11, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v6, v6, v12, vcc
-; SDAG-NEXT: v_cndmask_b32_e64 v10, v6, v10, s[4:5]
-; SDAG-NEXT: v_lshr_b64 v[6:7], v[8:9], v22
-; SDAG-NEXT: v_cndmask_b32_e32 v13, 0, v7, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v12, 0, v6, vcc
-; SDAG-NEXT: v_mov_b32_e32 v8, 0
-; SDAG-NEXT: v_mov_b32_e32 v9, 0
-; SDAG-NEXT: s_mov_b64 s[4:5], 0
-; SDAG-NEXT: v_mov_b32_e32 v7, 0
-; SDAG-NEXT: v_mov_b32_e32 v14, 0
-; SDAG-NEXT: v_mov_b32_e32 v15, 0
-; SDAG-NEXT: .LBB1_9: ; %udiv-do-while
-; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: v_lshl_b64 v[12:13], v[12:13], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v6, 31, v11
-; SDAG-NEXT: v_or_b32_e32 v12, v12, v6
-; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v6, 31, v5
-; SDAG-NEXT: v_or_b32_e32 v10, v10, v6
-; SDAG-NEXT: v_lshl_b64 v[4:5], v[4:5], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v6, 31, v3
-; SDAG-NEXT: v_or_b32_e32 v4, v4, v6
-; SDAG-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
-; SDAG-NEXT: v_or_b32_e32 v5, v9, v5
-; SDAG-NEXT: v_or_b32_e32 v4, v8, v4
-; SDAG-NEXT: v_or_b32_e32 v3, v15, v3
-; SDAG-NEXT: v_or_b32_e32 v2, v14, v2
-; SDAG-NEXT: v_sub_i32_e32 v6, vcc, -1, v10
-; SDAG-NEXT: v_subb_u32_e32 v6, vcc, 1, v11, vcc
-; SDAG-NEXT: v_subb_u32_e32 v6, vcc, 0, v12, vcc
-; SDAG-NEXT: v_subb_u32_e32 v6, vcc, 0, v13, vcc
-; SDAG-NEXT: v_ashrrev_i32_e32 v14, 31, v6
-; SDAG-NEXT: v_and_b32_e32 v6, 1, v14
-; SDAG-NEXT: v_and_b32_e32 v14, 2, v14
-; SDAG-NEXT: v_subrev_i32_e32 v10, vcc, 0, v10
-; SDAG-NEXT: v_subb_u32_e32 v11, vcc, v11, v14, vcc
-; SDAG-NEXT: v_subbrev_u32_e32 v12, vcc, 0, v12, vcc
-; SDAG-NEXT: v_subbrev_u32_e32 v13, vcc, 0, v13, vcc
-; SDAG-NEXT: v_add_i32_e32 v22, vcc, -1, v22
-; SDAG-NEXT: v_addc_u32_e32 v23, vcc, -1, v23, vcc
-; SDAG-NEXT: v_addc_u32_e32 v24, vcc, -1, v24, vcc
-; SDAG-NEXT: v_addc_u32_e32 v25, vcc, -1, v25, vcc
-; SDAG-NEXT: v_or_b32_e32 v15, v23, v25
-; SDAG-NEXT: v_or_b32_e32 v14, v22, v24
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15]
-; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; SDAG-NEXT: v_mov_b32_e32 v15, v7
-; SDAG-NEXT: v_mov_b32_e32 v14, v6
-; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; SDAG-NEXT: s_cbranch_execnz .LBB1_9
-; SDAG-NEXT: ; %bb.10: ; %Flow
-; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
-; SDAG-NEXT: .LBB1_11: ; %Flow11
-; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
-; SDAG-NEXT: v_lshl_b64 v[12:13], v[4:5], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v4, 31, v3
-; SDAG-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
-; SDAG-NEXT: v_or_b32_e32 v12, v12, v4
-; SDAG-NEXT: v_or_b32_e32 v7, v7, v3
-; SDAG-NEXT: v_or_b32_e32 v6, v6, v2
-; SDAG-NEXT: .LBB1_12: ; %Flow12
-; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
-; SDAG-NEXT: v_xor_b32_e32 v3, v1, v19
-; SDAG-NEXT: v_xor_b32_e32 v2, v0, v18
-; SDAG-NEXT: v_xor_b32_e32 v1, v16, v19
-; SDAG-NEXT: v_xor_b32_e32 v0, v17, v18
-; SDAG-NEXT: v_xor_b32_e32 v8, v13, v21
-; SDAG-NEXT: v_xor_b32_e32 v9, v12, v20
-; SDAG-NEXT: v_xor_b32_e32 v5, v7, v21
-; SDAG-NEXT: v_sub_i32_e32 v0, vcc, v0, v18
-; SDAG-NEXT: v_subb_u32_e32 v1, vcc, v1, v19, vcc
-; SDAG-NEXT: v_subb_u32_e32 v2, vcc, v2, v18, vcc
-; SDAG-NEXT: v_subb_u32_e32 v3, vcc, v3, v19, vcc
-; SDAG-NEXT: v_xor_b32_e32 v4, v6, v20
-; SDAG-NEXT: v_sub_i32_e32 v4, vcc, v4, v20
-; SDAG-NEXT: v_subb_u32_e32 v5, vcc, v5, v21, vcc
-; SDAG-NEXT: v_subb_u32_e32 v6, vcc, v9, v20, vcc
-; SDAG-NEXT: v_subb_u32_e32 v7, vcc, v8, v21, vcc
+; SDAG-NEXT: v_ashrrev_i32_e32 v8, 31, v3
+; SDAG-NEXT: v_ashrrev_i32_e32 v10, 31, v7
+; SDAG-NEXT: v_mov_b32_e32 v9, v8
+; SDAG-NEXT: v_lshr_b64 v[8:9], v[8:9], 31
+; SDAG-NEXT: v_mov_b32_e32 v11, v10
+; SDAG-NEXT: v_lshr_b64 v[10:11], v[10:11], 31
+; SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; SDAG-NEXT: v_addc_u32_e32 v0, vcc, v1, v9, vcc
+; SDAG-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc
+; SDAG-NEXT: v_addc_u32_e32 v8, vcc, 0, v3, vcc
+; SDAG-NEXT: v_alignbit_b32 v0, v1, v0, 1
+; SDAG-NEXT: v_ashrrev_i32_e32 v2, 1, v8
+; SDAG-NEXT: v_ashrrev_i32_e32 v3, 31, v8
+; SDAG-NEXT: v_add_i32_e32 v4, vcc, v4, v10
+; SDAG-NEXT: v_alignbit_b32 v1, v8, v1, 1
+; SDAG-NEXT: v_addc_u32_e32 v4, vcc, v5, v11, vcc
+; SDAG-NEXT: v_addc_u32_e32 v5, vcc, 0, v6, vcc
+; SDAG-NEXT: v_addc_u32_e32 v8, vcc, 0, v7, vcc
+; SDAG-NEXT: v_alignbit_b32 v4, v5, v4, 1
+; SDAG-NEXT: v_ashrrev_i32_e32 v6, 1, v8
+; SDAG-NEXT: v_ashrrev_i32_e32 v7, 31, v8
+; SDAG-NEXT: v_alignbit_b32 v5, v8, v5, 1
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-LABEL: v_sdiv_v2i128_v_pow2k:
-; GISEL: ; %bb.0: ; %_udiv-special-cases_udiv-special-cases
+; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: s_mov_b64 s[8:9], 0
-; GISEL-NEXT: v_ashrrev_i32_e32 v18, 31, v3
-; GISEL-NEXT: v_mov_b32_e32 v14, 0x5e
-; GISEL-NEXT: v_mov_b32_e32 v8, 0x7f
-; GISEL-NEXT: v_mov_b32_e32 v9, 0
-; GISEL-NEXT: v_xor_b32_e32 v0, v18, v0
-; GISEL-NEXT: v_xor_b32_e32 v1, v18, v1
-; GISEL-NEXT: v_xor_b32_e32 v2, v18, v2
-; GISEL-NEXT: v_xor_b32_e32 v3, v18, v3
-; GISEL-NEXT: v_sub_i32_e32 v10, vcc, v0, v18
-; GISEL-NEXT: v_subb_u32_e32 v11, vcc, v1, v18, vcc
-; GISEL-NEXT: v_subb_u32_e32 v12, vcc, v2, v18, vcc
-; GISEL-NEXT: v_subb_u32_e32 v13, vcc, v3, v18, vcc
-; GISEL-NEXT: v_ffbh_u32_e32 v2, v11
-; GISEL-NEXT: v_ffbh_u32_e32 v3, v10
-; GISEL-NEXT: v_or_b32_e32 v0, v10, v12
-; GISEL-NEXT: v_or_b32_e32 v1, v11, v13
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, 32, v3
-; GISEL-NEXT: v_ffbh_u32_e32 v15, v13
-; GISEL-NEXT: v_ffbh_u32_e32 v16, v12
-; GISEL-NEXT: v_min_u32_e32 v2, v2, v3
-; GISEL-NEXT: v_add_i32_e32 v3, vcc, 32, v16
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GISEL-NEXT: v_cndmask_b32_e64 v16, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, 64, v2
-; GISEL-NEXT: v_min_u32_e32 v1, v15, v3
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13]
-; GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v14, v0
-; GISEL-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, vcc
-; GISEL-NEXT: v_subb_u32_e64 v0, s[4:5], 0, 0, s[4:5]
-; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, s[4:5]
-; GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[8:9]
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_xor_b32_e32 v8, 0x7f, v2
-; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, 0, v[0:1]
-; GISEL-NEXT: v_cndmask_b32_e64 v15, 0, 1, vcc
-; GISEL-NEXT: v_or_b32_e32 v8, v8, v0
-; GISEL-NEXT: v_or_b32_e32 v9, v3, v1
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GISEL-NEXT: v_cndmask_b32_e32 v14, v15, v14, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_or_b32_e32 v9, v16, v14
-; GISEL-NEXT: v_and_b32_e32 v14, 1, v9
-; GISEL-NEXT: v_or_b32_e32 v8, v9, v8
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v14
-; GISEL-NEXT: v_cndmask_b32_e64 v16, v10, 0, vcc
-; GISEL-NEXT: v_and_b32_e32 v14, 1, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v17, v11, 0, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v8, v12, 0, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v9, v13, 0, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v14
-; GISEL-NEXT: s_xor_b64 s[4:5], vcc, -1
-; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
-; GISEL-NEXT: s_cbranch_execz .LBB1_6
-; GISEL-NEXT: ; %bb.1: ; %udiv-bb15
-; GISEL-NEXT: v_add_i32_e32 v19, vcc, 1, v2
-; GISEL-NEXT: v_addc_u32_e64 v20, s[4:5], 0, v3, vcc
-; GISEL-NEXT: v_sub_i32_e32 v23, vcc, 0x7f, v2
-; GISEL-NEXT: v_not_b32_e32 v2, 63
-; GISEL-NEXT: v_addc_u32_e64 v21, vcc, 0, v0, s[4:5]
-; GISEL-NEXT: v_addc_u32_e32 v22, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e64 v14, s[4:5], v23, v2
-; GISEL-NEXT: v_sub_i32_e64 v8, s[4:5], 64, v23
-; GISEL-NEXT: v_lshl_b64 v[0:1], v[10:11], v23
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[12:13], v23
-; GISEL-NEXT: s_xor_b64 s[4:5], vcc, -1
-; GISEL-NEXT: v_lshr_b64 v[8:9], v[10:11], v8
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[10:11], v14
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v23
-; GISEL-NEXT: v_cndmask_b32_e32 v14, 0, v0, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v15, 0, v1, vcc
-; GISEL-NEXT: v_or_b32_e32 v0, v8, v2
-; GISEL-NEXT: v_or_b32_e32 v1, v9, v3
-; GISEL-NEXT: v_cndmask_b32_e32 v0, v16, v0, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v17, v1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v23
-; GISEL-NEXT: v_cndmask_b32_e32 v8, v0, v12, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v9, v1, v13, vcc
-; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
-; GISEL-NEXT: v_mov_b32_e32 v0, s8
-; GISEL-NEXT: v_mov_b32_e32 v1, s9
-; GISEL-NEXT: v_mov_b32_e32 v2, s10
-; GISEL-NEXT: v_mov_b32_e32 v3, s11
-; GISEL-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
-; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
-; GISEL-NEXT: s_cbranch_execz .LBB1_5
-; GISEL-NEXT: ; %bb.2: ; %udiv-preheader4
-; GISEL-NEXT: v_add_i32_e32 v16, vcc, 0xffffffc0, v19
-; GISEL-NEXT: v_sub_i32_e32 v17, vcc, 64, v19
-; GISEL-NEXT: v_lshr_b64 v[0:1], v[12:13], v19
-; GISEL-NEXT: v_lshr_b64 v[2:3], v[10:11], v19
-; GISEL-NEXT: s_mov_b64 s[8:9], 0
-; GISEL-NEXT: v_lshl_b64 v[23:24], v[12:13], v17
-; GISEL-NEXT: v_lshr_b64 v[12:13], v[12:13], v16
-; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v19
-; GISEL-NEXT: v_cndmask_b32_e32 v16, 0, v0, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v17, 0, v1, vcc
-; GISEL-NEXT: v_or_b32_e32 v0, v2, v23
-; GISEL-NEXT: v_or_b32_e32 v1, v3, v24
-; GISEL-NEXT: v_cndmask_b32_e32 v0, v12, v0, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v13, v1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v19
-; GISEL-NEXT: v_cndmask_b32_e32 v12, v0, v10, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v13, v1, v11, vcc
-; GISEL-NEXT: v_mov_b32_e32 v11, 0
-; GISEL-NEXT: v_mov_b32_e32 v0, s8
-; GISEL-NEXT: v_mov_b32_e32 v1, s9
-; GISEL-NEXT: v_mov_b32_e32 v2, s10
-; GISEL-NEXT: v_mov_b32_e32 v3, s11
-; GISEL-NEXT: .LBB1_3: ; %udiv-do-while3
-; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[14:15], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v10, 31, v15
-; GISEL-NEXT: v_lshl_b64 v[23:24], v[12:13], 1
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v12, 31, v13
-; GISEL-NEXT: v_lshrrev_b32_e32 v13, 31, v9
-; GISEL-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
-; GISEL-NEXT: v_add_i32_e32 v19, vcc, -1, v19
-; GISEL-NEXT: v_addc_u32_e32 v20, vcc, -1, v20, vcc
-; GISEL-NEXT: v_or_b32_e32 v14, v0, v2
-; GISEL-NEXT: v_or_b32_e32 v15, v1, v3
-; GISEL-NEXT: v_or_b32_e32 v2, v16, v12
-; GISEL-NEXT: v_or_b32_e32 v0, v23, v13
-; GISEL-NEXT: v_or_b32_e32 v8, v8, v10
-; GISEL-NEXT: v_addc_u32_e32 v21, vcc, -1, v21, vcc
-; GISEL-NEXT: v_addc_u32_e32 v22, vcc, -1, v22, vcc
-; GISEL-NEXT: v_sub_i32_e32 v1, vcc, 1, v24
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, 0, v2, vcc
-; GISEL-NEXT: v_subrev_i32_e64 v12, s[4:5], 0, v0
-; GISEL-NEXT: v_or_b32_e32 v0, v19, v21
-; GISEL-NEXT: v_or_b32_e32 v1, v20, v22
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, 0, v17, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GISEL-NEXT: v_ashrrev_i32_e32 v0, 31, v3
-; GISEL-NEXT: v_and_b32_e32 v10, 1, v0
-; GISEL-NEXT: v_and_b32_e32 v3, 2, v0
-; GISEL-NEXT: v_mov_b32_e32 v0, v10
-; GISEL-NEXT: v_mov_b32_e32 v1, v11
-; GISEL-NEXT: v_sub_i32_e64 v13, s[4:5], v24, v3
-; GISEL-NEXT: v_subbrev_u32_e64 v16, s[4:5], 0, v2, s[4:5]
-; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GISEL-NEXT: v_subbrev_u32_e64 v17, vcc, 0, v17, s[4:5]
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GISEL-NEXT: s_cbranch_execnz .LBB1_3
-; GISEL-NEXT: ; %bb.4: ; %Flow13
-; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT: .LBB1_5: ; %Flow14
-; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[14:15], 1
-; GISEL-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v10, 31, v15
-; GISEL-NEXT: v_or_b32_e32 v8, v8, v10
-; GISEL-NEXT: v_or_b32_e32 v16, v0, v2
-; GISEL-NEXT: v_or_b32_e32 v17, v1, v3
-; GISEL-NEXT: .LBB1_6: ; %Flow16
-; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
-; GISEL-NEXT: s_mov_b64 s[8:9], 0
-; GISEL-NEXT: v_ashrrev_i32_e32 v19, 31, v7
-; GISEL-NEXT: v_mov_b32_e32 v2, 0x5e
-; GISEL-NEXT: v_mov_b32_e32 v12, 0x7f
-; GISEL-NEXT: v_mov_b32_e32 v13, 0
-; GISEL-NEXT: v_xor_b32_e32 v0, v19, v4
-; GISEL-NEXT: v_xor_b32_e32 v1, v19, v5
-; GISEL-NEXT: v_xor_b32_e32 v3, v19, v6
-; GISEL-NEXT: v_xor_b32_e32 v6, v19, v7
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, v0, v19
-; GISEL-NEXT: v_subb_u32_e32 v5, vcc, v1, v19, vcc
-; GISEL-NEXT: v_subb_u32_e32 v10, vcc, v3, v19, vcc
-; GISEL-NEXT: v_subb_u32_e32 v11, vcc, v6, v19, vcc
-; GISEL-NEXT: v_ffbh_u32_e32 v3, v5
-; GISEL-NEXT: v_ffbh_u32_e32 v6, v4
-; GISEL-NEXT: v_or_b32_e32 v0, v4, v10
-; GISEL-NEXT: v_or_b32_e32 v1, v5, v11
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, 32, v6
-; GISEL-NEXT: v_ffbh_u32_e32 v7, v11
-; GISEL-NEXT: v_ffbh_u32_e32 v14, v10
-; GISEL-NEXT: v_min_u32_e32 v3, v3, v6
-; GISEL-NEXT: v_add_i32_e32 v6, vcc, 32, v14
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GISEL-NEXT: v_cndmask_b32_e64 v14, 0, 1, vcc
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, 64, v3
-; GISEL-NEXT: v_min_u32_e32 v1, v7, v6
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
-; GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GISEL-NEXT: v_sub_i32_e32 v2, vcc, v2, v0
-; GISEL-NEXT: v_subb_u32_e64 v3, s[4:5], 0, 0, vcc
-; GISEL-NEXT: v_subb_u32_e64 v0, s[4:5], 0, 0, s[4:5]
-; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, s[4:5]
-; GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[2:3], v[12:13]
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_xor_b32_e32 v6, 0x7f, v2
-; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, 0, v[0:1]
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_or_b32_e32 v6, v6, v0
-; GISEL-NEXT: v_or_b32_e32 v7, v3, v1
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GISEL-NEXT: v_cndmask_b32_e32 v12, v13, v12, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
-; GISEL-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
-; GISEL-NEXT: v_or_b32_e32 v7, v14, v12
-; GISEL-NEXT: v_and_b32_e32 v12, 1, v7
-; GISEL-NEXT: v_or_b32_e32 v6, v7, v6
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, v4, 0, vcc
-; GISEL-NEXT: v_and_b32_e32 v14, 1, v6
-; GISEL-NEXT: v_cndmask_b32_e64 v13, v5, 0, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v6, v10, 0, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v7, v11, 0, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v14
-; GISEL-NEXT: s_xor_b64 s[4:5], vcc, -1
-; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
-; GISEL-NEXT: s_cbranch_execz .LBB1_12
-; GISEL-NEXT: ; %bb.7: ; %udiv-bb1
-; GISEL-NEXT: v_add_i32_e32 v20, vcc, 1, v2
-; GISEL-NEXT: v_addc_u32_e64 v21, s[4:5], 0, v3, vcc
-; GISEL-NEXT: v_sub_i32_e32 v24, vcc, 0x7f, v2
-; GISEL-NEXT: v_not_b32_e32 v2, 63
-; GISEL-NEXT: v_addc_u32_e64 v22, vcc, 0, v0, s[4:5]
-; GISEL-NEXT: v_addc_u32_e32 v23, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e64 v12, s[4:5], v24, v2
-; GISEL-NEXT: v_sub_i32_e64 v6, s[4:5], 64, v24
-; GISEL-NEXT: v_lshl_b64 v[0:1], v[4:5], v24
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[10:11], v24
-; GISEL-NEXT: s_xor_b64 s[4:5], vcc, -1
-; GISEL-NEXT: v_lshr_b64 v[6:7], v[4:5], v6
-; GISEL-NEXT: v_lshl_b64 v[14:15], v[4:5], v12
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v24
-; GISEL-NEXT: v_cndmask_b32_e32 v12, 0, v0, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v13, 0, v1, vcc
-; GISEL-NEXT: v_or_b32_e32 v0, v6, v2
-; GISEL-NEXT: v_or_b32_e32 v1, v7, v3
-; GISEL-NEXT: v_cndmask_b32_e32 v0, v14, v0, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v1, v15, v1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v24
-; GISEL-NEXT: v_cndmask_b32_e32 v6, v0, v10, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v7, v1, v11, vcc
-; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
-; GISEL-NEXT: v_mov_b32_e32 v0, s8
-; GISEL-NEXT: v_mov_b32_e32 v1, s9
-; GISEL-NEXT: v_mov_b32_e32 v2, s10
-; GISEL-NEXT: v_mov_b32_e32 v3, s11
-; GISEL-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
-; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
-; GISEL-NEXT: s_cbranch_execz .LBB1_11
-; GISEL-NEXT: ; %bb.8: ; %udiv-preheader
-; GISEL-NEXT: v_add_i32_e32 v24, vcc, 0xffffffc0, v20
-; GISEL-NEXT: v_sub_i32_e32 v14, vcc, 64, v20
-; GISEL-NEXT: v_lshr_b64 v[0:1], v[10:11], v20
-; GISEL-NEXT: v_lshr_b64 v[2:3], v[4:5], v20
-; GISEL-NEXT: s_mov_b64 s[8:9], 0
-; GISEL-NEXT: v_lshl_b64 v[14:15], v[10:11], v14
-; GISEL-NEXT: v_lshr_b64 v[10:11], v[10:11], v24
-; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
-; GISEL-NEXT: v_or_b32_e32 v2, v2, v14
-; GISEL-NEXT: v_or_b32_e32 v3, v3, v15
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v20
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v10, v2, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v11, v3, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v14, 0, v0, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v15, 0, v1, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v20
-; GISEL-NEXT: v_cndmask_b32_e32 v10, v2, v4, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v11, v3, v5, vcc
-; GISEL-NEXT: v_mov_b32_e32 v4, 0
-; GISEL-NEXT: v_mov_b32_e32 v0, s8
-; GISEL-NEXT: v_mov_b32_e32 v1, s9
-; GISEL-NEXT: v_mov_b32_e32 v2, s10
-; GISEL-NEXT: v_mov_b32_e32 v3, s11
-; GISEL-NEXT: .LBB1_9: ; %udiv-do-while
-; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_lshl_b64 v[24:25], v[10:11], 1
-; GISEL-NEXT: v_lshl_b64 v[14:15], v[14:15], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v5, 31, v11
-; GISEL-NEXT: v_lshrrev_b32_e32 v10, 31, v7
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[12:13], 1
-; GISEL-NEXT: v_lshl_b64 v[6:7], v[6:7], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v11, 31, v13
-; GISEL-NEXT: v_add_i32_e32 v20, vcc, -1, v20
-; GISEL-NEXT: v_addc_u32_e32 v21, vcc, -1, v21, vcc
-; GISEL-NEXT: v_or_b32_e32 v5, v14, v5
-; GISEL-NEXT: v_or_b32_e32 v10, v24, v10
-; GISEL-NEXT: v_or_b32_e32 v6, v6, v11
-; GISEL-NEXT: v_or_b32_e32 v12, v0, v2
-; GISEL-NEXT: v_or_b32_e32 v13, v1, v3
-; GISEL-NEXT: v_addc_u32_e32 v22, vcc, -1, v22, vcc
-; GISEL-NEXT: v_addc_u32_e32 v23, vcc, -1, v23, vcc
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, 1, v25
-; GISEL-NEXT: v_subb_u32_e32 v0, vcc, 0, v5, vcc
-; GISEL-NEXT: v_subrev_i32_e64 v10, s[4:5], 0, v10
-; GISEL-NEXT: v_or_b32_e32 v0, v20, v22
-; GISEL-NEXT: v_or_b32_e32 v1, v21, v23
-; GISEL-NEXT: v_subb_u32_e32 v2, vcc, 0, v15, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GISEL-NEXT: v_ashrrev_i32_e32 v0, 31, v2
-; GISEL-NEXT: v_and_b32_e32 v3, 1, v0
-; GISEL-NEXT: v_and_b32_e32 v2, 2, v0
-; GISEL-NEXT: v_mov_b32_e32 v0, v3
-; GISEL-NEXT: v_mov_b32_e32 v1, v4
-; GISEL-NEXT: v_sub_i32_e64 v11, s[4:5], v25, v2
-; GISEL-NEXT: v_subbrev_u32_e64 v14, s[4:5], 0, v5, s[4:5]
-; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GISEL-NEXT: v_subbrev_u32_e64 v15, vcc, 0, v15, s[4:5]
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GISEL-NEXT: s_cbranch_execnz .LBB1_9
-; GISEL-NEXT: ; %bb.10: ; %Flow
-; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT: .LBB1_11: ; %Flow11
-; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[12:13], 1
-; GISEL-NEXT: v_lshl_b64 v[6:7], v[6:7], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 31, v13
-; GISEL-NEXT: v_or_b32_e32 v6, v6, v4
-; GISEL-NEXT: v_or_b32_e32 v12, v0, v2
-; GISEL-NEXT: v_or_b32_e32 v13, v1, v3
-; GISEL-NEXT: .LBB1_12: ; %Flow12
-; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
-; GISEL-NEXT: v_xor_b32_e32 v0, v16, v18
-; GISEL-NEXT: v_xor_b32_e32 v1, v17, v18
-; GISEL-NEXT: v_xor_b32_e32 v2, v8, v18
-; GISEL-NEXT: v_xor_b32_e32 v3, v9, v18
-; GISEL-NEXT: v_xor_b32_e32 v4, v12, v19
-; GISEL-NEXT: v_xor_b32_e32 v5, v13, v19
-; GISEL-NEXT: v_xor_b32_e32 v6, v6, v19
-; GISEL-NEXT: v_xor_b32_e32 v7, v7, v19
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, v0, v18
-; GISEL-NEXT: v_subb_u32_e32 v1, vcc, v1, v18, vcc
-; GISEL-NEXT: v_sub_i32_e64 v4, s[4:5], v4, v19
-; GISEL-NEXT: v_subb_u32_e64 v5, s[4:5], v5, v19, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v2, vcc, v2, v18, vcc
-; GISEL-NEXT: v_subb_u32_e32 v3, vcc, v3, v18, vcc
-; GISEL-NEXT: v_subb_u32_e64 v6, vcc, v6, v19, s[4:5]
-; GISEL-NEXT: v_subb_u32_e32 v7, vcc, v7, v19, vcc
+; GISEL-NEXT: v_ashrrev_i32_e32 v8, 31, v3
+; GISEL-NEXT: v_ashrrev_i32_e32 v10, 31, v7
+; GISEL-NEXT: v_mov_b32_e32 v9, v8
+; GISEL-NEXT: v_lshr_b64 v[8:9], v[8:9], 31
+; GISEL-NEXT: v_mov_b32_e32 v11, v10
+; GISEL-NEXT: v_lshr_b64 v[10:11], v[10:11], 31
+; GISEL-NEXT: v_add_i32_e32 v0, vcc, v0, v8
+; GISEL-NEXT: v_addc_u32_e32 v0, vcc, v1, v9, vcc
+; GISEL-NEXT: v_add_i32_e64 v1, s[4:5], v4, v10
+; GISEL-NEXT: v_addc_u32_e64 v4, s[4:5], v5, v11, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v1, vcc, 0, v2, vcc
+; GISEL-NEXT: v_addc_u32_e32 v2, vcc, 0, v3, vcc
+; GISEL-NEXT: v_lshrrev_b32_e32 v8, 1, v0
+; GISEL-NEXT: v_addc_u32_e64 v5, vcc, 0, v6, s[4:5]
+; GISEL-NEXT: v_addc_u32_e32 v6, vcc, 0, v7, vcc
+; GISEL-NEXT: v_lshrrev_b32_e32 v9, 1, v4
+; GISEL-NEXT: v_lshl_b64 v[0:1], v[1:2], 31
+; GISEL-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GISEL-NEXT: v_ashrrev_i32_e32 v2, 1, v2
+; GISEL-NEXT: v_lshl_b64 v[4:5], v[5:6], 31
+; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GISEL-NEXT: v_or_b32_e32 v4, v4, v9
+; GISEL-NEXT: v_ashrrev_i32_e32 v6, 1, v6
; GISEL-NEXT: s_setpc_b64 s[30:31]
%shl = sdiv <2 x i128> %lhs, <i128 8589934592, i128 8589934592>
ret <2 x i128> %shl
@@ -2224,625 +1602,33 @@ define <2 x i128> @v_udiv_v2i128_vv(<2 x i128> %lhs, <2 x i128> %rhs) {
define <2 x i128> @v_udiv_v2i128_v_pow2k(<2 x i128> %lhs) {
; SDAG-LABEL: v_udiv_v2i128_v_pow2k:
-; SDAG: ; %bb.0: ; %_udiv-special-cases_udiv-special-cases
+; SDAG: ; %bb.0:
; SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; SDAG-NEXT: v_mov_b32_e32 v9, v3
-; SDAG-NEXT: v_mov_b32_e32 v8, v2
-; SDAG-NEXT: v_or_b32_e32 v3, v1, v9
-; SDAG-NEXT: v_or_b32_e32 v2, v0, v8
-; SDAG-NEXT: v_ffbh_u32_e32 v10, v8
-; SDAG-NEXT: v_ffbh_u32_e32 v11, v9
-; SDAG-NEXT: v_ffbh_u32_e32 v12, v0
-; SDAG-NEXT: v_ffbh_u32_e32 v13, v1
-; SDAG-NEXT: v_mov_b32_e32 v15, 0
-; SDAG-NEXT: s_mov_b64 s[6:7], 0x7f
-; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[2:3]
-; SDAG-NEXT: v_add_i32_e32 v2, vcc, 32, v10
-; SDAG-NEXT: v_add_i32_e32 v3, vcc, 32, v12
-; SDAG-NEXT: v_min_u32_e32 v2, v2, v11
-; SDAG-NEXT: v_min_u32_e32 v3, v3, v13
-; SDAG-NEXT: v_add_i32_e32 v3, vcc, 64, v3
-; SDAG-NEXT: v_addc_u32_e64 v10, s[8:9], 0, 0, vcc
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; SDAG-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
-; SDAG-NEXT: v_sub_i32_e32 v12, vcc, 0x5e, v2
-; SDAG-NEXT: v_subb_u32_e32 v13, vcc, 0, v10, vcc
-; SDAG-NEXT: v_xor_b32_e32 v2, 0x7f, v12
-; SDAG-NEXT: v_subb_u32_e32 v14, vcc, 0, v15, vcc
-; SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], s[6:7], v[12:13]
-; SDAG-NEXT: v_cndmask_b32_e64 v10, 0, 1, s[6:7]
-; SDAG-NEXT: v_subb_u32_e32 v15, vcc, 0, v15, vcc
-; SDAG-NEXT: v_or_b32_e32 v2, v2, v14
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[14:15]
-; SDAG-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; SDAG-NEXT: v_or_b32_e32 v3, v13, v15
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15]
-; SDAG-NEXT: v_cndmask_b32_e32 v10, v11, v10, vcc
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
-; SDAG-NEXT: v_and_b32_e32 v2, 1, v10
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v2
-; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
-; SDAG-NEXT: v_cndmask_b32_e64 v3, v9, 0, s[4:5]
-; SDAG-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; SDAG-NEXT: v_cndmask_b32_e64 v2, v8, 0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v10, v1, 0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v11, v0, 0, s[4:5]
-; SDAG-NEXT: s_and_b64 s[4:5], s[6:7], vcc
-; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
-; SDAG-NEXT: s_cbranch_execz .LBB3_6
-; SDAG-NEXT: ; %bb.1: ; %udiv-bb15
-; SDAG-NEXT: v_add_i32_e32 v18, vcc, 1, v12
-; SDAG-NEXT: v_sub_i32_e64 v2, s[4:5], 63, v12
-; SDAG-NEXT: v_addc_u32_e32 v19, vcc, 0, v13, vcc
-; SDAG-NEXT: v_lshl_b64 v[2:3], v[0:1], v2
-; SDAG-NEXT: v_addc_u32_e32 v20, vcc, 0, v14, vcc
-; SDAG-NEXT: v_addc_u32_e32 v21, vcc, 0, v15, vcc
-; SDAG-NEXT: v_or_b32_e32 v10, v18, v20
-; SDAG-NEXT: v_sub_i32_e32 v16, vcc, 0x7f, v12
-; SDAG-NEXT: v_or_b32_e32 v11, v19, v21
-; SDAG-NEXT: v_lshl_b64 v[12:13], v[8:9], v16
-; SDAG-NEXT: v_sub_i32_e32 v17, vcc, 64, v16
-; SDAG-NEXT: v_lshl_b64 v[14:15], v[0:1], v16
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
-; SDAG-NEXT: v_lshr_b64 v[10:11], v[0:1], v17
-; SDAG-NEXT: v_or_b32_e32 v11, v13, v11
-; SDAG-NEXT: v_or_b32_e32 v10, v12, v10
-; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v16
-; SDAG-NEXT: v_cndmask_b32_e64 v3, v3, v11, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v2, v2, v10, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v11, 0, v15, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v10, 0, v14, s[4:5]
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; SDAG-NEXT: v_cndmask_b32_e64 v3, v3, v9, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v2, v2, v8, s[4:5]
-; SDAG-NEXT: v_mov_b32_e32 v12, 0
-; SDAG-NEXT: v_mov_b32_e32 v13, 0
-; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; SDAG-NEXT: s_cbranch_execz .LBB3_5
-; SDAG-NEXT: ; %bb.2: ; %udiv-preheader4
-; SDAG-NEXT: v_lshr_b64 v[12:13], v[0:1], v18
-; SDAG-NEXT: v_sub_i32_e32 v14, vcc, 64, v18
-; SDAG-NEXT: v_lshl_b64 v[14:15], v[8:9], v14
-; SDAG-NEXT: v_or_b32_e32 v15, v13, v15
-; SDAG-NEXT: v_or_b32_e32 v14, v12, v14
-; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v18
-; SDAG-NEXT: v_subrev_i32_e64 v12, s[4:5], 64, v18
-; SDAG-NEXT: v_lshr_b64 v[12:13], v[8:9], v12
-; SDAG-NEXT: v_cndmask_b32_e32 v13, v13, v15, vcc
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v18
-; SDAG-NEXT: v_cndmask_b32_e64 v15, v13, v1, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v1, v12, v14, vcc
-; SDAG-NEXT: v_cndmask_b32_e64 v14, v1, v0, s[4:5]
-; SDAG-NEXT: v_lshr_b64 v[0:1], v[8:9], v18
-; SDAG-NEXT: v_cndmask_b32_e32 v17, 0, v1, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v16, 0, v0, vcc
-; SDAG-NEXT: v_mov_b32_e32 v0, 0
-; SDAG-NEXT: v_mov_b32_e32 v1, 0
-; SDAG-NEXT: s_mov_b64 s[4:5], 0
-; SDAG-NEXT: v_mov_b32_e32 v13, 0
-; SDAG-NEXT: v_mov_b32_e32 v8, 0
-; SDAG-NEXT: v_mov_b32_e32 v9, 0
-; SDAG-NEXT: .LBB3_3: ; %udiv-do-while3
-; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v12, 31, v15
-; SDAG-NEXT: v_lshl_b64 v[14:15], v[14:15], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v22, 31, v3
-; SDAG-NEXT: v_or_b32_e32 v16, v16, v12
-; SDAG-NEXT: v_or_b32_e32 v12, v14, v22
-; SDAG-NEXT: v_sub_i32_e32 v14, vcc, -1, v12
-; SDAG-NEXT: v_subb_u32_e32 v14, vcc, 1, v15, vcc
-; SDAG-NEXT: v_subb_u32_e32 v14, vcc, 0, v16, vcc
-; SDAG-NEXT: v_subb_u32_e32 v14, vcc, 0, v17, vcc
-; SDAG-NEXT: v_ashrrev_i32_e32 v22, 31, v14
-; SDAG-NEXT: v_subrev_i32_e32 v14, vcc, 0, v12
-; SDAG-NEXT: v_and_b32_e32 v12, 2, v22
-; SDAG-NEXT: v_subb_u32_e32 v15, vcc, v15, v12, vcc
-; SDAG-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v12, 31, v11
-; SDAG-NEXT: v_or_b32_e32 v2, v2, v12
-; SDAG-NEXT: v_or_b32_e32 v3, v1, v3
-; SDAG-NEXT: v_or_b32_e32 v2, v0, v2
-; SDAG-NEXT: v_lshl_b64 v[10:11], v[10:11], 1
-; SDAG-NEXT: v_and_b32_e32 v12, 1, v22
-; SDAG-NEXT: v_subbrev_u32_e32 v16, vcc, 0, v16, vcc
-; SDAG-NEXT: v_subbrev_u32_e32 v17, vcc, 0, v17, vcc
-; SDAG-NEXT: v_add_i32_e32 v18, vcc, -1, v18
-; SDAG-NEXT: v_addc_u32_e32 v19, vcc, -1, v19, vcc
-; SDAG-NEXT: v_addc_u32_e32 v20, vcc, -1, v20, vcc
-; SDAG-NEXT: v_addc_u32_e32 v21, vcc, -1, v21, vcc
-; SDAG-NEXT: v_or_b32_e32 v22, v18, v20
-; SDAG-NEXT: v_or_b32_e32 v23, v19, v21
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[22:23]
-; SDAG-NEXT: v_or_b32_e32 v11, v9, v11
-; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; SDAG-NEXT: v_or_b32_e32 v10, v8, v10
-; SDAG-NEXT: v_mov_b32_e32 v8, v12
-; SDAG-NEXT: v_mov_b32_e32 v9, v13
-; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; SDAG-NEXT: s_cbranch_execnz .LBB3_3
-; SDAG-NEXT: ; %bb.4: ; %Flow13
-; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
-; SDAG-NEXT: .LBB3_5: ; %Flow14
-; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
-; SDAG-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v8, 31, v11
-; SDAG-NEXT: v_lshl_b64 v[0:1], v[10:11], 1
-; SDAG-NEXT: v_or_b32_e32 v2, v2, v8
-; SDAG-NEXT: v_or_b32_e32 v10, v13, v1
-; SDAG-NEXT: v_or_b32_e32 v11, v12, v0
-; SDAG-NEXT: .LBB3_6: ; %Flow16
-; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
-; SDAG-NEXT: v_or_b32_e32 v1, v5, v7
-; SDAG-NEXT: v_or_b32_e32 v0, v4, v6
-; SDAG-NEXT: v_ffbh_u32_e32 v8, v6
-; SDAG-NEXT: v_ffbh_u32_e32 v9, v7
-; SDAG-NEXT: v_ffbh_u32_e32 v12, v4
-; SDAG-NEXT: v_ffbh_u32_e32 v13, v5
-; SDAG-NEXT: v_mov_b32_e32 v15, 0
-; SDAG-NEXT: s_mov_b64 s[6:7], 0x7f
-; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], 0, v[0:1]
-; SDAG-NEXT: v_add_i32_e32 v0, vcc, 32, v8
-; SDAG-NEXT: v_add_i32_e32 v1, vcc, 32, v12
-; SDAG-NEXT: v_min_u32_e32 v0, v0, v9
-; SDAG-NEXT: v_min_u32_e32 v1, v1, v13
-; SDAG-NEXT: v_add_i32_e32 v1, vcc, 64, v1
-; SDAG-NEXT: v_addc_u32_e64 v8, s[8:9], 0, 0, vcc
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
-; SDAG-NEXT: v_cndmask_b32_e64 v8, v8, 0, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; SDAG-NEXT: v_sub_i32_e32 v0, vcc, 0x5e, v0
-; SDAG-NEXT: v_subb_u32_e32 v1, vcc, 0, v8, vcc
-; SDAG-NEXT: v_xor_b32_e32 v8, 0x7f, v0
-; SDAG-NEXT: v_subb_u32_e32 v14, vcc, 0, v15, vcc
-; SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
-; SDAG-NEXT: v_cndmask_b32_e64 v12, 0, 1, s[6:7]
-; SDAG-NEXT: v_subb_u32_e32 v15, vcc, 0, v15, vcc
-; SDAG-NEXT: v_or_b32_e32 v8, v8, v14
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[14:15]
-; SDAG-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; SDAG-NEXT: v_or_b32_e32 v9, v1, v15
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15]
-; SDAG-NEXT: v_cndmask_b32_e32 v12, v13, v12, vcc
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
-; SDAG-NEXT: v_and_b32_e32 v8, 1, v12
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[6:7], 1, v8
-; SDAG-NEXT: s_or_b64 s[4:5], s[4:5], s[6:7]
-; SDAG-NEXT: v_cndmask_b32_e64 v9, v7, 0, s[4:5]
-; SDAG-NEXT: s_xor_b64 s[6:7], s[4:5], -1
-; SDAG-NEXT: v_cndmask_b32_e64 v8, v6, 0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v12, v5, 0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v13, v4, 0, s[4:5]
-; SDAG-NEXT: s_and_b64 s[4:5], s[6:7], vcc
-; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
-; SDAG-NEXT: s_cbranch_execz .LBB3_12
-; SDAG-NEXT: ; %bb.7: ; %udiv-bb1
-; SDAG-NEXT: v_add_i32_e32 v18, vcc, 1, v0
-; SDAG-NEXT: v_sub_i32_e64 v8, s[4:5], 63, v0
-; SDAG-NEXT: v_addc_u32_e32 v19, vcc, 0, v1, vcc
-; SDAG-NEXT: v_lshl_b64 v[8:9], v[4:5], v8
-; SDAG-NEXT: v_addc_u32_e32 v20, vcc, 0, v14, vcc
-; SDAG-NEXT: v_addc_u32_e32 v21, vcc, 0, v15, vcc
-; SDAG-NEXT: v_or_b32_e32 v12, v18, v20
-; SDAG-NEXT: v_sub_i32_e32 v16, vcc, 0x7f, v0
-; SDAG-NEXT: v_or_b32_e32 v13, v19, v21
-; SDAG-NEXT: v_lshl_b64 v[0:1], v[6:7], v16
-; SDAG-NEXT: v_sub_i32_e32 v17, vcc, 64, v16
-; SDAG-NEXT: v_lshl_b64 v[14:15], v[4:5], v16
-; SDAG-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
-; SDAG-NEXT: v_lshr_b64 v[12:13], v[4:5], v17
-; SDAG-NEXT: v_or_b32_e32 v1, v1, v13
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v12
-; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v16
-; SDAG-NEXT: v_cndmask_b32_e64 v1, v9, v1, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v0, v8, v0, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v13, 0, v15, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v12, 0, v14, s[4:5]
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v16
-; SDAG-NEXT: v_cndmask_b32_e64 v1, v1, v7, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e64 v0, v0, v6, s[4:5]
-; SDAG-NEXT: v_mov_b32_e32 v14, 0
-; SDAG-NEXT: v_mov_b32_e32 v15, 0
-; SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
-; SDAG-NEXT: s_xor_b64 s[8:9], exec, s[4:5]
-; SDAG-NEXT: s_cbranch_execz .LBB3_11
-; SDAG-NEXT: ; %bb.8: ; %udiv-preheader
-; SDAG-NEXT: v_lshr_b64 v[8:9], v[4:5], v18
-; SDAG-NEXT: v_sub_i32_e32 v14, vcc, 64, v18
-; SDAG-NEXT: v_lshl_b64 v[14:15], v[6:7], v14
-; SDAG-NEXT: v_or_b32_e32 v15, v9, v15
-; SDAG-NEXT: v_or_b32_e32 v14, v8, v14
-; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v18
-; SDAG-NEXT: v_subrev_i32_e64 v8, s[4:5], 64, v18
-; SDAG-NEXT: v_lshr_b64 v[8:9], v[6:7], v8
-; SDAG-NEXT: v_cndmask_b32_e32 v9, v9, v15, vcc
-; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v18
-; SDAG-NEXT: v_cndmask_b32_e64 v9, v9, v5, s[4:5]
-; SDAG-NEXT: v_cndmask_b32_e32 v5, v8, v14, vcc
-; SDAG-NEXT: v_cndmask_b32_e64 v8, v5, v4, s[4:5]
-; SDAG-NEXT: v_lshr_b64 v[4:5], v[6:7], v18
-; SDAG-NEXT: v_cndmask_b32_e32 v17, 0, v5, vcc
-; SDAG-NEXT: v_cndmask_b32_e32 v16, 0, v4, vcc
-; SDAG-NEXT: v_mov_b32_e32 v4, 0
-; SDAG-NEXT: v_mov_b32_e32 v5, 0
-; SDAG-NEXT: s_mov_b64 s[4:5], 0
-; SDAG-NEXT: v_mov_b32_e32 v15, 0
-; SDAG-NEXT: v_mov_b32_e32 v6, 0
+; SDAG-NEXT: v_alignbit_b32 v0, v2, v1, 1
+; SDAG-NEXT: v_lshrrev_b32_e32 v8, 1, v3
+; SDAG-NEXT: v_alignbit_b32 v4, v6, v5, 1
+; SDAG-NEXT: v_lshrrev_b32_e32 v9, 1, v7
+; SDAG-NEXT: v_alignbit_b32 v1, v3, v2, 1
+; SDAG-NEXT: v_alignbit_b32 v5, v7, v6, 1
+; SDAG-NEXT: v_mov_b32_e32 v3, 0
; SDAG-NEXT: v_mov_b32_e32 v7, 0
-; SDAG-NEXT: .LBB3_9: ; %udiv-do-while
-; SDAG-NEXT: ; =>This Inner Loop Header: Depth=1
-; SDAG-NEXT: v_lshl_b64 v[16:17], v[16:17], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v14, 31, v9
-; SDAG-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v22, 31, v1
-; SDAG-NEXT: v_or_b32_e32 v16, v16, v14
-; SDAG-NEXT: v_or_b32_e32 v8, v8, v22
-; SDAG-NEXT: v_sub_i32_e32 v14, vcc, -1, v8
-; SDAG-NEXT: v_subb_u32_e32 v14, vcc, 1, v9, vcc
-; SDAG-NEXT: v_subb_u32_e32 v14, vcc, 0, v16, vcc
-; SDAG-NEXT: v_subb_u32_e32 v14, vcc, 0, v17, vcc
-; SDAG-NEXT: v_ashrrev_i32_e32 v14, 31, v14
-; SDAG-NEXT: v_subrev_i32_e32 v8, vcc, 0, v8
-; SDAG-NEXT: v_and_b32_e32 v22, 2, v14
-; SDAG-NEXT: v_subb_u32_e32 v9, vcc, v9, v22, vcc
-; SDAG-NEXT: v_lshl_b64 v[0:1], v[0:1], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v22, 31, v13
-; SDAG-NEXT: v_or_b32_e32 v0, v0, v22
-; SDAG-NEXT: v_or_b32_e32 v1, v5, v1
-; SDAG-NEXT: v_or_b32_e32 v0, v4, v0
-; SDAG-NEXT: v_lshl_b64 v[12:13], v[12:13], 1
-; SDAG-NEXT: v_and_b32_e32 v14, 1, v14
-; SDAG-NEXT: v_subbrev_u32_e32 v16, vcc, 0, v16, vcc
-; SDAG-NEXT: v_subbrev_u32_e32 v17, vcc, 0, v17, vcc
-; SDAG-NEXT: v_add_i32_e32 v18, vcc, -1, v18
-; SDAG-NEXT: v_addc_u32_e32 v19, vcc, -1, v19, vcc
-; SDAG-NEXT: v_addc_u32_e32 v20, vcc, -1, v20, vcc
-; SDAG-NEXT: v_addc_u32_e32 v21, vcc, -1, v21, vcc
-; SDAG-NEXT: v_or_b32_e32 v22, v18, v20
-; SDAG-NEXT: v_or_b32_e32 v23, v19, v21
-; SDAG-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[22:23]
-; SDAG-NEXT: v_or_b32_e32 v13, v7, v13
-; SDAG-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
-; SDAG-NEXT: v_or_b32_e32 v12, v6, v12
-; SDAG-NEXT: v_mov_b32_e32 v6, v14
-; SDAG-NEXT: v_mov_b32_e32 v7, v15
-; SDAG-NEXT: s_andn2_b64 exec, exec, s[4:5]
-; SDAG-NEXT: s_cbranch_execnz .LBB3_9
-; SDAG-NEXT: ; %bb.10: ; %Flow
-; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
-; SDAG-NEXT: .LBB3_11: ; %Flow11
-; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
-; SDAG-NEXT: v_lshl_b64 v[8:9], v[0:1], 1
-; SDAG-NEXT: v_lshrrev_b32_e32 v4, 31, v13
-; SDAG-NEXT: v_lshl_b64 v[0:1], v[12:13], 1
-; SDAG-NEXT: v_or_b32_e32 v8, v8, v4
-; SDAG-NEXT: v_or_b32_e32 v12, v15, v1
-; SDAG-NEXT: v_or_b32_e32 v13, v14, v0
-; SDAG-NEXT: .LBB3_12: ; %Flow12
-; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
-; SDAG-NEXT: v_mov_b32_e32 v0, v11
-; SDAG-NEXT: v_mov_b32_e32 v1, v10
-; SDAG-NEXT: v_mov_b32_e32 v4, v13
-; SDAG-NEXT: v_mov_b32_e32 v5, v12
-; SDAG-NEXT: v_mov_b32_e32 v6, v8
-; SDAG-NEXT: v_mov_b32_e32 v7, v9
+; SDAG-NEXT: v_mov_b32_e32 v2, v8
+; SDAG-NEXT: v_mov_b32_e32 v6, v9
; SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GISEL-LABEL: v_udiv_v2i128_v_pow2k:
-; GISEL: ; %bb.0: ; %_udiv-special-cases_udiv-special-cases
+; GISEL: ; %bb.0:
; GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GISEL-NEXT: v_mov_b32_e32 v8, v2
-; GISEL-NEXT: v_mov_b32_e32 v9, v3
-; GISEL-NEXT: s_mov_b64 s[8:9], 0
-; GISEL-NEXT: v_or_b32_e32 v2, v0, v8
-; GISEL-NEXT: v_or_b32_e32 v3, v1, v9
-; GISEL-NEXT: v_ffbh_u32_e32 v12, v1
-; GISEL-NEXT: v_ffbh_u32_e32 v13, v0
-; GISEL-NEXT: v_ffbh_u32_e32 v14, v9
-; GISEL-NEXT: v_ffbh_u32_e32 v15, v8
-; GISEL-NEXT: v_mov_b32_e32 v16, 0x5e
-; GISEL-NEXT: v_mov_b32_e32 v10, 0x7f
-; GISEL-NEXT: v_mov_b32_e32 v11, 0
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, 32, v13
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, 32, v15
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
-; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
-; GISEL-NEXT: v_min_u32_e32 v2, v12, v13
-; GISEL-NEXT: v_min_u32_e32 v3, v14, v15
-; GISEL-NEXT: v_add_i32_e32 v2, vcc, 64, v2
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
-; GISEL-NEXT: v_sub_i32_e32 v14, vcc, v16, v2
-; GISEL-NEXT: v_subb_u32_e64 v15, s[4:5], 0, 0, vcc
-; GISEL-NEXT: v_subb_u32_e64 v12, s[4:5], 0, 0, s[4:5]
-; GISEL-NEXT: v_subb_u32_e64 v13, s[4:5], 0, 0, s[4:5]
-; GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[14:15], v[10:11]
-; GISEL-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
-; GISEL-NEXT: v_xor_b32_e32 v2, 0x7f, v14
-; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, 0, v[12:13]
-; GISEL-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
-; GISEL-NEXT: v_or_b32_e32 v2, v2, v12
-; GISEL-NEXT: v_or_b32_e32 v3, v15, v13
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[12:13]
-; GISEL-NEXT: v_cndmask_b32_e32 v10, v11, v10, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[2:3]
-; GISEL-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc
-; GISEL-NEXT: v_or_b32_e32 v3, v17, v10
-; GISEL-NEXT: v_and_b32_e32 v10, 1, v3
-; GISEL-NEXT: v_or_b32_e32 v2, v3, v2
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v10
-; GISEL-NEXT: v_cndmask_b32_e64 v10, v0, 0, vcc
-; GISEL-NEXT: v_and_b32_e32 v16, 1, v2
-; GISEL-NEXT: v_cndmask_b32_e64 v11, v1, 0, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v2, v8, 0, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v3, v9, 0, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; GISEL-NEXT: s_xor_b64 s[4:5], vcc, -1
-; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
-; GISEL-NEXT: s_cbranch_execz .LBB3_6
-; GISEL-NEXT: ; %bb.1: ; %udiv-bb15
-; GISEL-NEXT: v_add_i32_e32 v18, vcc, 1, v14
-; GISEL-NEXT: v_addc_u32_e64 v19, s[4:5], 0, v15, vcc
-; GISEL-NEXT: v_sub_i32_e32 v22, vcc, 0x7f, v14
-; GISEL-NEXT: v_not_b32_e32 v2, 63
-; GISEL-NEXT: v_addc_u32_e64 v20, vcc, 0, v12, s[4:5]
-; GISEL-NEXT: v_addc_u32_e32 v21, vcc, 0, v13, vcc
-; GISEL-NEXT: v_add_i32_e64 v14, s[4:5], v22, v2
-; GISEL-NEXT: v_sub_i32_e64 v12, s[4:5], 64, v22
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[0:1], v22
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[8:9], v22
-; GISEL-NEXT: s_xor_b64 s[4:5], vcc, -1
-; GISEL-NEXT: v_lshr_b64 v[12:13], v[0:1], v12
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[0:1], v14
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v22
-; GISEL-NEXT: v_cndmask_b32_e32 v14, 0, v2, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v15, 0, v3, vcc
-; GISEL-NEXT: v_or_b32_e32 v2, v12, v10
-; GISEL-NEXT: v_or_b32_e32 v3, v13, v11
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v16, v2, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v17, v3, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v22
-; GISEL-NEXT: v_cndmask_b32_e32 v2, v2, v8, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v3, v3, v9, vcc
-; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
-; GISEL-NEXT: v_mov_b32_e32 v13, s11
-; GISEL-NEXT: v_mov_b32_e32 v12, s10
-; GISEL-NEXT: v_mov_b32_e32 v11, s9
-; GISEL-NEXT: v_mov_b32_e32 v10, s8
-; GISEL-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
-; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
-; GISEL-NEXT: s_cbranch_execz .LBB3_5
-; GISEL-NEXT: ; %bb.2: ; %udiv-preheader4
-; GISEL-NEXT: v_add_i32_e32 v22, vcc, 0xffffffc0, v18
-; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 64, v18
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v18
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v18
-; GISEL-NEXT: v_lshr_b64 v[10:11], v[8:9], v18
-; GISEL-NEXT: v_lshr_b64 v[12:13], v[0:1], v18
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[8:9], v16
-; GISEL-NEXT: v_or_b32_e32 v12, v12, v16
-; GISEL-NEXT: v_or_b32_e32 v13, v13, v17
-; GISEL-NEXT: s_mov_b64 s[8:9], 0
-; GISEL-NEXT: v_lshr_b64 v[8:9], v[8:9], v22
-; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v12, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v13, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v8, v8, v0, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v9, v9, v1, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v16, 0, v10, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v17, 0, v11, vcc
-; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
-; GISEL-NEXT: v_mov_b32_e32 v1, 0
-; GISEL-NEXT: v_mov_b32_e32 v13, s11
-; GISEL-NEXT: v_mov_b32_e32 v12, s10
-; GISEL-NEXT: v_mov_b32_e32 v11, s9
-; GISEL-NEXT: v_mov_b32_e32 v10, s8
-; GISEL-NEXT: .LBB3_3: ; %udiv-do-while3
-; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_lshl_b64 v[12:13], v[14:15], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v0, 31, v15
-; GISEL-NEXT: v_or_b32_e32 v14, v10, v12
-; GISEL-NEXT: v_or_b32_e32 v15, v11, v13
-; GISEL-NEXT: v_lshl_b64 v[10:11], v[8:9], 1
-; GISEL-NEXT: v_lshl_b64 v[12:13], v[16:17], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v8, 31, v9
-; GISEL-NEXT: v_lshrrev_b32_e32 v9, 31, v3
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
-; GISEL-NEXT: v_add_i32_e32 v18, vcc, -1, v18
-; GISEL-NEXT: v_addc_u32_e32 v19, vcc, -1, v19, vcc
-; GISEL-NEXT: v_or_b32_e32 v12, v12, v8
-; GISEL-NEXT: v_or_b32_e32 v8, v10, v9
-; GISEL-NEXT: v_or_b32_e32 v2, v2, v0
-; GISEL-NEXT: v_addc_u32_e32 v20, vcc, -1, v20, vcc
-; GISEL-NEXT: v_addc_u32_e32 v21, vcc, -1, v21, vcc
-; GISEL-NEXT: v_sub_i32_e32 v0, vcc, 1, v11
-; GISEL-NEXT: v_subb_u32_e32 v0, vcc, 0, v12, vcc
-; GISEL-NEXT: v_subrev_i32_e64 v8, s[4:5], 0, v8
-; GISEL-NEXT: v_or_b32_e32 v9, v18, v20
-; GISEL-NEXT: v_or_b32_e32 v10, v19, v21
-; GISEL-NEXT: v_subb_u32_e32 v0, vcc, 0, v13, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[9:10]
-; GISEL-NEXT: v_ashrrev_i32_e32 v9, 31, v0
-; GISEL-NEXT: v_and_b32_e32 v0, 1, v9
-; GISEL-NEXT: v_and_b32_e32 v9, 2, v9
-; GISEL-NEXT: v_sub_i32_e64 v9, s[4:5], v11, v9
-; GISEL-NEXT: v_mov_b32_e32 v11, v1
-; GISEL-NEXT: v_mov_b32_e32 v10, v0
-; GISEL-NEXT: v_subbrev_u32_e64 v16, s[4:5], 0, v12, s[4:5]
-; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GISEL-NEXT: v_subbrev_u32_e64 v17, vcc, 0, v13, s[4:5]
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GISEL-NEXT: s_cbranch_execnz .LBB3_3
-; GISEL-NEXT: ; %bb.4: ; %Flow13
-; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT: .LBB3_5: ; %Flow14
-; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
-; GISEL-NEXT: v_lshl_b64 v[0:1], v[14:15], 1
-; GISEL-NEXT: v_lshl_b64 v[2:3], v[2:3], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v8, 31, v15
-; GISEL-NEXT: v_or_b32_e32 v2, v2, v8
-; GISEL-NEXT: v_or_b32_e32 v10, v10, v0
-; GISEL-NEXT: v_or_b32_e32 v11, v11, v1
-; GISEL-NEXT: .LBB3_6: ; %Flow16
-; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
-; GISEL-NEXT: s_mov_b64 s[8:9], 0
-; GISEL-NEXT: v_or_b32_e32 v0, v4, v6
-; GISEL-NEXT: v_or_b32_e32 v1, v5, v7
-; GISEL-NEXT: v_ffbh_u32_e32 v12, v5
-; GISEL-NEXT: v_ffbh_u32_e32 v13, v4
-; GISEL-NEXT: v_ffbh_u32_e32 v14, v7
-; GISEL-NEXT: v_ffbh_u32_e32 v15, v6
-; GISEL-NEXT: v_mov_b32_e32 v16, 0x5e
-; GISEL-NEXT: v_mov_b32_e32 v8, 0x7f
-; GISEL-NEXT: v_mov_b32_e32 v9, 0
-; GISEL-NEXT: v_add_i32_e32 v13, vcc, 32, v13
-; GISEL-NEXT: v_add_i32_e32 v15, vcc, 32, v15
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GISEL-NEXT: v_cndmask_b32_e64 v17, 0, 1, vcc
-; GISEL-NEXT: v_min_u32_e32 v0, v12, v13
-; GISEL-NEXT: v_min_u32_e32 v1, v14, v15
-; GISEL-NEXT: v_add_i32_e32 v0, vcc, 64, v0
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
-; GISEL-NEXT: v_cndmask_b32_e32 v0, v1, v0, vcc
-; GISEL-NEXT: v_sub_i32_e32 v14, vcc, v16, v0
-; GISEL-NEXT: v_subb_u32_e64 v15, s[4:5], 0, 0, vcc
-; GISEL-NEXT: v_subb_u32_e64 v0, s[4:5], 0, 0, s[4:5]
-; GISEL-NEXT: v_subb_u32_e64 v1, s[4:5], 0, 0, s[4:5]
-; GISEL-NEXT: v_cmp_gt_u64_e32 vcc, v[14:15], v[8:9]
-; GISEL-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
-; GISEL-NEXT: v_xor_b32_e32 v8, 0x7f, v14
-; GISEL-NEXT: v_cmp_lt_u64_e32 vcc, 0, v[0:1]
-; GISEL-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
-; GISEL-NEXT: v_or_b32_e32 v8, v8, v0
-; GISEL-NEXT: v_or_b32_e32 v9, v15, v1
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[0:1]
-; GISEL-NEXT: v_cndmask_b32_e32 v12, v13, v12, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
-; GISEL-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
-; GISEL-NEXT: v_or_b32_e32 v9, v17, v12
-; GISEL-NEXT: v_and_b32_e32 v12, 1, v9
-; GISEL-NEXT: v_or_b32_e32 v8, v9, v8
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v12
-; GISEL-NEXT: v_cndmask_b32_e64 v12, v4, 0, vcc
-; GISEL-NEXT: v_and_b32_e32 v16, 1, v8
-; GISEL-NEXT: v_cndmask_b32_e64 v13, v5, 0, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v8, v6, 0, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v9, v7, 0, vcc
-; GISEL-NEXT: v_cmp_ne_u32_e32 vcc, 0, v16
-; GISEL-NEXT: s_xor_b64 s[4:5], vcc, -1
-; GISEL-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
-; GISEL-NEXT: s_cbranch_execz .LBB3_12
-; GISEL-NEXT: ; %bb.7: ; %udiv-bb1
-; GISEL-NEXT: v_add_i32_e32 v18, vcc, 1, v14
-; GISEL-NEXT: v_addc_u32_e64 v19, s[4:5], 0, v15, vcc
-; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 0x7f, v14
-; GISEL-NEXT: v_not_b32_e32 v8, 63
-; GISEL-NEXT: v_addc_u32_e64 v20, vcc, 0, v0, s[4:5]
-; GISEL-NEXT: v_addc_u32_e32 v21, vcc, 0, v1, vcc
-; GISEL-NEXT: v_add_i32_e64 v14, s[4:5], v16, v8
-; GISEL-NEXT: v_sub_i32_e64 v12, s[4:5], 64, v16
-; GISEL-NEXT: v_lshl_b64 v[0:1], v[4:5], v16
-; GISEL-NEXT: v_lshl_b64 v[8:9], v[6:7], v16
-; GISEL-NEXT: s_xor_b64 s[4:5], vcc, -1
-; GISEL-NEXT: v_lshr_b64 v[12:13], v[4:5], v12
-; GISEL-NEXT: v_lshl_b64 v[14:15], v[4:5], v14
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v16
-; GISEL-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
-; GISEL-NEXT: v_or_b32_e32 v8, v12, v8
-; GISEL-NEXT: v_or_b32_e32 v9, v13, v9
-; GISEL-NEXT: v_cndmask_b32_e32 v8, v14, v8, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v9, v15, v9, vcc
-; GISEL-NEXT: v_cmp_eq_u32_e32 vcc, 0, v16
-; GISEL-NEXT: v_cndmask_b32_e32 v8, v8, v6, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v9, v9, v7, vcc
-; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
-; GISEL-NEXT: v_mov_b32_e32 v15, s11
-; GISEL-NEXT: v_mov_b32_e32 v14, s10
-; GISEL-NEXT: v_mov_b32_e32 v13, s9
-; GISEL-NEXT: v_mov_b32_e32 v12, s8
-; GISEL-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
-; GISEL-NEXT: s_xor_b64 s[12:13], exec, s[8:9]
-; GISEL-NEXT: s_cbranch_execz .LBB3_11
-; GISEL-NEXT: ; %bb.8: ; %udiv-preheader
-; GISEL-NEXT: v_add_i32_e32 v22, vcc, 0xffffffc0, v18
-; GISEL-NEXT: v_sub_i32_e32 v16, vcc, 64, v18
-; GISEL-NEXT: v_cmp_gt_u32_e32 vcc, 64, v18
-; GISEL-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v18
-; GISEL-NEXT: v_lshr_b64 v[12:13], v[6:7], v18
-; GISEL-NEXT: v_lshr_b64 v[14:15], v[4:5], v18
-; GISEL-NEXT: v_lshl_b64 v[16:17], v[6:7], v16
-; GISEL-NEXT: v_or_b32_e32 v14, v14, v16
-; GISEL-NEXT: v_or_b32_e32 v15, v15, v17
-; GISEL-NEXT: s_mov_b64 s[8:9], 0
-; GISEL-NEXT: v_lshr_b64 v[6:7], v[6:7], v22
-; GISEL-NEXT: v_cndmask_b32_e32 v6, v6, v14, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v7, v7, v15, vcc
-; GISEL-NEXT: v_cndmask_b32_e64 v6, v6, v4, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e64 v7, v7, v5, s[4:5]
-; GISEL-NEXT: v_cndmask_b32_e32 v16, 0, v12, vcc
-; GISEL-NEXT: v_cndmask_b32_e32 v17, 0, v13, vcc
-; GISEL-NEXT: s_mov_b64 s[10:11], s[8:9]
-; GISEL-NEXT: v_mov_b32_e32 v5, 0
-; GISEL-NEXT: v_mov_b32_e32 v15, s11
-; GISEL-NEXT: v_mov_b32_e32 v14, s10
-; GISEL-NEXT: v_mov_b32_e32 v13, s9
-; GISEL-NEXT: v_mov_b32_e32 v12, s8
-; GISEL-NEXT: .LBB3_9: ; %udiv-do-while
-; GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
-; GISEL-NEXT: v_lshl_b64 v[14:15], v[0:1], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v4, 31, v1
-; GISEL-NEXT: v_or_b32_e32 v0, v12, v14
-; GISEL-NEXT: v_or_b32_e32 v1, v13, v15
-; GISEL-NEXT: v_lshl_b64 v[12:13], v[6:7], 1
-; GISEL-NEXT: v_lshl_b64 v[14:15], v[16:17], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v6, 31, v7
-; GISEL-NEXT: v_lshrrev_b32_e32 v7, 31, v9
-; GISEL-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
-; GISEL-NEXT: v_add_i32_e32 v18, vcc, -1, v18
-; GISEL-NEXT: v_addc_u32_e32 v19, vcc, -1, v19, vcc
-; GISEL-NEXT: v_or_b32_e32 v14, v14, v6
-; GISEL-NEXT: v_or_b32_e32 v6, v12, v7
-; GISEL-NEXT: v_or_b32_e32 v8, v8, v4
-; GISEL-NEXT: v_addc_u32_e32 v20, vcc, -1, v20, vcc
-; GISEL-NEXT: v_addc_u32_e32 v21, vcc, -1, v21, vcc
-; GISEL-NEXT: v_sub_i32_e32 v4, vcc, 1, v13
-; GISEL-NEXT: v_subb_u32_e32 v4, vcc, 0, v14, vcc
-; GISEL-NEXT: v_subrev_i32_e64 v6, s[4:5], 0, v6
-; GISEL-NEXT: v_or_b32_e32 v16, v18, v20
-; GISEL-NEXT: v_or_b32_e32 v17, v19, v21
-; GISEL-NEXT: v_subb_u32_e32 v4, vcc, 0, v15, vcc
-; GISEL-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[16:17]
-; GISEL-NEXT: v_ashrrev_i32_e32 v7, 31, v4
-; GISEL-NEXT: v_and_b32_e32 v4, 1, v7
-; GISEL-NEXT: v_and_b32_e32 v7, 2, v7
-; GISEL-NEXT: v_sub_i32_e64 v7, s[4:5], v13, v7
-; GISEL-NEXT: v_mov_b32_e32 v13, v5
-; GISEL-NEXT: v_mov_b32_e32 v12, v4
-; GISEL-NEXT: v_subbrev_u32_e64 v16, s[4:5], 0, v14, s[4:5]
-; GISEL-NEXT: s_or_b64 s[8:9], vcc, s[8:9]
-; GISEL-NEXT: v_subbrev_u32_e64 v17, vcc, 0, v15, s[4:5]
-; GISEL-NEXT: s_andn2_b64 exec, exec, s[8:9]
-; GISEL-NEXT: s_cbranch_execnz .LBB3_9
-; GISEL-NEXT: ; %bb.10: ; %Flow
-; GISEL-NEXT: s_or_b64 exec, exec, s[8:9]
-; GISEL-NEXT: .LBB3_11: ; %Flow11
-; GISEL-NEXT: s_or_b64 exec, exec, s[12:13]
-; GISEL-NEXT: v_lshl_b64 v[4:5], v[0:1], 1
-; GISEL-NEXT: v_lshl_b64 v[8:9], v[8:9], 1
-; GISEL-NEXT: v_lshrrev_b32_e32 v0, 31, v1
-; GISEL-NEXT: v_or_b32_e32 v8, v8, v0
-; GISEL-NEXT: v_or_b32_e32 v12, v12, v4
-; GISEL-NEXT: v_or_b32_e32 v13, v13, v5
-; GISEL-NEXT: .LBB3_12: ; %Flow12
-; GISEL-NEXT: s_or_b64 exec, exec, s[6:7]
-; GISEL-NEXT: v_mov_b32_e32 v0, v10
-; GISEL-NEXT: v_mov_b32_e32 v1, v11
-; GISEL-NEXT: v_mov_b32_e32 v4, v12
-; GISEL-NEXT: v_mov_b32_e32 v5, v13
-; GISEL-NEXT: v_mov_b32_e32 v6, v8
-; GISEL-NEXT: v_mov_b32_e32 v7, v9
+; GISEL-NEXT: v_lshrrev_b32_e32 v8, 1, v1
+; GISEL-NEXT: v_lshl_b64 v[0:1], v[2:3], 31
+; GISEL-NEXT: v_lshrrev_b32_e32 v2, 1, v3
+; GISEL-NEXT: v_lshrrev_b32_e32 v3, 1, v5
+; GISEL-NEXT: v_lshl_b64 v[4:5], v[6:7], 31
+; GISEL-NEXT: v_lshrrev_b32_e32 v6, 1, v7
+; GISEL-NEXT: v_or_b32_e32 v0, v0, v8
+; GISEL-NEXT: v_or_b32_e32 v4, v4, v3
+; GISEL-NEXT: v_mov_b32_e32 v3, 0
+; GISEL-NEXT: v_mov_b32_e32 v7, 0
; GISEL-NEXT: s_setpc_b64 s[30:31]
%shl = udiv <2 x i128> %lhs, <i128 8589934592, i128 8589934592>
ret <2 x i128> %shl
diff --git a/llvm/test/CodeGen/AMDGPU/rem_i128.ll b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
index 941b1fa66c49e..0ead77e1a9cf1 100644
--- a/llvm/test/CodeGen/AMDGPU/rem_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/rem_i128.ll
@@ -2657,49 +2657,56 @@ define i128 @v_srem_i128_v_pow2k(i128 %lhs) {
; GFX9-O0-LABEL: v_srem_i128_v_pow2k:
; GFX9-O0: ; %bb.0:
; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2
+; GFX9-O0-NEXT: buffer_store_dword v3, off, s[0:3], s32 ; 4-byte Folded Spill
+; GFX9-O0-NEXT: buffer_store_dword v2, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v1
-; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v7, v3
-; GFX9-O0-NEXT: ; kill: def $vgpr0 killed $vgpr0 def $vgpr0_vgpr1 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2
-; GFX9-O0-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-O0-NEXT: v_mov_b32_e32 v0, v1
-; GFX9-O0-NEXT: v_mov_b32_e32 v3, v6
-; GFX9-O0-NEXT: v_mov_b32_e32 v1, v7
+; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:4 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: v_mov_b32_e32 v3, v0
+; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 ; 4-byte Folded Reload
+; GFX9-O0-NEXT: ; kill: def $vgpr3 killed $vgpr3 def $vgpr3_vgpr4 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v4, v2
+; GFX9-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 def $vgpr1_vgpr2 killed $exec
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
+; GFX9-O0-NEXT: v_mov_b32_e32 v2, v0
; GFX9-O0-NEXT: s_mov_b32 s4, 63
-; GFX9-O0-NEXT: v_ashrrev_i64 v[6:7], s4, v[6:7]
+; GFX9-O0-NEXT: v_mov_b32_e32 v6, v2
+; GFX9-O0-NEXT: v_mov_b32_e32 v5, v1
+; GFX9-O0-NEXT: v_ashrrev_i64 v[5:6], s4, v[5:6]
; GFX9-O0-NEXT: s_mov_b32 s4, 31
-; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], s4, v[6:7]
+; GFX9-O0-NEXT: v_lshrrev_b64 v[6:7], s4, v[5:6]
+; GFX9-O0-NEXT: v_mov_b32_e32 v5, v3
+; GFX9-O0-NEXT: v_mov_b32_e32 v0, v4
+; GFX9-O0-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, v2
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v7
; GFX9-O0-NEXT: s_mov_b64 s[6:7], 0
; GFX9-O0-NEXT: s_mov_b32 s5, s6
; GFX9-O0-NEXT: s_mov_b32 s4, s7
; GFX9-O0-NEXT: v_add_co_u32_e32 v6, vcc, v5, v4
-; GFX9-O0-NEXT: v_addc_co_u32_e32 v4, vcc, v0, v2, vcc
-; GFX9-O0-NEXT: v_mov_b32_e32 v2, s5
-; GFX9-O0-NEXT: v_addc_co_u32_e32 v8, vcc, v3, v2, vcc
-; GFX9-O0-NEXT: v_mov_b32_e32 v2, s4
-; GFX9-O0-NEXT: v_addc_co_u32_e32 v2, vcc, v1, v2, vcc
+; GFX9-O0-NEXT: v_addc_co_u32_e32 v2, vcc, v0, v2, vcc
+; GFX9-O0-NEXT: v_mov_b32_e32 v4, s5
+; GFX9-O0-NEXT: v_addc_co_u32_e32 v8, vcc, v3, v4, vcc
+; GFX9-O0-NEXT: v_mov_b32_e32 v4, s4
+; GFX9-O0-NEXT: v_addc_co_u32_e32 v4, vcc, v1, v4, vcc
+; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v9, v4
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 def $vgpr6_vgpr7 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v7, v4
-; GFX9-O0-NEXT: v_mov_b32_e32 v4, v7
+; GFX9-O0-NEXT: v_mov_b32_e32 v7, v2
+; GFX9-O0-NEXT: v_mov_b32_e32 v2, v7
; GFX9-O0-NEXT: s_mov_b32 s6, -2
; GFX9-O0-NEXT: s_mov_b32 s4, 0
; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 def $sgpr4_sgpr5
; GFX9-O0-NEXT: s_mov_b32 s5, s6
; GFX9-O0-NEXT: s_mov_b32 s6, s5
-; GFX9-O0-NEXT: v_and_b32_e64 v4, v4, s6
-; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr6_vgpr7 killed $exec
+; GFX9-O0-NEXT: v_and_b32_e64 v2, v2, s6
+; GFX9-O0-NEXT: v_mov_b32_e32 v4, v6
; GFX9-O0-NEXT: ; kill: def $sgpr4 killed $sgpr4 killed $sgpr4_sgpr5
-; GFX9-O0-NEXT: v_and_b32_e64 v9, v6, s4
-; GFX9-O0-NEXT: ; kill: def $vgpr9 killed $vgpr9 def $vgpr9_vgpr10 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v10, v4
-; GFX9-O0-NEXT: v_mov_b32_e32 v7, v9
-; GFX9-O0-NEXT: v_mov_b32_e32 v6, v10
-; GFX9-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9 killed $exec
-; GFX9-O0-NEXT: v_mov_b32_e32 v9, v2
+; GFX9-O0-NEXT: v_and_b32_e64 v10, v4, s4
+; GFX9-O0-NEXT: ; kill: def $vgpr10 killed $vgpr10 def $vgpr10_vgpr11 killed $exec
+; GFX9-O0-NEXT: v_mov_b32_e32 v11, v2
+; GFX9-O0-NEXT: v_mov_b32_e32 v7, v10
+; GFX9-O0-NEXT: v_mov_b32_e32 v6, v11
; GFX9-O0-NEXT: v_mov_b32_e32 v4, v8
; GFX9-O0-NEXT: v_mov_b32_e32 v2, v9
; GFX9-O0-NEXT: v_sub_co_u32_e32 v5, vcc, v5, v7
diff --git a/llvm/test/CodeGen/RISCV/div_minsize.ll b/llvm/test/CodeGen/RISCV/div_minsize.ll
index 794af2f55d9a0..9a8ff822c78d5 100644
--- a/llvm/test/CodeGen/RISCV/div_minsize.ll
+++ b/llvm/test/CodeGen/RISCV/div_minsize.ll
@@ -88,8 +88,8 @@ define i128 @i128_sdiv(i128 %arg0) minsize nounwind {
; RV32IM-NEXT: slli a6, a6, 30
; RV32IM-NEXT: or a5, a5, a6
; RV32IM-NEXT: add a1, a3, a1
-; RV32IM-NEXT: srli a6, a1, 2
; RV32IM-NEXT: sltu a3, a1, a3
+; RV32IM-NEXT: srli a6, a1, 2
; RV32IM-NEXT: slli a1, a1, 30
; RV32IM-NEXT: add a2, a2, a3
; RV32IM-NEXT: or a1, a4, a1
@@ -168,8 +168,8 @@ define i256 @i256_sdiv(i256 %arg0) minsize nounwind {
; RV32IM-NEXT: and a4, a5, a4
; RV32IM-NEXT: or a1, t3, a6
; RV32IM-NEXT: add a4, a2, a4
-; RV32IM-NEXT: srli a5, a4, 2
; RV32IM-NEXT: sltu a2, a4, a2
+; RV32IM-NEXT: srli a5, a4, 2
; RV32IM-NEXT: slli a4, a4, 30
; RV32IM-NEXT: add a2, a3, a2
; RV32IM-NEXT: or a3, t2, a4
@@ -200,8 +200,8 @@ define i256 @i256_sdiv(i256 %arg0) minsize nounwind {
; RV64IM-NEXT: slli a6, a6, 62
; RV64IM-NEXT: or a5, a5, a6
; RV64IM-NEXT: add a1, a3, a1
-; RV64IM-NEXT: srli a6, a1, 2
; RV64IM-NEXT: sltu a3, a1, a3
+; RV64IM-NEXT: srli a6, a1, 2
; RV64IM-NEXT: slli a1, a1, 62
; RV64IM-NEXT: add a2, a2, a3
; RV64IM-NEXT: or a1, a4, a1
diff --git a/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll b/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
index 4d6d795e3beb8..165bd9e5947d7 100644
--- a/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
+++ b/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
@@ -16,10 +16,10 @@ define i129 @v_sdiv_i129_v_pow2k(i129 %lhs) nounwind {
; X64-NEXT: addq %rdi, %rax
; X64-NEXT: adcq $0, %rsi
; X64-NEXT: adcq $0, %rcx
-; X64-NEXT: shrdq $33, %rsi, %rax
; X64-NEXT: andl $1, %ecx
; X64-NEXT: movq %rcx, %rdx
; X64-NEXT: negq %rdx
+; X64-NEXT: shrdq $33, %rsi, %rax
; X64-NEXT: shldq $31, %rsi, %rdx
; X64-NEXT: retq
;
@@ -37,18 +37,19 @@ define i129 @v_sdiv_i129_v_pow2k(i129 %lhs) nounwind {
; X64-O0-NEXT: addq %rax, %rdi
; X64-O0-NEXT: adcq $0, %rsi
; X64-O0-NEXT: adcq $0, %rdx
-; X64-O0-NEXT: movq %rsi, %rax
-; X64-O0-NEXT: shldq $31, %rdi, %rax
-; X64-O0-NEXT: movl %edx, %ecx
-; X64-O0-NEXT: andl $1, %ecx
-; X64-O0-NEXT: # kill: def $rcx killed $ecx
+; X64-O0-NEXT: movl %edx, %eax
+; X64-O0-NEXT: andl $1, %eax
+; X64-O0-NEXT: movl %eax, %ecx
; X64-O0-NEXT: movq %rcx, %rdx
; X64-O0-NEXT: negq %rdx
+; X64-O0-NEXT: movq %rsi, %rax
+; X64-O0-NEXT: shldq $31, %rdi, %rax
; X64-O0-NEXT: shldq $31, %rsi, %rdx
; X64-O0-NEXT: retq
;
; X86-LABEL: v_sdiv_i129_v_pow2k:
; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
@@ -67,20 +68,21 @@ define i129 @v_sdiv_i129_v_pow2k(i129 %lhs) nounwind {
; X86-NEXT: adcl $0, %edx
; X86-NEXT: adcl $0, %ecx
; X86-NEXT: movl %ecx, %ebx
-; X86-NEXT: shldl $31, %edx, %ebx
+; X86-NEXT: andl $1, %ebx
+; X86-NEXT: movl %ebx, %ebp
+; X86-NEXT: negl %ebp
+; X86-NEXT: shldl $31, %edx, %ecx
; X86-NEXT: shldl $31, %esi, %edx
; X86-NEXT: shldl $31, %edi, %esi
-; X86-NEXT: andl $1, %ecx
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: negl %edi
; X86-NEXT: movl %esi, (%eax)
; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ebx, 8(%eax)
-; X86-NEXT: movl %edi, 12(%eax)
-; X86-NEXT: movb %cl, 16(%eax)
+; X86-NEXT: movl %ecx, 8(%eax)
+; X86-NEXT: movl %ebp, 12(%eax)
+; X86-NEXT: movb %bl, 16(%eax)
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
;
; X86-O0-LABEL: v_sdiv_i129_v_pow2k:
@@ -90,33 +92,34 @@ define i129 @v_sdiv_i129_v_pow2k(i129 %lhs) nounwind {
; X86-O0-NEXT: pushl %edi
; X86-O0-NEXT: pushl %esi
; X86-O0-NEXT: subl $8, %esp
-; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-O0-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-O0-NEXT: movl %ecx, %eax
; X86-O0-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %edi
; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-O0-NEXT: movl %edx, %ecx
-; X86-O0-NEXT: andl $1, %ecx
-; X86-O0-NEXT: negl %ecx
-; X86-O0-NEXT: movl %ecx, %edi
-; X86-O0-NEXT: andl $1, %edi
-; X86-O0-NEXT: addl %ecx, %eax
-; X86-O0-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-O0-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-O0-NEXT: adcl %edi, %esi
+; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-O0-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X86-O0-NEXT: movl %edi, %eax
+; X86-O0-NEXT: andl $1, %eax
+; X86-O0-NEXT: negl %eax
+; X86-O0-NEXT: movl %eax, %edx
+; X86-O0-NEXT: andl $1, %edx
+; X86-O0-NEXT: addl %eax, %esi
+; X86-O0-NEXT: movl (%esp), %eax # 4-byte Reload
+; X86-O0-NEXT: adcl %edx, %eax
; X86-O0-NEXT: adcl $0, %ebp
; X86-O0-NEXT: adcl $0, %ebx
-; X86-O0-NEXT: adcl $0, %edx
-; X86-O0-NEXT: movl %edx, %edi
-; X86-O0-NEXT: shldl $31, %ebx, %edi
-; X86-O0-NEXT: shldl $31, %ebp, %ebx
-; X86-O0-NEXT: shldl $31, %esi, %ebp
+; X86-O0-NEXT: adcl $0, %edi
+; X86-O0-NEXT: movl %edi, %edx
; X86-O0-NEXT: andl $1, %edx
; X86-O0-NEXT: movl %edx, %esi
; X86-O0-NEXT: negl %esi
+; X86-O0-NEXT: shldl $31, %ebx, %edi
+; X86-O0-NEXT: shldl $31, %ebp, %ebx
+; X86-O0-NEXT: shldl $31, %eax, %ebp
+; X86-O0-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
; X86-O0-NEXT: movl %ebp, (%ecx)
; X86-O0-NEXT: movl %ebx, 4(%ecx)
; X86-O0-NEXT: movl %edi, 8(%ecx)
@@ -137,35 +140,16 @@ define i129 @v_sdiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
; X64-LABEL: v_sdiv_exact_i129_v_pow2k:
; X64: # %bb.0:
; X64-NEXT: movq %rdx, %rcx
-; X64-NEXT: andl $1, %edx
-; X64-NEXT: negq %rdx
-; X64-NEXT: movl %edx, %eax
-; X64-NEXT: andl $1, %eax
-; X64-NEXT: shldq $32, %rdx, %rax
-; X64-NEXT: addq %rdi, %rax
-; X64-NEXT: adcq $0, %rsi
-; X64-NEXT: adcq $0, %rcx
-; X64-NEXT: shrdq $33, %rsi, %rax
+; X64-NEXT: movq %rdi, %rax
; X64-NEXT: andl $1, %ecx
; X64-NEXT: movq %rcx, %rdx
; X64-NEXT: negq %rdx
+; X64-NEXT: shrdq $33, %rsi, %rax
; X64-NEXT: shldq $31, %rsi, %rdx
; X64-NEXT: retq
;
; X64-O0-LABEL: v_sdiv_exact_i129_v_pow2k:
; X64-O0: # %bb.0:
-; X64-O0-NEXT: movl %edx, %eax
-; X64-O0-NEXT: andl $1, %eax
-; X64-O0-NEXT: movl %eax, %ecx
-; X64-O0-NEXT: negq %rcx
-; X64-O0-NEXT: movl %ecx, %r8d
-; X64-O0-NEXT: andl $1, %r8d
-; X64-O0-NEXT: # implicit-def: $rax
-; X64-O0-NEXT: movl %r8d, %eax
-; X64-O0-NEXT: shldq $32, %rcx, %rax
-; X64-O0-NEXT: addq %rax, %rdi
-; X64-O0-NEXT: adcq $0, %rsi
-; X64-O0-NEXT: adcq $0, %rdx
; X64-O0-NEXT: movq %rsi, %rax
; X64-O0-NEXT: shldq $31, %rdi, %rax
; X64-O0-NEXT: movl %edx, %ecx
@@ -178,38 +162,31 @@ define i129 @v_sdiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
;
; X86-LABEL: v_sdiv_exact_i129_v_pow2k:
; X86: # %bb.0:
+; X86-NEXT: pushl %ebp
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X86-NEXT: movl %ecx, %ebx
-; X86-NEXT: andl $1, %ebx
-; X86-NEXT: negl %ebx
-; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X86-NEXT: movl %ebx, %edi
-; X86-NEXT: andl $1, %edi
-; X86-NEXT: addl {{[0-9]+}}(%esp), %ebx
-; X86-NEXT: adcl {{[0-9]+}}(%esp), %edi
-; X86-NEXT: adcl $0, %esi
-; X86-NEXT: adcl $0, %edx
-; X86-NEXT: adcl $0, %ecx
-; X86-NEXT: movl %ecx, %ebx
-; X86-NEXT: shldl $31, %edx, %ebx
-; X86-NEXT: shldl $31, %esi, %edx
-; X86-NEXT: shldl $31, %edi, %esi
+; X86-NEXT: movl %edx, %ecx
; X86-NEXT: andl $1, %ecx
-; X86-NEXT: movl %ecx, %edi
-; X86-NEXT: negl %edi
-; X86-NEXT: movl %esi, (%eax)
-; X86-NEXT: movl %edx, 4(%eax)
-; X86-NEXT: movl %ebx, 8(%eax)
-; X86-NEXT: movl %edi, 12(%eax)
+; X86-NEXT: movl %ecx, %esi
+; X86-NEXT: negl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X86-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X86-NEXT: shrdl $1, %ebx, %edi
+; X86-NEXT: shldl $31, %ebp, %edx
+; X86-NEXT: shldl $31, %ebx, %ebp
+; X86-NEXT: movl %esi, 12(%eax)
+; X86-NEXT: movl %edx, 8(%eax)
+; X86-NEXT: movl %ebp, 4(%eax)
+; X86-NEXT: movl %edi, (%eax)
; X86-NEXT: movb %cl, 16(%eax)
; X86-NEXT: popl %esi
; X86-NEXT: popl %edi
; X86-NEXT: popl %ebx
+; X86-NEXT: popl %ebp
; X86-NEXT: retl $4
;
; X86-O0-LABEL: v_sdiv_exact_i129_v_pow2k:
@@ -218,27 +195,13 @@ define i129 @v_sdiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
; X86-O0-NEXT: pushl %ebx
; X86-O0-NEXT: pushl %edi
; X86-O0-NEXT: pushl %esi
-; X86-O0-NEXT: subl $8, %esp
-; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-O0-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X86-O0-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X86-O0-NEXT: movl %ecx, %eax
; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-O0-NEXT: movl %edx, %ecx
-; X86-O0-NEXT: andl $1, %ecx
-; X86-O0-NEXT: negl %ecx
-; X86-O0-NEXT: movl %ecx, %edi
-; X86-O0-NEXT: andl $1, %edi
-; X86-O0-NEXT: addl %ecx, %eax
-; X86-O0-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X86-O0-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X86-O0-NEXT: adcl %edi, %esi
-; X86-O0-NEXT: adcl $0, %ebp
-; X86-O0-NEXT: adcl $0, %ebx
-; X86-O0-NEXT: adcl $0, %edx
+; X86-O0-NEXT: movl {{[0-9]+}}(%esp), %esi
; X86-O0-NEXT: movl %edx, %edi
; X86-O0-NEXT: shldl $31, %ebx, %edi
; X86-O0-NEXT: shldl $31, %ebp, %ebx
@@ -252,7 +215,6 @@ define i129 @v_sdiv_exact_i129_v_pow2k(i129 %lhs) nounwind {
; X86-O0-NEXT: movl %esi, 12(%ecx)
; X86-O0-NEXT: # kill: def $dl killed $dl killed $edx
; X86-O0-NEXT: movb %dl, 16(%ecx)
-; X86-O0-NEXT: addl $8, %esp
; X86-O0-NEXT: popl %esi
; X86-O0-NEXT: popl %edi
; X86-O0-NEXT: popl %ebx
diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/sdiv129.ll b/llvm/test/Transforms/ExpandIRInsts/X86/sdiv129.ll
index 751bdbade15d9..f8f588310c541 100644
--- a/llvm/test/Transforms/ExpandIRInsts/X86/sdiv129.ll
+++ b/llvm/test/Transforms/ExpandIRInsts/X86/sdiv129.ll
@@ -76,6 +76,154 @@ define void @sdiv129(ptr %ptr, ptr %out) nounwind !prof !0 {
ret void
}
+define void @test_sdiv_pow2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_sdiv_pow2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 126
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
+; CHECK-NEXT: [[RES:%.*]] = ashr i129 [[ADJUSTED]], 3
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = sdiv i129 %a, 8
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_sdiv_neg_pow2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_sdiv_neg_pow2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 126
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
+; CHECK-NEXT: [[PRE_NEG:%.*]] = ashr i129 [[ADJUSTED]], 3
+; CHECK-NEXT: [[RES:%.*]] = sub i129 0, [[PRE_NEG]]
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = sdiv i129 %a, -8
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_sdiv_by_1(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_sdiv_by_1(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: store i129 [[A]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = sdiv i129 %a, 1
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_sdiv_by_neg1(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_sdiv_by_neg1(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[RES:%.*]] = sub i129 0, [[A]]
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = sdiv i129 %a, -1
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_sdiv_exact_pow2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_sdiv_exact_pow2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[RES:%.*]] = ashr exact i129 [[A]], 3
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = sdiv exact i129 %a, 8
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_sdiv_exact_neg_pow2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_sdiv_exact_neg_pow2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[PRE_NEG:%.*]] = ashr exact i129 [[A]], 3
+; CHECK-NEXT: [[RES:%.*]] = sub i129 0, [[PRE_NEG]]
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = sdiv exact i129 %a, -8
+ store i129 %res, ptr %out
+ ret void
+}
+
+; INT_MIN = -2^128
+define void @test_sdiv_intmin(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_sdiv_intmin(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 1
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
+; CHECK-NEXT: [[PRE_NEG:%.*]] = ashr i129 [[ADJUSTED]], 128
+; CHECK-NEXT: [[RES:%.*]] = sub i129 0, [[PRE_NEG]]
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = sdiv i129 %a, -340282366920938463463374607431768211456
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_sdiv_by_2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_sdiv_by_2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 128
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
+; CHECK-NEXT: [[RES:%.*]] = ashr i129 [[ADJUSTED]], 1
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = sdiv i129 %a, 2
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_sdiv_exact_by_2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_sdiv_exact_by_2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[RES:%.*]] = ashr exact i129 [[A]], 1
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = sdiv exact i129 %a, 2
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_sdiv_large_pow2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_sdiv_large_pow2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 65
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
+; CHECK-NEXT: [[RES:%.*]] = ashr i129 [[ADJUSTED]], 64
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = sdiv i129 %a, 18446744073709551616
+ store i129 %res, ptr %out
+ ret void
+}
+
!0 = !{!"function_entry_count", i64 1000}
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind }
diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/srem129.ll b/llvm/test/Transforms/ExpandIRInsts/X86/srem129.ll
index 45491ccda2b19..84fb376d32724 100644
--- a/llvm/test/Transforms/ExpandIRInsts/X86/srem129.ll
+++ b/llvm/test/Transforms/ExpandIRInsts/X86/srem129.ll
@@ -79,6 +79,121 @@ define void @test(ptr %ptr, ptr %out) nounwind !prof !0 {
ret void
}
+define void @test_srem_pow2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_srem_pow2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 126
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
+; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i129 [[ADJUSTED]], 3
+; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i129 [[SHIFTED]], 3
+; CHECK-NEXT: [[RES:%.*]] = sub i129 [[A]], [[TRUNCATED]]
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = srem i129 %a, 8
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_srem_neg_pow2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_srem_neg_pow2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 126
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
+; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i129 [[ADJUSTED]], 3
+; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i129 [[SHIFTED]], 3
+; CHECK-NEXT: [[RES:%.*]] = sub i129 [[A]], [[TRUNCATED]]
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = srem i129 %a, -8
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_srem_by_1(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_srem_by_1(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: store i129 0, ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = srem i129 %a, 1
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_srem_by_neg1(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_srem_by_neg1(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: store i129 0, ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = srem i129 %a, -1
+ store i129 %res, ptr %out
+ ret void
+}
+
+; INT_MIN = -2^128
+define void @test_srem_intmin(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_srem_intmin(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 1
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
+; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i129 [[ADJUSTED]], 128
+; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i129 [[SHIFTED]], 128
+; CHECK-NEXT: [[RES:%.*]] = sub i129 [[A]], [[TRUNCATED]]
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = srem i129 %a, -340282366920938463463374607431768211456
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_srem_by_2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_srem_by_2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 128
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
+; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i129 [[ADJUSTED]], 1
+; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i129 [[SHIFTED]], 1
+; CHECK-NEXT: [[RES:%.*]] = sub i129 [[A]], [[TRUNCATED]]
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = srem i129 %a, 2
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_srem_large_pow2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_srem_large_pow2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 65
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
+; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i129 [[ADJUSTED]], 64
+; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i129 [[SHIFTED]], 64
+; CHECK-NEXT: [[RES:%.*]] = sub i129 [[A]], [[TRUNCATED]]
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = srem i129 %a, 18446744073709551616
+ store i129 %res, ptr %out
+ ret void
+}
+
!0 = !{!"function_entry_count", i64 1000}
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind }
diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/udiv129.ll b/llvm/test/Transforms/ExpandIRInsts/X86/udiv129.ll
index 6ad696ae446fd..3f51f0279c585 100644
--- a/llvm/test/Transforms/ExpandIRInsts/X86/udiv129.ll
+++ b/llvm/test/Transforms/ExpandIRInsts/X86/udiv129.ll
@@ -65,6 +65,57 @@ define void @test(ptr %ptr, ptr %out) nounwind !prof !0 {
ret void
}
+define void @test_udiv_pow2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_udiv_pow2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[RES:%.*]] = lshr i129 [[A]], 3
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = udiv i129 %a, 8
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_udiv_by_1(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_udiv_by_1(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: store i129 [[A]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = udiv i129 %a, 1
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_udiv_large_pow2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_udiv_large_pow2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[RES:%.*]] = lshr i129 [[A]], 64
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = udiv i129 %a, 18446744073709551616
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_udiv_exact_pow2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_udiv_exact_pow2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[RES:%.*]] = lshr exact i129 [[A]], 3
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = udiv exact i129 %a, 8
+ store i129 %res, ptr %out
+ ret void
+}
+
!0 = !{!"function_entry_count", i64 1000}
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind }
diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/urem129.ll b/llvm/test/Transforms/ExpandIRInsts/X86/urem129.ll
index a4c4ac2cba329..b192cb840f074 100644
--- a/llvm/test/Transforms/ExpandIRInsts/X86/urem129.ll
+++ b/llvm/test/Transforms/ExpandIRInsts/X86/urem129.ll
@@ -69,6 +69,31 @@ define void @test(ptr %ptr, ptr %out) nounwind !prof !0 {
ret void
}
+define void @test_urem_pow2(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_urem_pow2(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: [[RES:%.*]] = and i129 [[A]], 7
+; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = urem i129 %a, 8
+ store i129 %res, ptr %out
+ ret void
+}
+
+define void @test_urem_by_1(ptr %ptr, ptr %out) nounwind {
+; CHECK-LABEL: @test_urem_by_1(
+; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
+; CHECK-NEXT: store i129 0, ptr [[OUT:%.*]], align 16
+; CHECK-NEXT: ret void
+;
+ %a = load i129, ptr %ptr
+ %res = urem i129 %a, 1
+ store i129 %res, ptr %out
+ ret void
+}
+
!0 = !{!"function_entry_count", i64 1000}
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind }
>From 9f4ed865711848d315525aeb461f425b050ebb4c Mon Sep 17 00:00:00 2001
From: Daniil Fukalov <dfukalov at gmail.com>
Date: Thu, 12 Feb 2026 20:38:21 +0100
Subject: [PATCH 2/3] Addressed review comments:
- Added proofs for power-of-2 div/rem expansion in ExpandIRInsts at
https://alive2.llvm.org/ce/z/Y-iWm-
- Tests updated as requested.
Also added CreateFreeze() where needed.
---
llvm/lib/CodeGen/ExpandIRInsts.cpp | 9 +
llvm/test/CodeGen/AMDGPU/div_i128.ll | 35 ++-
llvm/test/CodeGen/X86/div_i129_v_pow2k.ll | 4 +-
llvm/test/CodeGen/X86/i128-sdiv.ll | 2 +-
.../ExpandIRInsts/X86/divrem-pow2.ll | 255 ++++++++++++++++++
.../Transforms/ExpandIRInsts/X86/sdiv129.ll | 148 ----------
.../Transforms/ExpandIRInsts/X86/srem129.ll | 115 --------
.../Transforms/ExpandIRInsts/X86/udiv129.ll | 51 ----
.../Transforms/ExpandIRInsts/X86/urem129.ll | 25 --
9 files changed, 291 insertions(+), 353 deletions(-)
create mode 100644 llvm/test/Transforms/ExpandIRInsts/X86/divrem-pow2.ll
diff --git a/llvm/lib/CodeGen/ExpandIRInsts.cpp b/llvm/lib/CodeGen/ExpandIRInsts.cpp
index 5eed9c904edef..0656f5c171a89 100644
--- a/llvm/lib/CodeGen/ExpandIRInsts.cpp
+++ b/llvm/lib/CodeGen/ExpandIRInsts.cpp
@@ -129,6 +129,11 @@ static void expandPow2Division(BinaryOperator *Div) {
// X / 1 = X, X / -1 = -X.
Result = IsNegativeDivisor ? Builder.CreateNeg(X) : X;
} else if (IsSigned) {
+ // The signed expansion uses X multiple times (in the bias computation and
+ // the shift). Freeze X to ensure consistent behavior if it is undef/poison,
+ // matching the approach in IntegerDivision.cpp and expandFPToI.
+ if (!IsExact && !isGuaranteedNotToBeUndefOrPoison(X))
+ X = Builder.CreateFreeze(X, X->getName() + ".fr");
// For exact division, no bias is needed since there's no rounding.
Value *Dividend =
IsExact ? X : addSignedBias(Builder, X, BitWidth, ShiftAmt);
@@ -171,6 +176,10 @@ static void expandPow2Remainder(BinaryOperator *Rem) {
// Remainder by 1 or -1 is always 0.
Result = ConstantInt::get(Ty, 0);
} else if (IsSigned) {
+ // The signed expansion uses X multiple times (bias computation, shift, and
+ // final sub). Freeze X to ensure consistent behavior if it is undef/poison.
+ if (!isGuaranteedNotToBeUndefOrPoison(X))
+ X = Builder.CreateFreeze(X, X->getName() + ".fr");
Value *Adjusted = addSignedBias(Builder, X, BitWidth, ShiftAmt);
// Clear lower ShiftAmt bits via round-trip shift:
// Truncated = (Adjusted >> ShiftAmt) << ShiftAmt
diff --git a/llvm/test/CodeGen/AMDGPU/div_i128.ll b/llvm/test/CodeGen/AMDGPU/div_i128.ll
index 033149b17fe09..8193ee4104ab7 100644
--- a/llvm/test/CodeGen/AMDGPU/div_i128.ll
+++ b/llvm/test/CodeGen/AMDGPU/div_i128.ll
@@ -4335,25 +4335,38 @@ define i128 @v_sdiv_i128_v_pow2k(i128 %lhs) {
; GFX9-G-O0-LABEL: v_sdiv_i128_v_pow2k:
; GFX9-G-O0: ; %bb.0:
; GFX9-G-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v8, v0
+; GFX9-G-O0-NEXT: ; kill: def $vgpr8 killed $vgpr8 def $vgpr8_vgpr9_vgpr10_vgpr11 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v10, v2
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v11, v3
+; GFX9-G-O0-NEXT: ; kill: def $vgpr0_vgpr1 killed $vgpr8_vgpr9 killed $exec
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v11
+; GFX9-G-O0-NEXT: ; kill: def $vgpr2 killed $vgpr0 killed $exec
+; GFX9-G-O0-NEXT: ; kill: def $vgpr1 killed $vgpr1 killed $vgpr0_vgpr1 killed $exec
; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v0, v0, v3
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v0
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v0
+; GFX9-G-O0-NEXT: v_ashrrev_i32_e64 v0, v0, v1
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v0
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v0
; GFX9-G-O0-NEXT: s_mov_b32 s4, 31
; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s4
-; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v0, v[5:6]
+; GFX9-G-O0-NEXT: v_lshrrev_b64 v[6:7], v0, v[1:2]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v4, v8
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v1, v9
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, v10
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v11
; GFX9-G-O0-NEXT: v_mov_b32_e32 v5, v6
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, v7
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, v7
; GFX9-G-O0-NEXT: s_mov_b32 s8, 0
; GFX9-G-O0-NEXT: s_mov_b32 s5, 0
; GFX9-G-O0-NEXT: v_add_co_u32_e64 v4, s[6:7], v4, v5
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v1, s[6:7], v1, v0, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s8
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v2, v0, s[6:7]
-; GFX9-G-O0-NEXT: v_mov_b32_e32 v0, s5
-; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v4, s[6:7], v3, v0, s[6:7]
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v1, s[6:7], v1, v3, s[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v3, s8
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v5, s[6:7], v2, v3, s[6:7]
+; GFX9-G-O0-NEXT: v_mov_b32_e32 v2, s5
+; GFX9-G-O0-NEXT: v_addc_co_u32_e64 v4, s[6:7], v0, v2, s[6:7]
; GFX9-G-O0-NEXT: ; kill: def $vgpr5 killed $vgpr5 def $vgpr5_vgpr6 killed $exec
; GFX9-G-O0-NEXT: v_mov_b32_e32 v6, v4
; GFX9-G-O0-NEXT: s_mov_b32 s5, 1
diff --git a/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll b/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
index 165bd9e5947d7..9ea4cdbb92feb 100644
--- a/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
+++ b/llvm/test/CodeGen/X86/div_i129_v_pow2k.ll
@@ -53,13 +53,13 @@ define i129 @v_sdiv_i129_v_pow2k(i129 %lhs) nounwind {
; X86-NEXT: pushl %ebx
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NEXT: movl %ecx, %ebx
; X86-NEXT: andl $1, %ebx
; X86-NEXT: negl %ebx
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X86-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-NEXT: movl %ebx, %edi
; X86-NEXT: andl $1, %edi
; X86-NEXT: addl {{[0-9]+}}(%esp), %ebx
diff --git a/llvm/test/CodeGen/X86/i128-sdiv.ll b/llvm/test/CodeGen/X86/i128-sdiv.ll
index 7d5757392c982..f78e34ef60569 100644
--- a/llvm/test/CodeGen/X86/i128-sdiv.ll
+++ b/llvm/test/CodeGen/X86/i128-sdiv.ll
@@ -13,8 +13,8 @@ define i128 @test1(i128 %x) nounwind {
; X86-NEXT: pushl %edi
; X86-NEXT: pushl %esi
; X86-NEXT: andl $-16, %esp
-; X86-NEXT: movl 8(%ebp), %eax
; X86-NEXT: movl 36(%ebp), %ecx
+; X86-NEXT: movl 8(%ebp), %eax
; X86-NEXT: movl %ecx, %esi
; X86-NEXT: sarl $31, %esi
; X86-NEXT: movl %esi, %edx
diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/divrem-pow2.ll b/llvm/test/Transforms/ExpandIRInsts/X86/divrem-pow2.ll
new file mode 100644
index 0000000000000..d14b230358a62
--- /dev/null
+++ b/llvm/test/Transforms/ExpandIRInsts/X86/divrem-pow2.ll
@@ -0,0 +1,255 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt -S -mtriple=x86_64-- -expand-ir-insts -expand-div-rem-bits 0 < %s | FileCheck %s
+; RUN: opt -S -mtriple=x86_64-- -passes='require<libcall-lowering-info>,expand-ir-insts' -expand-div-rem-bits 0 < %s | FileCheck %s
+
+; Test power-of-2 division/remainder expansion.
+
+;--- udiv ---
+define i32 @udiv_by_8(i32 %x) {
+; CHECK-LABEL: @udiv_by_8(
+; CHECK-NEXT: [[R:%.*]] = lshr i32 [[X:%.*]], 3
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = udiv i32 %x, 8
+ ret i32 %r
+}
+
+define i32 @udiv_by_1(i32 %x) {
+; CHECK-LABEL: @udiv_by_1(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %r = udiv i32 %x, 1
+ ret i32 %r
+}
+
+define i32 @udiv_exact_by_8(i32 %x) {
+; CHECK-LABEL: @udiv_exact_by_8(
+; CHECK-NEXT: [[R:%.*]] = lshr exact i32 [[X:%.*]], 3
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = udiv exact i32 %x, 8
+ ret i32 %r
+}
+
+;--- sdiv ---
+define i32 @sdiv_by_8(i32 %x) {
+; CHECK-LABEL: @sdiv_by_8(
+; CHECK-NEXT: [[X:%.*]] = freeze i32 [[X1:%.*]]
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i32 [[X]], 31
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i32 [[SIGN]], 29
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i32 [[X]], [[BIAS]]
+; CHECK-NEXT: [[R:%.*]] = ashr i32 [[ADJUSTED]], 3
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = sdiv i32 %x, 8
+ ret i32 %r
+}
+
+define i32 @sdiv_by_neg8(i32 %x) {
+; CHECK-LABEL: @sdiv_by_neg8(
+; CHECK-NEXT: [[X:%.*]] = freeze i32 [[X1:%.*]]
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i32 [[X]], 31
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i32 [[SIGN]], 29
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i32 [[X]], [[BIAS]]
+; CHECK-NEXT: [[PRE_NEG:%.*]] = ashr i32 [[ADJUSTED]], 3
+; CHECK-NEXT: [[R:%.*]] = sub i32 0, [[PRE_NEG]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = sdiv i32 %x, -8
+ ret i32 %r
+}
+
+define i32 @sdiv_by_2(i32 %x) {
+; CHECK-LABEL: @sdiv_by_2(
+; CHECK-NEXT: [[X:%.*]] = freeze i32 [[X1:%.*]]
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i32 [[X]], 31
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i32 [[SIGN]], 31
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i32 [[X]], [[BIAS]]
+; CHECK-NEXT: [[R:%.*]] = ashr i32 [[ADJUSTED]], 1
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = sdiv i32 %x, 2
+ ret i32 %r
+}
+
+define i32 @sdiv_by_1(i32 %x) {
+; CHECK-LABEL: @sdiv_by_1(
+; CHECK-NEXT: ret i32 [[X:%.*]]
+;
+ %r = sdiv i32 %x, 1
+ ret i32 %r
+}
+
+define i32 @sdiv_by_neg1(i32 %x) {
+; CHECK-LABEL: @sdiv_by_neg1(
+; CHECK-NEXT: [[R:%.*]] = sub i32 0, [[X:%.*]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = sdiv i32 %x, -1
+ ret i32 %r
+}
+
+define i32 @sdiv_exact_by_8(i32 %x) {
+; CHECK-LABEL: @sdiv_exact_by_8(
+; CHECK-NEXT: [[R:%.*]] = ashr exact i32 [[X:%.*]], 3
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = sdiv exact i32 %x, 8
+ ret i32 %r
+}
+
+define i32 @sdiv_exact_by_neg8(i32 %x) {
+; CHECK-LABEL: @sdiv_exact_by_neg8(
+; CHECK-NEXT: [[PRE_NEG:%.*]] = ashr exact i32 [[X:%.*]], 3
+; CHECK-NEXT: [[R:%.*]] = sub i32 0, [[PRE_NEG]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = sdiv exact i32 %x, -8
+ ret i32 %r
+}
+
+define i32 @sdiv_exact_by_2(i32 %x) {
+; CHECK-LABEL: @sdiv_exact_by_2(
+; CHECK-NEXT: [[R:%.*]] = ashr exact i32 [[X:%.*]], 1
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = sdiv exact i32 %x, 2
+ ret i32 %r
+}
+
+define i32 @sdiv_by_intmin(i32 %x) {
+; CHECK-LABEL: @sdiv_by_intmin(
+; CHECK-NEXT: [[X:%.*]] = freeze i32 [[X1:%.*]]
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i32 [[X]], 31
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i32 [[SIGN]], 1
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i32 [[X]], [[BIAS]]
+; CHECK-NEXT: [[PRE_NEG:%.*]] = ashr i32 [[ADJUSTED]], 31
+; CHECK-NEXT: [[R:%.*]] = sub i32 0, [[PRE_NEG]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = sdiv i32 %x, -2147483648
+ ret i32 %r
+}
+
+define i32 @sdiv_large_pow2(i32 %x) {
+; CHECK-LABEL: @sdiv_large_pow2(
+; CHECK-NEXT: [[X:%.*]] = freeze i32 [[X1:%.*]]
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i32 [[X]], 31
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i32 [[SIGN]], 16
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i32 [[X]], [[BIAS]]
+; CHECK-NEXT: [[R:%.*]] = ashr i32 [[ADJUSTED]], 16
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = sdiv i32 %x, 65536
+ ret i32 %r
+}
+
+;--- urem ---
+define i32 @urem_by_8(i32 %x) {
+; CHECK-LABEL: @urem_by_8(
+; CHECK-NEXT: [[R:%.*]] = and i32 [[X:%.*]], 7
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = urem i32 %x, 8
+ ret i32 %r
+}
+
+define i32 @urem_by_1(i32 %x) {
+; CHECK-LABEL: @urem_by_1(
+; CHECK-NEXT: ret i32 0
+;
+ %r = urem i32 %x, 1
+ ret i32 %r
+}
+
+;--- srem ---
+define i32 @srem_by_8(i32 %x) {
+; CHECK-LABEL: @srem_by_8(
+; CHECK-NEXT: [[X:%.*]] = freeze i32 [[X1:%.*]]
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i32 [[X]], 31
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i32 [[SIGN]], 29
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i32 [[X]], [[BIAS]]
+; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i32 [[ADJUSTED]], 3
+; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i32 [[SHIFTED]], 3
+; CHECK-NEXT: [[R:%.*]] = sub i32 [[X]], [[TRUNCATED]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = srem i32 %x, 8
+ ret i32 %r
+}
+
+define i32 @srem_by_neg8(i32 %x) {
+; CHECK-LABEL: @srem_by_neg8(
+; CHECK-NEXT: [[X:%.*]] = freeze i32 [[X1:%.*]]
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i32 [[X]], 31
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i32 [[SIGN]], 29
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i32 [[X]], [[BIAS]]
+; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i32 [[ADJUSTED]], 3
+; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i32 [[SHIFTED]], 3
+; CHECK-NEXT: [[R:%.*]] = sub i32 [[X]], [[TRUNCATED]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = srem i32 %x, -8
+ ret i32 %r
+}
+
+define i32 @srem_by_2(i32 %x) {
+; CHECK-LABEL: @srem_by_2(
+; CHECK-NEXT: [[X:%.*]] = freeze i32 [[X1:%.*]]
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i32 [[X]], 31
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i32 [[SIGN]], 31
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i32 [[X]], [[BIAS]]
+; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i32 [[ADJUSTED]], 1
+; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i32 [[SHIFTED]], 1
+; CHECK-NEXT: [[R:%.*]] = sub i32 [[X]], [[TRUNCATED]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = srem i32 %x, 2
+ ret i32 %r
+}
+
+define i32 @srem_by_1(i32 %x) {
+; CHECK-LABEL: @srem_by_1(
+; CHECK-NEXT: ret i32 0
+;
+ %r = srem i32 %x, 1
+ ret i32 %r
+}
+
+define i32 @srem_by_neg1(i32 %x) {
+; CHECK-LABEL: @srem_by_neg1(
+; CHECK-NEXT: ret i32 0
+;
+ %r = srem i32 %x, -1
+ ret i32 %r
+}
+
+define i32 @srem_by_intmin(i32 %x) {
+; CHECK-LABEL: @srem_by_intmin(
+; CHECK-NEXT: [[X:%.*]] = freeze i32 [[X1:%.*]]
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i32 [[X]], 31
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i32 [[SIGN]], 1
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i32 [[X]], [[BIAS]]
+; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i32 [[ADJUSTED]], 31
+; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i32 [[SHIFTED]], 31
+; CHECK-NEXT: [[R:%.*]] = sub i32 [[X]], [[TRUNCATED]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = srem i32 %x, -2147483648
+ ret i32 %r
+}
+
+define i32 @srem_large_pow2(i32 %x) {
+; CHECK-LABEL: @srem_large_pow2(
+; CHECK-NEXT: [[X:%.*]] = freeze i32 [[X1:%.*]]
+; CHECK-NEXT: [[SIGN:%.*]] = ashr i32 [[X]], 31
+; CHECK-NEXT: [[BIAS:%.*]] = lshr i32 [[SIGN]], 16
+; CHECK-NEXT: [[ADJUSTED:%.*]] = add i32 [[X]], [[BIAS]]
+; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i32 [[ADJUSTED]], 16
+; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i32 [[SHIFTED]], 16
+; CHECK-NEXT: [[R:%.*]] = sub i32 [[X]], [[TRUNCATED]]
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = srem i32 %x, 65536
+ ret i32 %r
+}
diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/sdiv129.ll b/llvm/test/Transforms/ExpandIRInsts/X86/sdiv129.ll
index f8f588310c541..751bdbade15d9 100644
--- a/llvm/test/Transforms/ExpandIRInsts/X86/sdiv129.ll
+++ b/llvm/test/Transforms/ExpandIRInsts/X86/sdiv129.ll
@@ -76,154 +76,6 @@ define void @sdiv129(ptr %ptr, ptr %out) nounwind !prof !0 {
ret void
}
-define void @test_sdiv_pow2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_sdiv_pow2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
-; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 126
-; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
-; CHECK-NEXT: [[RES:%.*]] = ashr i129 [[ADJUSTED]], 3
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = sdiv i129 %a, 8
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_sdiv_neg_pow2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_sdiv_neg_pow2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
-; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 126
-; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
-; CHECK-NEXT: [[PRE_NEG:%.*]] = ashr i129 [[ADJUSTED]], 3
-; CHECK-NEXT: [[RES:%.*]] = sub i129 0, [[PRE_NEG]]
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = sdiv i129 %a, -8
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_sdiv_by_1(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_sdiv_by_1(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: store i129 [[A]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = sdiv i129 %a, 1
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_sdiv_by_neg1(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_sdiv_by_neg1(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[RES:%.*]] = sub i129 0, [[A]]
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = sdiv i129 %a, -1
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_sdiv_exact_pow2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_sdiv_exact_pow2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[RES:%.*]] = ashr exact i129 [[A]], 3
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = sdiv exact i129 %a, 8
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_sdiv_exact_neg_pow2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_sdiv_exact_neg_pow2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[PRE_NEG:%.*]] = ashr exact i129 [[A]], 3
-; CHECK-NEXT: [[RES:%.*]] = sub i129 0, [[PRE_NEG]]
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = sdiv exact i129 %a, -8
- store i129 %res, ptr %out
- ret void
-}
-
-; INT_MIN = -2^128
-define void @test_sdiv_intmin(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_sdiv_intmin(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
-; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 1
-; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
-; CHECK-NEXT: [[PRE_NEG:%.*]] = ashr i129 [[ADJUSTED]], 128
-; CHECK-NEXT: [[RES:%.*]] = sub i129 0, [[PRE_NEG]]
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = sdiv i129 %a, -340282366920938463463374607431768211456
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_sdiv_by_2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_sdiv_by_2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
-; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 128
-; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
-; CHECK-NEXT: [[RES:%.*]] = ashr i129 [[ADJUSTED]], 1
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = sdiv i129 %a, 2
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_sdiv_exact_by_2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_sdiv_exact_by_2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[RES:%.*]] = ashr exact i129 [[A]], 1
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = sdiv exact i129 %a, 2
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_sdiv_large_pow2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_sdiv_large_pow2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
-; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 65
-; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
-; CHECK-NEXT: [[RES:%.*]] = ashr i129 [[ADJUSTED]], 64
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = sdiv i129 %a, 18446744073709551616
- store i129 %res, ptr %out
- ret void
-}
-
!0 = !{!"function_entry_count", i64 1000}
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind }
diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/srem129.ll b/llvm/test/Transforms/ExpandIRInsts/X86/srem129.ll
index 84fb376d32724..45491ccda2b19 100644
--- a/llvm/test/Transforms/ExpandIRInsts/X86/srem129.ll
+++ b/llvm/test/Transforms/ExpandIRInsts/X86/srem129.ll
@@ -79,121 +79,6 @@ define void @test(ptr %ptr, ptr %out) nounwind !prof !0 {
ret void
}
-define void @test_srem_pow2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_srem_pow2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
-; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 126
-; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
-; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i129 [[ADJUSTED]], 3
-; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i129 [[SHIFTED]], 3
-; CHECK-NEXT: [[RES:%.*]] = sub i129 [[A]], [[TRUNCATED]]
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = srem i129 %a, 8
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_srem_neg_pow2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_srem_neg_pow2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
-; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 126
-; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
-; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i129 [[ADJUSTED]], 3
-; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i129 [[SHIFTED]], 3
-; CHECK-NEXT: [[RES:%.*]] = sub i129 [[A]], [[TRUNCATED]]
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = srem i129 %a, -8
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_srem_by_1(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_srem_by_1(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: store i129 0, ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = srem i129 %a, 1
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_srem_by_neg1(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_srem_by_neg1(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: store i129 0, ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = srem i129 %a, -1
- store i129 %res, ptr %out
- ret void
-}
-
-; INT_MIN = -2^128
-define void @test_srem_intmin(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_srem_intmin(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
-; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 1
-; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
-; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i129 [[ADJUSTED]], 128
-; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i129 [[SHIFTED]], 128
-; CHECK-NEXT: [[RES:%.*]] = sub i129 [[A]], [[TRUNCATED]]
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = srem i129 %a, -340282366920938463463374607431768211456
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_srem_by_2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_srem_by_2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
-; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 128
-; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
-; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i129 [[ADJUSTED]], 1
-; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i129 [[SHIFTED]], 1
-; CHECK-NEXT: [[RES:%.*]] = sub i129 [[A]], [[TRUNCATED]]
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = srem i129 %a, 2
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_srem_large_pow2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_srem_large_pow2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[SIGN:%.*]] = ashr i129 [[A]], 128
-; CHECK-NEXT: [[BIAS:%.*]] = lshr i129 [[SIGN]], 65
-; CHECK-NEXT: [[ADJUSTED:%.*]] = add i129 [[A]], [[BIAS]]
-; CHECK-NEXT: [[SHIFTED:%.*]] = ashr i129 [[ADJUSTED]], 64
-; CHECK-NEXT: [[TRUNCATED:%.*]] = shl i129 [[SHIFTED]], 64
-; CHECK-NEXT: [[RES:%.*]] = sub i129 [[A]], [[TRUNCATED]]
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = srem i129 %a, 18446744073709551616
- store i129 %res, ptr %out
- ret void
-}
-
!0 = !{!"function_entry_count", i64 1000}
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind }
diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/udiv129.ll b/llvm/test/Transforms/ExpandIRInsts/X86/udiv129.ll
index 3f51f0279c585..6ad696ae446fd 100644
--- a/llvm/test/Transforms/ExpandIRInsts/X86/udiv129.ll
+++ b/llvm/test/Transforms/ExpandIRInsts/X86/udiv129.ll
@@ -65,57 +65,6 @@ define void @test(ptr %ptr, ptr %out) nounwind !prof !0 {
ret void
}
-define void @test_udiv_pow2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_udiv_pow2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[RES:%.*]] = lshr i129 [[A]], 3
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = udiv i129 %a, 8
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_udiv_by_1(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_udiv_by_1(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: store i129 [[A]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = udiv i129 %a, 1
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_udiv_large_pow2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_udiv_large_pow2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[RES:%.*]] = lshr i129 [[A]], 64
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = udiv i129 %a, 18446744073709551616
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_udiv_exact_pow2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_udiv_exact_pow2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[RES:%.*]] = lshr exact i129 [[A]], 3
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = udiv exact i129 %a, 8
- store i129 %res, ptr %out
- ret void
-}
-
!0 = !{!"function_entry_count", i64 1000}
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind }
diff --git a/llvm/test/Transforms/ExpandIRInsts/X86/urem129.ll b/llvm/test/Transforms/ExpandIRInsts/X86/urem129.ll
index b192cb840f074..a4c4ac2cba329 100644
--- a/llvm/test/Transforms/ExpandIRInsts/X86/urem129.ll
+++ b/llvm/test/Transforms/ExpandIRInsts/X86/urem129.ll
@@ -69,31 +69,6 @@ define void @test(ptr %ptr, ptr %out) nounwind !prof !0 {
ret void
}
-define void @test_urem_pow2(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_urem_pow2(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: [[RES:%.*]] = and i129 [[A]], 7
-; CHECK-NEXT: store i129 [[RES]], ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = urem i129 %a, 8
- store i129 %res, ptr %out
- ret void
-}
-
-define void @test_urem_by_1(ptr %ptr, ptr %out) nounwind {
-; CHECK-LABEL: @test_urem_by_1(
-; CHECK-NEXT: [[A:%.*]] = load i129, ptr [[PTR:%.*]], align 16
-; CHECK-NEXT: store i129 0, ptr [[OUT:%.*]], align 16
-; CHECK-NEXT: ret void
-;
- %a = load i129, ptr %ptr
- %res = urem i129 %a, 1
- store i129 %res, ptr %out
- ret void
-}
-
!0 = !{!"function_entry_count", i64 1000}
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nounwind }
>From 9466b46a9c5144e1be45758dc4dd14159b72f829 Mon Sep 17 00:00:00 2001
From: Daniil Fukalov <dfukalov at gmail.com>
Date: Mon, 16 Feb 2026 18:47:03 +0100
Subject: [PATCH 3/3] Unify expandPow2Division/expandPow2Remainder into
expandPow2DivRem.
Merge the two functions into one to share the common signed-path logic (freeze, bias, ashr) and reduce code duplication, as suggested.
---
llvm/lib/CodeGen/ExpandIRInsts.cpp | 151 +++++++++++++----------------
1 file changed, 66 insertions(+), 85 deletions(-)
diff --git a/llvm/lib/CodeGen/ExpandIRInsts.cpp b/llvm/lib/CodeGen/ExpandIRInsts.cpp
index 0656f5c171a89..8e5b0f35adb8a 100644
--- a/llvm/lib/CodeGen/ExpandIRInsts.cpp
+++ b/llvm/lib/CodeGen/ExpandIRInsts.cpp
@@ -101,17 +101,30 @@ static Value *addSignedBias(IRBuilder<> &Builder, Value *X, unsigned BitWidth,
return Builder.CreateAdd(X, Bias, "adjusted");
}
-/// Expand division by a power-of-2 constant.
-/// udiv X, 2^C -> lshr X, C
-/// sdiv X, 2^C -> ashr (add X, Bias), C (Bias corrects rounding)
-/// sdiv exact X, 2^C -> ashr exact X, C (no bias needed)
-/// For negative power-of-2 divisors, the result is negated.
-static void expandPow2Division(BinaryOperator *Div) {
- bool IsSigned = isSigned(Div->getOpcode());
- bool IsExact = Div->isExact();
- Value *X = Div->getOperand(0);
- auto *C = cast<ConstantInt>(Div->getOperand(1));
- Type *Ty = Div->getType();
+/// Expand division or remainder by a power-of-2 constant.
+/// Division (let C = log2(|divisor|)):
+/// udiv X, 2^C -> lshr X, C
+/// sdiv X, 2^C -> ashr (add X, Bias), C (Bias corrects rounding)
+/// sdiv exact X, 2^C -> ashr exact X, C (no bias needed)
+/// For negative power-of-2 divisors, the division result is negated.
+/// Remainder (let C = log2(|divisor|)):
+/// urem X, 2^C -> and X, (2^C - 1)
+/// srem X, 2^C -> sub X, (shl (ashr (add X, Bias), C), C)
+static void expandPow2DivRem(BinaryOperator *BO) {
+ LLVM_DEBUG(dbgs() << "Expanding instruction: " << *BO << '\n');
+
+ unsigned Opcode = BO->getOpcode();
+ bool IsDiv = (Opcode == Instruction::UDiv || Opcode == Instruction::SDiv);
+ bool IsSigned = isSigned(Opcode);
+ // isExact() is only valid for div.
+ bool IsExact = IsDiv && BO->isExact();
+
+ assert(isConstantPowerOfTwo(BO->getOperand(1), IsSigned) &&
+ "Expected power-of-2 constant divisor");
+
+ Value *X = BO->getOperand(0);
+ auto *C = cast<ConstantInt>(BO->getOperand(1));
+ Type *Ty = BO->getType();
unsigned BitWidth = Ty->getIntegerBitWidth();
APInt DivisorVal = C->getValue();
@@ -121,82 +134,52 @@ static void expandPow2Division(BinaryOperator *Div) {
// INT_MIN, without needing to negate the value first.
unsigned ShiftAmt = DivisorVal.countr_zero();
- IRBuilder<> Builder(Div);
+ IRBuilder<> Builder(BO);
Value *Result;
if (ShiftAmt == 0) {
- // Division by 1 or -1.
- // X / 1 = X, X / -1 = -X.
- Result = IsNegativeDivisor ? Builder.CreateNeg(X) : X;
+ // Div by 1/-1: X / 1 = X, X / -1 = -X.
+ // Rem by 1/-1: always 0.
+ if (IsDiv)
+ Result = IsNegativeDivisor ? Builder.CreateNeg(X) : X;
+ else
+ Result = ConstantInt::get(Ty, 0);
} else if (IsSigned) {
- // The signed expansion uses X multiple times (in the bias computation and
- // the shift). Freeze X to ensure consistent behavior if it is undef/poison,
- // matching the approach in IntegerDivision.cpp and expandFPToI.
+ // The signed expansion uses X multiple times (bias computation, shift,
+ // and sub for remainder). Freeze X to ensure consistent behavior if it is
+ // undef/poison. For exact division, no bias is needed and X is used only
+ // once, so freeze is unnecessary.
if (!IsExact && !isGuaranteedNotToBeUndefOrPoison(X))
X = Builder.CreateFreeze(X, X->getName() + ".fr");
// For exact division, no bias is needed since there's no rounding.
Value *Dividend =
IsExact ? X : addSignedBias(Builder, X, BitWidth, ShiftAmt);
- Result = Builder.CreateAShr(Dividend, ShiftAmt,
- IsNegativeDivisor ? "pre.neg" : "", IsExact);
- if (IsNegativeDivisor)
- Result = Builder.CreateNeg(Result);
+ Value *Quotient = Builder.CreateAShr(
+ Dividend, ShiftAmt, IsDiv && IsNegativeDivisor ? "pre.neg" : "shifted",
+ IsExact);
+ if (IsDiv) {
+ Result = IsNegativeDivisor ? Builder.CreateNeg(Quotient) : Quotient;
+ } else {
+ // Rem = X - (Quotient << ShiftAmt):
+ // clear lower ShiftAmt bits via round-trip shift, then subtract.
+ Value *Truncated = Builder.CreateShl(Quotient, ShiftAmt, "truncated");
+ Result = Builder.CreateSub(X, Truncated);
+ }
} else {
- Result = Builder.CreateLShr(X, ShiftAmt, "", IsExact);
+ if (IsDiv) {
+ Result = Builder.CreateLShr(X, ShiftAmt, "", IsExact);
+ } else {
+ APInt Mask = APInt::getLowBitsSet(BitWidth, ShiftAmt);
+ Result = Builder.CreateAnd(X, ConstantInt::get(Ty, Mask));
+ }
}
- Div->replaceAllUsesWith(Result);
- // Transfer the name of the original instruction to its replacement,
- // unless the result is the original dividend itself (div by 1).
+ BO->replaceAllUsesWith(Result);
if (Result != X)
if (auto *RI = dyn_cast<Instruction>(Result))
- RI->takeName(Div);
- Div->dropAllReferences();
- Div->eraseFromParent();
-}
-
-/// Expand remainder by a power-of-2 constant (let C = log2(|divisor|)).
-/// urem X, 2^C -> and X, (2^C - 1)
-/// srem X, 2^C -> sub X, (shl (ashr (add X, Bias), C), C)
-static void expandPow2Remainder(BinaryOperator *Rem) {
- bool IsSigned = isSigned(Rem->getOpcode());
- Value *X = Rem->getOperand(0);
- auto *C = cast<ConstantInt>(Rem->getOperand(1));
- Type *Ty = Rem->getType();
- unsigned BitWidth = Ty->getIntegerBitWidth();
-
- // Use countr_zero() to get the shift amount directly from the bit pattern.
- // This works for both positive and negative powers of 2, including INT_MIN.
- unsigned ShiftAmt = C->getValue().countr_zero();
-
- IRBuilder<> Builder(Rem);
- Value *Result;
-
- if (ShiftAmt == 0) {
- // Remainder by 1 or -1 is always 0.
- Result = ConstantInt::get(Ty, 0);
- } else if (IsSigned) {
- // The signed expansion uses X multiple times (bias computation, shift, and
- // final sub). Freeze X to ensure consistent behavior if it is undef/poison.
- if (!isGuaranteedNotToBeUndefOrPoison(X))
- X = Builder.CreateFreeze(X, X->getName() + ".fr");
- Value *Adjusted = addSignedBias(Builder, X, BitWidth, ShiftAmt);
- // Clear lower ShiftAmt bits via round-trip shift:
- // Truncated = (Adjusted >> ShiftAmt) << ShiftAmt
- Value *Shifted = Builder.CreateAShr(Adjusted, ShiftAmt, "shifted");
- Value *Truncated = Builder.CreateShl(Shifted, ShiftAmt, "truncated");
- Result = Builder.CreateSub(X, Truncated);
- } else {
- APInt Mask = APInt::getLowBitsSet(BitWidth, ShiftAmt);
- Value *MaskVal = ConstantInt::get(Ty, Mask);
- Result = Builder.CreateAnd(X, MaskVal);
- }
-
- Rem->replaceAllUsesWith(Result);
- if (auto *RI = dyn_cast<Instruction>(Result))
- RI->takeName(Rem);
- Rem->dropAllReferences();
- Rem->eraseFromParent();
+ RI->takeName(BO);
+ BO->dropAllReferences();
+ BO->eraseFromParent();
}
/// This class implements a precise expansion of the frem instruction.
@@ -1285,21 +1268,19 @@ static bool runImpl(Function &F, const TargetLowering &TLI,
break;
case Instruction::UDiv:
- case Instruction::SDiv: {
- auto *BO = cast<BinaryOperator>(I);
- if (isConstantPowerOfTwo(BO->getOperand(1), isSigned(BO->getOpcode())))
- expandPow2Division(BO);
- else
- expandDivision(BO);
- break;
- }
+ case Instruction::SDiv:
case Instruction::URem:
case Instruction::SRem: {
auto *BO = cast<BinaryOperator>(I);
- if (isConstantPowerOfTwo(BO->getOperand(1), isSigned(BO->getOpcode())))
- expandPow2Remainder(BO);
- else
- expandRemainder(BO);
+ if (isConstantPowerOfTwo(BO->getOperand(1), isSigned(BO->getOpcode()))) {
+ expandPow2DivRem(BO);
+ } else {
+ unsigned Opc = BO->getOpcode();
+ if (Opc == Instruction::UDiv || Opc == Instruction::SDiv)
+ expandDivision(BO);
+ else
+ expandRemainder(BO);
+ }
break;
}
case Instruction::Call: {
More information about the llvm-commits
mailing list