[llvm] [X86] Prefer sub-of-not for vector inc-of-add (PR #207695)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 21 02:59:08 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/207695
>From 98f43cd175d1897fb26a9575b169a86036000b97 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <rshenoy at amd.com>
Date: Mon, 6 Jul 2026 15:38:08 +0530
Subject: [PATCH 1/4] [X86] Prefer sub-of-not for vector inc-of-add
Avoids lengthening the dependency chain for vector conditional
increments by preferring sub-of-not for vectors. Keeps existing
inc-of-add for scalars.
Fixes #167441.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 7 +
llvm/lib/Target/X86/X86ISelLowering.h | 2 +
.../CodeGen/X86/expand-vp-int-intrinsics.ll | 144 ++++++++--------
llvm/test/CodeGen/X86/inc-of-add.ll | 28 +--
llvm/test/CodeGen/X86/masked-sdiv.ll | 114 ++++++-------
llvm/test/CodeGen/X86/masked-srem.ll | 114 ++++++-------
llvm/test/CodeGen/X86/masked-udiv.ll | 114 ++++++-------
llvm/test/CodeGen/X86/masked-urem.ll | 114 ++++++-------
llvm/test/CodeGen/X86/sub-of-not.ll | 47 ++++--
llvm/test/CodeGen/X86/ucmp.ll | 14 +-
llvm/test/CodeGen/X86/vector-bo-select.ll | 65 +++----
llvm/test/CodeGen/X86/vector-shift-lut.ll | 159 +++++++++---------
12 files changed, 481 insertions(+), 441 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b33b01f804c6b..e1393953588d6 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -3819,6 +3819,13 @@ unsigned X86TargetLowering::preferedOpcodeForCmpEqPiecesOfOperand(
return ISD::SRL;
}
+bool X86TargetLowering::preferIncOfAddToSubOfNot(EVT VT) const {
+ // Prefer inc-of-add for scalars and single-element vectors, and sub-of-not
+ // for multi-element vectors.
+ return VT.isScalarInteger() ||
+ (VT.isVector() && VT.getVectorElementCount().isScalar());
+}
+
TargetLoweringBase::CondMergingParams
X86TargetLowering::getJumpConditionMergingParams(Instruction::BinaryOps Opc,
const Value *Lhs,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 5283c377b97c0..cc3b41c1f248e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -272,6 +272,8 @@ namespace llvm {
const APInt &ShiftOrRotateAmt,
const std::optional<APInt> &AndMask) const override;
+ bool preferIncOfAddToSubOfNot(EVT VT) const override;
+
bool preferScalarizeSplat(SDNode *N) const override;
CondMergingParams
diff --git a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
index f9f4cf99f24a3..471b86754c775 100644
--- a/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/expand-vp-int-intrinsics.ll
@@ -129,40 +129,40 @@ define void @vp_sdiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
; SSE-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: paddd %xmm2, %xmm1
-; SSE-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %eax
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pxor %xmm2, %xmm3
+; SSE-NEXT: pand %xmm1, %xmm2
+; SSE-NEXT: psubd %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %eax, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE-NEXT: movd %eax, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE-NEXT: movd %xmm3, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE-NEXT: movd %xmm3, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
; SSE-NEXT: movd %eax, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %eax, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: movd %eax, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
; SSE-NEXT: movd %eax, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT: movdqa %xmm2, (%rdi)
+; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT: movdqa %xmm1, (%rdi)
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_sdiv_v4i32:
@@ -308,40 +308,40 @@ define void @vp_udiv_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
; SSE-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: paddd %xmm2, %xmm1
-; SSE-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %eax
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pxor %xmm2, %xmm3
+; SSE-NEXT: pand %xmm1, %xmm2
+; SSE-NEXT: psubd %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
-; SSE-NEXT: movd %eax, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE-NEXT: movd %eax, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE-NEXT: movd %xmm3, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE-NEXT: movd %xmm3, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
; SSE-NEXT: movd %eax, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
-; SSE-NEXT: movd %eax, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: movd %eax, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
; SSE-NEXT: movd %eax, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT: movdqa %xmm2, (%rdi)
+; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT: movdqa %xmm1, (%rdi)
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_udiv_v4i32:
@@ -487,40 +487,40 @@ define void @vp_srem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
; SSE-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: paddd %xmm2, %xmm1
-; SSE-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %eax
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pxor %xmm2, %xmm3
+; SSE-NEXT: pand %xmm1, %xmm2
+; SSE-NEXT: psubd %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %edx, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE-NEXT: movd %edx, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE-NEXT: movd %xmm3, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE-NEXT: movd %xmm3, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
; SSE-NEXT: movd %edx, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
-; SSE-NEXT: movd %edx, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: movd %edx, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: cltd
; SSE-NEXT: idivl %ecx
; SSE-NEXT: movd %edx, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT: movdqa %xmm2, (%rdi)
+; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT: movdqa %xmm1, (%rdi)
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_srem_v4i32:
@@ -666,40 +666,40 @@ define void @vp_urem_v4i32(<4 x i32> %a0, <4 x i32> %a1, ptr %out, i32 %vp) noun
; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
; SSE-NEXT: pxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
; SSE-NEXT: pcmpgtd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
-; SSE-NEXT: pand %xmm2, %xmm1
-; SSE-NEXT: paddd %xmm2, %xmm1
-; SSE-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE-NEXT: psubd %xmm2, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %ecx
-; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE-NEXT: movd %xmm2, %eax
+; SSE-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE-NEXT: pxor %xmm2, %xmm3
+; SSE-NEXT: pand %xmm1, %xmm2
+; SSE-NEXT: psubd %xmm3, %xmm2
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE-NEXT: movd %xmm1, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
-; SSE-NEXT: movd %edx, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE-NEXT: movd %edx, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE-NEXT: movd %xmm3, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE-NEXT: movd %xmm3, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
; SSE-NEXT: movd %edx, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
-; SSE-NEXT: movd %edx, %xmm2
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE-NEXT: movd %xmm1, %ecx
+; SSE-NEXT: movd %edx, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE-NEXT: movd %xmm2, %ecx
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: xorl %edx, %edx
; SSE-NEXT: divl %ecx
; SSE-NEXT: movd %edx, %xmm0
-; SSE-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE-NEXT: movdqa %xmm2, (%rdi)
+; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE-NEXT: movdqa %xmm1, (%rdi)
; SSE-NEXT: retq
;
; AVX1-LABEL: vp_urem_v4i32:
diff --git a/llvm/test/CodeGen/X86/inc-of-add.ll b/llvm/test/CodeGen/X86/inc-of-add.ll
index a899660031d45..b07efed2b9199 100644
--- a/llvm/test/CodeGen/X86/inc-of-add.ll
+++ b/llvm/test/CodeGen/X86/inc-of-add.ll
@@ -101,9 +101,10 @@ define i64 @scalar_i64(i64 %x, i64 %y) nounwind {
define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; ALL-LABEL: vector_i128_i8:
; ALL: # %bb.0:
-; ALL-NEXT: paddb %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubb %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubb %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
%t1 = add <16 x i8> %y, %t0
@@ -113,9 +114,10 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; ALL-LABEL: vector_i128_i16:
; ALL: # %bb.0:
-; ALL-NEXT: paddw %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubw %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubw %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <8 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
%t1 = add <8 x i16> %y, %t0
@@ -125,9 +127,10 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
; ALL-LABEL: vector_i128_i32:
; ALL: # %bb.0:
-; ALL-NEXT: paddd %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubd %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubd %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1>
%t1 = add <4 x i32> %y, %t0
@@ -137,9 +140,10 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; ALL-LABEL: vector_i128_i64:
; ALL: # %bb.0:
-; ALL-NEXT: paddq %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubq %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubq %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <2 x i64> %x, <i64 1, i64 1>
%t1 = add <2 x i64> %y, %t0
diff --git a/llvm/test/CodeGen/X86/masked-sdiv.ll b/llvm/test/CodeGen/X86/masked-sdiv.ll
index 30d7f116dcb47..747c8ac0867f0 100644
--- a/llvm/test/CodeGen/X86/masked-sdiv.ll
+++ b/llvm/test/CodeGen/X86/masked-sdiv.ll
@@ -10,40 +10,40 @@ define <4 x i32> @sdiv_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
; SSE2: # %bb.0:
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: sdiv_v4i32:
@@ -394,40 +394,40 @@ define <2 x i32> @sdiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: sdiv_v2i32:
@@ -535,30 +535,30 @@ define <4 x i16> @sdiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
; SSE2-NEXT: psllw $15, %xmm2
; SSE2-NEXT: psraw $15, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddw %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubw %xmm2, %xmm1
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubw %xmm3, %xmm2
+; SSE2-NEXT: pextrw $7, %xmm2, %ecx
; SSE2-NEXT: pextrw $7, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pextrw $6, %xmm2, %ecx
; SSE2-NEXT: pextrw $6, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: pextrw $5, %xmm2, %ecx
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; SSE2-NEXT: pextrw $5, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
-; SSE2-NEXT: pextrw $4, %xmm1, %esi
+; SSE2-NEXT: pextrw $4, %xmm2, %esi
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: pextrw $4, %xmm0, %ecx
@@ -567,21 +567,21 @@ define <4 x i16> @sdiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %si
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; SSE2-NEXT: pextrw $3, %xmm2, %ecx
; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; SSE2-NEXT: pextrw $2, %xmm2, %ecx
; SSE2-NEXT: movd %eax, %xmm3
; SSE2-NEXT: pextrw $2, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
-; SSE2-NEXT: pextrw $1, %xmm1, %esi
+; SSE2-NEXT: pextrw $1, %xmm2, %esi
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: pextrw $1, %xmm0, %ecx
@@ -595,13 +595,13 @@ define <4 x i16> @sdiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm0
; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: retq
;
; SSE42-LABEL: sdiv_v4i16:
@@ -1084,9 +1084,9 @@ define <3 x i10> @sdiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-NEXT: pslld $22, %xmm1
; SSE2-NEXT: psrad $22, %xmm1
; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: psubd %xmm3, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/masked-srem.ll b/llvm/test/CodeGen/X86/masked-srem.ll
index 5bf1b49166dac..cf546c53e199c 100644
--- a/llvm/test/CodeGen/X86/masked-srem.ll
+++ b/llvm/test/CodeGen/X86/masked-srem.ll
@@ -10,40 +10,40 @@ define <4 x i32> @srem_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
; SSE2: # %bb.0:
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: srem_v4i32:
@@ -394,40 +394,40 @@ define <2 x i32> @srem_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: cltd
; SSE2-NEXT: idivl %ecx
; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: srem_v2i32:
@@ -535,30 +535,30 @@ define <4 x i16> @srem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
; SSE2-NEXT: psllw $15, %xmm2
; SSE2-NEXT: psraw $15, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddw %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubw %xmm2, %xmm1
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubw %xmm3, %xmm2
+; SSE2-NEXT: pextrw $7, %xmm2, %ecx
; SSE2-NEXT: pextrw $7, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pextrw $6, %xmm2, %ecx
; SSE2-NEXT: pextrw $6, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: pextrw $5, %xmm2, %ecx
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; SSE2-NEXT: pextrw $5, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
-; SSE2-NEXT: pextrw $4, %xmm1, %esi
+; SSE2-NEXT: pextrw $4, %xmm2, %esi
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: pextrw $4, %xmm0, %eax
@@ -567,21 +567,21 @@ define <4 x i16> @srem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %si
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; SSE2-NEXT: pextrw $3, %xmm2, %ecx
; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; SSE2-NEXT: pextrw $2, %xmm2, %ecx
; SSE2-NEXT: movd %edx, %xmm3
; SSE2-NEXT: pextrw $2, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
-; SSE2-NEXT: pextrw $1, %xmm1, %esi
+; SSE2-NEXT: pextrw $1, %xmm2, %esi
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: pextrw $1, %xmm0, %eax
@@ -595,13 +595,13 @@ define <4 x i16> @srem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: cwtd
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: idivw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm0
; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: retq
;
; SSE42-LABEL: srem_v4i16:
@@ -1087,9 +1087,9 @@ define <3 x i10> @srem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-NEXT: pslld $22, %xmm1
; SSE2-NEXT: psrad $22, %xmm1
; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: psubd %xmm3, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/masked-udiv.ll b/llvm/test/CodeGen/X86/masked-udiv.ll
index e39ae9474872c..379ae8077d419 100644
--- a/llvm/test/CodeGen/X86/masked-udiv.ll
+++ b/llvm/test/CodeGen/X86/masked-udiv.ll
@@ -10,40 +10,40 @@ define <4 x i32> @udiv_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
; SSE2: # %bb.0:
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: udiv_v4i32:
@@ -393,40 +393,40 @@ define <2 x i32> @udiv_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: udiv_v2i32:
@@ -534,49 +534,49 @@ define <4 x i16> @udiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
; SSE2-NEXT: psllw $15, %xmm2
; SSE2-NEXT: psraw $15, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddw %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubw %xmm2, %xmm1
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubw %xmm3, %xmm2
+; SSE2-NEXT: pextrw $7, %xmm2, %ecx
; SSE2-NEXT: pextrw $7, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
-; SSE2-NEXT: movd %eax, %xmm2
+; SSE2-NEXT: pextrw $6, %xmm2, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
; SSE2-NEXT: pextrw $6, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
+; SSE2-NEXT: pextrw $5, %xmm2, %ecx
; SSE2-NEXT: pextrw $5, %xmm0, %eax
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm4
-; SSE2-NEXT: pextrw $4, %xmm1, %ecx
+; SSE2-NEXT: pextrw $4, %xmm2, %ecx
; SSE2-NEXT: pextrw $4, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
-; SSE2-NEXT: movd %eax, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; SSE2-NEXT: pextrw $3, %xmm2, %ecx
; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; SSE2-NEXT: pextrw $2, %xmm2, %ecx
; SSE2-NEXT: pextrw $2, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
@@ -584,14 +584,14 @@ define <4 x i16> @udiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm4
; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT: pextrw $1, %xmm1, %ecx
+; SSE2-NEXT: pextrw $1, %xmm2, %ecx
; SSE2-NEXT: pextrw $1, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $ax killed $ax def $eax
; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
@@ -600,7 +600,7 @@ define <4 x i16> @udiv_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: movd %eax, %xmm0
; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: retq
;
; SSE42-LABEL: udiv_v4i16:
@@ -1072,9 +1072,9 @@ define <3 x i10> @udiv_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-NEXT: pand %xmm3, %xmm1
; SSE2-NEXT: pand %xmm3, %xmm0
; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: paddd %xmm2, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm0
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: psubd %xmm3, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/masked-urem.ll b/llvm/test/CodeGen/X86/masked-urem.ll
index 95f4c51016389..7a3a8bd183cd8 100644
--- a/llvm/test/CodeGen/X86/masked-urem.ll
+++ b/llvm/test/CodeGen/X86/masked-urem.ll
@@ -10,40 +10,40 @@ define <4 x i32> @urem_v4i32(<4 x i32> %x, <4 x i32> %y, <4 x i1> %m) {
; SSE2: # %bb.0:
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: urem_v4i32:
@@ -393,40 +393,40 @@ define <2 x i32> @urem_v2i32(<2 x i32> %x, <2 x i32> %y, <2 x i1> %m) {
; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm3[2,3]
; SSE2-NEXT: pslld $31, %xmm2
; SSE2-NEXT: psrad $31, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %ecx
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: movd %xmm2, %eax
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubd %xmm3, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,1,1]
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,1,1]
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divl %ecx
; SSE2-NEXT: movd %edx, %xmm0
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: urem_v2i32:
@@ -534,49 +534,49 @@ define <4 x i16> @urem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: psrldq {{.*#+}} xmm2 = xmm2[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
; SSE2-NEXT: psllw $15, %xmm2
; SSE2-NEXT: psraw $15, %xmm2
-; SSE2-NEXT: pand %xmm2, %xmm1
-; SSE2-NEXT: paddw %xmm2, %xmm1
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubw %xmm2, %xmm1
-; SSE2-NEXT: pextrw $7, %xmm1, %ecx
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: pand %xmm1, %xmm2
+; SSE2-NEXT: psubw %xmm3, %xmm2
+; SSE2-NEXT: pextrw $7, %xmm2, %ecx
; SSE2-NEXT: pextrw $7, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: pextrw $6, %xmm1, %ecx
-; SSE2-NEXT: movd %edx, %xmm2
+; SSE2-NEXT: pextrw $6, %xmm2, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
; SSE2-NEXT: pextrw $6, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: pextrw $5, %xmm1, %ecx
+; SSE2-NEXT: pextrw $5, %xmm2, %ecx
; SSE2-NEXT: pextrw $5, %xmm0, %eax
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm4
-; SSE2-NEXT: pextrw $4, %xmm1, %ecx
+; SSE2-NEXT: pextrw $4, %xmm2, %ecx
; SSE2-NEXT: pextrw $4, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
-; SSE2-NEXT: movd %edx, %xmm2
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; SSE2-NEXT: pextrw $3, %xmm1, %ecx
+; SSE2-NEXT: movd %edx, %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1],xmm1[2],xmm4[2],xmm1[3],xmm4[3]
+; SSE2-NEXT: pextrw $3, %xmm2, %ecx
; SSE2-NEXT: pextrw $3, %xmm0, %eax
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: pextrw $2, %xmm1, %ecx
+; SSE2-NEXT: pextrw $2, %xmm2, %ecx
; SSE2-NEXT: pextrw $2, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
@@ -584,14 +584,14 @@ define <4 x i16> @urem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm4
; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
-; SSE2-NEXT: pextrw $1, %xmm1, %ecx
+; SSE2-NEXT: pextrw $1, %xmm2, %ecx
; SSE2-NEXT: pextrw $1, %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
; SSE2-NEXT: divw %cx
; SSE2-NEXT: # kill: def $dx killed $dx def $edx
; SSE2-NEXT: movd %edx, %xmm3
-; SSE2-NEXT: movd %xmm1, %ecx
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: xorl %edx, %edx
@@ -600,7 +600,7 @@ define <4 x i16> @urem_v4i16(<4 x i16> %x, <4 x i16> %y, <4 x i1> %m) {
; SSE2-NEXT: movd %edx, %xmm0
; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1],xmm0[2],xmm3[2],xmm0[3],xmm3[3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
-; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSE2-NEXT: retq
;
; SSE42-LABEL: urem_v4i16:
@@ -1074,9 +1074,9 @@ define <3 x i10> @urem_v3i10(<3 x i10> %x, <3 x i10> %y, <3 x i1> %m) {
; SSE2-NEXT: pand %xmm3, %xmm1
; SSE2-NEXT: pand %xmm3, %xmm0
; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: paddd %xmm2, %xmm0
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubd %xmm2, %xmm0
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm3
+; SSE2-NEXT: psubd %xmm3, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/sub-of-not.ll b/llvm/test/CodeGen/X86/sub-of-not.ll
index 2b8f6c18ff086..8ad9092ebab1f 100644
--- a/llvm/test/CodeGen/X86/sub-of-not.ll
+++ b/llvm/test/CodeGen/X86/sub-of-not.ll
@@ -101,9 +101,10 @@ define i64 @scalar_i64(i64 %x, i64 %y) nounwind {
define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; ALL-LABEL: vector_i128_i8:
; ALL: # %bb.0:
-; ALL-NEXT: paddb %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubb %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubb %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = xor <16 x i8> %x, <i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1, i8 -1>
%t1 = sub <16 x i8> %y, %t0
@@ -113,9 +114,10 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; ALL-LABEL: vector_i128_i16:
; ALL: # %bb.0:
-; ALL-NEXT: paddw %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubw %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubw %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = xor <8 x i16> %x, <i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1, i16 -1>
%t1 = sub <8 x i16> %y, %t0
@@ -125,9 +127,10 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
; ALL-LABEL: vector_i128_i32:
; ALL: # %bb.0:
-; ALL-NEXT: paddd %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubd %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubd %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = xor <4 x i32> %x, <i32 -1, i32 -1, i32 -1, i32 -1>
%t1 = sub <4 x i32> %y, %t0
@@ -137,11 +140,31 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; ALL-LABEL: vector_i128_i64:
; ALL: # %bb.0:
-; ALL-NEXT: paddq %xmm1, %xmm0
-; ALL-NEXT: pcmpeqd %xmm1, %xmm1
-; ALL-NEXT: psubq %xmm1, %xmm0
+; ALL-NEXT: pcmpeqd %xmm2, %xmm2
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubq %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = xor <2 x i64> %x, <i64 -1, i64 -1>
%t1 = sub <2 x i64> %y, %t0
ret <2 x i64> %t1
}
+
+; From issue #167441
+define <16 x i8> @vector_cond_inc_or_disjoint_v16i8(<16 x i8> %x, <16 x i8> %condition) {
+; ALL-LABEL: vector_cond_inc_or_disjoint_v16i8:
+; ALL: # %bb.0:
+; ALL-NEXT: paddb %xmm0, %xmm0
+; ALL-NEXT: paddb %xmm0, %xmm0
+; ALL-NEXT: pxor %xmm2, %xmm2
+; ALL-NEXT: pcmpeqb %xmm1, %xmm2
+; ALL-NEXT: pcmpeqd %xmm1, %xmm1
+; ALL-NEXT: pxor %xmm2, %xmm1
+; ALL-NEXT: psubb %xmm1, %xmm0
+; ALL-NEXT: ret{{[l|q]}}
+ %shl = shl <16 x i8> %x, splat (i8 2)
+ %cmp = icmp ne <16 x i8> %condition, zeroinitializer
+ %z = zext <16 x i1> %cmp to <16 x i8>
+ %r = or disjoint <16 x i8> %shl, %z
+ ret <16 x i8> %r
+}
diff --git a/llvm/test/CodeGen/X86/ucmp.ll b/llvm/test/CodeGen/X86/ucmp.ll
index 65836384894ef..3aa92d4083d41 100644
--- a/llvm/test/CodeGen/X86/ucmp.ll
+++ b/llvm/test/CodeGen/X86/ucmp.ll
@@ -440,19 +440,25 @@ define <4 x i32> @ucmp_normal_vectors(<4 x i32> %x, <4 x i32> %y) nounwind {
; SSE2-LABEL: ucmp_normal_vectors:
; SSE2: # %bb.0:
; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pmaxud %xmm1, %xmm2
+; SSE2-NEXT: pminud %xmm1, %xmm2
; SSE2-NEXT: pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT: pminud %xmm0, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm3, %xmm2
+; SSE2-NEXT: pmaxud %xmm0, %xmm1
; SSE2-NEXT: pcmpeqd %xmm1, %xmm0
+; SSE2-NEXT: pxor %xmm3, %xmm0
; SSE2-NEXT: psubd %xmm2, %xmm0
; SSE2-NEXT: retq
;
; AVX2-LABEL: ucmp_normal_vectors:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpmaxud %xmm1, %xmm0, %xmm2
+; AVX2-NEXT: vpminud %xmm1, %xmm0, %xmm2
; AVX2-NEXT: vpcmpeqd %xmm2, %xmm0, %xmm2
-; AVX2-NEXT: vpminud %xmm1, %xmm0, %xmm1
+; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpmaxud %xmm1, %xmm0, %xmm1
; AVX2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpxor %xmm3, %xmm0, %xmm0
; AVX2-NEXT: vpsubd %xmm2, %xmm0, %xmm0
; AVX2-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-bo-select.ll b/llvm/test/CodeGen/X86/vector-bo-select.ll
index 0e37e5a2c72c6..ac27c26768c92 100644
--- a/llvm/test/CodeGen/X86/vector-bo-select.ll
+++ b/llvm/test/CodeGen/X86/vector-bo-select.ll
@@ -2720,17 +2720,18 @@ define <4 x i32> @mul_v4i32(<4 x i1> %b, <4 x i32> noundef %x, <4 x i32> noundef
; SSE2: # %bb.0:
; SSE2-NEXT: pslld $31, %xmm0
; SSE2-NEXT: psrad $31, %xmm0
-; SSE2-NEXT: pand %xmm0, %xmm2
-; SSE2-NEXT: paddd %xmm0, %xmm2
-; SSE2-NEXT: pcmpeqd %xmm0, %xmm0
-; SSE2-NEXT: psubd %xmm0, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,2,2,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm3, %xmm1
+; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm0, %xmm3
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: psubd %xmm3, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm0, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE42-LABEL: mul_v4i32:
@@ -2780,21 +2781,21 @@ define <8 x i32> @mul_v8i32_commute(<8 x i1> %b, <8 x i32> noundef %x, <8 x i32>
; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
; SSE2-NEXT: pslld $31, %xmm5
; SSE2-NEXT: psrad $31, %xmm5
+; SSE2-NEXT: pcmpeqd %xmm6, %xmm6
; SSE2-NEXT: pand %xmm5, %xmm4
-; SSE2-NEXT: paddd %xmm5, %xmm4
-; SSE2-NEXT: pcmpeqd %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm6, %xmm5
; SSE2-NEXT: psubd %xmm5, %xmm4
; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
; SSE2-NEXT: pslld $31, %xmm0
; SSE2-NEXT: psrad $31, %xmm0
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: paddd %xmm0, %xmm3
-; SSE2-NEXT: psubd %xmm5, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm1, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,2,2,3]
+; SSE2-NEXT: pxor %xmm0, %xmm6
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: psubd %xmm6, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm5, %xmm1
+; SSE2-NEXT: pmuludq %xmm3, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[1,1,3,3]
@@ -2866,22 +2867,22 @@ define <8 x i32> @mul_v8i32_cast_cond(i8 noundef zeroext %pb, <8 x i32> noundef
; SSE2-NEXT: movdqa %xmm4, %xmm6
; SSE2-NEXT: pand %xmm5, %xmm6
; SSE2-NEXT: pcmpeqd %xmm5, %xmm6
-; SSE2-NEXT: pand %xmm6, %xmm3
-; SSE2-NEXT: paddd %xmm6, %xmm3
; SSE2-NEXT: pcmpeqd %xmm5, %xmm5
-; SSE2-NEXT: psubd %xmm5, %xmm3
+; SSE2-NEXT: pand %xmm6, %xmm3
+; SSE2-NEXT: pxor %xmm5, %xmm6
+; SSE2-NEXT: psubd %xmm6, %xmm3
; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [1,2,4,8]
; SSE2-NEXT: pand %xmm6, %xmm4
; SSE2-NEXT: pcmpeqd %xmm6, %xmm4
-; SSE2-NEXT: pand %xmm4, %xmm2
-; SSE2-NEXT: paddd %xmm4, %xmm2
-; SSE2-NEXT: psubd %xmm5, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm2, %xmm0
+; SSE2-NEXT: pxor %xmm4, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm4
+; SSE2-NEXT: psubd %xmm5, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm4, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm4, %xmm2
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm2, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,3,3]
; SSE2-NEXT: pmuludq %xmm3, %xmm1
@@ -3157,7 +3158,7 @@ define <4 x i32> @shl_v4i32(<4 x i1> %b, <4 x i32> noundef %x, <4 x i32> noundef
; SSE2-NEXT: psrad $31, %xmm0
; SSE2-NEXT: pand %xmm2, %xmm0
; SSE2-NEXT: pslld $23, %xmm0
-; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE2-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [1065353216,1065353216,1065353216,1065353216]
; SSE2-NEXT: cvttps2dq %xmm0, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
; SSE2-NEXT: pmuludq %xmm2, %xmm1
@@ -3174,7 +3175,7 @@ define <4 x i32> @shl_v4i32(<4 x i1> %b, <4 x i32> noundef %x, <4 x i32> noundef
; SSE42-NEXT: psrad $31, %xmm0
; SSE42-NEXT: pand %xmm2, %xmm0
; SSE42-NEXT: pslld $23, %xmm0
-; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE42-NEXT: paddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0 # [1065353216,1065353216,1065353216,1065353216]
; SSE42-NEXT: cvttps2dq %xmm0, %xmm0
; SSE42-NEXT: pmulld %xmm1, %xmm0
; SSE42-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/vector-shift-lut.ll b/llvm/test/CodeGen/X86/vector-shift-lut.ll
index 44c28d67cf677..35a4fd5ed3f54 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lut.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lut.ll
@@ -16,12 +16,11 @@ define <16 x i8> @uniform_shl_v16i8(<16 x i8> %a) nounwind {
; SSE2-NEXT: movdqa %xmm0, %xmm1
; SSE2-NEXT: psllw $5, %xmm1
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pcmpgtb %xmm1, %xmm3
-; SSE2-NEXT: movdqa {{.*#+}} xmm0 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; SSE2-NEXT: pand %xmm3, %xmm0
-; SSE2-NEXT: paddb %xmm3, %xmm0
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pcmpgtb %xmm1, %xmm0
; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm0, %xmm3
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; SSE2-NEXT: psubb %xmm3, %xmm0
; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm3, %xmm3
@@ -1028,58 +1027,57 @@ define <64 x i8> @uniform_ashr_v64i8(<64 x i8> %a) nounwind {
define <32 x i8> @perlane_shl_v32i8(<32 x i8> %a) nounwind {
; SSE2-LABEL: perlane_shl_v32i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: movdqa %xmm1, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm2
; SSE2-NEXT: psllw $5, %xmm2
; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm1
-; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE2-NEXT: pcmpeqd %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm5, %xmm0
; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
-; SSE2-NEXT: por %xmm1, %xmm5
+; SSE2-NEXT: psubb %xmm0, %xmm5
; SSE2-NEXT: paddb %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pcmpgtb %xmm2, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm6
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm6
; SSE2-NEXT: pandn %xmm5, %xmm6
; SSE2-NEXT: psllw $2, %xmm5
; SSE2-NEXT: movdqa {{.*#+}} xmm4 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
-; SSE2-NEXT: pand %xmm4, %xmm1
-; SSE2-NEXT: pand %xmm5, %xmm1
-; SSE2-NEXT: por %xmm6, %xmm1
+; SSE2-NEXT: pand %xmm4, %xmm0
+; SSE2-NEXT: pand %xmm5, %xmm0
+; SSE2-NEXT: por %xmm6, %xmm0
; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
; SSE2-NEXT: movdqa %xmm5, %xmm2
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: paddb %xmm1, %xmm1
-; SSE2-NEXT: pand %xmm5, %xmm1
-; SSE2-NEXT: por %xmm2, %xmm1
-; SSE2-NEXT: psllw $5, %xmm0
-; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pcmpgtb %xmm0, %xmm2
-; SSE2-NEXT: movdqa {{.*#+}} xmm5 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; SSE2-NEXT: pand %xmm2, %xmm5
-; SSE2-NEXT: paddb %xmm2, %xmm5
-; SSE2-NEXT: pcmpeqd %xmm2, %xmm2
-; SSE2-NEXT: psubb %xmm2, %xmm5
+; SSE2-NEXT: pandn %xmm0, %xmm2
; SSE2-NEXT: paddb %xmm0, %xmm0
+; SSE2-NEXT: pand %xmm5, %xmm0
+; SSE2-NEXT: por %xmm2, %xmm0
+; SSE2-NEXT: psllw $5, %xmm1
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pcmpgtb %xmm1, %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; SSE2-NEXT: por %xmm2, %xmm5
+; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pcmpgtb %xmm0, %xmm2
+; SSE2-NEXT: pcmpgtb %xmm1, %xmm2
; SSE2-NEXT: movdqa %xmm2, %xmm6
; SSE2-NEXT: pandn %xmm5, %xmm6
; SSE2-NEXT: psllw $2, %xmm5
; SSE2-NEXT: pand %xmm4, %xmm2
; SSE2-NEXT: pand %xmm5, %xmm2
; SSE2-NEXT: por %xmm6, %xmm2
-; SSE2-NEXT: paddb %xmm0, %xmm0
-; SSE2-NEXT: pcmpgtb %xmm0, %xmm3
-; SSE2-NEXT: movdqa %xmm3, %xmm0
-; SSE2-NEXT: pandn %xmm2, %xmm0
+; SSE2-NEXT: paddb %xmm1, %xmm1
+; SSE2-NEXT: pcmpgtb %xmm1, %xmm3
+; SSE2-NEXT: movdqa %xmm3, %xmm1
+; SSE2-NEXT: pandn %xmm2, %xmm1
; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: pand %xmm3, %xmm2
-; SSE2-NEXT: por %xmm0, %xmm2
-; SSE2-NEXT: movdqa %xmm2, %xmm0
+; SSE2-NEXT: por %xmm1, %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm1
; SSE2-NEXT: retq
;
; SSE41-LABEL: perlane_shl_v32i8:
@@ -1358,7 +1356,7 @@ define <64 x i8> @perlane_shl_v64i8(<64 x i8> %a) nounwind {
; SSE2: # %bb.0:
; SSE2-NEXT: movdqa %xmm0, %xmm1
; SSE2-NEXT: psllw $5, %xmm1
-; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm6, %xmm6
; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: pcmpgtb %xmm1, %xmm0
; SSE2-NEXT: movdqa %xmm0, %xmm4
@@ -1368,12 +1366,12 @@ define <64 x i8> @perlane_shl_v64i8(<64 x i8> %a) nounwind {
; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pcmpgtb %xmm1, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm6
-; SSE2-NEXT: pandn %xmm0, %xmm6
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pandn %xmm0, %xmm5
; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: pand %xmm4, %xmm0
-; SSE2-NEXT: por %xmm6, %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
; SSE2-NEXT: paddb %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pcmpgtb %xmm1, %xmm4
@@ -1382,58 +1380,57 @@ define <64 x i8> @perlane_shl_v64i8(<64 x i8> %a) nounwind {
; SSE2-NEXT: paddb %xmm0, %xmm0
; SSE2-NEXT: pand %xmm4, %xmm0
; SSE2-NEXT: por %xmm1, %xmm0
-; SSE2-NEXT: psllw $5, %xmm3
-; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pcmpgtb %xmm3, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm1
-; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm6
-; SSE2-NEXT: por %xmm1, %xmm6
-; SSE2-NEXT: paddb %xmm3, %xmm3
+; SSE2-NEXT: psllw $5, %xmm2
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
+; SSE2-NEXT: pcmpeqd %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm5, %xmm1
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; SSE2-NEXT: psubb %xmm1, %xmm5
+; SSE2-NEXT: paddb %xmm2, %xmm2
; SSE2-NEXT: pxor %xmm4, %xmm4
-; SSE2-NEXT: pcmpgtb %xmm3, %xmm4
+; SSE2-NEXT: pcmpgtb %xmm2, %xmm4
; SSE2-NEXT: movdqa %xmm4, %xmm7
-; SSE2-NEXT: pandn %xmm6, %xmm7
-; SSE2-NEXT: psllw $2, %xmm6
+; SSE2-NEXT: pandn %xmm5, %xmm7
+; SSE2-NEXT: psllw $2, %xmm5
; SSE2-NEXT: movdqa {{.*#+}} xmm1 = [252,252,252,252,252,252,252,252,252,252,252,252,252,252,252,252]
; SSE2-NEXT: pand %xmm1, %xmm4
-; SSE2-NEXT: pand %xmm6, %xmm4
+; SSE2-NEXT: pand %xmm5, %xmm4
; SSE2-NEXT: por %xmm7, %xmm4
-; SSE2-NEXT: paddb %xmm3, %xmm3
-; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pcmpgtb %xmm3, %xmm6
-; SSE2-NEXT: movdqa %xmm6, %xmm3
-; SSE2-NEXT: pandn %xmm4, %xmm3
-; SSE2-NEXT: paddb %xmm4, %xmm4
-; SSE2-NEXT: pand %xmm6, %xmm4
-; SSE2-NEXT: por %xmm3, %xmm4
-; SSE2-NEXT: psllw $5, %xmm2
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pcmpgtb %xmm2, %xmm3
-; SSE2-NEXT: movdqa {{.*#+}} xmm6 = [16,16,16,16,16,16,16,16,16,16,16,16,16,16,16,16]
-; SSE2-NEXT: pand %xmm3, %xmm6
-; SSE2-NEXT: paddb %xmm3, %xmm6
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT: psubb %xmm3, %xmm6
-; SSE2-NEXT: paddb %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pcmpgtb %xmm2, %xmm3
-; SSE2-NEXT: movdqa %xmm3, %xmm7
-; SSE2-NEXT: pandn %xmm6, %xmm7
-; SSE2-NEXT: psllw $2, %xmm6
-; SSE2-NEXT: pand %xmm1, %xmm3
-; SSE2-NEXT: pand %xmm6, %xmm3
-; SSE2-NEXT: por %xmm7, %xmm3
; SSE2-NEXT: paddb %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: pcmpgtb %xmm2, %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm1
-; SSE2-NEXT: pandn %xmm3, %xmm1
+; SSE2-NEXT: movdqa %xmm5, %xmm2
+; SSE2-NEXT: pandn %xmm4, %xmm2
+; SSE2-NEXT: paddb %xmm4, %xmm4
+; SSE2-NEXT: pand %xmm5, %xmm4
+; SSE2-NEXT: por %xmm2, %xmm4
+; SSE2-NEXT: psllw $5, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pcmpgtb %xmm3, %xmm2
+; SSE2-NEXT: movdqa %xmm2, %xmm5
+; SSE2-NEXT: pandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm5
+; SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm2
+; SSE2-NEXT: por %xmm5, %xmm2
; SSE2-NEXT: paddb %xmm3, %xmm3
-; SSE2-NEXT: pand %xmm5, %xmm3
-; SSE2-NEXT: por %xmm1, %xmm3
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pcmpgtb %xmm3, %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm7
+; SSE2-NEXT: pandn %xmm2, %xmm7
+; SSE2-NEXT: psllw $2, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm5
+; SSE2-NEXT: por %xmm7, %xmm5
+; SSE2-NEXT: paddb %xmm3, %xmm3
+; SSE2-NEXT: pcmpgtb %xmm3, %xmm6
+; SSE2-NEXT: movdqa %xmm6, %xmm1
+; SSE2-NEXT: pandn %xmm5, %xmm1
+; SSE2-NEXT: paddb %xmm5, %xmm5
+; SSE2-NEXT: pand %xmm6, %xmm5
+; SSE2-NEXT: por %xmm1, %xmm5
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm4, %xmm3
+; SSE2-NEXT: movdqa %xmm4, %xmm2
+; SSE2-NEXT: movdqa %xmm5, %xmm3
; SSE2-NEXT: retq
;
; SSE41-LABEL: perlane_shl_v64i8:
>From 89dad1b2b62eaf55ac21d4fcc2efdbbc5560df5f Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <RohanRadhesh.Shenoy at amd.com>
Date: Wed, 2 Sep 2026 01:18:15 +0530
Subject: [PATCH 2/4] patching up some regressions
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 6 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 6 +
llvm/test/CodeGen/AArch64/arm64-vhadd.ll | 8 +-
llvm/test/CodeGen/AArch64/avg.ll | 20 +-
llvm/test/CodeGen/AArch64/inc-of-add.ll | 16 +-
llvm/test/CodeGen/AArch64/sve-hadd.ll | 94 ++++-----
llvm/test/CodeGen/ARM/inc-of-add.ll | 89 ++++-----
llvm/test/CodeGen/PowerPC/inc-of-add.ll | 188 +++++++++---------
llvm/test/CodeGen/PowerPC/vavg.ll | 48 ++---
llvm/test/CodeGen/Thumb2/mve-halving.ll | 48 ++---
llvm/test/CodeGen/Thumb2/mve-vhadd.ll | 20 +-
llvm/test/CodeGen/X86/inc-of-add.ll | 20 +-
llvm/test/CodeGen/X86/ucmp.ll | 14 +-
13 files changed, 286 insertions(+), 291 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 35571c0dea8ce..50691e51a41d5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -3156,14 +3156,14 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) {
// Look for:
// add (add x, y), 1
// And if the target does not like this form then turn into:
- // sub y, (xor x, -1)
+ // sub x, (xor y, -1)
if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD &&
N0.hasOneUse() &&
// Limit this to after legalization if the add has wrap flags
(Level >= AfterLegalizeDAG || (!N->getFlags().hasNoUnsignedWrap() &&
!N->getFlags().hasNoSignedWrap()))) {
- SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT);
- return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not);
+ SDValue Not = DAG.getNOT(DL, N0.getOperand(1), VT);
+ return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Not);
}
}
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index afc253442cac5..dbd796a695e90 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -60600,6 +60600,12 @@ static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(ISD::SUB, DL, VT, ADC.getValue(0), Op1.getOperand(0));
}
+ // Fold SUB(~A,~B) -> SUB(B,A)
+ if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
+ isBitwiseNot(Op1) && Op1.hasOneUse()) {
+ return DAG.getNode(ISD::SUB, DL, VT, Op1.getOperand(0), Op0.getOperand(0));
+ }
+
if (SDValue V = combineXorSubCTLZ(N, DL, DAG, Subtarget))
return V;
diff --git a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
index 9034a39b0ac51..527061b412eae 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
@@ -1330,13 +1330,13 @@ define <2 x i16> @rhadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {
define <2 x i16> @rhadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
; CHECK-SD-LABEL: rhadd8x2_sext_lsr:
; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: shl.2s v0, v0, #24
; CHECK-SD-NEXT: shl.2s v1, v1, #24
+; CHECK-SD-NEXT: shl.2s v0, v0, #24
; CHECK-SD-NEXT: movi d2, #0x00ffff0000ffff
-; CHECK-SD-NEXT: sshr.2s v0, v0, #24
; CHECK-SD-NEXT: sshr.2s v1, v1, #24
-; CHECK-SD-NEXT: mvn.8b v0, v0
-; CHECK-SD-NEXT: sub.2s v0, v1, v0
+; CHECK-SD-NEXT: sshr.2s v0, v0, #24
+; CHECK-SD-NEXT: mvn.8b v1, v1
+; CHECK-SD-NEXT: sub.2s v0, v0, v1
; CHECK-SD-NEXT: and.8b v0, v0, v2
; CHECK-SD-NEXT: ushr.2s v0, v0, #1
; CHECK-SD-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/avg.ll b/llvm/test/CodeGen/AArch64/avg.ll
index fbe8391ae45b5..4c6f2217bfc55 100644
--- a/llvm/test/CodeGen/AArch64/avg.ll
+++ b/llvm/test/CodeGen/AArch64/avg.ll
@@ -234,8 +234,8 @@ define <8 x i16> @add_avgceilu_mismatch1(<8 x i16> %a0, <8 x i16> %a1) {
define <8 x i16> @add_avgceilu_mismatch2(<8 x i16> %a0, <8 x i16> %a1) {
; CHECK-LABEL: add_avgceilu_mismatch2:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v1.16b, v1.16b
-; CHECK-NEXT: sub v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: mvn v0.16b, v0.16b
+; CHECK-NEXT: sub v0.8h, v1.8h, v0.8h
; CHECK-NEXT: ushr v0.8h, v0.8h, #1
; CHECK-NEXT: ret
%add0 = add nuw <8 x i16> %a1, %a0
@@ -247,8 +247,8 @@ define <8 x i16> @add_avgceilu_mismatch2(<8 x i16> %a0, <8 x i16> %a1) {
define <8 x i16> @add_avgceilu_mismatch3(<8 x i16> %a0, <8 x i16> %a1) {
; CHECK-LABEL: add_avgceilu_mismatch3:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v1.16b, v1.16b
-; CHECK-NEXT: sub v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: mvn v0.16b, v0.16b
+; CHECK-NEXT: sub v0.8h, v1.8h, v0.8h
; CHECK-NEXT: ushr v0.8h, v0.8h, #1
; CHECK-NEXT: ret
%add0 = add nuw <8 x i16> %a1, %a0
@@ -355,8 +355,8 @@ define <8 x i16> @add_avgceils_mismatch1(<8 x i16> %a0, <8 x i16> %a1) {
define <8 x i16> @add_avgceils_mismatch2(<8 x i16> %a0, <8 x i16> %a1) {
; CHECK-LABEL: add_avgceils_mismatch2:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v1.16b, v1.16b
-; CHECK-NEXT: sub v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: mvn v0.16b, v0.16b
+; CHECK-NEXT: sub v0.8h, v1.8h, v0.8h
; CHECK-NEXT: sshr v0.8h, v0.8h, #1
; CHECK-NEXT: ret
%add0 = add nsw <8 x i16> %a1, %a0
@@ -368,8 +368,8 @@ define <8 x i16> @add_avgceils_mismatch2(<8 x i16> %a0, <8 x i16> %a1) {
define <8 x i16> @add_avgceils_mismatch3(<8 x i16> %a0, <8 x i16> %a1) {
; CHECK-LABEL: add_avgceils_mismatch3:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v1.16b, v1.16b
-; CHECK-NEXT: sub v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: mvn v0.16b, v0.16b
+; CHECK-NEXT: sub v0.8h, v1.8h, v0.8h
; CHECK-NEXT: sshr v0.8h, v0.8h, #1
; CHECK-NEXT: ret
%add0 = add nsw <8 x i16> %a1, %a0
@@ -381,8 +381,8 @@ define <8 x i16> @add_avgceils_mismatch3(<8 x i16> %a0, <8 x i16> %a1) {
define <8 x i16> @add_avgceils_mismatch4(<8 x i16> %a0, <8 x i16> %a1) {
; CHECK-LABEL: add_avgceils_mismatch4:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v0.16b, v0.16b
-; CHECK-NEXT: sub v0.8h, v1.8h, v0.8h
+; CHECK-NEXT: mvn v1.16b, v1.16b
+; CHECK-NEXT: sub v0.8h, v0.8h, v1.8h
; CHECK-NEXT: sshr v0.8h, v0.8h, #2
; CHECK-NEXT: ret
%add0 = add nsw <8 x i16> %a0, splat(i16 1)
diff --git a/llvm/test/CodeGen/AArch64/inc-of-add.ll b/llvm/test/CodeGen/AArch64/inc-of-add.ll
index b5ce2ab5d409f..dc7c018194b0a 100644
--- a/llvm/test/CodeGen/AArch64/inc-of-add.ll
+++ b/llvm/test/CodeGen/AArch64/inc-of-add.ll
@@ -53,8 +53,8 @@ define i64 @scalar_i64(i64 %x, i64 %y) nounwind {
define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; CHECK-LABEL: vector_i128_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v0.16b, v0.16b
-; CHECK-NEXT: sub v0.16b, v1.16b, v0.16b
+; CHECK-NEXT: mvn v1.16b, v1.16b
+; CHECK-NEXT: sub v0.16b, v0.16b, v1.16b
; CHECK-NEXT: ret
%t0 = add <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
%t1 = add <16 x i8> %y, %t0
@@ -64,8 +64,8 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; CHECK-LABEL: vector_i128_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v0.16b, v0.16b
-; CHECK-NEXT: sub v0.8h, v1.8h, v0.8h
+; CHECK-NEXT: mvn v1.16b, v1.16b
+; CHECK-NEXT: sub v0.8h, v0.8h, v1.8h
; CHECK-NEXT: ret
%t0 = add <8 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
%t1 = add <8 x i16> %y, %t0
@@ -75,8 +75,8 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
; CHECK-LABEL: vector_i128_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v0.16b, v0.16b
-; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s
+; CHECK-NEXT: mvn v1.16b, v1.16b
+; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s
; CHECK-NEXT: ret
%t0 = add <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1>
%t1 = add <4 x i32> %y, %t0
@@ -86,8 +86,8 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; CHECK-LABEL: vector_i128_i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v0.16b, v0.16b
-; CHECK-NEXT: sub v0.2d, v1.2d, v0.2d
+; CHECK-NEXT: mvn v1.16b, v1.16b
+; CHECK-NEXT: sub v0.2d, v0.2d, v1.2d
; CHECK-NEXT: ret
%t0 = add <2 x i64> %x, <i64 1, i64 1>
%t1 = add <2 x i64> %y, %t0
diff --git a/llvm/test/CodeGen/AArch64/sve-hadd.ll b/llvm/test/CodeGen/AArch64/sve-hadd.ll
index 31681d57dd9da..24e4d0482c0d5 100644
--- a/llvm/test/CodeGen/AArch64/sve-hadd.ll
+++ b/llvm/test/CodeGen/AArch64/sve-hadd.ll
@@ -714,10 +714,10 @@ define <vscale x 2 x i32> @rhadds_v2i32(<vscale x 2 x i32> %s0, <vscale x 2 x i3
; SVE-LABEL: rhadds_v2i32:
; SVE: // %bb.0: // %entry
; SVE-NEXT: ptrue p0.d
-; SVE-NEXT: sxtw z0.d, p0/m, z0.d
; SVE-NEXT: sxtw z1.d, p0/m, z1.d
-; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
-; SVE-NEXT: sub z0.d, z1.d, z0.d
+; SVE-NEXT: sxtw z0.d, p0/m, z0.d
+; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
+; SVE-NEXT: sub z0.d, z0.d, z1.d
; SVE-NEXT: asr z0.d, z0.d, #1
; SVE-NEXT: ret
;
@@ -742,10 +742,10 @@ define <vscale x 2 x i32> @rhadds_v2i32_lsh(<vscale x 2 x i32> %s0, <vscale x 2
; CHECK-LABEL: rhadds_v2i32_lsh:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: sxtw z0.d, p0/m, z0.d
; CHECK-NEXT: sxtw z1.d, p0/m, z1.d
-; CHECK-NEXT: subr z0.b, z0.b, #255 // =0xff
-; CHECK-NEXT: sub z0.d, z1.d, z0.d
+; CHECK-NEXT: sxtw z0.d, p0/m, z0.d
+; CHECK-NEXT: subr z1.b, z1.b, #255 // =0xff
+; CHECK-NEXT: sub z0.d, z0.d, z1.d
; CHECK-NEXT: lsr z0.d, z0.d, #1
; CHECK-NEXT: ret
entry:
@@ -761,10 +761,10 @@ entry:
define <vscale x 2 x i32> @rhaddu_v2i32(<vscale x 2 x i32> %s0, <vscale x 2 x i32> %s1) {
; SVE-LABEL: rhaddu_v2i32:
; SVE: // %bb.0: // %entry
-; SVE-NEXT: and z0.d, z0.d, #0xffffffff
; SVE-NEXT: and z1.d, z1.d, #0xffffffff
-; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
-; SVE-NEXT: sub z0.d, z1.d, z0.d
+; SVE-NEXT: and z0.d, z0.d, #0xffffffff
+; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
+; SVE-NEXT: sub z0.d, z0.d, z1.d
; SVE-NEXT: lsr z0.d, z0.d, #1
; SVE-NEXT: ret
;
@@ -861,10 +861,10 @@ define <vscale x 2 x i16> @rhadds_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i1
; SVE-LABEL: rhadds_v2i16:
; SVE: // %bb.0: // %entry
; SVE-NEXT: ptrue p0.d
-; SVE-NEXT: sxth z0.d, p0/m, z0.d
; SVE-NEXT: sxth z1.d, p0/m, z1.d
-; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
-; SVE-NEXT: sub z0.d, z1.d, z0.d
+; SVE-NEXT: sxth z0.d, p0/m, z0.d
+; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
+; SVE-NEXT: sub z0.d, z0.d, z1.d
; SVE-NEXT: asr z0.d, z0.d, #1
; SVE-NEXT: ret
;
@@ -889,10 +889,10 @@ define <vscale x 2 x i16> @rhadds_v2i16_lsh(<vscale x 2 x i16> %s0, <vscale x 2
; CHECK-LABEL: rhadds_v2i16_lsh:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: sxth z0.d, p0/m, z0.d
; CHECK-NEXT: sxth z1.d, p0/m, z1.d
-; CHECK-NEXT: subr z0.b, z0.b, #255 // =0xff
-; CHECK-NEXT: sub z0.d, z1.d, z0.d
+; CHECK-NEXT: sxth z0.d, p0/m, z0.d
+; CHECK-NEXT: subr z1.b, z1.b, #255 // =0xff
+; CHECK-NEXT: sub z0.d, z0.d, z1.d
; CHECK-NEXT: and z0.d, z0.d, #0xffffffff
; CHECK-NEXT: lsr z0.d, z0.d, #1
; CHECK-NEXT: ret
@@ -909,10 +909,10 @@ entry:
define <vscale x 2 x i16> @rhaddu_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {
; SVE-LABEL: rhaddu_v2i16:
; SVE: // %bb.0: // %entry
-; SVE-NEXT: and z0.d, z0.d, #0xffff
; SVE-NEXT: and z1.d, z1.d, #0xffff
-; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
-; SVE-NEXT: sub z0.d, z1.d, z0.d
+; SVE-NEXT: and z0.d, z0.d, #0xffff
+; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
+; SVE-NEXT: sub z0.d, z0.d, z1.d
; SVE-NEXT: lsr z0.d, z0.d, #1
; SVE-NEXT: ret
;
@@ -937,10 +937,10 @@ define <vscale x 4 x i16> @rhadds_v4i16(<vscale x 4 x i16> %s0, <vscale x 4 x i1
; SVE-LABEL: rhadds_v4i16:
; SVE: // %bb.0: // %entry
; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: sxth z0.s, p0/m, z0.s
; SVE-NEXT: sxth z1.s, p0/m, z1.s
-; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
-; SVE-NEXT: sub z0.s, z1.s, z0.s
+; SVE-NEXT: sxth z0.s, p0/m, z0.s
+; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
+; SVE-NEXT: sub z0.s, z0.s, z1.s
; SVE-NEXT: asr z0.s, z0.s, #1
; SVE-NEXT: ret
;
@@ -965,10 +965,10 @@ define <vscale x 4 x i16> @rhadds_v4i16_lsh(<vscale x 4 x i16> %s0, <vscale x 4
; CHECK-LABEL: rhadds_v4i16_lsh:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: sxth z0.s, p0/m, z0.s
; CHECK-NEXT: sxth z1.s, p0/m, z1.s
-; CHECK-NEXT: subr z0.b, z0.b, #255 // =0xff
-; CHECK-NEXT: sub z0.s, z1.s, z0.s
+; CHECK-NEXT: sxth z0.s, p0/m, z0.s
+; CHECK-NEXT: subr z1.b, z1.b, #255 // =0xff
+; CHECK-NEXT: sub z0.s, z0.s, z1.s
; CHECK-NEXT: lsr z0.s, z0.s, #1
; CHECK-NEXT: ret
entry:
@@ -984,10 +984,10 @@ entry:
define <vscale x 4 x i16> @rhaddu_v4i16(<vscale x 4 x i16> %s0, <vscale x 4 x i16> %s1) {
; SVE-LABEL: rhaddu_v4i16:
; SVE: // %bb.0: // %entry
-; SVE-NEXT: and z0.s, z0.s, #0xffff
; SVE-NEXT: and z1.s, z1.s, #0xffff
-; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
-; SVE-NEXT: sub z0.s, z1.s, z0.s
+; SVE-NEXT: and z0.s, z0.s, #0xffff
+; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
+; SVE-NEXT: sub z0.s, z0.s, z1.s
; SVE-NEXT: lsr z0.s, z0.s, #1
; SVE-NEXT: ret
;
@@ -1084,10 +1084,10 @@ define <vscale x 4 x i8> @rhadds_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %
; SVE-LABEL: rhadds_v4i8:
; SVE: // %bb.0: // %entry
; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: sxtb z0.s, p0/m, z0.s
; SVE-NEXT: sxtb z1.s, p0/m, z1.s
-; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
-; SVE-NEXT: sub z0.s, z1.s, z0.s
+; SVE-NEXT: sxtb z0.s, p0/m, z0.s
+; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
+; SVE-NEXT: sub z0.s, z0.s, z1.s
; SVE-NEXT: asr z0.s, z0.s, #1
; SVE-NEXT: ret
;
@@ -1112,10 +1112,10 @@ define <vscale x 4 x i8> @rhadds_v4i8_lsh(<vscale x 4 x i8> %s0, <vscale x 4 x i
; CHECK-LABEL: rhadds_v4i8_lsh:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: sxtb z0.s, p0/m, z0.s
; CHECK-NEXT: sxtb z1.s, p0/m, z1.s
-; CHECK-NEXT: subr z0.b, z0.b, #255 // =0xff
-; CHECK-NEXT: sub z0.s, z1.s, z0.s
+; CHECK-NEXT: sxtb z0.s, p0/m, z0.s
+; CHECK-NEXT: subr z1.b, z1.b, #255 // =0xff
+; CHECK-NEXT: sub z0.s, z0.s, z1.s
; CHECK-NEXT: and z0.s, z0.s, #0xffff
; CHECK-NEXT: lsr z0.s, z0.s, #1
; CHECK-NEXT: ret
@@ -1132,10 +1132,10 @@ entry:
define <vscale x 4 x i8> @rhaddu_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {
; SVE-LABEL: rhaddu_v4i8:
; SVE: // %bb.0: // %entry
-; SVE-NEXT: and z0.s, z0.s, #0xff
; SVE-NEXT: and z1.s, z1.s, #0xff
-; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
-; SVE-NEXT: sub z0.s, z1.s, z0.s
+; SVE-NEXT: and z0.s, z0.s, #0xff
+; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
+; SVE-NEXT: sub z0.s, z0.s, z1.s
; SVE-NEXT: lsr z0.s, z0.s, #1
; SVE-NEXT: ret
;
@@ -1160,10 +1160,10 @@ define <vscale x 8 x i8> @rhadds_v8i8(<vscale x 8 x i8> %s0, <vscale x 8 x i8> %
; SVE-LABEL: rhadds_v8i8:
; SVE: // %bb.0: // %entry
; SVE-NEXT: ptrue p0.h
-; SVE-NEXT: sxtb z0.h, p0/m, z0.h
; SVE-NEXT: sxtb z1.h, p0/m, z1.h
-; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
-; SVE-NEXT: sub z0.h, z1.h, z0.h
+; SVE-NEXT: sxtb z0.h, p0/m, z0.h
+; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
+; SVE-NEXT: sub z0.h, z0.h, z1.h
; SVE-NEXT: asr z0.h, z0.h, #1
; SVE-NEXT: ret
;
@@ -1188,10 +1188,10 @@ define <vscale x 8 x i8> @rhadds_v8i8_lsh(<vscale x 8 x i8> %s0, <vscale x 8 x i
; CHECK-LABEL: rhadds_v8i8_lsh:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ptrue p0.h
-; CHECK-NEXT: sxtb z0.h, p0/m, z0.h
; CHECK-NEXT: sxtb z1.h, p0/m, z1.h
-; CHECK-NEXT: subr z0.b, z0.b, #255 // =0xff
-; CHECK-NEXT: sub z0.h, z1.h, z0.h
+; CHECK-NEXT: sxtb z0.h, p0/m, z0.h
+; CHECK-NEXT: subr z1.b, z1.b, #255 // =0xff
+; CHECK-NEXT: sub z0.h, z0.h, z1.h
; CHECK-NEXT: lsr z0.h, z0.h, #1
; CHECK-NEXT: ret
entry:
@@ -1207,10 +1207,10 @@ entry:
define <vscale x 8 x i8> @rhaddu_v8i8(<vscale x 8 x i8> %s0, <vscale x 8 x i8> %s1) {
; SVE-LABEL: rhaddu_v8i8:
; SVE: // %bb.0: // %entry
-; SVE-NEXT: and z0.h, z0.h, #0xff
; SVE-NEXT: and z1.h, z1.h, #0xff
-; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
-; SVE-NEXT: sub z0.h, z1.h, z0.h
+; SVE-NEXT: and z0.h, z0.h, #0xff
+; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
+; SVE-NEXT: sub z0.h, z0.h, z1.h
; SVE-NEXT: lsr z0.h, z0.h, #1
; SVE-NEXT: ret
;
@@ -1375,8 +1375,8 @@ define void @zext_mload_avgflooru(ptr %p1, ptr %p2, <vscale x 8 x i1> %mask) {
define void @zext_mload_avgceilu(ptr %p1, ptr %p2, <vscale x 8 x i1> %mask) {
; SVE-LABEL: zext_mload_avgceilu:
; SVE: // %bb.0:
-; SVE-NEXT: ld1b { z0.h }, p0/z, [x0]
-; SVE-NEXT: ld1b { z1.h }, p0/z, [x1]
+; SVE-NEXT: ld1b { z0.h }, p0/z, [x1]
+; SVE-NEXT: ld1b { z1.h }, p0/z, [x0]
; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
; SVE-NEXT: sub z0.h, z1.h, z0.h
; SVE-NEXT: lsr z0.h, z0.h, #1
diff --git a/llvm/test/CodeGen/ARM/inc-of-add.ll b/llvm/test/CodeGen/ARM/inc-of-add.ll
index 3079dbceb9844..4a1400371934c 100644
--- a/llvm/test/CodeGen/ARM/inc-of-add.ll
+++ b/llvm/test/CodeGen/ARM/inc-of-add.ll
@@ -91,11 +91,10 @@ define i64 @scalar_i64(i64 %x, i64 %y) nounwind {
;
; THUMB6-LABEL: scalar_i64:
; THUMB6: @ %bb.0:
-; THUMB6-NEXT: mvns r1, r1
-; THUMB6-NEXT: mvns r0, r0
-; THUMB6-NEXT: subs r0, r2, r0
-; THUMB6-NEXT: sbcs r3, r1
-; THUMB6-NEXT: mov r1, r3
+; THUMB6-NEXT: mvns r3, r3
+; THUMB6-NEXT: mvns r2, r2
+; THUMB6-NEXT: subs r0, r0, r2
+; THUMB6-NEXT: sbcs r1, r3
; THUMB6-NEXT: bx lr
;
; THUMB78-LABEL: scalar_i64:
@@ -195,11 +194,11 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
;
; ARM78-LABEL: vector_i128_i8:
; ARM78: @ %bb.0:
-; ARM78-NEXT: vmov d17, r2, r3
-; ARM78-NEXT: vmov d16, r0, r1
-; ARM78-NEXT: mov r0, sp
+; ARM78-NEXT: mov r12, sp
+; ARM78-NEXT: vld1.64 {d16, d17}, [r12]
; ARM78-NEXT: vmvn q8, q8
-; ARM78-NEXT: vld1.64 {d18, d19}, [r0]
+; ARM78-NEXT: vmov d19, r2, r3
+; ARM78-NEXT: vmov d18, r0, r1
; ARM78-NEXT: vsub.i8 q8, q9, q8
; ARM78-NEXT: vmov r0, r1, d16
; ARM78-NEXT: vmov r2, r3, d17
@@ -290,11 +289,11 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
;
; THUMB78-LABEL: vector_i128_i8:
; THUMB78: @ %bb.0:
-; THUMB78-NEXT: vmov d17, r2, r3
-; THUMB78-NEXT: vmov d16, r0, r1
-; THUMB78-NEXT: mov r0, sp
+; THUMB78-NEXT: mov r12, sp
+; THUMB78-NEXT: vld1.64 {d16, d17}, [r12]
; THUMB78-NEXT: vmvn q8, q8
-; THUMB78-NEXT: vld1.64 {d18, d19}, [r0]
+; THUMB78-NEXT: vmov d19, r2, r3
+; THUMB78-NEXT: vmov d18, r0, r1
; THUMB78-NEXT: vsub.i8 q8, q9, q8
; THUMB78-NEXT: vmov r0, r1, d16
; THUMB78-NEXT: vmov r2, r3, d17
@@ -349,11 +348,11 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
;
; ARM78-LABEL: vector_i128_i16:
; ARM78: @ %bb.0:
-; ARM78-NEXT: vmov d17, r2, r3
-; ARM78-NEXT: vmov d16, r0, r1
-; ARM78-NEXT: mov r0, sp
+; ARM78-NEXT: mov r12, sp
+; ARM78-NEXT: vld1.64 {d16, d17}, [r12]
; ARM78-NEXT: vmvn q8, q8
-; ARM78-NEXT: vld1.64 {d18, d19}, [r0]
+; ARM78-NEXT: vmov d19, r2, r3
+; ARM78-NEXT: vmov d18, r0, r1
; ARM78-NEXT: vsub.i16 q8, q9, q8
; ARM78-NEXT: vmov r0, r1, d16
; ARM78-NEXT: vmov r2, r3, d17
@@ -404,11 +403,11 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
;
; THUMB78-LABEL: vector_i128_i16:
; THUMB78: @ %bb.0:
-; THUMB78-NEXT: vmov d17, r2, r3
-; THUMB78-NEXT: vmov d16, r0, r1
-; THUMB78-NEXT: mov r0, sp
+; THUMB78-NEXT: mov r12, sp
+; THUMB78-NEXT: vld1.64 {d16, d17}, [r12]
; THUMB78-NEXT: vmvn q8, q8
-; THUMB78-NEXT: vld1.64 {d18, d19}, [r0]
+; THUMB78-NEXT: vmov d19, r2, r3
+; THUMB78-NEXT: vmov d18, r0, r1
; THUMB78-NEXT: vsub.i16 q8, q9, q8
; THUMB78-NEXT: vmov r0, r1, d16
; THUMB78-NEXT: vmov r2, r3, d17
@@ -437,11 +436,11 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
;
; ARM78-LABEL: vector_i128_i32:
; ARM78: @ %bb.0:
-; ARM78-NEXT: vmov d17, r2, r3
-; ARM78-NEXT: vmov d16, r0, r1
-; ARM78-NEXT: mov r0, sp
+; ARM78-NEXT: mov r12, sp
+; ARM78-NEXT: vld1.64 {d16, d17}, [r12]
; ARM78-NEXT: vmvn q8, q8
-; ARM78-NEXT: vld1.64 {d18, d19}, [r0]
+; ARM78-NEXT: vmov d19, r2, r3
+; ARM78-NEXT: vmov d18, r0, r1
; ARM78-NEXT: vsub.i32 q8, q9, q8
; ARM78-NEXT: vmov r0, r1, d16
; ARM78-NEXT: vmov r2, r3, d17
@@ -466,11 +465,11 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
;
; THUMB78-LABEL: vector_i128_i32:
; THUMB78: @ %bb.0:
-; THUMB78-NEXT: vmov d17, r2, r3
-; THUMB78-NEXT: vmov d16, r0, r1
-; THUMB78-NEXT: mov r0, sp
+; THUMB78-NEXT: mov r12, sp
+; THUMB78-NEXT: vld1.64 {d16, d17}, [r12]
; THUMB78-NEXT: vmvn q8, q8
-; THUMB78-NEXT: vld1.64 {d18, d19}, [r0]
+; THUMB78-NEXT: vmov d19, r2, r3
+; THUMB78-NEXT: vmov d18, r0, r1
; THUMB78-NEXT: vsub.i32 q8, q9, q8
; THUMB78-NEXT: vmov r0, r1, d16
; THUMB78-NEXT: vmov r2, r3, d17
@@ -500,11 +499,11 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
;
; ARM78-LABEL: vector_i128_i64:
; ARM78: @ %bb.0:
-; ARM78-NEXT: vmov d17, r2, r3
-; ARM78-NEXT: vmov d16, r0, r1
-; ARM78-NEXT: mov r0, sp
+; ARM78-NEXT: mov r12, sp
+; ARM78-NEXT: vld1.64 {d16, d17}, [r12]
; ARM78-NEXT: vmvn q8, q8
-; ARM78-NEXT: vld1.64 {d18, d19}, [r0]
+; ARM78-NEXT: vmov d19, r2, r3
+; ARM78-NEXT: vmov d18, r0, r1
; ARM78-NEXT: vsub.i64 q8, q9, q8
; ARM78-NEXT: vmov r0, r1, d16
; ARM78-NEXT: vmov r2, r3, d17
@@ -513,27 +512,27 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; THUMB6-LABEL: vector_i128_i64:
; THUMB6: @ %bb.0:
; THUMB6-NEXT: push {r4, r5, r7, lr}
-; THUMB6-NEXT: mvns r4, r1
-; THUMB6-NEXT: mvns r0, r0
-; THUMB6-NEXT: ldr r1, [sp, #20]
+; THUMB6-NEXT: ldr r4, [sp, #20]
+; THUMB6-NEXT: mvns r4, r4
; THUMB6-NEXT: ldr r5, [sp, #16]
-; THUMB6-NEXT: subs r0, r5, r0
+; THUMB6-NEXT: mvns r5, r5
+; THUMB6-NEXT: subs r0, r0, r5
; THUMB6-NEXT: sbcs r1, r4
-; THUMB6-NEXT: mvns r4, r3
-; THUMB6-NEXT: mvns r2, r2
-; THUMB6-NEXT: ldr r3, [sp, #28]
+; THUMB6-NEXT: ldr r4, [sp, #28]
+; THUMB6-NEXT: mvns r4, r4
; THUMB6-NEXT: ldr r5, [sp, #24]
-; THUMB6-NEXT: subs r2, r5, r2
+; THUMB6-NEXT: mvns r5, r5
+; THUMB6-NEXT: subs r2, r2, r5
; THUMB6-NEXT: sbcs r3, r4
; THUMB6-NEXT: pop {r4, r5, r7, pc}
;
; THUMB78-LABEL: vector_i128_i64:
; THUMB78: @ %bb.0:
-; THUMB78-NEXT: vmov d17, r2, r3
-; THUMB78-NEXT: vmov d16, r0, r1
-; THUMB78-NEXT: mov r0, sp
+; THUMB78-NEXT: mov r12, sp
+; THUMB78-NEXT: vld1.64 {d16, d17}, [r12]
; THUMB78-NEXT: vmvn q8, q8
-; THUMB78-NEXT: vld1.64 {d18, d19}, [r0]
+; THUMB78-NEXT: vmov d19, r2, r3
+; THUMB78-NEXT: vmov d18, r0, r1
; THUMB78-NEXT: vsub.i64 q8, q9, q8
; THUMB78-NEXT: vmov r0, r1, d16
; THUMB78-NEXT: vmov r2, r3, d17
diff --git a/llvm/test/CodeGen/PowerPC/inc-of-add.ll b/llvm/test/CodeGen/PowerPC/inc-of-add.ll
index 432b5a6b362fe..20702fb08dac6 100644
--- a/llvm/test/CodeGen/PowerPC/inc-of-add.ll
+++ b/llvm/test/CodeGen/PowerPC/inc-of-add.ll
@@ -70,59 +70,59 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; PPC32-NEXT: lbz 4, 115(1)
; PPC32-NEXT: lbz 22, 119(1)
; PPC32-NEXT: lbz 21, 123(1)
-; PPC32-NEXT: add 4, 4, 5
-; PPC32-NEXT: add 5, 22, 6
+; PPC32-NEXT: add 4, 5, 4
+; PPC32-NEXT: add 5, 6, 22
; PPC32-NEXT: lbz 22, 131(1)
-; PPC32-NEXT: add 6, 21, 7
+; PPC32-NEXT: add 6, 7, 21
; PPC32-NEXT: lbz 21, 135(1)
; PPC32-NEXT: addi 6, 6, 1
; PPC32-NEXT: stw 20, 16(1) # 4-byte Folded Spill
-; PPC32-NEXT: add 9, 22, 9
+; PPC32-NEXT: add 9, 9, 22
; PPC32-NEXT: lbz 20, 127(1)
-; PPC32-NEXT: add 10, 21, 10
+; PPC32-NEXT: add 10, 10, 21
; PPC32-NEXT: stw 25, 36(1) # 4-byte Folded Spill
; PPC32-NEXT: addi 5, 5, 1
-; PPC32-NEXT: lbz 25, 83(1)
-; PPC32-NEXT: add 7, 20, 8
-; PPC32-NEXT: lbz 21, 147(1)
+; PPC32-NEXT: lbz 25, 147(1)
+; PPC32-NEXT: add 7, 8, 20
+; PPC32-NEXT: lbz 21, 83(1)
; PPC32-NEXT: addi 7, 7, 1
; PPC32-NEXT: stw 24, 32(1) # 4-byte Folded Spill
; PPC32-NEXT: addi 4, 4, 1
-; PPC32-NEXT: lbz 24, 79(1)
+; PPC32-NEXT: lbz 24, 143(1)
; PPC32-NEXT: add 25, 21, 25
-; PPC32-NEXT: lbz 22, 143(1)
+; PPC32-NEXT: lbz 22, 79(1)
; PPC32-NEXT: stw 23, 28(1) # 4-byte Folded Spill
-; PPC32-NEXT: lbz 23, 75(1)
+; PPC32-NEXT: lbz 23, 139(1)
; PPC32-NEXT: add 24, 22, 24
-; PPC32-NEXT: lbz 8, 139(1)
+; PPC32-NEXT: lbz 8, 75(1)
; PPC32-NEXT: stw 28, 48(1) # 4-byte Folded Spill
-; PPC32-NEXT: lbz 28, 95(1)
+; PPC32-NEXT: lbz 28, 159(1)
; PPC32-NEXT: add 8, 8, 23
-; PPC32-NEXT: lbz 21, 159(1)
+; PPC32-NEXT: lbz 21, 95(1)
; PPC32-NEXT: addi 8, 8, 1
; PPC32-NEXT: stw 27, 44(1) # 4-byte Folded Spill
-; PPC32-NEXT: lbz 27, 91(1)
+; PPC32-NEXT: lbz 27, 155(1)
; PPC32-NEXT: add 28, 21, 28
-; PPC32-NEXT: lbz 22, 155(1)
+; PPC32-NEXT: lbz 22, 91(1)
; PPC32-NEXT: stw 26, 40(1) # 4-byte Folded Spill
-; PPC32-NEXT: lbz 26, 87(1)
+; PPC32-NEXT: lbz 26, 151(1)
; PPC32-NEXT: add 27, 22, 27
-; PPC32-NEXT: lbz 23, 151(1)
-; PPC32-NEXT: lbz 11, 111(1)
-; PPC32-NEXT: lbz 21, 175(1)
+; PPC32-NEXT: lbz 23, 87(1)
+; PPC32-NEXT: lbz 11, 175(1)
+; PPC32-NEXT: lbz 21, 111(1)
; PPC32-NEXT: add 26, 23, 26
-; PPC32-NEXT: lbz 12, 107(1)
-; PPC32-NEXT: lbz 0, 171(1)
+; PPC32-NEXT: lbz 12, 171(1)
+; PPC32-NEXT: lbz 0, 107(1)
; PPC32-NEXT: add 11, 21, 11
; PPC32-NEXT: stw 30, 56(1) # 4-byte Folded Spill
; PPC32-NEXT: addi 11, 11, 1
-; PPC32-NEXT: lbz 30, 103(1)
+; PPC32-NEXT: lbz 30, 167(1)
; PPC32-NEXT: add 12, 0, 12
-; PPC32-NEXT: lbz 22, 167(1)
+; PPC32-NEXT: lbz 22, 103(1)
; PPC32-NEXT: stw 29, 52(1) # 4-byte Folded Spill
-; PPC32-NEXT: lbz 29, 99(1)
+; PPC32-NEXT: lbz 29, 163(1)
; PPC32-NEXT: add 30, 22, 30
-; PPC32-NEXT: lbz 23, 163(1)
+; PPC32-NEXT: lbz 23, 99(1)
; PPC32-NEXT: stb 11, 15(3)
; PPC32-NEXT: addi 11, 12, 1
; PPC32-NEXT: add 29, 23, 29
@@ -179,50 +179,50 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; AIX-PPC64-NEXT: std 31, -8(1) # 8-byte Folded Spill
; AIX-PPC64-NEXT: lbz 24, 199(1)
; AIX-PPC64-NEXT: lbz 25, 191(1)
-; AIX-PPC64-NEXT: add 6, 23, 6
+; AIX-PPC64-NEXT: add 6, 6, 23
; AIX-PPC64-NEXT: lbz 23, 231(1)
-; AIX-PPC64-NEXT: add 5, 24, 5
+; AIX-PPC64-NEXT: add 5, 5, 24
; AIX-PPC64-NEXT: lbz 24, 223(1)
-; AIX-PPC64-NEXT: add 4, 25, 4
+; AIX-PPC64-NEXT: add 4, 4, 25
; AIX-PPC64-NEXT: lbz 25, 215(1)
-; AIX-PPC64-NEXT: add 9, 23, 9
-; AIX-PPC64-NEXT: lbz 27, 127(1)
-; AIX-PPC64-NEXT: add 8, 24, 8
-; AIX-PPC64-NEXT: lbz 23, 255(1)
-; AIX-PPC64-NEXT: add 7, 25, 7
-; AIX-PPC64-NEXT: lbz 26, 119(1)
+; AIX-PPC64-NEXT: add 9, 9, 23
+; AIX-PPC64-NEXT: lbz 27, 255(1)
+; AIX-PPC64-NEXT: add 8, 8, 24
+; AIX-PPC64-NEXT: lbz 23, 127(1)
+; AIX-PPC64-NEXT: add 7, 7, 25
+; AIX-PPC64-NEXT: lbz 26, 247(1)
; AIX-PPC64-NEXT: addi 9, 9, 1
-; AIX-PPC64-NEXT: lbz 24, 247(1)
+; AIX-PPC64-NEXT: lbz 24, 119(1)
; AIX-PPC64-NEXT: add 27, 23, 27
; AIX-PPC64-NEXT: lbz 25, 239(1)
; AIX-PPC64-NEXT: addi 8, 8, 1
-; AIX-PPC64-NEXT: lbz 30, 151(1)
+; AIX-PPC64-NEXT: lbz 30, 279(1)
; AIX-PPC64-NEXT: add 26, 24, 26
-; AIX-PPC64-NEXT: lbz 23, 279(1)
-; AIX-PPC64-NEXT: add 10, 25, 10
-; AIX-PPC64-NEXT: lbz 29, 143(1)
+; AIX-PPC64-NEXT: lbz 23, 151(1)
+; AIX-PPC64-NEXT: add 10, 10, 25
+; AIX-PPC64-NEXT: lbz 29, 271(1)
; AIX-PPC64-NEXT: addi 10, 10, 1
-; AIX-PPC64-NEXT: lbz 24, 271(1)
+; AIX-PPC64-NEXT: lbz 24, 143(1)
; AIX-PPC64-NEXT: add 30, 23, 30
-; AIX-PPC64-NEXT: lbz 28, 135(1)
+; AIX-PPC64-NEXT: lbz 28, 263(1)
; AIX-PPC64-NEXT: addi 7, 7, 1
-; AIX-PPC64-NEXT: lbz 25, 263(1)
+; AIX-PPC64-NEXT: lbz 25, 135(1)
; AIX-PPC64-NEXT: add 29, 24, 29
-; AIX-PPC64-NEXT: lbz 11, 183(1)
+; AIX-PPC64-NEXT: lbz 11, 311(1)
; AIX-PPC64-NEXT: addi 6, 6, 1
-; AIX-PPC64-NEXT: lbz 23, 311(1)
+; AIX-PPC64-NEXT: lbz 23, 183(1)
; AIX-PPC64-NEXT: add 28, 25, 28
-; AIX-PPC64-NEXT: lbz 12, 175(1)
+; AIX-PPC64-NEXT: lbz 12, 303(1)
; AIX-PPC64-NEXT: addi 5, 5, 1
-; AIX-PPC64-NEXT: lbz 0, 303(1)
+; AIX-PPC64-NEXT: lbz 0, 175(1)
; AIX-PPC64-NEXT: add 11, 23, 11
-; AIX-PPC64-NEXT: lbz 2, 167(1)
+; AIX-PPC64-NEXT: lbz 2, 295(1)
; AIX-PPC64-NEXT: addi 11, 11, 1
-; AIX-PPC64-NEXT: lbz 24, 295(1)
+; AIX-PPC64-NEXT: lbz 24, 167(1)
; AIX-PPC64-NEXT: add 12, 0, 12
-; AIX-PPC64-NEXT: lbz 31, 159(1)
+; AIX-PPC64-NEXT: lbz 31, 287(1)
; AIX-PPC64-NEXT: addi 4, 4, 1
-; AIX-PPC64-NEXT: lbz 25, 287(1)
+; AIX-PPC64-NEXT: lbz 25, 159(1)
; AIX-PPC64-NEXT: add 2, 24, 2
; AIX-PPC64-NEXT: stb 11, 15(3)
; AIX-PPC64-NEXT: addi 11, 12, 1
@@ -263,8 +263,8 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
;
; PPC64LE-LABEL: vector_i128_i8:
; PPC64LE: # %bb.0:
-; PPC64LE-NEXT: xxlnor 34, 34, 34
-; PPC64LE-NEXT: vsububm 2, 3, 2
+; PPC64LE-NEXT: xxlnor 35, 35, 35
+; PPC64LE-NEXT: vsububm 2, 2, 3
; PPC64LE-NEXT: blr
%t0 = add <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
%t1 = add <16 x i8> %y, %t0
@@ -283,20 +283,20 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; PPC32-NEXT: lhz 11, 70(1)
; PPC32-NEXT: lhz 12, 66(1)
; PPC32-NEXT: lhz 0, 62(1)
-; PPC32-NEXT: add 10, 11, 10
+; PPC32-NEXT: add 10, 10, 11
; PPC32-NEXT: lhz 30, 58(1)
-; PPC32-NEXT: add 9, 12, 9
+; PPC32-NEXT: add 9, 9, 12
; PPC32-NEXT: lhz 29, 50(1)
-; PPC32-NEXT: add 8, 0, 8
+; PPC32-NEXT: add 8, 8, 0
; PPC32-NEXT: lhz 28, 42(1)
-; PPC32-NEXT: add 7, 30, 7
+; PPC32-NEXT: add 7, 7, 30
; PPC32-NEXT: lhz 27, 46(1)
-; PPC32-NEXT: add 5, 29, 5
+; PPC32-NEXT: add 5, 5, 29
; PPC32-NEXT: lhz 26, 54(1)
-; PPC32-NEXT: add 3, 28, 3
-; PPC32-NEXT: add 4, 27, 4
+; PPC32-NEXT: add 3, 3, 28
+; PPC32-NEXT: add 4, 4, 27
; PPC32-NEXT: addi 3, 3, 1
-; PPC32-NEXT: add 6, 26, 6
+; PPC32-NEXT: add 6, 6, 26
; PPC32-NEXT: addi 4, 4, 1
; PPC32-NEXT: addi 5, 5, 1
; PPC32-NEXT: addi 6, 6, 1
@@ -320,24 +320,24 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; AIX-PPC64-NEXT: std 30, -16(1) # 8-byte Folded Spill
; AIX-PPC64-NEXT: std 31, -8(1) # 8-byte Folded Spill
; AIX-PPC64-NEXT: std 2, -48(1) # 8-byte Folded Spill
-; AIX-PPC64-NEXT: lhz 11, 118(1)
-; AIX-PPC64-NEXT: lhz 12, 182(1)
+; AIX-PPC64-NEXT: lhz 11, 182(1)
+; AIX-PPC64-NEXT: lhz 12, 118(1)
; AIX-PPC64-NEXT: lhz 0, 174(1)
; AIX-PPC64-NEXT: lhz 2, 166(1)
; AIX-PPC64-NEXT: add 11, 12, 11
; AIX-PPC64-NEXT: lhz 31, 158(1)
-; AIX-PPC64-NEXT: add 10, 0, 10
+; AIX-PPC64-NEXT: add 10, 10, 0
; AIX-PPC64-NEXT: lhz 30, 142(1)
-; AIX-PPC64-NEXT: add 9, 2, 9
+; AIX-PPC64-NEXT: add 9, 9, 2
; AIX-PPC64-NEXT: lhz 29, 126(1)
-; AIX-PPC64-NEXT: add 8, 31, 8
+; AIX-PPC64-NEXT: add 8, 8, 31
; AIX-PPC64-NEXT: lhz 28, 134(1)
-; AIX-PPC64-NEXT: add 6, 30, 6
+; AIX-PPC64-NEXT: add 6, 6, 30
; AIX-PPC64-NEXT: lhz 27, 150(1)
-; AIX-PPC64-NEXT: add 4, 29, 4
-; AIX-PPC64-NEXT: add 5, 28, 5
+; AIX-PPC64-NEXT: add 4, 4, 29
+; AIX-PPC64-NEXT: add 5, 5, 28
; AIX-PPC64-NEXT: addi 11, 11, 1
-; AIX-PPC64-NEXT: add 7, 27, 7
+; AIX-PPC64-NEXT: add 7, 7, 27
; AIX-PPC64-NEXT: addi 10, 10, 1
; AIX-PPC64-NEXT: addi 9, 9, 1
; AIX-PPC64-NEXT: addi 8, 8, 1
@@ -363,8 +363,8 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
;
; PPC64LE-LABEL: vector_i128_i16:
; PPC64LE: # %bb.0:
-; PPC64LE-NEXT: xxlnor 34, 34, 34
-; PPC64LE-NEXT: vsubuhm 2, 3, 2
+; PPC64LE-NEXT: xxlnor 35, 35, 35
+; PPC64LE-NEXT: vsubuhm 2, 2, 3
; PPC64LE-NEXT: blr
%t0 = add <8 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
%t1 = add <8 x i16> %y, %t0
@@ -374,10 +374,10 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
; PPC32-LABEL: vector_i128_i32:
; PPC32: # %bb.0:
-; PPC32-NEXT: add 3, 7, 3
-; PPC32-NEXT: add 4, 8, 4
-; PPC32-NEXT: add 5, 9, 5
-; PPC32-NEXT: add 6, 10, 6
+; PPC32-NEXT: add 3, 3, 7
+; PPC32-NEXT: add 4, 4, 8
+; PPC32-NEXT: add 5, 5, 9
+; PPC32-NEXT: add 6, 6, 10
; PPC32-NEXT: addi 3, 3, 1
; PPC32-NEXT: addi 4, 4, 1
; PPC32-NEXT: addi 5, 5, 1
@@ -386,10 +386,10 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
;
; PPC64BE-LABEL: vector_i128_i32:
; PPC64BE: # %bb.0:
-; PPC64BE-NEXT: add 6, 10, 6
-; PPC64BE-NEXT: add 5, 9, 5
-; PPC64BE-NEXT: add 4, 8, 4
-; PPC64BE-NEXT: add 3, 7, 3
+; PPC64BE-NEXT: add 6, 6, 10
+; PPC64BE-NEXT: add 5, 5, 9
+; PPC64BE-NEXT: add 4, 4, 8
+; PPC64BE-NEXT: add 3, 3, 7
; PPC64BE-NEXT: addi 6, 6, 1
; PPC64BE-NEXT: addi 5, 5, 1
; PPC64BE-NEXT: addi 4, 4, 1
@@ -398,8 +398,8 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
;
; PPC64LE-LABEL: vector_i128_i32:
; PPC64LE: # %bb.0:
-; PPC64LE-NEXT: xxlnor 34, 34, 34
-; PPC64LE-NEXT: vsubuwm 2, 3, 2
+; PPC64LE-NEXT: xxlnor 35, 35, 35
+; PPC64LE-NEXT: vsubuwm 2, 2, 3
; PPC64LE-NEXT: blr
%t0 = add <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1>
%t1 = add <4 x i32> %y, %t0
@@ -409,28 +409,28 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; PPC32-LABEL: vector_i128_i64:
; PPC32: # %bb.0:
-; PPC32-NEXT: not 4, 4
-; PPC32-NEXT: not 3, 3
-; PPC32-NEXT: subc 4, 8, 4
-; PPC32-NEXT: subfe 3, 3, 7
-; PPC32-NEXT: not 6, 6
-; PPC32-NEXT: not 5, 5
-; PPC32-NEXT: subc 6, 10, 6
-; PPC32-NEXT: subfe 5, 5, 9
+; PPC32-NEXT: not 8, 8
+; PPC32-NEXT: not 7, 7
+; PPC32-NEXT: subc 4, 4, 8
+; PPC32-NEXT: subfe 3, 7, 3
+; PPC32-NEXT: not 7, 10
+; PPC32-NEXT: not 9, 9
+; PPC32-NEXT: subc 6, 6, 7
+; PPC32-NEXT: subfe 5, 9, 5
; PPC32-NEXT: blr
;
; PPC64BE-LABEL: vector_i128_i64:
; PPC64BE: # %bb.0:
-; PPC64BE-NEXT: add 3, 5, 3
-; PPC64BE-NEXT: add 4, 6, 4
+; PPC64BE-NEXT: add 3, 3, 5
+; PPC64BE-NEXT: add 4, 4, 6
; PPC64BE-NEXT: addi 3, 3, 1
; PPC64BE-NEXT: addi 4, 4, 1
; PPC64BE-NEXT: blr
;
; PPC64LE-LABEL: vector_i128_i64:
; PPC64LE: # %bb.0:
-; PPC64LE-NEXT: xxlnor 34, 34, 34
-; PPC64LE-NEXT: vsubudm 2, 3, 2
+; PPC64LE-NEXT: xxlnor 35, 35, 35
+; PPC64LE-NEXT: vsubudm 2, 2, 3
; PPC64LE-NEXT: blr
%t0 = add <2 x i64> %x, <i64 1, i64 1>
%t1 = add <2 x i64> %y, %t0
diff --git a/llvm/test/CodeGen/PowerPC/vavg.ll b/llvm/test/CodeGen/PowerPC/vavg.ll
index e473cbcd3f281..7f6a91ad5fd4f 100644
--- a/llvm/test/CodeGen/PowerPC/vavg.ll
+++ b/llvm/test/CodeGen/PowerPC/vavg.ll
@@ -197,25 +197,25 @@ entry:
define <4 x i32> @test_v4i32_negative(<4 x i32> %m, <4 x i32> %n) {
; CHECK-P9-LABEL: test_v4i32_negative:
; CHECK-P9: # %bb.0: # %entry
-; CHECK-P9-NEXT: xxlnor 34, 34, 34
-; CHECK-P9-NEXT: vsubuwm 2, 3, 2
+; CHECK-P9-NEXT: xxlnor 35, 35, 35
+; CHECK-P9-NEXT: vsubuwm 2, 2, 3
; CHECK-P9-NEXT: vspltisw 3, 2
; CHECK-P9-NEXT: vsrw 2, 2, 3
; CHECK-P9-NEXT: blr
;
; CHECK-P8-LABEL: test_v4i32_negative:
; CHECK-P8: # %bb.0: # %entry
-; CHECK-P8-NEXT: xxlnor 34, 34, 34
+; CHECK-P8-NEXT: xxlnor 35, 35, 35
; CHECK-P8-NEXT: vspltisw 4, 2
-; CHECK-P8-NEXT: vsubuwm 2, 3, 2
+; CHECK-P8-NEXT: vsubuwm 2, 2, 3
; CHECK-P8-NEXT: vsrw 2, 2, 4
; CHECK-P8-NEXT: blr
;
; CHECK-P7-LABEL: test_v4i32_negative:
; CHECK-P7: # %bb.0: # %entry
-; CHECK-P7-NEXT: xxlnor 34, 34, 34
+; CHECK-P7-NEXT: xxlnor 35, 35, 35
; CHECK-P7-NEXT: vspltisw 4, 2
-; CHECK-P7-NEXT: vsubuwm 2, 3, 2
+; CHECK-P7-NEXT: vsubuwm 2, 2, 3
; CHECK-P7-NEXT: vsrw 2, 2, 4
; CHECK-P7-NEXT: blr
entry:
@@ -262,25 +262,25 @@ entry:
define <8 x i16> @test_v8i16_overflow(<8 x i16> %m, <8 x i16> %n) {
; CHECK-P9-LABEL: test_v8i16_overflow:
; CHECK-P9: # %bb.0: # %entry
-; CHECK-P9-NEXT: xxlnor 34, 34, 34
+; CHECK-P9-NEXT: xxlnor 35, 35, 35
; CHECK-P9-NEXT: vspltish 4, 1
-; CHECK-P9-NEXT: vsubuhm 2, 3, 2
+; CHECK-P9-NEXT: vsubuhm 2, 2, 3
; CHECK-P9-NEXT: vsrh 2, 2, 4
; CHECK-P9-NEXT: blr
;
; CHECK-P8-LABEL: test_v8i16_overflow:
; CHECK-P8: # %bb.0: # %entry
-; CHECK-P8-NEXT: xxlnor 34, 34, 34
+; CHECK-P8-NEXT: xxlnor 35, 35, 35
; CHECK-P8-NEXT: vspltish 4, 1
-; CHECK-P8-NEXT: vsubuhm 2, 3, 2
+; CHECK-P8-NEXT: vsubuhm 2, 2, 3
; CHECK-P8-NEXT: vsrh 2, 2, 4
; CHECK-P8-NEXT: blr
;
; CHECK-P7-LABEL: test_v8i16_overflow:
; CHECK-P7: # %bb.0: # %entry
-; CHECK-P7-NEXT: xxlnor 34, 34, 34
+; CHECK-P7-NEXT: xxlnor 35, 35, 35
; CHECK-P7-NEXT: vspltish 4, 1
-; CHECK-P7-NEXT: vsubuhm 2, 3, 2
+; CHECK-P7-NEXT: vsubuhm 2, 2, 3
; CHECK-P7-NEXT: vsrh 2, 2, 4
; CHECK-P7-NEXT: blr
entry:
@@ -293,25 +293,25 @@ entry:
define <4 x i32> @test_v4i32_overflow(<4 x i32> %m, <4 x i32> %n) {
; CHECK-P9-LABEL: test_v4i32_overflow:
; CHECK-P9: # %bb.0: # %entry
-; CHECK-P9-NEXT: xxlnor 34, 34, 34
+; CHECK-P9-NEXT: xxlnor 35, 35, 35
; CHECK-P9-NEXT: vspltisw 4, 1
-; CHECK-P9-NEXT: vsubuwm 2, 3, 2
+; CHECK-P9-NEXT: vsubuwm 2, 2, 3
; CHECK-P9-NEXT: vsrw 2, 2, 4
; CHECK-P9-NEXT: blr
;
; CHECK-P8-LABEL: test_v4i32_overflow:
; CHECK-P8: # %bb.0: # %entry
-; CHECK-P8-NEXT: xxlnor 34, 34, 34
+; CHECK-P8-NEXT: xxlnor 35, 35, 35
; CHECK-P8-NEXT: vspltisw 4, 1
-; CHECK-P8-NEXT: vsubuwm 2, 3, 2
+; CHECK-P8-NEXT: vsubuwm 2, 2, 3
; CHECK-P8-NEXT: vsrw 2, 2, 4
; CHECK-P8-NEXT: blr
;
; CHECK-P7-LABEL: test_v4i32_overflow:
; CHECK-P7: # %bb.0: # %entry
-; CHECK-P7-NEXT: xxlnor 34, 34, 34
+; CHECK-P7-NEXT: xxlnor 35, 35, 35
; CHECK-P7-NEXT: vspltisw 4, 1
-; CHECK-P7-NEXT: vsubuwm 2, 3, 2
+; CHECK-P7-NEXT: vsubuwm 2, 2, 3
; CHECK-P7-NEXT: vsrw 2, 2, 4
; CHECK-P7-NEXT: blr
entry:
@@ -324,25 +324,25 @@ entry:
define <16 x i8> @test_v16i8_overflow(<16 x i8> %m, <16 x i8> %n) {
; CHECK-P9-LABEL: test_v16i8_overflow:
; CHECK-P9: # %bb.0: # %entry
-; CHECK-P9-NEXT: xxlnor 34, 34, 34
+; CHECK-P9-NEXT: xxlnor 35, 35, 35
; CHECK-P9-NEXT: xxspltib 36, 1
-; CHECK-P9-NEXT: vsububm 2, 3, 2
+; CHECK-P9-NEXT: vsububm 2, 2, 3
; CHECK-P9-NEXT: vsrb 2, 2, 4
; CHECK-P9-NEXT: blr
;
; CHECK-P8-LABEL: test_v16i8_overflow:
; CHECK-P8: # %bb.0: # %entry
-; CHECK-P8-NEXT: xxlnor 34, 34, 34
+; CHECK-P8-NEXT: xxlnor 35, 35, 35
; CHECK-P8-NEXT: vspltisb 4, 1
-; CHECK-P8-NEXT: vsububm 2, 3, 2
+; CHECK-P8-NEXT: vsububm 2, 2, 3
; CHECK-P8-NEXT: vsrb 2, 2, 4
; CHECK-P8-NEXT: blr
;
; CHECK-P7-LABEL: test_v16i8_overflow:
; CHECK-P7: # %bb.0: # %entry
-; CHECK-P7-NEXT: xxlnor 34, 34, 34
+; CHECK-P7-NEXT: xxlnor 35, 35, 35
; CHECK-P7-NEXT: vspltisb 4, 1
-; CHECK-P7-NEXT: vsububm 2, 3, 2
+; CHECK-P7-NEXT: vsububm 2, 2, 3
; CHECK-P7-NEXT: vsrb 2, 2, 4
; CHECK-P7-NEXT: blr
entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-halving.ll b/llvm/test/CodeGen/Thumb2/mve-halving.ll
index 524818e8d210d..10c968a1b01b9 100644
--- a/llvm/test/CodeGen/Thumb2/mve-halving.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-halving.ll
@@ -233,8 +233,8 @@ define arm_aapcs_vfpcc <4 x i32> @vhsubu_v4i32_nw(<4 x i32> %x, <4 x i32> %y) {
define arm_aapcs_vfpcc <16 x i8> @vrhadds_v16i8(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: vrhadds_v16i8:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i8 q0, q1, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i8 q0, q0, q1
; CHECK-NEXT: vshr.s8 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add <16 x i8> %x, %y
@@ -245,8 +245,8 @@ define arm_aapcs_vfpcc <16 x i8> @vrhadds_v16i8(<16 x i8> %x, <16 x i8> %y) {
define arm_aapcs_vfpcc <16 x i8> @vrhaddu_v16i8(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: vrhaddu_v16i8:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i8 q0, q1, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i8 q0, q0, q1
; CHECK-NEXT: vshr.u8 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add <16 x i8> %x, %y
@@ -257,8 +257,8 @@ define arm_aapcs_vfpcc <16 x i8> @vrhaddu_v16i8(<16 x i8> %x, <16 x i8> %y) {
define arm_aapcs_vfpcc <8 x i16> @vrhadds_v8i16(<8 x i16> %x, <8 x i16> %y) {
; CHECK-LABEL: vrhadds_v8i16:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i16 q0, q1, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i16 q0, q0, q1
; CHECK-NEXT: vshr.s16 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add <8 x i16> %x, %y
@@ -269,8 +269,8 @@ define arm_aapcs_vfpcc <8 x i16> @vrhadds_v8i16(<8 x i16> %x, <8 x i16> %y) {
define arm_aapcs_vfpcc <8 x i16> @vrhaddu_v8i16(<8 x i16> %x, <8 x i16> %y) {
; CHECK-LABEL: vrhaddu_v8i16:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i16 q0, q1, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i16 q0, q0, q1
; CHECK-NEXT: vshr.u16 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add <8 x i16> %x, %y
@@ -281,8 +281,8 @@ define arm_aapcs_vfpcc <8 x i16> @vrhaddu_v8i16(<8 x i16> %x, <8 x i16> %y) {
define arm_aapcs_vfpcc <4 x i32> @vrhadds_v4i32(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: vrhadds_v4i32:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i32 q0, q1, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i32 q0, q0, q1
; CHECK-NEXT: vshr.s32 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add <4 x i32> %x, %y
@@ -293,8 +293,8 @@ define arm_aapcs_vfpcc <4 x i32> @vrhadds_v4i32(<4 x i32> %x, <4 x i32> %y) {
define arm_aapcs_vfpcc <4 x i32> @vrhaddu_v4i32(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: vrhaddu_v4i32:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i32 q0, q1, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i32 q0, q0, q1
; CHECK-NEXT: vshr.u32 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add <4 x i32> %x, %y
@@ -305,8 +305,8 @@ define arm_aapcs_vfpcc <4 x i32> @vrhaddu_v4i32(<4 x i32> %x, <4 x i32> %y) {
define arm_aapcs_vfpcc <16 x i8> @vrhadds_v16i8_nwop(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: vrhadds_v16i8_nwop:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i8 q0, q1, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i8 q0, q0, q1
; CHECK-NEXT: vshr.s8 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add nsw <16 x i8> %x, %y
@@ -317,8 +317,8 @@ define arm_aapcs_vfpcc <16 x i8> @vrhadds_v16i8_nwop(<16 x i8> %x, <16 x i8> %y)
define arm_aapcs_vfpcc <16 x i8> @vrhaddu_v16i8_nwop(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: vrhaddu_v16i8_nwop:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i8 q0, q1, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i8 q0, q0, q1
; CHECK-NEXT: vshr.u8 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add nuw <16 x i8> %x, %y
@@ -329,8 +329,8 @@ define arm_aapcs_vfpcc <16 x i8> @vrhaddu_v16i8_nwop(<16 x i8> %x, <16 x i8> %y)
define arm_aapcs_vfpcc <8 x i16> @vrhadds_v8i16_nwop(<8 x i16> %x, <8 x i16> %y) {
; CHECK-LABEL: vrhadds_v8i16_nwop:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i16 q0, q1, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i16 q0, q0, q1
; CHECK-NEXT: vshr.s16 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add nsw <8 x i16> %x, %y
@@ -341,8 +341,8 @@ define arm_aapcs_vfpcc <8 x i16> @vrhadds_v8i16_nwop(<8 x i16> %x, <8 x i16> %y)
define arm_aapcs_vfpcc <8 x i16> @vrhaddu_v8i16_nwop(<8 x i16> %x, <8 x i16> %y) {
; CHECK-LABEL: vrhaddu_v8i16_nwop:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i16 q0, q1, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i16 q0, q0, q1
; CHECK-NEXT: vshr.u16 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add nuw <8 x i16> %x, %y
@@ -353,8 +353,8 @@ define arm_aapcs_vfpcc <8 x i16> @vrhaddu_v8i16_nwop(<8 x i16> %x, <8 x i16> %y)
define arm_aapcs_vfpcc <4 x i32> @vrhadds_v4i32_nwop(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: vrhadds_v4i32_nwop:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i32 q0, q1, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i32 q0, q0, q1
; CHECK-NEXT: vshr.s32 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add nsw <4 x i32> %x, %y
@@ -365,8 +365,8 @@ define arm_aapcs_vfpcc <4 x i32> @vrhadds_v4i32_nwop(<4 x i32> %x, <4 x i32> %y)
define arm_aapcs_vfpcc <4 x i32> @vrhaddu_v4i32_nwop(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: vrhaddu_v4i32_nwop:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i32 q0, q1, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i32 q0, q0, q1
; CHECK-NEXT: vshr.u32 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add nuw <4 x i32> %x, %y
diff --git a/llvm/test/CodeGen/Thumb2/mve-vhadd.ll b/llvm/test/CodeGen/Thumb2/mve-vhadd.ll
index 207c0f377e34a..3fc88d7a93b61 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vhadd.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vhadd.ll
@@ -221,10 +221,10 @@ entry:
define arm_aapcs_vfpcc <4 x i16> @vrhadds_v4i16(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-LABEL: vrhadds_v4i16:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmovlb.s16 q0, q0
; CHECK-NEXT: vmovlb.s16 q1, q1
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i32 q0, q1, q0
+; CHECK-NEXT: vmovlb.s16 q0, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i32 q0, q0, q1
; CHECK-NEXT: vshr.u32 q0, q0, #1
; CHECK-NEXT: bx lr
entry:
@@ -287,12 +287,12 @@ entry:
define arm_aapcs_vfpcc <4 x i8> @vrhadds_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
; CHECK-LABEL: vrhadds_v4i8:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmovlb.s8 q0, q0
; CHECK-NEXT: vmovlb.s8 q1, q1
-; CHECK-NEXT: vmovlb.s16 q0, q0
+; CHECK-NEXT: vmovlb.s8 q0, q0
; CHECK-NEXT: vmovlb.s16 q1, q1
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i32 q0, q1, q0
+; CHECK-NEXT: vmovlb.s16 q0, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i32 q0, q0, q1
; CHECK-NEXT: vmovlb.u16 q0, q0
; CHECK-NEXT: vshr.u32 q0, q0, #1
; CHECK-NEXT: bx lr
@@ -327,10 +327,10 @@ entry:
define arm_aapcs_vfpcc <8 x i8> @vrhadds_v8i8(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-LABEL: vrhadds_v8i8:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmovlb.s8 q0, q0
; CHECK-NEXT: vmovlb.s8 q1, q1
-; CHECK-NEXT: vmvn q0, q0
-; CHECK-NEXT: vsub.i16 q0, q1, q0
+; CHECK-NEXT: vmovlb.s8 q0, q0
+; CHECK-NEXT: vmvn q1, q1
+; CHECK-NEXT: vsub.i16 q0, q0, q1
; CHECK-NEXT: vshr.u16 q0, q0, #1
; CHECK-NEXT: bx lr
entry:
diff --git a/llvm/test/CodeGen/X86/inc-of-add.ll b/llvm/test/CodeGen/X86/inc-of-add.ll
index b07efed2b9199..1070f49ecf214 100644
--- a/llvm/test/CodeGen/X86/inc-of-add.ll
+++ b/llvm/test/CodeGen/X86/inc-of-add.ll
@@ -102,9 +102,8 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; ALL-LABEL: vector_i128_i8:
; ALL: # %bb.0:
; ALL-NEXT: pcmpeqd %xmm2, %xmm2
-; ALL-NEXT: pxor %xmm0, %xmm2
-; ALL-NEXT: psubb %xmm2, %xmm1
-; ALL-NEXT: movdqa %xmm1, %xmm0
+; ALL-NEXT: pxor %xmm1, %xmm2
+; ALL-NEXT: psubb %xmm2, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
%t1 = add <16 x i8> %y, %t0
@@ -115,9 +114,8 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; ALL-LABEL: vector_i128_i16:
; ALL: # %bb.0:
; ALL-NEXT: pcmpeqd %xmm2, %xmm2
-; ALL-NEXT: pxor %xmm0, %xmm2
-; ALL-NEXT: psubw %xmm2, %xmm1
-; ALL-NEXT: movdqa %xmm1, %xmm0
+; ALL-NEXT: pxor %xmm1, %xmm2
+; ALL-NEXT: psubw %xmm2, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <8 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
%t1 = add <8 x i16> %y, %t0
@@ -128,9 +126,8 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
; ALL-LABEL: vector_i128_i32:
; ALL: # %bb.0:
; ALL-NEXT: pcmpeqd %xmm2, %xmm2
-; ALL-NEXT: pxor %xmm0, %xmm2
-; ALL-NEXT: psubd %xmm2, %xmm1
-; ALL-NEXT: movdqa %xmm1, %xmm0
+; ALL-NEXT: pxor %xmm1, %xmm2
+; ALL-NEXT: psubd %xmm2, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1>
%t1 = add <4 x i32> %y, %t0
@@ -141,9 +138,8 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; ALL-LABEL: vector_i128_i64:
; ALL: # %bb.0:
; ALL-NEXT: pcmpeqd %xmm2, %xmm2
-; ALL-NEXT: pxor %xmm0, %xmm2
-; ALL-NEXT: psubq %xmm2, %xmm1
-; ALL-NEXT: movdqa %xmm1, %xmm0
+; ALL-NEXT: pxor %xmm1, %xmm2
+; ALL-NEXT: psubq %xmm2, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <2 x i64> %x, <i64 1, i64 1>
%t1 = add <2 x i64> %y, %t0
diff --git a/llvm/test/CodeGen/X86/ucmp.ll b/llvm/test/CodeGen/X86/ucmp.ll
index 3aa92d4083d41..65836384894ef 100644
--- a/llvm/test/CodeGen/X86/ucmp.ll
+++ b/llvm/test/CodeGen/X86/ucmp.ll
@@ -440,25 +440,19 @@ define <4 x i32> @ucmp_normal_vectors(<4 x i32> %x, <4 x i32> %y) nounwind {
; SSE2-LABEL: ucmp_normal_vectors:
; SSE2: # %bb.0:
; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pminud %xmm1, %xmm2
+; SSE2-NEXT: pmaxud %xmm1, %xmm2
; SSE2-NEXT: pcmpeqd %xmm0, %xmm2
-; SSE2-NEXT: pcmpeqd %xmm3, %xmm3
-; SSE2-NEXT: pxor %xmm3, %xmm2
-; SSE2-NEXT: pmaxud %xmm0, %xmm1
+; SSE2-NEXT: pminud %xmm0, %xmm1
; SSE2-NEXT: pcmpeqd %xmm1, %xmm0
-; SSE2-NEXT: pxor %xmm3, %xmm0
; SSE2-NEXT: psubd %xmm2, %xmm0
; SSE2-NEXT: retq
;
; AVX2-LABEL: ucmp_normal_vectors:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpminud %xmm1, %xmm0, %xmm2
+; AVX2-NEXT: vpmaxud %xmm1, %xmm0, %xmm2
; AVX2-NEXT: vpcmpeqd %xmm2, %xmm0, %xmm2
-; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
-; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpmaxud %xmm1, %xmm0, %xmm1
+; AVX2-NEXT: vpminud %xmm1, %xmm0, %xmm1
; AVX2-NEXT: vpcmpeqd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm3, %xmm0, %xmm0
; AVX2-NEXT: vpsubd %xmm2, %xmm0, %xmm0
; AVX2-NEXT: retq
;
>From c334df7bb4c96c2a9fbb72467a2f61e21cb0f957 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <RohanRadhesh.Shenoy at amd.com>
Date: Wed, 2 Sep 2026 01:31:54 +0530
Subject: [PATCH 3/4] format
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index dbd796a695e90..59c2b89ac80d3 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -60601,8 +60601,8 @@ static SDValue combineSub(SDNode *N, SelectionDAG &DAG,
}
// Fold SUB(~A,~B) -> SUB(B,A)
- if (isBitwiseNot(Op0) && Op0.hasOneUse() &&
- isBitwiseNot(Op1) && Op1.hasOneUse()) {
+ if (isBitwiseNot(Op0) && Op0.hasOneUse() && isBitwiseNot(Op1) &&
+ Op1.hasOneUse()) {
return DAG.getNode(ISD::SUB, DL, VT, Op1.getOperand(0), Op0.getOperand(0));
}
>From 7d03378d190f1b0b79dd905a90686d979b114db0 Mon Sep 17 00:00:00 2001
From: Rohan Shenoy <RohanRadhesh.Shenoy at amd.com>
Date: Wed, 16 Sep 2026 11:53:46 +0530
Subject: [PATCH 4/4] revert generic dag change
---
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 6 +-
llvm/test/CodeGen/AArch64/arm64-vhadd.ll | 8 +-
llvm/test/CodeGen/AArch64/avg.ll | 20 +-
llvm/test/CodeGen/AArch64/inc-of-add.ll | 16 +-
llvm/test/CodeGen/AArch64/sve-hadd.ll | 94 ++++-----
llvm/test/CodeGen/ARM/inc-of-add.ll | 89 +++++----
llvm/test/CodeGen/PowerPC/inc-of-add.ll | 188 +++++++++---------
llvm/test/CodeGen/PowerPC/vavg.ll | 48 ++---
llvm/test/CodeGen/Thumb2/mve-halving.ll | 48 ++---
llvm/test/CodeGen/Thumb2/mve-vhadd.ll | 20 +-
llvm/test/CodeGen/X86/inc-of-add.ll | 20 +-
11 files changed, 281 insertions(+), 276 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 71f2761a1de7c..e08eaf1ba81d4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -3156,14 +3156,14 @@ SDValue DAGCombiner::visitADDLike(SDNode *N) {
// Look for:
// add (add x, y), 1
// And if the target does not like this form then turn into:
- // sub x, (xor y, -1)
+ // sub y, (xor x, -1)
if (!TLI.preferIncOfAddToSubOfNot(VT) && N0.getOpcode() == ISD::ADD &&
N0.hasOneUse() &&
// Limit this to after legalization if the add has wrap flags
(Level >= AfterLegalizeDAG || (!N->getFlags().hasNoUnsignedWrap() &&
!N->getFlags().hasNoSignedWrap()))) {
- SDValue Not = DAG.getNOT(DL, N0.getOperand(1), VT);
- return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(0), Not);
+ SDValue Not = DAG.getNOT(DL, N0.getOperand(0), VT);
+ return DAG.getNode(ISD::SUB, DL, VT, N0.getOperand(1), Not);
}
}
diff --git a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
index 527061b412eae..9034a39b0ac51 100644
--- a/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-vhadd.ll
@@ -1330,13 +1330,13 @@ define <2 x i16> @rhadd8x2_zext_asr(<2 x i8> %src1, <2 x i8> %src2) {
define <2 x i16> @rhadd8x2_sext_lsr(<2 x i8> %src1, <2 x i8> %src2) {
; CHECK-SD-LABEL: rhadd8x2_sext_lsr:
; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: shl.2s v1, v1, #24
; CHECK-SD-NEXT: shl.2s v0, v0, #24
+; CHECK-SD-NEXT: shl.2s v1, v1, #24
; CHECK-SD-NEXT: movi d2, #0x00ffff0000ffff
-; CHECK-SD-NEXT: sshr.2s v1, v1, #24
; CHECK-SD-NEXT: sshr.2s v0, v0, #24
-; CHECK-SD-NEXT: mvn.8b v1, v1
-; CHECK-SD-NEXT: sub.2s v0, v0, v1
+; CHECK-SD-NEXT: sshr.2s v1, v1, #24
+; CHECK-SD-NEXT: mvn.8b v0, v0
+; CHECK-SD-NEXT: sub.2s v0, v1, v0
; CHECK-SD-NEXT: and.8b v0, v0, v2
; CHECK-SD-NEXT: ushr.2s v0, v0, #1
; CHECK-SD-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/avg.ll b/llvm/test/CodeGen/AArch64/avg.ll
index 4c6f2217bfc55..fbe8391ae45b5 100644
--- a/llvm/test/CodeGen/AArch64/avg.ll
+++ b/llvm/test/CodeGen/AArch64/avg.ll
@@ -234,8 +234,8 @@ define <8 x i16> @add_avgceilu_mismatch1(<8 x i16> %a0, <8 x i16> %a1) {
define <8 x i16> @add_avgceilu_mismatch2(<8 x i16> %a0, <8 x i16> %a1) {
; CHECK-LABEL: add_avgceilu_mismatch2:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v0.16b, v0.16b
-; CHECK-NEXT: sub v0.8h, v1.8h, v0.8h
+; CHECK-NEXT: mvn v1.16b, v1.16b
+; CHECK-NEXT: sub v0.8h, v0.8h, v1.8h
; CHECK-NEXT: ushr v0.8h, v0.8h, #1
; CHECK-NEXT: ret
%add0 = add nuw <8 x i16> %a1, %a0
@@ -247,8 +247,8 @@ define <8 x i16> @add_avgceilu_mismatch2(<8 x i16> %a0, <8 x i16> %a1) {
define <8 x i16> @add_avgceilu_mismatch3(<8 x i16> %a0, <8 x i16> %a1) {
; CHECK-LABEL: add_avgceilu_mismatch3:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v0.16b, v0.16b
-; CHECK-NEXT: sub v0.8h, v1.8h, v0.8h
+; CHECK-NEXT: mvn v1.16b, v1.16b
+; CHECK-NEXT: sub v0.8h, v0.8h, v1.8h
; CHECK-NEXT: ushr v0.8h, v0.8h, #1
; CHECK-NEXT: ret
%add0 = add nuw <8 x i16> %a1, %a0
@@ -355,8 +355,8 @@ define <8 x i16> @add_avgceils_mismatch1(<8 x i16> %a0, <8 x i16> %a1) {
define <8 x i16> @add_avgceils_mismatch2(<8 x i16> %a0, <8 x i16> %a1) {
; CHECK-LABEL: add_avgceils_mismatch2:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v0.16b, v0.16b
-; CHECK-NEXT: sub v0.8h, v1.8h, v0.8h
+; CHECK-NEXT: mvn v1.16b, v1.16b
+; CHECK-NEXT: sub v0.8h, v0.8h, v1.8h
; CHECK-NEXT: sshr v0.8h, v0.8h, #1
; CHECK-NEXT: ret
%add0 = add nsw <8 x i16> %a1, %a0
@@ -368,8 +368,8 @@ define <8 x i16> @add_avgceils_mismatch2(<8 x i16> %a0, <8 x i16> %a1) {
define <8 x i16> @add_avgceils_mismatch3(<8 x i16> %a0, <8 x i16> %a1) {
; CHECK-LABEL: add_avgceils_mismatch3:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v0.16b, v0.16b
-; CHECK-NEXT: sub v0.8h, v1.8h, v0.8h
+; CHECK-NEXT: mvn v1.16b, v1.16b
+; CHECK-NEXT: sub v0.8h, v0.8h, v1.8h
; CHECK-NEXT: sshr v0.8h, v0.8h, #1
; CHECK-NEXT: ret
%add0 = add nsw <8 x i16> %a1, %a0
@@ -381,8 +381,8 @@ define <8 x i16> @add_avgceils_mismatch3(<8 x i16> %a0, <8 x i16> %a1) {
define <8 x i16> @add_avgceils_mismatch4(<8 x i16> %a0, <8 x i16> %a1) {
; CHECK-LABEL: add_avgceils_mismatch4:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v1.16b, v1.16b
-; CHECK-NEXT: sub v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: mvn v0.16b, v0.16b
+; CHECK-NEXT: sub v0.8h, v1.8h, v0.8h
; CHECK-NEXT: sshr v0.8h, v0.8h, #2
; CHECK-NEXT: ret
%add0 = add nsw <8 x i16> %a0, splat(i16 1)
diff --git a/llvm/test/CodeGen/AArch64/inc-of-add.ll b/llvm/test/CodeGen/AArch64/inc-of-add.ll
index dc7c018194b0a..b5ce2ab5d409f 100644
--- a/llvm/test/CodeGen/AArch64/inc-of-add.ll
+++ b/llvm/test/CodeGen/AArch64/inc-of-add.ll
@@ -53,8 +53,8 @@ define i64 @scalar_i64(i64 %x, i64 %y) nounwind {
define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; CHECK-LABEL: vector_i128_i8:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v1.16b, v1.16b
-; CHECK-NEXT: sub v0.16b, v0.16b, v1.16b
+; CHECK-NEXT: mvn v0.16b, v0.16b
+; CHECK-NEXT: sub v0.16b, v1.16b, v0.16b
; CHECK-NEXT: ret
%t0 = add <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
%t1 = add <16 x i8> %y, %t0
@@ -64,8 +64,8 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; CHECK-LABEL: vector_i128_i16:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v1.16b, v1.16b
-; CHECK-NEXT: sub v0.8h, v0.8h, v1.8h
+; CHECK-NEXT: mvn v0.16b, v0.16b
+; CHECK-NEXT: sub v0.8h, v1.8h, v0.8h
; CHECK-NEXT: ret
%t0 = add <8 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
%t1 = add <8 x i16> %y, %t0
@@ -75,8 +75,8 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
; CHECK-LABEL: vector_i128_i32:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v1.16b, v1.16b
-; CHECK-NEXT: sub v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: mvn v0.16b, v0.16b
+; CHECK-NEXT: sub v0.4s, v1.4s, v0.4s
; CHECK-NEXT: ret
%t0 = add <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1>
%t1 = add <4 x i32> %y, %t0
@@ -86,8 +86,8 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; CHECK-LABEL: vector_i128_i64:
; CHECK: // %bb.0:
-; CHECK-NEXT: mvn v1.16b, v1.16b
-; CHECK-NEXT: sub v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: mvn v0.16b, v0.16b
+; CHECK-NEXT: sub v0.2d, v1.2d, v0.2d
; CHECK-NEXT: ret
%t0 = add <2 x i64> %x, <i64 1, i64 1>
%t1 = add <2 x i64> %y, %t0
diff --git a/llvm/test/CodeGen/AArch64/sve-hadd.ll b/llvm/test/CodeGen/AArch64/sve-hadd.ll
index 24e4d0482c0d5..31681d57dd9da 100644
--- a/llvm/test/CodeGen/AArch64/sve-hadd.ll
+++ b/llvm/test/CodeGen/AArch64/sve-hadd.ll
@@ -714,10 +714,10 @@ define <vscale x 2 x i32> @rhadds_v2i32(<vscale x 2 x i32> %s0, <vscale x 2 x i3
; SVE-LABEL: rhadds_v2i32:
; SVE: // %bb.0: // %entry
; SVE-NEXT: ptrue p0.d
-; SVE-NEXT: sxtw z1.d, p0/m, z1.d
; SVE-NEXT: sxtw z0.d, p0/m, z0.d
-; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
-; SVE-NEXT: sub z0.d, z0.d, z1.d
+; SVE-NEXT: sxtw z1.d, p0/m, z1.d
+; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
+; SVE-NEXT: sub z0.d, z1.d, z0.d
; SVE-NEXT: asr z0.d, z0.d, #1
; SVE-NEXT: ret
;
@@ -742,10 +742,10 @@ define <vscale x 2 x i32> @rhadds_v2i32_lsh(<vscale x 2 x i32> %s0, <vscale x 2
; CHECK-LABEL: rhadds_v2i32_lsh:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: sxtw z1.d, p0/m, z1.d
; CHECK-NEXT: sxtw z0.d, p0/m, z0.d
-; CHECK-NEXT: subr z1.b, z1.b, #255 // =0xff
-; CHECK-NEXT: sub z0.d, z0.d, z1.d
+; CHECK-NEXT: sxtw z1.d, p0/m, z1.d
+; CHECK-NEXT: subr z0.b, z0.b, #255 // =0xff
+; CHECK-NEXT: sub z0.d, z1.d, z0.d
; CHECK-NEXT: lsr z0.d, z0.d, #1
; CHECK-NEXT: ret
entry:
@@ -761,10 +761,10 @@ entry:
define <vscale x 2 x i32> @rhaddu_v2i32(<vscale x 2 x i32> %s0, <vscale x 2 x i32> %s1) {
; SVE-LABEL: rhaddu_v2i32:
; SVE: // %bb.0: // %entry
-; SVE-NEXT: and z1.d, z1.d, #0xffffffff
; SVE-NEXT: and z0.d, z0.d, #0xffffffff
-; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
-; SVE-NEXT: sub z0.d, z0.d, z1.d
+; SVE-NEXT: and z1.d, z1.d, #0xffffffff
+; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
+; SVE-NEXT: sub z0.d, z1.d, z0.d
; SVE-NEXT: lsr z0.d, z0.d, #1
; SVE-NEXT: ret
;
@@ -861,10 +861,10 @@ define <vscale x 2 x i16> @rhadds_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i1
; SVE-LABEL: rhadds_v2i16:
; SVE: // %bb.0: // %entry
; SVE-NEXT: ptrue p0.d
-; SVE-NEXT: sxth z1.d, p0/m, z1.d
; SVE-NEXT: sxth z0.d, p0/m, z0.d
-; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
-; SVE-NEXT: sub z0.d, z0.d, z1.d
+; SVE-NEXT: sxth z1.d, p0/m, z1.d
+; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
+; SVE-NEXT: sub z0.d, z1.d, z0.d
; SVE-NEXT: asr z0.d, z0.d, #1
; SVE-NEXT: ret
;
@@ -889,10 +889,10 @@ define <vscale x 2 x i16> @rhadds_v2i16_lsh(<vscale x 2 x i16> %s0, <vscale x 2
; CHECK-LABEL: rhadds_v2i16_lsh:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ptrue p0.d
-; CHECK-NEXT: sxth z1.d, p0/m, z1.d
; CHECK-NEXT: sxth z0.d, p0/m, z0.d
-; CHECK-NEXT: subr z1.b, z1.b, #255 // =0xff
-; CHECK-NEXT: sub z0.d, z0.d, z1.d
+; CHECK-NEXT: sxth z1.d, p0/m, z1.d
+; CHECK-NEXT: subr z0.b, z0.b, #255 // =0xff
+; CHECK-NEXT: sub z0.d, z1.d, z0.d
; CHECK-NEXT: and z0.d, z0.d, #0xffffffff
; CHECK-NEXT: lsr z0.d, z0.d, #1
; CHECK-NEXT: ret
@@ -909,10 +909,10 @@ entry:
define <vscale x 2 x i16> @rhaddu_v2i16(<vscale x 2 x i16> %s0, <vscale x 2 x i16> %s1) {
; SVE-LABEL: rhaddu_v2i16:
; SVE: // %bb.0: // %entry
-; SVE-NEXT: and z1.d, z1.d, #0xffff
; SVE-NEXT: and z0.d, z0.d, #0xffff
-; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
-; SVE-NEXT: sub z0.d, z0.d, z1.d
+; SVE-NEXT: and z1.d, z1.d, #0xffff
+; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
+; SVE-NEXT: sub z0.d, z1.d, z0.d
; SVE-NEXT: lsr z0.d, z0.d, #1
; SVE-NEXT: ret
;
@@ -937,10 +937,10 @@ define <vscale x 4 x i16> @rhadds_v4i16(<vscale x 4 x i16> %s0, <vscale x 4 x i1
; SVE-LABEL: rhadds_v4i16:
; SVE: // %bb.0: // %entry
; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: sxth z1.s, p0/m, z1.s
; SVE-NEXT: sxth z0.s, p0/m, z0.s
-; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
-; SVE-NEXT: sub z0.s, z0.s, z1.s
+; SVE-NEXT: sxth z1.s, p0/m, z1.s
+; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
+; SVE-NEXT: sub z0.s, z1.s, z0.s
; SVE-NEXT: asr z0.s, z0.s, #1
; SVE-NEXT: ret
;
@@ -965,10 +965,10 @@ define <vscale x 4 x i16> @rhadds_v4i16_lsh(<vscale x 4 x i16> %s0, <vscale x 4
; CHECK-LABEL: rhadds_v4i16_lsh:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: sxth z1.s, p0/m, z1.s
; CHECK-NEXT: sxth z0.s, p0/m, z0.s
-; CHECK-NEXT: subr z1.b, z1.b, #255 // =0xff
-; CHECK-NEXT: sub z0.s, z0.s, z1.s
+; CHECK-NEXT: sxth z1.s, p0/m, z1.s
+; CHECK-NEXT: subr z0.b, z0.b, #255 // =0xff
+; CHECK-NEXT: sub z0.s, z1.s, z0.s
; CHECK-NEXT: lsr z0.s, z0.s, #1
; CHECK-NEXT: ret
entry:
@@ -984,10 +984,10 @@ entry:
define <vscale x 4 x i16> @rhaddu_v4i16(<vscale x 4 x i16> %s0, <vscale x 4 x i16> %s1) {
; SVE-LABEL: rhaddu_v4i16:
; SVE: // %bb.0: // %entry
-; SVE-NEXT: and z1.s, z1.s, #0xffff
; SVE-NEXT: and z0.s, z0.s, #0xffff
-; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
-; SVE-NEXT: sub z0.s, z0.s, z1.s
+; SVE-NEXT: and z1.s, z1.s, #0xffff
+; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
+; SVE-NEXT: sub z0.s, z1.s, z0.s
; SVE-NEXT: lsr z0.s, z0.s, #1
; SVE-NEXT: ret
;
@@ -1084,10 +1084,10 @@ define <vscale x 4 x i8> @rhadds_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %
; SVE-LABEL: rhadds_v4i8:
; SVE: // %bb.0: // %entry
; SVE-NEXT: ptrue p0.s
-; SVE-NEXT: sxtb z1.s, p0/m, z1.s
; SVE-NEXT: sxtb z0.s, p0/m, z0.s
-; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
-; SVE-NEXT: sub z0.s, z0.s, z1.s
+; SVE-NEXT: sxtb z1.s, p0/m, z1.s
+; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
+; SVE-NEXT: sub z0.s, z1.s, z0.s
; SVE-NEXT: asr z0.s, z0.s, #1
; SVE-NEXT: ret
;
@@ -1112,10 +1112,10 @@ define <vscale x 4 x i8> @rhadds_v4i8_lsh(<vscale x 4 x i8> %s0, <vscale x 4 x i
; CHECK-LABEL: rhadds_v4i8_lsh:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ptrue p0.s
-; CHECK-NEXT: sxtb z1.s, p0/m, z1.s
; CHECK-NEXT: sxtb z0.s, p0/m, z0.s
-; CHECK-NEXT: subr z1.b, z1.b, #255 // =0xff
-; CHECK-NEXT: sub z0.s, z0.s, z1.s
+; CHECK-NEXT: sxtb z1.s, p0/m, z1.s
+; CHECK-NEXT: subr z0.b, z0.b, #255 // =0xff
+; CHECK-NEXT: sub z0.s, z1.s, z0.s
; CHECK-NEXT: and z0.s, z0.s, #0xffff
; CHECK-NEXT: lsr z0.s, z0.s, #1
; CHECK-NEXT: ret
@@ -1132,10 +1132,10 @@ entry:
define <vscale x 4 x i8> @rhaddu_v4i8(<vscale x 4 x i8> %s0, <vscale x 4 x i8> %s1) {
; SVE-LABEL: rhaddu_v4i8:
; SVE: // %bb.0: // %entry
-; SVE-NEXT: and z1.s, z1.s, #0xff
; SVE-NEXT: and z0.s, z0.s, #0xff
-; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
-; SVE-NEXT: sub z0.s, z0.s, z1.s
+; SVE-NEXT: and z1.s, z1.s, #0xff
+; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
+; SVE-NEXT: sub z0.s, z1.s, z0.s
; SVE-NEXT: lsr z0.s, z0.s, #1
; SVE-NEXT: ret
;
@@ -1160,10 +1160,10 @@ define <vscale x 8 x i8> @rhadds_v8i8(<vscale x 8 x i8> %s0, <vscale x 8 x i8> %
; SVE-LABEL: rhadds_v8i8:
; SVE: // %bb.0: // %entry
; SVE-NEXT: ptrue p0.h
-; SVE-NEXT: sxtb z1.h, p0/m, z1.h
; SVE-NEXT: sxtb z0.h, p0/m, z0.h
-; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
-; SVE-NEXT: sub z0.h, z0.h, z1.h
+; SVE-NEXT: sxtb z1.h, p0/m, z1.h
+; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
+; SVE-NEXT: sub z0.h, z1.h, z0.h
; SVE-NEXT: asr z0.h, z0.h, #1
; SVE-NEXT: ret
;
@@ -1188,10 +1188,10 @@ define <vscale x 8 x i8> @rhadds_v8i8_lsh(<vscale x 8 x i8> %s0, <vscale x 8 x i
; CHECK-LABEL: rhadds_v8i8_lsh:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: ptrue p0.h
-; CHECK-NEXT: sxtb z1.h, p0/m, z1.h
; CHECK-NEXT: sxtb z0.h, p0/m, z0.h
-; CHECK-NEXT: subr z1.b, z1.b, #255 // =0xff
-; CHECK-NEXT: sub z0.h, z0.h, z1.h
+; CHECK-NEXT: sxtb z1.h, p0/m, z1.h
+; CHECK-NEXT: subr z0.b, z0.b, #255 // =0xff
+; CHECK-NEXT: sub z0.h, z1.h, z0.h
; CHECK-NEXT: lsr z0.h, z0.h, #1
; CHECK-NEXT: ret
entry:
@@ -1207,10 +1207,10 @@ entry:
define <vscale x 8 x i8> @rhaddu_v8i8(<vscale x 8 x i8> %s0, <vscale x 8 x i8> %s1) {
; SVE-LABEL: rhaddu_v8i8:
; SVE: // %bb.0: // %entry
-; SVE-NEXT: and z1.h, z1.h, #0xff
; SVE-NEXT: and z0.h, z0.h, #0xff
-; SVE-NEXT: subr z1.b, z1.b, #255 // =0xff
-; SVE-NEXT: sub z0.h, z0.h, z1.h
+; SVE-NEXT: and z1.h, z1.h, #0xff
+; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
+; SVE-NEXT: sub z0.h, z1.h, z0.h
; SVE-NEXT: lsr z0.h, z0.h, #1
; SVE-NEXT: ret
;
@@ -1375,8 +1375,8 @@ define void @zext_mload_avgflooru(ptr %p1, ptr %p2, <vscale x 8 x i1> %mask) {
define void @zext_mload_avgceilu(ptr %p1, ptr %p2, <vscale x 8 x i1> %mask) {
; SVE-LABEL: zext_mload_avgceilu:
; SVE: // %bb.0:
-; SVE-NEXT: ld1b { z0.h }, p0/z, [x1]
-; SVE-NEXT: ld1b { z1.h }, p0/z, [x0]
+; SVE-NEXT: ld1b { z0.h }, p0/z, [x0]
+; SVE-NEXT: ld1b { z1.h }, p0/z, [x1]
; SVE-NEXT: subr z0.b, z0.b, #255 // =0xff
; SVE-NEXT: sub z0.h, z1.h, z0.h
; SVE-NEXT: lsr z0.h, z0.h, #1
diff --git a/llvm/test/CodeGen/ARM/inc-of-add.ll b/llvm/test/CodeGen/ARM/inc-of-add.ll
index 4a1400371934c..3079dbceb9844 100644
--- a/llvm/test/CodeGen/ARM/inc-of-add.ll
+++ b/llvm/test/CodeGen/ARM/inc-of-add.ll
@@ -91,10 +91,11 @@ define i64 @scalar_i64(i64 %x, i64 %y) nounwind {
;
; THUMB6-LABEL: scalar_i64:
; THUMB6: @ %bb.0:
-; THUMB6-NEXT: mvns r3, r3
-; THUMB6-NEXT: mvns r2, r2
-; THUMB6-NEXT: subs r0, r0, r2
-; THUMB6-NEXT: sbcs r1, r3
+; THUMB6-NEXT: mvns r1, r1
+; THUMB6-NEXT: mvns r0, r0
+; THUMB6-NEXT: subs r0, r2, r0
+; THUMB6-NEXT: sbcs r3, r1
+; THUMB6-NEXT: mov r1, r3
; THUMB6-NEXT: bx lr
;
; THUMB78-LABEL: scalar_i64:
@@ -194,11 +195,11 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
;
; ARM78-LABEL: vector_i128_i8:
; ARM78: @ %bb.0:
-; ARM78-NEXT: mov r12, sp
-; ARM78-NEXT: vld1.64 {d16, d17}, [r12]
+; ARM78-NEXT: vmov d17, r2, r3
+; ARM78-NEXT: vmov d16, r0, r1
+; ARM78-NEXT: mov r0, sp
; ARM78-NEXT: vmvn q8, q8
-; ARM78-NEXT: vmov d19, r2, r3
-; ARM78-NEXT: vmov d18, r0, r1
+; ARM78-NEXT: vld1.64 {d18, d19}, [r0]
; ARM78-NEXT: vsub.i8 q8, q9, q8
; ARM78-NEXT: vmov r0, r1, d16
; ARM78-NEXT: vmov r2, r3, d17
@@ -289,11 +290,11 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
;
; THUMB78-LABEL: vector_i128_i8:
; THUMB78: @ %bb.0:
-; THUMB78-NEXT: mov r12, sp
-; THUMB78-NEXT: vld1.64 {d16, d17}, [r12]
+; THUMB78-NEXT: vmov d17, r2, r3
+; THUMB78-NEXT: vmov d16, r0, r1
+; THUMB78-NEXT: mov r0, sp
; THUMB78-NEXT: vmvn q8, q8
-; THUMB78-NEXT: vmov d19, r2, r3
-; THUMB78-NEXT: vmov d18, r0, r1
+; THUMB78-NEXT: vld1.64 {d18, d19}, [r0]
; THUMB78-NEXT: vsub.i8 q8, q9, q8
; THUMB78-NEXT: vmov r0, r1, d16
; THUMB78-NEXT: vmov r2, r3, d17
@@ -348,11 +349,11 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
;
; ARM78-LABEL: vector_i128_i16:
; ARM78: @ %bb.0:
-; ARM78-NEXT: mov r12, sp
-; ARM78-NEXT: vld1.64 {d16, d17}, [r12]
+; ARM78-NEXT: vmov d17, r2, r3
+; ARM78-NEXT: vmov d16, r0, r1
+; ARM78-NEXT: mov r0, sp
; ARM78-NEXT: vmvn q8, q8
-; ARM78-NEXT: vmov d19, r2, r3
-; ARM78-NEXT: vmov d18, r0, r1
+; ARM78-NEXT: vld1.64 {d18, d19}, [r0]
; ARM78-NEXT: vsub.i16 q8, q9, q8
; ARM78-NEXT: vmov r0, r1, d16
; ARM78-NEXT: vmov r2, r3, d17
@@ -403,11 +404,11 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
;
; THUMB78-LABEL: vector_i128_i16:
; THUMB78: @ %bb.0:
-; THUMB78-NEXT: mov r12, sp
-; THUMB78-NEXT: vld1.64 {d16, d17}, [r12]
+; THUMB78-NEXT: vmov d17, r2, r3
+; THUMB78-NEXT: vmov d16, r0, r1
+; THUMB78-NEXT: mov r0, sp
; THUMB78-NEXT: vmvn q8, q8
-; THUMB78-NEXT: vmov d19, r2, r3
-; THUMB78-NEXT: vmov d18, r0, r1
+; THUMB78-NEXT: vld1.64 {d18, d19}, [r0]
; THUMB78-NEXT: vsub.i16 q8, q9, q8
; THUMB78-NEXT: vmov r0, r1, d16
; THUMB78-NEXT: vmov r2, r3, d17
@@ -436,11 +437,11 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
;
; ARM78-LABEL: vector_i128_i32:
; ARM78: @ %bb.0:
-; ARM78-NEXT: mov r12, sp
-; ARM78-NEXT: vld1.64 {d16, d17}, [r12]
+; ARM78-NEXT: vmov d17, r2, r3
+; ARM78-NEXT: vmov d16, r0, r1
+; ARM78-NEXT: mov r0, sp
; ARM78-NEXT: vmvn q8, q8
-; ARM78-NEXT: vmov d19, r2, r3
-; ARM78-NEXT: vmov d18, r0, r1
+; ARM78-NEXT: vld1.64 {d18, d19}, [r0]
; ARM78-NEXT: vsub.i32 q8, q9, q8
; ARM78-NEXT: vmov r0, r1, d16
; ARM78-NEXT: vmov r2, r3, d17
@@ -465,11 +466,11 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
;
; THUMB78-LABEL: vector_i128_i32:
; THUMB78: @ %bb.0:
-; THUMB78-NEXT: mov r12, sp
-; THUMB78-NEXT: vld1.64 {d16, d17}, [r12]
+; THUMB78-NEXT: vmov d17, r2, r3
+; THUMB78-NEXT: vmov d16, r0, r1
+; THUMB78-NEXT: mov r0, sp
; THUMB78-NEXT: vmvn q8, q8
-; THUMB78-NEXT: vmov d19, r2, r3
-; THUMB78-NEXT: vmov d18, r0, r1
+; THUMB78-NEXT: vld1.64 {d18, d19}, [r0]
; THUMB78-NEXT: vsub.i32 q8, q9, q8
; THUMB78-NEXT: vmov r0, r1, d16
; THUMB78-NEXT: vmov r2, r3, d17
@@ -499,11 +500,11 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
;
; ARM78-LABEL: vector_i128_i64:
; ARM78: @ %bb.0:
-; ARM78-NEXT: mov r12, sp
-; ARM78-NEXT: vld1.64 {d16, d17}, [r12]
+; ARM78-NEXT: vmov d17, r2, r3
+; ARM78-NEXT: vmov d16, r0, r1
+; ARM78-NEXT: mov r0, sp
; ARM78-NEXT: vmvn q8, q8
-; ARM78-NEXT: vmov d19, r2, r3
-; ARM78-NEXT: vmov d18, r0, r1
+; ARM78-NEXT: vld1.64 {d18, d19}, [r0]
; ARM78-NEXT: vsub.i64 q8, q9, q8
; ARM78-NEXT: vmov r0, r1, d16
; ARM78-NEXT: vmov r2, r3, d17
@@ -512,27 +513,27 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; THUMB6-LABEL: vector_i128_i64:
; THUMB6: @ %bb.0:
; THUMB6-NEXT: push {r4, r5, r7, lr}
-; THUMB6-NEXT: ldr r4, [sp, #20]
-; THUMB6-NEXT: mvns r4, r4
+; THUMB6-NEXT: mvns r4, r1
+; THUMB6-NEXT: mvns r0, r0
+; THUMB6-NEXT: ldr r1, [sp, #20]
; THUMB6-NEXT: ldr r5, [sp, #16]
-; THUMB6-NEXT: mvns r5, r5
-; THUMB6-NEXT: subs r0, r0, r5
+; THUMB6-NEXT: subs r0, r5, r0
; THUMB6-NEXT: sbcs r1, r4
-; THUMB6-NEXT: ldr r4, [sp, #28]
-; THUMB6-NEXT: mvns r4, r4
+; THUMB6-NEXT: mvns r4, r3
+; THUMB6-NEXT: mvns r2, r2
+; THUMB6-NEXT: ldr r3, [sp, #28]
; THUMB6-NEXT: ldr r5, [sp, #24]
-; THUMB6-NEXT: mvns r5, r5
-; THUMB6-NEXT: subs r2, r2, r5
+; THUMB6-NEXT: subs r2, r5, r2
; THUMB6-NEXT: sbcs r3, r4
; THUMB6-NEXT: pop {r4, r5, r7, pc}
;
; THUMB78-LABEL: vector_i128_i64:
; THUMB78: @ %bb.0:
-; THUMB78-NEXT: mov r12, sp
-; THUMB78-NEXT: vld1.64 {d16, d17}, [r12]
+; THUMB78-NEXT: vmov d17, r2, r3
+; THUMB78-NEXT: vmov d16, r0, r1
+; THUMB78-NEXT: mov r0, sp
; THUMB78-NEXT: vmvn q8, q8
-; THUMB78-NEXT: vmov d19, r2, r3
-; THUMB78-NEXT: vmov d18, r0, r1
+; THUMB78-NEXT: vld1.64 {d18, d19}, [r0]
; THUMB78-NEXT: vsub.i64 q8, q9, q8
; THUMB78-NEXT: vmov r0, r1, d16
; THUMB78-NEXT: vmov r2, r3, d17
diff --git a/llvm/test/CodeGen/PowerPC/inc-of-add.ll b/llvm/test/CodeGen/PowerPC/inc-of-add.ll
index 20702fb08dac6..432b5a6b362fe 100644
--- a/llvm/test/CodeGen/PowerPC/inc-of-add.ll
+++ b/llvm/test/CodeGen/PowerPC/inc-of-add.ll
@@ -70,59 +70,59 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; PPC32-NEXT: lbz 4, 115(1)
; PPC32-NEXT: lbz 22, 119(1)
; PPC32-NEXT: lbz 21, 123(1)
-; PPC32-NEXT: add 4, 5, 4
-; PPC32-NEXT: add 5, 6, 22
+; PPC32-NEXT: add 4, 4, 5
+; PPC32-NEXT: add 5, 22, 6
; PPC32-NEXT: lbz 22, 131(1)
-; PPC32-NEXT: add 6, 7, 21
+; PPC32-NEXT: add 6, 21, 7
; PPC32-NEXT: lbz 21, 135(1)
; PPC32-NEXT: addi 6, 6, 1
; PPC32-NEXT: stw 20, 16(1) # 4-byte Folded Spill
-; PPC32-NEXT: add 9, 9, 22
+; PPC32-NEXT: add 9, 22, 9
; PPC32-NEXT: lbz 20, 127(1)
-; PPC32-NEXT: add 10, 10, 21
+; PPC32-NEXT: add 10, 21, 10
; PPC32-NEXT: stw 25, 36(1) # 4-byte Folded Spill
; PPC32-NEXT: addi 5, 5, 1
-; PPC32-NEXT: lbz 25, 147(1)
-; PPC32-NEXT: add 7, 8, 20
-; PPC32-NEXT: lbz 21, 83(1)
+; PPC32-NEXT: lbz 25, 83(1)
+; PPC32-NEXT: add 7, 20, 8
+; PPC32-NEXT: lbz 21, 147(1)
; PPC32-NEXT: addi 7, 7, 1
; PPC32-NEXT: stw 24, 32(1) # 4-byte Folded Spill
; PPC32-NEXT: addi 4, 4, 1
-; PPC32-NEXT: lbz 24, 143(1)
+; PPC32-NEXT: lbz 24, 79(1)
; PPC32-NEXT: add 25, 21, 25
-; PPC32-NEXT: lbz 22, 79(1)
+; PPC32-NEXT: lbz 22, 143(1)
; PPC32-NEXT: stw 23, 28(1) # 4-byte Folded Spill
-; PPC32-NEXT: lbz 23, 139(1)
+; PPC32-NEXT: lbz 23, 75(1)
; PPC32-NEXT: add 24, 22, 24
-; PPC32-NEXT: lbz 8, 75(1)
+; PPC32-NEXT: lbz 8, 139(1)
; PPC32-NEXT: stw 28, 48(1) # 4-byte Folded Spill
-; PPC32-NEXT: lbz 28, 159(1)
+; PPC32-NEXT: lbz 28, 95(1)
; PPC32-NEXT: add 8, 8, 23
-; PPC32-NEXT: lbz 21, 95(1)
+; PPC32-NEXT: lbz 21, 159(1)
; PPC32-NEXT: addi 8, 8, 1
; PPC32-NEXT: stw 27, 44(1) # 4-byte Folded Spill
-; PPC32-NEXT: lbz 27, 155(1)
+; PPC32-NEXT: lbz 27, 91(1)
; PPC32-NEXT: add 28, 21, 28
-; PPC32-NEXT: lbz 22, 91(1)
+; PPC32-NEXT: lbz 22, 155(1)
; PPC32-NEXT: stw 26, 40(1) # 4-byte Folded Spill
-; PPC32-NEXT: lbz 26, 151(1)
+; PPC32-NEXT: lbz 26, 87(1)
; PPC32-NEXT: add 27, 22, 27
-; PPC32-NEXT: lbz 23, 87(1)
-; PPC32-NEXT: lbz 11, 175(1)
-; PPC32-NEXT: lbz 21, 111(1)
+; PPC32-NEXT: lbz 23, 151(1)
+; PPC32-NEXT: lbz 11, 111(1)
+; PPC32-NEXT: lbz 21, 175(1)
; PPC32-NEXT: add 26, 23, 26
-; PPC32-NEXT: lbz 12, 171(1)
-; PPC32-NEXT: lbz 0, 107(1)
+; PPC32-NEXT: lbz 12, 107(1)
+; PPC32-NEXT: lbz 0, 171(1)
; PPC32-NEXT: add 11, 21, 11
; PPC32-NEXT: stw 30, 56(1) # 4-byte Folded Spill
; PPC32-NEXT: addi 11, 11, 1
-; PPC32-NEXT: lbz 30, 167(1)
+; PPC32-NEXT: lbz 30, 103(1)
; PPC32-NEXT: add 12, 0, 12
-; PPC32-NEXT: lbz 22, 103(1)
+; PPC32-NEXT: lbz 22, 167(1)
; PPC32-NEXT: stw 29, 52(1) # 4-byte Folded Spill
-; PPC32-NEXT: lbz 29, 163(1)
+; PPC32-NEXT: lbz 29, 99(1)
; PPC32-NEXT: add 30, 22, 30
-; PPC32-NEXT: lbz 23, 99(1)
+; PPC32-NEXT: lbz 23, 163(1)
; PPC32-NEXT: stb 11, 15(3)
; PPC32-NEXT: addi 11, 12, 1
; PPC32-NEXT: add 29, 23, 29
@@ -179,50 +179,50 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; AIX-PPC64-NEXT: std 31, -8(1) # 8-byte Folded Spill
; AIX-PPC64-NEXT: lbz 24, 199(1)
; AIX-PPC64-NEXT: lbz 25, 191(1)
-; AIX-PPC64-NEXT: add 6, 6, 23
+; AIX-PPC64-NEXT: add 6, 23, 6
; AIX-PPC64-NEXT: lbz 23, 231(1)
-; AIX-PPC64-NEXT: add 5, 5, 24
+; AIX-PPC64-NEXT: add 5, 24, 5
; AIX-PPC64-NEXT: lbz 24, 223(1)
-; AIX-PPC64-NEXT: add 4, 4, 25
+; AIX-PPC64-NEXT: add 4, 25, 4
; AIX-PPC64-NEXT: lbz 25, 215(1)
-; AIX-PPC64-NEXT: add 9, 9, 23
-; AIX-PPC64-NEXT: lbz 27, 255(1)
-; AIX-PPC64-NEXT: add 8, 8, 24
-; AIX-PPC64-NEXT: lbz 23, 127(1)
-; AIX-PPC64-NEXT: add 7, 7, 25
-; AIX-PPC64-NEXT: lbz 26, 247(1)
+; AIX-PPC64-NEXT: add 9, 23, 9
+; AIX-PPC64-NEXT: lbz 27, 127(1)
+; AIX-PPC64-NEXT: add 8, 24, 8
+; AIX-PPC64-NEXT: lbz 23, 255(1)
+; AIX-PPC64-NEXT: add 7, 25, 7
+; AIX-PPC64-NEXT: lbz 26, 119(1)
; AIX-PPC64-NEXT: addi 9, 9, 1
-; AIX-PPC64-NEXT: lbz 24, 119(1)
+; AIX-PPC64-NEXT: lbz 24, 247(1)
; AIX-PPC64-NEXT: add 27, 23, 27
; AIX-PPC64-NEXT: lbz 25, 239(1)
; AIX-PPC64-NEXT: addi 8, 8, 1
-; AIX-PPC64-NEXT: lbz 30, 279(1)
+; AIX-PPC64-NEXT: lbz 30, 151(1)
; AIX-PPC64-NEXT: add 26, 24, 26
-; AIX-PPC64-NEXT: lbz 23, 151(1)
-; AIX-PPC64-NEXT: add 10, 10, 25
-; AIX-PPC64-NEXT: lbz 29, 271(1)
+; AIX-PPC64-NEXT: lbz 23, 279(1)
+; AIX-PPC64-NEXT: add 10, 25, 10
+; AIX-PPC64-NEXT: lbz 29, 143(1)
; AIX-PPC64-NEXT: addi 10, 10, 1
-; AIX-PPC64-NEXT: lbz 24, 143(1)
+; AIX-PPC64-NEXT: lbz 24, 271(1)
; AIX-PPC64-NEXT: add 30, 23, 30
-; AIX-PPC64-NEXT: lbz 28, 263(1)
+; AIX-PPC64-NEXT: lbz 28, 135(1)
; AIX-PPC64-NEXT: addi 7, 7, 1
-; AIX-PPC64-NEXT: lbz 25, 135(1)
+; AIX-PPC64-NEXT: lbz 25, 263(1)
; AIX-PPC64-NEXT: add 29, 24, 29
-; AIX-PPC64-NEXT: lbz 11, 311(1)
+; AIX-PPC64-NEXT: lbz 11, 183(1)
; AIX-PPC64-NEXT: addi 6, 6, 1
-; AIX-PPC64-NEXT: lbz 23, 183(1)
+; AIX-PPC64-NEXT: lbz 23, 311(1)
; AIX-PPC64-NEXT: add 28, 25, 28
-; AIX-PPC64-NEXT: lbz 12, 303(1)
+; AIX-PPC64-NEXT: lbz 12, 175(1)
; AIX-PPC64-NEXT: addi 5, 5, 1
-; AIX-PPC64-NEXT: lbz 0, 175(1)
+; AIX-PPC64-NEXT: lbz 0, 303(1)
; AIX-PPC64-NEXT: add 11, 23, 11
-; AIX-PPC64-NEXT: lbz 2, 295(1)
+; AIX-PPC64-NEXT: lbz 2, 167(1)
; AIX-PPC64-NEXT: addi 11, 11, 1
-; AIX-PPC64-NEXT: lbz 24, 167(1)
+; AIX-PPC64-NEXT: lbz 24, 295(1)
; AIX-PPC64-NEXT: add 12, 0, 12
-; AIX-PPC64-NEXT: lbz 31, 287(1)
+; AIX-PPC64-NEXT: lbz 31, 159(1)
; AIX-PPC64-NEXT: addi 4, 4, 1
-; AIX-PPC64-NEXT: lbz 25, 159(1)
+; AIX-PPC64-NEXT: lbz 25, 287(1)
; AIX-PPC64-NEXT: add 2, 24, 2
; AIX-PPC64-NEXT: stb 11, 15(3)
; AIX-PPC64-NEXT: addi 11, 12, 1
@@ -263,8 +263,8 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
;
; PPC64LE-LABEL: vector_i128_i8:
; PPC64LE: # %bb.0:
-; PPC64LE-NEXT: xxlnor 35, 35, 35
-; PPC64LE-NEXT: vsububm 2, 2, 3
+; PPC64LE-NEXT: xxlnor 34, 34, 34
+; PPC64LE-NEXT: vsububm 2, 3, 2
; PPC64LE-NEXT: blr
%t0 = add <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
%t1 = add <16 x i8> %y, %t0
@@ -283,20 +283,20 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; PPC32-NEXT: lhz 11, 70(1)
; PPC32-NEXT: lhz 12, 66(1)
; PPC32-NEXT: lhz 0, 62(1)
-; PPC32-NEXT: add 10, 10, 11
+; PPC32-NEXT: add 10, 11, 10
; PPC32-NEXT: lhz 30, 58(1)
-; PPC32-NEXT: add 9, 9, 12
+; PPC32-NEXT: add 9, 12, 9
; PPC32-NEXT: lhz 29, 50(1)
-; PPC32-NEXT: add 8, 8, 0
+; PPC32-NEXT: add 8, 0, 8
; PPC32-NEXT: lhz 28, 42(1)
-; PPC32-NEXT: add 7, 7, 30
+; PPC32-NEXT: add 7, 30, 7
; PPC32-NEXT: lhz 27, 46(1)
-; PPC32-NEXT: add 5, 5, 29
+; PPC32-NEXT: add 5, 29, 5
; PPC32-NEXT: lhz 26, 54(1)
-; PPC32-NEXT: add 3, 3, 28
-; PPC32-NEXT: add 4, 4, 27
+; PPC32-NEXT: add 3, 28, 3
+; PPC32-NEXT: add 4, 27, 4
; PPC32-NEXT: addi 3, 3, 1
-; PPC32-NEXT: add 6, 6, 26
+; PPC32-NEXT: add 6, 26, 6
; PPC32-NEXT: addi 4, 4, 1
; PPC32-NEXT: addi 5, 5, 1
; PPC32-NEXT: addi 6, 6, 1
@@ -320,24 +320,24 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; AIX-PPC64-NEXT: std 30, -16(1) # 8-byte Folded Spill
; AIX-PPC64-NEXT: std 31, -8(1) # 8-byte Folded Spill
; AIX-PPC64-NEXT: std 2, -48(1) # 8-byte Folded Spill
-; AIX-PPC64-NEXT: lhz 11, 182(1)
-; AIX-PPC64-NEXT: lhz 12, 118(1)
+; AIX-PPC64-NEXT: lhz 11, 118(1)
+; AIX-PPC64-NEXT: lhz 12, 182(1)
; AIX-PPC64-NEXT: lhz 0, 174(1)
; AIX-PPC64-NEXT: lhz 2, 166(1)
; AIX-PPC64-NEXT: add 11, 12, 11
; AIX-PPC64-NEXT: lhz 31, 158(1)
-; AIX-PPC64-NEXT: add 10, 10, 0
+; AIX-PPC64-NEXT: add 10, 0, 10
; AIX-PPC64-NEXT: lhz 30, 142(1)
-; AIX-PPC64-NEXT: add 9, 9, 2
+; AIX-PPC64-NEXT: add 9, 2, 9
; AIX-PPC64-NEXT: lhz 29, 126(1)
-; AIX-PPC64-NEXT: add 8, 8, 31
+; AIX-PPC64-NEXT: add 8, 31, 8
; AIX-PPC64-NEXT: lhz 28, 134(1)
-; AIX-PPC64-NEXT: add 6, 6, 30
+; AIX-PPC64-NEXT: add 6, 30, 6
; AIX-PPC64-NEXT: lhz 27, 150(1)
-; AIX-PPC64-NEXT: add 4, 4, 29
-; AIX-PPC64-NEXT: add 5, 5, 28
+; AIX-PPC64-NEXT: add 4, 29, 4
+; AIX-PPC64-NEXT: add 5, 28, 5
; AIX-PPC64-NEXT: addi 11, 11, 1
-; AIX-PPC64-NEXT: add 7, 7, 27
+; AIX-PPC64-NEXT: add 7, 27, 7
; AIX-PPC64-NEXT: addi 10, 10, 1
; AIX-PPC64-NEXT: addi 9, 9, 1
; AIX-PPC64-NEXT: addi 8, 8, 1
@@ -363,8 +363,8 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
;
; PPC64LE-LABEL: vector_i128_i16:
; PPC64LE: # %bb.0:
-; PPC64LE-NEXT: xxlnor 35, 35, 35
-; PPC64LE-NEXT: vsubuhm 2, 2, 3
+; PPC64LE-NEXT: xxlnor 34, 34, 34
+; PPC64LE-NEXT: vsubuhm 2, 3, 2
; PPC64LE-NEXT: blr
%t0 = add <8 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
%t1 = add <8 x i16> %y, %t0
@@ -374,10 +374,10 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
; PPC32-LABEL: vector_i128_i32:
; PPC32: # %bb.0:
-; PPC32-NEXT: add 3, 3, 7
-; PPC32-NEXT: add 4, 4, 8
-; PPC32-NEXT: add 5, 5, 9
-; PPC32-NEXT: add 6, 6, 10
+; PPC32-NEXT: add 3, 7, 3
+; PPC32-NEXT: add 4, 8, 4
+; PPC32-NEXT: add 5, 9, 5
+; PPC32-NEXT: add 6, 10, 6
; PPC32-NEXT: addi 3, 3, 1
; PPC32-NEXT: addi 4, 4, 1
; PPC32-NEXT: addi 5, 5, 1
@@ -386,10 +386,10 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
;
; PPC64BE-LABEL: vector_i128_i32:
; PPC64BE: # %bb.0:
-; PPC64BE-NEXT: add 6, 6, 10
-; PPC64BE-NEXT: add 5, 5, 9
-; PPC64BE-NEXT: add 4, 4, 8
-; PPC64BE-NEXT: add 3, 3, 7
+; PPC64BE-NEXT: add 6, 10, 6
+; PPC64BE-NEXT: add 5, 9, 5
+; PPC64BE-NEXT: add 4, 8, 4
+; PPC64BE-NEXT: add 3, 7, 3
; PPC64BE-NEXT: addi 6, 6, 1
; PPC64BE-NEXT: addi 5, 5, 1
; PPC64BE-NEXT: addi 4, 4, 1
@@ -398,8 +398,8 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
;
; PPC64LE-LABEL: vector_i128_i32:
; PPC64LE: # %bb.0:
-; PPC64LE-NEXT: xxlnor 35, 35, 35
-; PPC64LE-NEXT: vsubuwm 2, 2, 3
+; PPC64LE-NEXT: xxlnor 34, 34, 34
+; PPC64LE-NEXT: vsubuwm 2, 3, 2
; PPC64LE-NEXT: blr
%t0 = add <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1>
%t1 = add <4 x i32> %y, %t0
@@ -409,28 +409,28 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; PPC32-LABEL: vector_i128_i64:
; PPC32: # %bb.0:
-; PPC32-NEXT: not 8, 8
-; PPC32-NEXT: not 7, 7
-; PPC32-NEXT: subc 4, 4, 8
-; PPC32-NEXT: subfe 3, 7, 3
-; PPC32-NEXT: not 7, 10
-; PPC32-NEXT: not 9, 9
-; PPC32-NEXT: subc 6, 6, 7
-; PPC32-NEXT: subfe 5, 9, 5
+; PPC32-NEXT: not 4, 4
+; PPC32-NEXT: not 3, 3
+; PPC32-NEXT: subc 4, 8, 4
+; PPC32-NEXT: subfe 3, 3, 7
+; PPC32-NEXT: not 6, 6
+; PPC32-NEXT: not 5, 5
+; PPC32-NEXT: subc 6, 10, 6
+; PPC32-NEXT: subfe 5, 5, 9
; PPC32-NEXT: blr
;
; PPC64BE-LABEL: vector_i128_i64:
; PPC64BE: # %bb.0:
-; PPC64BE-NEXT: add 3, 3, 5
-; PPC64BE-NEXT: add 4, 4, 6
+; PPC64BE-NEXT: add 3, 5, 3
+; PPC64BE-NEXT: add 4, 6, 4
; PPC64BE-NEXT: addi 3, 3, 1
; PPC64BE-NEXT: addi 4, 4, 1
; PPC64BE-NEXT: blr
;
; PPC64LE-LABEL: vector_i128_i64:
; PPC64LE: # %bb.0:
-; PPC64LE-NEXT: xxlnor 35, 35, 35
-; PPC64LE-NEXT: vsubudm 2, 2, 3
+; PPC64LE-NEXT: xxlnor 34, 34, 34
+; PPC64LE-NEXT: vsubudm 2, 3, 2
; PPC64LE-NEXT: blr
%t0 = add <2 x i64> %x, <i64 1, i64 1>
%t1 = add <2 x i64> %y, %t0
diff --git a/llvm/test/CodeGen/PowerPC/vavg.ll b/llvm/test/CodeGen/PowerPC/vavg.ll
index 7f6a91ad5fd4f..e473cbcd3f281 100644
--- a/llvm/test/CodeGen/PowerPC/vavg.ll
+++ b/llvm/test/CodeGen/PowerPC/vavg.ll
@@ -197,25 +197,25 @@ entry:
define <4 x i32> @test_v4i32_negative(<4 x i32> %m, <4 x i32> %n) {
; CHECK-P9-LABEL: test_v4i32_negative:
; CHECK-P9: # %bb.0: # %entry
-; CHECK-P9-NEXT: xxlnor 35, 35, 35
-; CHECK-P9-NEXT: vsubuwm 2, 2, 3
+; CHECK-P9-NEXT: xxlnor 34, 34, 34
+; CHECK-P9-NEXT: vsubuwm 2, 3, 2
; CHECK-P9-NEXT: vspltisw 3, 2
; CHECK-P9-NEXT: vsrw 2, 2, 3
; CHECK-P9-NEXT: blr
;
; CHECK-P8-LABEL: test_v4i32_negative:
; CHECK-P8: # %bb.0: # %entry
-; CHECK-P8-NEXT: xxlnor 35, 35, 35
+; CHECK-P8-NEXT: xxlnor 34, 34, 34
; CHECK-P8-NEXT: vspltisw 4, 2
-; CHECK-P8-NEXT: vsubuwm 2, 2, 3
+; CHECK-P8-NEXT: vsubuwm 2, 3, 2
; CHECK-P8-NEXT: vsrw 2, 2, 4
; CHECK-P8-NEXT: blr
;
; CHECK-P7-LABEL: test_v4i32_negative:
; CHECK-P7: # %bb.0: # %entry
-; CHECK-P7-NEXT: xxlnor 35, 35, 35
+; CHECK-P7-NEXT: xxlnor 34, 34, 34
; CHECK-P7-NEXT: vspltisw 4, 2
-; CHECK-P7-NEXT: vsubuwm 2, 2, 3
+; CHECK-P7-NEXT: vsubuwm 2, 3, 2
; CHECK-P7-NEXT: vsrw 2, 2, 4
; CHECK-P7-NEXT: blr
entry:
@@ -262,25 +262,25 @@ entry:
define <8 x i16> @test_v8i16_overflow(<8 x i16> %m, <8 x i16> %n) {
; CHECK-P9-LABEL: test_v8i16_overflow:
; CHECK-P9: # %bb.0: # %entry
-; CHECK-P9-NEXT: xxlnor 35, 35, 35
+; CHECK-P9-NEXT: xxlnor 34, 34, 34
; CHECK-P9-NEXT: vspltish 4, 1
-; CHECK-P9-NEXT: vsubuhm 2, 2, 3
+; CHECK-P9-NEXT: vsubuhm 2, 3, 2
; CHECK-P9-NEXT: vsrh 2, 2, 4
; CHECK-P9-NEXT: blr
;
; CHECK-P8-LABEL: test_v8i16_overflow:
; CHECK-P8: # %bb.0: # %entry
-; CHECK-P8-NEXT: xxlnor 35, 35, 35
+; CHECK-P8-NEXT: xxlnor 34, 34, 34
; CHECK-P8-NEXT: vspltish 4, 1
-; CHECK-P8-NEXT: vsubuhm 2, 2, 3
+; CHECK-P8-NEXT: vsubuhm 2, 3, 2
; CHECK-P8-NEXT: vsrh 2, 2, 4
; CHECK-P8-NEXT: blr
;
; CHECK-P7-LABEL: test_v8i16_overflow:
; CHECK-P7: # %bb.0: # %entry
-; CHECK-P7-NEXT: xxlnor 35, 35, 35
+; CHECK-P7-NEXT: xxlnor 34, 34, 34
; CHECK-P7-NEXT: vspltish 4, 1
-; CHECK-P7-NEXT: vsubuhm 2, 2, 3
+; CHECK-P7-NEXT: vsubuhm 2, 3, 2
; CHECK-P7-NEXT: vsrh 2, 2, 4
; CHECK-P7-NEXT: blr
entry:
@@ -293,25 +293,25 @@ entry:
define <4 x i32> @test_v4i32_overflow(<4 x i32> %m, <4 x i32> %n) {
; CHECK-P9-LABEL: test_v4i32_overflow:
; CHECK-P9: # %bb.0: # %entry
-; CHECK-P9-NEXT: xxlnor 35, 35, 35
+; CHECK-P9-NEXT: xxlnor 34, 34, 34
; CHECK-P9-NEXT: vspltisw 4, 1
-; CHECK-P9-NEXT: vsubuwm 2, 2, 3
+; CHECK-P9-NEXT: vsubuwm 2, 3, 2
; CHECK-P9-NEXT: vsrw 2, 2, 4
; CHECK-P9-NEXT: blr
;
; CHECK-P8-LABEL: test_v4i32_overflow:
; CHECK-P8: # %bb.0: # %entry
-; CHECK-P8-NEXT: xxlnor 35, 35, 35
+; CHECK-P8-NEXT: xxlnor 34, 34, 34
; CHECK-P8-NEXT: vspltisw 4, 1
-; CHECK-P8-NEXT: vsubuwm 2, 2, 3
+; CHECK-P8-NEXT: vsubuwm 2, 3, 2
; CHECK-P8-NEXT: vsrw 2, 2, 4
; CHECK-P8-NEXT: blr
;
; CHECK-P7-LABEL: test_v4i32_overflow:
; CHECK-P7: # %bb.0: # %entry
-; CHECK-P7-NEXT: xxlnor 35, 35, 35
+; CHECK-P7-NEXT: xxlnor 34, 34, 34
; CHECK-P7-NEXT: vspltisw 4, 1
-; CHECK-P7-NEXT: vsubuwm 2, 2, 3
+; CHECK-P7-NEXT: vsubuwm 2, 3, 2
; CHECK-P7-NEXT: vsrw 2, 2, 4
; CHECK-P7-NEXT: blr
entry:
@@ -324,25 +324,25 @@ entry:
define <16 x i8> @test_v16i8_overflow(<16 x i8> %m, <16 x i8> %n) {
; CHECK-P9-LABEL: test_v16i8_overflow:
; CHECK-P9: # %bb.0: # %entry
-; CHECK-P9-NEXT: xxlnor 35, 35, 35
+; CHECK-P9-NEXT: xxlnor 34, 34, 34
; CHECK-P9-NEXT: xxspltib 36, 1
-; CHECK-P9-NEXT: vsububm 2, 2, 3
+; CHECK-P9-NEXT: vsububm 2, 3, 2
; CHECK-P9-NEXT: vsrb 2, 2, 4
; CHECK-P9-NEXT: blr
;
; CHECK-P8-LABEL: test_v16i8_overflow:
; CHECK-P8: # %bb.0: # %entry
-; CHECK-P8-NEXT: xxlnor 35, 35, 35
+; CHECK-P8-NEXT: xxlnor 34, 34, 34
; CHECK-P8-NEXT: vspltisb 4, 1
-; CHECK-P8-NEXT: vsububm 2, 2, 3
+; CHECK-P8-NEXT: vsububm 2, 3, 2
; CHECK-P8-NEXT: vsrb 2, 2, 4
; CHECK-P8-NEXT: blr
;
; CHECK-P7-LABEL: test_v16i8_overflow:
; CHECK-P7: # %bb.0: # %entry
-; CHECK-P7-NEXT: xxlnor 35, 35, 35
+; CHECK-P7-NEXT: xxlnor 34, 34, 34
; CHECK-P7-NEXT: vspltisb 4, 1
-; CHECK-P7-NEXT: vsububm 2, 2, 3
+; CHECK-P7-NEXT: vsububm 2, 3, 2
; CHECK-P7-NEXT: vsrb 2, 2, 4
; CHECK-P7-NEXT: blr
entry:
diff --git a/llvm/test/CodeGen/Thumb2/mve-halving.ll b/llvm/test/CodeGen/Thumb2/mve-halving.ll
index 10c968a1b01b9..524818e8d210d 100644
--- a/llvm/test/CodeGen/Thumb2/mve-halving.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-halving.ll
@@ -233,8 +233,8 @@ define arm_aapcs_vfpcc <4 x i32> @vhsubu_v4i32_nw(<4 x i32> %x, <4 x i32> %y) {
define arm_aapcs_vfpcc <16 x i8> @vrhadds_v16i8(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: vrhadds_v16i8:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i8 q0, q0, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i8 q0, q1, q0
; CHECK-NEXT: vshr.s8 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add <16 x i8> %x, %y
@@ -245,8 +245,8 @@ define arm_aapcs_vfpcc <16 x i8> @vrhadds_v16i8(<16 x i8> %x, <16 x i8> %y) {
define arm_aapcs_vfpcc <16 x i8> @vrhaddu_v16i8(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: vrhaddu_v16i8:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i8 q0, q0, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i8 q0, q1, q0
; CHECK-NEXT: vshr.u8 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add <16 x i8> %x, %y
@@ -257,8 +257,8 @@ define arm_aapcs_vfpcc <16 x i8> @vrhaddu_v16i8(<16 x i8> %x, <16 x i8> %y) {
define arm_aapcs_vfpcc <8 x i16> @vrhadds_v8i16(<8 x i16> %x, <8 x i16> %y) {
; CHECK-LABEL: vrhadds_v8i16:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i16 q0, q0, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i16 q0, q1, q0
; CHECK-NEXT: vshr.s16 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add <8 x i16> %x, %y
@@ -269,8 +269,8 @@ define arm_aapcs_vfpcc <8 x i16> @vrhadds_v8i16(<8 x i16> %x, <8 x i16> %y) {
define arm_aapcs_vfpcc <8 x i16> @vrhaddu_v8i16(<8 x i16> %x, <8 x i16> %y) {
; CHECK-LABEL: vrhaddu_v8i16:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i16 q0, q0, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i16 q0, q1, q0
; CHECK-NEXT: vshr.u16 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add <8 x i16> %x, %y
@@ -281,8 +281,8 @@ define arm_aapcs_vfpcc <8 x i16> @vrhaddu_v8i16(<8 x i16> %x, <8 x i16> %y) {
define arm_aapcs_vfpcc <4 x i32> @vrhadds_v4i32(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: vrhadds_v4i32:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i32 q0, q0, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i32 q0, q1, q0
; CHECK-NEXT: vshr.s32 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add <4 x i32> %x, %y
@@ -293,8 +293,8 @@ define arm_aapcs_vfpcc <4 x i32> @vrhadds_v4i32(<4 x i32> %x, <4 x i32> %y) {
define arm_aapcs_vfpcc <4 x i32> @vrhaddu_v4i32(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: vrhaddu_v4i32:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i32 q0, q0, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i32 q0, q1, q0
; CHECK-NEXT: vshr.u32 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add <4 x i32> %x, %y
@@ -305,8 +305,8 @@ define arm_aapcs_vfpcc <4 x i32> @vrhaddu_v4i32(<4 x i32> %x, <4 x i32> %y) {
define arm_aapcs_vfpcc <16 x i8> @vrhadds_v16i8_nwop(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: vrhadds_v16i8_nwop:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i8 q0, q0, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i8 q0, q1, q0
; CHECK-NEXT: vshr.s8 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add nsw <16 x i8> %x, %y
@@ -317,8 +317,8 @@ define arm_aapcs_vfpcc <16 x i8> @vrhadds_v16i8_nwop(<16 x i8> %x, <16 x i8> %y)
define arm_aapcs_vfpcc <16 x i8> @vrhaddu_v16i8_nwop(<16 x i8> %x, <16 x i8> %y) {
; CHECK-LABEL: vrhaddu_v16i8_nwop:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i8 q0, q0, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i8 q0, q1, q0
; CHECK-NEXT: vshr.u8 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add nuw <16 x i8> %x, %y
@@ -329,8 +329,8 @@ define arm_aapcs_vfpcc <16 x i8> @vrhaddu_v16i8_nwop(<16 x i8> %x, <16 x i8> %y)
define arm_aapcs_vfpcc <8 x i16> @vrhadds_v8i16_nwop(<8 x i16> %x, <8 x i16> %y) {
; CHECK-LABEL: vrhadds_v8i16_nwop:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i16 q0, q0, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i16 q0, q1, q0
; CHECK-NEXT: vshr.s16 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add nsw <8 x i16> %x, %y
@@ -341,8 +341,8 @@ define arm_aapcs_vfpcc <8 x i16> @vrhadds_v8i16_nwop(<8 x i16> %x, <8 x i16> %y)
define arm_aapcs_vfpcc <8 x i16> @vrhaddu_v8i16_nwop(<8 x i16> %x, <8 x i16> %y) {
; CHECK-LABEL: vrhaddu_v8i16_nwop:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i16 q0, q0, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i16 q0, q1, q0
; CHECK-NEXT: vshr.u16 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add nuw <8 x i16> %x, %y
@@ -353,8 +353,8 @@ define arm_aapcs_vfpcc <8 x i16> @vrhaddu_v8i16_nwop(<8 x i16> %x, <8 x i16> %y)
define arm_aapcs_vfpcc <4 x i32> @vrhadds_v4i32_nwop(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: vrhadds_v4i32_nwop:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i32 q0, q0, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i32 q0, q1, q0
; CHECK-NEXT: vshr.s32 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add nsw <4 x i32> %x, %y
@@ -365,8 +365,8 @@ define arm_aapcs_vfpcc <4 x i32> @vrhadds_v4i32_nwop(<4 x i32> %x, <4 x i32> %y)
define arm_aapcs_vfpcc <4 x i32> @vrhaddu_v4i32_nwop(<4 x i32> %x, <4 x i32> %y) {
; CHECK-LABEL: vrhaddu_v4i32_nwop:
; CHECK: @ %bb.0:
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i32 q0, q0, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i32 q0, q1, q0
; CHECK-NEXT: vshr.u32 q0, q0, #1
; CHECK-NEXT: bx lr
%add = add nuw <4 x i32> %x, %y
diff --git a/llvm/test/CodeGen/Thumb2/mve-vhadd.ll b/llvm/test/CodeGen/Thumb2/mve-vhadd.ll
index 3fc88d7a93b61..207c0f377e34a 100644
--- a/llvm/test/CodeGen/Thumb2/mve-vhadd.ll
+++ b/llvm/test/CodeGen/Thumb2/mve-vhadd.ll
@@ -221,10 +221,10 @@ entry:
define arm_aapcs_vfpcc <4 x i16> @vrhadds_v4i16(<4 x i16> %s0, <4 x i16> %s1) {
; CHECK-LABEL: vrhadds_v4i16:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmovlb.s16 q1, q1
; CHECK-NEXT: vmovlb.s16 q0, q0
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i32 q0, q0, q1
+; CHECK-NEXT: vmovlb.s16 q1, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i32 q0, q1, q0
; CHECK-NEXT: vshr.u32 q0, q0, #1
; CHECK-NEXT: bx lr
entry:
@@ -287,12 +287,12 @@ entry:
define arm_aapcs_vfpcc <4 x i8> @vrhadds_v4i8(<4 x i8> %s0, <4 x i8> %s1) {
; CHECK-LABEL: vrhadds_v4i8:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmovlb.s8 q1, q1
; CHECK-NEXT: vmovlb.s8 q0, q0
-; CHECK-NEXT: vmovlb.s16 q1, q1
+; CHECK-NEXT: vmovlb.s8 q1, q1
; CHECK-NEXT: vmovlb.s16 q0, q0
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i32 q0, q0, q1
+; CHECK-NEXT: vmovlb.s16 q1, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i32 q0, q1, q0
; CHECK-NEXT: vmovlb.u16 q0, q0
; CHECK-NEXT: vshr.u32 q0, q0, #1
; CHECK-NEXT: bx lr
@@ -327,10 +327,10 @@ entry:
define arm_aapcs_vfpcc <8 x i8> @vrhadds_v8i8(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-LABEL: vrhadds_v8i8:
; CHECK: @ %bb.0: @ %entry
-; CHECK-NEXT: vmovlb.s8 q1, q1
; CHECK-NEXT: vmovlb.s8 q0, q0
-; CHECK-NEXT: vmvn q1, q1
-; CHECK-NEXT: vsub.i16 q0, q0, q1
+; CHECK-NEXT: vmovlb.s8 q1, q1
+; CHECK-NEXT: vmvn q0, q0
+; CHECK-NEXT: vsub.i16 q0, q1, q0
; CHECK-NEXT: vshr.u16 q0, q0, #1
; CHECK-NEXT: bx lr
entry:
diff --git a/llvm/test/CodeGen/X86/inc-of-add.ll b/llvm/test/CodeGen/X86/inc-of-add.ll
index 1070f49ecf214..b07efed2b9199 100644
--- a/llvm/test/CodeGen/X86/inc-of-add.ll
+++ b/llvm/test/CodeGen/X86/inc-of-add.ll
@@ -102,8 +102,9 @@ define <16 x i8> @vector_i128_i8(<16 x i8> %x, <16 x i8> %y) nounwind {
; ALL-LABEL: vector_i128_i8:
; ALL: # %bb.0:
; ALL-NEXT: pcmpeqd %xmm2, %xmm2
-; ALL-NEXT: pxor %xmm1, %xmm2
-; ALL-NEXT: psubb %xmm2, %xmm0
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubb %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <16 x i8> %x, <i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1, i8 1>
%t1 = add <16 x i8> %y, %t0
@@ -114,8 +115,9 @@ define <8 x i16> @vector_i128_i16(<8 x i16> %x, <8 x i16> %y) nounwind {
; ALL-LABEL: vector_i128_i16:
; ALL: # %bb.0:
; ALL-NEXT: pcmpeqd %xmm2, %xmm2
-; ALL-NEXT: pxor %xmm1, %xmm2
-; ALL-NEXT: psubw %xmm2, %xmm0
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubw %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <8 x i16> %x, <i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1, i16 1>
%t1 = add <8 x i16> %y, %t0
@@ -126,8 +128,9 @@ define <4 x i32> @vector_i128_i32(<4 x i32> %x, <4 x i32> %y) nounwind {
; ALL-LABEL: vector_i128_i32:
; ALL: # %bb.0:
; ALL-NEXT: pcmpeqd %xmm2, %xmm2
-; ALL-NEXT: pxor %xmm1, %xmm2
-; ALL-NEXT: psubd %xmm2, %xmm0
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubd %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <4 x i32> %x, <i32 1, i32 1, i32 1, i32 1>
%t1 = add <4 x i32> %y, %t0
@@ -138,8 +141,9 @@ define <2 x i64> @vector_i128_i64(<2 x i64> %x, <2 x i64> %y) nounwind {
; ALL-LABEL: vector_i128_i64:
; ALL: # %bb.0:
; ALL-NEXT: pcmpeqd %xmm2, %xmm2
-; ALL-NEXT: pxor %xmm1, %xmm2
-; ALL-NEXT: psubq %xmm2, %xmm0
+; ALL-NEXT: pxor %xmm0, %xmm2
+; ALL-NEXT: psubq %xmm2, %xmm1
+; ALL-NEXT: movdqa %xmm1, %xmm0
; ALL-NEXT: ret{{[l|q]}}
%t0 = add <2 x i64> %x, <i64 1, i64 1>
%t1 = add <2 x i64> %y, %t0
More information about the llvm-commits
mailing list