[llvm] [AArch64][GlobalISel] Clamp TruncSat num elements. (PR #217248)
David Green via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 02:15:13 PDT 2026
https://github.com/davemgreen created https://github.com/llvm/llvm-project/pull/217248
Just like v2i32, we can clamp v8i8 and v4i16 vectors to make use of truncsat in more places and split larger vectors with fewerElements.
>From a2f2bd86a0594717afe5a6dfbea828a3ebef8440 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 19 Aug 2026 07:22:35 +0100
Subject: [PATCH 1/2] [AArch64][GlobalISel] Add lowering for s/umul.fix.sat
This extends the existing s/umul.fix lowering to handle the saturating
versions of the instructions. They use a TruncSSatS or TruncUSatU instead of
the trunc, keeping the implementation simple. That does require truncsat
lowering to be added to allow those operations to subsequently lower if needed.
---
.../llvm/CodeGen/GlobalISel/LegalizerHelper.h | 1 +
.../CodeGen/GlobalISel/LegalizerHelper.cpp | 67 +-
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 7 +-
.../GlobalISel/legalizer-info-validation.mir | 22 +-
llvm/test/CodeGen/AArch64/smul_fix_sat.ll | 1089 +++++++++++------
llvm/test/CodeGen/AArch64/umul_fix_sat.ll | 688 +++++++----
6 files changed, 1277 insertions(+), 597 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
index ce7bf3c49712f..de00cf8966970 100644
--- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
+++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerHelper.h
@@ -575,6 +575,7 @@ class LegalizerHelper {
LLVM_ABI LegalizeResult lowerAddSubSatToMinMax(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerAddSubSatToAddoSubo(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerShlSat(MachineInstr &MI);
+ LLVM_ABI LegalizeResult lowerTruncSat(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerBswap(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerBitreverse(MachineInstr &MI);
LLVM_ABI LegalizeResult lowerReadWriteRegister(MachineInstr &MI);
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
index 6f85390fd4ede..3a9de29c55224 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerHelper.cpp
@@ -5058,6 +5058,10 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
case G_SSHLSAT:
case G_USHLSAT:
return lowerShlSat(MI);
+ case G_TRUNC_SSAT_S:
+ case G_TRUNC_USAT_U:
+ case G_TRUNC_SSAT_U:
+ return lowerTruncSat(MI);
case G_ABS:
return lowerAbsToAddXor(MI);
case G_ABDS:
@@ -5113,6 +5117,8 @@ LegalizerHelper::lower(MachineInstr &MI, unsigned TypeIdx, LLT LowerHintTy) {
}
case G_SMULFIX:
case G_UMULFIX:
+ case G_SMULFIXSAT:
+ case G_UMULFIXSAT:
return lowerMulfix(MI);
}
}
@@ -10297,6 +10303,39 @@ LegalizerHelper::lowerShlSat(MachineInstr &MI) {
return Legalized;
}
+LegalizerHelper::LegalizeResult
+LegalizerHelper::lowerTruncSat(MachineInstr &MI) {
+ unsigned Opc = MI.getOpcode();
+ auto [Dst, DstTy, Src, SrcTy] = MI.getFirst2RegLLTs();
+ unsigned DstSize = DstTy.getScalarSizeInBits();
+ unsigned SrcSize = SrcTy.getScalarSizeInBits();
+
+ if (Opc == TargetOpcode::G_TRUNC_SSAT_S) {
+ auto Max = MIRBuilder.buildConstant(
+ SrcTy, APInt::getSignedMaxValue(DstSize).sext(SrcSize));
+ Src = MIRBuilder.buildSMin(SrcTy, Src, Max).getReg(0);
+ auto Min = MIRBuilder.buildConstant(
+ SrcTy, APInt::getSignedMinValue(DstSize).sext(SrcSize));
+ Src = MIRBuilder.buildSMax(SrcTy, Src, Min).getReg(0);
+ } else if (Opc == TargetOpcode::G_TRUNC_USAT_U) {
+ auto Max = MIRBuilder.buildConstant(
+ SrcTy, APInt::getAllOnes(DstSize).zext(SrcSize));
+ Src = MIRBuilder.buildUMin(SrcTy, Src, Max).getReg(0);
+ } else if (Opc == TargetOpcode::G_TRUNC_SSAT_U) {
+ auto Max = MIRBuilder.buildConstant(
+ SrcTy, APInt::getAllOnes(DstSize).zext(SrcSize));
+ Src = MIRBuilder.buildSMin(SrcTy, Src, Max).getReg(0);
+ auto Min = MIRBuilder.buildConstant(SrcTy, APInt::getZero(SrcSize));
+ Src = MIRBuilder.buildSMax(SrcTy, Src, Min).getReg(0);
+ } else {
+ llvm_unreachable("Expected truncsat opcode!");
+ }
+
+ MIRBuilder.buildTrunc(Dst, Src);
+ MI.eraseFromParent();
+ return Legalized;
+}
+
LegalizerHelper::LegalizeResult LegalizerHelper::lowerBswap(MachineInstr &MI) {
auto [Dst, Src] = MI.getFirst2Regs();
const LLT Ty = MRI.getType(Src);
@@ -10933,25 +10972,32 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerVAArg(MachineInstr &MI) {
}
LegalizerHelper::LegalizeResult LegalizerHelper::lowerMulfix(MachineInstr &MI) {
- [[maybe_unused]] unsigned OpCode = MI.getOpcode();
+ unsigned OpCode = MI.getOpcode();
assert((OpCode == TargetOpcode::G_SMULFIX ||
- OpCode == TargetOpcode::G_UMULFIX) &&
- "Operator must be either G_SMULFIX or G_UMULFIX!");
+ OpCode == TargetOpcode::G_UMULFIX ||
+ OpCode == TargetOpcode::G_SMULFIXSAT ||
+ OpCode == TargetOpcode::G_UMULFIXSAT) &&
+ "Operator must be either G_SMULFIX[SAT] or G_UMULFIX[SAT]!");
auto [Dst, LHS, RHS] = MI.getFirst3Regs();
LLT Ty = MRI.getType(Dst);
unsigned Scale = MI.getOperand(3).getImm();
- if (Scale == 0) {
+ bool Saturating = (OpCode == TargetOpcode::G_SMULFIXSAT ||
+ OpCode == TargetOpcode::G_UMULFIXSAT);
+ bool IsSigned = (OpCode == TargetOpcode::G_SMULFIX ||
+ OpCode == TargetOpcode::G_SMULFIXSAT);
+
+ if (!Saturating && Scale == 0) {
MIRBuilder.buildMul(Dst, LHS, RHS);
MI.eraseFromParent();
return Legalized;
}
- // TODO: Port other lowerng paths from SelectionDAG.
+ // TODO: Port other lowering paths from SelectionDAG.
LLT WideTy = Ty.changeElementSize(Ty.getScalarSizeInBits() * 2);
auto ShiftAmt = MIRBuilder.buildConstant(WideTy, Scale);
MachineInstrBuilder ExtLHS{}, ExtRHS{}, Shift{};
- if (MI.getOpcode() == TargetOpcode::G_SMULFIX) {
+ if (IsSigned) {
ExtLHS = MIRBuilder.buildSExt(WideTy, LHS);
ExtRHS = MIRBuilder.buildSExt(WideTy, RHS);
} else {
@@ -10960,12 +11006,17 @@ LegalizerHelper::LegalizeResult LegalizerHelper::lowerMulfix(MachineInstr &MI) {
}
auto Mul = MIRBuilder.buildMul(WideTy, ExtLHS, ExtRHS);
- if (MI.getOpcode() == TargetOpcode::G_SMULFIX)
+ if (IsSigned)
Shift = MIRBuilder.buildAShr(WideTy, Mul, ShiftAmt);
else
Shift = MIRBuilder.buildLShr(WideTy, Mul, ShiftAmt);
- MIRBuilder.buildTrunc(Dst, Shift);
+ if (!Saturating)
+ MIRBuilder.buildTrunc(Dst, Shift);
+ else if (IsSigned)
+ MIRBuilder.buildTruncSSatS(Dst, Shift);
+ else
+ MIRBuilder.buildTruncUSatU(Dst, Shift);
MI.eraseFromParent();
return Legalized;
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 1274aab686bae..91ad1911818f6 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -296,7 +296,9 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
.legalFor({i64, v16i8, v8i16, v4i32})
.lower();
- getActionDefinitionsBuilder({G_SMULFIX, G_UMULFIX}).lower();
+ getActionDefinitionsBuilder(
+ {G_SMULFIX, G_UMULFIX, G_SMULFIXSAT, G_UMULFIXSAT})
+ .lower();
getActionDefinitionsBuilder({G_SMIN, G_SMAX, G_UMIN, G_UMAX})
.legalFor({v8i8, v16i8, v4i16, v8i16, v2i32, v4i32})
@@ -858,7 +860,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
getActionDefinitionsBuilder({G_TRUNC_SSAT_S, G_TRUNC_SSAT_U, G_TRUNC_USAT_U})
.legalFor({{v8i8, v8i16}, {v4i16, v4i32}, {v2i32, v2i64}})
- .clampNumElements(0, v2s32, v2s32);
+ .clampNumElements(0, v2s32, v2s32)
+ .lower();
getActionDefinitionsBuilder(G_SEXT_INREG)
.legalFor({i32, i64, v8i8, v16i8, v4i16, v8i16, v2i32, v4i32, v2i64})
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
index 4a88a30d56e01..de55cc6cc5393 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalizer-info-validation.mir
@@ -354,16 +354,16 @@
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_TRUNC_SSAT_S (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_TRUNC_SSAT_U (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_TRUNC_USAT_U (opcode {{[0-9]+}}): 2 type indices, 0 imm indices
# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
-# DEBUG-NEXT: .. the first uncovered type index: 2, OK
-# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_CONSTANT (opcode {{[0-9]+}}): 1 type index, 0 imm indices
# DEBUG-NEXT: .. the first uncovered type index: 1, OK
# DEBUG-NEXT: .. the first uncovered imm index: 0, OK
@@ -500,11 +500,13 @@
# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_SMULFIXSAT (opcode {{[0-9]+}}): 1 type index, 1 imm index
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_UMULFIXSAT (opcode {{[0-9]+}}): 1 type index, 1 imm index
-# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
-# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
+# DEBUG-NEXT: .. opcode {{[0-9]+}} is aliased to {{[0-9]+}}
+# DEBUG-NEXT: .. type index coverage check SKIPPED: user-defined predicate detected
+# DEBUG-NEXT: .. imm index coverage check SKIPPED: user-defined predicate detected
# DEBUG-NEXT: G_SDIVFIX (opcode {{[0-9]+}}): 1 type index, 1 imm index
# DEBUG-NEXT: .. type index coverage check SKIPPED: no rules defined
# DEBUG-NEXT: .. imm index coverage check SKIPPED: no rules defined
diff --git a/llvm/test/CodeGen/AArch64/smul_fix_sat.ll b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
index 0a087c949d4d3..865b11100c4f5 100644
--- a/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
+++ b/llvm/test/CodeGen/AArch64/smul_fix_sat.ll
@@ -2,455 +2,856 @@
; RUN: llc < %s -mtriple=aarch64-linux-gnu -global-isel=0 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc < %s -mtriple=aarch64-linux-gnu -global-isel=1 -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-; CHECK-GI: warning: Instruction selection used fallback path for func
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func2
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func3
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func4
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func5
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func6
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func7
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v8i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v16i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v4i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v8i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v2i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v4i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v8i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v2i64
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v4i64
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_sqdmulh_v8i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_sqdmulh_v4i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_sqdmulh_v4i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_sqdmulh_v2i32
define i32 @func(i32 %x, i32 %y) {
-; CHECK-LABEL: func:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull x9, w0, w1
-; CHECK-NEXT: mov w8, #2147483647 // =0x7fffffff
-; CHECK-NEXT: lsr x10, x9, #32
-; CHECK-NEXT: extr w9, w10, w9, #2
-; CHECK-NEXT: cmp w10, #1
-; CHECK-NEXT: csel w8, w8, w9, gt
-; CHECK-NEXT: cmn w10, #2
-; CHECK-NEXT: mov w9, #-2147483648 // =0x80000000
-; CHECK-NEXT: csel w0, w9, w8, lt
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull x9, w0, w1
+; CHECK-SD-NEXT: mov w8, #2147483647 // =0x7fffffff
+; CHECK-SD-NEXT: lsr x10, x9, #32
+; CHECK-SD-NEXT: extr w9, w10, w9, #2
+; CHECK-SD-NEXT: cmp w10, #1
+; CHECK-SD-NEXT: csel w8, w8, w9, gt
+; CHECK-SD-NEXT: cmn w10, #2
+; CHECK-SD-NEXT: mov w9, #-2147483648 // =0x80000000
+; CHECK-SD-NEXT: csel w0, w9, w8, lt
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull x8, w0, w1
+; CHECK-GI-NEXT: mov w9, #2147483647 // =0x7fffffff
+; CHECK-GI-NEXT: asr x8, x8, #2
+; CHECK-GI-NEXT: cmp x8, x9
+; CHECK-GI-NEXT: csel x8, x8, x9, lt
+; CHECK-GI-NEXT: mov x9, #-2147483648 // =0xffffffff80000000
+; CHECK-GI-NEXT: cmp x8, x9
+; CHECK-GI-NEXT: csel x0, x8, x9, gt
+; CHECK-GI-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-GI-NEXT: ret
%tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 2)
ret i32 %tmp
}
define i64 @func2(i64 %x, i64 %y) {
-; CHECK-LABEL: func2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-NEXT: smulh x10, x0, x1
-; CHECK-NEXT: extr x9, x10, x9, #2
-; CHECK-NEXT: cmp x10, #1
-; CHECK-NEXT: csel x8, x8, x9, gt
-; CHECK-NEXT: cmn x10, #2
-; CHECK-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: csel x0, x9, x8, lt
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func2:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-SD-NEXT: smulh x10, x0, x1
+; CHECK-SD-NEXT: extr x9, x10, x9, #2
+; CHECK-SD-NEXT: cmp x10, #1
+; CHECK-SD-NEXT: csel x8, x8, x9, gt
+; CHECK-SD-NEXT: cmn x10, #2
+; CHECK-SD-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
+; CHECK-SD-NEXT: csel x0, x9, x8, lt
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: asr x10, x1, #63
+; CHECK-GI-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-NEXT: mul x11, x0, x1
+; CHECK-GI-NEXT: madd x9, x0, x10, x9
+; CHECK-GI-NEXT: asr x10, x0, #63
+; CHECK-GI-NEXT: madd x9, x10, x1, x9
+; CHECK-GI-NEXT: extr x10, x9, x11, #2
+; CHECK-GI-NEXT: asr x9, x9, #2
+; CHECK-GI-NEXT: cmp x10, x8
+; CHECK-GI-NEXT: cset w11, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: cset w12, mi
+; CHECK-GI-NEXT: csel w11, w11, w12, eq
+; CHECK-GI-NEXT: tst w11, #0x1
+; CHECK-GI-NEXT: csel x8, x10, x8, ne
+; CHECK-GI-NEXT: mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT: csel x9, x9, xzr, ne
+; CHECK-GI-NEXT: cmp x8, x10
+; CHECK-GI-NEXT: cset w11, hi
+; CHECK-GI-NEXT: cmn x9, #1
+; CHECK-GI-NEXT: cset w9, gt
+; CHECK-GI-NEXT: csel w9, w11, w9, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csel x0, x8, x10, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 2)
ret i64 %tmp
}
define i4 @func3(i4 %x, i4 %y) {
-; CHECK-LABEL: func3:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sbfx w9, w1, #0, #4
-; CHECK-NEXT: lsl w10, w0, #28
-; CHECK-NEXT: mov w8, #2147483647 // =0x7fffffff
-; CHECK-NEXT: smull x9, w10, w9
-; CHECK-NEXT: lsr x10, x9, #32
-; CHECK-NEXT: extr w9, w10, w9, #2
-; CHECK-NEXT: cmp w10, #1
-; CHECK-NEXT: csel w8, w8, w9, gt
-; CHECK-NEXT: cmn w10, #2
-; CHECK-NEXT: mov w9, #-2147483648 // =0x80000000
-; CHECK-NEXT: csel w8, w9, w8, lt
-; CHECK-NEXT: asr w0, w8, #28
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func3:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: sbfx w9, w1, #0, #4
+; CHECK-SD-NEXT: lsl w10, w0, #28
+; CHECK-SD-NEXT: mov w8, #2147483647 // =0x7fffffff
+; CHECK-SD-NEXT: smull x9, w10, w9
+; CHECK-SD-NEXT: lsr x10, x9, #32
+; CHECK-SD-NEXT: extr w9, w10, w9, #2
+; CHECK-SD-NEXT: cmp w10, #1
+; CHECK-SD-NEXT: csel w8, w8, w9, gt
+; CHECK-SD-NEXT: cmn w10, #2
+; CHECK-SD-NEXT: mov w9, #-2147483648 // =0x80000000
+; CHECK-SD-NEXT: csel w8, w9, w8, lt
+; CHECK-SD-NEXT: asr w0, w8, #28
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: sbfx w9, w0, #0, #4
+; CHECK-GI-NEXT: sbfx w10, w1, #0, #4
+; CHECK-GI-NEXT: mov w8, #7 // =0x7
+; CHECK-GI-NEXT: mul w9, w9, w10
+; CHECK-GI-NEXT: sbfx w9, w9, #2, #6
+; CHECK-GI-NEXT: cmp w9, #7
+; CHECK-GI-NEXT: csel w8, w9, w8, lt
+; CHECK-GI-NEXT: mov w9, #-8 // =0xfffffff8
+; CHECK-GI-NEXT: cmn w8, #8
+; CHECK-GI-NEXT: csel w0, w8, w9, gt
+; CHECK-GI-NEXT: ret
%tmp = call i4 @llvm.smul.fix.sat.i4(i4 %x, i4 %y, i32 2)
ret i4 %tmp
}
;; These result in regular integer multiplication with a saturation check.
define i32 @func4(i32 %x, i32 %y) {
-; CHECK-LABEL: func4:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull x9, w0, w1
-; CHECK-NEXT: eor w10, w0, w1
-; CHECK-NEXT: mov w8, #-2147483648 // =0x80000000
-; CHECK-NEXT: cmp w10, #0
-; CHECK-NEXT: cinv w8, w8, pl
-; CHECK-NEXT: cmp x9, w9, sxtw
-; CHECK-NEXT: csel w0, w8, w9, ne
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func4:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull x9, w0, w1
+; CHECK-SD-NEXT: eor w10, w0, w1
+; CHECK-SD-NEXT: mov w8, #-2147483648 // =0x80000000
+; CHECK-SD-NEXT: cmp w10, #0
+; CHECK-SD-NEXT: cinv w8, w8, pl
+; CHECK-SD-NEXT: cmp x9, w9, sxtw
+; CHECK-SD-NEXT: csel w0, w8, w9, ne
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func4:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull x8, w0, w1
+; CHECK-GI-NEXT: mov w9, #2147483647 // =0x7fffffff
+; CHECK-GI-NEXT: cmp x8, x9
+; CHECK-GI-NEXT: csel x8, x8, x9, lt
+; CHECK-GI-NEXT: mov x9, #-2147483648 // =0xffffffff80000000
+; CHECK-GI-NEXT: cmp x8, x9
+; CHECK-GI-NEXT: csel x0, x8, x9, gt
+; CHECK-GI-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-GI-NEXT: ret
%tmp = call i32 @llvm.smul.fix.sat.i32(i32 %x, i32 %y, i32 0)
ret i32 %tmp
}
define i64 @func5(i64 %x, i64 %y) {
-; CHECK-LABEL: func5:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: eor x11, x0, x1
-; CHECK-NEXT: mov x8, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: cmp x11, #0
-; CHECK-NEXT: smulh x10, x0, x1
-; CHECK-NEXT: cinv x8, x8, pl
-; CHECK-NEXT: cmp x10, x9, asr #63
-; CHECK-NEXT: csel x0, x8, x9, ne
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func5:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: eor x11, x0, x1
+; CHECK-SD-NEXT: mov x8, #-9223372036854775808 // =0x8000000000000000
+; CHECK-SD-NEXT: cmp x11, #0
+; CHECK-SD-NEXT: smulh x10, x0, x1
+; CHECK-SD-NEXT: cinv x8, x8, pl
+; CHECK-SD-NEXT: cmp x10, x9, asr #63
+; CHECK-SD-NEXT: csel x0, x8, x9, ne
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func5:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umulh x8, x0, x1
+; CHECK-GI-NEXT: asr x9, x1, #63
+; CHECK-GI-NEXT: mul x10, x0, x1
+; CHECK-GI-NEXT: madd x8, x0, x9, x8
+; CHECK-GI-NEXT: asr x9, x0, #63
+; CHECK-GI-NEXT: madd x8, x9, x1, x8
+; CHECK-GI-NEXT: mov x9, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-NEXT: cmp x10, x9
+; CHECK-GI-NEXT: cset w11, lo
+; CHECK-GI-NEXT: cmp x8, #0
+; CHECK-GI-NEXT: cset w12, mi
+; CHECK-GI-NEXT: csel w11, w11, w12, eq
+; CHECK-GI-NEXT: tst w11, #0x1
+; CHECK-GI-NEXT: csel x9, x10, x9, ne
+; CHECK-GI-NEXT: mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT: csel x8, x8, xzr, ne
+; CHECK-GI-NEXT: cmp x9, x10
+; CHECK-GI-NEXT: cset w11, hi
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: cset w8, gt
+; CHECK-GI-NEXT: csel w8, w11, w8, eq
+; CHECK-GI-NEXT: tst w8, #0x1
+; CHECK-GI-NEXT: csel x0, x9, x10, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 0)
ret i64 %tmp
}
define i4 @func6(i4 %x, i4 %y) {
-; CHECK-LABEL: func6:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sbfx w9, w1, #0, #4
-; CHECK-NEXT: lsl w10, w0, #28
-; CHECK-NEXT: mov w8, #-2147483648 // =0x80000000
-; CHECK-NEXT: smull x11, w10, w9
-; CHECK-NEXT: eor w9, w10, w9
-; CHECK-NEXT: cmp w9, #0
-; CHECK-NEXT: cinv w8, w8, pl
-; CHECK-NEXT: cmp x11, w11, sxtw
-; CHECK-NEXT: csel w8, w8, w11, ne
-; CHECK-NEXT: asr w0, w8, #28
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func6:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: sbfx w9, w1, #0, #4
+; CHECK-SD-NEXT: lsl w10, w0, #28
+; CHECK-SD-NEXT: mov w8, #-2147483648 // =0x80000000
+; CHECK-SD-NEXT: smull x11, w10, w9
+; CHECK-SD-NEXT: eor w9, w10, w9
+; CHECK-SD-NEXT: cmp w9, #0
+; CHECK-SD-NEXT: cinv w8, w8, pl
+; CHECK-SD-NEXT: cmp x11, w11, sxtw
+; CHECK-SD-NEXT: csel w8, w8, w11, ne
+; CHECK-SD-NEXT: asr w0, w8, #28
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func6:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: sbfx w9, w0, #0, #4
+; CHECK-GI-NEXT: sbfx w10, w1, #0, #4
+; CHECK-GI-NEXT: mov w8, #7 // =0x7
+; CHECK-GI-NEXT: mul w9, w9, w10
+; CHECK-GI-NEXT: sxtb w9, w9
+; CHECK-GI-NEXT: cmp w9, #7
+; CHECK-GI-NEXT: csel w8, w9, w8, lt
+; CHECK-GI-NEXT: mov w9, #-8 // =0xfffffff8
+; CHECK-GI-NEXT: cmn w8, #8
+; CHECK-GI-NEXT: csel w0, w8, w9, gt
+; CHECK-GI-NEXT: ret
%tmp = call i4 @llvm.smul.fix.sat.i4(i4 %x, i4 %y, i32 0)
ret i4 %tmp
}
define i64 @func7(i64 %x, i64 %y) {
-; CHECK-LABEL: func7:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: mov w8, #2147483647 // =0x7fffffff
-; CHECK-NEXT: mov x11, #-2147483648 // =0xffffffff80000000
-; CHECK-NEXT: smulh x10, x0, x1
-; CHECK-NEXT: extr x9, x10, x9, #32
-; CHECK-NEXT: cmp x10, x8
-; CHECK-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-NEXT: csel x8, x8, x9, gt
-; CHECK-NEXT: cmp x10, x11
-; CHECK-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: csel x0, x9, x8, lt
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func7:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: mov w8, #2147483647 // =0x7fffffff
+; CHECK-SD-NEXT: mov x11, #-2147483648 // =0xffffffff80000000
+; CHECK-SD-NEXT: smulh x10, x0, x1
+; CHECK-SD-NEXT: extr x9, x10, x9, #32
+; CHECK-SD-NEXT: cmp x10, x8
+; CHECK-SD-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-SD-NEXT: csel x8, x8, x9, gt
+; CHECK-SD-NEXT: cmp x10, x11
+; CHECK-SD-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
+; CHECK-SD-NEXT: csel x0, x9, x8, lt
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func7:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: asr x10, x1, #63
+; CHECK-GI-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-NEXT: mul x11, x0, x1
+; CHECK-GI-NEXT: madd x9, x0, x10, x9
+; CHECK-GI-NEXT: asr x10, x0, #63
+; CHECK-GI-NEXT: madd x9, x10, x1, x9
+; CHECK-GI-NEXT: extr x10, x9, x11, #32
+; CHECK-GI-NEXT: asr x9, x9, #32
+; CHECK-GI-NEXT: cmp x10, x8
+; CHECK-GI-NEXT: cset w11, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: cset w12, mi
+; CHECK-GI-NEXT: csel w11, w11, w12, eq
+; CHECK-GI-NEXT: tst w11, #0x1
+; CHECK-GI-NEXT: csel x8, x10, x8, ne
+; CHECK-GI-NEXT: mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT: csel x9, x9, xzr, ne
+; CHECK-GI-NEXT: cmp x8, x10
+; CHECK-GI-NEXT: cset w11, hi
+; CHECK-GI-NEXT: cmn x9, #1
+; CHECK-GI-NEXT: cset w9, gt
+; CHECK-GI-NEXT: csel w9, w11, w9, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csel x0, x8, x10, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 32)
ret i64 %tmp
}
define i64 @func8(i64 %x, i64 %y) {
-; CHECK-LABEL: func8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: mov x8, #4611686018427387903 // =0x3fffffffffffffff
-; CHECK-NEXT: mov x11, #-4611686018427387904 // =0xc000000000000000
-; CHECK-NEXT: smulh x10, x0, x1
-; CHECK-NEXT: extr x9, x10, x9, #63
-; CHECK-NEXT: cmp x10, x8
-; CHECK-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-NEXT: csel x8, x8, x9, gt
-; CHECK-NEXT: cmp x10, x11
-; CHECK-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: csel x0, x9, x8, lt
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: mov x8, #4611686018427387903 // =0x3fffffffffffffff
+; CHECK-SD-NEXT: mov x11, #-4611686018427387904 // =0xc000000000000000
+; CHECK-SD-NEXT: smulh x10, x0, x1
+; CHECK-SD-NEXT: extr x9, x10, x9, #63
+; CHECK-SD-NEXT: cmp x10, x8
+; CHECK-SD-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-SD-NEXT: csel x8, x8, x9, gt
+; CHECK-SD-NEXT: cmp x10, x11
+; CHECK-SD-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
+; CHECK-SD-NEXT: csel x0, x9, x8, lt
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: asr x10, x1, #63
+; CHECK-GI-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-NEXT: mul x11, x0, x1
+; CHECK-GI-NEXT: madd x9, x0, x10, x9
+; CHECK-GI-NEXT: asr x10, x0, #63
+; CHECK-GI-NEXT: madd x9, x10, x1, x9
+; CHECK-GI-NEXT: extr x10, x9, x11, #63
+; CHECK-GI-NEXT: asr x9, x9, #63
+; CHECK-GI-NEXT: cmp x10, x8
+; CHECK-GI-NEXT: cset w11, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: cset w12, mi
+; CHECK-GI-NEXT: csel w11, w11, w12, eq
+; CHECK-GI-NEXT: tst w11, #0x1
+; CHECK-GI-NEXT: csel x8, x10, x8, ne
+; CHECK-GI-NEXT: mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT: csel x9, x9, xzr, ne
+; CHECK-GI-NEXT: cmp x8, x10
+; CHECK-GI-NEXT: cset w11, hi
+; CHECK-GI-NEXT: cmn x9, #1
+; CHECK-GI-NEXT: cset w9, gt
+; CHECK-GI-NEXT: csel w9, w11, w9, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csel x0, x8, x10, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.smul.fix.sat.i64(i64 %x, i64 %y, i32 63)
ret i64 %tmp
}
define <8 x i8> @vec_v8i8(<8 x i8> %x, <8 x i8> %y) {
-; CHECK-LABEL: vec_v8i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT: movi v2.8b, #1
-; CHECK-NEXT: movi v4.8b, #127
-; CHECK-NEXT: shrn v1.8b, v0.8h, #8
-; CHECK-NEXT: xtn v0.8b, v0.8h
-; CHECK-NEXT: shl v3.8b, v1.8b, #6
-; CHECK-NEXT: usra v3.8b, v0.8b, #2
-; CHECK-NEXT: cmgt v0.8b, v1.8b, v2.8b
-; CHECK-NEXT: movi v2.8b, #254
-; CHECK-NEXT: bsl v0.8b, v4.8b, v3.8b
-; CHECK-NEXT: movi v3.8b, #128
-; CHECK-NEXT: cmgt v1.8b, v2.8b, v1.8b
-; CHECK-NEXT: bit v0.8b, v3.8b, v1.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v8i8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT: movi v2.8b, #1
+; CHECK-SD-NEXT: movi v4.8b, #127
+; CHECK-SD-NEXT: shrn v1.8b, v0.8h, #8
+; CHECK-SD-NEXT: xtn v0.8b, v0.8h
+; CHECK-SD-NEXT: shl v3.8b, v1.8b, #6
+; CHECK-SD-NEXT: usra v3.8b, v0.8b, #2
+; CHECK-SD-NEXT: cmgt v0.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT: movi v2.8b, #254
+; CHECK-SD-NEXT: bsl v0.8b, v4.8b, v3.8b
+; CHECK-SD-NEXT: movi v3.8b, #128
+; CHECK-SD-NEXT: cmgt v1.8b, v2.8b, v1.8b
+; CHECK-SD-NEXT: bit v0.8b, v3.8b, v1.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v8i8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v0.8h, v0.8b, v1.8b
+; CHECK-GI-NEXT: sqshrn v0.8b, v0.8h, #2
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i8> @llvm.smul.fix.sat.v8i8(<8 x i8> %x, <8 x i8> %y, i32 2)
ret <8 x i8> %tmp
}
define <16 x i8> @vec_v16i8(<16 x i8> %x, <16 x i8> %y) {
-; CHECK-LABEL: vec_v16i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull2 v3.8h, v0.16b, v1.16b
-; CHECK-NEXT: smull v4.8h, v0.8b, v1.8b
-; CHECK-NEXT: movi v2.16b, #1
-; CHECK-NEXT: mul v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: uzp2 v3.16b, v4.16b, v3.16b
-; CHECK-NEXT: movi v4.16b, #127
-; CHECK-NEXT: shl v1.16b, v3.16b, #6
-; CHECK-NEXT: cmgt v2.16b, v3.16b, v2.16b
-; CHECK-NEXT: usra v1.16b, v0.16b, #2
-; CHECK-NEXT: movi v0.16b, #254
-; CHECK-NEXT: bit v1.16b, v4.16b, v2.16b
-; CHECK-NEXT: movi v2.16b, #128
-; CHECK-NEXT: cmgt v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: bsl v0.16b, v2.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v16i8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull2 v3.8h, v0.16b, v1.16b
+; CHECK-SD-NEXT: smull v4.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT: movi v2.16b, #1
+; CHECK-SD-NEXT: mul v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: uzp2 v3.16b, v4.16b, v3.16b
+; CHECK-SD-NEXT: movi v4.16b, #127
+; CHECK-SD-NEXT: shl v1.16b, v3.16b, #6
+; CHECK-SD-NEXT: cmgt v2.16b, v3.16b, v2.16b
+; CHECK-SD-NEXT: usra v1.16b, v0.16b, #2
+; CHECK-SD-NEXT: movi v0.16b, #254
+; CHECK-SD-NEXT: bit v1.16b, v4.16b, v2.16b
+; CHECK-SD-NEXT: movi v2.16b, #128
+; CHECK-SD-NEXT: cmgt v0.16b, v0.16b, v3.16b
+; CHECK-SD-NEXT: bsl v0.16b, v2.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v16i8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v2.8h, v0.8b, v1.8b
+; CHECK-GI-NEXT: smull2 v1.8h, v0.16b, v1.16b
+; CHECK-GI-NEXT: sqshrn v0.8b, v2.8h, #2
+; CHECK-GI-NEXT: sqshrn2 v0.16b, v1.8h, #2
+; CHECK-GI-NEXT: ret
%tmp = call <16 x i8> @llvm.smul.fix.sat.v16i8(<16 x i8> %x, <16 x i8> %y, i32 2)
ret <16 x i8> %tmp
}
define <4 x i16> @vec_v4i16(<4 x i16> %x, <4 x i16> %y) {
-; CHECK-LABEL: vec_v4i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull v0.4s, v0.4h, v1.4h
-; CHECK-NEXT: mvni v2.4h, #254, lsl #8
-; CHECK-NEXT: movi v4.4h, #128, lsl #8
-; CHECK-NEXT: shrn v1.4h, v0.4s, #16
-; CHECK-NEXT: xtn v0.4h, v0.4s
-; CHECK-NEXT: shl v3.4h, v1.4h, #6
-; CHECK-NEXT: usra v3.4h, v0.4h, #10
-; CHECK-NEXT: cmgt v0.4h, v1.4h, v2.4h
-; CHECK-NEXT: movi v2.4h, #254, lsl #8
-; CHECK-NEXT: bic v3.8b, v3.8b, v0.8b
-; CHECK-NEXT: bic v0.4h, #128, lsl #8
-; CHECK-NEXT: cmgt v1.4h, v2.4h, v1.4h
-; CHECK-NEXT: orr v0.8b, v0.8b, v3.8b
-; CHECK-NEXT: bit v0.8b, v4.8b, v1.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v4i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull v0.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT: mvni v2.4h, #254, lsl #8
+; CHECK-SD-NEXT: movi v4.4h, #128, lsl #8
+; CHECK-SD-NEXT: shrn v1.4h, v0.4s, #16
+; CHECK-SD-NEXT: xtn v0.4h, v0.4s
+; CHECK-SD-NEXT: shl v3.4h, v1.4h, #6
+; CHECK-SD-NEXT: usra v3.4h, v0.4h, #10
+; CHECK-SD-NEXT: cmgt v0.4h, v1.4h, v2.4h
+; CHECK-SD-NEXT: movi v2.4h, #254, lsl #8
+; CHECK-SD-NEXT: bic v3.8b, v3.8b, v0.8b
+; CHECK-SD-NEXT: bic v0.4h, #128, lsl #8
+; CHECK-SD-NEXT: cmgt v1.4h, v2.4h, v1.4h
+; CHECK-SD-NEXT: orr v0.8b, v0.8b, v3.8b
+; CHECK-SD-NEXT: bit v0.8b, v4.8b, v1.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v4i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v0.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT: sqshrn v0.4h, v0.4s, #10
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i16> @llvm.smul.fix.sat.v4i16(<4 x i16> %x, <4 x i16> %y, i32 10)
ret <4 x i16> %tmp
}
define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) {
-; CHECK-LABEL: vec_v8i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull2 v3.4s, v0.8h, v1.8h
-; CHECK-NEXT: smull v4.4s, v0.4h, v1.4h
-; CHECK-NEXT: movi v2.8h, #1
-; CHECK-NEXT: mul v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: uzp2 v3.8h, v4.8h, v3.8h
-; CHECK-NEXT: movi v4.8h, #128, lsl #8
-; CHECK-NEXT: shl v1.8h, v3.8h, #14
-; CHECK-NEXT: cmgt v2.8h, v3.8h, v2.8h
-; CHECK-NEXT: usra v1.8h, v0.8h, #2
-; CHECK-NEXT: bic v0.16b, v1.16b, v2.16b
-; CHECK-NEXT: bic v2.8h, #128, lsl #8
-; CHECK-NEXT: mvni v1.8h, #1
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
-; CHECK-NEXT: cmgt v1.8h, v1.8h, v3.8h
-; CHECK-NEXT: bit v0.16b, v4.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v8i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull2 v3.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT: smull v4.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT: movi v2.8h, #1
+; CHECK-SD-NEXT: mul v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: uzp2 v3.8h, v4.8h, v3.8h
+; CHECK-SD-NEXT: movi v4.8h, #128, lsl #8
+; CHECK-SD-NEXT: shl v1.8h, v3.8h, #14
+; CHECK-SD-NEXT: cmgt v2.8h, v3.8h, v2.8h
+; CHECK-SD-NEXT: usra v1.8h, v0.8h, #2
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT: bic v2.8h, #128, lsl #8
+; CHECK-SD-NEXT: mvni v1.8h, #1
+; CHECK-SD-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-SD-NEXT: cmgt v1.8h, v1.8h, v3.8h
+; CHECK-SD-NEXT: bit v0.16b, v4.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v8i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v2.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT: smull2 v1.4s, v0.8h, v1.8h
+; CHECK-GI-NEXT: sqshrn v0.4h, v2.4s, #2
+; CHECK-GI-NEXT: sqshrn2 v0.8h, v1.4s, #2
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i16> @llvm.smul.fix.sat.v8i16(<8 x i16> %x, <8 x i16> %y, i32 2)
ret <8 x i16> %tmp
}
define <2 x i32> @vec_v2i32(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: vec_v2i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull v0.2d, v0.2s, v1.2s
-; CHECK-NEXT: movi v2.2s, #128, lsl #24
-; CHECK-NEXT: shrn v1.2s, v0.2d, #32
-; CHECK-NEXT: xtn v0.2s, v0.2d
-; CHECK-NEXT: cmlt v3.2s, v1.2s, #0
-; CHECK-NEXT: add v4.2s, v1.2s, v1.2s
-; CHECK-NEXT: cmlt v6.2s, v0.2s, #0
-; CHECK-NEXT: mvn v5.8b, v3.8b
-; CHECK-NEXT: shl v4.2s, v4.2s, #31
-; CHECK-NEXT: cmeq v1.2s, v1.2s, v6.2s
-; CHECK-NEXT: bsl v2.8b, v3.8b, v5.8b
-; CHECK-NEXT: orr v0.8b, v4.8b, v0.8b
-; CHECK-NEXT: bif v0.8b, v2.8b, v1.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v2i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT: movi v2.2s, #128, lsl #24
+; CHECK-SD-NEXT: shrn v1.2s, v0.2d, #32
+; CHECK-SD-NEXT: xtn v0.2s, v0.2d
+; CHECK-SD-NEXT: cmlt v3.2s, v1.2s, #0
+; CHECK-SD-NEXT: add v4.2s, v1.2s, v1.2s
+; CHECK-SD-NEXT: cmlt v6.2s, v0.2s, #0
+; CHECK-SD-NEXT: mvn v5.8b, v3.8b
+; CHECK-SD-NEXT: shl v4.2s, v4.2s, #31
+; CHECK-SD-NEXT: cmeq v1.2s, v1.2s, v6.2s
+; CHECK-SD-NEXT: bsl v2.8b, v3.8b, v5.8b
+; CHECK-SD-NEXT: orr v0.8b, v4.8b, v0.8b
+; CHECK-SD-NEXT: bif v0.8b, v2.8b, v1.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v2i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v0.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT: sqxtn v0.2s, v0.2d
+; CHECK-GI-NEXT: ret
%tmp = call <2 x i32> @llvm.smul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> %y, i32 0)
ret <2 x i32> %tmp
}
define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: vec_v4i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull2 v3.2d, v0.4s, v1.4s
-; CHECK-NEXT: smull v4.2d, v0.2s, v1.2s
-; CHECK-NEXT: movi v2.4s, #63, msl #8
-; CHECK-NEXT: mul v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: uzp2 v3.4s, v4.4s, v3.4s
-; CHECK-NEXT: movi v4.4s, #128, lsl #24
-; CHECK-NEXT: shl v1.4s, v3.4s, #17
-; CHECK-NEXT: cmgt v2.4s, v3.4s, v2.4s
-; CHECK-NEXT: usra v1.4s, v0.4s, #15
-; CHECK-NEXT: bic v0.16b, v1.16b, v2.16b
-; CHECK-NEXT: bic v2.4s, #128, lsl #24
-; CHECK-NEXT: mvni v1.4s, #63, msl #8
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
-; CHECK-NEXT: cmgt v1.4s, v1.4s, v3.4s
-; CHECK-NEXT: bit v0.16b, v4.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v4i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull2 v3.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT: smull v4.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT: movi v2.4s, #63, msl #8
+; CHECK-SD-NEXT: mul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: uzp2 v3.4s, v4.4s, v3.4s
+; CHECK-SD-NEXT: movi v4.4s, #128, lsl #24
+; CHECK-SD-NEXT: shl v1.4s, v3.4s, #17
+; CHECK-SD-NEXT: cmgt v2.4s, v3.4s, v2.4s
+; CHECK-SD-NEXT: usra v1.4s, v0.4s, #15
+; CHECK-SD-NEXT: bic v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT: bic v2.4s, #128, lsl #24
+; CHECK-SD-NEXT: mvni v1.4s, #63, msl #8
+; CHECK-SD-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-SD-NEXT: cmgt v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT: bit v0.16b, v4.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v4i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v2.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT: smull2 v1.2d, v0.4s, v1.4s
+; CHECK-GI-NEXT: sqshrn v0.2s, v2.2d, #15
+; CHECK-GI-NEXT: sqshrn2 v0.4s, v1.2d, #15
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 15)
ret <4 x i32> %tmp
}
define <8 x i32> @vec_v8i32(<8 x i32> %x, <8 x i32> %y) {
-; CHECK-LABEL: vec_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: smull2 v4.2d, v1.4s, v3.4s
-; CHECK-NEXT: smull2 v5.2d, v0.4s, v2.4s
-; CHECK-NEXT: smull v6.2d, v0.2s, v2.2s
-; CHECK-NEXT: smull v7.2d, v1.2s, v3.2s
-; CHECK-NEXT: mul v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: mul v1.4s, v1.4s, v3.4s
-; CHECK-NEXT: movi v16.4s, #128, lsl #24
-; CHECK-NEXT: uzp2 v5.4s, v6.4s, v5.4s
-; CHECK-NEXT: uzp2 v4.4s, v7.4s, v4.4s
-; CHECK-NEXT: cmlt v18.4s, v0.4s, #0
-; CHECK-NEXT: cmlt v20.4s, v1.4s, #0
-; CHECK-NEXT: cmlt v2.4s, v5.4s, #0
-; CHECK-NEXT: add v3.4s, v5.4s, v5.4s
-; CHECK-NEXT: cmeq v5.4s, v5.4s, v18.4s
-; CHECK-NEXT: cmlt v6.4s, v4.4s, #0
-; CHECK-NEXT: add v7.4s, v4.4s, v4.4s
-; CHECK-NEXT: mvn v17.16b, v2.16b
-; CHECK-NEXT: shl v3.4s, v3.4s, #31
-; CHECK-NEXT: mvn v19.16b, v6.16b
-; CHECK-NEXT: shl v7.4s, v7.4s, #31
-; CHECK-NEXT: bif v2.16b, v17.16b, v16.16b
-; CHECK-NEXT: orr v0.16b, v3.16b, v0.16b
-; CHECK-NEXT: cmeq v3.4s, v4.4s, v20.4s
-; CHECK-NEXT: bif v6.16b, v19.16b, v16.16b
-; CHECK-NEXT: orr v1.16b, v7.16b, v1.16b
-; CHECK-NEXT: bif v0.16b, v2.16b, v5.16b
-; CHECK-NEXT: bif v1.16b, v6.16b, v3.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v8i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: smull2 v4.2d, v1.4s, v3.4s
+; CHECK-SD-NEXT: smull2 v5.2d, v0.4s, v2.4s
+; CHECK-SD-NEXT: smull v6.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT: smull v7.2d, v1.2s, v3.2s
+; CHECK-SD-NEXT: mul v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT: mul v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT: movi v16.4s, #128, lsl #24
+; CHECK-SD-NEXT: uzp2 v5.4s, v6.4s, v5.4s
+; CHECK-SD-NEXT: uzp2 v4.4s, v7.4s, v4.4s
+; CHECK-SD-NEXT: cmlt v18.4s, v0.4s, #0
+; CHECK-SD-NEXT: cmlt v20.4s, v1.4s, #0
+; CHECK-SD-NEXT: cmlt v2.4s, v5.4s, #0
+; CHECK-SD-NEXT: add v3.4s, v5.4s, v5.4s
+; CHECK-SD-NEXT: cmeq v5.4s, v5.4s, v18.4s
+; CHECK-SD-NEXT: cmlt v6.4s, v4.4s, #0
+; CHECK-SD-NEXT: add v7.4s, v4.4s, v4.4s
+; CHECK-SD-NEXT: mvn v17.16b, v2.16b
+; CHECK-SD-NEXT: shl v3.4s, v3.4s, #31
+; CHECK-SD-NEXT: mvn v19.16b, v6.16b
+; CHECK-SD-NEXT: shl v7.4s, v7.4s, #31
+; CHECK-SD-NEXT: bif v2.16b, v17.16b, v16.16b
+; CHECK-SD-NEXT: orr v0.16b, v3.16b, v0.16b
+; CHECK-SD-NEXT: cmeq v3.4s, v4.4s, v20.4s
+; CHECK-SD-NEXT: bif v6.16b, v19.16b, v16.16b
+; CHECK-SD-NEXT: orr v1.16b, v7.16b, v1.16b
+; CHECK-SD-NEXT: bif v0.16b, v2.16b, v5.16b
+; CHECK-SD-NEXT: bif v1.16b, v6.16b, v3.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v8i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v4.2d, v0.2s, v2.2s
+; CHECK-GI-NEXT: smull v5.2d, v1.2s, v3.2s
+; CHECK-GI-NEXT: smull2 v2.2d, v0.4s, v2.4s
+; CHECK-GI-NEXT: smull2 v3.2d, v1.4s, v3.4s
+; CHECK-GI-NEXT: sqxtn v0.2s, v4.2d
+; CHECK-GI-NEXT: sqxtn v1.2s, v5.2d
+; CHECK-GI-NEXT: sqxtn2 v0.4s, v2.2d
+; CHECK-GI-NEXT: sqxtn2 v1.4s, v3.2d
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i32> @llvm.smul.fix.sat.v8i32(<8 x i32> %x, <8 x i32> %y, i32 0)
ret <8 x i32> %tmp
}
define <2 x i64> @vec_v2i64(<2 x i64> %x, <2 x i64> %y) {
-; CHECK-LABEL: vec_v2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov x9, v1.d[1]
-; CHECK-NEXT: mov x10, v0.d[1]
-; CHECK-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-NEXT: fmov x12, d0
-; CHECK-NEXT: mul x11, x10, x9
-; CHECK-NEXT: smulh x9, x10, x9
-; CHECK-NEXT: fmov x10, d1
-; CHECK-NEXT: mul x13, x12, x10
-; CHECK-NEXT: smulh x10, x12, x10
-; CHECK-NEXT: extr x11, x9, x11, #15
-; CHECK-NEXT: cmp x9, #4, lsl #12 // =16384
-; CHECK-NEXT: csel x11, x8, x11, ge
-; CHECK-NEXT: cmn x9, #4, lsl #12 // =16384
-; CHECK-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: csel x11, x9, x11, lt
-; CHECK-NEXT: extr x12, x10, x13, #15
-; CHECK-NEXT: cmp x10, #4, lsl #12 // =16384
-; CHECK-NEXT: csel x8, x8, x12, ge
-; CHECK-NEXT: cmn x10, #4, lsl #12 // =16384
-; CHECK-NEXT: csel x8, x9, x8, lt
-; CHECK-NEXT: fmov d0, x8
-; CHECK-NEXT: mov v0.d[1], x11
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v2i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov x9, v1.d[1]
+; CHECK-SD-NEXT: mov x10, v0.d[1]
+; CHECK-SD-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-SD-NEXT: fmov x12, d0
+; CHECK-SD-NEXT: mul x11, x10, x9
+; CHECK-SD-NEXT: smulh x9, x10, x9
+; CHECK-SD-NEXT: fmov x10, d1
+; CHECK-SD-NEXT: mul x13, x12, x10
+; CHECK-SD-NEXT: smulh x10, x12, x10
+; CHECK-SD-NEXT: extr x11, x9, x11, #15
+; CHECK-SD-NEXT: cmp x9, #4, lsl #12 // =16384
+; CHECK-SD-NEXT: csel x11, x8, x11, ge
+; CHECK-SD-NEXT: cmn x9, #4, lsl #12 // =16384
+; CHECK-SD-NEXT: mov x9, #-9223372036854775808 // =0x8000000000000000
+; CHECK-SD-NEXT: csel x11, x9, x11, lt
+; CHECK-SD-NEXT: extr x12, x10, x13, #15
+; CHECK-SD-NEXT: cmp x10, #4, lsl #12 // =16384
+; CHECK-SD-NEXT: csel x8, x8, x12, ge
+; CHECK-SD-NEXT: cmn x10, #4, lsl #12 // =16384
+; CHECK-SD-NEXT: csel x8, x9, x8, lt
+; CHECK-SD-NEXT: fmov d0, x8
+; CHECK-SD-NEXT: mov v0.d[1], x11
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v2i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: fmov x9, d0
+; CHECK-GI-NEXT: fmov x10, d1
+; CHECK-GI-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-NEXT: mov d0, v0.d[1]
+; CHECK-GI-NEXT: mov d1, v1.d[1]
+; CHECK-GI-NEXT: umulh x11, x9, x10
+; CHECK-GI-NEXT: asr x12, x10, #63
+; CHECK-GI-NEXT: fmov x14, d1
+; CHECK-GI-NEXT: mul x13, x9, x10
+; CHECK-GI-NEXT: madd x11, x9, x12, x11
+; CHECK-GI-NEXT: fmov x12, d0
+; CHECK-GI-NEXT: asr x9, x9, #63
+; CHECK-GI-NEXT: asr x16, x14, #63
+; CHECK-GI-NEXT: umulh x15, x12, x14
+; CHECK-GI-NEXT: madd x9, x9, x10, x11
+; CHECK-GI-NEXT: asr x11, x12, #63
+; CHECK-GI-NEXT: madd x10, x12, x16, x15
+; CHECK-GI-NEXT: mul x12, x12, x14
+; CHECK-GI-NEXT: madd x10, x11, x14, x10
+; CHECK-GI-NEXT: extr x11, x9, x13, #15
+; CHECK-GI-NEXT: asr x9, x9, #15
+; CHECK-GI-NEXT: cmp x11, x8
+; CHECK-GI-NEXT: cset w13, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: cset w14, mi
+; CHECK-GI-NEXT: extr x12, x10, x12, #15
+; CHECK-GI-NEXT: asr x10, x10, #15
+; CHECK-GI-NEXT: csel w13, w13, w14, eq
+; CHECK-GI-NEXT: cmp x12, x8
+; CHECK-GI-NEXT: cset w14, lo
+; CHECK-GI-NEXT: cmp x10, #0
+; CHECK-GI-NEXT: cset w15, mi
+; CHECK-GI-NEXT: csel w14, w14, w15, eq
+; CHECK-GI-NEXT: tst w13, #0x1
+; CHECK-GI-NEXT: mov x13, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT: csel x11, x11, x8, ne
+; CHECK-GI-NEXT: csel x9, x9, xzr, ne
+; CHECK-GI-NEXT: tst w14, #0x1
+; CHECK-GI-NEXT: csel x8, x12, x8, ne
+; CHECK-GI-NEXT: csel x10, x10, xzr, ne
+; CHECK-GI-NEXT: cmp x11, x13
+; CHECK-GI-NEXT: cset w12, hi
+; CHECK-GI-NEXT: cmn x9, #1
+; CHECK-GI-NEXT: cset w9, gt
+; CHECK-GI-NEXT: csel w9, w12, w9, eq
+; CHECK-GI-NEXT: cmp x8, x13
+; CHECK-GI-NEXT: cset w12, hi
+; CHECK-GI-NEXT: cmn x10, #1
+; CHECK-GI-NEXT: cset w10, gt
+; CHECK-GI-NEXT: csel w10, w12, w10, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csel x9, x11, x13, ne
+; CHECK-GI-NEXT: tst w10, #0x1
+; CHECK-GI-NEXT: fmov d0, x9
+; CHECK-GI-NEXT: csel x8, x8, x13, ne
+; CHECK-GI-NEXT: mov v0.d[1], x8
+; CHECK-GI-NEXT: ret
%tmp = call <2 x i64> @llvm.smul.fix.sat.v2i64(<2 x i64> %x, <2 x i64> %y, i32 15)
ret <2 x i64> %tmp
}
define <4 x i64> @vec_v4i64(<4 x i64> %x, <4 x i64> %y) {
-; CHECK-LABEL: vec_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov x8, v2.d[1]
-; CHECK-NEXT: mov x9, v0.d[1]
-; CHECK-NEXT: mov w16, #2147483647 // =0x7fffffff
-; CHECK-NEXT: fmov x10, d2
-; CHECK-NEXT: fmov x11, d0
-; CHECK-NEXT: mov x18, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-NEXT: mov x14, v3.d[1]
-; CHECK-NEXT: mov x15, v1.d[1]
-; CHECK-NEXT: mul x13, x9, x8
-; CHECK-NEXT: smulh x8, x9, x8
-; CHECK-NEXT: mul x12, x11, x10
-; CHECK-NEXT: smulh x9, x11, x10
-; CHECK-NEXT: extr x13, x8, x13, #32
-; CHECK-NEXT: cmp x8, x16
-; CHECK-NEXT: mul x10, x15, x14
-; CHECK-NEXT: csel x13, x18, x13, gt
-; CHECK-NEXT: smulh x11, x15, x14
-; CHECK-NEXT: fmov x14, d3
-; CHECK-NEXT: fmov x15, d1
-; CHECK-NEXT: extr x12, x9, x12, #32
-; CHECK-NEXT: mul x17, x15, x14
-; CHECK-NEXT: smulh x14, x15, x14
-; CHECK-NEXT: mov x15, #-2147483648 // =0xffffffff80000000
-; CHECK-NEXT: cmp x8, x15
-; CHECK-NEXT: mov x8, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: csel x13, x8, x13, lt
-; CHECK-NEXT: cmp x9, x16
-; CHECK-NEXT: csel x12, x18, x12, gt
-; CHECK-NEXT: cmp x9, x15
-; CHECK-NEXT: extr x9, x11, x10, #32
-; CHECK-NEXT: csel x10, x8, x12, lt
-; CHECK-NEXT: cmp x11, x16
-; CHECK-NEXT: csel x9, x18, x9, gt
-; CHECK-NEXT: cmp x11, x15
-; CHECK-NEXT: extr x11, x14, x17, #32
-; CHECK-NEXT: csel x9, x8, x9, lt
-; CHECK-NEXT: cmp x14, x16
-; CHECK-NEXT: fmov d0, x10
-; CHECK-NEXT: csel x11, x18, x11, gt
-; CHECK-NEXT: cmp x14, x15
-; CHECK-NEXT: csel x8, x8, x11, lt
-; CHECK-NEXT: fmov d1, x8
-; CHECK-NEXT: mov v0.d[1], x13
-; CHECK-NEXT: mov v1.d[1], x9
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v4i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov x8, v2.d[1]
+; CHECK-SD-NEXT: mov x9, v0.d[1]
+; CHECK-SD-NEXT: mov w16, #2147483647 // =0x7fffffff
+; CHECK-SD-NEXT: fmov x10, d2
+; CHECK-SD-NEXT: fmov x11, d0
+; CHECK-SD-NEXT: mov x18, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-SD-NEXT: mov x14, v3.d[1]
+; CHECK-SD-NEXT: mov x15, v1.d[1]
+; CHECK-SD-NEXT: mul x13, x9, x8
+; CHECK-SD-NEXT: smulh x8, x9, x8
+; CHECK-SD-NEXT: mul x12, x11, x10
+; CHECK-SD-NEXT: smulh x9, x11, x10
+; CHECK-SD-NEXT: extr x13, x8, x13, #32
+; CHECK-SD-NEXT: cmp x8, x16
+; CHECK-SD-NEXT: mul x10, x15, x14
+; CHECK-SD-NEXT: csel x13, x18, x13, gt
+; CHECK-SD-NEXT: smulh x11, x15, x14
+; CHECK-SD-NEXT: fmov x14, d3
+; CHECK-SD-NEXT: fmov x15, d1
+; CHECK-SD-NEXT: extr x12, x9, x12, #32
+; CHECK-SD-NEXT: mul x17, x15, x14
+; CHECK-SD-NEXT: smulh x14, x15, x14
+; CHECK-SD-NEXT: mov x15, #-2147483648 // =0xffffffff80000000
+; CHECK-SD-NEXT: cmp x8, x15
+; CHECK-SD-NEXT: mov x8, #-9223372036854775808 // =0x8000000000000000
+; CHECK-SD-NEXT: csel x13, x8, x13, lt
+; CHECK-SD-NEXT: cmp x9, x16
+; CHECK-SD-NEXT: csel x12, x18, x12, gt
+; CHECK-SD-NEXT: cmp x9, x15
+; CHECK-SD-NEXT: extr x9, x11, x10, #32
+; CHECK-SD-NEXT: csel x10, x8, x12, lt
+; CHECK-SD-NEXT: cmp x11, x16
+; CHECK-SD-NEXT: csel x9, x18, x9, gt
+; CHECK-SD-NEXT: cmp x11, x15
+; CHECK-SD-NEXT: extr x11, x14, x17, #32
+; CHECK-SD-NEXT: csel x9, x8, x9, lt
+; CHECK-SD-NEXT: cmp x14, x16
+; CHECK-SD-NEXT: fmov d0, x10
+; CHECK-SD-NEXT: csel x11, x18, x11, gt
+; CHECK-SD-NEXT: cmp x14, x15
+; CHECK-SD-NEXT: csel x8, x8, x11, lt
+; CHECK-SD-NEXT: fmov d1, x8
+; CHECK-SD-NEXT: mov v0.d[1], x13
+; CHECK-SD-NEXT: mov v1.d[1], x9
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v4i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: fmov x8, d0
+; CHECK-GI-NEXT: fmov x9, d2
+; CHECK-GI-NEXT: mov d0, v0.d[1]
+; CHECK-GI-NEXT: mov d2, v2.d[1]
+; CHECK-GI-NEXT: umulh x10, x8, x9
+; CHECK-GI-NEXT: asr x11, x9, #63
+; CHECK-GI-NEXT: fmov x13, d2
+; CHECK-GI-NEXT: mul x12, x8, x9
+; CHECK-GI-NEXT: madd x10, x8, x11, x10
+; CHECK-GI-NEXT: fmov x11, d0
+; CHECK-GI-NEXT: asr x8, x8, #63
+; CHECK-GI-NEXT: asr x15, x13, #63
+; CHECK-GI-NEXT: mov d0, v1.d[1]
+; CHECK-GI-NEXT: umulh x14, x11, x13
+; CHECK-GI-NEXT: madd x8, x8, x9, x10
+; CHECK-GI-NEXT: asr x10, x11, #63
+; CHECK-GI-NEXT: madd x9, x11, x15, x14
+; CHECK-GI-NEXT: mul x11, x11, x13
+; CHECK-GI-NEXT: extr x12, x8, x12, #32
+; CHECK-GI-NEXT: asr x8, x8, #32
+; CHECK-GI-NEXT: madd x9, x10, x13, x9
+; CHECK-GI-NEXT: fmov x10, d1
+; CHECK-GI-NEXT: fmov x13, d3
+; CHECK-GI-NEXT: mov d1, v3.d[1]
+; CHECK-GI-NEXT: umulh x14, x10, x13
+; CHECK-GI-NEXT: asr x15, x13, #63
+; CHECK-GI-NEXT: fmov x17, d1
+; CHECK-GI-NEXT: mul x16, x10, x13
+; CHECK-GI-NEXT: extr x11, x9, x11, #32
+; CHECK-GI-NEXT: asr x9, x9, #32
+; CHECK-GI-NEXT: madd x14, x10, x15, x14
+; CHECK-GI-NEXT: fmov x15, d0
+; CHECK-GI-NEXT: asr x10, x10, #63
+; CHECK-GI-NEXT: asr x0, x17, #63
+; CHECK-GI-NEXT: umulh x18, x15, x17
+; CHECK-GI-NEXT: madd x10, x10, x13, x14
+; CHECK-GI-NEXT: mov x14, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-NEXT: cmp x12, x14
+; CHECK-GI-NEXT: madd x13, x15, x0, x18
+; CHECK-GI-NEXT: asr x18, x15, #63
+; CHECK-GI-NEXT: cset w0, lo
+; CHECK-GI-NEXT: cmp x8, #0
+; CHECK-GI-NEXT: mul x15, x15, x17
+; CHECK-GI-NEXT: cset w1, mi
+; CHECK-GI-NEXT: extr x16, x10, x16, #32
+; CHECK-GI-NEXT: asr x10, x10, #32
+; CHECK-GI-NEXT: madd x13, x18, x17, x13
+; CHECK-GI-NEXT: csel w17, w0, w1, eq
+; CHECK-GI-NEXT: cmp x11, x14
+; CHECK-GI-NEXT: cset w18, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: cset w0, mi
+; CHECK-GI-NEXT: csel w18, w18, w0, eq
+; CHECK-GI-NEXT: cmp x16, x14
+; CHECK-GI-NEXT: cset w0, lo
+; CHECK-GI-NEXT: cmp x10, #0
+; CHECK-GI-NEXT: extr x15, x13, x15, #32
+; CHECK-GI-NEXT: asr x13, x13, #32
+; CHECK-GI-NEXT: cset w1, mi
+; CHECK-GI-NEXT: csel w0, w0, w1, eq
+; CHECK-GI-NEXT: cmp x15, x14
+; CHECK-GI-NEXT: cset w1, lo
+; CHECK-GI-NEXT: cmp x13, #0
+; CHECK-GI-NEXT: cset w2, mi
+; CHECK-GI-NEXT: csel w1, w1, w2, eq
+; CHECK-GI-NEXT: tst w17, #0x1
+; CHECK-GI-NEXT: mov x17, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-NEXT: csel x12, x12, x14, ne
+; CHECK-GI-NEXT: csel x8, x8, xzr, ne
+; CHECK-GI-NEXT: tst w18, #0x1
+; CHECK-GI-NEXT: csel x11, x11, x14, ne
+; CHECK-GI-NEXT: csel x9, x9, xzr, ne
+; CHECK-GI-NEXT: tst w0, #0x1
+; CHECK-GI-NEXT: csel x16, x16, x14, ne
+; CHECK-GI-NEXT: csel x10, x10, xzr, ne
+; CHECK-GI-NEXT: tst w1, #0x1
+; CHECK-GI-NEXT: csel x14, x15, x14, ne
+; CHECK-GI-NEXT: csel x13, x13, xzr, ne
+; CHECK-GI-NEXT: cmp x12, x17
+; CHECK-GI-NEXT: cset w15, hi
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: cset w8, gt
+; CHECK-GI-NEXT: csel w8, w15, w8, eq
+; CHECK-GI-NEXT: cmp x11, x17
+; CHECK-GI-NEXT: cset w15, hi
+; CHECK-GI-NEXT: cmn x9, #1
+; CHECK-GI-NEXT: cset w9, gt
+; CHECK-GI-NEXT: csel w9, w15, w9, eq
+; CHECK-GI-NEXT: cmp x16, x17
+; CHECK-GI-NEXT: cset w15, hi
+; CHECK-GI-NEXT: cmn x10, #1
+; CHECK-GI-NEXT: cset w10, gt
+; CHECK-GI-NEXT: csel w10, w15, w10, eq
+; CHECK-GI-NEXT: cmp x14, x17
+; CHECK-GI-NEXT: cset w15, hi
+; CHECK-GI-NEXT: cmn x13, #1
+; CHECK-GI-NEXT: cset w13, gt
+; CHECK-GI-NEXT: csel w13, w15, w13, eq
+; CHECK-GI-NEXT: tst w8, #0x1
+; CHECK-GI-NEXT: csel x8, x12, x17, ne
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csel x9, x11, x17, ne
+; CHECK-GI-NEXT: tst w10, #0x1
+; CHECK-GI-NEXT: fmov d0, x8
+; CHECK-GI-NEXT: csel x10, x16, x17, ne
+; CHECK-GI-NEXT: tst w13, #0x1
+; CHECK-GI-NEXT: fmov d1, x10
+; CHECK-GI-NEXT: csel x11, x14, x17, ne
+; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: mov v1.d[1], x11
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i64> @llvm.smul.fix.sat.v4i64(<4 x i64> %x, <4 x i64> %y, i32 32)
ret <4 x i64> %tmp
}
define <8 x i16> @vec_sqdmulh_v8i16(<8 x i16> %x, <8 x i16> %y) {
-; CHECK-LABEL: vec_sqdmulh_v8i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sqdmulh v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_sqdmulh_v8i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: sqdmulh v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_sqdmulh_v8i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v2.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT: smull2 v1.4s, v0.8h, v1.8h
+; CHECK-GI-NEXT: sqshrn v0.4h, v2.4s, #15
+; CHECK-GI-NEXT: sqshrn2 v0.8h, v1.4s, #15
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i16> @llvm.smul.fix.sat.v8i16(<8 x i16> %x, <8 x i16> %y, i32 15)
ret <8 x i16> %tmp
}
define <4 x i16> @vec_sqdmulh_v4i16(<4 x i16> %x, <4 x i16> %y) {
-; CHECK-LABEL: vec_sqdmulh_v4i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sqdmulh v0.4h, v0.4h, v1.4h
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_sqdmulh_v4i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: sqdmulh v0.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_sqdmulh_v4i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v0.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT: sqshrn v0.4h, v0.4s, #15
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i16> @llvm.smul.fix.sat.v4i16(<4 x i16> %x, <4 x i16> %y, i32 15)
ret <4 x i16> %tmp
}
define <4 x i32> @vec_sqdmulh_v4i32(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: vec_sqdmulh_v4i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sqdmulh v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_sqdmulh_v4i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: sqdmulh v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_sqdmulh_v4i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v2.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT: smull2 v1.2d, v0.4s, v1.4s
+; CHECK-GI-NEXT: sqshrn v0.2s, v2.2d, #31
+; CHECK-GI-NEXT: sqshrn2 v0.4s, v1.2d, #31
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i32> @llvm.smul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 31)
ret <4 x i32> %tmp
}
define <2 x i32> @vec_sqdmulh_v2i32(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: vec_sqdmulh_v2i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: sqdmulh v0.2s, v0.2s, v1.2s
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_sqdmulh_v2i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: sqdmulh v0.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_sqdmulh_v2i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: smull v0.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT: sqshrn v0.2s, v0.2d, #31
+; CHECK-GI-NEXT: ret
%tmp = call <2 x i32> @llvm.smul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> %y, i32 31)
ret <2 x i32> %tmp
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-GI: {{.*}}
-; CHECK-SD: {{.*}}
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/umul_fix_sat.ll b/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
index 18a94a3c38711..f7ffb9ec61abf 100644
--- a/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
+++ b/llvm/test/CodeGen/AArch64/umul_fix_sat.ll
@@ -2,324 +2,546 @@
; RUN: llc < %s -mtriple=aarch64-linux-gnu -global-isel=0 | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc < %s -mtriple=aarch64-linux-gnu -global-isel=1 -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-; CHECK-GI: warning: Instruction selection used fallback path for func
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func2
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func3
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func4
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func5
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func6
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func7
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for func8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v8i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v16i8
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v4i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v8i16
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v2i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v4i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v8i32
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v2i64
-; CHECK-GI-NEXT: warning: Instruction selection used fallback path for vec_v4i64
define i32 @func(i32 %x, i32 %y) {
-; CHECK-LABEL: func:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull x8, w0, w1
-; CHECK-NEXT: lsr x9, x8, #32
-; CHECK-NEXT: extr w8, w9, w8, #2
-; CHECK-NEXT: cmp w9, #3
-; CHECK-NEXT: csinv w0, w8, wzr, ls
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull x8, w0, w1
+; CHECK-SD-NEXT: lsr x9, x8, #32
+; CHECK-SD-NEXT: extr w8, w9, w8, #2
+; CHECK-SD-NEXT: cmp w9, #3
+; CHECK-SD-NEXT: csinv w0, w8, wzr, ls
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull x8, w0, w1
+; CHECK-GI-NEXT: mov w9, #-1 // =0xffffffff
+; CHECK-GI-NEXT: lsr x8, x8, #2
+; CHECK-GI-NEXT: cmp x8, x9
+; CHECK-GI-NEXT: csel x0, x8, x9, lo
+; CHECK-GI-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-GI-NEXT: ret
%tmp = call i32 @llvm.umul.fix.sat.i32(i32 %x, i32 %y, i32 2)
ret i32 %tmp
}
define i64 @func2(i64 %x, i64 %y) {
-; CHECK-LABEL: func2:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x8, x0, x1
-; CHECK-NEXT: umulh x9, x0, x1
-; CHECK-NEXT: extr x8, x9, x8, #2
-; CHECK-NEXT: cmp x9, #3
-; CHECK-NEXT: csinv x0, x8, xzr, ls
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func2:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x8, x0, x1
+; CHECK-SD-NEXT: umulh x9, x0, x1
+; CHECK-SD-NEXT: extr x8, x9, x8, #2
+; CHECK-SD-NEXT: cmp x9, #3
+; CHECK-SD-NEXT: csinv x0, x8, xzr, ls
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func2:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mul x8, x0, x1
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: extr x8, x9, x8, #2
+; CHECK-GI-NEXT: lsr x9, x9, #2
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: cset w10, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: csel w9, w10, wzr, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csinv x0, x8, xzr, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 2)
ret i64 %tmp
}
define i4 @func3(i4 %x, i4 %y) {
-; CHECK-LABEL: func3:
-; CHECK: // %bb.0:
-; CHECK-NEXT: lsl w8, w0, #28
-; CHECK-NEXT: and w9, w1, #0xf
-; CHECK-NEXT: umull x8, w8, w9
-; CHECK-NEXT: lsr x9, x8, #32
-; CHECK-NEXT: extr w8, w9, w8, #2
-; CHECK-NEXT: cmp w9, #3
-; CHECK-NEXT: csinv w8, w8, wzr, ls
-; CHECK-NEXT: lsr w0, w8, #28
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func3:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: lsl w8, w0, #28
+; CHECK-SD-NEXT: and w9, w1, #0xf
+; CHECK-SD-NEXT: umull x8, w8, w9
+; CHECK-SD-NEXT: lsr x9, x8, #32
+; CHECK-SD-NEXT: extr w8, w9, w8, #2
+; CHECK-SD-NEXT: cmp w9, #3
+; CHECK-SD-NEXT: csinv w8, w8, wzr, ls
+; CHECK-SD-NEXT: lsr w0, w8, #28
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func3:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: and w9, w0, #0xf
+; CHECK-GI-NEXT: and w10, w1, #0xf
+; CHECK-GI-NEXT: mov w8, #15 // =0xf
+; CHECK-GI-NEXT: mul w9, w9, w10
+; CHECK-GI-NEXT: lsr w9, w9, #2
+; CHECK-GI-NEXT: cmp w9, #15
+; CHECK-GI-NEXT: csel w0, w9, w8, lo
+; CHECK-GI-NEXT: ret
%tmp = call i4 @llvm.umul.fix.sat.i4(i4 %x, i4 %y, i32 2)
ret i4 %tmp
}
;; These result in regular integer multiplication with a saturation check.
define i32 @func4(i32 %x, i32 %y) {
-; CHECK-LABEL: func4:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull x8, w0, w1
-; CHECK-NEXT: tst x8, #0xffffffff00000000
-; CHECK-NEXT: csinv w0, w8, wzr, eq
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func4:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull x8, w0, w1
+; CHECK-SD-NEXT: tst x8, #0xffffffff00000000
+; CHECK-SD-NEXT: csinv w0, w8, wzr, eq
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func4:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull x8, w0, w1
+; CHECK-GI-NEXT: mov w9, #-1 // =0xffffffff
+; CHECK-GI-NEXT: cmp x8, x9
+; CHECK-GI-NEXT: csel x0, x8, x9, lo
+; CHECK-GI-NEXT: // kill: def $w0 killed $w0 killed $x0
+; CHECK-GI-NEXT: ret
%tmp = call i32 @llvm.umul.fix.sat.i32(i32 %x, i32 %y, i32 0)
ret i32 %tmp
}
define i64 @func5(i64 %x, i64 %y) {
-; CHECK-LABEL: func5:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umulh x8, x0, x1
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: cmp xzr, x8
-; CHECK-NEXT: csinv x0, x9, xzr, eq
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func5:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umulh x8, x0, x1
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: cmp xzr, x8
+; CHECK-SD-NEXT: csinv x0, x9, xzr, eq
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func5:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mul x8, x0, x1
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: cset w10, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: csel w9, w10, wzr, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csinv x0, x8, xzr, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 0)
ret i64 %tmp
}
define i4 @func6(i4 %x, i4 %y) {
-; CHECK-LABEL: func6:
-; CHECK: // %bb.0:
-; CHECK-NEXT: lsl w8, w0, #28
-; CHECK-NEXT: and w9, w1, #0xf
-; CHECK-NEXT: umull x8, w8, w9
-; CHECK-NEXT: tst x8, #0xffffffff00000000
-; CHECK-NEXT: csinv w8, w8, wzr, eq
-; CHECK-NEXT: lsr w0, w8, #28
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func6:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: lsl w8, w0, #28
+; CHECK-SD-NEXT: and w9, w1, #0xf
+; CHECK-SD-NEXT: umull x8, w8, w9
+; CHECK-SD-NEXT: tst x8, #0xffffffff00000000
+; CHECK-SD-NEXT: csinv w8, w8, wzr, eq
+; CHECK-SD-NEXT: lsr w0, w8, #28
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func6:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: and w9, w0, #0xf
+; CHECK-GI-NEXT: and w10, w1, #0xf
+; CHECK-GI-NEXT: mov w8, #15 // =0xf
+; CHECK-GI-NEXT: mul w9, w9, w10
+; CHECK-GI-NEXT: cmp w9, #15
+; CHECK-GI-NEXT: csel w0, w9, w8, lo
+; CHECK-GI-NEXT: ret
%tmp = call i4 @llvm.umul.fix.sat.i4(i4 %x, i4 %y, i32 0)
ret i4 %tmp
}
define i64 @func7(i64 %x, i64 %y) {
-; CHECK-LABEL: func7:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: mov w8, #-1 // =0xffffffff
-; CHECK-NEXT: umulh x10, x0, x1
-; CHECK-NEXT: extr x9, x10, x9, #32
-; CHECK-NEXT: cmp x10, x8
-; CHECK-NEXT: csinv x0, x9, xzr, ls
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func7:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: mov w8, #-1 // =0xffffffff
+; CHECK-SD-NEXT: umulh x10, x0, x1
+; CHECK-SD-NEXT: extr x9, x10, x9, #32
+; CHECK-SD-NEXT: cmp x10, x8
+; CHECK-SD-NEXT: csinv x0, x9, xzr, ls
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func7:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mul x8, x0, x1
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: extr x8, x9, x8, #32
+; CHECK-GI-NEXT: lsr x9, x9, #32
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: cset w10, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: csel w9, w10, wzr, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csinv x0, x8, xzr, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 32)
ret i64 %tmp
}
define i64 @func8(i64 %x, i64 %y) {
-; CHECK-LABEL: func8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mul x9, x0, x1
-; CHECK-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-NEXT: umulh x10, x0, x1
-; CHECK-NEXT: extr x9, x10, x9, #63
-; CHECK-NEXT: cmp x10, x8
-; CHECK-NEXT: csinv x0, x9, xzr, ls
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: func8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mul x9, x0, x1
+; CHECK-SD-NEXT: mov x8, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-SD-NEXT: umulh x10, x0, x1
+; CHECK-SD-NEXT: extr x9, x10, x9, #63
+; CHECK-SD-NEXT: cmp x10, x8
+; CHECK-SD-NEXT: csinv x0, x9, xzr, ls
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: func8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mul x8, x0, x1
+; CHECK-GI-NEXT: umulh x9, x0, x1
+; CHECK-GI-NEXT: extr x8, x9, x8, #63
+; CHECK-GI-NEXT: lsr x9, x9, #63
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: cset w10, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: csel w9, w10, wzr, eq
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: csinv x0, x8, xzr, ne
+; CHECK-GI-NEXT: ret
%tmp = call i64 @llvm.umul.fix.sat.i64(i64 %x, i64 %y, i32 63)
ret i64 %tmp
}
define <8 x i8> @vec_v8i8(<8 x i8> %x, <8 x i8> %y) {
-; CHECK-LABEL: vec_v8i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull v0.8h, v0.8b, v1.8b
-; CHECK-NEXT: movi v2.8b, #3
-; CHECK-NEXT: shrn v1.8b, v0.8h, #8
-; CHECK-NEXT: xtn v0.8b, v0.8h
-; CHECK-NEXT: shl v3.8b, v1.8b, #6
-; CHECK-NEXT: usra v3.8b, v0.8b, #2
-; CHECK-NEXT: cmhi v0.8b, v1.8b, v2.8b
-; CHECK-NEXT: orr v0.8b, v3.8b, v0.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v8i8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull v0.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT: movi v2.8b, #3
+; CHECK-SD-NEXT: shrn v1.8b, v0.8h, #8
+; CHECK-SD-NEXT: xtn v0.8b, v0.8h
+; CHECK-SD-NEXT: shl v3.8b, v1.8b, #6
+; CHECK-SD-NEXT: usra v3.8b, v0.8b, #2
+; CHECK-SD-NEXT: cmhi v0.8b, v1.8b, v2.8b
+; CHECK-SD-NEXT: orr v0.8b, v3.8b, v0.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v8i8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v0.8h, v0.8b, v1.8b
+; CHECK-GI-NEXT: uqshrn v0.8b, v0.8h, #2
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i8> @llvm.umul.fix.sat.v8i8(<8 x i8> %x, <8 x i8> %y, i32 2)
ret <8 x i8> %tmp
}
define <16 x i8> @vec_v16i8(<16 x i8> %x, <16 x i8> %y) {
-; CHECK-LABEL: vec_v16i8:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull2 v3.8h, v0.16b, v1.16b
-; CHECK-NEXT: umull v4.8h, v0.8b, v1.8b
-; CHECK-NEXT: movi v2.16b, #3
-; CHECK-NEXT: mul v0.16b, v0.16b, v1.16b
-; CHECK-NEXT: uzp2 v3.16b, v4.16b, v3.16b
-; CHECK-NEXT: shl v1.16b, v3.16b, #6
-; CHECK-NEXT: cmhi v2.16b, v3.16b, v2.16b
-; CHECK-NEXT: usra v1.16b, v0.16b, #2
-; CHECK-NEXT: orr v0.16b, v1.16b, v2.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v16i8:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull2 v3.8h, v0.16b, v1.16b
+; CHECK-SD-NEXT: umull v4.8h, v0.8b, v1.8b
+; CHECK-SD-NEXT: movi v2.16b, #3
+; CHECK-SD-NEXT: mul v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT: uzp2 v3.16b, v4.16b, v3.16b
+; CHECK-SD-NEXT: shl v1.16b, v3.16b, #6
+; CHECK-SD-NEXT: cmhi v2.16b, v3.16b, v2.16b
+; CHECK-SD-NEXT: usra v1.16b, v0.16b, #2
+; CHECK-SD-NEXT: orr v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v16i8:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v2.8h, v0.8b, v1.8b
+; CHECK-GI-NEXT: umull2 v1.8h, v0.16b, v1.16b
+; CHECK-GI-NEXT: uqshrn v0.8b, v2.8h, #2
+; CHECK-GI-NEXT: uqshrn2 v0.16b, v1.8h, #2
+; CHECK-GI-NEXT: ret
%tmp = call <16 x i8> @llvm.umul.fix.sat.v16i8(<16 x i8> %x, <16 x i8> %y, i32 2)
ret <16 x i8> %tmp
}
define <4 x i16> @vec_v4i16(<4 x i16> %x, <4 x i16> %y) {
-; CHECK-LABEL: vec_v4i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull v0.4s, v0.4h, v1.4h
-; CHECK-NEXT: mvni v2.4h, #252, lsl #8
-; CHECK-NEXT: shrn v1.4h, v0.4s, #16
-; CHECK-NEXT: xtn v0.4h, v0.4s
-; CHECK-NEXT: shl v3.4h, v1.4h, #6
-; CHECK-NEXT: usra v3.4h, v0.4h, #10
-; CHECK-NEXT: cmhi v0.4h, v1.4h, v2.4h
-; CHECK-NEXT: orr v0.8b, v3.8b, v0.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v4i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull v0.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT: mvni v2.4h, #252, lsl #8
+; CHECK-SD-NEXT: shrn v1.4h, v0.4s, #16
+; CHECK-SD-NEXT: xtn v0.4h, v0.4s
+; CHECK-SD-NEXT: shl v3.4h, v1.4h, #6
+; CHECK-SD-NEXT: usra v3.4h, v0.4h, #10
+; CHECK-SD-NEXT: cmhi v0.4h, v1.4h, v2.4h
+; CHECK-SD-NEXT: orr v0.8b, v3.8b, v0.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v4i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v0.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT: uqshrn v0.4h, v0.4s, #10
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i16> @llvm.umul.fix.sat.v4i16(<4 x i16> %x, <4 x i16> %y, i32 10)
ret <4 x i16> %tmp
}
define <8 x i16> @vec_v8i16(<8 x i16> %x, <8 x i16> %y) {
-; CHECK-LABEL: vec_v8i16:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull2 v3.4s, v0.8h, v1.8h
-; CHECK-NEXT: umull v4.4s, v0.4h, v1.4h
-; CHECK-NEXT: movi v2.8h, #3
-; CHECK-NEXT: mul v0.8h, v0.8h, v1.8h
-; CHECK-NEXT: uzp2 v3.8h, v4.8h, v3.8h
-; CHECK-NEXT: shl v1.8h, v3.8h, #14
-; CHECK-NEXT: cmhi v2.8h, v3.8h, v2.8h
-; CHECK-NEXT: usra v1.8h, v0.8h, #2
-; CHECK-NEXT: orr v0.16b, v1.16b, v2.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v8i16:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull2 v3.4s, v0.8h, v1.8h
+; CHECK-SD-NEXT: umull v4.4s, v0.4h, v1.4h
+; CHECK-SD-NEXT: movi v2.8h, #3
+; CHECK-SD-NEXT: mul v0.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT: uzp2 v3.8h, v4.8h, v3.8h
+; CHECK-SD-NEXT: shl v1.8h, v3.8h, #14
+; CHECK-SD-NEXT: cmhi v2.8h, v3.8h, v2.8h
+; CHECK-SD-NEXT: usra v1.8h, v0.8h, #2
+; CHECK-SD-NEXT: orr v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v8i16:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v2.4s, v0.4h, v1.4h
+; CHECK-GI-NEXT: umull2 v1.4s, v0.8h, v1.8h
+; CHECK-GI-NEXT: uqshrn v0.4h, v2.4s, #2
+; CHECK-GI-NEXT: uqshrn2 v0.8h, v1.4s, #2
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i16> @llvm.umul.fix.sat.v8i16(<8 x i16> %x, <8 x i16> %y, i32 2)
ret <8 x i16> %tmp
}
define <2 x i32> @vec_v2i32(<2 x i32> %x, <2 x i32> %y) {
-; CHECK-LABEL: vec_v2i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull v0.2d, v0.2s, v1.2s
-; CHECK-NEXT: movi v2.2d, #0000000000000000
-; CHECK-NEXT: shrn v1.2s, v0.2d, #32
-; CHECK-NEXT: xtn v0.2s, v0.2d
-; CHECK-NEXT: add v3.2s, v1.2s, v1.2s
-; CHECK-NEXT: cmhi v1.2s, v1.2s, v2.2s
-; CHECK-NEXT: shl v2.2s, v3.2s, #31
-; CHECK-NEXT: orr v0.8b, v0.8b, v1.8b
-; CHECK-NEXT: orr v0.8b, v2.8b, v0.8b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v2i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull v0.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT: movi v2.2d, #0000000000000000
+; CHECK-SD-NEXT: shrn v1.2s, v0.2d, #32
+; CHECK-SD-NEXT: xtn v0.2s, v0.2d
+; CHECK-SD-NEXT: add v3.2s, v1.2s, v1.2s
+; CHECK-SD-NEXT: cmhi v1.2s, v1.2s, v2.2s
+; CHECK-SD-NEXT: shl v2.2s, v3.2s, #31
+; CHECK-SD-NEXT: orr v0.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT: orr v0.8b, v2.8b, v0.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v2i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v0.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT: uqxtn v0.2s, v0.2d
+; CHECK-GI-NEXT: ret
%tmp = call <2 x i32> @llvm.umul.fix.sat.v2i32(<2 x i32> %x, <2 x i32> %y, i32 0)
ret <2 x i32> %tmp
}
define <4 x i32> @vec_v4i32(<4 x i32> %x, <4 x i32> %y) {
-; CHECK-LABEL: vec_v4i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull2 v3.2d, v0.4s, v1.4s
-; CHECK-NEXT: umull v4.2d, v0.2s, v1.2s
-; CHECK-NEXT: movi v2.4s, #127, msl #8
-; CHECK-NEXT: mul v0.4s, v0.4s, v1.4s
-; CHECK-NEXT: uzp2 v3.4s, v4.4s, v3.4s
-; CHECK-NEXT: shl v1.4s, v3.4s, #17
-; CHECK-NEXT: cmhi v2.4s, v3.4s, v2.4s
-; CHECK-NEXT: usra v1.4s, v0.4s, #15
-; CHECK-NEXT: orr v0.16b, v1.16b, v2.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v4i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull2 v3.2d, v0.4s, v1.4s
+; CHECK-SD-NEXT: umull v4.2d, v0.2s, v1.2s
+; CHECK-SD-NEXT: movi v2.4s, #127, msl #8
+; CHECK-SD-NEXT: mul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT: uzp2 v3.4s, v4.4s, v3.4s
+; CHECK-SD-NEXT: shl v1.4s, v3.4s, #17
+; CHECK-SD-NEXT: cmhi v2.4s, v3.4s, v2.4s
+; CHECK-SD-NEXT: usra v1.4s, v0.4s, #15
+; CHECK-SD-NEXT: orr v0.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v4i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v2.2d, v0.2s, v1.2s
+; CHECK-GI-NEXT: umull2 v1.2d, v0.4s, v1.4s
+; CHECK-GI-NEXT: uqshrn v0.2s, v2.2d, #15
+; CHECK-GI-NEXT: uqshrn2 v0.4s, v1.2d, #15
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i32> @llvm.umul.fix.sat.v4i32(<4 x i32> %x, <4 x i32> %y, i32 15)
ret <4 x i32> %tmp
}
define <8 x i32> @vec_v8i32(<8 x i32> %x, <8 x i32> %y) {
-; CHECK-LABEL: vec_v8i32:
-; CHECK: // %bb.0:
-; CHECK-NEXT: umull2 v4.2d, v0.4s, v2.4s
-; CHECK-NEXT: umull v5.2d, v0.2s, v2.2s
-; CHECK-NEXT: umull2 v6.2d, v1.4s, v3.4s
-; CHECK-NEXT: umull v7.2d, v1.2s, v3.2s
-; CHECK-NEXT: mul v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: mul v1.4s, v1.4s, v3.4s
-; CHECK-NEXT: uzp2 v4.4s, v5.4s, v4.4s
-; CHECK-NEXT: uzp2 v5.4s, v7.4s, v6.4s
-; CHECK-NEXT: movi v6.2d, #0000000000000000
-; CHECK-NEXT: add v2.4s, v4.4s, v4.4s
-; CHECK-NEXT: cmhi v3.4s, v4.4s, v6.4s
-; CHECK-NEXT: cmhi v4.4s, v5.4s, v6.4s
-; CHECK-NEXT: add v5.4s, v5.4s, v5.4s
-; CHECK-NEXT: shl v2.4s, v2.4s, #31
-; CHECK-NEXT: orr v0.16b, v0.16b, v3.16b
-; CHECK-NEXT: shl v3.4s, v5.4s, #31
-; CHECK-NEXT: orr v1.16b, v1.16b, v4.16b
-; CHECK-NEXT: orr v0.16b, v2.16b, v0.16b
-; CHECK-NEXT: orr v1.16b, v3.16b, v1.16b
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v8i32:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: umull2 v4.2d, v0.4s, v2.4s
+; CHECK-SD-NEXT: umull v5.2d, v0.2s, v2.2s
+; CHECK-SD-NEXT: umull2 v6.2d, v1.4s, v3.4s
+; CHECK-SD-NEXT: umull v7.2d, v1.2s, v3.2s
+; CHECK-SD-NEXT: mul v0.4s, v0.4s, v2.4s
+; CHECK-SD-NEXT: mul v1.4s, v1.4s, v3.4s
+; CHECK-SD-NEXT: uzp2 v4.4s, v5.4s, v4.4s
+; CHECK-SD-NEXT: uzp2 v5.4s, v7.4s, v6.4s
+; CHECK-SD-NEXT: movi v6.2d, #0000000000000000
+; CHECK-SD-NEXT: add v2.4s, v4.4s, v4.4s
+; CHECK-SD-NEXT: cmhi v3.4s, v4.4s, v6.4s
+; CHECK-SD-NEXT: cmhi v4.4s, v5.4s, v6.4s
+; CHECK-SD-NEXT: add v5.4s, v5.4s, v5.4s
+; CHECK-SD-NEXT: shl v2.4s, v2.4s, #31
+; CHECK-SD-NEXT: orr v0.16b, v0.16b, v3.16b
+; CHECK-SD-NEXT: shl v3.4s, v5.4s, #31
+; CHECK-SD-NEXT: orr v1.16b, v1.16b, v4.16b
+; CHECK-SD-NEXT: orr v0.16b, v2.16b, v0.16b
+; CHECK-SD-NEXT: orr v1.16b, v3.16b, v1.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v8i32:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: umull v4.2d, v0.2s, v2.2s
+; CHECK-GI-NEXT: umull v5.2d, v1.2s, v3.2s
+; CHECK-GI-NEXT: umull2 v2.2d, v0.4s, v2.4s
+; CHECK-GI-NEXT: umull2 v3.2d, v1.4s, v3.4s
+; CHECK-GI-NEXT: uqxtn v0.2s, v4.2d
+; CHECK-GI-NEXT: uqxtn v1.2s, v5.2d
+; CHECK-GI-NEXT: uqxtn2 v0.4s, v2.2d
+; CHECK-GI-NEXT: uqxtn2 v1.4s, v3.2d
+; CHECK-GI-NEXT: ret
%tmp = call <8 x i32> @llvm.umul.fix.sat.v8i32(<8 x i32> %x, <8 x i32> %y, i32 0)
ret <8 x i32> %tmp
}
define <2 x i64> @vec_v2i64(<2 x i64> %x, <2 x i64> %y) {
-; CHECK-LABEL: vec_v2i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov x8, v1.d[1]
-; CHECK-NEXT: mov x9, v0.d[1]
-; CHECK-NEXT: fmov x11, d0
-; CHECK-NEXT: mul x10, x9, x8
-; CHECK-NEXT: umulh x8, x9, x8
-; CHECK-NEXT: fmov x9, d1
-; CHECK-NEXT: mul x12, x11, x9
-; CHECK-NEXT: umulh x9, x11, x9
-; CHECK-NEXT: extr x10, x8, x10, #15
-; CHECK-NEXT: cmp x8, #8, lsl #12 // =32768
-; CHECK-NEXT: csinv x10, x10, xzr, lo
-; CHECK-NEXT: extr x8, x9, x12, #15
-; CHECK-NEXT: cmp x9, #8, lsl #12 // =32768
-; CHECK-NEXT: csinv x8, x8, xzr, lo
-; CHECK-NEXT: fmov d0, x8
-; CHECK-NEXT: mov v0.d[1], x10
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v2i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov x8, v1.d[1]
+; CHECK-SD-NEXT: mov x9, v0.d[1]
+; CHECK-SD-NEXT: fmov x11, d0
+; CHECK-SD-NEXT: mul x10, x9, x8
+; CHECK-SD-NEXT: umulh x8, x9, x8
+; CHECK-SD-NEXT: fmov x9, d1
+; CHECK-SD-NEXT: mul x12, x11, x9
+; CHECK-SD-NEXT: umulh x9, x11, x9
+; CHECK-SD-NEXT: extr x10, x8, x10, #15
+; CHECK-SD-NEXT: cmp x8, #8, lsl #12 // =32768
+; CHECK-SD-NEXT: csinv x10, x10, xzr, lo
+; CHECK-SD-NEXT: extr x8, x9, x12, #15
+; CHECK-SD-NEXT: cmp x9, #8, lsl #12 // =32768
+; CHECK-SD-NEXT: csinv x8, x8, xzr, lo
+; CHECK-SD-NEXT: fmov d0, x8
+; CHECK-SD-NEXT: mov v0.d[1], x10
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v2i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov d2, v0.d[1]
+; CHECK-GI-NEXT: mov d3, v1.d[1]
+; CHECK-GI-NEXT: fmov x8, d0
+; CHECK-GI-NEXT: fmov x9, d1
+; CHECK-GI-NEXT: mul x10, x8, x9
+; CHECK-GI-NEXT: fmov x11, d3
+; CHECK-GI-NEXT: umulh x8, x8, x9
+; CHECK-GI-NEXT: fmov x9, d2
+; CHECK-GI-NEXT: mul x12, x9, x11
+; CHECK-GI-NEXT: umulh x9, x9, x11
+; CHECK-GI-NEXT: extr x10, x8, x10, #15
+; CHECK-GI-NEXT: lsr x8, x8, #15
+; CHECK-GI-NEXT: cmn x10, #1
+; CHECK-GI-NEXT: extr x11, x9, x12, #15
+; CHECK-GI-NEXT: lsr x9, x9, #15
+; CHECK-GI-NEXT: cset w12, lo
+; CHECK-GI-NEXT: cmp x8, #0
+; CHECK-GI-NEXT: csel w8, w12, wzr, eq
+; CHECK-GI-NEXT: cmn x11, #1
+; CHECK-GI-NEXT: cset w12, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: csel w9, w12, wzr, eq
+; CHECK-GI-NEXT: tst w8, #0x1
+; CHECK-GI-NEXT: csinv x8, x10, xzr, ne
+; CHECK-GI-NEXT: tst w9, #0x1
+; CHECK-GI-NEXT: fmov d0, x8
+; CHECK-GI-NEXT: csinv x9, x11, xzr, ne
+; CHECK-GI-NEXT: mov v0.d[1], x9
+; CHECK-GI-NEXT: ret
%tmp = call <2 x i64> @llvm.umul.fix.sat.v2i64(<2 x i64> %x, <2 x i64> %y, i32 15)
ret <2 x i64> %tmp
}
define <4 x i64> @vec_v4i64(<4 x i64> %x, <4 x i64> %y) {
-; CHECK-LABEL: vec_v4i64:
-; CHECK: // %bb.0:
-; CHECK-NEXT: mov x8, v2.d[1]
-; CHECK-NEXT: mov x9, v0.d[1]
-; CHECK-NEXT: mov x14, v3.d[1]
-; CHECK-NEXT: mov x15, v1.d[1]
-; CHECK-NEXT: fmov x10, d2
-; CHECK-NEXT: fmov x11, d0
-; CHECK-NEXT: mul x12, x11, x10
-; CHECK-NEXT: mul x13, x9, x8
-; CHECK-NEXT: umulh x8, x9, x8
-; CHECK-NEXT: umulh x9, x11, x10
-; CHECK-NEXT: mul x10, x15, x14
-; CHECK-NEXT: extr x13, x8, x13, #32
-; CHECK-NEXT: umulh x11, x15, x14
-; CHECK-NEXT: fmov x14, d3
-; CHECK-NEXT: fmov x15, d1
-; CHECK-NEXT: mul x16, x15, x14
-; CHECK-NEXT: umulh x14, x15, x14
-; CHECK-NEXT: mov w15, #-1 // =0xffffffff
-; CHECK-NEXT: cmp x8, x15
-; CHECK-NEXT: extr x8, x9, x12, #32
-; CHECK-NEXT: csinv x12, x13, xzr, ls
-; CHECK-NEXT: cmp x9, x15
-; CHECK-NEXT: extr x9, x11, x10, #32
-; CHECK-NEXT: csinv x8, x8, xzr, ls
-; CHECK-NEXT: cmp x11, x15
-; CHECK-NEXT: csinv x9, x9, xzr, ls
-; CHECK-NEXT: fmov d0, x8
-; CHECK-NEXT: extr x10, x14, x16, #32
-; CHECK-NEXT: cmp x14, x15
-; CHECK-NEXT: csinv x10, x10, xzr, ls
-; CHECK-NEXT: mov v0.d[1], x12
-; CHECK-NEXT: fmov d1, x10
-; CHECK-NEXT: mov v1.d[1], x9
-; CHECK-NEXT: ret
+; CHECK-SD-LABEL: vec_v4i64:
+; CHECK-SD: // %bb.0:
+; CHECK-SD-NEXT: mov x8, v2.d[1]
+; CHECK-SD-NEXT: mov x9, v0.d[1]
+; CHECK-SD-NEXT: mov x14, v3.d[1]
+; CHECK-SD-NEXT: mov x15, v1.d[1]
+; CHECK-SD-NEXT: fmov x10, d2
+; CHECK-SD-NEXT: fmov x11, d0
+; CHECK-SD-NEXT: mul x12, x11, x10
+; CHECK-SD-NEXT: mul x13, x9, x8
+; CHECK-SD-NEXT: umulh x8, x9, x8
+; CHECK-SD-NEXT: umulh x9, x11, x10
+; CHECK-SD-NEXT: mul x10, x15, x14
+; CHECK-SD-NEXT: extr x13, x8, x13, #32
+; CHECK-SD-NEXT: umulh x11, x15, x14
+; CHECK-SD-NEXT: fmov x14, d3
+; CHECK-SD-NEXT: fmov x15, d1
+; CHECK-SD-NEXT: mul x16, x15, x14
+; CHECK-SD-NEXT: umulh x14, x15, x14
+; CHECK-SD-NEXT: mov w15, #-1 // =0xffffffff
+; CHECK-SD-NEXT: cmp x8, x15
+; CHECK-SD-NEXT: extr x8, x9, x12, #32
+; CHECK-SD-NEXT: csinv x12, x13, xzr, ls
+; CHECK-SD-NEXT: cmp x9, x15
+; CHECK-SD-NEXT: extr x9, x11, x10, #32
+; CHECK-SD-NEXT: csinv x8, x8, xzr, ls
+; CHECK-SD-NEXT: cmp x11, x15
+; CHECK-SD-NEXT: csinv x9, x9, xzr, ls
+; CHECK-SD-NEXT: fmov d0, x8
+; CHECK-SD-NEXT: extr x10, x14, x16, #32
+; CHECK-SD-NEXT: cmp x14, x15
+; CHECK-SD-NEXT: csinv x10, x10, xzr, ls
+; CHECK-SD-NEXT: mov v0.d[1], x12
+; CHECK-SD-NEXT: fmov d1, x10
+; CHECK-SD-NEXT: mov v1.d[1], x9
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: vec_v4i64:
+; CHECK-GI: // %bb.0:
+; CHECK-GI-NEXT: mov d4, v0.d[1]
+; CHECK-GI-NEXT: mov d5, v2.d[1]
+; CHECK-GI-NEXT: fmov x8, d0
+; CHECK-GI-NEXT: fmov x9, d2
+; CHECK-GI-NEXT: mov d0, v1.d[1]
+; CHECK-GI-NEXT: mov d2, v3.d[1]
+; CHECK-GI-NEXT: fmov x13, d3
+; CHECK-GI-NEXT: mul x10, x8, x9
+; CHECK-GI-NEXT: fmov x11, d5
+; CHECK-GI-NEXT: fmov x15, d2
+; CHECK-GI-NEXT: umulh x8, x8, x9
+; CHECK-GI-NEXT: fmov x9, d4
+; CHECK-GI-NEXT: mul x12, x9, x11
+; CHECK-GI-NEXT: umulh x9, x9, x11
+; CHECK-GI-NEXT: fmov x11, d1
+; CHECK-GI-NEXT: extr x10, x8, x10, #32
+; CHECK-GI-NEXT: lsr x8, x8, #32
+; CHECK-GI-NEXT: cmn x10, #1
+; CHECK-GI-NEXT: mul x14, x11, x13
+; CHECK-GI-NEXT: umulh x11, x11, x13
+; CHECK-GI-NEXT: fmov x13, d0
+; CHECK-GI-NEXT: extr x12, x9, x12, #32
+; CHECK-GI-NEXT: lsr x9, x9, #32
+; CHECK-GI-NEXT: mul x16, x13, x15
+; CHECK-GI-NEXT: umulh x13, x13, x15
+; CHECK-GI-NEXT: cset w15, lo
+; CHECK-GI-NEXT: cmp x8, #0
+; CHECK-GI-NEXT: extr x8, x11, x14, #32
+; CHECK-GI-NEXT: csel w14, w15, wzr, eq
+; CHECK-GI-NEXT: cmn x12, #1
+; CHECK-GI-NEXT: lsr x11, x11, #32
+; CHECK-GI-NEXT: cset w15, lo
+; CHECK-GI-NEXT: cmp x9, #0
+; CHECK-GI-NEXT: csel w15, w15, wzr, eq
+; CHECK-GI-NEXT: cmn x8, #1
+; CHECK-GI-NEXT: extr x9, x13, x16, #32
+; CHECK-GI-NEXT: lsr x13, x13, #32
+; CHECK-GI-NEXT: cset w16, lo
+; CHECK-GI-NEXT: cmp x11, #0
+; CHECK-GI-NEXT: csel w11, w16, wzr, eq
+; CHECK-GI-NEXT: cmn x9, #1
+; CHECK-GI-NEXT: cset w16, lo
+; CHECK-GI-NEXT: cmp x13, #0
+; CHECK-GI-NEXT: csel w13, w16, wzr, eq
+; CHECK-GI-NEXT: tst w14, #0x1
+; CHECK-GI-NEXT: csinv x10, x10, xzr, ne
+; CHECK-GI-NEXT: tst w15, #0x1
+; CHECK-GI-NEXT: csinv x12, x12, xzr, ne
+; CHECK-GI-NEXT: tst w11, #0x1
+; CHECK-GI-NEXT: fmov d0, x10
+; CHECK-GI-NEXT: csinv x8, x8, xzr, ne
+; CHECK-GI-NEXT: tst w13, #0x1
+; CHECK-GI-NEXT: fmov d1, x8
+; CHECK-GI-NEXT: csinv x9, x9, xzr, ne
+; CHECK-GI-NEXT: mov v0.d[1], x12
+; CHECK-GI-NEXT: mov v1.d[1], x9
+; CHECK-GI-NEXT: ret
%tmp = call <4 x i64> @llvm.umul.fix.sat.v4i64(<4 x i64> %x, <4 x i64> %y, i32 32)
ret <4 x i64> %tmp
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-GI: {{.*}}
-; CHECK-SD: {{.*}}
+; CHECK: {{.*}}
>From 69bb3ac7df9fa05fd7f33a2cd0a51c4f2286e588 Mon Sep 17 00:00:00 2001
From: David Green <david.green at arm.com>
Date: Wed, 19 Aug 2026 10:08:36 +0100
Subject: [PATCH 2/2] [AArch64][GlobalISel] Clamp TruncSat num elements.
Just like v2i32, we can clamp v8i8 and v4i16 vectors to make use of truncsat in
more places and split larger vectors with fewerElements.
---
.../AArch64/GISel/AArch64LegalizerInfo.cpp | 2 +
llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll | 38 ++++---------------
2 files changed, 10 insertions(+), 30 deletions(-)
diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 91ad1911818f6..5c1d43139f0a4 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -860,6 +860,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
getActionDefinitionsBuilder({G_TRUNC_SSAT_S, G_TRUNC_SSAT_U, G_TRUNC_USAT_U})
.legalFor({{v8i8, v8i16}, {v4i16, v4i32}, {v2i32, v2i64}})
+ .clampNumElements(0, v8s8, v8s8)
+ .clampNumElements(0, v4s16, v4s16)
.clampNumElements(0, v2s32, v2s32)
.lower();
diff --git a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
index dcb0051bd4c3c..b406a8c43b12d 100644
--- a/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
+++ b/llvm/test/CodeGen/AArch64/fpclamptosat_vec.ll
@@ -459,20 +459,10 @@ define <8 x i16> @utest_f16i16(<8 x half> %x) {
; CHECK-CVT-NEXT: uqxtn2 v0.8h, v2.4s
; CHECK-CVT-NEXT: ret
;
-; CHECK-FP16-SD-LABEL: utest_f16i16:
-; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: fcvtzu v0.8h, v0.8h
-; CHECK-FP16-SD-NEXT: ret
-;
-; CHECK-FP16-GI-LABEL: utest_f16i16:
-; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-FP16-GI-NEXT: fcvtzu v1.4s, v1.4s
-; CHECK-FP16-GI-NEXT: fcvtzu v2.4s, v0.4s
-; CHECK-FP16-GI-NEXT: uqxtn v0.4h, v1.4s
-; CHECK-FP16-GI-NEXT: uqxtn2 v0.8h, v2.4s
-; CHECK-FP16-GI-NEXT: ret
+; CHECK-FP16-LABEL: utest_f16i16:
+; CHECK-FP16: // %bb.0: // %entry
+; CHECK-FP16-NEXT: fcvtzu v0.8h, v0.8h
+; CHECK-FP16-NEXT: ret
entry:
%conv = fptoui <8 x half> %x to <8 x i32>
%0 = icmp ult <8 x i32> %conv, <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>
@@ -2252,20 +2242,10 @@ define <8 x i16> @utest_f16i16_mm(<8 x half> %x) {
; CHECK-CVT-NEXT: uqxtn2 v0.8h, v2.4s
; CHECK-CVT-NEXT: ret
;
-; CHECK-FP16-SD-LABEL: utest_f16i16_mm:
-; CHECK-FP16-SD: // %bb.0: // %entry
-; CHECK-FP16-SD-NEXT: fcvtzu v0.8h, v0.8h
-; CHECK-FP16-SD-NEXT: ret
-;
-; CHECK-FP16-GI-LABEL: utest_f16i16_mm:
-; CHECK-FP16-GI: // %bb.0: // %entry
-; CHECK-FP16-GI-NEXT: fcvtl v1.4s, v0.4h
-; CHECK-FP16-GI-NEXT: fcvtl2 v0.4s, v0.8h
-; CHECK-FP16-GI-NEXT: fcvtzu v1.4s, v1.4s
-; CHECK-FP16-GI-NEXT: fcvtzu v2.4s, v0.4s
-; CHECK-FP16-GI-NEXT: uqxtn v0.4h, v1.4s
-; CHECK-FP16-GI-NEXT: uqxtn2 v0.8h, v2.4s
-; CHECK-FP16-GI-NEXT: ret
+; CHECK-FP16-LABEL: utest_f16i16_mm:
+; CHECK-FP16: // %bb.0: // %entry
+; CHECK-FP16-NEXT: fcvtzu v0.8h, v0.8h
+; CHECK-FP16-NEXT: ret
entry:
%conv = fptoui <8 x half> %x to <8 x i32>
%spec.store.select = call <8 x i32> @llvm.umin.v8i32(<8 x i32> %conv, <8 x i32> <i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535, i32 65535>)
@@ -3648,5 +3628,3 @@ declare <4 x i64> @llvm.umin.v4i64(<4 x i64>, <4 x i64>)
declare <2 x i128> @llvm.smin.v2i128(<2 x i128>, <2 x i128>)
declare <2 x i128> @llvm.smax.v2i128(<2 x i128>, <2 x i128>)
declare <2 x i128> @llvm.umin.v2i128(<2 x i128>, <2 x i128>)
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; CHECK-FP16: {{.*}}
More information about the llvm-commits
mailing list