[llvm-branch-commits] [llvm] [ConstantTime] Native ct.select support for X86 and i386 (PR #166704)
Julius Alexandre via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Sep 1 10:49:52 PDT 2026
https://github.com/wizardengineer updated https://github.com/llvm/llvm-project/pull/166704
>From e9aa37a2c1eb0c050454ab2118c268b895fc085e Mon Sep 17 00:00:00 2001
From: wizardengineer <juliuswoosebert at gmail.com>
Date: Wed, 5 Nov 2025 17:09:23 -0500
Subject: [PATCH 1/4] [LLVM][X86] Add native ct.select support for X86 and i386
Add native X86 implementation with CMOV instructions and comprehensive tests:
- X86 ISelLowering with CMOV for x86_64 and i386
- Fallback bitwise operations for i386 targets without CMOV
- Post-RA expansion for pseudo-instructions
- Comprehensive test coverage:
- Edge cases (zero conditions, large integers)
- i386-specific tests (FP, MMX, non-CMOV fallback)
- Vector operations
- Optimization patterns
The basic test demonstrating fallback is in the core infrastructure PR.
---
llvm/lib/Target/X86/X86.td | 8 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 798 ++++++++++-
llvm/lib/Target/X86/X86ISelLowering.h | 3 +
llvm/lib/Target/X86/X86InstrCMovSetCC.td | 205 +++
llvm/lib/Target/X86/X86InstrCompiler.td | 81 ++
llvm/lib/Target/X86/X86InstrFragments.td | 6 +
llvm/lib/Target/X86/X86InstrInfo.cpp | 603 +++++++-
llvm/lib/Target/X86/X86InstrInfo.h | 6 +
llvm/lib/Target/X86/X86InstrPredicates.td | 5 +
llvm/lib/Target/X86/X86TargetMachine.cpp | 5 +-
llvm/test/CodeGen/X86/ctselect-edge-cases.ll | 409 ++++++
llvm/test/CodeGen/X86/ctselect-i386-fp.ll | 722 ++++++++++
llvm/test/CodeGen/X86/ctselect-i386-mmx.ll | 428 ++++++
llvm/test/CodeGen/X86/ctselect-i386.ll | 267 ++++
.../test/CodeGen/X86/ctselect-optimization.ll | 304 ++++
llvm/test/CodeGen/X86/ctselect-vector.ll | 1274 +++++++++++++++++
16 files changed, 5081 insertions(+), 43 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/ctselect-edge-cases.ll
create mode 100644 llvm/test/CodeGen/X86/ctselect-i386-fp.ll
create mode 100644 llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
create mode 100644 llvm/test/CodeGen/X86/ctselect-i386.ll
create mode 100644 llvm/test/CodeGen/X86/ctselect-optimization.ll
create mode 100644 llvm/test/CodeGen/X86/ctselect-vector.ll
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 5786c659b0f98..d02f51be95f3d 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -959,9 +959,10 @@ include "X86ScheduleC864GM8.td"
def ProcessorFeatures {
// x86-64 micro-architecture levels: x86-64 and x86-64-v[234]
- list<SubtargetFeature> X86_64V1Features = [
- FeatureX87, FeatureCX8, FeatureCMOV, FeatureMMX, FeatureSSE2,
- FeatureFXSR, FeatureNOPL, FeatureX86_64,
+ list<SubtargetFeature> X86_64V1Features = [FeatureX87, FeatureCX8,
+ FeatureCMOV, FeatureMMX,
+ FeatureSSE2, FeatureFXSR,
+ FeatureNOPL, FeatureX86_64,
];
list<SubtargetFeature> X86_64V1Tuning = [
TuningMacroFusion,
@@ -1292,6 +1293,7 @@ def ProcessorFeatures {
FeatureAVXNECONVERT,
FeatureAVXVNNIINT8,
FeatureAVXVNNIINT16,
+ FeatureUSERMSR,
FeatureSHA512,
FeatureSM3,
FeatureEGPR,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 71cba0b21af82..208707e12978b 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "X86ISelLowering.h"
+#include "MCTargetDesc/X86MCTargetDesc.h"
#include "MCTargetDesc/X86ShuffleDecode.h"
#include "X86.h"
#include "X86FrameLowering.h"
@@ -29,6 +30,8 @@
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/ISDOpcodes.h"
+#include "llvm/CodeGen/IntrinsicLowering.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -48,6 +51,7 @@
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/PatternMatch.h"
@@ -501,6 +505,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// X86 wants to expand cmov itself.
for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
@@ -509,6 +514,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
if (VT == MVT::i64 && !Subtarget.is64Bit())
continue;
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::SETCC, VT, Custom);
}
@@ -516,6 +522,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// Custom action for SELECT MMX and expand action for SELECT_CC MMX
setOperationAction(ISD::SELECT, MVT::x86mmx, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::x86mmx, Custom);
setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand);
setOperationAction(ISD::EH_RETURN , MVT::Other, Custom);
@@ -645,6 +652,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::BR_CC, VT, Action);
setOperationAction(ISD::SETCC, VT, Action);
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::SELECT_CC, VT, Action);
setOperationAction(ISD::FROUND, VT, Action);
setOperationAction(ISD::FROUNDEVEN, VT, Action);
@@ -1102,7 +1110,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
- setOperationAction(ISD::FCANONICALIZE, MVT::v4f32, Expand);
+ setOperationAction(ISD::CT_SELECT, MVT::v4f32, Custom);
+ setOperationAction(ISD::FCANONICALIZE, MVT::v4f32, Expand);
setOperationAction(ISD::LOAD, MVT::v2f32, Custom);
setOperationAction(ISD::STORE, MVT::v2f32, Custom);
@@ -1272,6 +1281,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SELECT, MVT::v8f16, Custom);
setOperationAction(ISD::SELECT, MVT::v16i8, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v2f64, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v2i64, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v4i32, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v8i16, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v8f16, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v16i8, Custom);
+
setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Custom);
setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Custom);
setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom);
@@ -1614,6 +1630,14 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SELECT, MVT::v32i8, Custom);
setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v4f64, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v4i64, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v8i32, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v16i16, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v16f16, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v32i8, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v8f32, Custom);
+
for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
@@ -1827,6 +1851,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
setOperationAction(ISD::SELECT, MVT::v1i1, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v1i1, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom);
setOperationAction(ISD::BUILD_VECTOR, MVT::v1i1, Custom);
@@ -1872,6 +1897,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {
setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::TRUNCATE, VT, Custom);
setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
@@ -2169,6 +2195,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::VSELECT, VT, Custom);
setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
@@ -2369,6 +2396,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
@@ -2445,6 +2473,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VSELECT, VT, Legal);
setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::FNEG, VT, Custom);
setOperationAction(ISD::FABS, VT, Custom);
@@ -2705,6 +2734,22 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addRegisterClass(MVT::x86amx, &X86::TILERegClass);
}
+ // Handle 512-bit vector CT_SELECT without AVX512 by setting them to Expand
+ // This allows type legalization to split them into smaller vectors
+ for (auto VT : {MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64, MVT::v32f16,
+ MVT::v16f32, MVT::v8f64}) {
+ setOperationAction(ISD::CT_SELECT, VT, Expand);
+ }
+
+ // Handle 256-bit vector CT_SELECT without AVX by setting them to Expand
+ // This allows type legalization to split them into 128-bit vectors
+ if (!Subtarget.hasAVX()) {
+ for (auto VT : {MVT::v4f64, MVT::v4i64, MVT::v8i32, MVT::v16i16,
+ MVT::v16f16, MVT::v32i8, MVT::v8f32}) {
+ setOperationAction(ISD::CT_SELECT, VT, Expand);
+ }
+ }
+
// We want to custom lower some of our intrinsics.
setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
@@ -2815,6 +2860,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
ISD::BITCAST,
ISD::VSELECT,
ISD::SELECT,
+ ISD::CT_SELECT,
ISD::SHL,
ISD::SRA,
ISD::SRL,
@@ -26068,6 +26114,174 @@ static SDValue LowerSIGN_EXTEND_Mask(SDValue Op, const SDLoc &dl,
return V;
}
+SDValue X86TargetLowering::LowerCT_SELECT(SDValue Op, SelectionDAG &DAG) const {
+ SDValue Cond = Op.getOperand(0); // condition
+ SDValue TrueOp = Op.getOperand(1); // true_value
+ SDValue FalseOp = Op.getOperand(2); // false_value
+ SDLoc DL(Op);
+ MVT VT = TrueOp.getSimpleValueType();
+
+ // Special handling for i386 targets (no CMOV) - route to post-RA expansion
+ // pseudos Let standard type legalization handle i64 automatically (splits
+ // into EDX:EAX)
+
+ // Handle soft float16 by converting to integer operations
+ if (isSoftF16(VT, Subtarget)) {
+ MVT NVT = VT.changeTypeToInteger();
+ SDValue CtSelect =
+ DAG.getNode(ISD::CT_SELECT, DL, NVT, Cond, DAG.getBitcast(NVT, FalseOp),
+ DAG.getBitcast(NVT, TrueOp));
+ return DAG.getBitcast(VT, CtSelect);
+ }
+
+ // Handle vector types
+ if (VT.isVector()) {
+ // Handle soft float16 vectors
+ if (isSoftF16(VT, Subtarget)) {
+ MVT NVT = VT.changeVectorElementTypeToInteger();
+ SDValue CtSelect = DAG.getNode(ISD::CT_SELECT, DL, NVT, Cond,
+ DAG.getBitcast(NVT, FalseOp),
+ DAG.getBitcast(NVT, TrueOp));
+ return DAG.getBitcast(VT, CtSelect);
+ }
+
+ unsigned VectorWidth = VT.getSizeInBits();
+ MVT EltVT = VT.getVectorElementType();
+
+ // 512-bit vectors without AVX512 are now handled by type legalization
+ // (Expand action) 256-bit vectors without AVX are now handled by type
+ // legalization (Expand action)
+
+ if (VectorWidth == 128 && !Subtarget.hasSSE1())
+ return SDValue();
+
+ // Handle special cases for floating point vectors
+ if (EltVT.isFloatingPoint()) {
+ // For vector floating point with AVX, use VBLENDV-style operations
+ if (Subtarget.hasAVX() && (VectorWidth == 256 || VectorWidth == 128)) {
+ // Convert to bitwise operations using the condition
+ MVT IntVT = VT.changeVectorElementTypeToInteger();
+ SDValue IntOp1 = DAG.getBitcast(IntVT, TrueOp);
+ SDValue IntOp2 = DAG.getBitcast(IntVT, FalseOp);
+
+ // Create the CT_SELECT node with integer types
+ SDValue IntResult =
+ DAG.getNode(X86ISD::CT_SELECT, DL, IntVT, IntOp2, IntOp1,
+ DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8),
+ EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget));
+ return DAG.getBitcast(VT, IntResult);
+ }
+ }
+
+ // For integer vectors or when we don't have advanced SIMD support,
+ // use the generic X86 CT_SELECT node which will be matched by the patterns
+ SDValue CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
+ SDValue EFLAGS = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget);
+ // Create the X86 CT_SELECT node - note operand order: true, false, cc, flags
+ return DAG.getNode(X86ISD::CT_SELECT, DL, VT, FalseOp, TrueOp, CC, EFLAGS);
+ }
+
+ // Look past (and (setcc_carry (cmp ...)), 1)
+ if (Cond.getOpcode() == ISD::AND &&
+ Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY &&
+ isOneConstant(Cond.getOperand(1)))
+ Cond = Cond.getOperand(0);
+
+ /// Process condition flags and prepare for CT_SELECT node creation
+ auto ProcessConditionFlags =
+ [&](SDValue Cond, MVT VT, SDLoc DL, SelectionDAG &DAG,
+ const X86Subtarget &Subtarget) -> std::pair<SDValue, SDValue> {
+ SDValue CC;
+ bool AddTest = true;
+
+ unsigned CondOpcode = Cond.getOpcode();
+ if (CondOpcode == X86ISD::SETCC || CondOpcode == X86ISD::SETCC_CARRY) {
+ CC = Cond.getOperand(0);
+ SDValue Cmp = Cond.getOperand(1);
+
+ if ((isX86LogicalCmp(Cmp)) || Cmp.getOpcode() == X86ISD::BT) {
+ Cond = Cmp;
+ AddTest = false;
+ }
+ } else if (CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO ||
+ CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO ||
+ CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) {
+ SDValue Value;
+ X86::CondCode X86Cond;
+ std::tie(Value, Cond) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
+ CC = DAG.getTargetConstant(X86Cond, DL, MVT::i8);
+ AddTest = false;
+ }
+
+ if (AddTest) {
+ // Look past the truncate if the high bits are known zero
+ if (isTruncWithZeroHighBitsInput(Cond, DAG))
+ Cond = Cond.getOperand(0);
+
+ // Try to match AND to BT instruction
+ if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
+ X86::CondCode X86CondCode;
+ if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, X86CondCode)) {
+ CC = DAG.getTargetConstant(X86CondCode, DL, MVT::i8);
+ Cond = BT;
+ AddTest = false;
+ }
+ }
+ }
+
+ if (AddTest) {
+ CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
+ Cond = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget);
+ }
+
+ return {CC, Cond};
+ };
+
+ // Process condition flags and prepare for CT_SELECT
+ auto [CC, ProcessedCond] =
+ ProcessConditionFlags(Cond, VT, DL, DAG, Subtarget);
+
+ // Handle i8 CT_SELECT with truncate optimization
+ if (Op.getValueType() == MVT::i8 && TrueOp.getOpcode() == ISD::TRUNCATE &&
+ FalseOp.getOpcode() == ISD::TRUNCATE) {
+ SDValue T1 = TrueOp.getOperand(0), T2 = FalseOp.getOperand(0);
+ if (T1.getValueType() == T2.getValueType() &&
+ T1.getOpcode() != ISD::CopyFromReg &&
+ T2.getOpcode() != ISD::CopyFromReg) {
+ SDValue CtSelect = DAG.getNode(X86ISD::CT_SELECT, DL, T1.getValueType(),
+ T2, T1, CC, ProcessedCond);
+ return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), CtSelect);
+ }
+ }
+
+ // Promote small integer types to avoid partial register stalls
+ // Exception: For i8 without CMOV, we can generate a shorter instruction
+ // sequence without movzx so keep it as is.
+ if ((Op.getValueType() == MVT::i8 && Subtarget.hasCMOV()) ||
+ (Op.getValueType() == MVT::i16 && !X86::mayFoldLoad(TrueOp, Subtarget) &&
+ !X86::mayFoldLoad(FalseOp, Subtarget))) {
+ TrueOp = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, TrueOp);
+ FalseOp = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, FalseOp);
+ SDValue Ops[] = {FalseOp, TrueOp, CC, ProcessedCond};
+ SDValue CtSelect = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Ops);
+ return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), CtSelect);
+ }
+
+ if (isScalarFPTypeInSSEReg(VT)) {
+ MVT IntVT = (VT == MVT::f32) ? MVT::i32 : MVT::i64;
+ TrueOp = DAG.getBitcast(IntVT, TrueOp);
+ FalseOp = DAG.getBitcast(IntVT, FalseOp);
+ SDValue Ops[] = {FalseOp, TrueOp, CC, ProcessedCond};
+ SDValue CtSelect = DAG.getNode(X86ISD::CT_SELECT, DL, IntVT, Ops);
+ return DAG.getBitcast(VT, CtSelect);
+ }
+
+ // Create final CT_SELECT node
+ SDValue Ops[] = {FalseOp, TrueOp, CC, ProcessedCond};
+ return DAG.getNode(X86ISD::CT_SELECT, DL, Op.getValueType(), Ops,
+ Op->getFlags());
+}
+
static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
SDValue In = Op->getOperand(0);
@@ -30524,30 +30738,65 @@ static SDValue LowervXi8MulWithUNPCK(SDValue A, SDValue B, const SDLoc &dl,
const X86Subtarget &Subtarget,
SelectionDAG &DAG,
SDValue *Low = nullptr) {
+ unsigned NumElts = VT.getVectorNumElements();
+
// For vXi8 we will unpack the low and high half of each 128 bit lane to widen
// to a vXi16 type. Do the multiplies, shift the results and pack the half
// lane results back together.
// We'll take different approaches for signed and unsigned.
- // For unsigned we'll use punpcklbw/punpckhbw to zero extend the bytes to
- // words and use pmullw to calculate the full 16-bit product.
+ // For unsigned we'll use punpcklbw/punpckhbw to put zero extend the bytes
+ // and use pmullw to calculate the full 16-bit product.
// For signed we'll use punpcklbw/punpckbw to extend the bytes to words and
// shift them left into the upper byte of each word. This allows us to use
// pmulhw to calculate the full 16-bit product. This trick means we don't
// need to sign extend the bytes to use pmullw.
- MVT ExVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2);
+
+ MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
SDValue Zero = DAG.getConstant(0, dl, VT);
- SDValue ALo, AHi, BLo, BHi;
+ SDValue ALo, AHi;
if (IsSigned) {
ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, A));
- BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, B));
AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, A));
- BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, B));
} else {
ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Zero));
- BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Zero));
AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Zero));
+ }
+
+ SDValue BLo, BHi;
+ if (ISD::isBuildVectorOfConstantSDNodes(B.getNode())) {
+ // If the RHS is a constant, manually unpackl/unpackh and extend.
+ SmallVector<SDValue, 16> LoOps, HiOps;
+ for (unsigned i = 0; i != NumElts; i += 16) {
+ for (unsigned j = 0; j != 8; ++j) {
+ SDValue LoOp = B.getOperand(i + j);
+ SDValue HiOp = B.getOperand(i + j + 8);
+
+ if (IsSigned) {
+ LoOp = DAG.getAnyExtOrTrunc(LoOp, dl, MVT::i16);
+ HiOp = DAG.getAnyExtOrTrunc(HiOp, dl, MVT::i16);
+ LoOp = DAG.getNode(ISD::SHL, dl, MVT::i16, LoOp,
+ DAG.getConstant(8, dl, MVT::i16));
+ HiOp = DAG.getNode(ISD::SHL, dl, MVT::i16, HiOp,
+ DAG.getConstant(8, dl, MVT::i16));
+ } else {
+ LoOp = DAG.getZExtOrTrunc(LoOp, dl, MVT::i16);
+ HiOp = DAG.getZExtOrTrunc(HiOp, dl, MVT::i16);
+ }
+
+ LoOps.push_back(LoOp);
+ HiOps.push_back(HiOp);
+ }
+ }
+
+ BLo = DAG.getBuildVector(ExVT, dl, LoOps);
+ BHi = DAG.getBuildVector(ExVT, dl, HiOps);
+ } else if (IsSigned) {
+ BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, B));
+ BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, B));
+ } else {
+ BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Zero));
BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Zero));
}
@@ -30560,7 +30809,7 @@ static SDValue LowervXi8MulWithUNPCK(SDValue A, SDValue B, const SDLoc &dl,
if (Low)
*Low = getPack(DAG, Subtarget, dl, VT, RLo, RHi);
- return getPack(DAG, Subtarget, dl, VT, RLo, RHi, /*PackHiHalf=*/true);
+ return getPack(DAG, Subtarget, dl, VT, RLo, RHi, /*PackHiHalf*/ true);
}
static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
@@ -34633,6 +34882,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::STRICT_FSETCCS: return LowerSETCC(Op, DAG);
case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG);
case ISD::SELECT: return LowerSELECT(Op, DAG);
+ case ISD::CT_SELECT: return LowerCT_SELECT(Op, DAG);
case ISD::COND_LOOP:
case ISD::BRCOND: return LowerConditionalBranch(Op, DAG);
case ISD::JumpTable: return LowerJumpTable(Op, DAG);
@@ -34729,6 +34979,12 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
}
}
+bool X86TargetLowering::isSelectSupported(SelectSupportKind Kind) const {
+ if (Kind == SelectSupportKind::CtSelect) {
+ return true;
+ }
+ return TargetLoweringBase::isSelectSupported(Kind);
+}
/// Replace a node with an illegal result type with a new node built out of
/// custom code.
void X86TargetLowering::ReplaceNodeResults(SDNode *N,
@@ -36381,12 +36637,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
}
}
-/// Return true if the addressing mode represented by AM is legal for this
-/// target, for a load/store of the specified type.
-bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
- const AddrMode &AM, Type *Ty,
- unsigned AS,
- Instruction *I) const {
// X86 supports extremely general addressing modes.
CodeModel::Model M = getTargetMachine().getCodeModel();
@@ -38718,6 +38968,480 @@ X86TargetLowering::emitPatchableEventCall(MachineInstr &MI,
return BB;
}
+/// Helper function to emit i386 CT_SELECT with condition materialization.
+/// This converts EFLAGS-based CT_SELECT into a condition byte that can be
+/// shared across multiple operations (critical for i64 type legalization).
+///
+/// Phase 1: Materialize condition byte from EFLAGS using SETCC
+/// Phase 2: Create internal pseudo with condition byte for post-RA expansion
+///
+/// This approach ensures that when i64 is type-legalized into two i32
+/// operations, both operations share the same condition byte rather than
+/// each independently reading (and destroying) EFLAGS.
+static MachineBasicBlock *
+emitCTSelectI386WithConditionMaterialization(MachineInstr &MI,
+ MachineBasicBlock *BB,
+ unsigned InternalPseudoOpcode) {
+ const TargetInstrInfo *TII = BB->getParent()->getSubtarget().getInstrInfo();
+ const MIMetadata MIMD(MI);
+ MachineFunction *MF = BB->getParent();
+ MachineRegisterInfo &MRI = MF->getRegInfo();
+
+ // Original pseudo operands: (outs dst), (ins src1, src2, cond)
+ Register Src1Reg = MI.getOperand(1).getReg();
+ Register Src2Reg = MI.getOperand(2).getReg();
+ X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(3).getImm());
+
+ // Get opposite condition (SETCC sets to 1 when condition is TRUE,
+ // but we want to select src1 when condition is FALSE for X86 semantics)
+ X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
+
+ // Step 1: Materialize condition byte from EFLAGS
+ // This is done OUTSIDE the constant-time bundle, before any EFLAGS corruption
+ Register CondByteReg = MRI.createVirtualRegister(&X86::GR8RegClass);
+ BuildMI(*BB, MI, MIMD, TII->get(X86::SETCCr), CondByteReg).addImm(OppCC);
+
+ // Step 2: Create internal pseudo that takes condition byte as input
+ // This pseudo will be expanded post-RA into the actual constant-time bundle
+ // The condition byte can now be safely shared between multiple pseudos
+
+ // Internal pseudo has operands: (outs dst, tmp_byte, tmp_mask), (ins src1,
+ // src2, cond_byte)
+ Register DstReg = MI.getOperand(0).getReg();
+
+ // Create virtual registers for the temporary outputs
+ Register TmpByteReg = MRI.createVirtualRegister(&X86::GR8RegClass);
+ Register TmpMaskReg;
+
+ // Determine the register class for tmp_mask based on the data type
+ if (InternalPseudoOpcode == X86::CT_SELECT_I386_INT_GR8rr) {
+ TmpMaskReg = MRI.createVirtualRegister(&X86::GR8RegClass);
+ } else if (InternalPseudoOpcode == X86::CT_SELECT_I386_INT_GR16rr) {
+ TmpMaskReg = MRI.createVirtualRegister(&X86::GR16RegClass);
+ } else if (InternalPseudoOpcode == X86::CT_SELECT_I386_INT_GR32rr) {
+ TmpMaskReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+ } else {
+ llvm_unreachable("Unknown internal pseudo opcode");
+ }
+
+ BuildMI(*BB, MI, MIMD, TII->get(InternalPseudoOpcode))
+ .addDef(DstReg) // dst (output)
+ .addDef(TmpByteReg) // tmp_byte (output)
+ .addDef(TmpMaskReg) // tmp_mask (output)
+ .addReg(Src1Reg) // src1 (input)
+ .addReg(Src2Reg) // src2 (input)
+ .addReg(CondByteReg); // pre-materialized condition byte (input)
+
+ MI.eraseFromParent();
+ return BB;
+}
+
+// Helper structure to hold memory operand information for FP loads
+struct FPLoadMemOperands {
+ bool IsValid = false;
+ unsigned BaseReg = 0;
+ int64_t ScaleVal = 1;
+ unsigned IndexReg = 0;
+ int64_t Disp = 0;
+ unsigned SegReg = 0;
+ int FrameIndex = -1;
+ bool IsFrameIndex = false;
+ int ConstantPoolIndex = -1;
+ bool IsConstantPool = false;
+ const GlobalValue *Global = nullptr;
+ int64_t GlobalOffset = 0;
+ bool IsGlobal = false;
+};
+
+// Check if a virtual register is defined by a simple FP load instruction
+// Returns the memory operands if it's a simple load, otherwise returns invalid
+static FPLoadMemOperands getFPLoadMemOperands(Register Reg,
+ MachineRegisterInfo &MRI,
+ unsigned ExpectedLoadOpcode) {
+ FPLoadMemOperands Result;
+
+ if (!Reg.isVirtual())
+ return Result;
+
+ MachineInstr *DefMI = MRI.getVRegDef(Reg);
+ if (!DefMI)
+ return Result;
+
+ // Check if it's the expected load opcode (e.g., LD_Fp32m, LD_Fp64m, LD_Fp80m)
+ if (DefMI->getOpcode() != ExpectedLoadOpcode)
+ return Result;
+
+ // Check that this is a simple load - not volatile, not atomic, etc.
+ // FP loads have hasSideEffects = 0 in their definition for simple loads
+ if (DefMI->hasOrderedMemoryRef())
+ return Result;
+
+ // The load should have a single def (the destination register) and memory operands
+ // Format: %reg = LD_Fpxxm <fi#N>, 1, %noreg, 0, %noreg
+ // or: %reg = LD_Fpxxm %base, scale, %index, disp, %segment
+ if (DefMI->getNumOperands() < 6)
+ return Result;
+
+ // Operand 0 is the destination, operands 1-5 are the memory reference
+ MachineOperand &BaseMO = DefMI->getOperand(1);
+ MachineOperand &ScaleMO = DefMI->getOperand(2);
+ MachineOperand &IndexMO = DefMI->getOperand(3);
+ MachineOperand &DispMO = DefMI->getOperand(4);
+ MachineOperand &SegMO = DefMI->getOperand(5);
+
+ // Check if this is a frame index load
+ if (BaseMO.isFI()) {
+ Result.IsValid = true;
+ Result.IsFrameIndex = true;
+ Result.FrameIndex = BaseMO.getIndex();
+ Result.ScaleVal = ScaleMO.getImm();
+ Result.IndexReg = IndexMO.getReg();
+ Result.Disp = DispMO.getImm();
+ Result.SegReg = SegMO.getReg();
+ return Result;
+ }
+
+ // Check if this is a constant pool load
+ // Format: %reg = LD_Fpxxm $noreg, 1, $noreg, %const.N, $noreg
+ if (BaseMO.isReg() && BaseMO.getReg() == X86::NoRegister &&
+ ScaleMO.isImm() && IndexMO.isReg() &&
+ IndexMO.getReg() == X86::NoRegister &&
+ DispMO.isCPI() && SegMO.isReg()) {
+ Result.IsValid = true;
+ Result.IsConstantPool = true;
+ Result.ConstantPoolIndex = DispMO.getIndex();
+ Result.ScaleVal = ScaleMO.getImm();
+ Result.IndexReg = IndexMO.getReg();
+ Result.Disp = 0;
+ Result.SegReg = SegMO.getReg();
+ return Result;
+ }
+
+ // Check if this is a global variable load
+ // Format: %reg = LD_Fpxxm $noreg, 1, $noreg, @global_name, $noreg
+ if (BaseMO.isReg() && BaseMO.getReg() == X86::NoRegister &&
+ ScaleMO.isImm() && IndexMO.isReg() &&
+ IndexMO.getReg() == X86::NoRegister &&
+ DispMO.isGlobal() && SegMO.isReg()) {
+ Result.IsValid = true;
+ Result.IsGlobal = true;
+ Result.Global = DispMO.getGlobal();
+ Result.GlobalOffset = DispMO.getOffset();
+ Result.ScaleVal = ScaleMO.getImm();
+ Result.IndexReg = IndexMO.getReg();
+ Result.Disp = 0;
+ Result.SegReg = SegMO.getReg();
+ return Result;
+ }
+
+ // Regular memory operands (e.g., pointer loads)
+ if (BaseMO.isReg() && ScaleMO.isImm() && IndexMO.isReg() &&
+ DispMO.isImm() && SegMO.isReg()) {
+ Result.IsValid = true;
+ Result.IsFrameIndex = false;
+ Result.IsConstantPool = false;
+ Result.BaseReg = BaseMO.getReg();
+ Result.ScaleVal = ScaleMO.getImm();
+ Result.IndexReg = IndexMO.getReg();
+ Result.Disp = DispMO.getImm();
+ Result.SegReg = SegMO.getReg();
+ return Result;
+ }
+
+ return Result;
+}
+
+static MachineBasicBlock *emitCTSelectI386WithFpType(MachineInstr &MI,
+ MachineBasicBlock *BB,
+ unsigned pseudoInstr) {
+ const TargetInstrInfo *TII = BB->getParent()->getSubtarget().getInstrInfo();
+ const MIMetadata MIMD(MI);
+ MachineFunction *MF = BB->getParent();
+ MachineRegisterInfo &MRI = MF->getRegInfo();
+ MachineFrameInfo &MFI = MF->getFrameInfo();
+ unsigned RegSizeInByte = 4;
+
+ // Get operands
+ // MI operands: %result:rfp80 = CT_SELECT_I386 %false:rfp80, %true:rfp80, %cond:i8imm
+ unsigned DestReg = MI.getOperand(0).getReg();
+ unsigned FalseReg = MI.getOperand(1).getReg();
+ unsigned TrueReg = MI.getOperand(2).getReg();
+ X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(3).getImm());
+ X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
+
+ // Materialize condition byte from EFLAGS
+ Register CondByteReg = MRI.createVirtualRegister(&X86::GR8RegClass);
+ BuildMI(*BB, MI, MIMD, TII->get(X86::SETCCr), CondByteReg).addImm(OppCC);
+
+ auto storeFpToSlot = [&](unsigned Opcode, int Slot, Register Reg) {
+ addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(Opcode)), Slot)
+ .addReg(Reg, RegState::Kill);
+ };
+
+ // Helper to load integer from memory operands
+ auto loadIntFromMemOperands = [&](const FPLoadMemOperands &MemOps,
+ unsigned Offset) -> unsigned {
+ unsigned IntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+ MachineInstrBuilder MIB =
+ BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), IntReg);
+
+ if (MemOps.IsFrameIndex) {
+ // Frame index: addFrameIndex + scale + index + disp + segment
+ MIB.addFrameIndex(MemOps.FrameIndex)
+ .addImm(MemOps.ScaleVal)
+ .addReg(MemOps.IndexReg)
+ .addImm(MemOps.Disp + Offset)
+ .addReg(MemOps.SegReg);
+ } else if (MemOps.IsConstantPool) {
+ // Constant pool: base_reg + scale + index + CP_index + segment
+ // MOV32rm format: base, scale, index, displacement, segment
+ MIB.addReg(X86::NoRegister) // Base register
+ .addImm(MemOps.ScaleVal) // Scale
+ .addReg(MemOps.IndexReg) // Index register
+ .addConstantPoolIndex(MemOps.ConstantPoolIndex, Offset) // Displacement (CP index)
+ .addReg(MemOps.SegReg); // Segment
+ } else if (MemOps.IsGlobal) {
+ // Global variable: base_reg + scale + index + global + segment
+ // MOV32rm format: base, scale, index, displacement, segment
+ MIB.addReg(X86::NoRegister) // Base register
+ .addImm(MemOps.ScaleVal) // Scale
+ .addReg(MemOps.IndexReg) // Index register
+ .addGlobalAddress(MemOps.Global, MemOps.GlobalOffset + Offset) // Displacement (global address)
+ .addReg(MemOps.SegReg); // Segment
+ } else {
+ // Regular memory: base_reg + scale + index + disp + segment
+ MIB.addReg(MemOps.BaseReg)
+ .addImm(MemOps.ScaleVal)
+ .addReg(MemOps.IndexReg)
+ .addImm(MemOps.Disp + Offset)
+ .addReg(MemOps.SegReg);
+ }
+
+ return IntReg;
+ };
+
+ // Optimized path: load integers directly from memory when both operands are
+ // memory loads, avoiding FP register round-trip
+ auto emitCtSelectFromMemory = [&](unsigned NumValues,
+ const FPLoadMemOperands &TrueMemOps,
+ const FPLoadMemOperands &FalseMemOps,
+ int ResultSlot) {
+ for (unsigned Val = 0; Val < NumValues; ++Val) {
+ unsigned Offset = Val * RegSizeInByte;
+
+ // Load true and false values directly from their memory locations as integers
+ unsigned TrueIntReg = loadIntFromMemOperands(TrueMemOps, Offset);
+ unsigned FalseIntReg = loadIntFromMemOperands(FalseMemOps, Offset);
+
+ // Use CT_SELECT_I386_INT_GR32 pseudo instruction for constant-time selection
+ unsigned ResultIntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+ unsigned TmpByteReg = MRI.createVirtualRegister(&X86::GR8RegClass);
+ unsigned TmpMaskReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+
+ BuildMI(*BB, MI, MIMD, TII->get(X86::CT_SELECT_I386_INT_GR32rr))
+ .addDef(ResultIntReg) // dst (output)
+ .addDef(TmpByteReg) // tmp_byte (output)
+ .addDef(TmpMaskReg) // tmp_mask (output)
+ .addReg(FalseIntReg) // src1 (input) - false value
+ .addReg(TrueIntReg) // src2 (input) - true value
+ .addReg(CondByteReg); // pre-materialized condition byte (input)
+
+ // Store result back to result slot
+ BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32mr))
+ .addFrameIndex(ResultSlot)
+ .addImm(1)
+ .addReg(0)
+ .addImm(Offset)
+ .addReg(0)
+ .addReg(ResultIntReg, RegState::Kill);
+ }
+ };
+
+ auto emitCtSelectWithPseudo = [&](unsigned NumValues, int TrueSlot, int FalseSlot, int ResultSlot) {
+ for (unsigned Val = 0; Val < NumValues; ++Val) {
+ unsigned Offset = Val * RegSizeInByte;
+
+ // Load true and false values from stack as 32-bit integers
+ unsigned TrueIntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+ BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), TrueIntReg)
+ .addFrameIndex(TrueSlot)
+ .addImm(1)
+ .addReg(0)
+ .addImm(Offset)
+ .addReg(0);
+
+ unsigned FalseIntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+ BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), FalseIntReg)
+ .addFrameIndex(FalseSlot)
+ .addImm(1)
+ .addReg(0)
+ .addImm(Offset)
+ .addReg(0);
+
+ // Use CT_SELECT_I386_INT_GR32 pseudo instruction for constant-time selection
+ unsigned ResultIntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+ unsigned TmpByteReg = MRI.createVirtualRegister(&X86::GR8RegClass);
+ unsigned TmpMaskReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+
+ BuildMI(*BB, MI, MIMD, TII->get(X86::CT_SELECT_I386_INT_GR32rr))
+ .addDef(ResultIntReg) // dst (output)
+ .addDef(TmpByteReg) // tmp_byte (output)
+ .addDef(TmpMaskReg) // tmp_mask (output)
+ .addReg(FalseIntReg) // src1 (input) - false value
+ .addReg(TrueIntReg) // src2 (input) - true value
+ .addReg(CondByteReg); // pre-materialized condition byte (input)
+
+ // Store result back to result slot
+ BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32mr))
+ .addFrameIndex(ResultSlot)
+ .addImm(1)
+ .addReg(0)
+ .addImm(Offset)
+ .addReg(0)
+ .addReg(ResultIntReg, RegState::Kill);
+ }
+ };
+
+ switch (pseudoInstr) {
+ case X86::CT_SELECT_I386_FP32rr: {
+ // Check if both operands are simple memory loads
+ FPLoadMemOperands TrueMemOps =
+ getFPLoadMemOperands(TrueReg, MRI, X86::LD_Fp32m);
+ FPLoadMemOperands FalseMemOps =
+ getFPLoadMemOperands(FalseReg, MRI, X86::LD_Fp32m);
+
+ int ResultSlot = MFI.CreateStackObject(RegSizeInByte, Align(4), false);
+
+ if (TrueMemOps.IsValid && FalseMemOps.IsValid) {
+ // Optimized path: load directly from memory as integers
+ // Works for both frame index loads (stack parameters) and
+ // constant pool loads (constants)
+ emitCtSelectFromMemory(1, TrueMemOps, FalseMemOps, ResultSlot);
+
+ // Erase the original FP load instructions since we're not using them
+ // and have loaded the data directly as integers instead
+ if (MRI.hasOneUse(TrueReg)) {
+ if (MachineInstr *TrueDefMI = MRI.getVRegDef(TrueReg))
+ TrueDefMI->eraseFromParent();
+ }
+ if (MRI.hasOneUse(FalseReg)) {
+ if (MachineInstr *FalseDefMI = MRI.getVRegDef(FalseReg))
+ FalseDefMI->eraseFromParent();
+ }
+ } else {
+ // General path: spill FP registers to stack first
+ int TrueSlot = MFI.CreateStackObject(RegSizeInByte, Align(4), false);
+ int FalseSlot = MFI.CreateStackObject(RegSizeInByte, Align(4), false);
+
+ storeFpToSlot(X86::ST_Fp32m, TrueSlot, TrueReg);
+ storeFpToSlot(X86::ST_Fp32m, FalseSlot, FalseReg);
+
+ emitCtSelectWithPseudo(1, TrueSlot, FalseSlot, ResultSlot);
+ }
+
+ // Load result back as f32
+ addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::LD_Fp32m), DestReg),
+ ResultSlot);
+ break;
+ }
+ case X86::CT_SELECT_I386_FP64rr: {
+ unsigned StackSlotSize = 8;
+
+ // Check if both operands are simple memory loads
+ FPLoadMemOperands TrueMemOps =
+ getFPLoadMemOperands(TrueReg, MRI, X86::LD_Fp64m);
+ FPLoadMemOperands FalseMemOps =
+ getFPLoadMemOperands(FalseReg, MRI, X86::LD_Fp64m);
+
+ int ResultSlot = MFI.CreateStackObject(StackSlotSize, Align(4), false);
+
+ if (TrueMemOps.IsValid && FalseMemOps.IsValid) {
+ // Optimized path: load directly from memory as integers
+ // Works for both frame index loads (stack parameters) and
+ // constant pool loads (constants)
+ emitCtSelectFromMemory(StackSlotSize / RegSizeInByte, TrueMemOps,
+ FalseMemOps, ResultSlot);
+
+ // Erase the original FP load instructions since we're not using them
+ if (MRI.hasOneUse(TrueReg)) {
+ if (MachineInstr *TrueDefMI = MRI.getVRegDef(TrueReg))
+ TrueDefMI->eraseFromParent();
+ }
+ if (MRI.hasOneUse(FalseReg)) {
+ if (MachineInstr *FalseDefMI = MRI.getVRegDef(FalseReg))
+ FalseDefMI->eraseFromParent();
+ }
+ } else {
+ // General path: spill FP registers to stack first
+ int TrueSlot = MFI.CreateStackObject(StackSlotSize, Align(4), false);
+ int FalseSlot = MFI.CreateStackObject(StackSlotSize, Align(4), false);
+
+ storeFpToSlot(X86::ST_Fp64m, TrueSlot, TrueReg);
+ storeFpToSlot(X86::ST_Fp64m, FalseSlot, FalseReg);
+
+ emitCtSelectWithPseudo(StackSlotSize / RegSizeInByte, TrueSlot, FalseSlot,
+ ResultSlot);
+ }
+
+ // Load result back as f64
+ addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::LD_Fp64m), DestReg),
+ ResultSlot);
+ break;
+ }
+ case X86::CT_SELECT_I386_FP80rr: {
+ // f80 is 80 bits (10 bytes), but stored with 12-byte alignment
+ unsigned StackObjectSize = 12;
+
+ // Check if both operands are simple memory loads
+ FPLoadMemOperands TrueMemOps =
+ getFPLoadMemOperands(TrueReg, MRI, X86::LD_Fp80m);
+ FPLoadMemOperands FalseMemOps =
+ getFPLoadMemOperands(FalseReg, MRI, X86::LD_Fp80m);
+
+ int ResultSlot = MFI.CreateStackObject(StackObjectSize, Align(4), false);
+
+ if (TrueMemOps.IsValid && FalseMemOps.IsValid) {
+ // Optimized path: load directly from memory as integers
+ // Works for both frame index loads (stack parameters) and
+ // constant pool loads (constants)
+ emitCtSelectFromMemory(StackObjectSize / RegSizeInByte, TrueMemOps,
+ FalseMemOps, ResultSlot);
+
+ // Erase the original FP load instructions since we're not using them
+ if (MRI.hasOneUse(TrueReg)) {
+ if (MachineInstr *TrueDefMI = MRI.getVRegDef(TrueReg))
+ TrueDefMI->eraseFromParent();
+ }
+ if (MRI.hasOneUse(FalseReg)) {
+ if (MachineInstr *FalseDefMI = MRI.getVRegDef(FalseReg))
+ FalseDefMI->eraseFromParent();
+ }
+ } else {
+ // General path: spill FP registers to stack first
+ int TrueSlot = MFI.CreateStackObject(StackObjectSize, Align(4), false);
+ int FalseSlot = MFI.CreateStackObject(StackObjectSize, Align(4), false);
+
+ storeFpToSlot(X86::ST_FpP80m, TrueSlot, TrueReg);
+ storeFpToSlot(X86::ST_FpP80m, FalseSlot, FalseReg);
+
+ emitCtSelectWithPseudo(StackObjectSize / RegSizeInByte, TrueSlot,
+ FalseSlot, ResultSlot);
+ }
+
+ // Load result back as f80
+ addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::LD_Fp80m), DestReg),
+ ResultSlot);
+ break;
+ }
+ default:
+ llvm_unreachable("Invalid CT_SELECT opcode");
+ }
+
+ MI.eraseFromParent();
+
+ return BB;
+}
+
MachineBasicBlock *
X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
MachineBasicBlock *BB) const {
@@ -38775,6 +39499,25 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case X86::CMOV_VK64:
return EmitLoweredSelect(MI, BB);
+ case X86::CT_SELECT_I386_GR8rr:
+ return emitCTSelectI386WithConditionMaterialization(
+ MI, BB, X86::CT_SELECT_I386_INT_GR8rr);
+
+ case X86::CT_SELECT_I386_GR16rr:
+ return emitCTSelectI386WithConditionMaterialization(
+ MI, BB, X86::CT_SELECT_I386_INT_GR16rr);
+
+ case X86::CT_SELECT_I386_GR32rr:
+ return emitCTSelectI386WithConditionMaterialization(
+ MI, BB, X86::CT_SELECT_I386_INT_GR32rr);
+
+ case X86::CT_SELECT_I386_FP32rr:
+ return emitCTSelectI386WithFpType(MI, BB, X86::CT_SELECT_I386_FP32rr);
+ case X86::CT_SELECT_I386_FP64rr:
+ return emitCTSelectI386WithFpType(MI, BB, X86::CT_SELECT_I386_FP64rr);
+ case X86::CT_SELECT_I386_FP80rr:
+ return emitCTSelectI386WithFpType(MI, BB, X86::CT_SELECT_I386_FP80rr);
+
case X86::FP80_ADDr:
case X86::FP80_ADDm32: {
// Change the floating point control register to use double extended
@@ -42790,7 +43533,7 @@ static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL,
if (!X86::mayFoldLoad(peekThroughOneUseBitcasts(N0), Subtarget) ||
X86::mayFoldLoad(peekThroughOneUseBitcasts(N1), Subtarget))
return SDValue();
- Imm = llvm::rotl<uint8_t>(Imm, 4);
+ Imm = ((Imm & 0x0F) << 4) | ((Imm & 0xF0) >> 4);
return DAG.getNode(X86ISD::SHUFP, DL, VT, N1, N0,
DAG.getTargetConstant(Imm, DL, MVT::i8));
};
@@ -45859,16 +46602,10 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
}
case X86ISD::PCMPGT:
// icmp sgt(0, R) == ashr(R, BitWidth-1).
- if (ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode())) {
- // iff we only need the signbit then we can use R directly.
- if (OriginalDemandedBits.isSignMask())
- return TLO.CombineTo(Op, Op.getOperand(1));
- // otherwise we just need R's signbit for the comparison.
- APInt SignMask = APInt::getSignMask(BitWidth);
- if (SimplifyDemandedBits(Op.getOperand(1), SignMask, OriginalDemandedElts,
- Known, TLO, Depth + 1))
- return true;
- }
+ // iff we only need the sign bit then we can use R directly.
+ if (OriginalDemandedBits.isSignMask() &&
+ ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode()))
+ return TLO.CombineTo(Op, Op.getOperand(1));
break;
case X86ISD::MOVMSK: {
SDValue Src = Op.getOperand(0);
@@ -48775,15 +49512,6 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
DL, DAG, Subtarget))
return V;
- // If the sign bit is known then BLENDV can be folded away.
- if (N->getOpcode() == X86ISD::BLENDV) {
- KnownBits KnownCond = DAG.computeKnownBits(Cond);
- if (KnownCond.isNegative())
- return LHS;
- if (KnownCond.isNonNegative())
- return RHS;
- }
-
if (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::BLENDV) {
SmallVector<int, 64> CondMask;
if (createShuffleMaskFromVSELECT(CondMask, Cond,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index b220c1bbe7bc7..190e4e750092f 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -176,6 +176,8 @@ namespace llvm {
///
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
+ bool isSelectSupported(SelectSupportKind Kind) const override;
+
/// Replace the results of node with an illegal result
/// type with new values built out of custom code.
///
@@ -820,6 +822,7 @@ namespace llvm {
SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerCT_SELECT(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerConditionalBranch(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/X86/X86InstrCMovSetCC.td b/llvm/lib/Target/X86/X86InstrCMovSetCC.td
index 39663049b92bf..5904ebcded423 100644
--- a/llvm/lib/Target/X86/X86InstrCMovSetCC.td
+++ b/llvm/lib/Target/X86/X86InstrCMovSetCC.td
@@ -106,6 +106,211 @@ let Predicates = [HasCMOV, HasNDD] in {
def : Pat<(X86cmov (loadi64 ndd_addr:$src1), GR64:$src2, timm:$cond, EFLAGS),
(CMOV64rm_ND GR64:$src2, ndd_addr:$src1, (inv_cond_XFORM timm:$cond))>;
}
+
+// Create pseudo instruction and do the pattern matching to them.
+// We use a machine pass to lower these pseudos into cmov, in order
+// to avoid backend optimizations
+let Uses = [EFLAGS], isNotDuplicable = 1, isPseudo = 1 in {
+
+ multiclass CT_SELECT<X86TypeInfo t> {
+ // register-only
+ let isCommutable = 0, SchedRW = [WriteCMOV], Predicates = [HasNativeCMOV],
+ AsmString = "ctselect\\t$dst, $src1, $src2, $cond" in {
+ def rr : PseudoI<(outs t.RegClass:$dst),
+ (ins t.RegClass:$src1, t.RegClass:$src2, i8imm:$cond),
+ [(set t.RegClass:$dst, (X86ct_select t.RegClass:$src1, t.RegClass:$src2, timm:$cond, EFLAGS))]>;
+ }
+
+ // register-memory
+ let SchedRW = [WriteCMOV.Folded, WriteCMOV.ReadAfterFold], Predicates = [HasNativeCMOV],
+ AsmString = "ctselect\\t$dst, $src1, $src2, $cond" in {
+ def rm : PseudoI<(outs t.RegClass:$dst),
+ (ins t.RegClass:$src1, t.MemOperand:$src2, i8imm:$cond),
+ [(set t.RegClass:$dst, (X86ct_select t.RegClass:$src1, (t.LoadNode addr:$src2), timm:$cond, EFLAGS))]>;
+ }
+ }
+}
+
+let isCodeGenOnly = 1, hasSideEffects = 1, ForceDisassemble = 1 in {
+ let Constraints = "$dst = $src1" in {
+ defm CT_SELECT16 : CT_SELECT<Xi16>;
+ defm CT_SELECT32 : CT_SELECT<Xi32>;
+ defm CT_SELECT64 : CT_SELECT<Xi64>;
+ }
+}
+
+// CT_SELECT_VEC base class
+class CT_SELECT_VEC<RegisterClass VRc, RegisterClass GRc>
+ : PseudoI<
+ (outs VRc:$dst, VRc:$tmpx, GRc:$tmpg),
+ (ins VRc:$t, VRc:$f, i8imm:$cond),
+ []
+ > {
+ let Uses = [EFLAGS];
+ let isPseudo = 1;
+ let isNotDuplicable = 1;
+ let hasSideEffects = 1;
+ let AsmString = "ctselect\t$dst, $f, $t, $cond";
+ let SchedRW = [];
+}
+
+// Width-specific class aliases
+class CT_SELECT_VEC128 : CT_SELECT_VEC<VR128, GR32>;
+class CT_SELECT_VEC128X : CT_SELECT_VEC<VR128X, GR32>;
+class CT_SELECT_VEC256 : CT_SELECT_VEC<VR256, GR32>;
+class CT_SELECT_VEC512 : CT_SELECT_VEC<VR512, GR32>;
+
+
+//===----------------------------------------------------------------------===//
+// 128-bit pseudos (SSE2 baseline; we use PXOR/PAND/MOVD/PSHUFD in the expander)
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasSSE1] in {
+
+ def CT_SELECT_V4F32 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+}
+
+let Predicates = [HasSSE2] in {
+
+ def CT_SELECT_V2F64 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V4I32 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V2I64 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V8I16 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V16I8 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+
+ // If your build has v8f16, keep this; otherwise comment it out.
+ def CT_SELECT_V8F16 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+}
+
+let Predicates = [HasAVX] in {
+
+ def CT_SELECT_V4F32X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V2F64X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V4I32X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V2I64X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V8I16X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V16I8X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+
+ // If your build has v8f16, keep this; otherwise comment it out.
+ def CT_SELECT_V8F16X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// 256-bit pseudos
+//===----------------------------------------------------------------------===//
+let Predicates = [HasAVX] in {
+
+ def CT_SELECT_V8F32 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V4F64 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V8I32 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V4I64 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V16I16 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V32I8 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+
+ // If your build has v16f16, keep this; otherwise comment it out.
+ def CT_SELECT_V16F16 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// Selection patterns: X86ct_select(...), EFLAGS -> CT_SELECT_V*
+//
+// NOTE:
+// * The SDNode carries Glue from CMP/TEST (due to SDNPInGlue).
+// * We list EFLAGS explicitly in the pattern (X86 style) to model the arch read.
+// * Temps (tmpx/tmpy,tmpg) are not in the pattern; they’re outs allocated by RA.
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasSSE1] in {
+
+ // 128-bit float (bitwise-equivalent ops in expander)
+ def : Pat<(v4f32 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V4F32 VR128:$t, VR128:$f, timm:$cc)>;
+}
+
+let Predicates = [HasSSE2] in {
+
+ // 128-bit integer
+ def : Pat<(v4i32 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V4I32 VR128:$t, VR128:$f, timm:$cc)>;
+ def : Pat<(v2i64 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V2I64 VR128:$t, VR128:$f, timm:$cc)>;
+ def : Pat<(v8i16 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V8I16 VR128:$t, VR128:$f, timm:$cc)>;
+ def : Pat<(v16i8 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V16I8 VR128:$t, VR128:$f, timm:$cc)>;
+ def : Pat<(v2f64 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V2F64 VR128:$t, VR128:$f, timm:$cc)>;
+
+ // 128-bit f16 (optional)
+ def : Pat<(v8f16 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V8F16 VR128:$t, VR128:$f, timm:$cc)>;
+}
+
+let Predicates = [HasAVX] in {
+
+ // 256-bit integer
+ def : Pat<(v8i32 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V8I32 VR256:$t, VR256:$f, timm:$cc)>;
+ def : Pat<(v4i64 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V4I64 VR256:$t, VR256:$f, timm:$cc)>;
+ def : Pat<(v16i16 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V16I16 VR256:$t, VR256:$f, timm:$cc)>;
+ def : Pat<(v32i8 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V32I8 VR256:$t, VR256:$f, timm:$cc)>;
+
+ // 256-bit float (bitwise-equivalent ops in expander)
+ def : Pat<(v8f32 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V8F32 VR256:$t, VR256:$f, timm:$cc)>;
+ def : Pat<(v4f64 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V4F64 VR256:$t, VR256:$f, timm:$cc)>;
+
+ // 256-bit f16 (optional)
+ def : Pat<(v16f16 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V16F16 VR256:$t, VR256:$f, timm:$cc)>;
+}
+
let Predicates = [HasCMOV, HasCF] in {
def : Pat<(X86cmov GR16:$src1, 0, timm:$cond, EFLAGS),
(CFCMOV16rr GR16:$src1, (inv_cond_XFORM timm:$cond))>;
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 40ad1eb9b72cb..196389b5a144a 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -704,6 +704,87 @@ def : Pat<(v16f32 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)),
def : Pat<(v8f64 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)),
(CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>;
+// CT_SELECT
+// Enhanced CT_SELECT pseudos for i386 with temporary register allocation
+// These use a two-phase approach:
+// 1. Custom inserter materializes condition byte from EFLAGS
+// 2. Post-RA expansion generates constant-time instruction bundles
+
+let isPseudo = 1, isNotDuplicable = 1 in {
+ // Phase 1: Initial pseudos that consume EFLAGS (via custom inserter)
+ // These are matched by patterns and convert EFLAGS to condition byte
+ class CT_SELECT_I386_INITIAL<RegisterClass RC, ValueType VT>
+ : PseudoI<(outs RC:$dst),
+ (ins RC:$src1, RC:$src2, i8imm:$cond),
+ [(set RC:$dst, (VT(X86ct_select RC:$src1, RC:$src2, timm:$cond,
+ EFLAGS)))]> {
+ let Uses = [EFLAGS];
+ let Defs = [EFLAGS];
+ let usesCustomInserter = 1;
+ let hasNoSchedulingInfo = 1;
+ }
+
+ // Phase 2: Internal pseudos with pre-materialized condition byte (post-RA expansion)
+ // These generate the actual constant-time instruction bundles
+ class CT_SELECT_I386_INTERNAL<RegisterClass RC, RegisterClass ByteRC>
+ : PseudoI<(outs RC:$dst, ByteRC:$tmp_byte, RC:$tmp_mask),
+ (ins RC:$src1, RC:$src2, ByteRC:$cond_byte), []> {
+ let hasNoSchedulingInfo = 1;
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmp_byte, at earlyclobber $tmp_mask";
+ let Defs = [EFLAGS]; // NEG instruction in post-RA expansion clobbers EFLAGS
+ }
+}
+
+// Phase 1 pseudos for non-CMOV targets (custom inserter materializes condition)
+let isCodeGenOnly = 1, hasSideEffects = 1, ForceDisassemble = 1 in {
+ let Predicates = [NoNativeCMOV] in {
+ def CT_SELECT_I386_GR8rr : CT_SELECT_I386_INITIAL<GR8, i8>;
+ def CT_SELECT_I386_GR16rr : CT_SELECT_I386_INITIAL<GR16, i16>;
+ def CT_SELECT_I386_GR32rr : CT_SELECT_I386_INITIAL<GR32, i32>;
+ }
+}
+
+// Phase 2 pseudos (post-RA expansion with pre-materialized condition byte)
+let isCodeGenOnly = 1, hasSideEffects = 1, ForceDisassemble = 1 in {
+ let Predicates = [NoNativeCMOV] in {
+ def CT_SELECT_I386_INT_GR8rr :
+ CT_SELECT_I386_INTERNAL<GR8, GR8>;
+ def CT_SELECT_I386_INT_GR16rr :
+ CT_SELECT_I386_INTERNAL<GR16, GR8>;
+ def CT_SELECT_I386_INT_GR32rr :
+ CT_SELECT_I386_INTERNAL<GR32, GR8>;
+ }
+}
+
+let hasSideEffects = 1,
+ ForceDisassemble = 1,
+ Constraints = "$dst = $src1" in {
+
+ let Predicates = [FPStackf32] in
+ def CT_SELECT_I386_FP32rr : CT_SELECT_I386_INITIAL<RFP32, f32>;
+
+ let Predicates = [FPStackf64] in
+ def CT_SELECT_I386_FP64rr : CT_SELECT_I386_INITIAL<RFP64, f64>;
+
+ def CT_SELECT_I386_FP80rr : CT_SELECT_I386_INITIAL<RFP80, f80>;
+}
+
+// Pattern matching for non-native-CMOV CT_SELECT (routes to custom inserter for condition materialization)
+// NoNativeCMOV ensures these patterns are used when actual CMOV instruction is not available
+// even if canUseCMOV() is true (e.g., i386 with SSE which can emulate CMOV)
+let Predicates = [NoNativeCMOV] in {
+ def : Pat<(i8(X86ct_select GR8:$src1, GR8:$src2, timm:$cond, EFLAGS)),
+ (CT_SELECT_I386_GR8rr GR8:$src1, GR8:$src2, timm:$cond)>;
+
+ def : Pat<(i16(X86ct_select GR16:$src1, GR16:$src2, timm:$cond, EFLAGS)),
+ (CT_SELECT_I386_GR16rr GR16:$src1, GR16:$src2, timm:$cond)>;
+
+ def : Pat<(i32(X86ct_select GR32:$src1, GR32:$src2, timm:$cond, EFLAGS)),
+ (CT_SELECT_I386_GR32rr GR32:$src1, GR32:$src2, timm:$cond)>;
+
+ // i64 patterns handled automatically by type legalization
+}
+
//===----------------------------------------------------------------------===//
// Normal-Instructions-With-Lock-Prefix Pseudo Instructions
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/X86/X86InstrFragments.td b/llvm/lib/Target/X86/X86InstrFragments.td
index 43a07e5971952..af7ad745290a3 100644
--- a/llvm/lib/Target/X86/X86InstrFragments.td
+++ b/llvm/lib/Target/X86/X86InstrFragments.td
@@ -28,6 +28,10 @@ def SDTX86Cmov : SDTypeProfile<1, 4,
[SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>,
SDTCisVT<3, i8>, SDTCisVT<4, i32>]>;
+def SDTX86CtSelect : SDTypeProfile<1, 4,
+ [SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>,
+ SDTCisVT<3, i8>, SDTCisVT<4, i32>]>;
+
// Unary and binary operator instructions that set EFLAGS as a side-effect.
def SDTUnaryArithWithFlags : SDTypeProfile<2, 1,
[SDTCisSameAs<0, 2>,
@@ -181,6 +185,8 @@ def X86ctest : SDNode<"X86ISD::CTEST", SDTX86Ccmp>;
def X86cload : SDNode<"X86ISD::CLOAD", SDTX86Cload, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
def X86cstore : SDNode<"X86ISD::CSTORE", SDTX86Cstore, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+def X86ct_select: SDNode<"X86ISD::CT_SELECT", SDTX86CtSelect, [SDNPInGlue]>;
+
// X86 conditional moves. Operand 0 and operand 1 are the two values
// to select from. Operand 2 is the condition code, and operand 3 is the
// flag operand produced by a CMP or TEST instruction.
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 05ee6eeabef02..3f8da6323db81 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -479,6 +479,556 @@ bool X86InstrInfo::isFrameOperand(const MachineInstr &MI, unsigned int Op,
return false;
}
+struct CtSelectInstructions {
+ unsigned PAndOpc;
+ unsigned PAndnOpc;
+ unsigned POrOpc;
+ unsigned BroadcastOpc;
+ unsigned IntMoveOpc;
+ unsigned MoveOpc;
+ bool Use256;
+ bool UseBlendInstr;
+};
+
+static CtSelectInstructions
+getCtSelectInstructions(unsigned Opcode, const X86Subtarget &Subtarget) {
+ CtSelectInstructions Instructions = {};
+
+ switch (Opcode) {
+ case X86::CT_SELECT_V2F64:
+ if (Subtarget.hasSSE2()) {
+ Instructions.PAndOpc = X86::PANDrr;
+ Instructions.PAndnOpc = X86::PANDNrr;
+ Instructions.POrOpc = X86::PORrr;
+ Instructions.BroadcastOpc = X86::PSHUFDri;
+ Instructions.IntMoveOpc = X86::MOVDI2PDIrr;
+ Instructions.MoveOpc = X86::MOVAPDrr;
+ Instructions.UseBlendInstr = true;
+ } else {
+ llvm_unreachable("Double precision vectors require SSE2");
+ }
+ break;
+ case X86::CT_SELECT_V4F32:
+ if (Subtarget.hasSSE41()) {
+ Instructions.PAndOpc = X86::PANDrr;
+ Instructions.PAndnOpc = X86::PANDNrr;
+ Instructions.POrOpc = X86::PORrr;
+ Instructions.BroadcastOpc = X86::PSHUFDri;
+ Instructions.IntMoveOpc = X86::MOVDI2PDIrr;
+ Instructions.MoveOpc = X86::MOVAPSrr;
+ Instructions.UseBlendInstr = true;
+ } else if (Subtarget.hasSSE2()) {
+ Instructions.PAndOpc = X86::PANDrr;
+ Instructions.PAndnOpc = X86::PANDNrr;
+ Instructions.POrOpc = X86::PORrr;
+ Instructions.BroadcastOpc = X86::PSHUFDri;
+ Instructions.IntMoveOpc = X86::MOVDI2PDIrr;
+ Instructions.MoveOpc = X86::MOVAPSrr;
+ } else {
+ // fallback to SSE1, only support four 32-bit single precision
+ // floating-point values
+ Instructions.PAndOpc = X86::ANDPSrr;
+ Instructions.PAndnOpc = X86::ANDNPSrr;
+ Instructions.POrOpc = X86::ORPSrr;
+ Instructions.BroadcastOpc = X86::SHUFPSrri;
+ Instructions.IntMoveOpc = X86::MOVSS2DIrr;
+ Instructions.MoveOpc = X86::MOVAPSrr;
+ }
+ break;
+ case X86::CT_SELECT_V4I32:
+ case X86::CT_SELECT_V2I64:
+ case X86::CT_SELECT_V8I16:
+ case X86::CT_SELECT_V16I8:
+ if (Subtarget.hasSSE2()) {
+ Instructions.PAndOpc = X86::PANDrr;
+ Instructions.PAndnOpc = X86::PANDNrr;
+ Instructions.POrOpc = X86::PORrr;
+ Instructions.BroadcastOpc = X86::PSHUFDri;
+ Instructions.IntMoveOpc = X86::MOVDI2PDIrr;
+ Instructions.MoveOpc = X86::MOVDQArr;
+ } else {
+ llvm_unreachable("Integer vector operations require SSE2");
+ }
+ break;
+ case X86::CT_SELECT_V8F16:
+ if (Subtarget.hasSSE2()) {
+ Instructions.PAndOpc = X86::PANDrr;
+ Instructions.PAndnOpc = X86::PANDNrr;
+ Instructions.POrOpc = X86::PORrr;
+ Instructions.BroadcastOpc = X86::PSHUFDri;
+ Instructions.IntMoveOpc = X86::MOVDI2PDIrr;
+ Instructions.MoveOpc = X86::MOVDQArr;
+ } else {
+ llvm_unreachable("FP16 vector operations require SSE2");
+ }
+ break;
+ case X86::CT_SELECT_V4F32X:
+ case X86::CT_SELECT_V4I32X:
+ case X86::CT_SELECT_V2F64X:
+ case X86::CT_SELECT_V2I64X:
+ case X86::CT_SELECT_V8I16X:
+ case X86::CT_SELECT_V16I8X:
+ case X86::CT_SELECT_V8F16X:
+ if (Subtarget.hasAVX()) {
+ Instructions.PAndOpc = X86::VPANDrr;
+ Instructions.PAndnOpc = X86::VPANDNrr;
+ Instructions.POrOpc = X86::VPORrr;
+ Instructions.BroadcastOpc = X86::VPSHUFDri;
+ Instructions.IntMoveOpc = X86::VMOVDI2PDIrr;
+ Instructions.MoveOpc = (Opcode == X86::CT_SELECT_V4F32X) ? X86::VMOVAPSrr
+ : (Opcode == X86::CT_SELECT_V2F64X)
+ ? X86::VMOVAPDrr
+ : X86::VMOVDQArr;
+ } else {
+ llvm_unreachable("AVX variants require AVX support");
+ }
+ break;
+ case X86::CT_SELECT_V8F32:
+ case X86::CT_SELECT_V8I32:
+ if (Subtarget.hasAVX()) {
+ Instructions.PAndOpc = X86::VPANDYrr;
+ Instructions.PAndnOpc = X86::VPANDNYrr;
+ Instructions.POrOpc = X86::VPORYrr;
+ Instructions.BroadcastOpc = X86::VPERMILPSYri;
+ Instructions.IntMoveOpc = X86::VMOVDI2PDIrr;
+ Instructions.MoveOpc =
+ (Opcode == X86::CT_SELECT_V8F32) ? X86::VMOVAPSYrr : X86::VMOVDQAYrr;
+ Instructions.Use256 = true;
+ } else {
+ llvm_unreachable("256-bit vectors require AVX");
+ }
+ break;
+ case X86::CT_SELECT_V4F64:
+ case X86::CT_SELECT_V4I64:
+ if (Subtarget.hasAVX()) {
+ Instructions.PAndOpc = X86::VPANDYrr;
+ Instructions.PAndnOpc = X86::VPANDNYrr;
+ Instructions.POrOpc = X86::VPORYrr;
+ Instructions.BroadcastOpc = X86::VPERMILPDYri;
+ Instructions.IntMoveOpc = X86::VMOVDI2PDIrr;
+ Instructions.MoveOpc =
+ (Opcode == X86::CT_SELECT_V4F64) ? X86::VMOVAPDYrr : X86::VMOVDQAYrr;
+ Instructions.Use256 = true;
+ } else {
+ llvm_unreachable("256-bit vectors require AVX");
+ }
+ break;
+ case X86::CT_SELECT_V16I16:
+ case X86::CT_SELECT_V32I8:
+ case X86::CT_SELECT_V16F16:
+ if (Subtarget.hasAVX2()) {
+ Instructions.PAndOpc = X86::VPANDYrr;
+ Instructions.PAndnOpc = X86::VPANDNYrr;
+ Instructions.POrOpc = X86::VPORYrr;
+ Instructions.BroadcastOpc = X86::VPERMILPSYri;
+ Instructions.IntMoveOpc = X86::VMOVDI2PDIrr;
+ Instructions.MoveOpc = X86::VMOVDQAYrr;
+ Instructions.Use256 = true;
+ } else if (Subtarget.hasAVX()) {
+ Instructions.PAndOpc = X86::VPANDYrr;
+ Instructions.PAndnOpc = X86::VPANDNYrr;
+ Instructions.POrOpc = X86::VPORYrr;
+ Instructions.BroadcastOpc = X86::VPERMILPSYri;
+ Instructions.IntMoveOpc = X86::VMOVDI2PDIrr;
+ Instructions.MoveOpc = X86::VMOVDQAYrr;
+ Instructions.Use256 = true;
+ } else {
+ llvm_unreachable("256-bit integer vectors require AVX");
+ }
+ break;
+ default:
+ llvm_unreachable("Unexpected CT_SELECT opcode");
+ }
+
+ return Instructions;
+}
+
+bool X86InstrInfo::expandCtSelectVector(MachineInstr &MI) const {
+ unsigned Opcode = MI.getOpcode();
+ const DebugLoc &DL = MI.getDebugLoc();
+ auto Instruction = getCtSelectInstructions(Opcode, Subtarget);
+
+ MachineBasicBlock *MBB = MI.getParent();
+
+ // Operand layout matches the TableGen definition:
+ // (outs VR128:$dst, VR128:$tmpx, GR32:$tmpg),
+ // (ins VR128:$t, VR128:$f, i8imm:$cond)
+ Register Dst = MI.getOperand(0).getReg();
+ Register MaskReg = MI.getOperand(1).getReg(); // vector mask temp
+ Register TmpGPR = MI.getOperand(2).getReg(); // scalar mask temp (GPR32)
+ Register FalseVal = MI.getOperand(3).getReg(); // true_value
+ Register TrueVal = MI.getOperand(4).getReg(); // false_value
+ X86::CondCode CC = X86::CondCode(MI.getOperand(5).getImm()); // condition
+
+ MachineInstr *FirstInstr = nullptr;
+ MachineInstr *LastInstr = nullptr;
+ auto recordInstr = [&](MachineInstrBuilder MIB) {
+ MachineInstr *NewMI = MIB.getInstr();
+ LastInstr = NewMI;
+ if (!FirstInstr)
+ FirstInstr = NewMI;
+ };
+
+ // Create scalar mask in tempGPR and broadcast to vector mask
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::MOV32ri), TmpGPR)
+ .addImm(0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ const TargetRegisterInfo *TRI = &getRegisterInfo();
+ auto SubReg = TRI->getSubReg(TmpGPR, X86::sub_8bit);
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::SETCCr))
+ .addReg(SubReg)
+ .addImm(CC)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // Zero-extend byte to 32-bit register (movzbl %al, %eax)
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::MOVZX32rr8), TmpGPR)
+ .addReg(SubReg)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ if (Instruction.UseBlendInstr && Subtarget.hasSSE41()) {
+ // Shift left 31 bits to convert 1 -> 0x80000000, 0 -> 0x00000000 (shll $31,
+ // %eax)
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::SHL32ri), TmpGPR)
+ .addReg(TmpGPR)
+ .addImm(31));
+ } else {
+ // Negate to convert 1 -> 0xFFFFFFFF, 0 -> 0x00000000 (negl %eax)
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::NEG32r), TmpGPR)
+ .addReg(TmpGPR));
+ }
+
+ // Broadcast to TmpX (vector mask)
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::PXORrr), MaskReg)
+ .addReg(MaskReg)
+ .addReg(MaskReg)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // Move scalar mask to vector register
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.IntMoveOpc), MaskReg)
+ .addReg(TmpGPR)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ if (Instruction.Use256) {
+ // Broadcast to 256-bit vector register
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.BroadcastOpc), MaskReg)
+ .addReg(MaskReg)
+ .addImm(0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ } else {
+ if (Subtarget.hasSSE2() || Subtarget.hasAVX()) {
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.BroadcastOpc), MaskReg)
+ .addReg(MaskReg)
+ .addImm(0x00)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ } else {
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.BroadcastOpc), MaskReg)
+ .addReg(MaskReg)
+ .addReg(MaskReg)
+ .addImm(0x00)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ }
+ }
+
+ if (Instruction.UseBlendInstr && Subtarget.hasSSE41()) {
+ // Use dedicated blend instructions for SSE4.1+
+ unsigned BlendOpc;
+ switch (Opcode) {
+ case X86::CT_SELECT_V4F32:
+ BlendOpc = X86::BLENDVPSrr0;
+ break;
+ case X86::CT_SELECT_V2F64:
+ BlendOpc = X86::BLENDVPDrr0;
+ break;
+ default:
+ // alias for pblendvb that takes xmm0 as implicit mask register
+ BlendOpc = X86::PBLENDVBrr0;
+ break;
+ }
+
+ // Check if XMM0 is used as one of source registers, if yes then save it
+ // in Dst register and update FalseVal and TrueVal to Dst register
+ bool DidSaveXMM0 = false;
+ Register SavedXMM0 = X86::XMM0;
+ if (FalseVal == X86::XMM0 || TrueVal == X86::XMM0) {
+ Register SrcXMM0 = (FalseVal == X86::XMM0) ? FalseVal : TrueVal;
+
+ // if XMM0 is one of the source registers, it will not match with Dst
+ // registers, so we need to move it to Dst register
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), Dst)
+ .addReg(SrcXMM0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // update FalseVal and TrueVal to Dst register
+ if (FalseVal == X86::XMM0)
+ FalseVal = Dst;
+ if (TrueVal == X86::XMM0)
+ TrueVal = Dst;
+
+ // update SavedXMM0 to Dst register
+ SavedXMM0 = Dst;
+
+ // set DidSaveXMM0 to true to indicate that we saved XMM0 into Dst
+ // register
+ DidSaveXMM0 = true;
+ } else if (MaskReg != X86::XMM0 && Dst != X86::XMM0) {
+
+ // if XMM0 is not allocated for any of the register, we stil need to save
+ // and restore it after using as mask register
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), Dst)
+ .addReg(X86::XMM0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ SavedXMM0 = Dst;
+ DidSaveXMM0 = true;
+ }
+
+ if (MaskReg != X86::XMM0) {
+ // BLENDV uses XMM0 as implicit mask register
+ // https://www.felixcloutier.com/x86/pblendvb
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), X86::XMM0)
+ .addReg(MaskReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge));
+
+ // move FalseVal to mask (use MaskReg as the dst of the blend)
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::MOVAPSrr), MaskReg)
+ .addReg(FalseVal)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // MaskReg := blend(MaskReg /*false*/, TrueVal /*true*/) ; mask in
+ // xmm0
+ recordInstr(BuildMI(*MBB, MI, DL, get(BlendOpc), MaskReg)
+ .addReg(MaskReg)
+ .addReg(TrueVal)
+ .addReg(X86::XMM0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // restore XMM0 from SavedXMM0 if we saved it into Dst
+ if (DidSaveXMM0) {
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), X86::XMM0)
+ .addReg(SavedXMM0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ }
+ // dst = result (now in MaskReg)
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), Dst)
+ .addReg(MaskReg)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ } else {
+ // move FalseVal to Dst register since MaskReg is XMM0 and Dst is not
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), Dst)
+ .addReg(FalseVal)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // Dst := blend(Dst /*false*/, TrueVal /*true*/) ; mask in
+ // xmm0
+ recordInstr(BuildMI(*MBB, MI, DL, get(BlendOpc), Dst)
+ .addReg(Dst)
+ .addReg(TrueVal)
+ .addReg(X86::XMM0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ }
+ } else {
+
+ // dst = mask
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), Dst)
+ .addReg(MaskReg)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // mask &= true_val
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::PANDrr), MaskReg)
+ .addReg(MaskReg)
+ .addReg(TrueVal)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // dst = ~mask & false_val
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::PANDNrr), Dst)
+ .addReg(Dst)
+ .addReg(FalseVal)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // dst |= mask; (mask & t) | (~mask & f)
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::PORrr), Dst)
+ .addReg(Dst)
+ .addReg(MaskReg)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ }
+
+ assert(FirstInstr && LastInstr && "Expected at least one expanded instruction");
+ auto BundleEnd = LastInstr->getIterator();
+ finalizeBundle(*MBB, FirstInstr->getIterator(), std::next(BundleEnd));
+
+ MI.eraseFromParent();
+
+ return true;
+}
+
+bool X86InstrInfo::expandCtSelectWithCMOV(MachineInstr &MI) const {
+ MachineBasicBlock *MBB = MI.getParent();
+ DebugLoc DL = MI.getDebugLoc();
+
+ // CT_SELECT pseudo has: (outs dst), (ins true_val, false_val, cond)
+ MachineOperand &OperandRes = MI.getOperand(0); // destination register
+ MachineOperand &OperandTrue = MI.getOperand(1); // true value
+ MachineOperand &OperandCond = MI.getOperand(3); // condition code
+
+ assert(OperandTrue.isReg() && OperandRes.isReg() && OperandCond.isImm() &&
+ "Invalid operand types");
+ assert(OperandTrue.getReg() == OperandRes.getReg() &&
+ "Result register different from True register");
+
+ assert(Subtarget.hasCMOV() && "target does not support CMOV instructions");
+
+ unsigned Opcode = 0;
+
+ switch (MI.getOpcode()) {
+ case X86::CT_SELECT16rr:
+ Opcode = X86::CMOV16rr;
+ break;
+ case X86::CT_SELECT32rr:
+ Opcode = X86::CMOV32rr;
+ break;
+ case X86::CT_SELECT64rr:
+ Opcode = X86::CMOV64rr;
+ break;
+ case X86::CT_SELECT16rm:
+ Opcode = X86::CMOV16rm;
+ break;
+ case X86::CT_SELECT32rm:
+ Opcode = X86::CMOV32rm;
+ break;
+ case X86::CT_SELECT64rm:
+ Opcode = X86::CMOV64rm;
+ break;
+ default:
+ llvm_unreachable("Invalid CT_SELECT opcode");
+ }
+
+ if (!Subtarget.hasCMOV()) {
+ llvm_unreachable("target does not support cmov");
+ }
+
+ // Build CMOV instruction: copy the first 3 operands (dst, true, false)
+ // and add condition code
+ MachineInstrBuilder CmovBuilder = BuildMI(*MBB, MI, DL, get(Opcode));
+ for (unsigned i = 0u; i < MI.getNumOperands(); ++i) { // Copy
+ CmovBuilder.add(MI.getOperand(i));
+ }
+
+ // Remove the original CT_SELECT instruction
+ MI.eraseFromParent();
+ return true;
+}
+
+/// Expand i386-specific CT_SELECT pseudo instructions (post-RA, constant-time)
+/// These internal pseudos receive a pre-materialized condition byte from the
+/// custom inserter, avoiding EFLAGS corruption issues during i64 type legalization.
+bool X86InstrInfo::expandCtSelectIntWithoutCMOV(MachineInstr &MI) const {
+ MachineBasicBlock *MBB = MI.getParent();
+ DebugLoc DL = MI.getDebugLoc();
+
+ // CT_SELECT_I386_INT_GRxxrr has operands: (outs dst, tmp_byte, tmp_mask),
+ // (ins src1, src2, cond_byte)
+ // Note: cond_byte is pre-materialized by custom inserter, not EFLAGS-dependent
+ Register DstReg = MI.getOperand(0).getReg();
+ Register TmpByteReg = MI.getOperand(1).getReg();
+ Register TmpMaskReg = MI.getOperand(2).getReg();
+ Register Src1Reg = MI.getOperand(3).getReg();
+ Register Src2Reg = MI.getOperand(4).getReg();
+ Register CondByteReg = MI.getOperand(5).getReg(); // Pre-materialized condition byte
+
+ // Determine instruction opcodes based on register width
+ unsigned MovZXOp, NegOp, MovOp, AndOp, NotOp, OrOp;
+ if (MI.getOpcode() == X86::CT_SELECT_I386_INT_GR8rr) {
+ MovZXOp = 0; // No zero-extend needed for GR8
+ NegOp = X86::NEG8r;
+ MovOp = X86::MOV8rr;
+ AndOp = X86::AND8rr;
+ NotOp = X86::NOT8r;
+ OrOp = X86::OR8rr;
+ } else if (MI.getOpcode() == X86::CT_SELECT_I386_INT_GR16rr) {
+ MovZXOp = X86::MOVZX16rr8;
+ NegOp = X86::NEG16r;
+ MovOp = X86::MOV16rr;
+ AndOp = X86::AND16rr;
+ NotOp = X86::NOT16r;
+ OrOp = X86::OR16rr;
+ } else { // X86::CT_SELECT_I386_INT_GR32rr
+ MovZXOp = X86::MOVZX32rr8;
+ NegOp = X86::NEG32r;
+ MovOp = X86::MOV32rr;
+ AndOp = X86::AND32rr;
+ NotOp = X86::NOT32r;
+ OrOp = X86::OR32rr;
+ }
+
+ // 7-instruction constant-time selection bundle (no SETCC inside):
+ // result = (true_val & mask) | (false_val & ~mask)
+ // The condition byte is already materialized, avoiding EFLAGS dependency
+
+ // Step 1: Copy pre-materialized condition byte to TmpByteReg
+ // This allows the bundle to work with allocated temporaries
+ auto I1 = BuildMI(*MBB, MI, DL, get(X86::MOV8rr), TmpByteReg)
+ .addReg(CondByteReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+ auto BundleStart = I1->getIterator();
+
+ // Step 2: Zero-extend condition byte to register width (0 or 1)
+ if (MI.getOpcode() != X86::CT_SELECT_I386_INT_GR8rr) {
+ BuildMI(*MBB, MI, DL, get(MovZXOp), TmpMaskReg)
+ .addReg(TmpByteReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+ }
+
+ // Step 3: Convert condition to bitmask (NEG: 1 -> 0xFFFF..., 0 -> 0x0000...)
+ Register MaskReg = (MI.getOpcode() == X86::CT_SELECT_I386_INT_GR8rr) ? TmpByteReg : TmpMaskReg;
+ BuildMI(*MBB, MI, DL, get(NegOp), MaskReg)
+ .addReg(MaskReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+
+ // Step 4,5: Apply mask to true value - copy src1 to dest, then AND with mask
+ BuildMI(*MBB, MI, DL, get(MovOp), DstReg)
+ .addReg(Src1Reg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+
+ BuildMI(*MBB, MI, DL, get(AndOp), DstReg)
+ .addReg(DstReg)
+ .addReg(MaskReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+
+ // Step 6: Create inverted mask inline (~mask)
+ BuildMI(*MBB, MI, DL, get(NotOp), MaskReg)
+ .addReg(MaskReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+
+ // Step 7: Apply inverted mask to false value - reuse mask register directly
+ BuildMI(*MBB, MI, DL, get(AndOp), MaskReg)
+ .addReg(MaskReg)
+ .addReg(Src2Reg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+
+ // Step 8: Final result: (src1 & mask) | (src2 & ~mask)
+ auto LI = BuildMI(*MBB, MI, DL, get(OrOp), DstReg)
+ .addReg(DstReg)
+ .addReg(MaskReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+
+ // Bundle all generated instructions for atomic execution before removing MI
+ auto BundleEnd = std::next(LI->getIterator());
+ if (BundleStart != BundleEnd) {
+ // Only bundle if we have multiple instructions
+ finalizeBundle(*MBB, BundleStart, BundleEnd);
+ }
+
+ // TODO: Optimization opportunity - The register allocator may choose callee-saved
+ // registers (e.g., %ebx, %esi) for TmpByteReg/TmpMaskReg, causing unnecessary
+ // save/restore overhead. Consider constraining these to caller-saved register
+ // classes (e.g., GR8_AL, GR32_CallSaved) in the TableGen definitions to improve
+ // constant-time performance by eliminating prologue/epilogue instructions.
+
+ // Remove the original pseudo instruction
+ MI.eraseFromParent();
+ return true;
+}
+
static bool isFrameLoadOpcode(int Opcode, TypeSize &MemBytes) {
switch (Opcode) {
default:
@@ -6579,6 +7129,43 @@ bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
case X86::ADD64ri32_DB:
MIB->setDesc(get(X86::OR64ri32));
break;
+
+ case X86::CT_SELECT64rr:
+ case X86::CT_SELECT32rr:
+ case X86::CT_SELECT16rr:
+ case X86::CT_SELECT64rm:
+ case X86::CT_SELECT32rm:
+ case X86::CT_SELECT16rm:
+ // These CT_SELECT pseudos are only selected when CMOV is available
+ // Pattern matching ensures we use CT_SELECT_I386 when CMOV is not available
+ return expandCtSelectWithCMOV(MI);
+
+ // non-cmov CT_SELECT expansion (post-RA, constant-time)
+ // These are the internal pseudos with pre-materialized condition byte
+ case X86::CT_SELECT_I386_INT_GR8rr:
+ case X86::CT_SELECT_I386_INT_GR16rr:
+ case X86::CT_SELECT_I386_INT_GR32rr:
+ return expandCtSelectIntWithoutCMOV(MI);
+
+ case X86::CT_SELECT_V2F64:
+ case X86::CT_SELECT_V4F32:
+ case X86::CT_SELECT_V2I64:
+ case X86::CT_SELECT_V4I32:
+ case X86::CT_SELECT_V8I16:
+ case X86::CT_SELECT_V16I8:
+ case X86::CT_SELECT_V2F64X:
+ case X86::CT_SELECT_V4F32X:
+ case X86::CT_SELECT_V2I64X:
+ case X86::CT_SELECT_V4I32X:
+ case X86::CT_SELECT_V8I16X:
+ case X86::CT_SELECT_V16I8X:
+ case X86::CT_SELECT_V4I64:
+ case X86::CT_SELECT_V8I32:
+ case X86::CT_SELECT_V16I16:
+ case X86::CT_SELECT_V32I8:
+ case X86::CT_SELECT_V4F64:
+ case X86::CT_SELECT_V8F32:
+ return expandCtSelectVector(MI);
}
return false;
}
@@ -10882,18 +11469,25 @@ void X86InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
if (!ST.hasSSE1())
return;
- BuildMI(MBB, Iter, DL, get(X86::V_SET0), Reg);
+ // PXOR is safe to use because it doesn't affect flags.
+ BuildMI(MBB, Iter, DL, get(X86::PXORrr), Reg)
+ .addReg(Reg, RegState::Undef)
+ .addReg(Reg, RegState::Undef);
} else if (X86::VR256RegClass.contains(Reg)) {
// YMM#
if (!ST.hasAVX())
return;
+ // Clearing the XMM subregister zeroes the full YMM, and V_SET0 expands to
+ // a vector XOR, so this is safe to use because it doesn't affect flags.
BuildMI(MBB, Iter, DL, get(X86::V_SET0), TRI.getSubReg(Reg, X86::sub_xmm));
} else if (X86::VR512RegClass.contains(Reg)) {
// ZMM#
if (!ST.hasAVX512())
return;
+ // Clearing the XMM subregister zeroes the full ZMM, and AVX512_128_SET0
+ // expands to a vector XOR, so it doesn't affect flags.
BuildMI(MBB, Iter, DL, get(X86::AVX512_128_SET0),
TRI.getSubReg(Reg, X86::sub_xmm));
} else if (X86::VK1RegClass.contains(Reg) || X86::VK2RegClass.contains(Reg) ||
@@ -10902,8 +11496,11 @@ void X86InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
if (!ST.hasVLX())
return;
- unsigned Op = ST.hasBWI() ? X86::KSET0Q : X86::KSET0W;
- BuildMI(MBB, Iter, DL, get(Op), Reg);
+ // KXOR is safe to use because it doesn't affect flags.
+ unsigned Op = ST.hasBWI() ? X86::KXORQkk : X86::KXORWkk;
+ BuildMI(MBB, Iter, DL, get(Op), Reg)
+ .addReg(Reg, RegState::Undef)
+ .addReg(Reg, RegState::Undef);
}
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index b6fac55373d05..a610dd3458a31 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -736,6 +736,12 @@ class X86InstrInfo final : public X86GenInstrInfo {
bool isFrameOperand(const MachineInstr &MI, unsigned int Op,
int &FrameIndex) const;
+ /// Expand the CT_SELECT pseudo-instructions.
+ bool expandCtSelectWithCMOV(MachineInstr &MI) const;
+ bool expandCtSelectIntWithoutCMOV(MachineInstr &MI) const;
+
+ bool expandCtSelectVector(MachineInstr &MI) const;
+
/// Returns true iff the routine could find two commutable operands in the
/// given machine instruction with 3 vector inputs.
/// The 'SrcOpIdx1' and 'SrcOpIdx2' are INPUT and OUTPUT arguments. Their
diff --git a/llvm/lib/Target/X86/X86InstrPredicates.td b/llvm/lib/Target/X86/X86InstrPredicates.td
index 1b177db55d276..554fc84877787 100644
--- a/llvm/lib/Target/X86/X86InstrPredicates.td
+++ b/llvm/lib/Target/X86/X86InstrPredicates.td
@@ -54,6 +54,11 @@ def HasCF : Predicate<"Subtarget->hasCF()">;
def HasJMPABS : Predicate<"Subtarget->hasJMPABS()">;
def HasCMOV : Predicate<"Subtarget->canUseCMOV()">;
def NoCMOV : Predicate<"!Subtarget->canUseCMOV()">;
+// Predicates for native CMOV instruction (checks hasCMOV(), not canUseCMOV())
+// HasCMOV may be true even without native CMOV (e.g., via SSE emulation)
+// Use HasNativeCMOV/NoNativeCMOV for constant-time code that requires actual CMOV
+def HasNativeCMOV : Predicate<"Subtarget->hasCMOV()">;
+def NoNativeCMOV : Predicate<"!Subtarget->hasCMOV()">;
def HasNOPL : Predicate<"Subtarget->hasNOPL()">;
def HasMMX : Predicate<"Subtarget->hasMMX()">;
def HasSSE1 : Predicate<"Subtarget->hasSSE1()">;
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 886405a0c7bae..2dee1af7423f0 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -623,10 +623,11 @@ void X86PassConfig::addPreEmitPass2() {
// ObjC runtime functions present in the module.
const Function &F = MF.getFunction();
const Module *M = F.getParent();
- return M->getModuleFlag("kcfi") ||
+ return M->getModuleFlag("kcfi") || F.hasFnAttribute("ct-select") ||
(TT.isOSDarwin() &&
(M->getFunction("objc_retainAutoreleasedReturnValue") ||
- M->getFunction("objc_unsafeClaimAutoreleasedReturnValue")));
+ M->getFunction("objc_unsafeClaimAutoreleasedReturnValue"))) ||
+ F.hasFnAttribute("ct-select");
}));
// Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
diff --git a/llvm/test/CodeGen/X86/ctselect-edge-cases.ll b/llvm/test/CodeGen/X86/ctselect-edge-cases.ll
new file mode 100644
index 0000000000000..0797265972a1f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ctselect-edge-cases.ll
@@ -0,0 +1,409 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+cmov | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+cmov | FileCheck %s --check-prefix=X32
+
+; Test ct.select edge cases and corner cases
+
+; Test with very large integers
+define i128 @test_ctselect_i128(i1 %cond, i128 %a, i128 %b) {
+; X64-LABEL: test_ctselect_i128:
+; X64: # %bb.0:
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rsi, %rax
+; X64-NEXT: cmovneq %rdx, %r8
+; X64-NEXT: movq %r8, %rdx
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_i128:
+; X32: # %bb.0:
+; X32-NEXT: pushl %edi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: pushl %eax
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: .cfi_offset %esi, -12
+; X32-NEXT: .cfi_offset %edi, -8
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edx
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl %ecx, 12(%eax)
+; X32-NEXT: movl %edx, 8(%eax)
+; X32-NEXT: movl %edi, 4(%eax)
+; X32-NEXT: movl %esi, (%eax)
+; X32-NEXT: addl $4, %esp
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: popl %edi
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl $4
+ %result = call i128 @llvm.ct.select.i128(i1 %cond, i128 %a, i128 %b)
+ ret i128 %result
+}
+
+; Test with small integer types
+define i1 @test_ctselect_i1(i1 %cond, i1 %a, i1 %b) {
+; X64-LABEL: test_ctselect_i1:
+; X64: # %bb.0:
+; X64-NEXT: movl %edx, %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %esi, %eax
+; X64-NEXT: # kill: def $al killed $al killed $eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_i1:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: # kill: def $al killed $al killed $eax
+; X32-NEXT: retl
+ %result = call i1 @llvm.ct.select.i1(i1 %cond, i1 %a, i1 %b)
+ ret i1 %result
+}
+
+; Test with extremal values
+define i32 @test_ctselect_extremal_values(i1 %cond) {
+; X64-LABEL: test_ctselect_extremal_values:
+; X64: # %bb.0:
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X64-NEXT: movl $-2147483648, %eax # imm = 0x80000000
+; X64-NEXT: cmovnel %ecx, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_extremal_values:
+; X32: # %bb.0:
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: movl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X32-NEXT: movl $-2147483648, %eax # imm = 0x80000000
+; X32-NEXT: cmovnel %ecx, %eax
+; X32-NEXT: retl
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 2147483647, i32 -2147483648)
+ ret i32 %result
+}
+
+; Test with floating point special values
+define float @test_ctselect_f32_special_values(i1 %cond) {
+; X64-LABEL: test_ctselect_f32_special_values:
+; X64: # %bb.0:
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movl $2143289344, %eax # imm = 0x7FC00000
+; X64-NEXT: movl $2139095040, %ecx # imm = 0x7F800000
+; X64-NEXT: cmovnel %eax, %ecx
+; X64-NEXT: movd %ecx, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_f32_special_values:
+; X32: # %bb.0:
+; X32-NEXT: pushl %edi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: pushl %eax
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: .cfi_offset %esi, -12
+; X32-NEXT: .cfi_offset %edi, -8
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: sete %al
+; X32-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %ecx
+; X32-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %edx
+; X32-NEXT: movb %al, %ah
+; X32-NEXT: movzbl %ah, %edi
+; X32-NEXT: negl %edi
+; X32-NEXT: movl %edx, %esi
+; X32-NEXT: andl %edi, %esi
+; X32-NEXT: notl %edi
+; X32-NEXT: andl %ecx, %edi
+; X32-NEXT: orl %edi, %esi
+; X32-NEXT: movl %esi, (%esp)
+; X32-NEXT: flds (%esp)
+; X32-NEXT: addl $4, %esp
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: popl %edi
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl
+ %result = call float @llvm.ct.select.f32(i1 %cond, float 0x7FF8000000000000, float 0x7FF0000000000000)
+ ret float %result
+}
+
+define double @test_ctselect_f64_special_values(i1 %cond) {
+; X64-LABEL: test_ctselect_f64_special_values:
+; X64: # %bb.0:
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movabsq $9221120237041090560, %rax # imm = 0x7FF8000000000000
+; X64-NEXT: movabsq $9218868437227405312, %rcx # imm = 0x7FF0000000000000
+; X64-NEXT: cmovneq %rax, %rcx
+; X64-NEXT: movq %rcx, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_f64_special_values:
+; X32: # %bb.0:
+; X32-NEXT: pushl %edi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: subl $24, %esp
+; X32-NEXT: .cfi_def_cfa_offset 36
+; X32-NEXT: .cfi_offset %esi, -12
+; X32-NEXT: .cfi_offset %edi, -8
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X32-NEXT: sete %al
+; X32-NEXT: fxch %st(1)
+; X32-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NEXT: fstpl (%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl (%esp), %edx
+; X32-NEXT: movb %al, %ah
+; X32-NEXT: movzbl %ah, %edi
+; X32-NEXT: negl %edi
+; X32-NEXT: movl %edx, %esi
+; X32-NEXT: andl %edi, %esi
+; X32-NEXT: notl %edi
+; X32-NEXT: andl %ecx, %edi
+; X32-NEXT: orl %edi, %esi
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movb %al, %ah
+; X32-NEXT: movzbl %ah, %edi
+; X32-NEXT: negl %edi
+; X32-NEXT: movl %edx, %esi
+; X32-NEXT: andl %edi, %esi
+; X32-NEXT: notl %edi
+; X32-NEXT: andl %ecx, %edi
+; X32-NEXT: orl %edi, %esi
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT: fldl {{[0-9]+}}(%esp)
+; X32-NEXT: addl $24, %esp
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: popl %edi
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl
+ %result = call double @llvm.ct.select.f64(i1 %cond, double 0x7FF8000000000000, double 0x7FF0000000000000)
+ ret double %result
+}
+
+; Test with null pointers
+define ptr @test_ctselect_null_ptr(i1 %cond, ptr %ptr) {
+; X64-LABEL: test_ctselect_null_ptr:
+; X64: # %bb.0:
+; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rsi, %rax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_null_ptr:
+; X32: # %bb.0:
+; X32-NEXT: xorl %eax, %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: retl
+ %result = call ptr @llvm.ct.select.p0(i1 %cond, ptr %ptr, ptr null)
+ ret ptr %result
+}
+
+; Test with function pointers
+define ptr @test_ctselect_function_ptr(i1 %cond, ptr %func1, ptr %func2) {
+; X64-LABEL: test_ctselect_function_ptr:
+; X64: # %bb.0:
+; X64-NEXT: movq %rdx, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rsi, %rax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_function_ptr:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: retl
+ %result = call ptr @llvm.ct.select.p0(i1 %cond, ptr %func1, ptr %func2)
+ ret ptr %result
+}
+
+; Test with volatile loads
+define i32 @test_ctselect_volatile_load(i1 %cond, ptr %p1, ptr %p2) {
+; X64-LABEL: test_ctselect_volatile_load:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rsi), %ecx
+; X64-NEXT: movl (%rdx), %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %ecx, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_volatile_load:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl (%ecx), %ecx
+; X32-NEXT: movl (%eax), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %ecx, %eax
+; X32-NEXT: retl
+ %a = load volatile i32, ptr %p1
+ %b = load volatile i32, ptr %p2
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test with atomic loads
+define i32 @test_ctselect_atomic_load(i1 %cond, ptr %p1, ptr %p2) {
+; X64-LABEL: test_ctselect_atomic_load:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rsi), %ecx
+; X64-NEXT: movl (%rdx), %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %ecx, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_atomic_load:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl (%ecx), %ecx
+; X32-NEXT: movl (%eax), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %ecx, %eax
+; X32-NEXT: retl
+ %a = load atomic i32, ptr %p1 acquire, align 4
+ %b = load atomic i32, ptr %p2 acquire, align 4
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test with condition from icmp on pointers
+define ptr @test_ctselect_ptr_cmp(ptr %p1, ptr %p2, ptr %a, ptr %b) {
+; X64-LABEL: test_ctselect_ptr_cmp:
+; X64: # %bb.0:
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: cmpq %rsi, %rdi
+; X64-NEXT: sete %cl
+; X64-NEXT: testb %cl, %cl
+; X64-NEXT: cmovneq %rdx, %rax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_ptr_cmp:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: sete %cl
+; X32-NEXT: testb %cl, %cl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: retl
+ %cmp = icmp eq ptr %p1, %p2
+ %result = call ptr @llvm.ct.select.p0(i1 %cmp, ptr %a, ptr %b)
+ ret ptr %result
+}
+
+; Test with struct pointer types (struct types themselves may not be directly supported)
+%struct.pair = type { i32, i32 }
+
+define ptr @test_ctselect_struct_ptr(i1 %cond, ptr %a, ptr %b) {
+; X64-LABEL: test_ctselect_struct_ptr:
+; X64: # %bb.0:
+; X64-NEXT: movq %rdx, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rsi, %rax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_struct_ptr:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: retl
+ %result = call ptr @llvm.ct.select.p0(i1 %cond, ptr %a, ptr %b)
+ ret ptr %result
+}
+
+; Test with deeply nested conditions (stress test for instruction selection)
+define i32 @test_ctselect_deeply_nested(i1 %c1, i1 %c2, i1 %c3, i1 %c4, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) {
+; X64-LABEL: test_ctselect_deeply_nested:
+; X64: # %bb.0:
+; X64-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; X64-NEXT: movl {{[0-9]+}}(%rsp), %r10d
+; X64-NEXT: movl {{[0-9]+}}(%rsp), %r11d
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %r8d, %r9d
+; X64-NEXT: testb $1, %sil
+; X64-NEXT: cmovnel %r9d, %r11d
+; X64-NEXT: testb $1, %dl
+; X64-NEXT: cmovnel %r11d, %r10d
+; X64-NEXT: testb $1, %cl
+; X64-NEXT: cmovnel %r10d, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_deeply_nested:
+; X32: # %bb.0:
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: .cfi_offset %esi, -8
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %esi, %edx
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %edx, %ecx
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %ecx, %eax
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl
+ %sel1 = call i32 @llvm.ct.select.i32(i1 %c1, i32 %a, i32 %b)
+ %sel2 = call i32 @llvm.ct.select.i32(i1 %c2, i32 %sel1, i32 %c)
+ %sel3 = call i32 @llvm.ct.select.i32(i1 %c3, i32 %sel2, i32 %d)
+ %sel4 = call i32 @llvm.ct.select.i32(i1 %c4, i32 %sel3, i32 %e)
+ ret i32 %sel4
+}
+
+; Test with misaligned loads
+define i32 @test_ctselect_misaligned_load(i1 %cond, ptr %p1, ptr %p2) {
+; X64-LABEL: test_ctselect_misaligned_load:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rdx), %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel (%rsi), %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_misaligned_load:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl (%eax), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel (%ecx), %eax
+; X32-NEXT: retl
+ %a = load i32, ptr %p1, align 1
+ %b = load i32, ptr %p2, align 1
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Declare the intrinsics
+declare i1 @llvm.ct.select.i1(i1, i1, i1)
+declare i128 @llvm.ct.select.i128(i1, i128, i128)
+declare i32 @llvm.ct.select.i32(i1, i32, i32)
+declare float @llvm.ct.select.f32(i1, float, float)
+declare double @llvm.ct.select.f64(i1, double, double)
+declare ptr @llvm.ct.select.p0(i1, ptr, ptr)
diff --git a/llvm/test/CodeGen/X86/ctselect-i386-fp.ll b/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
new file mode 100644
index 0000000000000..b88ec72a37925
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
@@ -0,0 +1,722 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=-cmov | FileCheck %s --check-prefix=I386-NOCMOV
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+cmov | FileCheck %s --check-prefix=I386-CMOV
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=-cmov -verify-machineinstrs | FileCheck %s --check-prefix=I386-NOCMOV
+
+; Comprehensive CT_SELECT tests for i386 targets with floating-point types
+; - Without CMOV: constant-time implementation using FP->int conversion + existing post-RA CT_SELECT
+; - With CMOV: CMOV-based implementation
+; - Verifies security properties: no conditional branches, constant execution time
+; Strategy: FP values stored to memory, converted to integers, CT_SELECT on integers, converted back to FP
+
+; Test basic f32 functionality
+define float @test_ctselect_f32_basic(i1 %cond, float %a, float %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_basic:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %eax
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: flds (%esp)
+; I386-NOCMOV-NEXT: addl $4, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_basic:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: pushl %eax
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: flds (%esp)
+; I386-CMOV-NEXT: addl $4, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
+ ret float %result
+}
+
+; Test f32 with different condition codes
+define float @test_ctselect_f32_eq(float %x, float %y, float %a, float %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_eq:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %eax
+; I386-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fucompp
+; I386-NOCMOV-NEXT: fnstsw %ax
+; I386-NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
+; I386-NOCMOV-NEXT: sahf
+; I386-NOCMOV-NEXT: setnp %al
+; I386-NOCMOV-NEXT: sete %cl
+; I386-NOCMOV-NEXT: testb %al, %cl
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: flds (%esp)
+; I386-NOCMOV-NEXT: addl $4, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_eq:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: pushl %eax
+; I386-CMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fucompi %st(1), %st
+; I386-CMOV-NEXT: fstp %st(0)
+; I386-CMOV-NEXT: setnp %al
+; I386-CMOV-NEXT: sete %cl
+; I386-CMOV-NEXT: testb %al, %cl
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: flds (%esp)
+; I386-CMOV-NEXT: addl $4, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %cmp = fcmp oeq float %x, %y
+ %result = call float @llvm.ct.select.f32(i1 %cmp, float %a, float %b)
+ ret float %result
+}
+
+; Test basic f64 functionality
+define double @test_ctselect_f64_basic(i1 %cond, double %a, double %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f64_basic:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: subl $8, %esp
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldl (%esp)
+; I386-NOCMOV-NEXT: addl $8, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f64_basic:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: subl $8, %esp
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fldl (%esp)
+; I386-CMOV-NEXT: addl $8, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %result = call double @llvm.ct.select.f64(i1 %cond, double %a, double %b)
+ ret double %result
+}
+
+; Test basic x86_fp80 functionality
+define x86_fp80 @test_ctselect_f80_basic(i1 %cond, x86_fp80 %a, x86_fp80 %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f80_basic:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: subl $12, %esp
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldt (%esp)
+; I386-NOCMOV-NEXT: addl $12, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f80_basic:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: subl $12, %esp
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fldt (%esp)
+; I386-CMOV-NEXT: addl $12, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %result = call x86_fp80 @llvm.ct.select.f80(i1 %cond, x86_fp80 %a, x86_fp80 %b)
+ ret x86_fp80 %result
+}
+
+; Test f32 with complex conditions
+define float @test_ctselect_f32_gt(float %x, float %y, float %a, float %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_gt:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %eax
+; I386-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fucompp
+; I386-NOCMOV-NEXT: fnstsw %ax
+; I386-NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
+; I386-NOCMOV-NEXT: sahf
+; I386-NOCMOV-NEXT: seta %al
+; I386-NOCMOV-NEXT: testb %al, %al
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: flds (%esp)
+; I386-NOCMOV-NEXT: addl $4, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_gt:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: pushl %eax
+; I386-CMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fucompi %st(1), %st
+; I386-CMOV-NEXT: fstp %st(0)
+; I386-CMOV-NEXT: seta %al
+; I386-CMOV-NEXT: testb %al, %al
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: flds (%esp)
+; I386-CMOV-NEXT: addl $4, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %cmp = fcmp ogt float %x, %y
+ %result = call float @llvm.ct.select.f32(i1 %cmp, float %a, float %b)
+ ret float %result
+}
+
+; Test constant-time properties: verify no branches in generated code
+define float @test_ctselect_f32_no_branches(i1 %cond, float %a, float %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_no_branches:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %eax
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: flds (%esp)
+; I386-NOCMOV-NEXT: addl $4, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_no_branches:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: pushl %eax
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: flds (%esp)
+; I386-CMOV-NEXT: addl $4, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
+ ret float %result
+}
+
+; Test that BUNDLE directives are present for constant-time guarantees
+define float @test_ctselect_f32_bundled(i1 %cond, float %a, float %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_bundled:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %eax
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: flds (%esp)
+; I386-NOCMOV-NEXT: addl $4, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_bundled:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: pushl %eax
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: flds (%esp)
+; I386-CMOV-NEXT: addl $4, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
+ ret float %result
+}
+
+; Test edge case: NaN handling
+define float @test_ctselect_f32_nan(i1 %cond) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_nan:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: subl $12, %esp
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; I386-NOCMOV-NEXT: fldz
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: fxch %st(1)
+; I386-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fstps (%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl (%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: addl $12, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_nan:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: subl $12, %esp
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; I386-CMOV-NEXT: fldz
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: fxch %st(1)
+; I386-CMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fstps (%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl (%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: addl $12, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %nan = bitcast i32 2139095040 to float ; 0x7F800000 = +inf
+ %zero = bitcast i32 0 to float
+ %result = call float @llvm.ct.select.f32(i1 %cond, float %nan, float %zero)
+ ret float %result
+}
+
+; Test memory alignment for f80
+define x86_fp80 @test_ctselect_f80_alignment(i1 %cond, x86_fp80 %a, x86_fp80 %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f80_alignment:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: subl $12, %esp
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldt (%esp)
+; I386-NOCMOV-NEXT: addl $12, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f80_alignment:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: subl $12, %esp
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fldt (%esp)
+; I386-CMOV-NEXT: addl $12, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %result = call x86_fp80 @llvm.ct.select.f80(i1 %cond, x86_fp80 %a, x86_fp80 %b)
+ ret x86_fp80 %result
+}
+
+; Stress test: multiple CT_SELECT operations
+define float @test_ctselect_f32_multiple(i1 %cond1, i1 %cond2, float %a, float %b, float %c, float %d) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_multiple:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: subl $8, %esp
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: flds (%esp)
+; I386-NOCMOV-NEXT: addl $8, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_multiple:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: subl $8, %esp
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: flds (%esp)
+; I386-CMOV-NEXT: addl $8, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %sel1 = call float @llvm.ct.select.f32(i1 %cond1, float %a, float %b)
+ %sel2 = call float @llvm.ct.select.f32(i1 %cond2, float %sel1, float %c)
+ ret float %sel2
+}
+
+; Declare intrinsics
+declare float @llvm.ct.select.f32(i1, float, float)
+declare double @llvm.ct.select.f64(i1, double, double)
+declare x86_fp80 @llvm.ct.select.f80(i1, x86_fp80, x86_fp80)
diff --git a/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll b/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
new file mode 100644
index 0000000000000..6851c5babeb2d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
@@ -0,0 +1,428 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=i386-unknown-linux-gnu -mattr=-cmov,+mmx < %s | FileCheck %s --check-prefix=I386-NOCMOV
+; RUN: llc -mtriple=i386-unknown-linux-gnu -mattr=+cmov,+mmx < %s | FileCheck %s --check-prefix=I386-CMOV
+; RUN: llc -mtriple=i386-unknown-linux-gnu -mattr=-cmov,+mmx -verify-machineinstrs < %s | FileCheck %s --check-prefix=I386-NOCMOV
+
+; Test constant-time selection with MMX intrinsics to exercise VR64 CT_SELECT
+; These tests use MMX intrinsics to create <1 x i64> values that get allocated to VR64 registers
+
+; Test MMX ct.select using paddd intrinsic to force VR64 allocation
+define <1 x i64> @test_mmx_ctselect_with_paddd(i32 %cond, i64 %a, i64 %b) {
+; I386-NOCMOV-LABEL: test_mmx_ctselect_with_paddd:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: movb %bh, %al
+; I386-NOCMOV-NEXT: movzbl %al, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %esi, %edi
+; I386-NOCMOV-NEXT: andl %ebp, %edi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %ecx, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %ecx
+; I386-NOCMOV-NEXT: andl %esi, %ecx
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: orl %esi, %ecx
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-NOCMOV-NEXT: paddd %mm0, %mm0
+; I386-NOCMOV-NEXT: movq %mm0, (%esp)
+; I386-NOCMOV-NEXT: movl (%esp), %eax
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_mmx_ctselect_with_paddd:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %dl
+; I386-CMOV-NEXT: testb %dl, %dl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-CMOV-NEXT: paddd %mm0, %mm0
+; I386-CMOV-NEXT: movq %mm0, (%esp)
+; I386-CMOV-NEXT: movl (%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-CMOV-NEXT: retl
+ %mmx_a = bitcast i64 %a to <1 x i64>
+ %mmx_b = bitcast i64 %b to <1 x i64>
+ %cmp = icmp ne i32 %cond, 0
+ %sel = call <1 x i64> @llvm.ct.select.v1i64(i1 %cmp, <1 x i64> %mmx_a, <1 x i64> %mmx_b)
+ %result = call <1 x i64> @llvm.x86.mmx.padd.d(<1 x i64> %sel, <1 x i64> %sel)
+ ret <1 x i64> %result
+}
+
+; Test MMX ct.select using psllw intrinsic
+define <1 x i64> @test_mmx_ctselect_with_psllw(i32 %cond, i64 %a, i64 %b) {
+; I386-NOCMOV-LABEL: test_mmx_ctselect_with_psllw:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: movb %bh, %al
+; I386-NOCMOV-NEXT: movzbl %al, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %esi, %edi
+; I386-NOCMOV-NEXT: andl %ebp, %edi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %ecx, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %ecx
+; I386-NOCMOV-NEXT: andl %esi, %ecx
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: orl %esi, %ecx
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-NOCMOV-NEXT: psllw %mm0, %mm0
+; I386-NOCMOV-NEXT: movq %mm0, (%esp)
+; I386-NOCMOV-NEXT: movl (%esp), %eax
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_mmx_ctselect_with_psllw:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %dl
+; I386-CMOV-NEXT: testb %dl, %dl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-CMOV-NEXT: psllw %mm0, %mm0
+; I386-CMOV-NEXT: movq %mm0, (%esp)
+; I386-CMOV-NEXT: movl (%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-CMOV-NEXT: retl
+ %mmx_a = bitcast i64 %a to <1 x i64>
+ %mmx_b = bitcast i64 %b to <1 x i64>
+ %cmp = icmp ne i32 %cond, 0
+ %sel = call <1 x i64> @llvm.ct.select.v1i64(i1 %cmp, <1 x i64> %mmx_a, <1 x i64> %mmx_b)
+ %result = call <1 x i64> @llvm.x86.mmx.psll.w(<1 x i64> %sel, <1 x i64> %sel)
+ ret <1 x i64> %result
+}
+
+; Test nested MMX ct.selects with pand intrinsic
+define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64 %a, i64 %b, i64 %c) {
+; I386-NOCMOV-LABEL: test_mmx_nested_ctselect_with_pand:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: movb %bh, %cl
+; I386-NOCMOV-NEXT: movzbl %cl, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %edx, %edi
+; I386-NOCMOV-NEXT: andl %ebp, %edi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %eax, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: sete %dl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: movb %dl, %dh
+; I386-NOCMOV-NEXT: movzbl %dh, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %esi, %ebx
+; I386-NOCMOV-NEXT: andl %ebp, %ebx
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %eax, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %ebx
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %dl
+; I386-NOCMOV-NEXT: testb %dl, %dl
+; I386-NOCMOV-NEXT: sete %dh
+; I386-NOCMOV-NEXT: movb %dh, %al
+; I386-NOCMOV-NEXT: movzbl %al, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %ecx, %esi
+; I386-NOCMOV-NEXT: andl %ebp, %esi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %ebx, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %esi
+; I386-NOCMOV-NEXT: testb %dl, %dl
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; I386-NOCMOV-NEXT: movb %al, %dl
+; I386-NOCMOV-NEXT: movzbl %dl, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %ebx, %ecx
+; I386-NOCMOV-NEXT: andl %esi, %ecx
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: orl %esi, %ecx
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-NOCMOV-NEXT: pand %mm0, %mm0
+; I386-NOCMOV-NEXT: movq %mm0, (%esp)
+; I386-NOCMOV-NEXT: movl (%esp), %eax
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_mmx_nested_ctselect_with_pand:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %ebx
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 32
+; I386-CMOV-NEXT: .cfi_offset %esi, -12
+; I386-CMOV-NEXT: .cfi_offset %ebx, -8
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %bl
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %bh
+; I386-CMOV-NEXT: testb %bh, %bh
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: testb %bl, %bl
+; I386-CMOV-NEXT: cmovnel %esi, %edx
+; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel %ecx, %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-CMOV-NEXT: pand %mm0, %mm0
+; I386-CMOV-NEXT: movq %mm0, (%esp)
+; I386-CMOV-NEXT: movl (%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: popl %ebx
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-CMOV-NEXT: retl
+ %mmx_a = bitcast i64 %a to <1 x i64>
+ %mmx_b = bitcast i64 %b to <1 x i64>
+ %mmx_c = bitcast i64 %c to <1 x i64>
+ %cmp1 = icmp ne i32 %cond1, 0
+ %cmp2 = icmp ne i32 %cond2, 0
+ %sel1 = call <1 x i64> @llvm.ct.select.v1i64(i1 %cmp2, <1 x i64> %mmx_a, <1 x i64> %mmx_b)
+ %sel2 = call <1 x i64> @llvm.ct.select.v1i64(i1 %cmp1, <1 x i64> %sel1, <1 x i64> %mmx_c)
+ %result = call <1 x i64> @llvm.x86.mmx.pand(<1 x i64> %sel2, <1 x i64> %sel2)
+ ret <1 x i64> %result
+}
+
+; Test MMX ct.select with por intrinsic
+define <1 x i64> @test_mmx_ctselect_with_por(i32 %cond, i64 %a, i64 %b) {
+; I386-NOCMOV-LABEL: test_mmx_ctselect_with_por:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: movb %bh, %al
+; I386-NOCMOV-NEXT: movzbl %al, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %esi, %edi
+; I386-NOCMOV-NEXT: andl %ebp, %edi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %ecx, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %ecx
+; I386-NOCMOV-NEXT: andl %esi, %ecx
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: orl %esi, %ecx
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-NOCMOV-NEXT: por %mm0, %mm0
+; I386-NOCMOV-NEXT: movq %mm0, (%esp)
+; I386-NOCMOV-NEXT: movl (%esp), %eax
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_mmx_ctselect_with_por:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %dl
+; I386-CMOV-NEXT: testb %dl, %dl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-CMOV-NEXT: por %mm0, %mm0
+; I386-CMOV-NEXT: movq %mm0, (%esp)
+; I386-CMOV-NEXT: movl (%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-CMOV-NEXT: retl
+ %mmx_a = bitcast i64 %a to <1 x i64>
+ %mmx_b = bitcast i64 %b to <1 x i64>
+ %cmp = icmp ne i32 %cond, 0
+ %sel = call <1 x i64> @llvm.ct.select.v1i64(i1 %cmp, <1 x i64> %mmx_a, <1 x i64> %mmx_b)
+ %result = call <1 x i64> @llvm.x86.mmx.por(<1 x i64> %sel, <1 x i64> %sel)
+ ret <1 x i64> %result
+}
+
+; Declare MMX intrinsics
+declare <1 x i64> @llvm.x86.mmx.padd.d(<1 x i64>, <1 x i64>)
+declare <1 x i64> @llvm.x86.mmx.psll.w(<1 x i64>, <1 x i64>)
+declare <1 x i64> @llvm.x86.mmx.pand(<1 x i64>, <1 x i64>)
+declare <1 x i64> @llvm.x86.mmx.por(<1 x i64>, <1 x i64>)
+
+; Declare constant-time selection intrinsic
+declare <1 x i64> @llvm.ct.select.v1i64(i1, <1 x i64>, <1 x i64>)
diff --git a/llvm/test/CodeGen/X86/ctselect-i386.ll b/llvm/test/CodeGen/X86/ctselect-i386.ll
new file mode 100644
index 0000000000000..d1cc559f0c1c1
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ctselect-i386.ll
@@ -0,0 +1,267 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=-cmov | FileCheck %s --check-prefix=I386-NOCMOV
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+cmov | FileCheck %s --check-prefix=I386-CMOV
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=-cmov -verify-machineinstrs | FileCheck %s --check-prefix=I386-NOCMOV
+
+; Comprehensive CT_SELECT tests for i386 targets with scalar integer types
+; - Without CMOV: constant-time implementation using post-RA expansion with bundled instructions
+; - With CMOV: CMOV-based implementation
+; - Verifies security properties: no conditional branches, constant execution time
+; All expansion happens post-RA for better optimization control and constant-time guarantees
+
+; Test basic i32 functionality
+define i32 @test_ctselect_i32_basic(i1 %cond, i32 %a, i32 %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_i32_basic:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %bl
+; I386-NOCMOV-NEXT: movb %bl, %bh
+; I386-NOCMOV-NEXT: movzbl %bh, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %eax
+; I386-NOCMOV-NEXT: andl %esi, %eax
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_i32_basic:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: retl
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test i16 functionality
+define i16 @test_ctselect_i16_basic(i1 %cond, i16 %a, i16 %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_i16_basic:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %bl
+; I386-NOCMOV-NEXT: movb %bl, %bh
+; I386-NOCMOV-NEXT: movzbw %bh, %si
+; I386-NOCMOV-NEXT: negw %si
+; I386-NOCMOV-NEXT: movw %dx, %ax
+; I386-NOCMOV-NEXT: andw %si, %ax
+; I386-NOCMOV-NEXT: notw %si
+; I386-NOCMOV-NEXT: andw %cx, %si
+; I386-NOCMOV-NEXT: orw %si, %ax
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_i16_basic:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnew {{[0-9]+}}(%esp), %ax
+; I386-CMOV-NEXT: retl
+ %result = call i16 @llvm.ct.select.i16(i1 %cond, i16 %a, i16 %b)
+ ret i16 %result
+}
+
+; Test i8 functionality
+define i8 @test_ctselect_i8_basic(i1 %cond, i8 %a, i8 %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_i8_basic:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %ah
+; I386-NOCMOV-NEXT: movb %ah, %ch
+; I386-NOCMOV-NEXT: negb %ch
+; I386-NOCMOV-NEXT: movb %dl, %al
+; I386-NOCMOV-NEXT: andb %ch, %al
+; I386-NOCMOV-NEXT: notb %ch
+; I386-NOCMOV-NEXT: andb %cl, %ch
+; I386-NOCMOV-NEXT: orb %ch, %al
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_i8_basic:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: # kill: def $al killed $al killed $eax
+; I386-CMOV-NEXT: retl
+ %result = call i8 @llvm.ct.select.i8(i1 %cond, i8 %a, i8 %b)
+ ret i8 %result
+}
+
+; Test security property: constant-time execution for cryptographic use case
+define i32 @test_crypto_key_select(i32 %secret_bit, i32 %key1, i32 %key2) nounwind {
+; I386-NOCMOV-LABEL: test_crypto_key_select:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %al
+; I386-NOCMOV-NEXT: testb %al, %al
+; I386-NOCMOV-NEXT: sete %bl
+; I386-NOCMOV-NEXT: movb %bl, %bh
+; I386-NOCMOV-NEXT: movzbl %bh, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %eax
+; I386-NOCMOV-NEXT: andl %esi, %eax
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_crypto_key_select:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %cl
+; I386-CMOV-NEXT: testb %cl, %cl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: retl
+ %cond = icmp ne i32 %secret_bit, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %key1, i32 %key2)
+ ret i32 %result
+}
+
+; Test that no conditional branches appear in constant-time path
+define i32 @test_no_conditional_branches(i32 %secret, i32 %val1, i32 %val2) nounwind {
+; I386-NOCMOV-LABEL: test_no_conditional_branches:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %al
+; I386-NOCMOV-NEXT: testb %al, %al
+; I386-NOCMOV-NEXT: sete %bl
+; I386-NOCMOV-NEXT: movb %bl, %bh
+; I386-NOCMOV-NEXT: movzbl %bh, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %eax
+; I386-NOCMOV-NEXT: andl %esi, %eax
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_no_conditional_branches:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %cl
+; I386-CMOV-NEXT: testb %cl, %cl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: retl
+ %cond = icmp ne i32 %secret, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %val1, i32 %val2)
+ ret i32 %result
+}
+
+; Test with comparison condition
+define i32 @test_ctselect_i32_cmp(i32 %a, i32 %b, i32 %c) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_i32_cmp:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: cmpl %edx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: testb %al, %al
+; I386-NOCMOV-NEXT: sete %bl
+; I386-NOCMOV-NEXT: movb %bl, %bh
+; I386-NOCMOV-NEXT: movzbl %bh, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %eax
+; I386-NOCMOV-NEXT: andl %esi, %eax
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_i32_cmp:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: cmpl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %cl
+; I386-CMOV-NEXT: testb %cl, %cl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: retl
+ %cond = icmp eq i32 %a, %c
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %b, i32 %c)
+ ret i32 %result
+}
+
+; Test nested selects
+define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_nested:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %bl
+; I386-NOCMOV-NEXT: movb %bl, %bh
+; I386-NOCMOV-NEXT: movzbl %bh, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %eax, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %dl
+; I386-NOCMOV-NEXT: movb %dl, %dh
+; I386-NOCMOV-NEXT: movzbl %dh, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %ecx, %eax
+; I386-NOCMOV-NEXT: andl %edi, %eax
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %esi, %edi
+; I386-NOCMOV-NEXT: orl %edi, %eax
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_nested:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel %ecx, %eax
+; I386-CMOV-NEXT: retl
+ %sel1 = call i32 @llvm.ct.select.i32(i1 %cond2, i32 %a, i32 %b)
+ %sel2 = call i32 @llvm.ct.select.i32(i1 %cond1, i32 %sel1, i32 %c)
+ ret i32 %sel2
+}
+
+; Declare ct.select intrinsics
+declare i8 @llvm.ct.select.i8(i1, i8, i8)
+declare i16 @llvm.ct.select.i16(i1, i16, i16)
+declare i32 @llvm.ct.select.i32(i1, i32, i32)
diff --git a/llvm/test/CodeGen/X86/ctselect-optimization.ll b/llvm/test/CodeGen/X86/ctselect-optimization.ll
new file mode 100644
index 0000000000000..481d49971a937
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ctselect-optimization.ll
@@ -0,0 +1,304 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+cmov | FileCheck %s
+
+; Test ct.select optimization patterns
+
+; Test smin(x, 0) pattern optimization
+define i32 @test_ctselect_smin_zero(i32 %x) {
+; CHECK-LABEL: test_ctselect_smin_zero:
+; CHECK: # %bb.0:
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: sets %cl
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp slt i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 0)
+ ret i32 %result
+}
+
+; Test smax(x, 0) pattern optimization
+define i32 @test_ctselect_smax_zero(i32 %x) {
+; CHECK-LABEL: test_ctselect_smax_zero:
+; CHECK: # %bb.0:
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: setg %cl
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp sgt i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 0)
+ ret i32 %result
+}
+
+; Test generic smin pattern
+define i32 @test_ctselect_smin_generic(i32 %x, i32 %y) {
+; CHECK-LABEL: test_ctselect_smin_generic:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: cmpl %esi, %edi
+; CHECK-NEXT: setl %cl
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp slt i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 %y)
+ ret i32 %result
+}
+
+; Test generic smax pattern
+define i32 @test_ctselect_smax_generic(i32 %x, i32 %y) {
+; CHECK-LABEL: test_ctselect_smax_generic:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: cmpl %esi, %edi
+; CHECK-NEXT: setg %cl
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp sgt i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 %y)
+ ret i32 %result
+}
+
+; Test umin pattern
+define i32 @test_ctselect_umin_generic(i32 %x, i32 %y) {
+; CHECK-LABEL: test_ctselect_umin_generic:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: cmpl %esi, %edi
+; CHECK-NEXT: setb %cl
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp ult i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 %y)
+ ret i32 %result
+}
+
+; Test umax pattern
+define i32 @test_ctselect_umax_generic(i32 %x, i32 %y) {
+; CHECK-LABEL: test_ctselect_umax_generic:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: cmpl %esi, %edi
+; CHECK-NEXT: seta %cl
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp ugt i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 %y)
+ ret i32 %result
+}
+
+; Test abs pattern
+define i32 @test_ctselect_abs(i32 %x) {
+; CHECK-LABEL: test_ctselect_abs:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: movl %edi, %ecx
+; CHECK-NEXT: negl %ecx
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: sets %dl
+; CHECK-NEXT: testb %dl, %dl
+; CHECK-NEXT: cmovnel %ecx, %eax
+; CHECK-NEXT: retq
+ %neg = sub i32 0, %x
+ %cmp = icmp slt i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %neg, i32 %x)
+ ret i32 %result
+}
+
+; Test nabs pattern (negative abs)
+define i32 @test_ctselect_nabs(i32 %x) {
+; CHECK-LABEL: test_ctselect_nabs:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: negl %eax
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: sets %cl
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %neg = sub i32 0, %x
+ %cmp = icmp slt i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 %neg)
+ ret i32 %result
+}
+
+; Test sign extension pattern
+define i32 @test_ctselect_sign_extend(i32 %x) {
+; CHECK-LABEL: test_ctselect_sign_extend:
+; CHECK: # %bb.0:
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: sets %cl
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: movl $-1, %ecx
+; CHECK-NEXT: cmovnel %ecx, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp slt i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 -1, i32 0)
+ ret i32 %result
+}
+
+; Test zero extension pattern
+define i32 @test_ctselect_zero_extend(i32 %x) {
+; CHECK-LABEL: test_ctselect_zero_extend:
+; CHECK: # %bb.0:
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: setne %cl
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: movl $1, %ecx
+; CHECK-NEXT: cmovnel %ecx, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp ne i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 1, i32 0)
+ ret i32 %result
+}
+
+; Test mask generation pattern
+define i32 @test_ctselect_mask_generation(i32 %x) {
+; CHECK-LABEL: test_ctselect_mask_generation:
+; CHECK: # %bb.0:
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: sets %cl
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: movl $-1, %ecx
+; CHECK-NEXT: cmovnel %ecx, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp slt i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 -1, i32 0)
+ ret i32 %result
+}
+
+; Test constant folding with known condition
+define i32 @test_ctselect_constant_folding_true(i32 %a, i32 %b) {
+; CHECK-LABEL: test_ctselect_constant_folding_true:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: movb $1, %cl
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %result = call i32 @llvm.ct.select.i32(i1 true, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+define i32 @test_ctselect_constant_folding_false(i32 %a, i32 %b) {
+; CHECK-LABEL: test_ctselect_constant_folding_false:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: xorl %ecx, %ecx
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %result = call i32 @llvm.ct.select.i32(i1 false, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test with identical operands
+define i32 @test_ctselect_identical_operands(i1 %cond, i32 %x) {
+; CHECK-LABEL: test_ctselect_identical_operands:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: testb $1, %dil
+; CHECK-NEXT: cmovnel %esi, %eax
+; CHECK-NEXT: retq
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %x, i32 %x)
+ ret i32 %result
+}
+
+; Test with inverted condition
+define i32 @test_ctselect_inverted_condition(i32 %x, i32 %y, i32 %a, i32 %b) {
+; CHECK-LABEL: test_ctselect_inverted_condition:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edx, %eax
+; CHECK-NEXT: cmpl %esi, %edi
+; CHECK-NEXT: sete %dl
+; CHECK-NEXT: testb %dl, %dl
+; CHECK-NEXT: cmovnel %ecx, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp eq i32 %x, %y
+ %not_cmp = xor i1 %cmp, true
+ %result = call i32 @llvm.ct.select.i32(i1 %not_cmp, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test for 64-bit specific optimizations
+define i64 @test_ctselect_i64_smin_zero(i64 %x) {
+; CHECK-LABEL: test_ctselect_i64_smin_zero:
+; CHECK: # %bb.0:
+; CHECK-NEXT: testq %rdi, %rdi
+; CHECK-NEXT: sets %cl
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovneq %rdi, %rax
+; CHECK-NEXT: retq
+ %cmp = icmp slt i64 %x, 0
+ %result = call i64 @llvm.ct.select.i64(i1 %cmp, i64 %x, i64 0)
+ ret i64 %result
+}
+
+; Test for floating point optimizations
+define float @test_ctselect_f32_zero_positive(float %x) {
+; CHECK-LABEL: test_ctselect_f32_zero_positive:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: xorps %xmm1, %xmm1
+; CHECK-NEXT: ucomiss %xmm1, %xmm0
+; CHECK-NEXT: seta %cl
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %eax, %edx
+; CHECK-NEXT: movd %edx, %xmm0
+; CHECK-NEXT: retq
+ %cmp = fcmp ogt float %x, 0.0
+ %result = call float @llvm.ct.select.f32(i1 %cmp, float %x, float 0.0)
+ ret float %result
+}
+
+define double @test_ctselect_f64_zero_positive(double %x) {
+; CHECK-LABEL: test_ctselect_f64_zero_positive:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq %xmm0, %rax
+; CHECK-NEXT: xorpd %xmm1, %xmm1
+; CHECK-NEXT: ucomisd %xmm1, %xmm0
+; CHECK-NEXT: seta %cl
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovneq %rax, %rdx
+; CHECK-NEXT: movq %rdx, %xmm0
+; CHECK-NEXT: retq
+ %cmp = fcmp ogt double %x, 0.0
+ %result = call double @llvm.ct.select.f64(i1 %cmp, double %x, double 0.0)
+ ret double %result
+}
+
+; Test chain of ct.select operations
+define i32 @test_ctselect_chain(i1 %c1, i1 %c2, i1 %c3, i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: test_ctselect_chain:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT: testb $1, %dil
+; CHECK-NEXT: cmovnel %ecx, %r8d
+; CHECK-NEXT: testb $1, %sil
+; CHECK-NEXT: cmovnel %r8d, %r9d
+; CHECK-NEXT: testb $1, %dl
+; CHECK-NEXT: cmovnel %r9d, %eax
+; CHECK-NEXT: retq
+ %sel1 = call i32 @llvm.ct.select.i32(i1 %c1, i32 %a, i32 %b)
+ %sel2 = call i32 @llvm.ct.select.i32(i1 %c2, i32 %sel1, i32 %c)
+ %sel3 = call i32 @llvm.ct.select.i32(i1 %c3, i32 %sel2, i32 %d)
+ ret i32 %sel3
+}
+
+; Declare the intrinsics
+declare i32 @llvm.ct.select.i32(i1, i32, i32)
+declare i64 @llvm.ct.select.i64(i1, i64, i64)
+declare float @llvm.ct.select.f32(i1, float, float)
+declare double @llvm.ct.select.f64(i1, double, double)
diff --git a/llvm/test/CodeGen/X86/ctselect-vector.ll b/llvm/test/CodeGen/X86/ctselect-vector.ll
new file mode 100644
index 0000000000000..2206e32cd6d34
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ctselect-vector.ll
@@ -0,0 +1,1274 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx | FileCheck %s --check-prefix=AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
+
+; Test ct.select functionality for vector types
+
+; 128-bit vectors
+define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) {
+; SSE2-LABEL: test_ctselect_v4i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4i32:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB0_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %xmm0, %xmm1
+; AVX512-NEXT: .LBB0_2:
+; AVX512-NEXT: vmovaps %xmm1, %xmm0
+; AVX512-NEXT: retq
+ %result = call <4 x i32> @llvm.ct.select.v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %result
+}
+
+define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b) {
+; SSE2-LABEL: test_ctselect_v4f32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movaps %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4f32:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4f32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB1_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %xmm0, %xmm1
+; AVX512-NEXT: .LBB1_2:
+; AVX512-NEXT: vmovaps %xmm1, %xmm0
+; AVX512-NEXT: retq
+ %result = call <4 x float> @llvm.ct.select.v4f32(i1 %cond, <4 x float> %a, <4 x float> %b)
+ ret <4 x float> %result
+}
+
+define <2 x i64> @test_ctselect_v2i64(i1 %cond, <2 x i64> %a, <2 x i64> %b) {
+; SSE2-LABEL: test_ctselect_v2i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v2i64:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v2i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v2i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB2_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %xmm0, %xmm1
+; AVX512-NEXT: .LBB2_2:
+; AVX512-NEXT: vmovaps %xmm1, %xmm0
+; AVX512-NEXT: retq
+ %result = call <2 x i64> @llvm.ct.select.v2i64(i1 %cond, <2 x i64> %a, <2 x i64> %b)
+ ret <2 x i64> %result
+}
+
+define <2 x double> @test_ctselect_v2f64(i1 %cond, <2 x double> %a, <2 x double> %b) {
+; SSE2-LABEL: test_ctselect_v2f64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movapd %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v2f64:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v2f64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v2f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB3_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovapd %xmm0, %xmm1
+; AVX512-NEXT: .LBB3_2:
+; AVX512-NEXT: vmovapd %xmm1, %xmm0
+; AVX512-NEXT: retq
+ %result = call <2 x double> @llvm.ct.select.v2f64(i1 %cond, <2 x double> %a, <2 x double> %b)
+ ret <2 x double> %result
+}
+
+; 256-bit vectors
+define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
+; SSE2-LABEL: test_ctselect_v8i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: movd %eax, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm5, %xmm4
+; SSE2-NEXT: pand %xmm0, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm4
+; SSE2-NEXT: por %xmm5, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm3, %xmm2
+; SSE2-NEXT: por %xmm0, %xmm2
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v8i32:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm3, %ymm3
+; AVX-NEXT: vmovd %eax, %ymm3
+; AVX-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
+; AVX-NEXT: vmovdqa %ymm3, %ymm2
+; AVX-NEXT: pand %ymm0, %ymm3
+; AVX-NEXT: pandn %ymm1, %ymm2
+; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: vmovaps %ymm2, %ymm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v8i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm3, %ymm3
+; AVX2-NEXT: vmovd %eax, %ymm3
+; AVX2-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
+; AVX2-NEXT: vmovdqa %ymm3, %ymm2
+; AVX2-NEXT: pand %ymm0, %ymm3
+; AVX2-NEXT: pandn %ymm1, %ymm2
+; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: vmovaps %ymm2, %ymm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v8i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB4_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %ymm0, %ymm1
+; AVX512-NEXT: .LBB4_2:
+; AVX512-NEXT: vmovaps %ymm1, %ymm0
+; AVX512-NEXT: retq
+ %result = call <8 x i32> @llvm.ct.select.v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b)
+ ret <8 x i32> %result
+}
+
+define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b) {
+; SSE2-LABEL: test_ctselect_v8f32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: movd %eax, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
+; SSE2-NEXT: movaps %xmm5, %xmm4
+; SSE2-NEXT: pand %xmm0, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm4
+; SSE2-NEXT: por %xmm5, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movaps %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm3, %xmm2
+; SSE2-NEXT: por %xmm0, %xmm2
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v8f32:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm3, %ymm3
+; AVX-NEXT: vmovd %eax, %ymm3
+; AVX-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
+; AVX-NEXT: vmovdqa %ymm3, %ymm2
+; AVX-NEXT: pand %ymm0, %ymm3
+; AVX-NEXT: pandn %ymm1, %ymm2
+; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: vmovaps %ymm2, %ymm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v8f32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm3, %ymm3
+; AVX2-NEXT: vmovd %eax, %ymm3
+; AVX2-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
+; AVX2-NEXT: vmovdqa %ymm3, %ymm2
+; AVX2-NEXT: pand %ymm0, %ymm3
+; AVX2-NEXT: pandn %ymm1, %ymm2
+; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: vmovaps %ymm2, %ymm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v8f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB5_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %ymm0, %ymm1
+; AVX512-NEXT: .LBB5_2:
+; AVX512-NEXT: vmovaps %ymm1, %ymm0
+; AVX512-NEXT: retq
+ %result = call <8 x float> @llvm.ct.select.v8f32(i1 %cond, <8 x float> %a, <8 x float> %b)
+ ret <8 x float> %result
+}
+
+define <4 x i64> @test_ctselect_v4i64(i1 %cond, <4 x i64> %a, <4 x i64> %b) {
+; SSE2-LABEL: test_ctselect_v4i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: movd %eax, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm5, %xmm4
+; SSE2-NEXT: pand %xmm0, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm4
+; SSE2-NEXT: por %xmm5, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm3, %xmm2
+; SSE2-NEXT: por %xmm0, %xmm2
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4i64:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm3, %ymm3
+; AVX-NEXT: vmovd %eax, %ymm3
+; AVX-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
+; AVX-NEXT: vmovdqa %ymm3, %ymm2
+; AVX-NEXT: pand %ymm0, %ymm3
+; AVX-NEXT: pandn %ymm1, %ymm2
+; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: vmovaps %ymm2, %ymm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm3, %ymm3
+; AVX2-NEXT: vmovd %eax, %ymm3
+; AVX2-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
+; AVX2-NEXT: vmovdqa %ymm3, %ymm2
+; AVX2-NEXT: pand %ymm0, %ymm3
+; AVX2-NEXT: pandn %ymm1, %ymm2
+; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: vmovaps %ymm2, %ymm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB6_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %ymm0, %ymm1
+; AVX512-NEXT: .LBB6_2:
+; AVX512-NEXT: vmovaps %ymm1, %ymm0
+; AVX512-NEXT: retq
+ %result = call <4 x i64> @llvm.ct.select.v4i64(i1 %cond, <4 x i64> %a, <4 x i64> %b)
+ ret <4 x i64> %result
+}
+
+define <4 x double> @test_ctselect_v4f64(i1 %cond, <4 x double> %a, <4 x double> %b) {
+; SSE2-LABEL: test_ctselect_v4f64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: movd %eax, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
+; SSE2-NEXT: movapd %xmm5, %xmm4
+; SSE2-NEXT: pand %xmm0, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm4
+; SSE2-NEXT: por %xmm5, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movapd %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm3, %xmm2
+; SSE2-NEXT: por %xmm0, %xmm2
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4f64:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm3, %ymm3
+; AVX-NEXT: vmovd %eax, %ymm3
+; AVX-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
+; AVX-NEXT: vmovdqa %ymm3, %ymm2
+; AVX-NEXT: pand %ymm0, %ymm3
+; AVX-NEXT: pandn %ymm1, %ymm2
+; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: vmovaps %ymm2, %ymm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4f64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm3, %ymm3
+; AVX2-NEXT: vmovd %eax, %ymm3
+; AVX2-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
+; AVX2-NEXT: vmovdqa %ymm3, %ymm2
+; AVX2-NEXT: pand %ymm0, %ymm3
+; AVX2-NEXT: pandn %ymm1, %ymm2
+; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: vmovaps %ymm2, %ymm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB7_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovapd %ymm0, %ymm1
+; AVX512-NEXT: .LBB7_2:
+; AVX512-NEXT: vmovapd %ymm1, %ymm0
+; AVX512-NEXT: retq
+ %result = call <4 x double> @llvm.ct.select.v4f64(i1 %cond, <4 x double> %a, <4 x double> %b)
+ ret <4 x double> %result
+}
+
+; 512-bit vectors (AVX512 only)
+define <16 x i32> @test_ctselect_v16i32(i1 %cond, <16 x i32> %a, <16 x i32> %b) {
+; SSE2-LABEL: test_ctselect_v16i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm9, %xmm9
+; SSE2-NEXT: movd %eax, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm9, %xmm8
+; SSE2-NEXT: pand %xmm0, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm8
+; SSE2-NEXT: por %xmm9, %xmm8
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm5, %xmm4
+; SSE2-NEXT: por %xmm0, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm6, %xmm5
+; SSE2-NEXT: por %xmm0, %xmm5
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm6
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: pandn %xmm7, %xmm6
+; SSE2-NEXT: por %xmm0, %xmm6
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v16i32:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm5, %ymm5
+; AVX-NEXT: vmovd %eax, %ymm5
+; AVX-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
+; AVX-NEXT: vmovdqa %ymm5, %ymm4
+; AVX-NEXT: pand %ymm0, %ymm5
+; AVX-NEXT: pandn %ymm2, %ymm4
+; AVX-NEXT: por %ymm5, %ymm4
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm0, %ymm0
+; AVX-NEXT: vmovd %eax, %ymm0
+; AVX-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
+; AVX-NEXT: vmovdqa %ymm0, %ymm2
+; AVX-NEXT: pand %ymm1, %ymm0
+; AVX-NEXT: pandn %ymm3, %ymm2
+; AVX-NEXT: por %ymm0, %ymm2
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v16i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm5, %ymm5
+; AVX2-NEXT: vmovd %eax, %ymm5
+; AVX2-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
+; AVX2-NEXT: vmovdqa %ymm5, %ymm4
+; AVX2-NEXT: pand %ymm0, %ymm5
+; AVX2-NEXT: pandn %ymm2, %ymm4
+; AVX2-NEXT: por %ymm5, %ymm4
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm0, %ymm0
+; AVX2-NEXT: vmovd %eax, %ymm0
+; AVX2-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
+; AVX2-NEXT: vmovdqa %ymm0, %ymm2
+; AVX2-NEXT: pand %ymm1, %ymm0
+; AVX2-NEXT: pandn %ymm3, %ymm2
+; AVX2-NEXT: por %ymm0, %ymm2
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v16i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB8_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %zmm0, %zmm1
+; AVX512-NEXT: .LBB8_2:
+; AVX512-NEXT: vmovaps %zmm1, %zmm0
+; AVX512-NEXT: retq
+ %result = call <16 x i32> @llvm.ct.select.v16i32(i1 %cond, <16 x i32> %a, <16 x i32> %b)
+ ret <16 x i32> %result
+}
+
+define <16 x float> @test_ctselect_v16f32(i1 %cond, <16 x float> %a, <16 x float> %b) {
+; SSE2-LABEL: test_ctselect_v16f32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm9, %xmm9
+; SSE2-NEXT: movd %eax, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
+; SSE2-NEXT: movaps %xmm9, %xmm8
+; SSE2-NEXT: pand %xmm0, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm8
+; SSE2-NEXT: por %xmm9, %xmm8
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movaps %xmm0, %xmm4
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm5, %xmm4
+; SSE2-NEXT: por %xmm0, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movaps %xmm0, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm6, %xmm5
+; SSE2-NEXT: por %xmm0, %xmm5
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movaps %xmm0, %xmm6
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: pandn %xmm7, %xmm6
+; SSE2-NEXT: por %xmm0, %xmm6
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v16f32:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm5, %ymm5
+; AVX-NEXT: vmovd %eax, %ymm5
+; AVX-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
+; AVX-NEXT: vmovdqa %ymm5, %ymm4
+; AVX-NEXT: pand %ymm0, %ymm5
+; AVX-NEXT: pandn %ymm2, %ymm4
+; AVX-NEXT: por %ymm5, %ymm4
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm0, %ymm0
+; AVX-NEXT: vmovd %eax, %ymm0
+; AVX-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
+; AVX-NEXT: vmovdqa %ymm0, %ymm2
+; AVX-NEXT: pand %ymm1, %ymm0
+; AVX-NEXT: pandn %ymm3, %ymm2
+; AVX-NEXT: por %ymm0, %ymm2
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v16f32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm5, %ymm5
+; AVX2-NEXT: vmovd %eax, %ymm5
+; AVX2-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
+; AVX2-NEXT: vmovdqa %ymm5, %ymm4
+; AVX2-NEXT: pand %ymm0, %ymm5
+; AVX2-NEXT: pandn %ymm2, %ymm4
+; AVX2-NEXT: por %ymm5, %ymm4
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm0, %ymm0
+; AVX2-NEXT: vmovd %eax, %ymm0
+; AVX2-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
+; AVX2-NEXT: vmovdqa %ymm0, %ymm2
+; AVX2-NEXT: pand %ymm1, %ymm0
+; AVX2-NEXT: pandn %ymm3, %ymm2
+; AVX2-NEXT: por %ymm0, %ymm2
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v16f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB9_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %zmm0, %zmm1
+; AVX512-NEXT: .LBB9_2:
+; AVX512-NEXT: vmovaps %zmm1, %zmm0
+; AVX512-NEXT: retq
+ %result = call <16 x float> @llvm.ct.select.v16f32(i1 %cond, <16 x float> %a, <16 x float> %b)
+ ret <16 x float> %result
+}
+
+define <8 x i64> @test_ctselect_v8i64(i1 %cond, <8 x i64> %a, <8 x i64> %b) {
+; SSE2-LABEL: test_ctselect_v8i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm9, %xmm9
+; SSE2-NEXT: movd %eax, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm9, %xmm8
+; SSE2-NEXT: pand %xmm0, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm8
+; SSE2-NEXT: por %xmm9, %xmm8
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm5, %xmm4
+; SSE2-NEXT: por %xmm0, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm6, %xmm5
+; SSE2-NEXT: por %xmm0, %xmm5
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm6
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: pandn %xmm7, %xmm6
+; SSE2-NEXT: por %xmm0, %xmm6
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v8i64:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm5, %ymm5
+; AVX-NEXT: vmovd %eax, %ymm5
+; AVX-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX-NEXT: vmovdqa %ymm5, %ymm4
+; AVX-NEXT: pand %ymm0, %ymm5
+; AVX-NEXT: pandn %ymm2, %ymm4
+; AVX-NEXT: por %ymm5, %ymm4
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm0, %ymm0
+; AVX-NEXT: vmovd %eax, %ymm0
+; AVX-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
+; AVX-NEXT: vmovdqa %ymm0, %ymm2
+; AVX-NEXT: pand %ymm1, %ymm0
+; AVX-NEXT: pandn %ymm3, %ymm2
+; AVX-NEXT: por %ymm0, %ymm2
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v8i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm5, %ymm5
+; AVX2-NEXT: vmovd %eax, %ymm5
+; AVX2-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX2-NEXT: vmovdqa %ymm5, %ymm4
+; AVX2-NEXT: pand %ymm0, %ymm5
+; AVX2-NEXT: pandn %ymm2, %ymm4
+; AVX2-NEXT: por %ymm5, %ymm4
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm0, %ymm0
+; AVX2-NEXT: vmovd %eax, %ymm0
+; AVX2-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
+; AVX2-NEXT: vmovdqa %ymm0, %ymm2
+; AVX2-NEXT: pand %ymm1, %ymm0
+; AVX2-NEXT: pandn %ymm3, %ymm2
+; AVX2-NEXT: por %ymm0, %ymm2
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v8i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB10_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %zmm0, %zmm1
+; AVX512-NEXT: .LBB10_2:
+; AVX512-NEXT: vmovaps %zmm1, %zmm0
+; AVX512-NEXT: retq
+ %result = call <8 x i64> @llvm.ct.select.v8i64(i1 %cond, <8 x i64> %a, <8 x i64> %b)
+ ret <8 x i64> %result
+}
+
+define <8 x double> @test_ctselect_v8f64(i1 %cond, <8 x double> %a, <8 x double> %b) {
+; SSE2-LABEL: test_ctselect_v8f64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm9, %xmm9
+; SSE2-NEXT: movd %eax, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
+; SSE2-NEXT: movapd %xmm9, %xmm8
+; SSE2-NEXT: pand %xmm0, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm8
+; SSE2-NEXT: por %xmm9, %xmm8
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movapd %xmm0, %xmm4
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm5, %xmm4
+; SSE2-NEXT: por %xmm0, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movapd %xmm0, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm6, %xmm5
+; SSE2-NEXT: por %xmm0, %xmm5
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movapd %xmm0, %xmm6
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: pandn %xmm7, %xmm6
+; SSE2-NEXT: por %xmm0, %xmm6
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v8f64:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm5, %ymm5
+; AVX-NEXT: vmovd %eax, %ymm5
+; AVX-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX-NEXT: vmovdqa %ymm5, %ymm4
+; AVX-NEXT: pand %ymm0, %ymm5
+; AVX-NEXT: pandn %ymm2, %ymm4
+; AVX-NEXT: por %ymm5, %ymm4
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm0, %ymm0
+; AVX-NEXT: vmovd %eax, %ymm0
+; AVX-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
+; AVX-NEXT: vmovdqa %ymm0, %ymm2
+; AVX-NEXT: pand %ymm1, %ymm0
+; AVX-NEXT: pandn %ymm3, %ymm2
+; AVX-NEXT: por %ymm0, %ymm2
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v8f64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm5, %ymm5
+; AVX2-NEXT: vmovd %eax, %ymm5
+; AVX2-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX2-NEXT: vmovdqa %ymm5, %ymm4
+; AVX2-NEXT: pand %ymm0, %ymm5
+; AVX2-NEXT: pandn %ymm2, %ymm4
+; AVX2-NEXT: por %ymm5, %ymm4
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm0, %ymm0
+; AVX2-NEXT: vmovd %eax, %ymm0
+; AVX2-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
+; AVX2-NEXT: vmovdqa %ymm0, %ymm2
+; AVX2-NEXT: pand %ymm1, %ymm0
+; AVX2-NEXT: pandn %ymm3, %ymm2
+; AVX2-NEXT: por %ymm0, %ymm2
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v8f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB11_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovapd %zmm0, %zmm1
+; AVX512-NEXT: .LBB11_2:
+; AVX512-NEXT: vmovapd %zmm1, %zmm0
+; AVX512-NEXT: retq
+ %result = call <8 x double> @llvm.ct.select.v8f64(i1 %cond, <8 x double> %a, <8 x double> %b)
+ ret <8 x double> %result
+}
+
+; Test with constant conditions for vector types
+define <4 x i32> @test_ctselect_v4i32_const_true(<4 x i32> %a, <4 x i32> %b) {
+; SSE2-LABEL: test_ctselect_v4i32_const_true:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movb $1, %al
+; SSE2-NEXT: testb %al, %al
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4i32_const_true:
+; AVX: # %bb.0:
+; AVX-NEXT: movb $1, %al
+; AVX-NEXT: testb %al, %al
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4i32_const_true:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movb $1, %al
+; AVX2-NEXT: testb %al, %al
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4i32_const_true:
+; AVX512: # %bb.0:
+; AVX512-NEXT: retq
+ %result = call <4 x i32> @llvm.ct.select.v4i32(i1 true, <4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %result
+}
+
+define <4 x i32> @test_ctselect_v4i32_const_false(<4 x i32> %a, <4 x i32> %b) {
+; SSE2-LABEL: test_ctselect_v4i32_const_false:
+; SSE2: # %bb.0:
+; SSE2-NEXT: xorl %eax, %eax
+; SSE2-NEXT: testb %al, %al
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4i32_const_false:
+; AVX: # %bb.0:
+; AVX-NEXT: xorl %eax, %eax
+; AVX-NEXT: testb %al, %al
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4i32_const_false:
+; AVX2: # %bb.0:
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: testb %al, %al
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4i32_const_false:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm1, %xmm0
+; AVX512-NEXT: retq
+ %result = call <4 x i32> @llvm.ct.select.v4i32(i1 false, <4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %result
+}
+
+; Test with comparison conditions for vector types
+define <4 x i32> @test_ctselect_v4i32_icmp(i32 %x, i32 %y, <4 x i32> %a, <4 x i32> %b) {
+; SSE2-LABEL: test_ctselect_v4i32_icmp:
+; SSE2: # %bb.0:
+; SSE2-NEXT: cmpl %esi, %edi
+; SSE2-NEXT: sete %al
+; SSE2-NEXT: testb %al, %al
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4i32_icmp:
+; AVX: # %bb.0:
+; AVX-NEXT: cmpl %esi, %edi
+; AVX-NEXT: sete %al
+; AVX-NEXT: testb %al, %al
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4i32_icmp:
+; AVX2: # %bb.0:
+; AVX2-NEXT: cmpl %esi, %edi
+; AVX2-NEXT: sete %al
+; AVX2-NEXT: testb %al, %al
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4i32_icmp:
+; AVX512: # %bb.0:
+; AVX512-NEXT: cmpl %esi, %edi
+; AVX512-NEXT: je .LBB14_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %xmm1, %xmm0
+; AVX512-NEXT: .LBB14_2:
+; AVX512-NEXT: retq
+ %cond = icmp eq i32 %x, %y
+ %result = call <4 x i32> @llvm.ct.select.v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %result
+}
+
+; Declare the intrinsics
+declare <4 x i32> @llvm.ct.select.v4i32(i1, <4 x i32>, <4 x i32>)
+declare <4 x float> @llvm.ct.select.v4f32(i1, <4 x float>, <4 x float>)
+declare <2 x i64> @llvm.ct.select.v2i64(i1, <2 x i64>, <2 x i64>)
+declare <2 x double> @llvm.ct.select.v2f64(i1, <2 x double>, <2 x double>)
+declare <8 x i32> @llvm.ct.select.v8i32(i1, <8 x i32>, <8 x i32>)
+declare <8 x float> @llvm.ct.select.v8f32(i1, <8 x float>, <8 x float>)
+declare <4 x i64> @llvm.ct.select.v4i64(i1, <4 x i64>, <4 x i64>)
+declare <4 x double> @llvm.ct.select.v4f64(i1, <4 x double>, <4 x double>)
+declare <16 x i32> @llvm.ct.select.v16i32(i1, <16 x i32>, <16 x i32>)
+declare <16 x float> @llvm.ct.select.v16f32(i1, <16 x float>, <16 x float>)
+declare <8 x i64> @llvm.ct.select.v8i64(i1, <8 x i64>, <8 x i64>)
+declare <8 x double> @llvm.ct.select.v8f64(i1, <8 x double>, <8 x double>)
>From ef41b7e3c7087bf56957b6f3a5d05bf27d2c7f52 Mon Sep 17 00:00:00 2001
From: wizardengineer <juliuswoosebert at gmail.com>
Date: Wed, 5 Nov 2025 23:56:12 -0500
Subject: [PATCH 2/4] [LLVM][X86] Add f80 support for ct.select
Add special handling for x86_fp80 types in CTSELECT lowering by splitting
them into three 32-bit chunks, performing constant-time selection on each
chunk, and reassembling the result. This fixes crashes when compiling
tests with f80 types.
Also updated ctselect.ll to match current generic fallback implementation.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 75 +-
llvm/lib/Target/X86/X86ISelLowering.h | 2 -
llvm/lib/Target/X86/X86InstrFragments.td | 2 +-
llvm/lib/Target/X86/X86InstrInfo.cpp | 847 +++++++++++----------
llvm/lib/Target/X86/X86InstrInfo.h | 9 +-
llvm/test/CodeGen/X86/ctselect-i386-fp.ll | 272 +++----
llvm/test/CodeGen/X86/ctselect-i386-mmx.ll | 441 +++++------
llvm/test/CodeGen/X86/ctselect-vector.ll | 703 +++++------------
8 files changed, 1026 insertions(+), 1325 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 208707e12978b..058fbd718f0f9 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -26276,6 +26276,69 @@ SDValue X86TargetLowering::LowerCT_SELECT(SDValue Op, SelectionDAG &DAG) const {
return DAG.getBitcast(VT, CtSelect);
}
+ // Handle f80 types by splitting into three 32-bit chunks
+ if (VT == MVT::f80) {
+ SDValue Chain = DAG.getEntryNode();
+
+ // Create temporary stack slots for input f80 values
+ SDValue TrueSlot = DAG.CreateStackTemporary(MVT::f80);
+ SDValue FalseSlot = DAG.CreateStackTemporary(MVT::f80);
+
+ // Store f80 values to memory
+ SDValue StoreTrueF80 =
+ DAG.getStore(Chain, DL, TrueOp, TrueSlot, MachinePointerInfo());
+ SDValue StoreFalseF80 =
+ DAG.getStore(Chain, DL, FalseOp, FalseSlot, MachinePointerInfo());
+
+ // Load i32 parts from memory (3 chunks for 96-bit f80 storage)
+ SDValue TruePart0 =
+ DAG.getLoad(MVT::i32, DL, StoreTrueF80, TrueSlot, MachinePointerInfo());
+ SDValue TruePart1Ptr =
+ DAG.getMemBasePlusOffset(TrueSlot, TypeSize::getFixed(4), DL);
+ SDValue TruePart1 = DAG.getLoad(MVT::i32, DL, StoreTrueF80, TruePart1Ptr,
+ MachinePointerInfo());
+ SDValue TruePart2Ptr =
+ DAG.getMemBasePlusOffset(TrueSlot, TypeSize::getFixed(8), DL);
+ SDValue TruePart2 = DAG.getLoad(MVT::i32, DL, StoreTrueF80, TruePart2Ptr,
+ MachinePointerInfo());
+
+ SDValue FalsePart0 = DAG.getLoad(MVT::i32, DL, StoreFalseF80, FalseSlot,
+ MachinePointerInfo());
+ SDValue FalsePart1Ptr =
+ DAG.getMemBasePlusOffset(FalseSlot, TypeSize::getFixed(4), DL);
+ SDValue FalsePart1 = DAG.getLoad(MVT::i32, DL, StoreFalseF80, FalsePart1Ptr,
+ MachinePointerInfo());
+ SDValue FalsePart2Ptr =
+ DAG.getMemBasePlusOffset(FalseSlot, TypeSize::getFixed(8), DL);
+ SDValue FalsePart2 = DAG.getLoad(MVT::i32, DL, StoreFalseF80, FalsePart2Ptr,
+ MachinePointerInfo());
+
+ // Perform CT_SELECT on each 32-bit chunk
+ SDValue Part0Ops[] = {FalsePart0, TruePart0, CC, ProcessedCond};
+ SDValue Part0Select = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part0Ops);
+ SDValue Part1Ops[] = {FalsePart1, TruePart1, CC, ProcessedCond};
+ SDValue Part1Select = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part1Ops);
+ SDValue Part2Ops[] = {FalsePart2, TruePart2, CC, ProcessedCond};
+ SDValue Part2Select = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part2Ops);
+
+ // Create result stack slot and store the selected parts
+ SDValue ResultSlot = DAG.CreateStackTemporary(MVT::f80);
+ SDValue StorePart0 =
+ DAG.getStore(Chain, DL, Part0Select, ResultSlot, MachinePointerInfo());
+ SDValue ResPart1Ptr =
+ DAG.getMemBasePlusOffset(ResultSlot, TypeSize::getFixed(4), DL);
+ SDValue StorePart1 = DAG.getStore(StorePart0, DL, Part1Select, ResPart1Ptr,
+ MachinePointerInfo());
+ SDValue ResPart2Ptr =
+ DAG.getMemBasePlusOffset(ResultSlot, TypeSize::getFixed(8), DL);
+ SDValue StorePart2 = DAG.getStore(StorePart1, DL, Part2Select, ResPart2Ptr,
+ MachinePointerInfo());
+
+ // Load complete f80 result from memory
+ return DAG.getLoad(MVT::f80, DL, StorePart2, ResultSlot,
+ MachinePointerInfo());
+ }
+
// Create final CT_SELECT node
SDValue Ops[] = {FalseOp, TrueOp, CC, ProcessedCond};
return DAG.getNode(X86ISD::CT_SELECT, DL, Op.getValueType(), Ops,
@@ -34979,12 +35042,6 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
}
}
-bool X86TargetLowering::isSelectSupported(SelectSupportKind Kind) const {
- if (Kind == SelectSupportKind::CtSelect) {
- return true;
- }
- return TargetLoweringBase::isSelectSupported(Kind);
-}
/// Replace a node with an illegal result type with a new node built out of
/// custom code.
void X86TargetLowering::ReplaceNodeResults(SDNode *N,
@@ -36637,6 +36694,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
}
}
+/// Return true if the addressing mode represented by AM is legal for this
+/// target, for a load/store of the specified type.
+bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
+ const AddrMode &AM, Type *Ty,
+ unsigned AS,
+ Instruction *I) const {
// X86 supports extremely general addressing modes.
CodeModel::Model M = getTargetMachine().getCodeModel();
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 190e4e750092f..f1699c63de8d5 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -176,8 +176,6 @@ namespace llvm {
///
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
- bool isSelectSupported(SelectSupportKind Kind) const override;
-
/// Replace the results of node with an illegal result
/// type with new values built out of custom code.
///
diff --git a/llvm/lib/Target/X86/X86InstrFragments.td b/llvm/lib/Target/X86/X86InstrFragments.td
index af7ad745290a3..42ac36bdd5313 100644
--- a/llvm/lib/Target/X86/X86InstrFragments.td
+++ b/llvm/lib/Target/X86/X86InstrFragments.td
@@ -185,7 +185,7 @@ def X86ctest : SDNode<"X86ISD::CTEST", SDTX86Ccmp>;
def X86cload : SDNode<"X86ISD::CLOAD", SDTX86Cload, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
def X86cstore : SDNode<"X86ISD::CSTORE", SDTX86Cstore, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
-def X86ct_select: SDNode<"X86ISD::CT_SELECT", SDTX86CtSelect, [SDNPInGlue]>;
+def X86ct_select: SDNode<"X86ISD::CT_SELECT", SDTX86CtSelect>;
// X86 conditional moves. Operand 0 and operand 1 are the two values
// to select from. Operand 2 is the condition code, and operand 3 is the
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 3f8da6323db81..90f8deae0c39b 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -694,8 +694,7 @@ bool X86InstrInfo::expandCtSelectVector(MachineInstr &MI) const {
.addImm(31));
} else {
// Negate to convert 1 -> 0xFFFFFFFF, 0 -> 0x00000000 (negl %eax)
- recordInstr(BuildMI(*MBB, MI, DL, get(X86::NEG32r), TmpGPR)
- .addReg(TmpGPR));
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::NEG32r), TmpGPR).addReg(TmpGPR));
}
// Broadcast to TmpX (vector mask)
@@ -852,7 +851,8 @@ bool X86InstrInfo::expandCtSelectVector(MachineInstr &MI) const {
.setMIFlags(MachineInstr::MIFlag::NoMerge));
}
- assert(FirstInstr && LastInstr && "Expected at least one expanded instruction");
+ assert(FirstInstr && LastInstr &&
+ "Expected at least one expanded instruction");
auto BundleEnd = LastInstr->getIterator();
finalizeBundle(*MBB, FirstInstr->getIterator(), std::next(BundleEnd));
@@ -920,25 +920,28 @@ bool X86InstrInfo::expandCtSelectWithCMOV(MachineInstr &MI) const {
/// Expand i386-specific CT_SELECT pseudo instructions (post-RA, constant-time)
/// These internal pseudos receive a pre-materialized condition byte from the
-/// custom inserter, avoiding EFLAGS corruption issues during i64 type legalization.
+/// custom inserter, avoiding EFLAGS corruption issues during i64 type
+/// legalization.
bool X86InstrInfo::expandCtSelectIntWithoutCMOV(MachineInstr &MI) const {
MachineBasicBlock *MBB = MI.getParent();
DebugLoc DL = MI.getDebugLoc();
// CT_SELECT_I386_INT_GRxxrr has operands: (outs dst, tmp_byte, tmp_mask),
// (ins src1, src2, cond_byte)
- // Note: cond_byte is pre-materialized by custom inserter, not EFLAGS-dependent
+ // Note: cond_byte is pre-materialized by custom inserter, not
+ // EFLAGS-dependent
Register DstReg = MI.getOperand(0).getReg();
Register TmpByteReg = MI.getOperand(1).getReg();
Register TmpMaskReg = MI.getOperand(2).getReg();
Register Src1Reg = MI.getOperand(3).getReg();
Register Src2Reg = MI.getOperand(4).getReg();
- Register CondByteReg = MI.getOperand(5).getReg(); // Pre-materialized condition byte
+ Register CondByteReg =
+ MI.getOperand(5).getReg(); // Pre-materialized condition byte
// Determine instruction opcodes based on register width
unsigned MovZXOp, NegOp, MovOp, AndOp, NotOp, OrOp;
if (MI.getOpcode() == X86::CT_SELECT_I386_INT_GR8rr) {
- MovZXOp = 0; // No zero-extend needed for GR8
+ MovZXOp = 0; // No zero-extend needed for GR8
NegOp = X86::NEG8r;
MovOp = X86::MOV8rr;
AndOp = X86::AND8rr;
@@ -967,8 +970,8 @@ bool X86InstrInfo::expandCtSelectIntWithoutCMOV(MachineInstr &MI) const {
// Step 1: Copy pre-materialized condition byte to TmpByteReg
// This allows the bundle to work with allocated temporaries
auto I1 = BuildMI(*MBB, MI, DL, get(X86::MOV8rr), TmpByteReg)
- .addReg(CondByteReg)
- .setMIFlag(MachineInstr::MIFlag::NoMerge);
+ .addReg(CondByteReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
auto BundleStart = I1->getIterator();
// Step 2: Zero-extend condition byte to register width (0 or 1)
@@ -979,7 +982,9 @@ bool X86InstrInfo::expandCtSelectIntWithoutCMOV(MachineInstr &MI) const {
}
// Step 3: Convert condition to bitmask (NEG: 1 -> 0xFFFF..., 0 -> 0x0000...)
- Register MaskReg = (MI.getOpcode() == X86::CT_SELECT_I386_INT_GR8rr) ? TmpByteReg : TmpMaskReg;
+ Register MaskReg = (MI.getOpcode() == X86::CT_SELECT_I386_INT_GR8rr)
+ ? TmpByteReg
+ : TmpMaskReg;
BuildMI(*MBB, MI, DL, get(NegOp), MaskReg)
.addReg(MaskReg)
.setMIFlag(MachineInstr::MIFlag::NoMerge);
@@ -1007,9 +1012,9 @@ bool X86InstrInfo::expandCtSelectIntWithoutCMOV(MachineInstr &MI) const {
// Step 8: Final result: (src1 & mask) | (src2 & ~mask)
auto LI = BuildMI(*MBB, MI, DL, get(OrOp), DstReg)
- .addReg(DstReg)
- .addReg(MaskReg)
- .setMIFlag(MachineInstr::MIFlag::NoMerge);
+ .addReg(DstReg)
+ .addReg(MaskReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
// Bundle all generated instructions for atomic execution before removing MI
auto BundleEnd = std::next(LI->getIterator());
@@ -1018,11 +1023,12 @@ bool X86InstrInfo::expandCtSelectIntWithoutCMOV(MachineInstr &MI) const {
finalizeBundle(*MBB, BundleStart, BundleEnd);
}
- // TODO: Optimization opportunity - The register allocator may choose callee-saved
- // registers (e.g., %ebx, %esi) for TmpByteReg/TmpMaskReg, causing unnecessary
- // save/restore overhead. Consider constraining these to caller-saved register
- // classes (e.g., GR8_AL, GR32_CallSaved) in the TableGen definitions to improve
- // constant-time performance by eliminating prologue/epilogue instructions.
+ // TODO: Optimization opportunity - The register allocator may choose
+ // callee-saved registers (e.g., %ebx, %esi) for TmpByteReg/TmpMaskReg,
+ // causing unnecessary save/restore overhead. Consider constraining these to
+ // caller-saved register classes (e.g., GR8_AL, GR32_CallSaved) in the
+ // TableGen definitions to improve constant-time performance by eliminating
+ // prologue/epilogue instructions.
// Remove the original pseudo instruction
MI.eraseFromParent();
@@ -1310,8 +1316,7 @@ static bool regIsPICBase(Register BaseReg, const MachineRegisterInfo &MRI) {
return isPICBase;
}
-bool X86InstrInfo::isReMaterializableImpl(
- const MachineInstr &MI) const {
+bool X86InstrInfo::isReMaterializableImpl(const MachineInstr &MI) const {
switch (MI.getOpcode()) {
default:
// This function should only be called for opcodes with the ReMaterializable
@@ -1826,32 +1831,32 @@ MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA(unsigned MIOpc,
switch (MIOpc) {
default:
llvm_unreachable("Unreachable!");
- CASE_NF(SHL8ri)
- CASE_NF(SHL16ri) {
- unsigned ShAmt = MI.getOperand(2).getImm();
- MIB.addReg(0)
- .addImm(1LL << ShAmt)
- .addReg(InRegLEA, RegState::Kill)
- .addImm(0)
- .addReg(0);
- break;
- }
- CASE_NF(INC8r)
- CASE_NF(INC16r)
+ CASE_NF(SHL8ri)
+ CASE_NF(SHL16ri) {
+ unsigned ShAmt = MI.getOperand(2).getImm();
+ MIB.addReg(0)
+ .addImm(1LL << ShAmt)
+ .addReg(InRegLEA, RegState::Kill)
+ .addImm(0)
+ .addReg(0);
+ break;
+ }
+ CASE_NF(INC8r)
+ CASE_NF(INC16r)
addRegOffset(MIB, InRegLEA, true, 1);
break;
- CASE_NF(DEC8r)
- CASE_NF(DEC16r)
+ CASE_NF(DEC8r)
+ CASE_NF(DEC16r)
addRegOffset(MIB, InRegLEA, true, -1);
break;
- CASE_NF(ADD8ri)
- CASE_NF(ADD16ri)
+ CASE_NF(ADD8ri)
+ CASE_NF(ADD16ri)
case X86::ADD8ri_DB:
case X86::ADD16ri_DB:
addRegOffset(MIB, InRegLEA, true, MI.getOperand(2).getImm());
break;
- CASE_NF(ADD8rr)
- CASE_NF(ADD16rr)
+ CASE_NF(ADD8rr)
+ CASE_NF(ADD16rr)
case X86::ADD8rr_DB:
case X86::ADD16rr_DB: {
Src2 = MI.getOperand(2).getReg();
@@ -1989,128 +1994,129 @@ MachineInstr *X86InstrInfo::convertToThreeAddress(MachineInstr &MI,
switch (MIOpc) {
default:
llvm_unreachable("Unreachable!");
- CASE_NF(SHL64ri) {
- assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
- unsigned ShAmt = getTruncatedShiftCount(MI, 2);
- if (!isTruncatedShiftCountForLEA(ShAmt))
- return nullptr;
-
- // LEA can't handle RSP.
- if (Src.getReg().isVirtual() && !MF.getRegInfo().constrainRegClass(
- Src.getReg(), &X86::GR64_NOSPRegClass))
- return nullptr;
+ CASE_NF(SHL64ri) {
+ assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
+ unsigned ShAmt = getTruncatedShiftCount(MI, 2);
+ if (!isTruncatedShiftCountForLEA(ShAmt))
+ return nullptr;
- NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r))
- .add(Dest)
- .addReg(0)
- .addImm(1LL << ShAmt)
- .add(Src)
- .addImm(0)
- .addReg(0);
- break;
- }
- CASE_NF(SHL32ri) {
- assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
- unsigned ShAmt = getTruncatedShiftCount(MI, 2);
- if (!isTruncatedShiftCountForLEA(ShAmt))
- return nullptr;
+ // LEA can't handle RSP.
+ if (Src.getReg().isVirtual() &&
+ !MF.getRegInfo().constrainRegClass(Src.getReg(),
+ &X86::GR64_NOSPRegClass))
+ return nullptr;
- unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
+ NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r))
+ .add(Dest)
+ .addReg(0)
+ .addImm(1LL << ShAmt)
+ .add(Src)
+ .addImm(0)
+ .addReg(0);
+ break;
+ }
+ CASE_NF(SHL32ri) {
+ assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
+ unsigned ShAmt = getTruncatedShiftCount(MI, 2);
+ if (!isTruncatedShiftCountForLEA(ShAmt))
+ return nullptr;
- // LEA can't handle ESP.
- bool isKill;
- MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
- if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
- isKill, ImplicitOp, LV, LIS))
- return nullptr;
+ unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
- MachineInstrBuilder MIB =
- BuildMI(MF, MI.getDebugLoc(), get(Opc))
- .add(Dest)
- .addReg(0)
- .addImm(1LL << ShAmt)
- .addReg(SrcReg, getKillRegState(isKill), SrcSubReg)
- .addImm(0)
- .addReg(0);
- if (ImplicitOp.getReg() != 0)
- MIB.add(ImplicitOp);
- NewMI = MIB;
+ // LEA can't handle ESP.
+ bool isKill;
+ MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
+ if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
+ isKill, ImplicitOp, LV, LIS))
+ return nullptr;
- // Add kills if classifyLEAReg created a new register.
- if (LV && SrcReg != Src.getReg())
- LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
- break;
- }
- CASE_NF(SHL8ri)
+ MachineInstrBuilder MIB =
+ BuildMI(MF, MI.getDebugLoc(), get(Opc))
+ .add(Dest)
+ .addReg(0)
+ .addImm(1LL << ShAmt)
+ .addReg(SrcReg, getKillRegState(isKill), SrcSubReg)
+ .addImm(0)
+ .addReg(0);
+ if (ImplicitOp.getReg() != 0)
+ MIB.add(ImplicitOp);
+ NewMI = MIB;
+
+ // Add kills if classifyLEAReg created a new register.
+ if (LV && SrcReg != Src.getReg())
+ LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
+ break;
+ }
+ CASE_NF(SHL8ri)
Is8BitOp = true;
[[fallthrough]];
- CASE_NF(SHL16ri) {
- assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
- unsigned ShAmt = getTruncatedShiftCount(MI, 2);
- if (!isTruncatedShiftCountForLEA(ShAmt))
- return nullptr;
- return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
- }
- CASE_NF(INC64r)
- CASE_NF(INC32r) {
- assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!");
- unsigned Opc = (MIOpc == X86::INC64r || MIOpc == X86::INC64r_NF)
- ? X86::LEA64r
- : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
- bool isKill;
- MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
- if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
- isKill, ImplicitOp, LV, LIS))
- return nullptr;
+ CASE_NF(SHL16ri) {
+ assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
+ unsigned ShAmt = getTruncatedShiftCount(MI, 2);
+ if (!isTruncatedShiftCountForLEA(ShAmt))
+ return nullptr;
+ return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
+ }
+ CASE_NF(INC64r)
+ CASE_NF(INC32r) {
+ assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!");
+ unsigned Opc = (MIOpc == X86::INC64r || MIOpc == X86::INC64r_NF)
+ ? X86::LEA64r
+ : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
+ bool isKill;
+ MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
+ if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
+ isKill, ImplicitOp, LV, LIS))
+ return nullptr;
- MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
- .add(Dest)
- .addReg(SrcReg, getKillRegState(isKill));
- if (ImplicitOp.getReg() != 0)
- MIB.add(ImplicitOp);
+ MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
+ .add(Dest)
+ .addReg(SrcReg, getKillRegState(isKill));
+ if (ImplicitOp.getReg() != 0)
+ MIB.add(ImplicitOp);
- NewMI = addOffset(MIB, 1);
+ NewMI = addOffset(MIB, 1);
- // Add kills if classifyLEAReg created a new register.
- if (LV && SrcReg != Src.getReg())
- LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
- break;
- }
- CASE_NF(DEC64r)
- CASE_NF(DEC32r) {
- assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!");
- unsigned Opc = (MIOpc == X86::DEC64r || MIOpc == X86::DEC64r_NF)
- ? X86::LEA64r
- : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
-
- bool isKill;
- MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
- if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
- isKill, ImplicitOp, LV, LIS))
- return nullptr;
+ // Add kills if classifyLEAReg created a new register.
+ if (LV && SrcReg != Src.getReg())
+ LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
+ break;
+ }
+ CASE_NF(DEC64r)
+ CASE_NF(DEC32r) {
+ assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!");
+ unsigned Opc = (MIOpc == X86::DEC64r || MIOpc == X86::DEC64r_NF)
+ ? X86::LEA64r
+ : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
+
+ bool isKill;
+ MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
+ if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
+ isKill, ImplicitOp, LV, LIS))
+ return nullptr;
- MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
- .add(Dest)
- .addReg(SrcReg, getKillRegState(isKill));
- if (ImplicitOp.getReg() != 0)
- MIB.add(ImplicitOp);
+ MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
+ .add(Dest)
+ .addReg(SrcReg, getKillRegState(isKill));
+ if (ImplicitOp.getReg() != 0)
+ MIB.add(ImplicitOp);
- NewMI = addOffset(MIB, -1);
+ NewMI = addOffset(MIB, -1);
- // Add kills if classifyLEAReg created a new register.
- if (LV && SrcReg != Src.getReg())
- LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
- break;
- }
- CASE_NF(DEC8r)
- CASE_NF(INC8r)
+ // Add kills if classifyLEAReg created a new register.
+ if (LV && SrcReg != Src.getReg())
+ LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
+ break;
+ }
+ CASE_NF(DEC8r)
+ CASE_NF(INC8r)
Is8BitOp = true;
[[fallthrough]];
- CASE_NF(DEC16r)
- CASE_NF(INC16r)
+ CASE_NF(DEC16r)
+ CASE_NF(INC16r)
return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
- CASE_NF(ADD64rr)
- CASE_NF(ADD32rr)
+ CASE_NF(ADD64rr)
+ CASE_NF(ADD32rr)
case X86::ADD64rr_DB:
case X86::ADD32rr_DB: {
assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
@@ -2161,21 +2167,21 @@ MachineInstr *X86InstrInfo::convertToThreeAddress(MachineInstr &MI,
NumRegOperands = 3;
break;
}
- CASE_NF(ADD8rr)
+ CASE_NF(ADD8rr)
case X86::ADD8rr_DB:
Is8BitOp = true;
[[fallthrough]];
- CASE_NF(ADD16rr)
+ CASE_NF(ADD16rr)
case X86::ADD16rr_DB:
return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
- CASE_NF(ADD64ri32)
+ CASE_NF(ADD64ri32)
case X86::ADD64ri32_DB:
assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
NewMI = addOffset(
BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src),
MI.getOperand(2));
break;
- CASE_NF(ADD32ri)
+ CASE_NF(ADD32ri)
case X86::ADD32ri_DB: {
assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
@@ -2200,62 +2206,62 @@ MachineInstr *X86InstrInfo::convertToThreeAddress(MachineInstr &MI,
LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
break;
}
- CASE_NF(ADD8ri)
+ CASE_NF(ADD8ri)
case X86::ADD8ri_DB:
Is8BitOp = true;
[[fallthrough]];
- CASE_NF(ADD16ri)
+ CASE_NF(ADD16ri)
case X86::ADD16ri_DB:
return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
- CASE_NF(SUB8ri)
- CASE_NF(SUB16ri)
+ CASE_NF(SUB8ri)
+ CASE_NF(SUB16ri)
/// FIXME: Support these similar to ADD8ri/ADD16ri*.
return nullptr;
- CASE_NF(SUB32ri) {
- if (!MI.getOperand(2).isImm())
- return nullptr;
- int64_t Imm = MI.getOperand(2).getImm();
- if (!isInt<32>(-Imm))
- return nullptr;
+ CASE_NF(SUB32ri) {
+ if (!MI.getOperand(2).isImm())
+ return nullptr;
+ int64_t Imm = MI.getOperand(2).getImm();
+ if (!isInt<32>(-Imm))
+ return nullptr;
- assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
- unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
+ assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
+ unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
- bool isKill;
- MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
- if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, SrcSubReg,
- isKill, ImplicitOp, LV, LIS))
- return nullptr;
+ bool isKill;
+ MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
+ if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, SrcSubReg,
+ isKill, ImplicitOp, LV, LIS))
+ return nullptr;
- MachineInstrBuilder MIB =
- BuildMI(MF, MI.getDebugLoc(), get(Opc))
- .add(Dest)
- .addReg(SrcReg, getKillRegState(isKill), SrcSubReg);
- if (ImplicitOp.getReg() != 0)
- MIB.add(ImplicitOp);
+ MachineInstrBuilder MIB =
+ BuildMI(MF, MI.getDebugLoc(), get(Opc))
+ .add(Dest)
+ .addReg(SrcReg, getKillRegState(isKill), SrcSubReg);
+ if (ImplicitOp.getReg() != 0)
+ MIB.add(ImplicitOp);
- NewMI = addOffset(MIB, -Imm);
+ NewMI = addOffset(MIB, -Imm);
- // Add kills if classifyLEAReg created a new register.
- if (LV && SrcReg != Src.getReg())
- LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
- break;
- }
+ // Add kills if classifyLEAReg created a new register.
+ if (LV && SrcReg != Src.getReg())
+ LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
+ break;
+ }
- CASE_NF(SUB64ri32) {
- if (!MI.getOperand(2).isImm())
- return nullptr;
- int64_t Imm = MI.getOperand(2).getImm();
- if (!isInt<32>(-Imm))
- return nullptr;
+ CASE_NF(SUB64ri32) {
+ if (!MI.getOperand(2).isImm())
+ return nullptr;
+ int64_t Imm = MI.getOperand(2).getImm();
+ if (!isInt<32>(-Imm))
+ return nullptr;
- assert(MI.getNumOperands() >= 3 && "Unknown sub instruction!");
+ assert(MI.getNumOperands() >= 3 && "Unknown sub instruction!");
- MachineInstrBuilder MIB =
- BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src);
- NewMI = addOffset(MIB, -Imm);
- break;
- }
+ MachineInstrBuilder MIB =
+ BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src);
+ NewMI = addOffset(MIB, -Imm);
+ break;
+ }
case X86::VMOVDQU8Z128rmk:
case X86::VMOVDQU8Z256rmk:
@@ -2855,17 +2861,17 @@ MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
case X86::OP##_ND:
switch (Opc) {
- // SHLD B, C, I <-> SHRD C, B, (BitWidth - I)
- CASE_ND(SHRD16rri8)
- CASE_ND(SHLD16rri8)
- CASE_ND(SHRD32rri8)
- CASE_ND(SHLD32rri8)
- CASE_ND(SHRD64rri8)
- CASE_ND(SHLD64rri8) {
- unsigned Size;
- switch (Opc) {
- default:
- llvm_unreachable("Unreachable!");
+ // SHLD B, C, I <-> SHRD C, B, (BitWidth - I)
+ CASE_ND(SHRD16rri8)
+ CASE_ND(SHLD16rri8)
+ CASE_ND(SHRD32rri8)
+ CASE_ND(SHLD32rri8)
+ CASE_ND(SHRD64rri8)
+ CASE_ND(SHLD64rri8) {
+ unsigned Size;
+ switch (Opc) {
+ default:
+ llvm_unreachable("Unreachable!");
#define FROM_TO_SIZE(A, B, S) \
case X86::A: \
Opc = X86::B; \
@@ -2884,16 +2890,16 @@ MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
Size = S; \
break;
- FROM_TO_SIZE(SHRD16rri8, SHLD16rri8, 16)
- FROM_TO_SIZE(SHRD32rri8, SHLD32rri8, 32)
- FROM_TO_SIZE(SHRD64rri8, SHLD64rri8, 64)
+ FROM_TO_SIZE(SHRD16rri8, SHLD16rri8, 16)
+ FROM_TO_SIZE(SHRD32rri8, SHLD32rri8, 32)
+ FROM_TO_SIZE(SHRD64rri8, SHLD64rri8, 64)
#undef FROM_TO_SIZE
+ }
+ WorkingMI = CloneIfNew(MI);
+ WorkingMI->setDesc(get(Opc));
+ WorkingMI->getOperand(3).setImm(Size - MI.getOperand(3).getImm());
+ break;
}
- WorkingMI = CloneIfNew(MI);
- WorkingMI->setDesc(get(Opc));
- WorkingMI->getOperand(3).setImm(Size - MI.getOperand(3).getImm());
- break;
- }
case X86::PFSUBrr:
case X86::PFSUBRrr:
// PFSUB x, y: x = x - y
@@ -3177,15 +3183,16 @@ MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
WorkingMI = CloneIfNew(MI);
WorkingMI->setDesc(get(Opc));
break;
- CASE_ND(CMOV16rr)
- CASE_ND(CMOV32rr)
- CASE_ND(CMOV64rr) {
- WorkingMI = CloneIfNew(MI);
- unsigned OpNo = MI.getDesc().getNumOperands() - 1;
- X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(OpNo).getImm());
- WorkingMI->getOperand(OpNo).setImm(X86::GetOppositeBranchCondition(CC));
- break;
- }
+ CASE_ND(CMOV16rr)
+ CASE_ND(CMOV32rr)
+ CASE_ND(CMOV64rr) {
+ WorkingMI = CloneIfNew(MI);
+ unsigned OpNo = MI.getDesc().getNumOperands() - 1;
+ X86::CondCode CC =
+ static_cast<X86::CondCode>(MI.getOperand(OpNo).getImm());
+ WorkingMI->getOperand(OpNo).setImm(X86::GetOppositeBranchCondition(CC));
+ break;
+ }
case X86::VPTERNLOGDZrri:
case X86::VPTERNLOGDZrmi:
case X86::VPTERNLOGDZ128rri:
@@ -5410,29 +5417,29 @@ bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
CmpMask = CmpValue = 0;
}
return true;
- // A SUB can be used to perform comparison.
- CASE_ND(SUB64rm)
- CASE_ND(SUB32rm)
- CASE_ND(SUB16rm)
- CASE_ND(SUB8rm)
+ // A SUB can be used to perform comparison.
+ CASE_ND(SUB64rm)
+ CASE_ND(SUB32rm)
+ CASE_ND(SUB16rm)
+ CASE_ND(SUB8rm)
SrcReg = MI.getOperand(1).getReg();
SrcReg2 = 0;
CmpMask = 0;
CmpValue = 0;
return true;
- CASE_ND(SUB64rr)
- CASE_ND(SUB32rr)
- CASE_ND(SUB16rr)
- CASE_ND(SUB8rr)
+ CASE_ND(SUB64rr)
+ CASE_ND(SUB32rr)
+ CASE_ND(SUB16rr)
+ CASE_ND(SUB8rr)
SrcReg = MI.getOperand(1).getReg();
SrcReg2 = MI.getOperand(2).getReg();
CmpMask = 0;
CmpValue = 0;
return true;
- CASE_ND(SUB64ri32)
- CASE_ND(SUB32ri)
- CASE_ND(SUB16ri)
- CASE_ND(SUB8ri)
+ CASE_ND(SUB64ri32)
+ CASE_ND(SUB32ri)
+ CASE_ND(SUB16ri)
+ CASE_ND(SUB8ri)
SrcReg = MI.getOperand(1).getReg();
SrcReg2 = 0;
if (MI.getOperand(2).isImm()) {
@@ -5487,27 +5494,27 @@ bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI,
case X86::CMP32rr:
case X86::CMP16rr:
case X86::CMP8rr:
- CASE_ND(SUB64rr)
- CASE_ND(SUB32rr)
- CASE_ND(SUB16rr)
- CASE_ND(SUB8rr) {
- Register OISrcReg;
- Register OISrcReg2;
- int64_t OIMask;
- int64_t OIValue;
- if (!analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) ||
- OIMask != ImmMask || OIValue != ImmValue)
+ CASE_ND(SUB64rr)
+ CASE_ND(SUB32rr)
+ CASE_ND(SUB16rr)
+ CASE_ND(SUB8rr) {
+ Register OISrcReg;
+ Register OISrcReg2;
+ int64_t OIMask;
+ int64_t OIValue;
+ if (!analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) ||
+ OIMask != ImmMask || OIValue != ImmValue)
+ return false;
+ if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) {
+ *IsSwapped = false;
+ return true;
+ }
+ if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) {
+ *IsSwapped = true;
+ return true;
+ }
return false;
- if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) {
- *IsSwapped = false;
- return true;
}
- if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) {
- *IsSwapped = true;
- return true;
- }
- return false;
- }
case X86::CMP64ri32:
case X86::CMP32ri:
case X86::CMP16ri:
@@ -5516,10 +5523,10 @@ bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI,
case X86::TEST32ri:
case X86::TEST16ri:
case X86::TEST8ri:
- CASE_ND(SUB64ri32)
- CASE_ND(SUB32ri)
- CASE_ND(SUB16ri)
- CASE_ND(SUB8ri)
+ CASE_ND(SUB64ri32)
+ CASE_ND(SUB32ri)
+ CASE_ND(SUB16ri)
+ CASE_ND(SUB8ri)
case X86::TEST64rr:
case X86::TEST32rr:
case X86::TEST16rr:
@@ -5601,98 +5608,98 @@ inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag,
default:
return false;
- // The shift instructions only modify ZF if their shift count is non-zero.
- // N.B.: The processor truncates the shift count depending on the encoding.
- CASE_ND(SAR8ri)
- CASE_ND(SAR16ri)
- CASE_ND(SAR32ri)
- CASE_ND(SAR64ri)
- CASE_ND(SHR8ri)
- CASE_ND(SHR16ri)
- CASE_ND(SHR32ri)
- CASE_ND(SHR64ri)
+ // The shift instructions only modify ZF if their shift count is non-zero.
+ // N.B.: The processor truncates the shift count depending on the encoding.
+ CASE_ND(SAR8ri)
+ CASE_ND(SAR16ri)
+ CASE_ND(SAR32ri)
+ CASE_ND(SAR64ri)
+ CASE_ND(SHR8ri)
+ CASE_ND(SHR16ri)
+ CASE_ND(SHR32ri)
+ CASE_ND(SHR64ri)
return getTruncatedShiftCount(MI, 2) != 0;
- // Some left shift instructions can be turned into LEA instructions but only
- // if their flags aren't used. Avoid transforming such instructions.
- CASE_ND(SHL8ri)
- CASE_ND(SHL16ri)
- CASE_ND(SHL32ri)
- CASE_ND(SHL64ri) {
- unsigned ShAmt = getTruncatedShiftCount(MI, 2);
- if (isTruncatedShiftCountForLEA(ShAmt))
- return false;
- return ShAmt != 0;
- }
+ // Some left shift instructions can be turned into LEA instructions but only
+ // if their flags aren't used. Avoid transforming such instructions.
+ CASE_ND(SHL8ri)
+ CASE_ND(SHL16ri)
+ CASE_ND(SHL32ri)
+ CASE_ND(SHL64ri) {
+ unsigned ShAmt = getTruncatedShiftCount(MI, 2);
+ if (isTruncatedShiftCountForLEA(ShAmt))
+ return false;
+ return ShAmt != 0;
+ }
- CASE_ND(SHRD16rri8)
- CASE_ND(SHRD32rri8)
- CASE_ND(SHRD64rri8)
- CASE_ND(SHLD16rri8)
- CASE_ND(SHLD32rri8)
- CASE_ND(SHLD64rri8)
+ CASE_ND(SHRD16rri8)
+ CASE_ND(SHRD32rri8)
+ CASE_ND(SHRD64rri8)
+ CASE_ND(SHLD16rri8)
+ CASE_ND(SHLD32rri8)
+ CASE_ND(SHLD64rri8)
return getTruncatedShiftCount(MI, 3) != 0;
- CASE_ND(SUB64ri32)
- CASE_ND(SUB32ri)
- CASE_ND(SUB16ri)
- CASE_ND(SUB8ri)
- CASE_ND(SUB64rr)
- CASE_ND(SUB32rr)
- CASE_ND(SUB16rr)
- CASE_ND(SUB8rr)
- CASE_ND(SUB64rm)
- CASE_ND(SUB32rm)
- CASE_ND(SUB16rm)
- CASE_ND(SUB8rm)
- CASE_ND(DEC64r)
- CASE_ND(DEC32r)
- CASE_ND(DEC16r)
- CASE_ND(DEC8r)
- CASE_ND(ADD64ri32)
- CASE_ND(ADD32ri)
- CASE_ND(ADD16ri)
- CASE_ND(ADD8ri)
- CASE_ND(ADD64rr)
- CASE_ND(ADD32rr)
- CASE_ND(ADD16rr)
- CASE_ND(ADD8rr)
- CASE_ND(ADD64rm)
- CASE_ND(ADD32rm)
- CASE_ND(ADD16rm)
- CASE_ND(ADD8rm)
- CASE_ND(INC64r)
- CASE_ND(INC32r)
- CASE_ND(INC16r)
- CASE_ND(INC8r)
- CASE_ND(ADC64ri32)
- CASE_ND(ADC32ri)
- CASE_ND(ADC16ri)
- CASE_ND(ADC8ri)
- CASE_ND(ADC64rr)
- CASE_ND(ADC32rr)
- CASE_ND(ADC16rr)
- CASE_ND(ADC8rr)
- CASE_ND(ADC64rm)
- CASE_ND(ADC32rm)
- CASE_ND(ADC16rm)
- CASE_ND(ADC8rm)
- CASE_ND(SBB64ri32)
- CASE_ND(SBB32ri)
- CASE_ND(SBB16ri)
- CASE_ND(SBB8ri)
- CASE_ND(SBB64rr)
- CASE_ND(SBB32rr)
- CASE_ND(SBB16rr)
- CASE_ND(SBB8rr)
- CASE_ND(SBB64rm)
- CASE_ND(SBB32rm)
- CASE_ND(SBB16rm)
- CASE_ND(SBB8rm)
- CASE_ND(NEG8r)
- CASE_ND(NEG16r)
- CASE_ND(NEG32r)
- CASE_ND(NEG64r)
+ CASE_ND(SUB64ri32)
+ CASE_ND(SUB32ri)
+ CASE_ND(SUB16ri)
+ CASE_ND(SUB8ri)
+ CASE_ND(SUB64rr)
+ CASE_ND(SUB32rr)
+ CASE_ND(SUB16rr)
+ CASE_ND(SUB8rr)
+ CASE_ND(SUB64rm)
+ CASE_ND(SUB32rm)
+ CASE_ND(SUB16rm)
+ CASE_ND(SUB8rm)
+ CASE_ND(DEC64r)
+ CASE_ND(DEC32r)
+ CASE_ND(DEC16r)
+ CASE_ND(DEC8r)
+ CASE_ND(ADD64ri32)
+ CASE_ND(ADD32ri)
+ CASE_ND(ADD16ri)
+ CASE_ND(ADD8ri)
+ CASE_ND(ADD64rr)
+ CASE_ND(ADD32rr)
+ CASE_ND(ADD16rr)
+ CASE_ND(ADD8rr)
+ CASE_ND(ADD64rm)
+ CASE_ND(ADD32rm)
+ CASE_ND(ADD16rm)
+ CASE_ND(ADD8rm)
+ CASE_ND(INC64r)
+ CASE_ND(INC32r)
+ CASE_ND(INC16r)
+ CASE_ND(INC8r)
+ CASE_ND(ADC64ri32)
+ CASE_ND(ADC32ri)
+ CASE_ND(ADC16ri)
+ CASE_ND(ADC8ri)
+ CASE_ND(ADC64rr)
+ CASE_ND(ADC32rr)
+ CASE_ND(ADC16rr)
+ CASE_ND(ADC8rr)
+ CASE_ND(ADC64rm)
+ CASE_ND(ADC32rm)
+ CASE_ND(ADC16rm)
+ CASE_ND(ADC8rm)
+ CASE_ND(SBB64ri32)
+ CASE_ND(SBB32ri)
+ CASE_ND(SBB16ri)
+ CASE_ND(SBB8ri)
+ CASE_ND(SBB64rr)
+ CASE_ND(SBB32rr)
+ CASE_ND(SBB16rr)
+ CASE_ND(SBB8rr)
+ CASE_ND(SBB64rm)
+ CASE_ND(SBB32rm)
+ CASE_ND(SBB16rm)
+ CASE_ND(SBB8rm)
+ CASE_ND(NEG8r)
+ CASE_ND(NEG16r)
+ CASE_ND(NEG32r)
+ CASE_ND(NEG64r)
case X86::LZCNT16rr:
case X86::LZCNT16rm:
case X86::LZCNT32rr:
@@ -5825,15 +5832,17 @@ inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag,
}
/// Check whether the use can be converted to remove a comparison against zero.
-/// Returns the EFLAGS condition and the operand that we are comparing against zero.
-static std::pair<X86::CondCode, unsigned> isUseDefConvertible(const MachineInstr &MI) {
+/// Returns the EFLAGS condition and the operand that we are comparing against
+/// zero.
+static std::pair<X86::CondCode, unsigned>
+isUseDefConvertible(const MachineInstr &MI) {
switch (MI.getOpcode()) {
default:
return std::make_pair(X86::COND_INVALID, ~0U);
- CASE_ND(NEG8r)
- CASE_ND(NEG16r)
- CASE_ND(NEG32r)
- CASE_ND(NEG64r)
+ CASE_ND(NEG8r)
+ CASE_ND(NEG16r)
+ CASE_ND(NEG32r)
+ CASE_ND(NEG64r)
return std::make_pair(X86::COND_AE, 1U);
case X86::LZCNT16rr:
case X86::LZCNT32rr:
@@ -5969,51 +5978,53 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
switch (CmpInstr.getOpcode()) {
default:
break;
- CASE_ND(SUB64ri32)
- CASE_ND(SUB32ri)
- CASE_ND(SUB16ri)
- CASE_ND(SUB8ri)
- CASE_ND(SUB64rm)
- CASE_ND(SUB32rm)
- CASE_ND(SUB16rm)
- CASE_ND(SUB8rm)
- CASE_ND(SUB64rr)
- CASE_ND(SUB32rr)
- CASE_ND(SUB16rr)
- CASE_ND(SUB8rr) {
- if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
- return false;
- // There is no use of the destination register, we can replace SUB with CMP.
- unsigned NewOpcode = 0;
+ CASE_ND(SUB64ri32)
+ CASE_ND(SUB32ri)
+ CASE_ND(SUB16ri)
+ CASE_ND(SUB8ri)
+ CASE_ND(SUB64rm)
+ CASE_ND(SUB32rm)
+ CASE_ND(SUB16rm)
+ CASE_ND(SUB8rm)
+ CASE_ND(SUB64rr)
+ CASE_ND(SUB32rr)
+ CASE_ND(SUB16rr)
+ CASE_ND(SUB8rr) {
+ if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
+ return false;
+ // There is no use of the destination register, we can replace SUB with
+ // CMP.
+ unsigned NewOpcode = 0;
#define FROM_TO(A, B) \
CASE_ND(A) NewOpcode = X86::B; \
break;
- switch (CmpInstr.getOpcode()) {
- default:
- llvm_unreachable("Unreachable!");
- FROM_TO(SUB64rm, CMP64rm)
- FROM_TO(SUB32rm, CMP32rm)
- FROM_TO(SUB16rm, CMP16rm)
- FROM_TO(SUB8rm, CMP8rm)
- FROM_TO(SUB64rr, CMP64rr)
- FROM_TO(SUB32rr, CMP32rr)
- FROM_TO(SUB16rr, CMP16rr)
- FROM_TO(SUB8rr, CMP8rr)
- FROM_TO(SUB64ri32, CMP64ri32)
- FROM_TO(SUB32ri, CMP32ri)
- FROM_TO(SUB16ri, CMP16ri)
- FROM_TO(SUB8ri, CMP8ri)
- }
+ switch (CmpInstr.getOpcode()) {
+ default:
+ llvm_unreachable("Unreachable!");
+ FROM_TO(SUB64rm, CMP64rm)
+ FROM_TO(SUB32rm, CMP32rm)
+ FROM_TO(SUB16rm, CMP16rm)
+ FROM_TO(SUB8rm, CMP8rm)
+ FROM_TO(SUB64rr, CMP64rr)
+ FROM_TO(SUB32rr, CMP32rr)
+ FROM_TO(SUB16rr, CMP16rr)
+ FROM_TO(SUB8rr, CMP8rr)
+ FROM_TO(SUB64ri32, CMP64ri32)
+ FROM_TO(SUB32ri, CMP32ri)
+ FROM_TO(SUB16ri, CMP16ri)
+ FROM_TO(SUB8ri, CMP8ri)
+ }
#undef FROM_TO
- CmpInstr.setDesc(get(NewOpcode));
- CmpInstr.removeOperand(0);
- // Mutating this instruction invalidates any debug data associated with it.
- CmpInstr.dropDebugNumber();
- // Fall through to optimize Cmp if Cmp is CMPrr or CMPri.
- if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
- NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
- return false;
- }
+ CmpInstr.setDesc(get(NewOpcode));
+ CmpInstr.removeOperand(0);
+ // Mutating this instruction invalidates any debug data associated with
+ // it.
+ CmpInstr.dropDebugNumber();
+ // Fall through to optimize Cmp if Cmp is CMPrr or CMPri.
+ if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
+ NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
+ return false;
+ }
}
// The following code tries to remove the comparison by re-using EFLAGS
@@ -6413,14 +6424,14 @@ static bool canConvert2Copy(unsigned Opc) {
switch (Opc) {
default:
return false;
- CASE_ND(ADD64ri32)
- CASE_ND(SUB64ri32)
- CASE_ND(OR64ri32)
- CASE_ND(XOR64ri32)
- CASE_ND(ADD32ri)
- CASE_ND(SUB32ri)
- CASE_ND(OR32ri)
- CASE_ND(XOR32ri)
+ CASE_ND(ADD64ri32)
+ CASE_ND(SUB64ri32)
+ CASE_ND(OR64ri32)
+ CASE_ND(XOR64ri32)
+ CASE_ND(ADD32ri)
+ CASE_ND(SUB32ri)
+ CASE_ND(OR32ri)
+ CASE_ND(XOR32ri)
return true;
}
}
@@ -9926,7 +9937,7 @@ Register X86InstrInfo::getGlobalBaseReg(MachineFunction *MF) const {
static const uint16_t *lookup(unsigned opcode, unsigned domain,
ArrayRef<uint16_t[3]> Table) {
- for (const uint16_t(&Row)[3] : Table)
+ for (const uint16_t (&Row)[3] : Table)
if (Row[domain - 1] == opcode)
return Row;
return nullptr;
@@ -9935,7 +9946,7 @@ static const uint16_t *lookup(unsigned opcode, unsigned domain,
static const uint16_t *lookupAVX512(unsigned opcode, unsigned domain,
ArrayRef<uint16_t[4]> Table) {
// If this is the integer domain make sure to check both integer columns.
- for (const uint16_t(&Row)[4] : Table)
+ for (const uint16_t (&Row)[4] : Table)
if (Row[domain - 1] == opcode || (domain == 3 && Row[3] == opcode))
return Row;
return nullptr;
@@ -10691,25 +10702,25 @@ bool X86InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
if (Invert)
return false;
switch (Inst.getOpcode()) {
- CASE_ND(ADD8rr)
- CASE_ND(ADD16rr)
- CASE_ND(ADD32rr)
- CASE_ND(ADD64rr)
- CASE_ND(AND8rr)
- CASE_ND(AND16rr)
- CASE_ND(AND32rr)
- CASE_ND(AND64rr)
- CASE_ND(OR8rr)
- CASE_ND(OR16rr)
- CASE_ND(OR32rr)
- CASE_ND(OR64rr)
- CASE_ND(XOR8rr)
- CASE_ND(XOR16rr)
- CASE_ND(XOR32rr)
- CASE_ND(XOR64rr)
- CASE_ND(IMUL16rr)
- CASE_ND(IMUL32rr)
- CASE_ND(IMUL64rr)
+ CASE_ND(ADD8rr)
+ CASE_ND(ADD16rr)
+ CASE_ND(ADD32rr)
+ CASE_ND(ADD64rr)
+ CASE_ND(AND8rr)
+ CASE_ND(AND16rr)
+ CASE_ND(AND32rr)
+ CASE_ND(AND64rr)
+ CASE_ND(OR8rr)
+ CASE_ND(OR16rr)
+ CASE_ND(OR32rr)
+ CASE_ND(OR64rr)
+ CASE_ND(XOR8rr)
+ CASE_ND(XOR16rr)
+ CASE_ND(XOR32rr)
+ CASE_ND(XOR64rr)
+ CASE_ND(IMUL16rr)
+ CASE_ND(IMUL32rr)
+ CASE_ND(IMUL64rr)
case X86::PANDrr:
case X86::PORrr:
case X86::PXORrr:
@@ -11532,8 +11543,8 @@ bool X86InstrInfo::getMachineCombinerPatterns(
break;
}
}
- return TargetInstrInfo::getMachineCombinerPatterns(Root,
- Patterns, DoRegPressureReduce);
+ return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
+ DoRegPressureReduce);
}
static void
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index a610dd3458a31..727143b9f97fe 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -331,8 +331,7 @@ class X86InstrInfo final : public X86GenInstrInfo {
Register isLoadFromStackSlot(const MachineInstr &MI,
int &FrameIndex) const override;
- Register isLoadFromStackSlot(const MachineInstr &MI,
- int &FrameIndex,
+ Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex,
TypeSize &MemBytes) const override;
/// isLoadFromStackSlotPostFE - Check for post-frame ptr elimination
/// stack locations as well. This uses a heuristic so it isn't
@@ -342,8 +341,7 @@ class X86InstrInfo final : public X86GenInstrInfo {
Register isStoreToStackSlot(const MachineInstr &MI,
int &FrameIndex) const override;
- Register isStoreToStackSlot(const MachineInstr &MI,
- int &FrameIndex,
+ Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex,
TypeSize &MemBytes) const override;
/// isStoreToStackSlotPostFE - Check for post-frame ptr elimination
/// stack locations as well. This uses a heuristic so it isn't
@@ -760,8 +758,7 @@ class X86InstrInfo final : public X86GenInstrInfo {
///
/// If IsIntrinsic is set, operand 1 will be ignored for commuting.
bool findThreeSrcCommutedOpIndices(const MachineInstr &MI,
- unsigned &SrcOpIdx1,
- unsigned &SrcOpIdx2,
+ unsigned &SrcOpIdx1, unsigned &SrcOpIdx2,
bool IsIntrinsic = false) const;
/// Returns true when instruction \p FlagI produces the same flags as \p OI.
diff --git a/llvm/test/CodeGen/X86/ctselect-i386-fp.ll b/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
index b88ec72a37925..4b5f31bad8313 100644
--- a/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
+++ b/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
@@ -209,94 +209,84 @@ define double @test_ctselect_f64_basic(i1 %cond, double %a, double %b) nounwind
define x86_fp80 @test_ctselect_f80_basic(i1 %cond, x86_fp80 %a, x86_fp80 %b) nounwind {
; I386-NOCMOV-LABEL: test_ctselect_f80_basic:
; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebp
+; I386-NOCMOV-NEXT: pushl %ebx
; I386-NOCMOV-NEXT: pushl %edi
; I386-NOCMOV-NEXT: pushl %esi
-; I386-NOCMOV-NEXT: subl $12, %esp
-; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: sete %al
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: subl $40, %esp
+; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: testb $1, %cl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, (%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: sete %ch
+; I386-NOCMOV-NEXT: movb %ch, %al
+; I386-NOCMOV-NEXT: movzbl %al, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %edi, %ebx
+; I386-NOCMOV-NEXT: andl %ebp, %ebx
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %edx, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %ebx
+; I386-NOCMOV-NEXT: testb $1, %cl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: sete %ch
+; I386-NOCMOV-NEXT: movb %ch, %cl
+; I386-NOCMOV-NEXT: movzbl %cl, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %edx, %edi
+; I386-NOCMOV-NEXT: andl %ebp, %edi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %eax, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl (%esp), %ebx # 4-byte Reload
+; I386-NOCMOV-NEXT: movb %al, %dl
+; I386-NOCMOV-NEXT: movzbl %dl, %edi
; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, %ecx
+; I386-NOCMOV-NEXT: andl %edi, %ecx
; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: fldt (%esp)
-; I386-NOCMOV-NEXT: addl $12, %esp
+; I386-NOCMOV-NEXT: andl %ebx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %ecx
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: addl $40, %esp
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: popl %ebp
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_ctselect_f80_basic:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: pushl %edi
-; I386-CMOV-NEXT: pushl %esi
-; I386-CMOV-NEXT: subl $12, %esp
+; I386-CMOV-NEXT: subl $36, %esp
+; I386-CMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, (%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, (%esp)
; I386-CMOV-NEXT: fldt (%esp)
-; I386-CMOV-NEXT: addl $12, %esp
-; I386-CMOV-NEXT: popl %esi
-; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: addl $36, %esp
; I386-CMOV-NEXT: retl
%result = call x86_fp80 @llvm.ct.select.f80(i1 %cond, x86_fp80 %a, x86_fp80 %b)
ret x86_fp80 %result
@@ -543,94 +533,84 @@ define float @test_ctselect_f32_nan(i1 %cond) nounwind {
define x86_fp80 @test_ctselect_f80_alignment(i1 %cond, x86_fp80 %a, x86_fp80 %b) nounwind {
; I386-NOCMOV-LABEL: test_ctselect_f80_alignment:
; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebp
+; I386-NOCMOV-NEXT: pushl %ebx
; I386-NOCMOV-NEXT: pushl %edi
; I386-NOCMOV-NEXT: pushl %esi
-; I386-NOCMOV-NEXT: subl $12, %esp
-; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: sete %al
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: subl $40, %esp
+; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: testb $1, %cl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, (%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: sete %ch
+; I386-NOCMOV-NEXT: movb %ch, %al
+; I386-NOCMOV-NEXT: movzbl %al, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %edi, %ebx
+; I386-NOCMOV-NEXT: andl %ebp, %ebx
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %edx, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %ebx
+; I386-NOCMOV-NEXT: testb $1, %cl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: sete %ch
+; I386-NOCMOV-NEXT: movb %ch, %cl
+; I386-NOCMOV-NEXT: movzbl %cl, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %edx, %edi
+; I386-NOCMOV-NEXT: andl %ebp, %edi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %eax, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl (%esp), %ebx # 4-byte Reload
+; I386-NOCMOV-NEXT: movb %al, %dl
+; I386-NOCMOV-NEXT: movzbl %dl, %edi
; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, %ecx
+; I386-NOCMOV-NEXT: andl %edi, %ecx
; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: fldt (%esp)
-; I386-NOCMOV-NEXT: addl $12, %esp
+; I386-NOCMOV-NEXT: andl %ebx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %ecx
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: addl $40, %esp
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: popl %ebp
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_ctselect_f80_alignment:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: pushl %edi
-; I386-CMOV-NEXT: pushl %esi
-; I386-CMOV-NEXT: subl $12, %esp
+; I386-CMOV-NEXT: subl $36, %esp
+; I386-CMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, (%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, (%esp)
; I386-CMOV-NEXT: fldt (%esp)
-; I386-CMOV-NEXT: addl $12, %esp
-; I386-CMOV-NEXT: popl %esi
-; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: addl $36, %esp
; I386-CMOV-NEXT: retl
%result = call x86_fp80 @llvm.ct.select.f80(i1 %cond, x86_fp80 %a, x86_fp80 %b)
ret x86_fp80 %result
diff --git a/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll b/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
index 6851c5babeb2d..2e1f16c19ca8e 100644
--- a/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
+++ b/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
@@ -10,84 +10,67 @@
define <1 x i64> @test_mmx_ctselect_with_paddd(i32 %cond, i64 %a, i64 %b) {
; I386-NOCMOV-LABEL: test_mmx_ctselect_with_paddd:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %ebp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: pushl %ebx
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: pushl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; I386-NOCMOV-NEXT: pushl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: subl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
-; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
-; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
-; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: subl $16, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %ecx, %ecx
+; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: setne %bl
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: sete %bh
-; I386-NOCMOV-NEXT: movb %bh, %al
-; I386-NOCMOV-NEXT: movzbl %al, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %esi, %edi
-; I386-NOCMOV-NEXT: andl %ebp, %edi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %ecx, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %edi
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: sete %al
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %ecx
-; I386-NOCMOV-NEXT: andl %esi, %ecx
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: orl %esi, %ecx
-; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: xorl %edx, %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: xorl %edx, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: andl %ecx, %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: paddd %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: popl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-NOCMOV-NEXT: popl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: addl $16, %esp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_mmx_ctselect_with_paddd:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: subl $16, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-CMOV-NEXT: .cfi_offset %esi, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: setne %dl
-; I386-CMOV-NEXT: testb %dl, %dl
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: xorl %ecx, %ecx
+; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: sbbl %ecx, %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: xorl %edx, %esi
+; I386-CMOV-NEXT: andl %ecx, %esi
+; I386-CMOV-NEXT: xorl %edx, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: andl %ecx, %edx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: paddd %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: addl $16, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: popl %esi
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
; I386-CMOV-NEXT: retl
%mmx_a = bitcast i64 %a to <1 x i64>
@@ -102,84 +85,67 @@ define <1 x i64> @test_mmx_ctselect_with_paddd(i32 %cond, i64 %a, i64 %b) {
define <1 x i64> @test_mmx_ctselect_with_psllw(i32 %cond, i64 %a, i64 %b) {
; I386-NOCMOV-LABEL: test_mmx_ctselect_with_psllw:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %ebp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: pushl %ebx
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: pushl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; I386-NOCMOV-NEXT: pushl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: subl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
-; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
-; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
-; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: subl $16, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %ecx, %ecx
+; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: setne %bl
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: sete %bh
-; I386-NOCMOV-NEXT: movb %bh, %al
-; I386-NOCMOV-NEXT: movzbl %al, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %esi, %edi
-; I386-NOCMOV-NEXT: andl %ebp, %edi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %ecx, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %edi
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: sete %al
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %ecx
-; I386-NOCMOV-NEXT: andl %esi, %ecx
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: orl %esi, %ecx
-; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: xorl %edx, %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: xorl %edx, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: andl %ecx, %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: psllw %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: popl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-NOCMOV-NEXT: popl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: addl $16, %esp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_mmx_ctselect_with_psllw:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: subl $16, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-CMOV-NEXT: .cfi_offset %esi, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: setne %dl
-; I386-CMOV-NEXT: testb %dl, %dl
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: xorl %ecx, %ecx
+; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: sbbl %ecx, %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: xorl %edx, %esi
+; I386-CMOV-NEXT: andl %ecx, %esi
+; I386-CMOV-NEXT: xorl %edx, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: andl %ecx, %edx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: psllw %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: addl $16, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: popl %esi
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
; I386-CMOV-NEXT: retl
%mmx_a = bitcast i64 %a to <1 x i64>
@@ -194,86 +160,55 @@ define <1 x i64> @test_mmx_ctselect_with_psllw(i32 %cond, i64 %a, i64 %b) {
define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64 %a, i64 %b, i64 %c) {
; I386-NOCMOV-LABEL: test_mmx_nested_ctselect_with_pand:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %ebp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; I386-NOCMOV-NEXT: pushl %ebx
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; I386-NOCMOV-NEXT: pushl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
; I386-NOCMOV-NEXT: pushl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: subl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
-; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
-; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
-; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: subl $16, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 32
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -8
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: setne %bl
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: sete %bh
-; I386-NOCMOV-NEXT: movb %bh, %cl
-; I386-NOCMOV-NEXT: movzbl %cl, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %edx, %edi
-; I386-NOCMOV-NEXT: andl %ebp, %edi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %eax, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %edi
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: sete %dl
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-NOCMOV-NEXT: movb %dl, %dh
-; I386-NOCMOV-NEXT: movzbl %dh, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %esi, %ebx
-; I386-NOCMOV-NEXT: andl %ebp, %ebx
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %eax, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %ebx
-; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: setne %dl
-; I386-NOCMOV-NEXT: testb %dl, %dl
-; I386-NOCMOV-NEXT: sete %dh
-; I386-NOCMOV-NEXT: movb %dh, %al
-; I386-NOCMOV-NEXT: movzbl %al, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %ecx, %esi
-; I386-NOCMOV-NEXT: andl %ebp, %esi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %ebx, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %esi
-; I386-NOCMOV-NEXT: testb %dl, %dl
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: xorl %ecx, %ecx
+; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl $0, %edi
+; I386-NOCMOV-NEXT: sbbl %edi, %edi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: andl %edi, %edx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; I386-NOCMOV-NEXT: movb %al, %dl
-; I386-NOCMOV-NEXT: movzbl %dl, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %ebx, %ecx
-; I386-NOCMOV-NEXT: andl %esi, %ecx
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: orl %esi, %ecx
-; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: xorl %esi, %ebx
+; I386-NOCMOV-NEXT: andl %edi, %ebx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
+; I386-NOCMOV-NEXT: xorl %edi, %esi
+; I386-NOCMOV-NEXT: xorl %ebx, %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: xorl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: xorl %esi, %eax
+; I386-NOCMOV-NEXT: xorl %edx, %eax
+; I386-NOCMOV-NEXT: andl %ecx, %eax
+; I386-NOCMOV-NEXT: xorl %esi, %eax
+; I386-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: pand %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: addl $16, %esp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: popl %edi
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: popl %ebx
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
@@ -281,36 +216,51 @@ define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64
; I386-CMOV: # %bb.0:
; I386-CMOV-NEXT: pushl %ebx
; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: pushl %edi
; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-CMOV-NEXT: subl $16, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 32
-; I386-CMOV-NEXT: .cfi_offset %esi, -12
+; I386-CMOV-NEXT: .cfi_offset %esi, -16
+; I386-CMOV-NEXT: .cfi_offset %edi, -12
; I386-CMOV-NEXT: .cfi_offset %ebx, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: xorl %ecx, %ecx
+; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl $0, %edi
+; I386-CMOV-NEXT: sbbl %edi, %edi
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: andl %edi, %edx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; I386-CMOV-NEXT: xorl %esi, %ebx
+; I386-CMOV-NEXT: andl %edi, %ebx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: sbbl %ecx, %ecx
+; I386-CMOV-NEXT: xorl %edi, %esi
+; I386-CMOV-NEXT: xorl %ebx, %esi
+; I386-CMOV-NEXT: andl %ecx, %esi
+; I386-CMOV-NEXT: xorl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: setne %bl
-; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: setne %bh
-; I386-CMOV-NEXT: testb %bh, %bh
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: testb %bl, %bl
-; I386-CMOV-NEXT: cmovnel %esi, %edx
-; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: cmovnel %ecx, %eax
+; I386-CMOV-NEXT: xorl %esi, %eax
+; I386-CMOV-NEXT: xorl %edx, %eax
+; I386-CMOV-NEXT: andl %ecx, %eax
+; I386-CMOV-NEXT: xorl %esi, %eax
; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: pand %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $20, %esp
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-CMOV-NEXT: addl $16, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 16
; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-CMOV-NEXT: popl %edi
; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
; I386-CMOV-NEXT: popl %ebx
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
@@ -330,84 +280,67 @@ define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64
define <1 x i64> @test_mmx_ctselect_with_por(i32 %cond, i64 %a, i64 %b) {
; I386-NOCMOV-LABEL: test_mmx_ctselect_with_por:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %ebp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: pushl %ebx
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: pushl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; I386-NOCMOV-NEXT: pushl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: subl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
-; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
-; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
-; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: subl $16, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %ecx, %ecx
+; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: setne %bl
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: sete %bh
-; I386-NOCMOV-NEXT: movb %bh, %al
-; I386-NOCMOV-NEXT: movzbl %al, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %esi, %edi
-; I386-NOCMOV-NEXT: andl %ebp, %edi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %ecx, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %edi
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: sete %al
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %ecx
-; I386-NOCMOV-NEXT: andl %esi, %ecx
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: orl %esi, %ecx
-; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: xorl %edx, %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: xorl %edx, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: andl %ecx, %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: por %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: popl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-NOCMOV-NEXT: popl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: addl $16, %esp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_mmx_ctselect_with_por:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: subl $16, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-CMOV-NEXT: .cfi_offset %esi, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: setne %dl
-; I386-CMOV-NEXT: testb %dl, %dl
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: xorl %ecx, %ecx
+; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: sbbl %ecx, %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: xorl %edx, %esi
+; I386-CMOV-NEXT: andl %ecx, %esi
+; I386-CMOV-NEXT: xorl %edx, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: andl %ecx, %edx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: por %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: addl $16, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: popl %esi
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
; I386-CMOV-NEXT: retl
%mmx_a = bitcast i64 %a to <1 x i64>
diff --git a/llvm/test/CodeGen/X86/ctselect-vector.ll b/llvm/test/CodeGen/X86/ctselect-vector.ll
index 2206e32cd6d34..79b76d1afca5c 100644
--- a/llvm/test/CodeGen/X86/ctselect-vector.ll
+++ b/llvm/test/CodeGen/X86/ctselect-vector.ll
@@ -266,31 +266,17 @@ define <2 x double> @test_ctselect_v2f64(i1 %cond, <2 x double> %a, <2 x double>
define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
; SSE2-LABEL: test_ctselect_v8i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: movd %eax, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm0, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm4
-; SSE2-NEXT: por %xmm5, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pand %xmm1, %xmm0
-; SSE2-NEXT: pandn %xmm3, %xmm2
-; SSE2-NEXT: por %xmm0, %xmm2
-; SSE2-NEXT: movaps %xmm4, %xmm0
-; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: movd %edi, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm4
+; SSE2-NEXT: psrad $31, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm5
+; SSE2-NEXT: pand %xmm4, %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
+; SSE2-NEXT: pand %xmm4, %xmm1
+; SSE2-NEXT: pandn %xmm3, %xmm4
+; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8i32:
@@ -342,31 +328,17 @@ define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b) {
; SSE2-LABEL: test_ctselect_v8f32:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: movd %eax, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
-; SSE2-NEXT: movaps %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm0, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm4
-; SSE2-NEXT: por %xmm5, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movaps %xmm0, %xmm2
-; SSE2-NEXT: pand %xmm1, %xmm0
-; SSE2-NEXT: pandn %xmm3, %xmm2
-; SSE2-NEXT: por %xmm0, %xmm2
-; SSE2-NEXT: movaps %xmm4, %xmm0
-; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: movd %edi, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm4
+; SSE2-NEXT: psrad $31, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm5
+; SSE2-NEXT: pand %xmm4, %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
+; SSE2-NEXT: pand %xmm4, %xmm1
+; SSE2-NEXT: pandn %xmm3, %xmm4
+; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8f32:
@@ -418,31 +390,17 @@ define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b
define <4 x i64> @test_ctselect_v4i64(i1 %cond, <4 x i64> %a, <4 x i64> %b) {
; SSE2-LABEL: test_ctselect_v4i64:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: movd %eax, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm0, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm4
-; SSE2-NEXT: por %xmm5, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pand %xmm1, %xmm0
-; SSE2-NEXT: pandn %xmm3, %xmm2
-; SSE2-NEXT: por %xmm0, %xmm2
-; SSE2-NEXT: movaps %xmm4, %xmm0
-; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: movd %edi, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm4
+; SSE2-NEXT: psrad $31, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm5
+; SSE2-NEXT: pand %xmm4, %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
+; SSE2-NEXT: pand %xmm4, %xmm1
+; SSE2-NEXT: pandn %xmm3, %xmm4
+; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v4i64:
@@ -494,31 +452,17 @@ define <4 x i64> @test_ctselect_v4i64(i1 %cond, <4 x i64> %a, <4 x i64> %b) {
define <4 x double> @test_ctselect_v4f64(i1 %cond, <4 x double> %a, <4 x double> %b) {
; SSE2-LABEL: test_ctselect_v4f64:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: movd %eax, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
-; SSE2-NEXT: movapd %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm0, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm4
-; SSE2-NEXT: por %xmm5, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movapd %xmm0, %xmm2
-; SSE2-NEXT: pand %xmm1, %xmm0
-; SSE2-NEXT: pandn %xmm3, %xmm2
-; SSE2-NEXT: por %xmm0, %xmm2
-; SSE2-NEXT: movaps %xmm4, %xmm0
-; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: movd %edi, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm4
+; SSE2-NEXT: psrad $31, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm5
+; SSE2-NEXT: pand %xmm4, %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
+; SSE2-NEXT: pand %xmm4, %xmm1
+; SSE2-NEXT: pandn %xmm3, %xmm4
+; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v4f64:
@@ -571,113 +515,63 @@ define <4 x double> @test_ctselect_v4f64(i1 %cond, <4 x double> %a, <4 x double>
define <16 x i32> @test_ctselect_v16i32(i1 %cond, <16 x i32> %a, <16 x i32> %b) {
; SSE2-LABEL: test_ctselect_v16i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm9, %xmm9
-; SSE2-NEXT: movd %eax, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm9, %xmm8
-; SSE2-NEXT: pand %xmm0, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm8
-; SSE2-NEXT: por %xmm9, %xmm8
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm4
-; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: movd %edi, %xmm8
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm8
+; SSE2-NEXT: psrad $31, %xmm8
+; SSE2-NEXT: movdqa %xmm8, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm9
+; SSE2-NEXT: pand %xmm8, %xmm0
+; SSE2-NEXT: por %xmm9, %xmm0
+; SSE2-NEXT: movdqa %xmm8, %xmm4
; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: por %xmm0, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm5
-; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: pandn %xmm6, %xmm5
-; SSE2-NEXT: por %xmm0, %xmm5
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm6
-; SSE2-NEXT: pand %xmm3, %xmm0
-; SSE2-NEXT: pandn %xmm7, %xmm6
-; SSE2-NEXT: por %xmm0, %xmm6
-; SSE2-NEXT: movaps %xmm8, %xmm0
-; SSE2-NEXT: movaps %xmm4, %xmm1
-; SSE2-NEXT: movaps %xmm5, %xmm2
-; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: pand %xmm8, %xmm1
+; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: movdqa %xmm8, %xmm4
+; SSE2-NEXT: pandn %xmm6, %xmm4
+; SSE2-NEXT: pand %xmm8, %xmm2
+; SSE2-NEXT: por %xmm4, %xmm2
+; SSE2-NEXT: pand %xmm8, %xmm3
+; SSE2-NEXT: pandn %xmm7, %xmm8
+; SSE2-NEXT: por %xmm8, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v16i32:
; AVX: # %bb.0:
-; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm5, %ymm5
-; AVX-NEXT: vmovd %eax, %ymm5
-; AVX-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
-; AVX-NEXT: vmovdqa %ymm5, %ymm4
-; AVX-NEXT: pand %ymm0, %ymm5
-; AVX-NEXT: pandn %ymm2, %ymm4
-; AVX-NEXT: por %ymm5, %ymm4
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm0, %ymm0
-; AVX-NEXT: vmovd %eax, %ymm0
-; AVX-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
-; AVX-NEXT: vmovdqa %ymm0, %ymm2
-; AVX-NEXT: pand %ymm1, %ymm0
-; AVX-NEXT: pandn %ymm3, %ymm2
-; AVX-NEXT: por %ymm0, %ymm2
-; AVX-NEXT: vmovaps %ymm4, %ymm0
-; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: vmovd %edi, %xmm4
+; AVX-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX-NEXT: vpshufb %xmm5, %xmm4, %xmm4
+; AVX-NEXT: vpsllw $7, %xmm4, %xmm4
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
+; AVX-NEXT: vpcmpgtb %xmm4, %xmm5, %xmm4
+; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
+; AVX-NEXT: vpmovsxbd %xmm5, %xmm5
+; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
+; AVX-NEXT: vpmovsxbd %xmm6, %xmm6
+; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5
+; AVX-NEXT: vpmovsxbd %xmm4, %xmm6
+; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
+; AVX-NEXT: vpmovsxbd %xmm4, %xmm4
+; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4
+; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
+; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
+; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
+; AVX-NEXT: vandnps %ymm3, %ymm5, %ymm2
+; AVX-NEXT: vandps %ymm5, %ymm1, %ymm1
+; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v16i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm5, %ymm5
-; AVX2-NEXT: vmovd %eax, %ymm5
-; AVX2-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
-; AVX2-NEXT: vmovdqa %ymm5, %ymm4
-; AVX2-NEXT: pand %ymm0, %ymm5
-; AVX2-NEXT: pandn %ymm2, %ymm4
-; AVX2-NEXT: por %ymm5, %ymm4
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm0, %ymm0
-; AVX2-NEXT: vmovd %eax, %ymm0
-; AVX2-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
-; AVX2-NEXT: vmovdqa %ymm0, %ymm2
-; AVX2-NEXT: pand %ymm1, %ymm0
-; AVX2-NEXT: pandn %ymm3, %ymm2
-; AVX2-NEXT: por %ymm0, %ymm2
-; AVX2-NEXT: vmovaps %ymm4, %ymm0
-; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: vmovd %edi, %xmm4
+; AVX2-NEXT: vpbroadcastb %xmm4, %xmm4
+; AVX2-NEXT: vpsllw $7, %xmm4, %xmm4
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm5
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; AVX2-NEXT: vpmovsxbd %xmm5, %ymm5
+; AVX2-NEXT: vpmovsxbd %xmm4, %ymm4
+; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpblendvb %ymm5, %ymm1, %ymm3, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v16i32:
; AVX512: # %bb.0:
@@ -695,113 +589,60 @@ define <16 x i32> @test_ctselect_v16i32(i1 %cond, <16 x i32> %a, <16 x i32> %b)
define <16 x float> @test_ctselect_v16f32(i1 %cond, <16 x float> %a, <16 x float> %b) {
; SSE2-LABEL: test_ctselect_v16f32:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm9, %xmm9
-; SSE2-NEXT: movd %eax, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
-; SSE2-NEXT: movaps %xmm9, %xmm8
-; SSE2-NEXT: pand %xmm0, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm8
-; SSE2-NEXT: por %xmm9, %xmm8
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movaps %xmm0, %xmm4
-; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: movd %edi, %xmm8
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm8
+; SSE2-NEXT: psrad $31, %xmm8
+; SSE2-NEXT: movdqa %xmm8, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm9
+; SSE2-NEXT: pand %xmm8, %xmm0
+; SSE2-NEXT: por %xmm9, %xmm0
+; SSE2-NEXT: movdqa %xmm8, %xmm4
; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: por %xmm0, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movaps %xmm0, %xmm5
-; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: pandn %xmm6, %xmm5
-; SSE2-NEXT: por %xmm0, %xmm5
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movaps %xmm0, %xmm6
-; SSE2-NEXT: pand %xmm3, %xmm0
-; SSE2-NEXT: pandn %xmm7, %xmm6
-; SSE2-NEXT: por %xmm0, %xmm6
-; SSE2-NEXT: movaps %xmm8, %xmm0
-; SSE2-NEXT: movaps %xmm4, %xmm1
-; SSE2-NEXT: movaps %xmm5, %xmm2
-; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: pand %xmm8, %xmm1
+; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: movdqa %xmm8, %xmm4
+; SSE2-NEXT: pandn %xmm6, %xmm4
+; SSE2-NEXT: pand %xmm8, %xmm2
+; SSE2-NEXT: por %xmm4, %xmm2
+; SSE2-NEXT: pand %xmm8, %xmm3
+; SSE2-NEXT: pandn %xmm7, %xmm8
+; SSE2-NEXT: por %xmm8, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v16f32:
; AVX: # %bb.0:
-; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm5, %ymm5
-; AVX-NEXT: vmovd %eax, %ymm5
-; AVX-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
-; AVX-NEXT: vmovdqa %ymm5, %ymm4
-; AVX-NEXT: pand %ymm0, %ymm5
-; AVX-NEXT: pandn %ymm2, %ymm4
-; AVX-NEXT: por %ymm5, %ymm4
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm0, %ymm0
-; AVX-NEXT: vmovd %eax, %ymm0
-; AVX-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
-; AVX-NEXT: vmovdqa %ymm0, %ymm2
-; AVX-NEXT: pand %ymm1, %ymm0
-; AVX-NEXT: pandn %ymm3, %ymm2
-; AVX-NEXT: por %ymm0, %ymm2
-; AVX-NEXT: vmovaps %ymm4, %ymm0
-; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: vmovd %edi, %xmm4
+; AVX-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX-NEXT: vpshufb %xmm5, %xmm4, %xmm4
+; AVX-NEXT: vpsllw $7, %xmm4, %xmm4
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
+; AVX-NEXT: vpcmpgtb %xmm4, %xmm5, %xmm4
+; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
+; AVX-NEXT: vpmovsxbd %xmm5, %xmm5
+; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
+; AVX-NEXT: vpmovsxbd %xmm6, %xmm6
+; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5
+; AVX-NEXT: vpmovsxbd %xmm4, %xmm6
+; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
+; AVX-NEXT: vpmovsxbd %xmm4, %xmm4
+; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4
+; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
+; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
+; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
+; AVX-NEXT: vandnps %ymm3, %ymm5, %ymm2
+; AVX-NEXT: vandps %ymm5, %ymm1, %ymm1
+; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v16f32:
; AVX2: # %bb.0:
-; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm5, %ymm5
-; AVX2-NEXT: vmovd %eax, %ymm5
-; AVX2-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
-; AVX2-NEXT: vmovdqa %ymm5, %ymm4
-; AVX2-NEXT: pand %ymm0, %ymm5
-; AVX2-NEXT: pandn %ymm2, %ymm4
-; AVX2-NEXT: por %ymm5, %ymm4
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm0, %ymm0
-; AVX2-NEXT: vmovd %eax, %ymm0
-; AVX2-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
-; AVX2-NEXT: vmovdqa %ymm0, %ymm2
-; AVX2-NEXT: pand %ymm1, %ymm0
-; AVX2-NEXT: pandn %ymm3, %ymm2
-; AVX2-NEXT: por %ymm0, %ymm2
-; AVX2-NEXT: vmovaps %ymm4, %ymm0
-; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: vmovd %edi, %xmm4
+; AVX2-NEXT: vpbroadcastb %xmm4, %xmm4
+; AVX2-NEXT: vpsllw $7, %xmm4, %xmm4
+; AVX2-NEXT: vpmovsxbd %xmm4, %ymm4
+; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm3, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v16f32:
; AVX512: # %bb.0:
@@ -819,113 +660,52 @@ define <16 x float> @test_ctselect_v16f32(i1 %cond, <16 x float> %a, <16 x float
define <8 x i64> @test_ctselect_v8i64(i1 %cond, <8 x i64> %a, <8 x i64> %b) {
; SSE2-LABEL: test_ctselect_v8i64:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm9, %xmm9
-; SSE2-NEXT: movd %eax, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm9, %xmm8
-; SSE2-NEXT: pand %xmm0, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm8
-; SSE2-NEXT: por %xmm9, %xmm8
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm4
-; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: movd %edi, %xmm8
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm8
+; SSE2-NEXT: psrad $31, %xmm8
+; SSE2-NEXT: movdqa %xmm8, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm9
+; SSE2-NEXT: pand %xmm8, %xmm0
+; SSE2-NEXT: por %xmm9, %xmm0
+; SSE2-NEXT: movdqa %xmm8, %xmm4
; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: por %xmm0, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm5
-; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: pandn %xmm6, %xmm5
-; SSE2-NEXT: por %xmm0, %xmm5
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm6
-; SSE2-NEXT: pand %xmm3, %xmm0
-; SSE2-NEXT: pandn %xmm7, %xmm6
-; SSE2-NEXT: por %xmm0, %xmm6
-; SSE2-NEXT: movaps %xmm8, %xmm0
-; SSE2-NEXT: movaps %xmm4, %xmm1
-; SSE2-NEXT: movaps %xmm5, %xmm2
-; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: pand %xmm8, %xmm1
+; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: movdqa %xmm8, %xmm4
+; SSE2-NEXT: pandn %xmm6, %xmm4
+; SSE2-NEXT: pand %xmm8, %xmm2
+; SSE2-NEXT: por %xmm4, %xmm2
+; SSE2-NEXT: pand %xmm8, %xmm3
+; SSE2-NEXT: pandn %xmm7, %xmm8
+; SSE2-NEXT: por %xmm8, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8i64:
; AVX: # %bb.0:
-; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm5, %ymm5
-; AVX-NEXT: vmovd %eax, %ymm5
-; AVX-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
-; AVX-NEXT: vmovdqa %ymm5, %ymm4
-; AVX-NEXT: pand %ymm0, %ymm5
-; AVX-NEXT: pandn %ymm2, %ymm4
-; AVX-NEXT: por %ymm5, %ymm4
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm0, %ymm0
-; AVX-NEXT: vmovd %eax, %ymm0
-; AVX-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; AVX-NEXT: vmovdqa %ymm0, %ymm2
-; AVX-NEXT: pand %ymm1, %ymm0
-; AVX-NEXT: pandn %ymm3, %ymm2
-; AVX-NEXT: por %ymm0, %ymm2
-; AVX-NEXT: vmovaps %ymm4, %ymm0
-; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: vmovd %edi, %xmm4
+; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
+; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
+; AVX-NEXT: vpsllw $15, %xmm4, %xmm4
+; AVX-NEXT: vpsraw $15, %xmm4, %xmm4
+; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm4, %ymm4
+; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
+; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
+; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
+; AVX-NEXT: vandnps %ymm3, %ymm4, %ymm2
+; AVX-NEXT: vandps %ymm4, %ymm1, %ymm1
+; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v8i64:
; AVX2: # %bb.0:
-; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm5, %ymm5
-; AVX2-NEXT: vmovd %eax, %ymm5
-; AVX2-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
-; AVX2-NEXT: vmovdqa %ymm5, %ymm4
-; AVX2-NEXT: pand %ymm0, %ymm5
-; AVX2-NEXT: pandn %ymm2, %ymm4
-; AVX2-NEXT: por %ymm5, %ymm4
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm0, %ymm0
-; AVX2-NEXT: vmovd %eax, %ymm0
-; AVX2-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; AVX2-NEXT: vmovdqa %ymm0, %ymm2
-; AVX2-NEXT: pand %ymm1, %ymm0
-; AVX2-NEXT: pandn %ymm3, %ymm2
-; AVX2-NEXT: por %ymm0, %ymm2
-; AVX2-NEXT: vmovaps %ymm4, %ymm0
-; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: vmovd %edi, %xmm4
+; AVX2-NEXT: vpbroadcastw %xmm4, %xmm4
+; AVX2-NEXT: vpsllw $15, %xmm4, %xmm4
+; AVX2-NEXT: vpsraw $15, %xmm4, %xmm4
+; AVX2-NEXT: vpmovsxwq %xmm4, %ymm4
+; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm3, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v8i64:
; AVX512: # %bb.0:
@@ -943,113 +723,52 @@ define <8 x i64> @test_ctselect_v8i64(i1 %cond, <8 x i64> %a, <8 x i64> %b) {
define <8 x double> @test_ctselect_v8f64(i1 %cond, <8 x double> %a, <8 x double> %b) {
; SSE2-LABEL: test_ctselect_v8f64:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm9, %xmm9
-; SSE2-NEXT: movd %eax, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
-; SSE2-NEXT: movapd %xmm9, %xmm8
-; SSE2-NEXT: pand %xmm0, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm8
-; SSE2-NEXT: por %xmm9, %xmm8
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movapd %xmm0, %xmm4
-; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: movd %edi, %xmm8
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm8
+; SSE2-NEXT: psrad $31, %xmm8
+; SSE2-NEXT: movdqa %xmm8, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm9
+; SSE2-NEXT: pand %xmm8, %xmm0
+; SSE2-NEXT: por %xmm9, %xmm0
+; SSE2-NEXT: movdqa %xmm8, %xmm4
; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: por %xmm0, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movapd %xmm0, %xmm5
-; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: pandn %xmm6, %xmm5
-; SSE2-NEXT: por %xmm0, %xmm5
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movapd %xmm0, %xmm6
-; SSE2-NEXT: pand %xmm3, %xmm0
-; SSE2-NEXT: pandn %xmm7, %xmm6
-; SSE2-NEXT: por %xmm0, %xmm6
-; SSE2-NEXT: movaps %xmm8, %xmm0
-; SSE2-NEXT: movaps %xmm4, %xmm1
-; SSE2-NEXT: movaps %xmm5, %xmm2
-; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: pand %xmm8, %xmm1
+; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: movdqa %xmm8, %xmm4
+; SSE2-NEXT: pandn %xmm6, %xmm4
+; SSE2-NEXT: pand %xmm8, %xmm2
+; SSE2-NEXT: por %xmm4, %xmm2
+; SSE2-NEXT: pand %xmm8, %xmm3
+; SSE2-NEXT: pandn %xmm7, %xmm8
+; SSE2-NEXT: por %xmm8, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8f64:
; AVX: # %bb.0:
-; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm5, %ymm5
-; AVX-NEXT: vmovd %eax, %ymm5
-; AVX-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
-; AVX-NEXT: vmovdqa %ymm5, %ymm4
-; AVX-NEXT: pand %ymm0, %ymm5
-; AVX-NEXT: pandn %ymm2, %ymm4
-; AVX-NEXT: por %ymm5, %ymm4
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm0, %ymm0
-; AVX-NEXT: vmovd %eax, %ymm0
-; AVX-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; AVX-NEXT: vmovdqa %ymm0, %ymm2
-; AVX-NEXT: pand %ymm1, %ymm0
-; AVX-NEXT: pandn %ymm3, %ymm2
-; AVX-NEXT: por %ymm0, %ymm2
-; AVX-NEXT: vmovaps %ymm4, %ymm0
-; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: vmovd %edi, %xmm4
+; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
+; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
+; AVX-NEXT: vpsllw $15, %xmm4, %xmm4
+; AVX-NEXT: vpsraw $15, %xmm4, %xmm4
+; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm4, %ymm4
+; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
+; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
+; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
+; AVX-NEXT: vandnps %ymm3, %ymm4, %ymm2
+; AVX-NEXT: vandps %ymm4, %ymm1, %ymm1
+; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v8f64:
; AVX2: # %bb.0:
-; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm5, %ymm5
-; AVX2-NEXT: vmovd %eax, %ymm5
-; AVX2-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
-; AVX2-NEXT: vmovdqa %ymm5, %ymm4
-; AVX2-NEXT: pand %ymm0, %ymm5
-; AVX2-NEXT: pandn %ymm2, %ymm4
-; AVX2-NEXT: por %ymm5, %ymm4
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm0, %ymm0
-; AVX2-NEXT: vmovd %eax, %ymm0
-; AVX2-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; AVX2-NEXT: vmovdqa %ymm0, %ymm2
-; AVX2-NEXT: pand %ymm1, %ymm0
-; AVX2-NEXT: pandn %ymm3, %ymm2
-; AVX2-NEXT: por %ymm0, %ymm2
-; AVX2-NEXT: vmovaps %ymm4, %ymm0
-; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: vmovd %edi, %xmm4
+; AVX2-NEXT: vpbroadcastw %xmm4, %xmm4
+; AVX2-NEXT: vpsllw $15, %xmm4, %xmm4
+; AVX2-NEXT: vpsraw $15, %xmm4, %xmm4
+; AVX2-NEXT: vpmovsxwq %xmm4, %ymm4
+; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm3, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v8f64:
; AVX512: # %bb.0:
>From 7298add14ff08fe0487ac307f0a3da3a17a3d60b Mon Sep 17 00:00:00 2001
From: wizardengineer <juliuswoosebert at gmail.com>
Date: Fri, 22 May 2026 12:00:06 -0400
Subject: [PATCH 3/4] [X86] Restore combineSelect BLENDV sign-bit fold
The optimization that folds X86ISD::BLENDV away when the condition's
sign bit is statically known was inadvertently dropped during a prior
rebase conflict resolution. Its absence caused unrelated CHECK lines
in combine-sdiv.ll, vector-compress.ll, vector-idiv-{u,s}div-*.ll to
fail because main expects the post-optimization codegen.
Restores the block at llvm/lib/Target/X86/X86ISelLowering.cpp line
~49587 (after combineLogicBlendIntoConditionalNegate, before the
VSELECT/BLENDV shuffle handling), matching origin/main verbatim.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 18 +++++++++++++++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 058fbd718f0f9..62024f08dddb8 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -26315,11 +26315,14 @@ SDValue X86TargetLowering::LowerCT_SELECT(SDValue Op, SelectionDAG &DAG) const {
// Perform CT_SELECT on each 32-bit chunk
SDValue Part0Ops[] = {FalsePart0, TruePart0, CC, ProcessedCond};
- SDValue Part0Select = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part0Ops);
+ SDValue Part0Select =
+ DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part0Ops);
SDValue Part1Ops[] = {FalsePart1, TruePart1, CC, ProcessedCond};
- SDValue Part1Select = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part1Ops);
+ SDValue Part1Select =
+ DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part1Ops);
SDValue Part2Ops[] = {FalsePart2, TruePart2, CC, ProcessedCond};
- SDValue Part2Select = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part2Ops);
+ SDValue Part2Select =
+ DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part2Ops);
// Create result stack slot and store the selected parts
SDValue ResultSlot = DAG.CreateStackTemporary(MVT::f80);
@@ -49575,6 +49578,15 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
DL, DAG, Subtarget))
return V;
+ // If the sign bit is known then BLENDV can be folded away.
+ if (N->getOpcode() == X86ISD::BLENDV) {
+ KnownBits KnownCond = DAG.computeKnownBits(Cond);
+ if (KnownCond.isNegative())
+ return LHS;
+ if (KnownCond.isNonNegative())
+ return RHS;
+ }
+
if (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::BLENDV) {
SmallVector<int, 64> CondMask;
if (createShuffleMaskFromVSELECT(CondMask, Cond,
>From f08515eff72db94180b5f86e6f092e44d91ac23a Mon Sep 17 00:00:00 2001
From: wizardengineer <juliuswoosebert at gmail.com>
Date: Fri, 22 May 2026 12:11:42 -0400
Subject: [PATCH 4/4] [X86] Stop claiming f128 CT_SELECT Custom; regen ctselect
tests
X86's LowerCT_SELECT falls through to X86ISD::CT_SELECT for any FP
type it claims Custom but doesn't special-case (only f80 has the
three-i32-chunk path, and scalar f32/f64 the SSE register path).
For f128 the fall-through produces an X86ISD::CT_SELECT with f128
result type, for which no isel pattern exists, and selection
ICEs with "Cannot select".
Drop f128 from the CT_SELECT Custom loop so the generic legalizer's
memory-blend path (recently added in the core PR) handles it.
Regenerate CHECK lines for all X86 ctselect tests to reflect the
expanded coverage (half/bfloat/fp80/fp128) introduced upstream.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 177 +-
llvm/lib/Target/X86/X86TargetMachine.cpp | 2 +-
llvm/test/CodeGen/X86/ctselect-edge-cases.ll | 27 +-
llvm/test/CodeGen/X86/ctselect-i386-fp.ll | 234 +-
llvm/test/CodeGen/X86/ctselect-i386-mmx.ll | 371 ++-
llvm/test/CodeGen/X86/ctselect-i386.ll | 71 +-
llvm/test/CodeGen/X86/ctselect-vector.ll | 659 +---
llvm/test/CodeGen/X86/ctselect.ll | 2821 +++++++-----------
8 files changed, 1503 insertions(+), 2859 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 62024f08dddb8..1e4f0876a8a21 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -29,9 +29,9 @@
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/VectorUtils.h"
-#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/ISDOpcodes.h"
#include "llvm/CodeGen/IntrinsicLowering.h"
+#include "llvm/CodeGen/LivePhysRegs.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -505,11 +505,16 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// X86 wants to expand cmov itself.
for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
setOperationAction(ISD::SELECT, VT, Custom);
- setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
}
+ // CT_SELECT for f32/f64/f80: native lowering via LowerCT_SELECT.
+ // f128 has no isel pattern for X86ISD::CT_SELECT; fall through to the
+ // generic legalizer's memory-blend expansion.
+ for (auto VT : {MVT::f32, MVT::f64, MVT::f80}) {
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
+ }
for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
if (VT == MVT::i64 && !Subtarget.is64Bit())
continue;
@@ -1110,8 +1115,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
- setOperationAction(ISD::CT_SELECT, MVT::v4f32, Custom);
- setOperationAction(ISD::FCANONICALIZE, MVT::v4f32, Expand);
+ setOperationAction(ISD::CT_SELECT, MVT::v4f32, Custom);
+ setOperationAction(ISD::FCANONICALIZE, MVT::v4f32, Expand);
setOperationAction(ISD::LOAD, MVT::v2f32, Custom);
setOperationAction(ISD::STORE, MVT::v2f32, Custom);
@@ -26115,7 +26120,7 @@ static SDValue LowerSIGN_EXTEND_Mask(SDValue Op, const SDLoc &dl,
}
SDValue X86TargetLowering::LowerCT_SELECT(SDValue Op, SelectionDAG &DAG) const {
- SDValue Cond = Op.getOperand(0); // condition
+ SDValue Cond = Op.getOperand(0); // condition
SDValue TrueOp = Op.getOperand(1); // true_value
SDValue FalseOp = Op.getOperand(2); // false_value
SDLoc DL(Op);
@@ -26177,7 +26182,8 @@ SDValue X86TargetLowering::LowerCT_SELECT(SDValue Op, SelectionDAG &DAG) const {
// use the generic X86 CT_SELECT node which will be matched by the patterns
SDValue CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
SDValue EFLAGS = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget);
- // Create the X86 CT_SELECT node - note operand order: true, false, cc, flags
+ // Create the X86 CT_SELECT node - note operand order: true, false, cc,
+ // flags
return DAG.getNode(X86ISD::CT_SELECT, DL, VT, FalseOp, TrueOp, CC, EFLAGS);
}
@@ -39044,10 +39050,8 @@ X86TargetLowering::emitPatchableEventCall(MachineInstr &MI,
/// This approach ensures that when i64 is type-legalized into two i32
/// operations, both operations share the same condition byte rather than
/// each independently reading (and destroying) EFLAGS.
-static MachineBasicBlock *
-emitCTSelectI386WithConditionMaterialization(MachineInstr &MI,
- MachineBasicBlock *BB,
- unsigned InternalPseudoOpcode) {
+static MachineBasicBlock *emitCTSelectI386WithConditionMaterialization(
+ MachineInstr &MI, MachineBasicBlock *BB, unsigned InternalPseudoOpcode) {
const TargetInstrInfo *TII = BB->getParent()->getSubtarget().getInstrInfo();
const MIMetadata MIMD(MI);
MachineFunction *MF = BB->getParent();
@@ -39091,12 +39095,12 @@ emitCTSelectI386WithConditionMaterialization(MachineInstr &MI,
}
BuildMI(*BB, MI, MIMD, TII->get(InternalPseudoOpcode))
- .addDef(DstReg) // dst (output)
- .addDef(TmpByteReg) // tmp_byte (output)
- .addDef(TmpMaskReg) // tmp_mask (output)
- .addReg(Src1Reg) // src1 (input)
- .addReg(Src2Reg) // src2 (input)
- .addReg(CondByteReg); // pre-materialized condition byte (input)
+ .addDef(DstReg) // dst (output)
+ .addDef(TmpByteReg) // tmp_byte (output)
+ .addDef(TmpMaskReg) // tmp_mask (output)
+ .addReg(Src1Reg) // src1 (input)
+ .addReg(Src2Reg) // src2 (input)
+ .addReg(CondByteReg); // pre-materialized condition byte (input)
MI.eraseFromParent();
return BB;
@@ -39122,8 +39126,8 @@ struct FPLoadMemOperands {
// Check if a virtual register is defined by a simple FP load instruction
// Returns the memory operands if it's a simple load, otherwise returns invalid
static FPLoadMemOperands getFPLoadMemOperands(Register Reg,
- MachineRegisterInfo &MRI,
- unsigned ExpectedLoadOpcode) {
+ MachineRegisterInfo &MRI,
+ unsigned ExpectedLoadOpcode) {
FPLoadMemOperands Result;
if (!Reg.isVirtual())
@@ -39142,9 +39146,9 @@ static FPLoadMemOperands getFPLoadMemOperands(Register Reg,
if (DefMI->hasOrderedMemoryRef())
return Result;
- // The load should have a single def (the destination register) and memory operands
- // Format: %reg = LD_Fpxxm <fi#N>, 1, %noreg, 0, %noreg
- // or: %reg = LD_Fpxxm %base, scale, %index, disp, %segment
+ // The load should have a single def (the destination register) and memory
+ // operands Format: %reg = LD_Fpxxm <fi#N>, 1, %noreg, 0, %noreg or: %reg =
+ // LD_Fpxxm %base, scale, %index, disp, %segment
if (DefMI->getNumOperands() < 6)
return Result;
@@ -39169,9 +39173,8 @@ static FPLoadMemOperands getFPLoadMemOperands(Register Reg,
// Check if this is a constant pool load
// Format: %reg = LD_Fpxxm $noreg, 1, $noreg, %const.N, $noreg
- if (BaseMO.isReg() && BaseMO.getReg() == X86::NoRegister &&
- ScaleMO.isImm() && IndexMO.isReg() &&
- IndexMO.getReg() == X86::NoRegister &&
+ if (BaseMO.isReg() && BaseMO.getReg() == X86::NoRegister && ScaleMO.isImm() &&
+ IndexMO.isReg() && IndexMO.getReg() == X86::NoRegister &&
DispMO.isCPI() && SegMO.isReg()) {
Result.IsValid = true;
Result.IsConstantPool = true;
@@ -39185,9 +39188,8 @@ static FPLoadMemOperands getFPLoadMemOperands(Register Reg,
// Check if this is a global variable load
// Format: %reg = LD_Fpxxm $noreg, 1, $noreg, @global_name, $noreg
- if (BaseMO.isReg() && BaseMO.getReg() == X86::NoRegister &&
- ScaleMO.isImm() && IndexMO.isReg() &&
- IndexMO.getReg() == X86::NoRegister &&
+ if (BaseMO.isReg() && BaseMO.getReg() == X86::NoRegister && ScaleMO.isImm() &&
+ IndexMO.isReg() && IndexMO.getReg() == X86::NoRegister &&
DispMO.isGlobal() && SegMO.isReg()) {
Result.IsValid = true;
Result.IsGlobal = true;
@@ -39201,8 +39203,8 @@ static FPLoadMemOperands getFPLoadMemOperands(Register Reg,
}
// Regular memory operands (e.g., pointer loads)
- if (BaseMO.isReg() && ScaleMO.isImm() && IndexMO.isReg() &&
- DispMO.isImm() && SegMO.isReg()) {
+ if (BaseMO.isReg() && ScaleMO.isImm() && IndexMO.isReg() && DispMO.isImm() &&
+ SegMO.isReg()) {
Result.IsValid = true;
Result.IsFrameIndex = false;
Result.IsConstantPool = false;
@@ -39228,7 +39230,8 @@ static MachineBasicBlock *emitCTSelectI386WithFpType(MachineInstr &MI,
unsigned RegSizeInByte = 4;
// Get operands
- // MI operands: %result:rfp80 = CT_SELECT_I386 %false:rfp80, %true:rfp80, %cond:i8imm
+ // MI operands: %result:rfp80 = CT_SELECT_I386 %false:rfp80, %true:rfp80,
+ // %cond:i8imm
unsigned DestReg = MI.getOperand(0).getReg();
unsigned FalseReg = MI.getOperand(1).getReg();
unsigned TrueReg = MI.getOperand(2).getReg();
@@ -39246,7 +39249,7 @@ static MachineBasicBlock *emitCTSelectI386WithFpType(MachineInstr &MI,
// Helper to load integer from memory operands
auto loadIntFromMemOperands = [&](const FPLoadMemOperands &MemOps,
- unsigned Offset) -> unsigned {
+ unsigned Offset) -> unsigned {
unsigned IntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
MachineInstrBuilder MIB =
BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), IntReg);
@@ -39262,18 +39265,21 @@ static MachineBasicBlock *emitCTSelectI386WithFpType(MachineInstr &MI,
// Constant pool: base_reg + scale + index + CP_index + segment
// MOV32rm format: base, scale, index, displacement, segment
MIB.addReg(X86::NoRegister) // Base register
- .addImm(MemOps.ScaleVal) // Scale
- .addReg(MemOps.IndexReg) // Index register
- .addConstantPoolIndex(MemOps.ConstantPoolIndex, Offset) // Displacement (CP index)
- .addReg(MemOps.SegReg); // Segment
+ .addImm(MemOps.ScaleVal) // Scale
+ .addReg(MemOps.IndexReg) // Index register
+ .addConstantPoolIndex(MemOps.ConstantPoolIndex,
+ Offset) // Displacement (CP index)
+ .addReg(MemOps.SegReg); // Segment
} else if (MemOps.IsGlobal) {
// Global variable: base_reg + scale + index + global + segment
// MOV32rm format: base, scale, index, displacement, segment
MIB.addReg(X86::NoRegister) // Base register
- .addImm(MemOps.ScaleVal) // Scale
- .addReg(MemOps.IndexReg) // Index register
- .addGlobalAddress(MemOps.Global, MemOps.GlobalOffset + Offset) // Displacement (global address)
- .addReg(MemOps.SegReg); // Segment
+ .addImm(MemOps.ScaleVal) // Scale
+ .addReg(MemOps.IndexReg) // Index register
+ .addGlobalAddress(MemOps.Global,
+ MemOps.GlobalOffset +
+ Offset) // Displacement (global address)
+ .addReg(MemOps.SegReg); // Segment
} else {
// Regular memory: base_reg + scale + index + disp + segment
MIB.addReg(MemOps.BaseReg)
@@ -39288,45 +39294,47 @@ static MachineBasicBlock *emitCTSelectI386WithFpType(MachineInstr &MI,
// Optimized path: load integers directly from memory when both operands are
// memory loads, avoiding FP register round-trip
- auto emitCtSelectFromMemory = [&](unsigned NumValues,
- const FPLoadMemOperands &TrueMemOps,
- const FPLoadMemOperands &FalseMemOps,
- int ResultSlot) {
- for (unsigned Val = 0; Val < NumValues; ++Val) {
- unsigned Offset = Val * RegSizeInByte;
-
- // Load true and false values directly from their memory locations as integers
- unsigned TrueIntReg = loadIntFromMemOperands(TrueMemOps, Offset);
- unsigned FalseIntReg = loadIntFromMemOperands(FalseMemOps, Offset);
-
- // Use CT_SELECT_I386_INT_GR32 pseudo instruction for constant-time selection
- unsigned ResultIntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
- unsigned TmpByteReg = MRI.createVirtualRegister(&X86::GR8RegClass);
- unsigned TmpMaskReg = MRI.createVirtualRegister(&X86::GR32RegClass);
-
- BuildMI(*BB, MI, MIMD, TII->get(X86::CT_SELECT_I386_INT_GR32rr))
- .addDef(ResultIntReg) // dst (output)
- .addDef(TmpByteReg) // tmp_byte (output)
- .addDef(TmpMaskReg) // tmp_mask (output)
- .addReg(FalseIntReg) // src1 (input) - false value
- .addReg(TrueIntReg) // src2 (input) - true value
- .addReg(CondByteReg); // pre-materialized condition byte (input)
-
- // Store result back to result slot
- BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32mr))
- .addFrameIndex(ResultSlot)
- .addImm(1)
- .addReg(0)
- .addImm(Offset)
- .addReg(0)
- .addReg(ResultIntReg, RegState::Kill);
- }
- };
+ auto emitCtSelectFromMemory =
+ [&](unsigned NumValues, const FPLoadMemOperands &TrueMemOps,
+ const FPLoadMemOperands &FalseMemOps, int ResultSlot) {
+ for (unsigned Val = 0; Val < NumValues; ++Val) {
+ unsigned Offset = Val * RegSizeInByte;
+
+ // Load true and false values directly from their memory locations as
+ // integers
+ unsigned TrueIntReg = loadIntFromMemOperands(TrueMemOps, Offset);
+ unsigned FalseIntReg = loadIntFromMemOperands(FalseMemOps, Offset);
+
+ // Use CT_SELECT_I386_INT_GR32 pseudo instruction for constant-time
+ // selection
+ unsigned ResultIntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+ unsigned TmpByteReg = MRI.createVirtualRegister(&X86::GR8RegClass);
+ unsigned TmpMaskReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+
+ BuildMI(*BB, MI, MIMD, TII->get(X86::CT_SELECT_I386_INT_GR32rr))
+ .addDef(ResultIntReg) // dst (output)
+ .addDef(TmpByteReg) // tmp_byte (output)
+ .addDef(TmpMaskReg) // tmp_mask (output)
+ .addReg(FalseIntReg) // src1 (input) - false value
+ .addReg(TrueIntReg) // src2 (input) - true value
+ .addReg(CondByteReg); // pre-materialized condition byte (input)
+
+ // Store result back to result slot
+ BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32mr))
+ .addFrameIndex(ResultSlot)
+ .addImm(1)
+ .addReg(0)
+ .addImm(Offset)
+ .addReg(0)
+ .addReg(ResultIntReg, RegState::Kill);
+ }
+ };
- auto emitCtSelectWithPseudo = [&](unsigned NumValues, int TrueSlot, int FalseSlot, int ResultSlot) {
+ auto emitCtSelectWithPseudo = [&](unsigned NumValues, int TrueSlot,
+ int FalseSlot, int ResultSlot) {
for (unsigned Val = 0; Val < NumValues; ++Val) {
unsigned Offset = Val * RegSizeInByte;
-
+
// Load true and false values from stack as 32-bit integers
unsigned TrueIntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), TrueIntReg)
@@ -39344,18 +39352,19 @@ static MachineBasicBlock *emitCTSelectI386WithFpType(MachineInstr &MI,
.addImm(Offset)
.addReg(0);
- // Use CT_SELECT_I386_INT_GR32 pseudo instruction for constant-time selection
+ // Use CT_SELECT_I386_INT_GR32 pseudo instruction for constant-time
+ // selection
unsigned ResultIntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
unsigned TmpByteReg = MRI.createVirtualRegister(&X86::GR8RegClass);
unsigned TmpMaskReg = MRI.createVirtualRegister(&X86::GR32RegClass);
-
+
BuildMI(*BB, MI, MIMD, TII->get(X86::CT_SELECT_I386_INT_GR32rr))
- .addDef(ResultIntReg) // dst (output)
- .addDef(TmpByteReg) // tmp_byte (output)
- .addDef(TmpMaskReg) // tmp_mask (output)
- .addReg(FalseIntReg) // src1 (input) - false value
- .addReg(TrueIntReg) // src2 (input) - true value
- .addReg(CondByteReg); // pre-materialized condition byte (input)
+ .addDef(ResultIntReg) // dst (output)
+ .addDef(TmpByteReg) // tmp_byte (output)
+ .addDef(TmpMaskReg) // tmp_mask (output)
+ .addReg(FalseIntReg) // src1 (input) - false value
+ .addReg(TrueIntReg) // src2 (input) - true value
+ .addReg(CondByteReg); // pre-materialized condition byte (input)
// Store result back to result slot
BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32mr))
@@ -39583,7 +39592,7 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
return emitCTSelectI386WithFpType(MI, BB, X86::CT_SELECT_I386_FP64rr);
case X86::CT_SELECT_I386_FP80rr:
return emitCTSelectI386WithFpType(MI, BB, X86::CT_SELECT_I386_FP80rr);
-
+
case X86::FP80_ADDr:
case X86::FP80_ADDm32: {
// Change the floating point control register to use double extended
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 2dee1af7423f0..457847bdb7b03 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -627,7 +627,7 @@ void X86PassConfig::addPreEmitPass2() {
(TT.isOSDarwin() &&
(M->getFunction("objc_retainAutoreleasedReturnValue") ||
M->getFunction("objc_unsafeClaimAutoreleasedReturnValue"))) ||
- F.hasFnAttribute("ct-select");
+ F.hasFnAttribute("ct-select");
}));
// Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
diff --git a/llvm/test/CodeGen/X86/ctselect-edge-cases.ll b/llvm/test/CodeGen/X86/ctselect-edge-cases.ll
index 0797265972a1f..33375747f9637 100644
--- a/llvm/test/CodeGen/X86/ctselect-edge-cases.ll
+++ b/llvm/test/CodeGen/X86/ctselect-edge-cases.ll
@@ -117,14 +117,7 @@ define float @test_ctselect_f32_special_values(i1 %cond) {
; X32-NEXT: sete %al
; X32-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %ecx
; X32-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %edx
-; X32-NEXT: movb %al, %ah
-; X32-NEXT: movzbl %ah, %edi
-; X32-NEXT: negl %edi
-; X32-NEXT: movl %edx, %esi
-; X32-NEXT: andl %edi, %esi
-; X32-NEXT: notl %edi
-; X32-NEXT: andl %ecx, %edi
-; X32-NEXT: orl %edi, %esi
+; X32-NEXT: BUNDLE
; X32-NEXT: movl %esi, (%esp)
; X32-NEXT: flds (%esp)
; X32-NEXT: addl $4, %esp
@@ -167,25 +160,11 @@ define double @test_ctselect_f64_special_values(i1 %cond) {
; X32-NEXT: fstpl (%esp)
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl (%esp), %edx
-; X32-NEXT: movb %al, %ah
-; X32-NEXT: movzbl %ah, %edi
-; X32-NEXT: negl %edi
-; X32-NEXT: movl %edx, %esi
-; X32-NEXT: andl %edi, %esi
-; X32-NEXT: notl %edi
-; X32-NEXT: andl %ecx, %edi
-; X32-NEXT: orl %edi, %esi
+; X32-NEXT: BUNDLE
; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movb %al, %ah
-; X32-NEXT: movzbl %ah, %edi
-; X32-NEXT: negl %edi
-; X32-NEXT: movl %edx, %esi
-; X32-NEXT: andl %edi, %esi
-; X32-NEXT: notl %edi
-; X32-NEXT: andl %ecx, %edi
-; X32-NEXT: orl %edi, %esi
+; X32-NEXT: BUNDLE
; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X32-NEXT: fldl {{[0-9]+}}(%esp)
; X32-NEXT: addl $24, %esp
diff --git a/llvm/test/CodeGen/X86/ctselect-i386-fp.ll b/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
index 4b5f31bad8313..c322c5a091695 100644
--- a/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
+++ b/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
@@ -20,14 +20,7 @@ define float @test_ctselect_f32_basic(i1 %cond, float %a, float %b) nounwind {
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: flds (%esp)
; I386-NOCMOV-NEXT: addl $4, %esp
@@ -44,14 +37,7 @@ define float @test_ctselect_f32_basic(i1 %cond, float %a, float %b) nounwind {
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: flds (%esp)
; I386-CMOV-NEXT: addl $4, %esp
@@ -81,14 +67,7 @@ define float @test_ctselect_f32_eq(float %x, float %y, float %a, float %b) nounw
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: flds (%esp)
; I386-NOCMOV-NEXT: addl $4, %esp
@@ -111,14 +90,7 @@ define float @test_ctselect_f32_eq(float %x, float %y, float %a, float %b) nounw
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: flds (%esp)
; I386-CMOV-NEXT: addl $4, %esp
@@ -141,25 +113,11 @@ define double @test_ctselect_f64_basic(i1 %cond, double %a, double %b) nounwind
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: fldl (%esp)
; I386-NOCMOV-NEXT: addl $8, %esp
@@ -176,25 +134,11 @@ define double @test_ctselect_f64_basic(i1 %cond, double %a, double %b) nounwind
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: fldl (%esp)
; I386-CMOV-NEXT: addl $8, %esp
@@ -226,39 +170,18 @@ define x86_fp80 @test_ctselect_f80_basic(i1 %cond, x86_fp80 %a, x86_fp80 %b) nou
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
; I386-NOCMOV-NEXT: sete %ch
-; I386-NOCMOV-NEXT: movb %ch, %al
-; I386-NOCMOV-NEXT: movzbl %al, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %edi, %ebx
-; I386-NOCMOV-NEXT: andl %ebp, %ebx
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %edx, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %ebx
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: testb $1, %cl
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: sete %ch
-; I386-NOCMOV-NEXT: movb %ch, %cl
-; I386-NOCMOV-NEXT: movzbl %cl, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %edx, %edi
-; I386-NOCMOV-NEXT: andl %ebp, %edi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %eax, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl (%esp), %ebx # 4-byte Reload
-; I386-NOCMOV-NEXT: movb %al, %dl
-; I386-NOCMOV-NEXT: movzbl %dl, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %esi, %ecx
-; I386-NOCMOV-NEXT: andl %edi, %ecx
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ebx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %ecx
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: addl $40, %esp
@@ -310,14 +233,7 @@ define float @test_ctselect_f32_gt(float %x, float %y, float %a, float %b) nounw
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: flds (%esp)
; I386-NOCMOV-NEXT: addl $4, %esp
@@ -339,14 +255,7 @@ define float @test_ctselect_f32_gt(float %x, float %y, float %a, float %b) nounw
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: flds (%esp)
; I386-CMOV-NEXT: addl $4, %esp
@@ -369,14 +278,7 @@ define float @test_ctselect_f32_no_branches(i1 %cond, float %a, float %b) nounwi
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: flds (%esp)
; I386-NOCMOV-NEXT: addl $4, %esp
@@ -393,14 +295,7 @@ define float @test_ctselect_f32_no_branches(i1 %cond, float %a, float %b) nounwi
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: flds (%esp)
; I386-CMOV-NEXT: addl $4, %esp
@@ -422,14 +317,7 @@ define float @test_ctselect_f32_bundled(i1 %cond, float %a, float %b) nounwind {
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: flds (%esp)
; I386-NOCMOV-NEXT: addl $4, %esp
@@ -446,14 +334,7 @@ define float @test_ctselect_f32_bundled(i1 %cond, float %a, float %b) nounwind {
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: flds (%esp)
; I386-CMOV-NEXT: addl $4, %esp
@@ -480,14 +361,7 @@ define float @test_ctselect_f32_nan(i1 %cond) nounwind {
; I386-NOCMOV-NEXT: fstps (%esp)
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl (%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: addl $12, %esp
@@ -509,14 +383,7 @@ define float @test_ctselect_f32_nan(i1 %cond) nounwind {
; I386-CMOV-NEXT: fstps (%esp)
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl (%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: flds {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: addl $12, %esp
@@ -550,39 +417,18 @@ define x86_fp80 @test_ctselect_f80_alignment(i1 %cond, x86_fp80 %a, x86_fp80 %b)
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
; I386-NOCMOV-NEXT: sete %ch
-; I386-NOCMOV-NEXT: movb %ch, %al
-; I386-NOCMOV-NEXT: movzbl %al, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %edi, %ebx
-; I386-NOCMOV-NEXT: andl %ebp, %ebx
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %edx, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %ebx
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: testb $1, %cl
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: sete %ch
-; I386-NOCMOV-NEXT: movb %ch, %cl
-; I386-NOCMOV-NEXT: movzbl %cl, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %edx, %edi
-; I386-NOCMOV-NEXT: andl %ebp, %edi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %eax, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl (%esp), %ebx # 4-byte Reload
-; I386-NOCMOV-NEXT: movb %al, %dl
-; I386-NOCMOV-NEXT: movzbl %dl, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %esi, %ecx
-; I386-NOCMOV-NEXT: andl %edi, %ecx
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ebx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %ecx
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: addl $40, %esp
@@ -627,27 +473,13 @@ define float @test_ctselect_f32_multiple(i1 %cond1, i1 %cond2, float %a, float %
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: flds (%esp)
; I386-NOCMOV-NEXT: addl $8, %esp
@@ -664,27 +496,13 @@ define float @test_ctselect_f32_multiple(i1 %cond1, i1 %cond2, float %a, float %
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: flds (%esp)
; I386-CMOV-NEXT: addl $8, %esp
diff --git a/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll b/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
index 2e1f16c19ca8e..ea258070ff633 100644
--- a/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
+++ b/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
@@ -10,67 +10,70 @@
define <1 x i64> @test_mmx_ctselect_with_paddd(i32 %cond, i64 %a, i64 %b) {
; I386-NOCMOV-LABEL: test_mmx_ctselect_with_paddd:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: subl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 24
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -8
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %ecx, %ecx
-; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: xorl %edx, %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: xorl %edx, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: andl %ecx, %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: paddd %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_mmx_ctselect_with_paddd:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: pushl %esi
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: subl $16, %esp
+; I386-CMOV-NEXT: subl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
-; I386-CMOV-NEXT: .cfi_offset %esi, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %ecx, %ecx
-; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: sbbl %ecx, %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: xorl %edx, %esi
-; I386-CMOV-NEXT: andl %ecx, %esi
-; I386-CMOV-NEXT: xorl %edx, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: andl %ecx, %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %dl
+; I386-CMOV-NEXT: testb %dl, %dl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: paddd %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $16, %esp
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: addl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
; I386-CMOV-NEXT: retl
%mmx_a = bitcast i64 %a to <1 x i64>
@@ -85,67 +88,70 @@ define <1 x i64> @test_mmx_ctselect_with_paddd(i32 %cond, i64 %a, i64 %b) {
define <1 x i64> @test_mmx_ctselect_with_psllw(i32 %cond, i64 %a, i64 %b) {
; I386-NOCMOV-LABEL: test_mmx_ctselect_with_psllw:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: subl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 24
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -8
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %ecx, %ecx
-; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: xorl %edx, %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: xorl %edx, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: andl %ecx, %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: psllw %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_mmx_ctselect_with_psllw:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: pushl %esi
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: subl $16, %esp
+; I386-CMOV-NEXT: subl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
-; I386-CMOV-NEXT: .cfi_offset %esi, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %ecx, %ecx
-; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: sbbl %ecx, %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: xorl %edx, %esi
-; I386-CMOV-NEXT: andl %ecx, %esi
-; I386-CMOV-NEXT: xorl %edx, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: andl %ecx, %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %dl
+; I386-CMOV-NEXT: testb %dl, %dl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: psllw %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $16, %esp
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: addl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
; I386-CMOV-NEXT: retl
%mmx_a = bitcast i64 %a to <1 x i64>
@@ -160,55 +166,58 @@ define <1 x i64> @test_mmx_ctselect_with_psllw(i32 %cond, i64 %a, i64 %b) {
define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64 %a, i64 %b, i64 %c) {
; I386-NOCMOV-LABEL: test_mmx_nested_ctselect_with_pand:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %ebx
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %edi
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-NOCMOV-NEXT: subl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 32
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -16
-; I386-NOCMOV-NEXT: .cfi_offset %edi, -12
-; I386-NOCMOV-NEXT: .cfi_offset %ebx, -8
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: xorl %ecx, %ecx
-; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: movl $0, %edi
-; I386-NOCMOV-NEXT: sbbl %edi, %edi
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: andl %edi, %edx
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; I386-NOCMOV-NEXT: xorl %esi, %ebx
-; I386-NOCMOV-NEXT: andl %edi, %ebx
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
-; I386-NOCMOV-NEXT: xorl %edi, %esi
-; I386-NOCMOV-NEXT: xorl %ebx, %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: xorl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: xorl %esi, %eax
-; I386-NOCMOV-NEXT: xorl %edx, %eax
-; I386-NOCMOV-NEXT: andl %ecx, %eax
-; I386-NOCMOV-NEXT: xorl %esi, %eax
-; I386-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: sete %dl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %dl
+; I386-NOCMOV-NEXT: testb %dl, %dl
+; I386-NOCMOV-NEXT: sete %dh
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: testb %dl, %dl
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: pand %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
; I386-NOCMOV-NEXT: popl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; I386-NOCMOV-NEXT: popl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
@@ -216,51 +225,36 @@ define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64
; I386-CMOV: # %bb.0:
; I386-CMOV-NEXT: pushl %ebx
; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: pushl %edi
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
; I386-CMOV-NEXT: pushl %esi
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-CMOV-NEXT: subl $16, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-CMOV-NEXT: subl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 32
-; I386-CMOV-NEXT: .cfi_offset %esi, -16
-; I386-CMOV-NEXT: .cfi_offset %edi, -12
+; I386-CMOV-NEXT: .cfi_offset %esi, -12
; I386-CMOV-NEXT: .cfi_offset %ebx, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: xorl %ecx, %ecx
-; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl $0, %edi
-; I386-CMOV-NEXT: sbbl %edi, %edi
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: andl %edi, %edx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; I386-CMOV-NEXT: xorl %esi, %ebx
-; I386-CMOV-NEXT: andl %edi, %ebx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: sbbl %ecx, %ecx
-; I386-CMOV-NEXT: xorl %edi, %esi
-; I386-CMOV-NEXT: xorl %ebx, %esi
-; I386-CMOV-NEXT: andl %ecx, %esi
-; I386-CMOV-NEXT: xorl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: xorl %esi, %eax
-; I386-CMOV-NEXT: xorl %edx, %eax
-; I386-CMOV-NEXT: andl %ecx, %eax
-; I386-CMOV-NEXT: xorl %esi, %eax
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %bl
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %bh
+; I386-CMOV-NEXT: testb %bh, %bh
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: testb %bl, %bl
+; I386-CMOV-NEXT: cmovnel %esi, %edx
+; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel %ecx, %eax
; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: pand %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $16, %esp
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: addl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: popl %esi
; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
; I386-CMOV-NEXT: popl %ebx
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
@@ -280,67 +274,70 @@ define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64
define <1 x i64> @test_mmx_ctselect_with_por(i32 %cond, i64 %a, i64 %b) {
; I386-NOCMOV-LABEL: test_mmx_ctselect_with_por:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: subl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 24
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -8
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %ecx, %ecx
-; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: xorl %edx, %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: xorl %edx, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: andl %ecx, %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: por %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_mmx_ctselect_with_por:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: pushl %esi
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: subl $16, %esp
+; I386-CMOV-NEXT: subl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
-; I386-CMOV-NEXT: .cfi_offset %esi, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %ecx, %ecx
-; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: sbbl %ecx, %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: xorl %edx, %esi
-; I386-CMOV-NEXT: andl %ecx, %esi
-; I386-CMOV-NEXT: xorl %edx, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: andl %ecx, %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %dl
+; I386-CMOV-NEXT: testb %dl, %dl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: por %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $16, %esp
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: addl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
; I386-CMOV-NEXT: retl
%mmx_a = bitcast i64 %a to <1 x i64>
diff --git a/llvm/test/CodeGen/X86/ctselect-i386.ll b/llvm/test/CodeGen/X86/ctselect-i386.ll
index d1cc559f0c1c1..d5a28e8e578f4 100644
--- a/llvm/test/CodeGen/X86/ctselect-i386.ll
+++ b/llvm/test/CodeGen/X86/ctselect-i386.ll
@@ -19,14 +19,7 @@ define i32 @test_ctselect_i32_basic(i1 %cond, i32 %a, i32 %b) nounwind {
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %bl
-; I386-NOCMOV-NEXT: movb %bl, %bh
-; I386-NOCMOV-NEXT: movzbl %bh, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %eax
-; I386-NOCMOV-NEXT: andl %esi, %eax
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %ebx
; I386-NOCMOV-NEXT: retl
@@ -51,14 +44,7 @@ define i16 @test_ctselect_i16_basic(i1 %cond, i16 %a, i16 %b) nounwind {
; I386-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %bl
-; I386-NOCMOV-NEXT: movb %bl, %bh
-; I386-NOCMOV-NEXT: movzbw %bh, %si
-; I386-NOCMOV-NEXT: negw %si
-; I386-NOCMOV-NEXT: movw %dx, %ax
-; I386-NOCMOV-NEXT: andw %si, %ax
-; I386-NOCMOV-NEXT: notw %si
-; I386-NOCMOV-NEXT: andw %cx, %si
-; I386-NOCMOV-NEXT: orw %si, %ax
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %ebx
; I386-NOCMOV-NEXT: retl
@@ -81,13 +67,7 @@ define i8 @test_ctselect_i8_basic(i1 %cond, i8 %a, i8 %b) nounwind {
; I386-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %ah
-; I386-NOCMOV-NEXT: movb %ah, %ch
-; I386-NOCMOV-NEXT: negb %ch
-; I386-NOCMOV-NEXT: movb %dl, %al
-; I386-NOCMOV-NEXT: andb %ch, %al
-; I386-NOCMOV-NEXT: notb %ch
-; I386-NOCMOV-NEXT: andb %cl, %ch
-; I386-NOCMOV-NEXT: orb %ch, %al
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_ctselect_i8_basic:
@@ -113,14 +93,7 @@ define i32 @test_crypto_key_select(i32 %secret_bit, i32 %key1, i32 %key2) nounwi
; I386-NOCMOV-NEXT: setne %al
; I386-NOCMOV-NEXT: testb %al, %al
; I386-NOCMOV-NEXT: sete %bl
-; I386-NOCMOV-NEXT: movb %bl, %bh
-; I386-NOCMOV-NEXT: movzbl %bh, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %eax
-; I386-NOCMOV-NEXT: andl %esi, %eax
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %ebx
; I386-NOCMOV-NEXT: retl
@@ -150,14 +123,7 @@ define i32 @test_no_conditional_branches(i32 %secret, i32 %val1, i32 %val2) noun
; I386-NOCMOV-NEXT: setne %al
; I386-NOCMOV-NEXT: testb %al, %al
; I386-NOCMOV-NEXT: sete %bl
-; I386-NOCMOV-NEXT: movb %bl, %bh
-; I386-NOCMOV-NEXT: movzbl %bh, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %eax
-; I386-NOCMOV-NEXT: andl %esi, %eax
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %ebx
; I386-NOCMOV-NEXT: retl
@@ -187,14 +153,7 @@ define i32 @test_ctselect_i32_cmp(i32 %a, i32 %b, i32 %c) nounwind {
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: testb %al, %al
; I386-NOCMOV-NEXT: sete %bl
-; I386-NOCMOV-NEXT: movb %bl, %bh
-; I386-NOCMOV-NEXT: movzbl %bh, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %eax
-; I386-NOCMOV-NEXT: andl %esi, %eax
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %ebx
; I386-NOCMOV-NEXT: retl
@@ -224,24 +183,10 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) n
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %bl
-; I386-NOCMOV-NEXT: movb %bl, %bh
-; I386-NOCMOV-NEXT: movzbl %bh, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %eax, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %dl
-; I386-NOCMOV-NEXT: movb %dl, %dh
-; I386-NOCMOV-NEXT: movzbl %dh, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %ecx, %eax
-; I386-NOCMOV-NEXT: andl %edi, %eax
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %esi, %edi
-; I386-NOCMOV-NEXT: orl %edi, %eax
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %edi
; I386-NOCMOV-NEXT: popl %ebx
diff --git a/llvm/test/CodeGen/X86/ctselect-vector.ll b/llvm/test/CodeGen/X86/ctselect-vector.ll
index 79b76d1afca5c..dd59d16a04280 100644
--- a/llvm/test/CodeGen/X86/ctselect-vector.ll
+++ b/llvm/test/CodeGen/X86/ctselect-vector.ll
@@ -10,51 +10,21 @@ define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) {
; SSE2-LABEL: test_ctselect_v4i32:
; SSE2: # %bb.0:
; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v4i32:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v4i32:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4i32:
@@ -74,51 +44,21 @@ define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b
; SSE2-LABEL: test_ctselect_v4f32:
; SSE2: # %bb.0:
; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movaps %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v4f32:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v4f32:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4f32:
@@ -138,51 +78,21 @@ define <2 x i64> @test_ctselect_v2i64(i1 %cond, <2 x i64> %a, <2 x i64> %b) {
; SSE2-LABEL: test_ctselect_v2i64:
; SSE2: # %bb.0:
; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v2i64:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v2i64:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v2i64:
@@ -202,51 +112,21 @@ define <2 x double> @test_ctselect_v2f64(i1 %cond, <2 x double> %a, <2 x double>
; SSE2-LABEL: test_ctselect_v2f64:
; SSE2: # %bb.0:
; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movapd %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v2f64:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v2f64:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v2f64:
@@ -266,50 +146,24 @@ define <2 x double> @test_ctselect_v2f64(i1 %cond, <2 x double> %a, <2 x double>
define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
; SSE2-LABEL: test_ctselect_v8i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm4
-; SSE2-NEXT: psrad $31, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm5
-; SSE2-NEXT: pand %xmm4, %xmm0
-; SSE2-NEXT: por %xmm5, %xmm0
-; SSE2-NEXT: pand %xmm4, %xmm1
-; SSE2-NEXT: pandn %xmm3, %xmm4
-; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8i32:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm3, %ymm3
-; AVX-NEXT: vmovd %eax, %ymm3
-; AVX-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
-; AVX-NEXT: vmovdqa %ymm3, %ymm2
-; AVX-NEXT: pand %ymm0, %ymm3
-; AVX-NEXT: pandn %ymm1, %ymm2
-; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %ymm2, %ymm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v8i32:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm3, %ymm3
-; AVX2-NEXT: vmovd %eax, %ymm3
-; AVX2-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
-; AVX2-NEXT: vmovdqa %ymm3, %ymm2
-; AVX2-NEXT: pand %ymm0, %ymm3
-; AVX2-NEXT: pandn %ymm1, %ymm2
-; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %ymm2, %ymm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v8i32:
@@ -328,50 +182,24 @@ define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b) {
; SSE2-LABEL: test_ctselect_v8f32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm4
-; SSE2-NEXT: psrad $31, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm5
-; SSE2-NEXT: pand %xmm4, %xmm0
-; SSE2-NEXT: por %xmm5, %xmm0
-; SSE2-NEXT: pand %xmm4, %xmm1
-; SSE2-NEXT: pandn %xmm3, %xmm4
-; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8f32:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm3, %ymm3
-; AVX-NEXT: vmovd %eax, %ymm3
-; AVX-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
-; AVX-NEXT: vmovdqa %ymm3, %ymm2
-; AVX-NEXT: pand %ymm0, %ymm3
-; AVX-NEXT: pandn %ymm1, %ymm2
-; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %ymm2, %ymm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v8f32:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm3, %ymm3
-; AVX2-NEXT: vmovd %eax, %ymm3
-; AVX2-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
-; AVX2-NEXT: vmovdqa %ymm3, %ymm2
-; AVX2-NEXT: pand %ymm0, %ymm3
-; AVX2-NEXT: pandn %ymm1, %ymm2
-; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %ymm2, %ymm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v8f32:
@@ -390,50 +218,24 @@ define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b
define <4 x i64> @test_ctselect_v4i64(i1 %cond, <4 x i64> %a, <4 x i64> %b) {
; SSE2-LABEL: test_ctselect_v4i64:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm4
-; SSE2-NEXT: psrad $31, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm5
-; SSE2-NEXT: pand %xmm4, %xmm0
-; SSE2-NEXT: por %xmm5, %xmm0
-; SSE2-NEXT: pand %xmm4, %xmm1
-; SSE2-NEXT: pandn %xmm3, %xmm4
-; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v4i64:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm3, %ymm3
-; AVX-NEXT: vmovd %eax, %ymm3
-; AVX-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
-; AVX-NEXT: vmovdqa %ymm3, %ymm2
-; AVX-NEXT: pand %ymm0, %ymm3
-; AVX-NEXT: pandn %ymm1, %ymm2
-; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %ymm2, %ymm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v4i64:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm3, %ymm3
-; AVX2-NEXT: vmovd %eax, %ymm3
-; AVX2-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
-; AVX2-NEXT: vmovdqa %ymm3, %ymm2
-; AVX2-NEXT: pand %ymm0, %ymm3
-; AVX2-NEXT: pandn %ymm1, %ymm2
-; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %ymm2, %ymm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4i64:
@@ -452,50 +254,24 @@ define <4 x i64> @test_ctselect_v4i64(i1 %cond, <4 x i64> %a, <4 x i64> %b) {
define <4 x double> @test_ctselect_v4f64(i1 %cond, <4 x double> %a, <4 x double> %b) {
; SSE2-LABEL: test_ctselect_v4f64:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm4
-; SSE2-NEXT: psrad $31, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm5
-; SSE2-NEXT: pand %xmm4, %xmm0
-; SSE2-NEXT: por %xmm5, %xmm0
-; SSE2-NEXT: pand %xmm4, %xmm1
-; SSE2-NEXT: pandn %xmm3, %xmm4
-; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v4f64:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm3, %ymm3
-; AVX-NEXT: vmovd %eax, %ymm3
-; AVX-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
-; AVX-NEXT: vmovdqa %ymm3, %ymm2
-; AVX-NEXT: pand %ymm0, %ymm3
-; AVX-NEXT: pandn %ymm1, %ymm2
-; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %ymm2, %ymm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v4f64:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm3, %ymm3
-; AVX2-NEXT: vmovd %eax, %ymm3
-; AVX2-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
-; AVX2-NEXT: vmovdqa %ymm3, %ymm2
-; AVX2-NEXT: pand %ymm0, %ymm3
-; AVX2-NEXT: pandn %ymm1, %ymm2
-; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %ymm2, %ymm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4f64:
@@ -515,63 +291,33 @@ define <4 x double> @test_ctselect_v4f64(i1 %cond, <4 x double> %a, <4 x double>
define <16 x i32> @test_ctselect_v16i32(i1 %cond, <16 x i32> %a, <16 x i32> %b) {
; SSE2-LABEL: test_ctselect_v16i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm8
-; SSE2-NEXT: psrad $31, %xmm8
-; SSE2-NEXT: movdqa %xmm8, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm9
-; SSE2-NEXT: pand %xmm8, %xmm0
-; SSE2-NEXT: por %xmm9, %xmm0
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm1
-; SSE2-NEXT: por %xmm4, %xmm1
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm6, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm2
-; SSE2-NEXT: por %xmm4, %xmm2
-; SSE2-NEXT: pand %xmm8, %xmm3
-; SSE2-NEXT: pandn %xmm7, %xmm8
-; SSE2-NEXT: por %xmm8, %xmm3
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v16i32:
; AVX: # %bb.0:
-; AVX-NEXT: vmovd %edi, %xmm4
-; AVX-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX-NEXT: vpshufb %xmm5, %xmm4, %xmm4
-; AVX-NEXT: vpsllw $7, %xmm4, %xmm4
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
-; AVX-NEXT: vpcmpgtb %xmm4, %xmm5, %xmm4
-; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; AVX-NEXT: vpmovsxbd %xmm5, %xmm5
-; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
-; AVX-NEXT: vpmovsxbd %xmm6, %xmm6
-; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5
-; AVX-NEXT: vpmovsxbd %xmm4, %xmm6
-; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
-; AVX-NEXT: vpmovsxbd %xmm4, %xmm4
-; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4
-; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
-; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
-; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
-; AVX-NEXT: vandnps %ymm3, %ymm5, %ymm2
-; AVX-NEXT: vandps %ymm5, %ymm1, %ymm1
-; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v16i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovd %edi, %xmm4
-; AVX2-NEXT: vpbroadcastb %xmm4, %xmm4
-; AVX2-NEXT: vpsllw $7, %xmm4, %xmm4
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm5
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
-; AVX2-NEXT: vpmovsxbd %xmm5, %ymm5
-; AVX2-NEXT: vpmovsxbd %xmm4, %ymm4
-; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT: vpblendvb %ymm5, %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v16i32:
; AVX512: # %bb.0:
@@ -589,60 +335,33 @@ define <16 x i32> @test_ctselect_v16i32(i1 %cond, <16 x i32> %a, <16 x i32> %b)
define <16 x float> @test_ctselect_v16f32(i1 %cond, <16 x float> %a, <16 x float> %b) {
; SSE2-LABEL: test_ctselect_v16f32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm8
-; SSE2-NEXT: psrad $31, %xmm8
-; SSE2-NEXT: movdqa %xmm8, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm9
-; SSE2-NEXT: pand %xmm8, %xmm0
-; SSE2-NEXT: por %xmm9, %xmm0
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm1
-; SSE2-NEXT: por %xmm4, %xmm1
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm6, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm2
-; SSE2-NEXT: por %xmm4, %xmm2
-; SSE2-NEXT: pand %xmm8, %xmm3
-; SSE2-NEXT: pandn %xmm7, %xmm8
-; SSE2-NEXT: por %xmm8, %xmm3
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v16f32:
; AVX: # %bb.0:
-; AVX-NEXT: vmovd %edi, %xmm4
-; AVX-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX-NEXT: vpshufb %xmm5, %xmm4, %xmm4
-; AVX-NEXT: vpsllw $7, %xmm4, %xmm4
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
-; AVX-NEXT: vpcmpgtb %xmm4, %xmm5, %xmm4
-; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; AVX-NEXT: vpmovsxbd %xmm5, %xmm5
-; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
-; AVX-NEXT: vpmovsxbd %xmm6, %xmm6
-; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5
-; AVX-NEXT: vpmovsxbd %xmm4, %xmm6
-; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
-; AVX-NEXT: vpmovsxbd %xmm4, %xmm4
-; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4
-; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
-; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
-; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
-; AVX-NEXT: vandnps %ymm3, %ymm5, %ymm2
-; AVX-NEXT: vandps %ymm5, %ymm1, %ymm1
-; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v16f32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovd %edi, %xmm4
-; AVX2-NEXT: vpbroadcastb %xmm4, %xmm4
-; AVX2-NEXT: vpsllw $7, %xmm4, %xmm4
-; AVX2-NEXT: vpmovsxbd %xmm4, %ymm4
-; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v16f32:
; AVX512: # %bb.0:
@@ -660,52 +379,33 @@ define <16 x float> @test_ctselect_v16f32(i1 %cond, <16 x float> %a, <16 x float
define <8 x i64> @test_ctselect_v8i64(i1 %cond, <8 x i64> %a, <8 x i64> %b) {
; SSE2-LABEL: test_ctselect_v8i64:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm8
-; SSE2-NEXT: psrad $31, %xmm8
-; SSE2-NEXT: movdqa %xmm8, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm9
-; SSE2-NEXT: pand %xmm8, %xmm0
-; SSE2-NEXT: por %xmm9, %xmm0
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm1
-; SSE2-NEXT: por %xmm4, %xmm1
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm6, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm2
-; SSE2-NEXT: por %xmm4, %xmm2
-; SSE2-NEXT: pand %xmm8, %xmm3
-; SSE2-NEXT: pandn %xmm7, %xmm8
-; SSE2-NEXT: por %xmm8, %xmm3
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8i64:
; AVX: # %bb.0:
-; AVX-NEXT: vmovd %edi, %xmm4
-; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
-; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
-; AVX-NEXT: vpsllw $15, %xmm4, %xmm4
-; AVX-NEXT: vpsraw $15, %xmm4, %xmm4
-; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm4, %ymm4
-; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
-; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
-; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
-; AVX-NEXT: vandnps %ymm3, %ymm4, %ymm2
-; AVX-NEXT: vandps %ymm4, %ymm1, %ymm1
-; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v8i64:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovd %edi, %xmm4
-; AVX2-NEXT: vpbroadcastw %xmm4, %xmm4
-; AVX2-NEXT: vpsllw $15, %xmm4, %xmm4
-; AVX2-NEXT: vpsraw $15, %xmm4, %xmm4
-; AVX2-NEXT: vpmovsxwq %xmm4, %ymm4
-; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v8i64:
; AVX512: # %bb.0:
@@ -723,52 +423,33 @@ define <8 x i64> @test_ctselect_v8i64(i1 %cond, <8 x i64> %a, <8 x i64> %b) {
define <8 x double> @test_ctselect_v8f64(i1 %cond, <8 x double> %a, <8 x double> %b) {
; SSE2-LABEL: test_ctselect_v8f64:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm8
-; SSE2-NEXT: psrad $31, %xmm8
-; SSE2-NEXT: movdqa %xmm8, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm9
-; SSE2-NEXT: pand %xmm8, %xmm0
-; SSE2-NEXT: por %xmm9, %xmm0
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm1
-; SSE2-NEXT: por %xmm4, %xmm1
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm6, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm2
-; SSE2-NEXT: por %xmm4, %xmm2
-; SSE2-NEXT: pand %xmm8, %xmm3
-; SSE2-NEXT: pandn %xmm7, %xmm8
-; SSE2-NEXT: por %xmm8, %xmm3
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8f64:
; AVX: # %bb.0:
-; AVX-NEXT: vmovd %edi, %xmm4
-; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
-; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
-; AVX-NEXT: vpsllw $15, %xmm4, %xmm4
-; AVX-NEXT: vpsraw $15, %xmm4, %xmm4
-; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm4, %ymm4
-; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
-; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
-; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
-; AVX-NEXT: vandnps %ymm3, %ymm4, %ymm2
-; AVX-NEXT: vandps %ymm4, %ymm1, %ymm1
-; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v8f64:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovd %edi, %xmm4
-; AVX2-NEXT: vpbroadcastw %xmm4, %xmm4
-; AVX2-NEXT: vpsllw $15, %xmm4, %xmm4
-; AVX2-NEXT: vpsraw $15, %xmm4, %xmm4
-; AVX2-NEXT: vpmovsxwq %xmm4, %ymm4
-; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v8f64:
; AVX512: # %bb.0:
@@ -789,17 +470,7 @@ define <4 x i32> @test_ctselect_v4i32_const_true(<4 x i32> %a, <4 x i32> %b) {
; SSE2: # %bb.0:
; SSE2-NEXT: movb $1, %al
; SSE2-NEXT: testb %al, %al
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
@@ -807,17 +478,7 @@ define <4 x i32> @test_ctselect_v4i32_const_true(<4 x i32> %a, <4 x i32> %b) {
; AVX: # %bb.0:
; AVX-NEXT: movb $1, %al
; AVX-NEXT: testb %al, %al
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
@@ -825,17 +486,7 @@ define <4 x i32> @test_ctselect_v4i32_const_true(<4 x i32> %a, <4 x i32> %b) {
; AVX2: # %bb.0:
; AVX2-NEXT: movb $1, %al
; AVX2-NEXT: testb %al, %al
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4i32_const_true:
@@ -850,17 +501,7 @@ define <4 x i32> @test_ctselect_v4i32_const_false(<4 x i32> %a, <4 x i32> %b) {
; SSE2: # %bb.0:
; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: testb %al, %al
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
@@ -868,17 +509,7 @@ define <4 x i32> @test_ctselect_v4i32_const_false(<4 x i32> %a, <4 x i32> %b) {
; AVX: # %bb.0:
; AVX-NEXT: xorl %eax, %eax
; AVX-NEXT: testb %al, %al
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
@@ -886,17 +517,7 @@ define <4 x i32> @test_ctselect_v4i32_const_false(<4 x i32> %a, <4 x i32> %b) {
; AVX2: # %bb.0:
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: testb %al, %al
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4i32_const_false:
@@ -914,17 +535,7 @@ define <4 x i32> @test_ctselect_v4i32_icmp(i32 %x, i32 %y, <4 x i32> %a, <4 x i3
; SSE2-NEXT: cmpl %esi, %edi
; SSE2-NEXT: sete %al
; SSE2-NEXT: testb %al, %al
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
@@ -933,17 +544,7 @@ define <4 x i32> @test_ctselect_v4i32_icmp(i32 %x, i32 %y, <4 x i32> %a, <4 x i3
; AVX-NEXT: cmpl %esi, %edi
; AVX-NEXT: sete %al
; AVX-NEXT: testb %al, %al
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
@@ -952,17 +553,7 @@ define <4 x i32> @test_ctselect_v4i32_icmp(i32 %x, i32 %y, <4 x i32> %a, <4 x i3
; AVX2-NEXT: cmpl %esi, %edi
; AVX2-NEXT: sete %al
; AVX2-NEXT: testb %al, %al
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4i32_icmp:
diff --git a/llvm/test/CodeGen/X86/ctselect.ll b/llvm/test/CodeGen/X86/ctselect.ll
index f36b0d9f042b9..029eacc72b856 100644
--- a/llvm/test/CodeGen/X86/ctselect.ll
+++ b/llvm/test/CodeGen/X86/ctselect.ll
@@ -8,116 +8,58 @@
define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) #0 {
; X64-LABEL: test_ctselect_i8:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: andb $1, %al
-; X64-NEXT: xorl %edx, %esi
-; X64-NEXT: negb %al
-; X64-NEXT: andb %sil, %al
-; X64-NEXT: xorb %dl, %al
+; X64-NEXT: movl %edx, %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %esi, %eax
; X64-NEXT: # kill: def $al killed $al killed $eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_i8:
; X32: # %bb.0:
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorb %cl, %dl
-; X32-NEXT: negb %al
-; X32-NEXT: andb %dl, %al
-; X32-NEXT: xorb %cl, %al
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: # kill: def $al killed $al killed $eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_i8:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorb %cl, %dl
-; X32-NOCMOV-NEXT: negb %al
-; X32-NOCMOV-NEXT: andb %dl, %al
-; X32-NOCMOV-NEXT: xorb %cl, %al
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: retl
%result = call i8 @llvm.ct.select.i8(i1 %cond, i8 %a, i8 %b)
ret i8 %result
}
-define b8 @test_ctselect_b8(i1 %cond, b8 %a, b8 %b) #0 {
-; X64-LABEL: test_ctselect_b8:
-; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: andb $1, %al
-; X64-NEXT: xorl %edx, %esi
-; X64-NEXT: negb %al
-; X64-NEXT: andb %sil, %al
-; X64-NEXT: xorb %dl, %al
-; X64-NEXT: # kill: def $al killed $al killed $eax
-; X64-NEXT: retq
-;
-; X32-LABEL: test_ctselect_b8:
-; X32: # %bb.0:
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorb %cl, %dl
-; X32-NEXT: negb %al
-; X32-NEXT: andb %dl, %al
-; X32-NEXT: xorb %cl, %al
-; X32-NEXT: retl
-;
-; X32-NOCMOV-LABEL: test_ctselect_b8:
-; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorb %cl, %dl
-; X32-NOCMOV-NEXT: negb %al
-; X32-NOCMOV-NEXT: andb %dl, %al
-; X32-NOCMOV-NEXT: xorb %cl, %al
-; X32-NOCMOV-NEXT: retl
- %result = call b8 @llvm.ct.select.b8(i1 %cond, b8 %a, b8 %b)
- ret b8 %result
-}
-
define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_i32:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: xorl %edx, %esi
-; X64-NEXT: andl $1, %eax
-; X64-NEXT: negl %eax
-; X64-NEXT: andl %esi, %eax
-; X64-NEXT: xorl %edx, %eax
+; X64-NEXT: movl %edx, %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %esi, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_i32:
; X32: # %bb.0:
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_i32:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
ret i32 %result
@@ -126,56 +68,41 @@ define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) #0 {
define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) #0 {
; X64-LABEL: test_ctselect_i64:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: xorq %rdx, %rsi
-; X64-NEXT: andl $1, %eax
-; X64-NEXT: negq %rax
-; X64-NEXT: andq %rsi, %rax
-; X64-NEXT: xorq %rdx, %rax
+; X64-NEXT: movq %rdx, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rsi, %rax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_i64:
; X32: # %bb.0:
-; X32-NEXT: pushl %edi
-; X32-NEXT: pushl %esi
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: andb $1, %dl
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %esi, %eax
-; X32-NEXT: movzbl %dl, %edi
-; X32-NEXT: negl %edi
-; X32-NEXT: andl %edi, %eax
-; X32-NEXT: xorl %esi, %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl %edi, %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: popl %esi
-; X32-NEXT: popl %edi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edx
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_i64:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebp
+; X32-NOCMOV-NEXT: pushl %ebx
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: andb $1, %dl
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %esi, %eax
-; X32-NOCMOV-NEXT: movzbl %dl, %edi
-; X32-NOCMOV-NEXT: negl %edi
-; X32-NOCMOV-NEXT: andl %edi, %eax
-; X32-NOCMOV-NEXT: xorl %esi, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl %edi, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: testb $1, %bl
+; X32-NOCMOV-NEXT: sete %bh
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %bl
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
+; X32-NOCMOV-NEXT: popl %ebp
; X32-NOCMOV-NEXT: retl
%result = call i64 @llvm.ct.select.i64(i1 %cond, i64 %a, i64 %b)
ret i64 %result
@@ -184,56 +111,45 @@ define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) #0 {
define float @test_ctselect_f32(i1 %cond, float %a, float %b) #0 {
; X64-LABEL: test_ctselect_f32:
; X64: # %bb.0:
-; X64-NEXT: movd %xmm1, %eax
-; X64-NEXT: pxor %xmm1, %xmm0
-; X64-NEXT: movd %xmm0, %ecx
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl %ecx, %edi
-; X64-NEXT: xorl %eax, %edi
-; X64-NEXT: movd %edi, %xmm0
+; X64-NEXT: movd %xmm0, %eax
+; X64-NEXT: movd %xmm1, %ecx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %eax, %ecx
+; X64-NEXT: movd %ecx, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f32:
; X32: # %bb.0:
-; X32-NEXT: subl $12, %esp
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps (%esp)
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
+; X32-NEXT: pushl %edi
+; X32-NEXT: pushl %esi
+; X32-NEXT: pushl %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: sete %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: movl (%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl %eax, %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: addl $12, %esp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %esi, (%esp)
+; X32-NEXT: flds (%esp)
+; X32-NEXT: addl $4, %esp
+; X32-NEXT: popl %esi
+; X32-NEXT: popl %edi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f32:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: subl $12, %esp
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps (%esp)
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: pushl %eax
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: movl (%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl %eax, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: addl $12, %esp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, (%esp)
+; X32-NOCMOV-NEXT: flds (%esp)
+; X32-NOCMOV-NEXT: addl $4, %esp
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: retl
%result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
ret float %result
@@ -242,73 +158,53 @@ define float @test_ctselect_f32(i1 %cond, float %a, float %b) #0 {
define double @test_ctselect_f64(i1 %cond, double %a, double %b) #0 {
; X64-LABEL: test_ctselect_f64:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-NEXT: movq %xmm1, %rax
-; X64-NEXT: pxor %xmm1, %xmm0
-; X64-NEXT: movq %xmm0, %rcx
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negq %rdi
-; X64-NEXT: andq %rcx, %rdi
-; X64-NEXT: xorq %rax, %rdi
-; X64-NEXT: movq %rdi, %xmm0
+; X64-NEXT: movq %xmm0, %rax
+; X64-NEXT: movq %xmm1, %rcx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rax, %rcx
+; X64-NEXT: movq %rcx, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f64:
; X32: # %bb.0:
+; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $16, %esp
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: fldl {{[0-9]+}}(%esp)
-; X32-NEXT: fldl {{[0-9]+}}(%esp)
-; X32-NEXT: fstpl {{[0-9]+}}(%esp)
-; X32-NEXT: fstpl (%esp)
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: subl $8, %esp
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: sete %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl (%esp), %esi
-; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: andl %eax, %esi
-; X32-NEXT: xorl %edx, %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: BUNDLE
; X32-NEXT: movl %esi, (%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl %eax, %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X32-NEXT: fldl (%esp)
-; X32-NEXT: addl $16, %esp
+; X32-NEXT: addl $8, %esp
; X32-NEXT: popl %esi
+; X32-NEXT: popl %edi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f64:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $16, %esp
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: fldl {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fldl {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstpl (%esp)
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: subl $8, %esp
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl (%esp), %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: andl %eax, %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: movl %esi, (%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl %eax, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fldl (%esp)
-; X32-NOCMOV-NEXT: addl $16, %esp
+; X32-NOCMOV-NEXT: addl $8, %esp
; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: retl
%result = call double @llvm.ct.select.f64(i1 %cond, double %a, double %b)
ret double %result
@@ -317,42 +213,31 @@ define double @test_ctselect_f64(i1 %cond, double %a, double %b) #0 {
define half @test_ctselect_f16(i1 %cond, half %a, half %b) #0 {
; X64-LABEL: test_ctselect_f16:
; X64: # %bb.0:
-; X64-NEXT: pextrw $0, %xmm1, %eax
-; X64-NEXT: pextrw $0, %xmm0, %ecx
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl %ecx, %edi
-; X64-NEXT: xorl %eax, %edi
-; X64-NEXT: pinsrw $0, %edi, %xmm0
+; X64-NEXT: pextrw $0, %xmm0, %eax
+; X64-NEXT: pextrw $0, %xmm1, %ecx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %ecx, %eax
+; X64-NEXT: pinsrw $0, %eax, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f16:
; X32: # %bb.0:
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorw %cx, %dx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: # kill: def $ax killed $ax killed $eax
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %ax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f16:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorw %cx, %dx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: # kill: def $ax killed $ax killed $eax
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%result = call half @llvm.ct.select.f16(i1 %cond, half %a, half %b)
ret half %result
@@ -363,12 +248,9 @@ define bfloat @test_ctselect_bf16(i1 %cond, bfloat %a, bfloat %b) #0 {
; X64: # %bb.0:
; X64-NEXT: pextrw $0, %xmm1, %eax
; X64-NEXT: pextrw $0, %xmm0, %ecx
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl %ecx, %edi
-; X64-NEXT: xorl %eax, %edi
-; X64-NEXT: pinsrw $0, %edi, %xmm0
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %ecx, %eax
+; X64-NEXT: pinsrw $0, %eax, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_bf16:
@@ -383,15 +265,10 @@ define bfloat @test_ctselect_bf16(i1 %cond, bfloat %a, bfloat %b) #0 {
; X32-NEXT: fstps (%esp)
; X32-NEXT: calll __truncsfbf2
; X32-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: andb $1, %cl
-; X32-NEXT: xorl %esi, %eax
-; X32-NEXT: movzbl %cl, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: andl %eax, %ecx
-; X32-NEXT: xorl %esi, %ecx
-; X32-NEXT: shll $16, %ecx
-; X32-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %eax, %esi
+; X32-NEXT: shll $16, %esi
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: addl $8, %esp
; X32-NEXT: popl %esi
@@ -399,8 +276,9 @@ define bfloat @test_ctselect_bf16(i1 %cond, bfloat %a, bfloat %b) #0 {
;
; X32-NOCMOV-LABEL: test_ctselect_bf16:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $8, %esp
+; X32-NOCMOV-NEXT: subl $20, %esp
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
@@ -409,18 +287,15 @@ define bfloat @test_ctselect_bf16(i1 %cond, bfloat %a, bfloat %b) #0 {
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: andb $1, %cl
-; X32-NOCMOV-NEXT: xorl %esi, %eax
-; X32-NOCMOV-NEXT: movzbl %cl, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl %eax, %ecx
-; X32-NOCMOV-NEXT: xorl %esi, %ecx
-; X32-NOCMOV-NEXT: shll $16, %ecx
-; X32-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: shll $16, %edx
+; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: addl $8, %esp
+; X32-NOCMOV-NEXT: addl $20, %esp
; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: retl
%result = call bfloat @llvm.ct.select.bf16(i1 %cond, bfloat %a, bfloat %b)
ret bfloat %result
@@ -429,66 +304,40 @@ define bfloat @test_ctselect_bf16(i1 %cond, bfloat %a, bfloat %b) #0 {
define fp128 @test_ctselect_f128(i1 %cond, fp128 %a, fp128 %b) #0 {
; X64-LABEL: test_ctselect_f128:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
; X64-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negq %rdi
; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rax
; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rcx
-; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rdx
-; X64-NEXT: xorq %rax, %rdx
-; X64-NEXT: andq %rdi, %rdx
-; X64-NEXT: xorq %rax, %rdx
-; X64-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rax
-; X64-NEXT: xorq %rcx, %rax
-; X64-NEXT: andq %rdi, %rax
-; X64-NEXT: xorq %rcx, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq -{{[0-9]+}}(%rsp), %rax
; X64-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NEXT: cmovneq -{{[0-9]+}}(%rsp), %rcx
+; X64-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NEXT: movaps -{{[0-9]+}}(%rsp), %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f128:
; X32: # %bb.0:
-; X32-NEXT: pushl %ebp
-; X32-NEXT: pushl %ebx
; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $12, %esp
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: andb $1, %bl
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %edi, %ecx
-; X32-NEXT: movzbl %bl, %ebp
-; X32-NEXT: negl %ebp
-; X32-NEXT: andl %ebp, %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: andl %ebp, %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: andl %ebp, %ebx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: pushl %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %edx, %eax
-; X32-NEXT: andl %ebp, %eax
-; X32-NEXT: xorl %edi, %ecx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl %edx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl %eax, 12(%edx)
-; X32-NEXT: movl %ebx, 8(%edx)
-; X32-NEXT: movl %esi, 4(%edx)
-; X32-NEXT: movl %ecx, (%edx)
-; X32-NEXT: movl %edx, %eax
-; X32-NEXT: addl $12, %esp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edx
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl %ecx, 12(%eax)
+; X32-NEXT: movl %edx, 8(%eax)
+; X32-NEXT: movl %edi, 4(%eax)
+; X32-NEXT: movl %esi, (%eax)
+; X32-NEXT: addl $4, %esp
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
-; X32-NEXT: popl %ebx
-; X32-NEXT: popl %ebp
; X32-NEXT: retl $4
;
; X32-NOCMOV-LABEL: test_ctselect_f128:
@@ -498,34 +347,36 @@ define fp128 @test_ctselect_f128(i1 %cond, fp128 %a, fp128 %b) #0 {
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: subl $12, %esp
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: andb $1, %bl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: testb $1, %bl
+; X32-NOCMOV-NEXT: sete %bh
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %bl
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %bl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %edi, %ecx
-; X32-NOCMOV-NEXT: movzbl %bl, %ebp
-; X32-NOCMOV-NEXT: negl %ebp
-; X32-NOCMOV-NEXT: andl %ebp, %ecx
+; X32-NOCMOV-NEXT: sete %bh
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: andl %ebp, %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: andl %ebp, %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %bl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %edx, %eax
-; X32-NOCMOV-NEXT: andl %ebp, %eax
-; X32-NOCMOV-NEXT: xorl %edi, %ecx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl %edx, %eax
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl %eax, 12(%edx)
-; X32-NOCMOV-NEXT: movl %ebx, 8(%edx)
-; X32-NOCMOV-NEXT: movl %esi, 4(%edx)
-; X32-NOCMOV-NEXT: movl %ecx, (%edx)
-; X32-NOCMOV-NEXT: movl %edx, %eax
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, 12(%eax)
+; X32-NOCMOV-NEXT: movl %edi, 8(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, 4(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, (%eax)
; X32-NOCMOV-NEXT: addl $12, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
@@ -539,96 +390,82 @@ define fp128 @test_ctselect_f128(i1 %cond, fp128 %a, fp128 %b) #0 {
define x86_fp80 @test_ctselect_f80(i1 %cond, x86_fp80 %a, x86_fp80 %b) #0 {
; X64-LABEL: test_ctselect_f80:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
; X64-NEXT: fldt {{[0-9]+}}(%rsp)
; X64-NEXT: fldt {{[0-9]+}}(%rsp)
; X64-NEXT: fstpt -{{[0-9]+}}(%rsp)
; X64-NEXT: fstpt -{{[0-9]+}}(%rsp)
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negq %rdi
-; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rax
-; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rcx
-; X64-NEXT: xorq %rax, %rcx
-; X64-NEXT: andq %rdi, %rcx
-; X64-NEXT: xorq %rax, %rcx
-; X64-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %ecx
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: andl %ecx, %edi
-; X64-NEXT: xorl %eax, %edi
-; X64-NEXT: movw %di, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT: movl %eax, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT: movl -{{[0-9]+}}(%rsp), %ecx
+; X64-NEXT: cmovnel -{{[0-9]+}}(%rsp), %ecx
+; X64-NEXT: movl %ecx, -{{[0-9]+}}(%rsp)
+; X64-NEXT: cmovnel -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT: movl %eax, -{{[0-9]+}}(%rsp)
; X64-NEXT: fldt -{{[0-9]+}}(%rsp)
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f80:
; X32: # %bb.0:
-; X32-NEXT: pushl %esi
-; X32-NEXT: subl $24, %esp
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
+; X32-NEXT: subl $36, %esp
; X32-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NEXT: fstpt {{[0-9]+}}(%esp)
-; X32-NEXT: fstpt (%esp)
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: fstpt {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl (%esp), %esi
-; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: andl %eax, %esi
-; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: movl %esi, (%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl %eax, %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl %eax, %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movw %dx, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl %eax, (%esp)
; X32-NEXT: fldt (%esp)
-; X32-NEXT: addl $24, %esp
-; X32-NEXT: popl %esi
+; X32-NEXT: addl $36, %esp
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f80:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebp
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $24, %esp
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: subl $40, %esp
; X32-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstpt (%esp)
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl (%esp), %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: andl %eax, %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: movl %esi, (%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: sete %ch
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl %eax, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl %eax, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movw %dx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fldt (%esp)
-; X32-NOCMOV-NEXT: addl $24, %esp
+; X32-NOCMOV-NEXT: sete %ch
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl (%esp), %ebx # 4-byte Reload
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: addl $40, %esp
; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
+; X32-NOCMOV-NEXT: popl %ebp
; X32-NOCMOV-NEXT: retl
%result = call x86_fp80 @llvm.ct.select.f80(i1 %cond, x86_fp80 %a, x86_fp80 %b)
ret x86_fp80 %result
@@ -637,38 +474,29 @@ define x86_fp80 @test_ctselect_f80(i1 %cond, x86_fp80 %a, x86_fp80 %b) #0 {
define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) #0 {
; X64-LABEL: test_ctselect_ptr:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: xorq %rdx, %rsi
-; X64-NEXT: andl $1, %eax
-; X64-NEXT: negq %rax
-; X64-NEXT: andq %rsi, %rax
-; X64-NEXT: xorq %rdx, %rax
+; X64-NEXT: movq %rdx, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rsi, %rax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_ptr:
; X32: # %bb.0:
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_ptr:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%result = call ptr @llvm.ct.select.p0(i1 %cond, ptr %a, ptr %b)
ret ptr %result
@@ -678,25 +506,32 @@ define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) #0 {
define i32 @test_ctselect_const_true(i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_const_true:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: xorl %esi, %eax
-; X64-NEXT: xorl %esi, %eax
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: movb $1, %cl
+; X64-NEXT: testb %cl, %cl
+; X64-NEXT: cmovnel %edi, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_const_true:
; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movb $1, %cl
+; X32-NEXT: testb %cl, %cl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_const_true:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movb $1, %al
+; X32-NOCMOV-NEXT: testb %al, %al
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%result = call i32 @llvm.ct.select.i32(i1 true, i32 %a, i32 %b)
ret i32 %result
@@ -706,18 +541,31 @@ define i32 @test_ctselect_const_false(i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_const_false:
; X64: # %bb.0:
; X64-NEXT: movl %esi, %eax
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: testb %cl, %cl
+; X64-NEXT: cmovnel %edi, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_const_false:
; X32: # %bb.0:
-; X32-NEXT: xorl %eax, %eax
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %ecx, %ecx
+; X32-NEXT: testb %cl, %cl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_const_false:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: xorl %eax, %eax
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: testb %al, %al
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%result = call i32 @llvm.ct.select.i32(i1 false, i32 %a, i32 %b)
ret i32 %result
@@ -727,41 +575,37 @@ define i32 @test_ctselect_const_false(i32 %a, i32 %b) #0 {
define i32 @test_ctselect_icmp_eq(i32 %x, i32 %y, i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_icmp_eq:
; X64: # %bb.0:
-; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: movl %ecx, %eax
; X64-NEXT: cmpl %esi, %edi
-; X64-NEXT: sete %al
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: negl %eax
-; X64-NEXT: andl %edx, %eax
-; X64-NEXT: xorl %ecx, %eax
+; X64-NEXT: sete %cl
+; X64-NEXT: testb %cl, %cl
+; X64-NEXT: cmovnel %edx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_icmp_eq:
; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %eax, %eax
-; X32-NEXT: cmpl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: sete %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: sete %cl
+; X32-NEXT: testb %cl, %cl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_icmp_eq:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %eax, %eax
-; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: sete %al
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: testb %al, %al
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%cond = icmp eq i32 %x, %y
%result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
@@ -771,38 +615,37 @@ define i32 @test_ctselect_icmp_eq(i32 %x, i32 %y, i32 %a, i32 %b) #0 {
define i32 @test_ctselect_icmp_ult(i32 %x, i32 %y, i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_icmp_ult:
; X64: # %bb.0:
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: movl %ecx, %eax
; X64-NEXT: cmpl %esi, %edi
-; X64-NEXT: sbbl %eax, %eax
-; X64-NEXT: andl %edx, %eax
-; X64-NEXT: xorl %ecx, %eax
+; X64-NEXT: setb %cl
+; X64-NEXT: testb %cl, %cl
+; X64-NEXT: cmovnel %edx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_icmp_ult:
; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %edx, %edx
-; X32-NEXT: cmpl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: sbbl %edx, %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: setb %cl
+; X32-NEXT: testb %cl, %cl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_icmp_ult:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %edx, %edx
; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: sbbl %edx, %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: setb %al
+; X32-NOCMOV-NEXT: testb %al, %al
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%cond = icmp ult i32 %x, %y
%result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
@@ -812,49 +655,44 @@ define i32 @test_ctselect_icmp_ult(i32 %x, i32 %y, i32 %a, i32 %b) #0 {
define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) #0 {
; X64-LABEL: test_ctselect_fcmp_oeq:
; X64: # %bb.0:
-; X64-NEXT: movd %xmm3, %eax
-; X64-NEXT: cmpeqss %xmm1, %xmm0
-; X64-NEXT: pxor %xmm3, %xmm2
-; X64-NEXT: pand %xmm0, %xmm2
-; X64-NEXT: movd %xmm2, %ecx
-; X64-NEXT: xorl %eax, %ecx
+; X64-NEXT: movd %xmm2, %eax
+; X64-NEXT: movd %xmm3, %ecx
+; X64-NEXT: ucomiss %xmm1, %xmm0
+; X64-NEXT: setnp %dl
+; X64-NEXT: sete %sil
+; X64-NEXT: testb %dl, %sil
+; X64-NEXT: cmovnel %eax, %ecx
; X64-NEXT: movd %ecx, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_fcmp_oeq:
; X32: # %bb.0:
-; X32-NEXT: subl $12, %esp
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps (%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: pushl %edi
+; X32-NEXT: pushl %esi
+; X32-NEXT: pushl %eax
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fucompi %st(1), %st
; X32-NEXT: fstp %st(0)
-; X32-NEXT: setnp %cl
-; X32-NEXT: sete %dl
-; X32-NEXT: andb %cl, %dl
-; X32-NEXT: movzbl %dl, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: movl (%esp), %edx
-; X32-NEXT: xorl %eax, %edx
-; X32-NEXT: andl %ecx, %edx
-; X32-NEXT: xorl %eax, %edx
-; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: addl $12, %esp
+; X32-NEXT: setnp %al
+; X32-NEXT: sete %cl
+; X32-NEXT: testb %al, %cl
+; X32-NEXT: sete %al
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %esi, (%esp)
+; X32-NEXT: flds (%esp)
+; X32-NEXT: addl $4, %esp
+; X32-NEXT: popl %esi
+; X32-NEXT: popl %edi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_fcmp_oeq:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: subl $12, %esp
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps (%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: pushl %eax
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fucompp
@@ -862,17 +700,17 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) #0
; X32-NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
; X32-NOCMOV-NEXT: sahf
; X32-NOCMOV-NEXT: setnp %al
-; X32-NOCMOV-NEXT: sete %dl
-; X32-NOCMOV-NEXT: andb %al, %dl
-; X32-NOCMOV-NEXT: movzbl %dl, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: movl (%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl %eax, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: addl $12, %esp
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: testb %al, %cl
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, (%esp)
+; X32-NOCMOV-NEXT: flds (%esp)
+; X32-NOCMOV-NEXT: addl $4, %esp
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: retl
%cond = fcmp oeq float %x, %y
%result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
@@ -883,43 +721,33 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) #0
define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) #0 {
; X64-LABEL: test_ctselect_load:
; X64: # %bb.0:
-; X64-NEXT: movl (%rdx), %ecx
-; X64-NEXT: movl (%rsi), %eax
-; X64-NEXT: xorl %ecx, %eax
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl %edi, %eax
-; X64-NEXT: xorl %ecx, %eax
+; X64-NEXT: movl (%rdx), %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel (%rsi), %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_load:
; X32: # %bb.0:
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl (%edx), %edx
-; X32-NEXT: movl (%ecx), %ecx
-; X32-NEXT: xorl %edx, %ecx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %ecx, %eax
-; X32-NEXT: xorl %edx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl (%eax), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel (%ecx), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_load:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl (%edx), %edx
; X32-NOCMOV-NEXT: movl (%ecx), %ecx
-; X32-NOCMOV-NEXT: xorl %edx, %ecx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %ecx, %eax
-; X32-NOCMOV-NEXT: xorl %edx, %eax
+; X32-NOCMOV-NEXT: movl (%eax), %edx
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%a = load i32, ptr %p1
%b = load i32, ptr %p2
@@ -931,67 +759,40 @@ define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) #0 {
define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) #0 {
; X64-LABEL: test_ctselect_nested:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: andl $1, %esi
-; X64-NEXT: negl %esi
-; X64-NEXT: andl %edx, %esi
-; X64-NEXT: xorl %r8d, %ecx
-; X64-NEXT: xorl %esi, %ecx
-; X64-NEXT: andl $1, %eax
-; X64-NEXT: negl %eax
-; X64-NEXT: andl %ecx, %eax
-; X64-NEXT: xorl %r8d, %eax
+; X64-NEXT: movl %r8d, %eax
+; X64-NEXT: testb $1, %sil
+; X64-NEXT: cmovnel %edx, %ecx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %ecx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_nested:
; X32: # %bb.0:
-; X32-NEXT: pushl %edi
-; X32-NEXT: pushl %esi
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movb {{[0-9]+}}(%esp), %ah
-; X32-NEXT: andb $1, %ah
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: movzbl %ah, %edi
-; X32-NEXT: negl %edi
-; X32-NEXT: andl %esi, %edi
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: xorl %edi, %edx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: popl %esi
-; X32-NEXT: popl %edi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %ecx, %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_nested:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebx
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %ah
-; X32-NOCMOV-NEXT: andb $1, %ah
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: movzbl %ah, %edi
-; X32-NOCMOV-NEXT: negl %edi
-; X32-NOCMOV-NEXT: andl %esi, %edi
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: xorl %edi, %edx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %dl
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%inner = call i32 @llvm.ct.select.i32(i1 %cond2, i32 %a, i32 %b)
%result = call i32 @llvm.ct.select.i32(i1 %cond1, i32 %inner, i32 %c)
@@ -1004,44 +805,49 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) #
define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) #0 {
; X64-LABEL: test_ctselect_nested_and_i1_to_i32:
; X64: # %bb.0:
+; X64-NEXT: movl %ecx, %eax
; X64-NEXT: andl %esi, %edi
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: movzbl %dil, %eax
-; X64-NEXT: negl %eax
-; X64-NEXT: andl %edx, %eax
-; X64-NEXT: xorl %ecx, %eax
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movl $1, %esi
+; X64-NEXT: cmovnel %esi, %ecx
+; X64-NEXT: testb $1, %cl
+; X64-NEXT: cmovnel %edx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_nested_and_i1_to_i32:
; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NEXT: andb $1, %al
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: andb {{[0-9]+}}(%esp), %cl
+; X32-NEXT: xorl %edx, %edx
+; X32-NEXT: testb $1, %cl
+; X32-NEXT: movl $1, %ecx
+; X32-NEXT: cmovnel %ecx, %edx
+; X32-NEXT: testb $1, %dl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_nested_and_i1_to_i32:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: xorl %ebx, %ebx
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movb $1, %al
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%inner = call i1 @llvm.ct.select.i1(i1 %c1, i1 true, i1 false)
%cond = call i1 @llvm.ct.select.i1(i1 %c0, i1 %inner, i1 false)
@@ -1055,44 +861,49 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) #
define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) #0 {
; X64-LABEL: test_ctselect_nested_or_i1_to_i32:
; X64: # %bb.0:
+; X64-NEXT: movl %ecx, %eax
; X64-NEXT: orl %esi, %edi
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: movzbl %dil, %eax
-; X64-NEXT: negl %eax
-; X64-NEXT: andl %edx, %eax
-; X64-NEXT: xorl %ecx, %eax
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movl $1, %esi
+; X64-NEXT: cmovnel %esi, %ecx
+; X64-NEXT: testb $1, %cl
+; X64-NEXT: cmovnel %edx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_nested_or_i1_to_i32:
; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: orb {{[0-9]+}}(%esp), %al
-; X32-NEXT: andb $1, %al
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: orb {{[0-9]+}}(%esp), %cl
+; X32-NEXT: xorl %edx, %edx
+; X32-NEXT: testb $1, %cl
+; X32-NEXT: movl $1, %ecx
+; X32-NEXT: cmovnel %ecx, %edx
+; X32-NEXT: testb $1, %dl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_nested_or_i1_to_i32:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: orb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: xorl %ebx, %ebx
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movb $1, %al
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%inner = call i1 @llvm.ct.select.i1(i1 %c1, i1 true, i1 false)
%cond = call i1 @llvm.ct.select.i1(i1 %c0, i1 true, i1 %inner)
@@ -1108,47 +919,52 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) #0
define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i32 %y) #0 {
; X64-LABEL: test_ctselect_double_nested_and_i1:
; X64: # %bb.0:
+; X64-NEXT: movl %r8d, %eax
; X64-NEXT: andl %esi, %edi
; X64-NEXT: andl %edx, %edi
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: xorl %r8d, %ecx
-; X64-NEXT: movzbl %dil, %eax
-; X64-NEXT: negl %eax
-; X64-NEXT: andl %ecx, %eax
-; X64-NEXT: xorl %r8d, %eax
+; X64-NEXT: xorl %edx, %edx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movl $1, %esi
+; X64-NEXT: cmovnel %esi, %edx
+; X64-NEXT: testb $1, %dl
+; X64-NEXT: cmovnel %ecx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_double_nested_and_i1:
; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NEXT: andb $1, %al
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: andb {{[0-9]+}}(%esp), %cl
+; X32-NEXT: andb {{[0-9]+}}(%esp), %cl
+; X32-NEXT: xorl %edx, %edx
+; X32-NEXT: testb $1, %cl
+; X32-NEXT: movl $1, %ecx
+; X32-NEXT: cmovnel %ecx, %edx
+; X32-NEXT: testb $1, %dl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_double_nested_and_i1:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: xorl %ebx, %ebx
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movb $1, %al
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%inner2 = call i1 @llvm.ct.select.i1(i1 %c2, i1 true, i1 false)
%inner1 = call i1 @llvm.ct.select.i1(i1 %c1, i1 %inner2, i1 false)
@@ -1160,327 +976,36 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
; Vector CT_SELECT Tests
; ============================================================================
-; Byte-typed vector: bytes lower like the same-width integer vector.
-define <16 x b8> @test_ctselect_v16b8(i1 %cond, <16 x b8> %a, <16 x b8> %b) #0 {
-; X64-LABEL: test_ctselect_v16b8:
-; X64: # %bb.0:
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: pxor %xmm1, %xmm0
-; X64-NEXT: negb %dil
-; X64-NEXT: movzbl %dil, %eax
-; X64-NEXT: movd %eax, %xmm2
-; X64-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
-; X64-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
-; X64-NEXT: pand %xmm2, %xmm0
-; X64-NEXT: pxor %xmm1, %xmm0
-; X64-NEXT: retq
-;
-; X32-LABEL: test_ctselect_v16b8:
-; X32: # %bb.0:
-; X32-NEXT: pushl %ebx
-; X32-NEXT: subl $12, %esp
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NEXT: negb %al
-; X32-NEXT: andb %al, %cl
-; X32-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NEXT: andb %al, %cl
-; X32-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NEXT: andb %al, %cl
-; X32-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NEXT: andb %al, %cl
-; X32-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NEXT: andb %al, %cl
-; X32-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NEXT: andb %al, %cl
-; X32-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NEXT: andb %al, %cl
-; X32-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NEXT: andb %al, %cl
-; X32-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NEXT: andb %al, %cl
-; X32-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NEXT: andb %al, %cl
-; X32-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NEXT: movb {{[0-9]+}}(%esp), %bh
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %bh
-; X32-NEXT: andb %al, %bh
-; X32-NEXT: movb {{[0-9]+}}(%esp), %bl
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %bl
-; X32-NEXT: andb %al, %bl
-; X32-NEXT: movb {{[0-9]+}}(%esp), %dh
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %dh
-; X32-NEXT: andb %al, %dh
-; X32-NEXT: movb {{[0-9]+}}(%esp), %ch
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %ch
-; X32-NEXT: andb %al, %ch
-; X32-NEXT: movb {{[0-9]+}}(%esp), %dl
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %dl
-; X32-NEXT: andb %al, %dl
-; X32-NEXT: movb {{[0-9]+}}(%esp), %ah
-; X32-NEXT: movb {{[0-9]+}}(%esp), %cl
-; X32-NEXT: xorb %ah, %cl
-; X32-NEXT: andb %al, %cl
-; X32-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %al
-; X32-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %al
-; X32-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %bh
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %bl
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %dh
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %ch
-; X32-NEXT: xorb {{[0-9]+}}(%esp), %dl
-; X32-NEXT: xorb %ah, %cl
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movb %cl, 15(%eax)
-; X32-NEXT: movb %dl, 14(%eax)
-; X32-NEXT: movb %ch, 13(%eax)
-; X32-NEXT: movb %dh, 12(%eax)
-; X32-NEXT: movb %bl, 11(%eax)
-; X32-NEXT: movb %bh, 10(%eax)
-; X32-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT: movb %cl, 9(%eax)
-; X32-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT: movb %cl, 8(%eax)
-; X32-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT: movb %cl, 7(%eax)
-; X32-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT: movb %cl, 6(%eax)
-; X32-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT: movb %cl, 5(%eax)
-; X32-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT: movb %cl, 4(%eax)
-; X32-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT: movb %cl, 3(%eax)
-; X32-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT: movb %cl, 2(%eax)
-; X32-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT: movb %cl, 1(%eax)
-; X32-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NEXT: movb %cl, (%eax)
-; X32-NEXT: addl $12, %esp
-; X32-NEXT: popl %ebx
-; X32-NEXT: retl $4
-;
-; X32-NOCMOV-LABEL: test_ctselect_v16b8:
-; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: pushl %ebx
-; X32-NOCMOV-NEXT: subl $12, %esp
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NOCMOV-NEXT: negb %al
-; X32-NOCMOV-NEXT: andb %al, %cl
-; X32-NOCMOV-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NOCMOV-NEXT: andb %al, %cl
-; X32-NOCMOV-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NOCMOV-NEXT: andb %al, %cl
-; X32-NOCMOV-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NOCMOV-NEXT: andb %al, %cl
-; X32-NOCMOV-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NOCMOV-NEXT: andb %al, %cl
-; X32-NOCMOV-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NOCMOV-NEXT: andb %al, %cl
-; X32-NOCMOV-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NOCMOV-NEXT: andb %al, %cl
-; X32-NOCMOV-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NOCMOV-NEXT: andb %al, %cl
-; X32-NOCMOV-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NOCMOV-NEXT: andb %al, %cl
-; X32-NOCMOV-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %cl
-; X32-NOCMOV-NEXT: andb %al, %cl
-; X32-NOCMOV-NEXT: movb %cl, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %bh
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %bh
-; X32-NOCMOV-NEXT: andb %al, %bh
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %bl
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %bl
-; X32-NOCMOV-NEXT: andb %al, %bl
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %dh
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %dh
-; X32-NOCMOV-NEXT: andb %al, %dh
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %ch
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %ch
-; X32-NOCMOV-NEXT: andb %al, %ch
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %dl
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %dl
-; X32-NOCMOV-NEXT: andb %al, %dl
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %ah
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %cl
-; X32-NOCMOV-NEXT: xorb %ah, %cl
-; X32-NOCMOV-NEXT: andb %al, %cl
-; X32-NOCMOV-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: xorb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Folded Spill
-; X32-NOCMOV-NEXT: movb {{[-0-9]+}}(%e{{[sb]}}p), %al # 1-byte Reload
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: movb %al, {{[-0-9]+}}(%e{{[sb]}}p) # 1-byte Spill
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %bh
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %bl
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %dh
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %ch
-; X32-NOCMOV-NEXT: xorb {{[0-9]+}}(%esp), %dl
-; X32-NOCMOV-NEXT: xorb %ah, %cl
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movb %cl, 15(%eax)
-; X32-NOCMOV-NEXT: movb %dl, 14(%eax)
-; X32-NOCMOV-NEXT: movb %ch, 13(%eax)
-; X32-NOCMOV-NEXT: movb %dh, 12(%eax)
-; X32-NOCMOV-NEXT: movb %bl, 11(%eax)
-; X32-NOCMOV-NEXT: movb %bh, 10(%eax)
-; X32-NOCMOV-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NOCMOV-NEXT: movb %cl, 9(%eax)
-; X32-NOCMOV-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NOCMOV-NEXT: movb %cl, 8(%eax)
-; X32-NOCMOV-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NOCMOV-NEXT: movb %cl, 7(%eax)
-; X32-NOCMOV-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NOCMOV-NEXT: movb %cl, 6(%eax)
-; X32-NOCMOV-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NOCMOV-NEXT: movb %cl, 5(%eax)
-; X32-NOCMOV-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NOCMOV-NEXT: movb %cl, 4(%eax)
-; X32-NOCMOV-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NOCMOV-NEXT: movb %cl, 3(%eax)
-; X32-NOCMOV-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NOCMOV-NEXT: movb %cl, 2(%eax)
-; X32-NOCMOV-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NOCMOV-NEXT: movb %cl, 1(%eax)
-; X32-NOCMOV-NEXT: movzbl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 1-byte Folded Reload
-; X32-NOCMOV-NEXT: movb %cl, (%eax)
-; X32-NOCMOV-NEXT: addl $12, %esp
-; X32-NOCMOV-NEXT: popl %ebx
-; X32-NOCMOV-NEXT: retl $4
- %result = call <16 x b8> @llvm.ct.select.v16b8(i1 %cond, <16 x b8> %a, <16 x b8> %b)
- ret <16 x b8> %result
-}
-
; Test vector CT_SELECT with v4i32 (128-bit vector with single i1 mask)
; NOW CONSTANT-TIME: Uses bitwise XOR/AND operations instead of branches!
define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) #0 {
; X64-LABEL: test_ctselect_v4i32:
; X64: # %bb.0:
-; X64-NEXT: pxor %xmm1, %xmm0
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: movd %edi, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; X64-NEXT: pand %xmm2, %xmm0
-; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: BUNDLE
+; X64-NEXT: movaps %xmm2, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_v4i32:
; X32: # %bb.0:
-; X32-NEXT: pushl %ebp
-; X32-NEXT: pushl %ebx
; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: andb $1, %bl
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %edi, %ecx
-; X32-NEXT: movzbl %bl, %ebp
-; X32-NEXT: negl %ebp
-; X32-NEXT: andl %ebp, %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: andl %ebp, %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: andl %ebp, %ebx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %edx, %eax
-; X32-NEXT: andl %ebp, %eax
-; X32-NEXT: xorl %edi, %ecx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl %edx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl %eax, 12(%edx)
-; X32-NEXT: movl %ebx, 8(%edx)
-; X32-NEXT: movl %esi, 4(%edx)
-; X32-NEXT: movl %ecx, (%edx)
-; X32-NEXT: movl %edx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edx
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl %ecx, 12(%eax)
+; X32-NEXT: movl %edx, 8(%eax)
+; X32-NEXT: movl %esi, 4(%eax)
+; X32-NEXT: movl %edi, (%eax)
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
-; X32-NEXT: popl %ebx
-; X32-NEXT: popl %ebp
; X32-NEXT: retl $4
;
; X32-NOCMOV-LABEL: test_ctselect_v4i32:
@@ -1489,34 +1014,38 @@ define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) #0 {
; X32-NOCMOV-NEXT: pushl %ebx
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: subl $8, %esp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: andb $1, %bl
+; X32-NOCMOV-NEXT: testb $1, %bl
+; X32-NOCMOV-NEXT: sete %bh
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %bl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: sete %bh
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %bl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %edi, %ecx
-; X32-NOCMOV-NEXT: movzbl %bl, %ebp
-; X32-NOCMOV-NEXT: negl %ebp
-; X32-NOCMOV-NEXT: andl %ebp, %ecx
+; X32-NOCMOV-NEXT: sete %bh
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: andl %ebp, %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: andl %ebp, %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %bl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %edx, %eax
-; X32-NOCMOV-NEXT: andl %ebp, %eax
-; X32-NOCMOV-NEXT: xorl %edi, %ecx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl %edx, %eax
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl %eax, 12(%edx)
-; X32-NOCMOV-NEXT: movl %ebx, 8(%edx)
-; X32-NOCMOV-NEXT: movl %esi, 4(%edx)
-; X32-NOCMOV-NEXT: movl %ecx, (%edx)
-; X32-NOCMOV-NEXT: movl %edx, %eax
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, 12(%eax)
+; X32-NOCMOV-NEXT: movl %edi, 8(%eax)
+; X32-NOCMOV-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, 4(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, (%eax)
+; X32-NOCMOV-NEXT: addl $8, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: popl %ebx
@@ -1528,147 +1057,101 @@ define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) #0 {
define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b) #0 {
; X64-LABEL: test_ctselect_v4f32:
; X64: # %bb.0:
-; X64-NEXT: pxor %xmm1, %xmm0
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: movd %edi, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; X64-NEXT: pand %xmm2, %xmm0
-; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: BUNDLE
+; X64-NEXT: movaps %xmm2, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_v4f32:
; X32: # %bb.0:
-; X32-NEXT: pushl %ebp
; X32-NEXT: pushl %ebx
; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $48, %esp
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps (%esp)
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: andb $1, %dl
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: subl $16, %esp
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: movzbl %dl, %edx
-; X32-NEXT: negl %edx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorl %ebx, %ebp
-; X32-NEXT: andl %edx, %ebp
-; X32-NEXT: xorl %ebx, %ebp
-; X32-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl %edi, %ebx
-; X32-NEXT: andl %edx, %ebx
-; X32-NEXT: xorl %edi, %ebx
-; X32-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: xorl %esi, %edi
-; X32-NEXT: andl %edx, %edi
-; X32-NEXT: xorl %esi, %edi
+; X32-NEXT: BUNDLE
; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X32-NEXT: movl (%esp), %esi
-; X32-NEXT: xorl %ecx, %esi
-; X32-NEXT: andl %edx, %esi
-; X32-NEXT: xorl %ecx, %esi
-; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, (%esp)
+; X32-NEXT: flds (%esp)
; X32-NEXT: fstps 12(%eax)
; X32-NEXT: fstps 8(%eax)
; X32-NEXT: fstps 4(%eax)
; X32-NEXT: fstps (%eax)
-; X32-NEXT: addl $48, %esp
+; X32-NEXT: addl $16, %esp
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
; X32-NEXT: popl %ebx
-; X32-NEXT: popl %ebp
; X32-NEXT: retl $4
;
; X32-NOCMOV-LABEL: test_ctselect_v4f32:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: pushl %ebp
; X32-NOCMOV-NEXT: pushl %ebx
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $48, %esp
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps (%esp)
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: andb $1, %dl
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: subl $16, %esp
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: movzbl %dl, %edx
-; X32-NOCMOV-NEXT: negl %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorl %ebx, %ebp
-; X32-NOCMOV-NEXT: andl %edx, %ebp
-; X32-NOCMOV-NEXT: xorl %ebx, %ebp
-; X32-NOCMOV-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl %edi, %ebx
-; X32-NOCMOV-NEXT: andl %edx, %ebx
-; X32-NOCMOV-NEXT: xorl %edi, %ebx
-; X32-NOCMOV-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: xorl %esi, %edi
-; X32-NOCMOV-NEXT: andl %edx, %edi
-; X32-NOCMOV-NEXT: xorl %esi, %edi
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl (%esp), %esi
-; X32-NOCMOV-NEXT: xorl %ecx, %esi
-; X32-NOCMOV-NEXT: andl %edx, %esi
-; X32-NOCMOV-NEXT: xorl %ecx, %esi
-; X32-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, (%esp)
+; X32-NOCMOV-NEXT: flds (%esp)
; X32-NOCMOV-NEXT: fstps 12(%eax)
; X32-NOCMOV-NEXT: fstps 8(%eax)
; X32-NOCMOV-NEXT: fstps 4(%eax)
; X32-NOCMOV-NEXT: fstps (%eax)
-; X32-NOCMOV-NEXT: addl $48, %esp
+; X32-NOCMOV-NEXT: addl $16, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: popl %ebx
-; X32-NOCMOV-NEXT: popl %ebp
; X32-NOCMOV-NEXT: retl $4
%result = call <4 x float> @llvm.ct.select.v4f32(i1 %cond, <4 x float> %a, <4 x float> %b)
ret <4 x float> %result
@@ -1677,16 +1160,11 @@ define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b
define <8 x i32> @test_ctselect_v8i32_avx(i1 %cond, <8 x i32> %a, <8 x i32> %b) #0 {
; X64-LABEL: test_ctselect_v8i32_avx:
; X64: # %bb.0:
-; X64-NEXT: pxor %xmm2, %xmm0
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: movd %edi, %xmm4
-; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; X64-NEXT: pand %xmm4, %xmm0
-; X64-NEXT: pxor %xmm2, %xmm0
-; X64-NEXT: pxor %xmm3, %xmm1
-; X64-NEXT: pand %xmm4, %xmm1
-; X64-NEXT: pxor %xmm3, %xmm1
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: BUNDLE
+; X64-NEXT: BUNDLE
+; X64-NEXT: movaps %xmm4, %xmm0
+; X64-NEXT: movaps %xmm2, %xmm1
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_v8i32_avx:
@@ -1696,58 +1174,36 @@ define <8 x i32> @test_ctselect_v8i32_avx(i1 %cond, <8 x i32> %a, <8 x i32> %b)
; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
; X32-NEXT: subl $8, %esp
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movzbl %al, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: andl %ecx, %edx
-; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andl %ecx, %eax
-; X32-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: andl %ecx, %ebp
; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: andl %ecx, %ebx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: andl %ecx, %edi
; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: andl %ecx, %esi
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: andl %ecx, %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andl %ecx, %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %ecx, (%esp) # 4-byte Folded Spill
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edx
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl %ecx, (%esp) # 4-byte Spill
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl %eax, 28(%ecx)
-; X32-NEXT: movl %edx, 24(%ecx)
-; X32-NEXT: movl %esi, 20(%ecx)
-; X32-NEXT: movl %edi, 16(%ecx)
-; X32-NEXT: movl %ebx, 12(%ecx)
-; X32-NEXT: movl %ebp, 8(%ecx)
-; X32-NEXT: movl (%esp), %eax # 4-byte Reload
-; X32-NEXT: movl %eax, 4(%ecx)
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT: movl %eax, (%ecx)
-; X32-NEXT: movl %ecx, %eax
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl %ecx, 28(%eax)
+; X32-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X32-NEXT: movl %ecx, 24(%eax)
+; X32-NEXT: movl %ebp, 20(%eax)
+; X32-NEXT: movl %ebx, 16(%eax)
+; X32-NEXT: movl %edi, 12(%eax)
+; X32-NEXT: movl %esi, 8(%eax)
+; X32-NEXT: movl %edx, 4(%eax)
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT: movl %ecx, (%eax)
; X32-NEXT: addl $8, %esp
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
@@ -1761,60 +1217,68 @@ define <8 x i32> @test_ctselect_v8i32_avx(i1 %cond, <8 x i32> %a, <8 x i32> %b)
; X32-NOCMOV-NEXT: pushl %ebx
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $8, %esp
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: subl $20, %esp
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movzbl %al, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl %ecx, %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andl %ecx, %eax
-; X32-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: andl %ecx, %ebp
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: andl %ecx, %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: andl %ecx, %edi
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, (%esp) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: sete %dl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: andl %ecx, %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: andl %ecx, %edx
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andl %ecx, %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %ecx, (%esp) # 4-byte Folded Spill
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl %eax, 28(%ecx)
-; X32-NOCMOV-NEXT: movl %edx, 24(%ecx)
-; X32-NOCMOV-NEXT: movl %esi, 20(%ecx)
-; X32-NOCMOV-NEXT: movl %edi, 16(%ecx)
-; X32-NOCMOV-NEXT: movl %ebx, 12(%ecx)
-; X32-NOCMOV-NEXT: movl %ebp, 8(%ecx)
-; X32-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: movl %eax, 4(%ecx)
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: movl %eax, (%ecx)
-; X32-NOCMOV-NEXT: movl %ecx, %eax
-; X32-NOCMOV-NEXT: addl $8, %esp
+; X32-NOCMOV-NEXT: movl %esi, 28(%eax)
+; X32-NOCMOV-NEXT: movl %edi, 24(%eax)
+; X32-NOCMOV-NEXT: movl %ebp, 20(%eax)
+; X32-NOCMOV-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, 16(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, 12(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, 8(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, 4(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, (%eax)
+; X32-NOCMOV-NEXT: addl $20, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: popl %ebx
@@ -1827,122 +1291,77 @@ define <8 x i32> @test_ctselect_v8i32_avx(i1 %cond, <8 x i32> %a, <8 x i32> %b)
define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b) #0 {
; X64-LABEL: test_ctselect_v8f32:
; X64: # %bb.0:
-; X64-NEXT: pxor %xmm2, %xmm0
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: movd %edi, %xmm4
-; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; X64-NEXT: pand %xmm4, %xmm0
-; X64-NEXT: pxor %xmm2, %xmm0
-; X64-NEXT: pxor %xmm3, %xmm1
-; X64-NEXT: pand %xmm4, %xmm1
-; X64-NEXT: pxor %xmm3, %xmm1
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: BUNDLE
+; X64-NEXT: BUNDLE
+; X64-NEXT: movaps %xmm4, %xmm0
+; X64-NEXT: movaps %xmm2, %xmm1
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_v8f32:
; X32: # %bb.0:
-; X32-NEXT: pushl %ebp
; X32-NEXT: pushl %ebx
; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $104, %esp
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: andb $1, %dl
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: subl $36, %esp
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: movzbl %dl, %edx
-; X32-NEXT: negl %edx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %ebx, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ebx, %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %ebp, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ebp, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %ebx, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ebx, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %edi, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %edi, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %esi, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %esi, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps (%esp) # 4-byte Folded Spill
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps 28(%eax)
; X32-NEXT: fstps 24(%eax)
@@ -1953,117 +1372,76 @@ define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b
; X32-NEXT: fstps 4(%eax)
; X32-NEXT: flds (%esp) # 4-byte Folded Reload
; X32-NEXT: fstps (%eax)
-; X32-NEXT: addl $104, %esp
+; X32-NEXT: addl $36, %esp
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
; X32-NEXT: popl %ebx
-; X32-NEXT: popl %ebp
; X32-NEXT: retl $4
;
; X32-NOCMOV-LABEL: test_ctselect_v8f32:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: pushl %ebp
; X32-NOCMOV-NEXT: pushl %ebx
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $104, %esp
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: andb $1, %dl
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: subl $36, %esp
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: movzbl %dl, %edx
-; X32-NOCMOV-NEXT: negl %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %ebx, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ebx, %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %ebp, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ebp, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %ebx, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ebx, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %edi, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %edi, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %esi, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %esi, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp) # 4-byte Folded Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps 28(%eax)
; X32-NOCMOV-NEXT: fstps 24(%eax)
@@ -2074,11 +1452,10 @@ define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b
; X32-NOCMOV-NEXT: fstps 4(%eax)
; X32-NOCMOV-NEXT: flds (%esp) # 4-byte Folded Reload
; X32-NOCMOV-NEXT: fstps (%eax)
-; X32-NOCMOV-NEXT: addl $104, %esp
+; X32-NOCMOV-NEXT: addl $36, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: popl %ebx
-; X32-NOCMOV-NEXT: popl %ebp
; X32-NOCMOV-NEXT: retl $4
%result = call <8 x float> @llvm.ct.select.v8f32(i1 %cond, <8 x float> %a, <8 x float> %b)
ret <8 x float> %result
@@ -2087,14 +1464,9 @@ define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b
define <8 x half> @test_ctselect_v8f16(i1 %cond, <8 x half> %a, <8 x half> %b) #0 {
; X64-LABEL: test_ctselect_v8f16:
; X64: # %bb.0:
-; X64-NEXT: pxor %xmm1, %xmm0
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: movd %edi, %xmm2
-; X64-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
-; X64-NEXT: pand %xmm2, %xmm0
-; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: BUNDLE
+; X64-NEXT: movaps %xmm2, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_v8f16:
@@ -2103,69 +1475,38 @@ define <8 x half> @test_ctselect_v8f16(i1 %cond, <8 x half> %a, <8 x half> %b) #
; X32-NEXT: pushl %ebx
; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $12, %esp
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorw %di, %cx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %eax, %ecx
-; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorw %si, %cx
-; X32-NEXT: andl %eax, %ecx
-; X32-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X32-NEXT: pushl %eax
; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorw %dx, %cx
-; X32-NEXT: andl %eax, %ecx
-; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorw %cx, %bx
-; X32-NEXT: andl %eax, %ebx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorw %cx, %bp
-; X32-NEXT: andl %eax, %ebp
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edi
; X32-NEXT: movzwl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorw %cx, %si
-; X32-NEXT: andl %eax, %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorw %cx, %dx
-; X32-NEXT: andl %eax, %edx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %ax
+; X32-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %dx
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %si
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %di
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %bx
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %bp
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %cx
+; X32-NEXT: movw %cx, (%esp) # 2-byte Spill
; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: xorw %di, %cx
-; X32-NEXT: andl %eax, %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %eax, (%esp) # 4-byte Folded Spill
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %cx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NEXT: movw %cx, 14(%eax)
-; X32-NEXT: movw %dx, 12(%eax)
-; X32-NEXT: movw %si, 10(%eax)
-; X32-NEXT: movw %bp, 8(%eax)
-; X32-NEXT: movw %bx, 6(%eax)
-; X32-NEXT: movw %di, 4(%eax)
-; X32-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X32-NEXT: movw %cx, 2(%eax)
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT: movzwl (%esp), %ecx # 2-byte Folded Reload
+; X32-NEXT: movw %cx, 12(%eax)
+; X32-NEXT: movw %bp, 10(%eax)
+; X32-NEXT: movw %bx, 8(%eax)
+; X32-NEXT: movw %di, 6(%eax)
+; X32-NEXT: movw %si, 4(%eax)
+; X32-NEXT: movw %dx, 2(%eax)
+; X32-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 2-byte Folded Reload
; X32-NEXT: movw %cx, (%eax)
-; X32-NEXT: addl $12, %esp
+; X32-NEXT: addl $4, %esp
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
; X32-NEXT: popl %ebx
@@ -2179,66 +1520,65 @@ define <8 x half> @test_ctselect_v8f16(i1 %cond, <8 x half> %a, <8 x half> %b) #
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: subl $12, %esp
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorw %di, %cx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %eax, %ecx
-; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorw %si, %cx
-; X32-NOCMOV-NEXT: andl %eax, %ecx
-; X32-NOCMOV-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movw %cx, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movw %cx, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movw %dx, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorw %dx, %cx
-; X32-NOCMOV-NEXT: andl %eax, %ecx
-; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorw %cx, %bx
-; X32-NOCMOV-NEXT: andl %eax, %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movw %si, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorw %cx, %bp
-; X32-NOCMOV-NEXT: andl %eax, %ebp
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorw %cx, %si
-; X32-NOCMOV-NEXT: andl %eax, %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorw %cx, %dx
-; X32-NOCMOV-NEXT: andl %eax, %edx
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movw %di, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: xorw %di, %cx
-; X32-NOCMOV-NEXT: andl %eax, %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %eax, (%esp) # 4-byte Folded Spill
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: sete %dl
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movw %cx, 14(%eax)
-; X32-NOCMOV-NEXT: movw %dx, 12(%eax)
-; X32-NOCMOV-NEXT: movw %si, 10(%eax)
-; X32-NOCMOV-NEXT: movw %bp, 8(%eax)
-; X32-NOCMOV-NEXT: movw %bx, 6(%eax)
-; X32-NOCMOV-NEXT: movw %di, 4(%eax)
-; X32-NOCMOV-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %si, 14(%eax)
+; X32-NOCMOV-NEXT: movw %di, 12(%eax)
+; X32-NOCMOV-NEXT: movw %bp, 10(%eax)
+; X32-NOCMOV-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 2-byte Folded Reload
+; X32-NOCMOV-NEXT: movw %cx, 8(%eax)
+; X32-NOCMOV-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 2-byte Folded Reload
+; X32-NOCMOV-NEXT: movw %cx, 6(%eax)
+; X32-NOCMOV-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 2-byte Folded Reload
+; X32-NOCMOV-NEXT: movw %cx, 4(%eax)
+; X32-NOCMOV-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 2-byte Folded Reload
; X32-NOCMOV-NEXT: movw %cx, 2(%eax)
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 2-byte Folded Reload
; X32-NOCMOV-NEXT: movw %cx, (%eax)
; X32-NOCMOV-NEXT: addl $12, %esp
; X32-NOCMOV-NEXT: popl %esi
@@ -2253,79 +1593,79 @@ define <8 x half> @test_ctselect_v8f16(i1 %cond, <8 x half> %a, <8 x half> %b) #
define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat> %b) #0 {
; X64-LABEL: test_ctselect_v8bf16:
; X64: # %bb.0:
-; X64-NEXT: movq %xmm1, %rax
-; X64-NEXT: movq %xmm0, %rcx
-; X64-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1,1]
-; X64-NEXT: movq %xmm1, %rsi
+; X64-NEXT: pushq %rbp
+; X64-NEXT: pushq %r15
+; X64-NEXT: pushq %r14
+; X64-NEXT: pushq %r13
+; X64-NEXT: pushq %r12
+; X64-NEXT: pushq %rbx
+; X64-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; X64-NEXT: movq %xmm0, %rdi
; X64-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
-; X64-NEXT: movq %xmm0, %r9
-; X64-NEXT: movq %r9, %rdx
-; X64-NEXT: movl %esi, %r8d
-; X64-NEXT: shrl $16, %r8d
-; X64-NEXT: movl %r9d, %r10d
-; X64-NEXT: shrl $16, %r10d
-; X64-NEXT: xorl %r8d, %r10d
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl %edi, %r10d
-; X64-NEXT: xorl %r8d, %r10d
-; X64-NEXT: movq %r9, %r8
-; X64-NEXT: xorl %esi, %r9d
-; X64-NEXT: andl %edi, %r9d
-; X64-NEXT: shll $16, %r10d
-; X64-NEXT: xorl %esi, %r9d
-; X64-NEXT: movzwl %r9w, %r9d
-; X64-NEXT: orl %r10d, %r9d
-; X64-NEXT: movq %rsi, %r10
-; X64-NEXT: shrq $48, %r10
-; X64-NEXT: shrq $48, %r8
-; X64-NEXT: xorl %r10d, %r8d
-; X64-NEXT: andl %edi, %r8d
-; X64-NEXT: xorl %r10d, %r8d
-; X64-NEXT: shrq $32, %rsi
-; X64-NEXT: shrq $32, %rdx
-; X64-NEXT: xorl %esi, %edx
-; X64-NEXT: andl %edi, %edx
-; X64-NEXT: xorl %esi, %edx
-; X64-NEXT: movq %rcx, %rsi
-; X64-NEXT: shll $16, %r8d
-; X64-NEXT: movzwl %dx, %edx
-; X64-NEXT: orl %r8d, %edx
-; X64-NEXT: movl %eax, %r8d
-; X64-NEXT: shrl $16, %r8d
-; X64-NEXT: shlq $32, %rdx
-; X64-NEXT: orq %r9, %rdx
-; X64-NEXT: movl %ecx, %r9d
+; X64-NEXT: movq %xmm0, %r14
+; X64-NEXT: movl %r14d, %eax
+; X64-NEXT: shrl $16, %eax
+; X64-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; X64-NEXT: movq %xmm1, %r10
+; X64-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1,1]
+; X64-NEXT: movq %xmm1, %r11
+; X64-NEXT: movl %r11d, %r9d
; X64-NEXT: shrl $16, %r9d
-; X64-NEXT: xorl %r8d, %r9d
-; X64-NEXT: andl %edi, %r9d
-; X64-NEXT: xorl %r8d, %r9d
-; X64-NEXT: movq %rcx, %r8
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: andl %edi, %ecx
-; X64-NEXT: shll $16, %r9d
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: movzwl %cx, %ecx
-; X64-NEXT: orl %r9d, %ecx
-; X64-NEXT: movq %rax, %r9
+; X64-NEXT: movq %r11, %rdx
+; X64-NEXT: shrq $48, %rdx
+; X64-NEXT: movq %r14, %r12
+; X64-NEXT: shrq $48, %r12
+; X64-NEXT: movq %r11, %rbx
+; X64-NEXT: shrq $32, %rbx
+; X64-NEXT: movq %r14, %r13
+; X64-NEXT: shrq $32, %r13
+; X64-NEXT: movl %edi, %ebp
+; X64-NEXT: shrl $16, %ebp
+; X64-NEXT: movl %r10d, %esi
+; X64-NEXT: shrl $16, %esi
+; X64-NEXT: movq %r10, %rcx
+; X64-NEXT: shrq $48, %rcx
+; X64-NEXT: movq %rdi, %r15
+; X64-NEXT: shrq $48, %r15
+; X64-NEXT: movq %r10, %r8
+; X64-NEXT: shrq $32, %r8
+; X64-NEXT: movq %rdi, %rax
; X64-NEXT: shrq $32, %rax
-; X64-NEXT: shrq $32, %rsi
-; X64-NEXT: shrq $48, %r9
-; X64-NEXT: shrq $48, %r8
-; X64-NEXT: xorl %r9d, %r8d
-; X64-NEXT: andl %edi, %r8d
-; X64-NEXT: xorl %eax, %esi
-; X64-NEXT: andl %edi, %esi
-; X64-NEXT: xorl %r9d, %r8d
-; X64-NEXT: xorl %eax, %esi
-; X64-NEXT: shll $16, %r8d
-; X64-NEXT: movzwl %si, %eax
-; X64-NEXT: orl %r8d, %eax
-; X64-NEXT: shlq $32, %rax
-; X64-NEXT: orq %rcx, %rax
-; X64-NEXT: movq %rax, %xmm0
-; X64-NEXT: movq %rdx, %xmm1
+; X64-NEXT: testb $1, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Folded Reload
+; X64-NEXT: cmovnel %eax, %r8d
+; X64-NEXT: cmovnel %r15d, %ecx
+; X64-NEXT: cmovnel %edi, %r10d
+; X64-NEXT: cmovnel %ebp, %esi
+; X64-NEXT: cmovnel %r13d, %ebx
+; X64-NEXT: cmovnel %r12d, %edx
+; X64-NEXT: cmovnel %r14d, %r11d
+; X64-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; X64-NEXT: cmovnel %eax, %r9d
+; X64-NEXT: shll $16, %r9d
+; X64-NEXT: movzwl %r11w, %eax
+; X64-NEXT: orl %r9d, %eax
+; X64-NEXT: shll $16, %edx
+; X64-NEXT: movzwl %bx, %edi
+; X64-NEXT: orl %edx, %edi
+; X64-NEXT: shlq $32, %rdi
+; X64-NEXT: orq %rax, %rdi
+; X64-NEXT: shll $16, %esi
+; X64-NEXT: movzwl %r10w, %eax
+; X64-NEXT: orl %esi, %eax
+; X64-NEXT: shll $16, %ecx
+; X64-NEXT: movzwl %r8w, %edx
+; X64-NEXT: orl %ecx, %edx
+; X64-NEXT: shlq $32, %rdx
+; X64-NEXT: orq %rax, %rdx
+; X64-NEXT: movq %rdx, %xmm0
+; X64-NEXT: movq %rdi, %xmm1
; X64-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X64-NEXT: popq %rbx
+; X64-NEXT: popq %r12
+; X64-NEXT: popq %r13
+; X64-NEXT: popq %r14
+; X64-NEXT: popq %r15
+; X64-NEXT: popq %rbp
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_v8bf16:
@@ -2343,7 +1683,8 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps (%esp)
; X32-NEXT: calll __truncsfbf2
-; X32-NEXT: movl %eax, %ebp
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps (%esp)
; X32-NEXT: calll __truncsfbf2
@@ -2377,6 +1718,10 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps (%esp)
; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: movl %eax, %esi
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
; X32-NEXT: # kill: def $ax killed $ax def $eax
; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NEXT: flds {{[0-9]+}}(%esp)
@@ -2401,73 +1746,48 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps (%esp)
; X32-NEXT: calll __truncsfbf2
-; X32-NEXT: movl %eax, %esi
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps (%esp)
-; X32-NEXT: calll __truncsfbf2
-; X32-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: movl %eax, %ebx
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps (%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X32-NEXT: calll __truncsfbf2
; X32-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: andb $1, %cl
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT: movzbl %cl, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: andl %ecx, %eax
-; X32-NEXT: movl %eax, %ebx
-; X32-NEXT: movl %ebp, %eax
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %eax
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %ebp
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %edx
-; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %edx
-; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %edx
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %edi
-; X32-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %esi
-; X32-NEXT: movl %esi, %ecx
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %eax, %ebx
; X32-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: cmovnel %eax, %edi
+; X32-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: cmovnel %eax, %edi
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: cmovnel %eax, %esi
+; X32-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: cmovnel %eax, %esi
; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT: movw %bx, 14(%ecx)
-; X32-NEXT: movw %ax, 12(%ecx)
-; X32-NEXT: movw %dx, 10(%ecx)
-; X32-NEXT: movw %si, 8(%ecx)
-; X32-NEXT: movw %di, 6(%ecx)
-; X32-NEXT: movw %bp, 4(%ecx)
; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT: movw %ax, 2(%ecx)
+; X32-NEXT: cmovnel %eax, %ebx
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: cmovnel %eax, %edx
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT: cmovnel %ecx, %eax
+; X32-NEXT: movw %ax, 14(%ebp)
+; X32-NEXT: movw %dx, 12(%ebp)
+; X32-NEXT: movw %bx, 10(%ebp)
+; X32-NEXT: movw %si, 8(%ebp)
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: movw %ax, 6(%ebp)
+; X32-NEXT: movw %di, 4(%ebp)
; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT: movw %ax, (%ecx)
-; X32-NEXT: movl %ecx, %eax
+; X32-NEXT: movw %ax, 2(%ebp)
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: movw %ax, (%ebp)
+; X32-NEXT: movl %ebp, %eax
; X32-NEXT: addl $60, %esp
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
@@ -2490,17 +1810,25 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: movl %eax, %ebp
+; X32-NOCMOV-NEXT: movl %eax, %edi
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl %eax, %ebx
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %ch
+; X32-NOCMOV-NEXT: testb $1, %ch
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %ch
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
@@ -2509,13 +1837,15 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl %eax, %esi
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
@@ -2524,13 +1854,15 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl %eax, %esi
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
@@ -2539,7 +1871,8 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: movl %eax, %edi
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
@@ -2548,73 +1881,50 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: movl %eax, %esi
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl %eax, %ebx
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: andb $1, %cl
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: movzbl %cl, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl %ecx, %eax
-; X32-NOCMOV-NEXT: movl %eax, %ebx
-; X32-NOCMOV-NEXT: movl %ebp, %eax
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %eax
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %dl
; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %ebp
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %edx
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %edi
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %esi
-; X32-NOCMOV-NEXT: movl %esi, %ecx
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movw %si, 14(%eax)
; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NOCMOV-NEXT: movw %bx, 14(%ecx)
-; X32-NOCMOV-NEXT: movw %ax, 12(%ecx)
-; X32-NOCMOV-NEXT: movw %dx, 10(%ecx)
-; X32-NOCMOV-NEXT: movw %si, 8(%ecx)
-; X32-NOCMOV-NEXT: movw %di, 6(%ecx)
-; X32-NOCMOV-NEXT: movw %bp, 4(%ecx)
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: movw %ax, 2(%ecx)
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: movw %ax, (%ecx)
-; X32-NOCMOV-NEXT: movl %ecx, %eax
+; X32-NOCMOV-NEXT: movw %si, 12(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %cx, 10(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %cx, 8(%eax)
+; X32-NOCMOV-NEXT: movw %dx, 6(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %cx, 4(%eax)
+; X32-NOCMOV-NEXT: movw %bx, 2(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %cx, (%eax)
; X32-NOCMOV-NEXT: addl $60, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
@@ -2628,39 +1938,45 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
define float @test_ctselect_f32_nan_inf(i1 %cond) #0 {
; X64-LABEL: test_ctselect_f32_nan_inf:
; X64: # %bb.0:
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl $4194304, %edi # imm = 0x400000
-; X64-NEXT: xorl $2139095040, %edi # imm = 0x7F800000
-; X64-NEXT: movd %edi, %xmm0
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movl $2143289344, %eax # imm = 0x7FC00000
+; X64-NEXT: movl $2139095040, %ecx # imm = 0x7F800000
+; X64-NEXT: cmovnel %eax, %ecx
+; X64-NEXT: movd %ecx, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f32_nan_inf:
; X32: # %bb.0:
+; X32-NEXT: pushl %edi
+; X32-NEXT: pushl %esi
; X32-NEXT: pushl %eax
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl $4194304, %eax # imm = 0x400000
-; X32-NEXT: xorl $2139095040, %eax # imm = 0x7F800000
-; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: sete %al
+; X32-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %ecx
+; X32-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %edx
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %esi, (%esp)
; X32-NEXT: flds (%esp)
-; X32-NEXT: popl %eax
+; X32-NEXT: addl $4, %esp
+; X32-NEXT: popl %esi
+; X32-NEXT: popl %edi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f32_nan_inf:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: pushl %eax
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl $4194304, %eax # imm = 0x400000
-; X32-NOCMOV-NEXT: xorl $2139095040, %eax # imm = 0x7F800000
-; X32-NOCMOV-NEXT: movl %eax, (%esp)
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %ecx
+; X32-NOCMOV-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %edx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, (%esp)
; X32-NOCMOV-NEXT: flds (%esp)
-; X32-NOCMOV-NEXT: popl %eax
+; X32-NOCMOV-NEXT: addl $4, %esp
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: retl
%result = call float @llvm.ct.select.f32(i1 %cond, float 0x7FF8000000000000, float 0x7FF0000000000000)
ret float %result
@@ -2669,72 +1985,63 @@ define float @test_ctselect_f32_nan_inf(i1 %cond) #0 {
define double @test_ctselect_f64_nan_inf(i1 %cond) #0 {
; X64-LABEL: test_ctselect_f64_nan_inf:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negq %rdi
-; X64-NEXT: movabsq $2251799813685248, %rax # imm = 0x8000000000000
-; X64-NEXT: andq %rdi, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movabsq $9221120237041090560, %rax # imm = 0x7FF8000000000000
; X64-NEXT: movabsq $9218868437227405312, %rcx # imm = 0x7FF0000000000000
-; X64-NEXT: xorq %rax, %rcx
+; X64-NEXT: cmovneq %rax, %rcx
; X64-NEXT: movq %rcx, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f64_nan_inf:
; X32: # %bb.0:
+; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $16, %esp
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
+; X32-NEXT: subl $24, %esp
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
-; X32-NEXT: fstpl {{[0-9]+}}(%esp)
; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X32-NEXT: sete %al
+; X32-NEXT: fxch %st(1)
+; X32-NEXT: fstpl {{[0-9]+}}(%esp)
; X32-NEXT: fstpl (%esp)
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl (%esp), %esi
-; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: andl %eax, %esi
-; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: movl %esi, (%esp)
+; X32-NEXT: movl (%esp), %edx
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl %eax, %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NEXT: fldl (%esp)
-; X32-NEXT: addl $16, %esp
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT: fldl {{[0-9]+}}(%esp)
+; X32-NEXT: addl $24, %esp
; X32-NEXT: popl %esi
+; X32-NEXT: popl %edi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f64_nan_inf:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $16, %esp
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: subl $24, %esp
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
-; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: fxch %st(1)
+; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstpl (%esp)
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl (%esp), %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: andl %eax, %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: movl %esi, (%esp)
+; X32-NOCMOV-NEXT: movl (%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl %eax, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fldl (%esp)
-; X32-NOCMOV-NEXT: addl $16, %esp
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fldl {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: addl $24, %esp
; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: retl
%result = call double @llvm.ct.select.f64(i1 %cond, double 0x7FF8000000000000, double 0x7FF0000000000000)
ret double %result
@@ -2743,7 +2050,6 @@ define double @test_ctselect_f64_nan_inf(i1 %cond) #0 {
; Declare the intrinsics
declare i1 @llvm.ct.select.i1(i1, i1, i1)
declare i8 @llvm.ct.select.i8(i1, i8, i8)
-declare b8 @llvm.ct.select.b8(i1, b8, b8)
declare i16 @llvm.ct.select.i16(i1, i16, i16)
declare i32 @llvm.ct.select.i32(i1, i32, i32)
declare i64 @llvm.ct.select.i64(i1, i64, i64)
@@ -2760,7 +2066,6 @@ declare <4 x i32> @llvm.ct.select.v4i32(i1, <4 x i32>, <4 x i32>)
declare <2 x i64> @llvm.ct.select.v2i64(i1, <2 x i64>, <2 x i64>)
declare <8 x i16> @llvm.ct.select.v8i16(i1, <8 x i16>, <8 x i16>)
declare <16 x i8> @llvm.ct.select.v16i8(i1, <16 x i8>, <16 x i8>)
-declare <16 x b8> @llvm.ct.select.v16b8(i1, <16 x b8>, <16 x b8>)
declare <4 x float> @llvm.ct.select.v4f32(i1, <4 x float>, <4 x float>)
declare <2 x double> @llvm.ct.select.v2f64(i1, <2 x double>, <2 x double>)
declare <8 x i32> @llvm.ct.select.v8i32(i1, <8 x i32>, <8 x i32>)
More information about the llvm-branch-commits
mailing list