[llvm-branch-commits] [clang] [llvm] [ConstantTime] Native ct.select support for X86 and i386 (PR #166704)
Julius Alexandre via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Fri May 22 18:10:16 PDT 2026
https://github.com/wizardengineer updated https://github.com/llvm/llvm-project/pull/166704
>From e0c4d094ba28bd501961e498aefa92376fe529eb Mon Sep 17 00:00:00 2001
From: wizardengineer <juliuswoosebert at gmail.com>
Date: Wed, 5 Nov 2025 10:56:34 -0500
Subject: [PATCH 1/5] [ConstantTime][Clang] Add __builtin_ct_select for
constant-time selection
---
clang/docs/LanguageExtensions.rst | 44 ++
clang/include/clang/Basic/Builtins.td | 8 +
clang/lib/CodeGen/CGBuiltin.cpp | 13 +
clang/lib/Sema/SemaChecking.cpp | 64 ++
.../test/Sema/builtin-ct-select-edge-cases.c | 373 ++++++++++
clang/test/Sema/builtin-ct-select.c | 683 ++++++++++++++++++
6 files changed, 1185 insertions(+)
create mode 100644 clang/test/Sema/builtin-ct-select-edge-cases.c
create mode 100644 clang/test/Sema/builtin-ct-select.c
diff --git a/clang/docs/LanguageExtensions.rst b/clang/docs/LanguageExtensions.rst
index 03cb02deb5e7f..6f5cd5f95cdb0 100644
--- a/clang/docs/LanguageExtensions.rst
+++ b/clang/docs/LanguageExtensions.rst
@@ -7332,3 +7332,47 @@ Clang fails to reject some code that should be rejected. e.g.,
// own initializer rather than rejecting the code with an undeclared identifier
// diagnostic.
auto x = x;
+
+.. _langext-__builtin_ct_select:
+
+``__builtin_ct_select``
+-----------------------
+
+``__builtin_ct_select`` performs a constant-time conditional selection between
+two values. Unlike the ternary operator ``?:``, this builtin is designed to
+execute in constant time regardless of the condition value, making it suitable
+for cryptographic and security-sensitive code where timing side-channels must
+be avoided.
+
+**Syntax**:
+
+.. code-block:: c++
+
+ __builtin_ct_select(condition, true_value, false_value)
+
+**Examples**:
+
+.. code-block:: c++
+
+ // Select between two integers
+ int result = __builtin_ct_select(secret_bit, value_a, value_b);
+
+ // Select between two pointers
+ int *ptr = __builtin_ct_select(condition, ptr_a, ptr_b);
+
+ // Select between two floating-point values
+ double d = __builtin_ct_select(flag, 1.0, 2.0);
+
+**Description**:
+
+The first argument is an integer condition that is converted to a boolean
+(non-zero is true, zero is false). The second and third arguments must have
+the same scalar or vector type. The builtin returns the second argument if
+the condition is true, otherwise the third argument.
+
+The operation is guaranteed to be lowered to constant-time machine code that
+does not branch on the condition value, preventing timing-based side-channel
+attacks.
+
+Query for this feature with ``__has_builtin(__builtin_ct_select)``.
+
diff --git a/clang/include/clang/Basic/Builtins.td b/clang/include/clang/Basic/Builtins.td
index 40ec94ab75046..389754a37f7e3 100644
--- a/clang/include/clang/Basic/Builtins.td
+++ b/clang/include/clang/Basic/Builtins.td
@@ -5810,3 +5810,11 @@ def CountedByRef : Builtin {
let Attributes = [NoThrow, CustomTypeChecking];
let Prototype = "int(...)";
}
+
+// Constant-time select builtin
+def CtSelect : Builtin {
+ let Spellings = ["__builtin_ct_select"];
+ let Attributes = [NoThrow, Const, UnevaluatedArguments,
+ ConstIgnoringExceptions, CustomTypeChecking];
+ let Prototype = "void(...)";
+}
diff --git a/clang/lib/CodeGen/CGBuiltin.cpp b/clang/lib/CodeGen/CGBuiltin.cpp
index cac1628e68721..f69390b4ace57 100644
--- a/clang/lib/CodeGen/CGBuiltin.cpp
+++ b/clang/lib/CodeGen/CGBuiltin.cpp
@@ -6668,6 +6668,19 @@ RValue CodeGenFunction::EmitBuiltinExpr(const GlobalDecl GD, unsigned BuiltinID,
auto Str = CGM.GetAddrOfConstantCString(Name, "");
return RValue::get(Str.getPointer());
}
+ case Builtin::BI__builtin_ct_select: {
+ auto *Cond = EmitScalarExpr(E->getArg(0));
+ auto *A = EmitScalarExpr(E->getArg(1));
+ auto *B = EmitScalarExpr(E->getArg(2));
+
+ if (Cond->getType()->getIntegerBitWidth() != 1)
+ Cond = Builder.CreateICmpNE(
+ Cond, llvm::ConstantInt::get(Cond->getType(), 0), "cond.bool");
+
+ llvm::Function *Fn =
+ CGM.getIntrinsic(llvm::Intrinsic::ct_select, {A->getType()});
+ return RValue::get(Builder.CreateCall(Fn, {Cond, A, B}));
+ }
}
// If this is an alias for a lib function (e.g. __builtin_sin), emit
diff --git a/clang/lib/Sema/SemaChecking.cpp b/clang/lib/Sema/SemaChecking.cpp
index cc834bbee23c4..e5a15c84de8d3 100644
--- a/clang/lib/Sema/SemaChecking.cpp
+++ b/clang/lib/Sema/SemaChecking.cpp
@@ -3928,6 +3928,70 @@ Sema::CheckBuiltinFunctionCall(FunctionDecl *FDecl, unsigned BuiltinID,
if (BuiltinCountedByRef(TheCall))
return ExprError();
break;
+
+ case Builtin::BI__builtin_ct_select: {
+ if (TheCall->getNumArgs() != 3) {
+ // Simple argument count check without complex diagnostics
+ if (TheCall->getNumArgs() < 3) {
+ return Diag(TheCall->getEndLoc(),
+ diag::err_typecheck_call_too_few_args_at_least)
+ << 0 << 3 << TheCall->getNumArgs() << 0
+ << TheCall->getCallee()->getSourceRange();
+ } else {
+ return Diag(TheCall->getEndLoc(),
+ diag::err_typecheck_call_too_many_args)
+ << 0 << 3 << TheCall->getNumArgs() << 0
+ << TheCall->getCallee()->getSourceRange();
+ }
+ }
+ auto *Cond = TheCall->getArg(0);
+ auto *A = TheCall->getArg(1);
+ auto *B = TheCall->getArg(2);
+
+ QualType CondTy = Cond->getType();
+ if (!CondTy->isIntegerType()) {
+ return Diag(Cond->getBeginLoc(), diag::err_typecheck_cond_expect_scalar)
+ << CondTy << Cond->getSourceRange();
+ }
+
+ ExprResult ARes = DefaultFunctionArrayLvalueConversion(A);
+ ExprResult BRes = DefaultFunctionArrayLvalueConversion(B);
+ if (ARes.isInvalid() || BRes.isInvalid())
+ return ExprError();
+
+ A = ARes.get();
+ B = BRes.get();
+ TheCall->setArg(1, A);
+ TheCall->setArg(2, B);
+
+ QualType ATy = A->getType();
+ QualType BTy = B->getType();
+
+ // check for scalar or vector scalar type
+ if ((!ATy->isScalarType() && !ATy->isVectorType()) ||
+ (!BTy->isScalarType() && !BTy->isVectorType())) {
+ return Diag(A->getBeginLoc(),
+ diag::err_typecheck_cond_incompatible_operands)
+ << ATy << BTy << A->getSourceRange() << B->getSourceRange();
+ }
+
+ // Check if both operands have the same type or can be implicitly converted
+ if (!Context.hasSameType(ATy, BTy)) {
+ // For non-arithmetic types, they must be exactly the same
+ return Diag(A->getBeginLoc(),
+ diag::err_typecheck_cond_incompatible_operands)
+ << ATy << BTy << A->getSourceRange() << B->getSourceRange();
+ }
+
+ QualType ResultTy = ATy;
+ ExprResult CondRes = PerformContextuallyConvertToBool(Cond);
+ if (CondRes.isInvalid())
+ return ExprError();
+
+ TheCall->setArg(0, CondRes.get());
+ TheCall->setType(ResultTy);
+ return TheCall;
+ }
}
if (getLangOpts().HLSL && HLSL().CheckBuiltinFunctionCall(BuiltinID, TheCall))
diff --git a/clang/test/Sema/builtin-ct-select-edge-cases.c b/clang/test/Sema/builtin-ct-select-edge-cases.c
new file mode 100644
index 0000000000000..167b19bf20663
--- /dev/null
+++ b/clang/test/Sema/builtin-ct-select-edge-cases.c
@@ -0,0 +1,373 @@
+// RUN: %clang_cc1 -fsyntax-only -verify %s
+// RUN: %clang_cc1 -fsyntax-only -verify %s -fexperimental-new-constant-interpreter
+
+// Test with various condition expressions
+int test_conditional_expressions(int x, int y, int a, int b) {
+ // Logical expressions
+ int result1 = __builtin_ct_select(x && y, a, b);
+ int result2 = __builtin_ct_select(x || y, a, b);
+ int result3 = __builtin_ct_select(!x, a, b);
+
+ // Comparison expressions
+ int result4 = __builtin_ct_select(x == y, a, b);
+ int result5 = __builtin_ct_select(x != y, a, b);
+ int result6 = __builtin_ct_select(x < y, a, b);
+ int result7 = __builtin_ct_select(x > y, a, b);
+ int result8 = __builtin_ct_select(x <= y, a, b);
+ int result9 = __builtin_ct_select(x >= y, a, b);
+
+ // Bitwise expressions
+ int result10 = __builtin_ct_select(x & y, a, b);
+ int result11 = __builtin_ct_select(x | y, a, b);
+ int result12 = __builtin_ct_select(x ^ y, a, b);
+ int result13 = __builtin_ct_select(~x, a, b);
+
+ // Arithmetic expressions
+ int result14 = __builtin_ct_select(x + y, a, b);
+ int result15 = __builtin_ct_select(x - y, a, b);
+ int result16 = __builtin_ct_select(x * y, a, b);
+ int result17 = __builtin_ct_select(x / y, a, b);
+ int result18 = __builtin_ct_select(x % y, a, b);
+
+ return result1 + result2 + result3 + result4 + result5 + result6 + result7 + result8 + result9 + result10 + result11 + result12 + result13 + result14 + result15 + result16 + result17 + result18;
+}
+
+// Test with extreme values
+int test_extreme_values(int cond) {
+ // Maximum and minimum values
+ int max_int = __builtin_ct_select(cond, __INT_MAX__, -__INT_MAX__ - 1);
+
+ // Very large numbers
+ long long max_ll = __builtin_ct_select(cond, __LONG_LONG_MAX__, -__LONG_LONG_MAX__ - 1);
+
+ // Floating point extremes
+ float max_float = __builtin_ct_select(cond, __FLT_MAX__, -__FLT_MAX__);
+ double max_double = __builtin_ct_select(cond, __DBL_MAX__, -__DBL_MAX__);
+
+ return max_int;
+}
+
+// Test with zero and negative zero
+int test_zero_values(int cond) {
+ // Integer zeros
+ int zero_int = __builtin_ct_select(cond, 0, -0);
+
+ // Floating point zeros
+ float zero_float = __builtin_ct_select(cond, 0.0f, -0.0f);
+ double zero_double = __builtin_ct_select(cond, 0.0, -0.0);
+
+ return zero_int;
+}
+
+// Test with infinity and NaN
+int test_special_float_values(int cond) {
+ // Infinity
+ float inf_float = __builtin_ct_select(cond, __builtin_inff(), -__builtin_inff());
+ double inf_double = __builtin_ct_select(cond, __builtin_inf(), -__builtin_inf());
+
+ // NaN
+ float nan_float = __builtin_ct_select(cond, __builtin_nanf(""), __builtin_nanf(""));
+ double nan_double = __builtin_ct_select(cond, __builtin_nan(""), __builtin_nan(""));
+
+ return 0;
+}
+
+// Test with complex pointer scenarios
+int test_pointer_edge_cases(int cond) {
+ int arr[10];
+ int *ptr1 = arr;
+ int *ptr2 = arr + 5;
+
+ // Array pointers
+ int *result1 = __builtin_ct_select(cond, ptr1, ptr2);
+
+ // Pointer arithmetic
+ int *result2 = __builtin_ct_select(cond, arr + 1, arr + 2);
+
+ // NULL vs non-NULL
+ int *result3 = __builtin_ct_select(cond, ptr1, (int*)0);
+
+ // Different pointer types (should fail)
+ float *fptr = (float*)0;
+ int *result4 = __builtin_ct_select(cond, ptr1, fptr); // expected-error {{incompatible operand types ('int *' and 'float *')}}
+
+ return *result1;
+}
+
+// Test with function pointers
+int func1(int x) { return x; }
+int func2(int x) { return x * 2; }
+float func3(float x) { return x; }
+
+int test_function_pointers(int cond, int x) {
+ // Same signature function pointer
+ int (*fptr)(int) = __builtin_ct_select(cond, &func1, &func2);
+
+ // Different signature function pointers (should fail)
+ int (*bad_fptr)(int) = __builtin_ct_select(cond, &func1, &func3); // expected-error {{incompatible operand types ('int (*)(int)' and 'float (*)(float)')}}
+
+ return fptr(x);
+}
+
+// Test with void pointers
+void *test_void_pointers(int cond, void *a, void *b) {
+ return __builtin_ct_select(cond, a, b);
+}
+
+// Test with const/volatile qualifiers
+int test_qualifiers(int cond) {
+ const int ca = 10;
+ const int cb = 20;
+ volatile int va = 30;
+ volatile int vb = 40;
+ const volatile int cva = 50;
+ const volatile int cvb = 60;
+
+ // const to const
+ const int result1 = __builtin_ct_select(cond, ca, cb);
+
+ // volatile to volatile
+ volatile int result2 = __builtin_ct_select(cond, va, vb);
+
+ // const volatile to const volatile
+ const volatile int result3 = __builtin_ct_select(cond, cva, cvb);
+
+ return result1 + result2 + result3;
+}
+
+// Test with arrays (should fail as they're not arithmetic or pointer)
+int test_arrays(int cond) {
+ int arr1[5] = {1, 2, 3, 4, 5};
+ int arr2[5] = {6, 7, 8, 9, 10};
+
+ // This should fail??
+ int *result = __builtin_ct_select(cond, arr1, arr2);
+
+ return result[0];
+}
+
+// Test with structures (should fail)
+struct Point {
+ int x, y;
+};
+
+struct Point test_structs(int cond) {
+ struct Point p1 = {1, 2};
+ struct Point p2 = {3, 4};
+
+ return __builtin_ct_select(cond, p1, p2); // expected-error {{incompatible operand types ('struct Point' and 'struct Point')}}
+}
+
+// Test with unions (should fail)
+union Data {
+ int i;
+ float f;
+};
+
+union Data test_unions(int cond) {
+ union Data d1 = {.i = 10};
+ union Data d2 = {.i = 20};
+
+ return __builtin_ct_select(cond, d1, d2); // expected-error {{incompatible operand types ('union Data' and 'union Data')}}
+}
+
+// Test with bit fields (should work as they're integers)
+struct BitField {
+ int a : 4;
+ int b : 4;
+};
+
+int test_bit_fields(int cond) {
+ struct BitField bf1 = {1, 2};
+ struct BitField bf2 = {3, 4};
+
+ // Individual bit fields should work
+ int result1 = __builtin_ct_select(cond, bf1.a, bf2.a);
+ int result2 = __builtin_ct_select(cond, bf1.b, bf2.b);
+
+ return result1 + result2;
+}
+
+// Test with designated initializers
+int test_designated_init(int cond) {
+ int arr1[3] = {[0] = 1, [1] = 2, [2] = 3};
+ int arr2[3] = {[0] = 4, [1] = 5, [2] = 6};
+
+ // Access specific elements
+ int result1 = __builtin_ct_select(cond, arr1[0], arr2[0]);
+ int result2 = __builtin_ct_select(cond, arr1[1], arr2[1]);
+
+ return result1 + result2;
+}
+
+// Test with complex expressions in arguments
+int complex_expr(int x) { return x * x; }
+
+int test_complex_arguments(int cond, int x, int y) {
+ // Function calls as arguments
+ int result1 = __builtin_ct_select(cond, complex_expr(x), complex_expr(y));
+
+ // Ternary operator as arguments
+ int result2 = __builtin_ct_select(cond, x > 0 ? x : -x, y > 0 ? y : -y);
+
+ // Compound literals
+ int result3 = __builtin_ct_select(cond, (int){x}, (int){y});
+
+ return result1 + result2 + result3;
+}
+
+// Test with preprocessor macros
+#define MACRO_A 42
+#define MACRO_B 24
+#define MACRO_COND(x) (x > 0)
+
+int test_macros(int x) {
+ int result1 = __builtin_ct_select(MACRO_COND(x), MACRO_A, MACRO_B);
+
+ // Nested macros
+ #define NESTED_SELECT(c, a, b) __builtin_ct_select(c, a, b)
+ int result2 = NESTED_SELECT(x, 10, 20);
+
+ return result1 + result2;
+}
+
+// Test with string literals (should fail)
+const char *test_strings(int cond) {
+ return __builtin_ct_select(cond, "hello", "world");
+}
+
+// Test with variable length arrays (VLA)
+int test_vla(int cond, int n) {
+ int vla1[n];
+ int vla2[n];
+
+ // Individual elements should work
+ vla1[0] = 1;
+ vla2[0] = 2;
+ int result = __builtin_ct_select(cond, vla1[0], vla2[0]);
+
+ return result;
+}
+
+// Test with typedef
+typedef int MyInt;
+typedef float MyFloat;
+
+MyInt test_typedef(int cond, MyInt a, MyInt b) {
+ return __builtin_ct_select(cond, a, b);
+}
+
+// Test with different typedef types (should fail)
+MyInt test_different_typedef(int cond, MyInt a, MyFloat b) {
+ return __builtin_ct_select(cond, a, b); // expected-error {{incompatible operand types ('MyInt' (aka 'int') and 'MyFloat' (aka 'float'))}}
+}
+
+// Test with side effects (should be evaluated)
+int side_effect_counter = 0;
+int side_effect_func(int x) {
+ side_effect_counter++;
+ return x;
+}
+
+int test_side_effects(int cond) {
+ // Both arguments should be evaluated
+ int result = __builtin_ct_select(cond, side_effect_func(10), side_effect_func(20));
+ return result;
+}
+
+// Test with goto labels (context where expressions are used)
+int test_goto_context(int cond, int a, int b) {
+ int result = __builtin_ct_select(cond, a, b);
+
+ if (result > 0) {
+ goto positive;
+ } else {
+ goto negative;
+ }
+
+positive:
+ return result;
+
+negative:
+ return -result;
+}
+
+// Test with switch statements
+int test_switch_context(int cond, int a, int b) {
+ int result = __builtin_ct_select(cond, a, b);
+
+ switch (result) {
+ case 0:
+ return 0;
+ case 1:
+ return 1;
+ default:
+ return -1;
+ }
+}
+
+// Test with loops
+int test_loop_context(int cond, int a, int b) {
+ int result = __builtin_ct_select(cond, a, b);
+ int sum = 0;
+
+ for (int i = 0; i < result; i++) {
+ sum += i;
+ }
+
+ return sum;
+}
+
+// Test with recursive functions
+int factorial(int n) {
+ if (n <= 1) return 1;
+ return n * factorial(n - 1);
+}
+
+int test_recursive(int cond, int n) {
+ int result = __builtin_ct_select(cond, n, n + 1);
+ return factorial(result);
+}
+
+// Test with inline functions
+static inline int inline_func(int x) {
+ return x * 2;
+}
+
+int test_inline(int cond, int a, int b) {
+ return __builtin_ct_select(cond, inline_func(a), inline_func(b));
+}
+
+// Test with static variables
+int test_static_vars(int cond) {
+ static int static_a = 10;
+ static int static_b = 20;
+
+ return __builtin_ct_select(cond, static_a, static_b);
+}
+
+// Test with extern variables
+extern int extern_a;
+extern int extern_b;
+
+int test_extern_vars(int cond) {
+ return __builtin_ct_select(cond, extern_a, extern_b);
+}
+
+// Test with register variables
+int test_register_vars(int cond) {
+ register int reg_a = 30;
+ register int reg_b = 40;
+
+ return __builtin_ct_select(cond, reg_a, reg_b);
+}
+
+// Test with thread-local variables (C11)
+#if __STDC_VERSION__ >= 201112L
+_Thread_local int tls_a = 50;
+_Thread_local int tls_b = 60;
+
+int test_tls_vars(int cond) {
+ return __builtin_ct_select(cond, tls_a, tls_b);
+}
+#endif
diff --git a/clang/test/Sema/builtin-ct-select.c b/clang/test/Sema/builtin-ct-select.c
new file mode 100644
index 0000000000000..7f2d9291299d6
--- /dev/null
+++ b/clang/test/Sema/builtin-ct-select.c
@@ -0,0 +1,683 @@
+// RUN: %clang_cc1 -emit-llvm -o - %s | FileCheck %s
+
+// Test integer types
+int test_int(int cond, int a, int b) {
+ // CHECK-LABEL: define {{.*}} @test_int
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+long long test_long(int cond, long long a, long long b) {
+ // CHECK-LABEL: define {{.*}} @test_long
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call i64 @llvm.ct.select.i64(i1 [[COND]], i64 %{{.*}}, i64 %{{.*}})
+ // CHECK: ret i64 [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+short test_short(int cond, short a, short b) {
+ // CHECK-LABEL: define {{.*}} @test_short
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call i16 @llvm.ct.select.i16(i1 [[COND]], i16 %{{.*}}, i16 %{{.*}})
+ // CHECK: ret i16 [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+unsigned char test_uchar(int cond, unsigned char a, unsigned char b) {
+ // CHECK-LABEL: define {{.*}} @test_uchar
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call i8 @llvm.ct.select.i8(i1 [[COND]], i8 %{{.*}}, i8 %{{.*}})
+ // CHECK: ret i8 [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+long long test_longlong(int cond, long long a, long long b) {
+ // CHECK-LABEL: define {{.*}} @test_longlong
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call i64 @llvm.ct.select.i64(i1 [[COND]], i64 %{{.*}}, i64 %{{.*}})
+ // CHECK: ret i64 [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+// Test floating point types
+float test_float(int cond, float a, float b) {
+ // CHECK-LABEL: define {{.*}} @test_float
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call float @llvm.ct.select.f32(i1 [[COND]], float %{{.*}}, float %{{.*}})
+ // CHECK: ret float [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+double test_double(int cond, double a, double b) {
+ // CHECK-LABEL: define {{.*}} @test_double
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call double @llvm.ct.select.f64(i1 [[COND]], double %{{.*}}, double %{{.*}})
+ // CHECK: ret double [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+// Test pointer types
+int *test_pointer(int cond, int *a, int *b) {
+ // CHECK-LABEL: define {{.*}} @test_pointer
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call ptr @llvm.ct.select.p0(i1 [[COND]], ptr %{{.*}}, ptr %{{.*}})
+ // CHECK: ret ptr [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+// Test with different condition types
+int test_char_cond(char cond, int a, int b) {
+ // CHECK-LABEL: define {{.*}} @test_char_cond
+ // CHECK: [[COND:%.*]] = icmp ne i8 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+int test_long_cond(long long cond, int a, int b) {
+ // CHECK-LABEL: define {{.*}} @test_long_cond
+ // CHECK: [[COND:%.*]] = icmp ne i64 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+// Test with boolean condition
+int test_bool_cond(_Bool cond, int a, int b) {
+ // CHECK-LABEL: define {{.*}} @test_bool_cond
+ // CHECK: [[COND:%.*]] = icmp ne i8 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+// Test with constants
+int test_constant_cond(void) {
+ // CHECK-LABEL: define {{.*}} @test_constant_cond
+ // CHECK: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 true, i32 42, i32 24)
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(1, 42, 24);
+}
+
+int test_zero_cond(void) {
+ // CHECK-LABEL: define {{.*}} @test_zero_cond
+ // CHECK: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 false, i32 42, i32 24)
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(0, 42, 24);
+}
+
+// Test type promotion
+int test_promotion(int cond, short a, short b) {
+ // CHECK-LABEL: define {{.*}} @test_promotion
+ // CHECK-DAG: [[A_EXT:%.*]] = sext i16 %{{.*}} to i32
+ // CHECK-DAG: [[B_EXT:%.*]] = sext i16 %{{.*}} to i32
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 [[A_EXT]], i32 [[B_EXT]])
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(cond, (int)a, (int)b);
+}
+
+// Test mixed signedness
+unsigned int test_mixed_signedness(int cond, int a, unsigned int b) {
+ // CHECK-LABEL: define {{.*}} @test_mixed_signedness
+ // CHECK-DAG: [[A_EXT:%.*]] = sext i32 %{{.*}} to i64
+ // CHECK-DAG: [[B_EXT:%.*]] = zext i32 %{{.*}} to i64
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call i64 @llvm.ct.select.i64(i1 [[COND]], i64 [[A_EXT]], i64 [[B_EXT]])
+ // CHECK: [[RESULT_TRUNC:%.*]] = trunc i64 [[RESULT]] to i32
+ // CHECK: ret i32 [[RESULT_TRUNC]]
+ return __builtin_ct_select(cond, (long long)a, (long long)b);
+}
+
+// Test complex expression
+int test_complex_expr_alt(int x, int y) {
+ // CHECK-LABEL: define {{.*}} @test_complex_expr_alt
+ // CHECK-DAG: [[CMP:%.*]] = icmp sgt i32 %{{.*}}, 0
+ // CHECK-DAG: [[ADD:%.*]] = add nsw i32 %{{.*}}, %{{.*}}
+ // CHECK-DAG: [[SUB:%.*]] = sub nsw i32 %{{.*}}, %{{.*}}
+ // Separate the final sequence to ensure proper ordering
+ // CHECK-NEXT: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[CMP]], i32 [[ADD]], i32 [[SUB]])
+ // CHECK-NEXT: ret i32 [[RESULT]]
+ return __builtin_ct_select(x > 0, x + y, x - y);
+}
+
+// Test nested calls
+int test_nested_structured(int cond1, int cond2, int a, int b, int c) {
+ // CHECK-LABEL: define {{.*}} @test_nested_structured
+ // Phase 1: Conditions (order doesn't matter)
+ // CHECK-DAG: [[COND1:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[COND2:%.*]] = icmp ne i32 %{{.*}}, 0
+
+ // Phase 2: Inner select (must happen before outer)
+ // CHECK: [[INNER:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND2]], i32 %{{.*}}, i32 %{{.*}})
+
+ // Phase 3: Outer select (must use inner result)
+ // CHECK: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND1]], i32 [[INNER]], i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(cond1, __builtin_ct_select(cond2, a, b), c);
+}
+
+// Test with function calls
+int helper(int x) { return x * 2; }
+int test_function_calls(int cond, int x, int y) {
+ // CHECK-LABEL: define {{.*}} @test_function_calls
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[CALL1:%.*]] = call i32 @helper(i32 noundef %{{.*}})
+ // CHECK-DAG: [[CALL2:%.*]] = call i32 @helper(i32 noundef %{{.*}})
+ // CHECK-DAG: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 [[CALL1]], i32 [[CALL2]])
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(cond, helper(x), helper(y));
+}
+
+// Test using ct_select as condition for another ct_select
+int test_intrinsic_condition(int cond1, int cond2, int a, int b, int c, int d) {
+ // CHECK-LABEL: define {{.*}} @test_intrinsic_condition
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[INNER_COND:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK-DAG: [[FINAL_COND:%.*]] = icmp ne i32 [[INNER_COND]], 0
+ // CHECK-DAG: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[FINAL_COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(__builtin_ct_select(cond1, cond2, a), b, c);
+}
+
+// Test using comparison result of ct_select as condition
+int test_comparison_condition(int cond, int a, int b, int c, int d) {
+ // CHECK-LABEL: define {{.*}} @test_comparison_condition
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[FIRST_SELECT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: [[CMP:%.*]] = icmp sgt i32 [[FIRST_SELECT]], %{{.*}}
+ // CHECK: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[CMP]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(__builtin_ct_select(cond, a, b) > c, d, a);
+}
+
+// Test using ct_select result in arithmetic as condition
+int test_arithmetic_condition(int cond, int a, int b, int c, int d) {
+ // CHECK-LABEL: define {{.*}} @test_arithmetic_condition
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[FIRST_SELECT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: [[ADD:%.*]] = add nsw i32 [[FIRST_SELECT]], %{{.*}}
+ // CHECK: [[FINAL_COND:%.*]] = icmp ne i32 [[ADD]], 0
+ // CHECK: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[FINAL_COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(__builtin_ct_select(cond, a, b) + c, d, a);
+}
+
+// Test chained ct_select as conditions
+int test_chained_conditions(int cond1, int cond2, int cond3, int a, int b, int c, int d, int e) {
+ // CHECK-LABEL: define {{.*}} @test_chained_conditions
+ // CHECK: [[COND1:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[FIRST:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND1]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK-DAG: [[COND2:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[SECOND:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND2]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK-DAG: [[FINAL_COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[FINAL_COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ int first_select = __builtin_ct_select(cond1, a, b);
+ int second_select = __builtin_ct_select(cond2, first_select, c);
+ return __builtin_ct_select(second_select, d, e);
+}
+
+// Test using ct_select with pointer condition
+//int test_pointer_condition(int *ptr1, int *ptr2, int a, int b, int c) {
+ // NO-CHECK-LABEL: define {{.*}} @test_pointer_condition
+ // NO-CHECK: [[PTR_COND:%.*]] = icmp ne ptr %{{.*}}, null
+ // NO-CHECK: [[PTR_SELECT:%.*]] = call ptr @llvm.ct.select.p0(i1 [[PTR_COND]], ptr %{{.*}}, ptr %{{.*}})
+ // NO-CHECK: [[FINAL_COND:%.*]] = icmp ne ptr [[PTR_SELECT]], null
+ // NO-CHECK: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[FINAL_COND]], i32 %{{.*}}, i32 %{{.*}})
+ // NO-CHECK: ret i32 [[RESULT]]
+// return __builtin_ct_select(__builtin_ct_select(ptr1, ptr1, ptr2), a, b);
+//}
+
+
+// Test using ct_select result in logical operations as condition
+int test_logical_condition(int cond1, int cond2, int a, int b, int c, int d) {
+ // CHECK-LABEL: define {{.*}} @test_logical_condition
+ // CHECK-DAG: [[COND1:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[COND2:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[FIRST_SELECT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND1]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK-DAG: [[SELECT_BOOL:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 %{{.*}}, i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(__builtin_ct_select(cond1, a, b) && cond2, c, d);
+}
+
+// Test multiple levels of ct_select as conditions
+int test_deep_condition_nesting(int cond1, int cond2, int cond3, int a, int b, int c, int d, int e, int f) {
+ // CHECK-LABEL: define {{.*}} @test_deep_condition_nesting
+ // CHECK-DAG: [[COND1:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[COND2:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[INNER1:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND2]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK-DAG: [[INNER1_COND:%.*]] = icmp ne i32 [[INNER1]], 0
+ // CHECK-DAG: [[INNER2:%.*]] = call i32 @llvm.ct.select.i32(i1 [[INNER1_COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK-DAG: [[OUTER:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND1]], i32 [[INNER2]], i32 %{{.*}})
+ // CHECK-DAG: [[FINAL_COND:%.*]] = icmp ne i32 [[OUTER]], 0
+ // CHECK-DAG: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[FINAL_COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(__builtin_ct_select(cond1, __builtin_ct_select(__builtin_ct_select(cond2, a, b), c, d), e), f, a);
+}
+
+// Test ct_select with complex condition expressions
+int test_complex_condition_expr(int x, int y, int z, int a, int b) {
+ // CHECK-LABEL: define {{.*}} @test_complex_condition_expr
+ // CHECK: [[CMP1:%.*]] = icmp sgt i32 %{{.*}}, %{{.*}}
+ // CHECK: [[SELECT1:%.*]] = call i32 @llvm.ct.select.i32(i1 [[CMP1]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: [[CMP2:%.*]] = icmp slt i32 [[SELECT1]], %{{.*}}
+ // CHECK: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[CMP2]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ return __builtin_ct_select(__builtin_ct_select(x > y, x, y) < z, a, b);
+}
+
+// Test vector types - 128-bit vectors
+typedef int __attribute__((vector_size(16))) int4;
+typedef float __attribute__((vector_size(16))) float4;
+typedef short __attribute__((vector_size(16))) short8;
+typedef char __attribute__((vector_size(16))) char16;
+
+int4 test_vector_int4(int cond, int4 a, int4 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_int4
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call <4 x i32> @llvm.ct.select.v4i32(i1 [[COND]], <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
+ // CHECK: ret <4 x i32> [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+float4 test_vector_float4(int cond, float4 a, float4 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_float4
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call <4 x float> @llvm.ct.select.v4f32(i1 [[COND]], <4 x float> %{{.*}}, <4 x float> %{{.*}})
+ // CHECK: ret <4 x float> [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+short8 test_vector_short8(int cond, short8 a, short8 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_short8
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call <8 x i16> @llvm.ct.select.v8i16(i1 [[COND]], <8 x i16> %{{.*}}, <8 x i16> %{{.*}})
+ // CHECK: ret <8 x i16> [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+char16 test_vector_char16(int cond, char16 a, char16 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_char16
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call <16 x i8> @llvm.ct.select.v16i8(i1 [[COND]], <16 x i8> %{{.*}}, <16 x i8> %{{.*}})
+ // CHECK: ret <16 x i8> [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+// Test 256-bit vectors
+typedef int __attribute__((vector_size(32))) int8;
+typedef float __attribute__((vector_size(32))) float8;
+typedef double __attribute__((vector_size(32))) double4;
+
+int8 test_vector_int8(int cond, int8 a, int8 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_int8
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call <8 x i32> @llvm.ct.select.v8i32(i1 [[COND]], <8 x i32> %{{.*}}, <8 x i32> %{{.*}})
+ return __builtin_ct_select(cond, a, b);
+}
+
+float8 test_vector_float8(int cond, float8 a, float8 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_float8
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call <8 x float> @llvm.ct.select.v8f32(i1 [[COND]], <8 x float> %{{.*}}, <8 x float> %{{.*}})
+ return __builtin_ct_select(cond, a, b);
+}
+
+double4 test_vector_double4(int cond, double4 a, double4 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_double4
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call <4 x double> @llvm.ct.select.v4f64(i1 [[COND]], <4 x double> %{{.*}}, <4 x double> %{{.*}})
+ return __builtin_ct_select(cond, a, b);
+}
+
+// Test 512-bit vectors
+typedef int __attribute__((vector_size(64))) int16;
+typedef float __attribute__((vector_size(64))) float16;
+
+int16 test_vector_int16(int cond, int16 a, int16 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_int16
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call <16 x i32> @llvm.ct.select.v16i32(i1 [[COND]], <16 x i32> %{{.*}}, <16 x i32> %{{.*}})
+ return __builtin_ct_select(cond, a, b);
+}
+
+float16 test_vector_float16(int cond, float16 a, float16 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_float16
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call <16 x float> @llvm.ct.select.v16f32(i1 [[COND]], <16 x float> %{{.*}}, <16 x float> %{{.*}})
+ return __builtin_ct_select(cond, a, b);
+}
+
+// Test vector operations with different condition types
+int4 test_vector_char_cond(char cond, int4 a, int4 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_char_cond
+ // CHECK: [[COND:%.*]] = icmp ne i8 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call <4 x i32> @llvm.ct.select.v4i32(i1 [[COND]], <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
+ // CHECK: ret <4 x i32> [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+float4 test_vector_long_cond(long long cond, float4 a, float4 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_long_cond
+ // CHECK: [[COND:%.*]] = icmp ne i64 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call <4 x float> @llvm.ct.select.v4f32(i1 [[COND]], <4 x float> %{{.*}}, <4 x float> %{{.*}})
+ // CHECK: ret <4 x float> [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+// Test vector constants
+int4 test_vector_constant_cond(void) {
+ // CHECK-LABEL: define {{.*}} @test_vector_constant_cond
+ // CHECK: [[RESULT:%.*]] = call <4 x i32> @llvm.ct.select.v4i32(i1 true, <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
+ // CHECK: ret <4 x i32> [[RESULT]]
+ int4 a = {1, 2, 3, 4};
+ int4 b = {5, 6, 7, 8};
+ return __builtin_ct_select(1, a, b);
+}
+
+float4 test_vector_zero_cond(void) {
+ // CHECK-LABEL: define {{.*}} @test_vector_zero_cond
+ // CHECK: [[RESULT:%.*]] = call <4 x float> @llvm.ct.select.v4f32(i1 false, <4 x float> %{{.*}}, <4 x float> %{{.*}})
+ // CHECK: ret <4 x float> [[RESULT]]
+ float4 a = {1.0f, 2.0f, 3.0f, 4.0f};
+ float4 b = {5.0f, 6.0f, 7.0f, 8.0f};
+ return __builtin_ct_select(0, a, b);
+}
+
+// Test nested vector selections
+int4 test_vector_nested(int cond1, int cond2, int4 a, int4 b, int4 c) {
+ // CHECK-LABEL: define {{.*}} @test_vector_nested
+ // CHECK-DAG: [[COND1:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[COND2:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[INNER:%.*]] = call <4 x i32> @llvm.ct.select.v4i32(i1 [[COND2]], <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
+ // CHECK: [[RESULT:%.*]] = call <4 x i32> @llvm.ct.select.v4i32(i1 [[COND1]], <4 x i32> [[INNER]], <4 x i32> %{{.*}})
+ // CHECK: ret <4 x i32> [[RESULT]]
+ return __builtin_ct_select(cond1, __builtin_ct_select(cond2, a, b), c);
+}
+
+// Test vector selection with complex expressions
+float4 test_vector_complex_expr(int x, int y, float4 a, float4 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_complex_expr
+ // CHECK: [[CMP:%.*]] = icmp sgt i32 %{{.*}}, %{{.*}}
+ // CHECK: [[RESULT:%.*]] = call <4 x float> @llvm.ct.select.v4f32(i1 [[CMP]], <4 x float> %{{.*}}, <4 x float> %{{.*}})
+ // CHECK: ret <4 x float> [[RESULT]]
+ return __builtin_ct_select(x > y, a, b);
+}
+
+// Test vector with different element sizes
+typedef long long __attribute__((vector_size(16))) long2;
+typedef double __attribute__((vector_size(16))) double2;
+
+long2 test_vector_long2(int cond, long2 a, long2 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_long2
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call <2 x i64> @llvm.ct.select.v2i64(i1 [[COND]], <2 x i64> %{{.*}}, <2 x i64> %{{.*}})
+ // CHECK: ret <2 x i64> [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+double2 test_vector_double2(int cond, double2 a, double2 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_double2
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call <2 x double> @llvm.ct.select.v2f64(i1 [[COND]], <2 x double> %{{.*}}, <2 x double> %{{.*}})
+ // CHECK: ret <2 x double> [[RESULT]]
+ return __builtin_ct_select(cond, a, b);
+}
+
+// Test mixed vector operations
+int4 test_vector_from_scalar_condition(int4 vec_cond, int4 a, int4 b) {
+ // CHECK-LABEL: define {{.*}} @test_vector_from_scalar_condition
+ // Extract first element and use as condition
+ int scalar_cond = vec_cond[0];
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call <4 x i32> @llvm.ct.select.v4i32(i1 [[COND]], <4 x i32> %{{.*}}, <4 x i32> %{{.*}})
+ // CHECK: ret <4 x i32> [[RESULT]]
+ return __builtin_ct_select(scalar_cond, a, b);
+}
+
+// Test vector chaining
+float4 test_vector_chaining(int cond1, int cond2, int cond3, float4 a, float4 b, float4 c, float4 d) {
+ // CHECK-LABEL: define {{.*}} @test_vector_chaining
+ // CHECK-DAG: [[COND1:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[COND2:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[COND3:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[FIRST:%.*]] = call <4 x float> @llvm.ct.select.v4f32(i1 [[COND1]], <4 x float> %{{.*}}, <4 x float> %{{.*}})
+ // CHECK-DAG: [[SECOND:%.*]] = call <4 x float> @llvm.ct.select.v4f32(i1 [[COND2]], <4 x float> %{{.*}}, <4 x float> %{{.*}})
+ // CHECK-DAG: [[RESULT:%.*]] = call <4 x float> @llvm.ct.select.v4f32(i1 [[COND3]], <4 x float> %{{.*}}, <4 x float> %{{.*}})
+ // CHECK: ret <4 x float> [[RESULT]]
+ float4 first = __builtin_ct_select(cond1, a, b);
+ float4 second = __builtin_ct_select(cond2, first, c);
+ return __builtin_ct_select(cond3, second, d);
+}
+
+// Test special floating point values - NaN
+float test_nan_operands(int cond) {
+ // CHECK-LABEL: define {{.*}} @test_nan_operands
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call float @llvm.ct.select.f32(i1 [[COND]], float %{{.*}}, float 1.000000e+00)
+ // CHECK: ret float [[RESULT]]
+ float nan_val = __builtin_nanf("");
+ return __builtin_ct_select(cond, nan_val, 1.0f);
+}
+
+double test_nan_double_operands(int cond) {
+ // CHECK-LABEL: define {{.*}} @test_nan_double_operands
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call double @llvm.ct.select.f64(i1 [[COND]], double %{{.*}}, double 2.000000e+00)
+ // CHECK: ret double [[RESULT]]
+ double nan_val = __builtin_nan("");
+ return __builtin_ct_select(cond, nan_val, 2.0);
+}
+
+// Test infinity values
+float test_infinity_operands(int cond) {
+ // CHECK-LABEL: define {{.*}} @test_infinity_operands
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call float @llvm.ct.select.f32(i1 [[COND]], float %{{.*}}, float %{{.*}})
+ // CHECK: ret float [[RESULT]]
+ float pos_inf = __builtin_inff();
+ float neg_inf = -__builtin_inff();
+ return __builtin_ct_select(cond, pos_inf, neg_inf);
+}
+
+double test_infinity_double_operands(int cond) {
+ // CHECK-LABEL: define {{.*}} @test_infinity_double_operands
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call double @llvm.ct.select.f64(i1 [[COND]], double %{{.*}}, double %{{.*}})
+ // CHECK: ret double [[RESULT]]
+ double pos_inf = __builtin_inf();
+ double neg_inf = -__builtin_inf();
+ return __builtin_ct_select(cond, pos_inf, neg_inf);
+}
+
+// Test subnormal/denormal values
+float test_subnormal_operands(int cond) {
+ // CHECK-LABEL: define {{.*}} @test_subnormal_operands
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call float @llvm.ct.select.f32(i1 [[COND]], float %{{.*}}, float %{{.*}})
+ // CHECK: ret float [[RESULT]]
+ // Very small subnormal values
+ float subnormal1 = 1e-40f;
+ float subnormal2 = 1e-45f;
+ return __builtin_ct_select(cond, subnormal1, subnormal2);
+}
+
+// Test integer overflow boundaries
+int test_integer_overflow_operands(int cond) {
+ // CHECK-LABEL: define {{.*}} @test_integer_overflow_operands
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ int max_int = __INT_MAX__;
+ int min_int = (-__INT_MAX__ - 1);
+ return __builtin_ct_select(cond, max_int, min_int);
+}
+
+long long test_longlong_overflow_operands(int cond) {
+ // CHECK-LABEL: define {{.*}} @test_longlong_overflow_operands
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call i64 @llvm.ct.select.i64(i1 [[COND]], i64 %{{.*}}, i64 %{{.*}})
+ // CHECK: ret i64 [[RESULT]]
+ long long max_ll = __LONG_LONG_MAX__;
+ long long min_ll = (-__LONG_LONG_MAX__ - 1);
+ return __builtin_ct_select(cond, max_ll, min_ll);
+}
+
+// Test unsigned overflow boundaries
+unsigned int test_unsigned_overflow_operands(int cond) {
+ // CHECK-LABEL: define {{.*}} @test_unsigned_overflow_operands
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ unsigned int max_uint = 4294967295;
+ unsigned int min_uint = 0;
+ return __builtin_ct_select(cond, max_uint, min_uint);
+}
+
+// Test null pointer dereference avoidance
+int* test_null_pointer_operands(int cond, int* valid_ptr) {
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call ptr @llvm.ct.select.p0(i1 [[COND]], ptr %{{.*}}, ptr %{{.*}})
+ // CHECK: ret ptr [[RESULT]]
+ int* null_ptr = (int*)0;
+ return __builtin_ct_select(cond, null_ptr, valid_ptr);
+}
+
+// Test volatile operations
+volatile int global_volatile = 42;
+int test_volatile_operands(int cond) {
+ // CHECK-LABEL: define {{.*}} @test_volatile_operands
+ // CHECK-DAG: [[VOLATILE_LOAD:%.*]] = load volatile i32, ptr {{.*}}
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 100)
+ // CHECK: ret i32 [[RESULT]]
+ volatile int vol_val = global_volatile;
+ return __builtin_ct_select(cond, vol_val, 100);
+}
+
+// Test uninitialized variable behavior (should still work with ct_select)
+int test_uninitialized_operands(int cond, int initialized) {
+ // CHECK-LABEL: define {{.*}} @test_uninitialized_operands
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ int uninitialized; // Intentionally uninitialized
+ return __builtin_ct_select(cond, uninitialized, initialized);
+}
+
+// Test zero division avoidance patterns
+int test_division_by_zero_avoidance(int cond, int dividend, int divisor) {
+ // CHECK-LABEL: define {{.*}} @test_division_by_zero_avoidance
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[DIV_RESULT:%.*]] = sdiv i32 %{{.*}}, %{{.*}}
+ // CHECK-DAG: [[SAFE_DIVISOR:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 1)
+ // First get a safe divisor (never zero)
+ int safe_divisor = __builtin_ct_select(divisor != 0, divisor, 1);
+ // Then perform division with guaranteed non-zero divisor
+ return dividend / safe_divisor;
+}
+
+// Test array bounds checking patterns
+int test_array_bounds_protection(int cond, int index, int* array) {
+ // CHECK-LABEL: define {{.*}} @test_array_bounds_protection
+ // CHECK-DAG: [[SAFE_INDEX:%.*]] = call i32 @llvm.ct.select.i32(i1 {{.*}}, i32 %{{.*}}, i32 0)
+ // Use ct_select to ensure safe array indexing
+ int safe_index = __builtin_ct_select(index >= 0 && index < 10, index, 0);
+ return array[safe_index];
+}
+
+// Test bit manipulation edge cases
+unsigned int test_bit_manipulation_edge_cases(int cond, unsigned int value) {
+ // CHECK-LABEL: define {{.*}} @test_bit_manipulation_edge_cases
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[SHIFT_LEFT:%.*]] = shl i32 %{{.*}}, 31
+ // CHECK-DAG: [[SHIFT_RIGHT:%.*]] = lshr i32 %{{.*}}, 31
+ // CHECK-DAG: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ // Test extreme bit shifts that could cause undefined behavior
+ unsigned int left_shift = value << 31; // Could overflow
+ unsigned int right_shift = value >> 31; // Extract sign bit
+ return __builtin_ct_select(cond, left_shift, right_shift);
+}
+
+// Test signed integer wraparound
+int test_signed_wraparound(int cond, int a, int b) {
+ // CHECK-LABEL: define {{.*}} @test_signed_wraparound
+ // CHECK-DAG: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK-DAG: [[ADD:%.*]] = add nsw i32 %{{.*}}, %{{.*}}
+ // CHECK-DAG: [[SUB:%.*]] = sub nsw i32 %{{.*}}, %{{.*}}
+ // CHECK-DAG: [[RESULT:%.*]] = call i32 @llvm.ct.select.i32(i1 [[COND]], i32 %{{.*}}, i32 %{{.*}})
+ // CHECK: ret i32 [[RESULT]]
+ int sum = a + b; // Could overflow
+ int diff = a - b; // Could underflow
+ return __builtin_ct_select(cond, sum, diff);
+}
+
+// Test vector NaN handling
+float4 test_vector_nan_operands(int cond) {
+ // CHECK-LABEL: define {{.*}} @test_vector_nan_operands
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call <4 x float> @llvm.ct.select.v4f32(i1 [[COND]], <4 x float> %{{.*}}, <4 x float> %{{.*}})
+ // CHECK: ret <4 x float> [[RESULT]]
+ float nan_val = __builtin_nanf("");
+ float4 nan_vec = {nan_val, nan_val, nan_val, nan_val};
+ float4 normal_vec = {1.0f, 2.0f, 3.0f, 4.0f};
+ return __builtin_ct_select(cond, nan_vec, normal_vec);
+}
+
+// Test vector infinity handling
+float4 test_vector_infinity_operands(int cond) {
+ // CHECK-LABEL: define {{.*}} @test_vector_infinity_operands
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call <4 x float> @llvm.ct.select.v4f32(i1 [[COND]], <4 x float> %{{.*}}, <4 x float> %{{.*}})
+ // CHECK: ret <4 x float> [[RESULT]]
+ float pos_inf = __builtin_inff();
+ float neg_inf = -__builtin_inff();
+ float4 inf_vec = {pos_inf, neg_inf, pos_inf, neg_inf};
+ float4 zero_vec = {0.0f, 0.0f, 0.0f, 0.0f};
+ return __builtin_ct_select(cond, inf_vec, zero_vec);
+}
+
+// Test mixed special values
+double test_mixed_special_values(int cond) {
+ // CHECK-LABEL: define {{.*}} @test_mixed_special_values
+ // CHECK: [[COND:%.*]] = icmp ne i32 %{{.*}}, 0
+ // CHECK: [[RESULT:%.*]] = call double @llvm.ct.select.f64(i1 [[COND]], double %{{.*}}, double %{{.*}})
+ // CHECK: ret double [[RESULT]]
+ double nan_val = __builtin_nan("");
+ double inf_val = __builtin_inf();
+ return __builtin_ct_select(cond, nan_val, inf_val);
+}
+
+// Test constant-time memory access pattern
+int test_constant_time_memory_access(int secret_index, int* data_array) {
+ // CHECK-LABEL: define {{.*}} @test_constant_time_memory_access
+ // This pattern ensures constant-time memory access regardless of secret_index value
+ int result = 0;
+ // Use ct_select to accumulate values without revealing the secret index
+ for (int i = 0; i < 8; i++) {
+ int is_target = (i == secret_index);
+ int current_value = data_array[i];
+ int selected_value = __builtin_ct_select(is_target, current_value, 0);
+ result += selected_value;
+ }
+ return result;
+}
+
+// Test timing-attack resistant comparison
+int test_timing_resistant_comparison(const char* secret, const char* guess) {
+ // CHECK-LABEL: define {{.*}} @test_timing_resistant_comparison
+ // Constant-time string comparison using ct_select
+ int match = 1;
+ for (int i = 0; i < 32; i++) {
+ int chars_equal = (secret[i] == guess[i]);
+ int both_null = (secret[i] == 0) && (guess[i] == 0);
+ int still_matching = __builtin_ct_select(chars_equal || both_null, match, 0);
+ match = __builtin_ct_select(both_null, match, still_matching);
+ }
+ return match;
+}
>From 700d2156f42687914775cef092bd79a8138c33e1 Mon Sep 17 00:00:00 2001
From: wizardengineer <juliuswoosebert at gmail.com>
Date: Wed, 5 Nov 2025 17:09:23 -0500
Subject: [PATCH 2/5] [LLVM][X86] Add native ct.select support for X86 and i386
Add native X86 implementation with CMOV instructions and comprehensive tests:
- X86 ISelLowering with CMOV for x86_64 and i386
- Fallback bitwise operations for i386 targets without CMOV
- Post-RA expansion for pseudo-instructions
- Comprehensive test coverage:
- Edge cases (zero conditions, large integers)
- i386-specific tests (FP, MMX, non-CMOV fallback)
- Vector operations
- Optimization patterns
The basic test demonstrating fallback is in the core infrastructure PR.
---
llvm/lib/Target/X86/X86.td | 8 +-
llvm/lib/Target/X86/X86ISelLowering.cpp | 798 ++++++++++-
llvm/lib/Target/X86/X86ISelLowering.h | 3 +
llvm/lib/Target/X86/X86InstrCMovSetCC.td | 205 +++
llvm/lib/Target/X86/X86InstrCompiler.td | 81 ++
llvm/lib/Target/X86/X86InstrFragments.td | 6 +
llvm/lib/Target/X86/X86InstrInfo.cpp | 609 +++++++-
llvm/lib/Target/X86/X86InstrInfo.h | 6 +
llvm/lib/Target/X86/X86InstrPredicates.td | 5 +
llvm/lib/Target/X86/X86TargetMachine.cpp | 5 +-
llvm/test/CodeGen/X86/ctselect-edge-cases.ll | 409 ++++++
llvm/test/CodeGen/X86/ctselect-i386-fp.ll | 722 ++++++++++
llvm/test/CodeGen/X86/ctselect-i386-mmx.ll | 428 ++++++
llvm/test/CodeGen/X86/ctselect-i386.ll | 267 ++++
.../test/CodeGen/X86/ctselect-optimization.ll | 304 ++++
llvm/test/CodeGen/X86/ctselect-vector.ll | 1274 +++++++++++++++++
16 files changed, 5085 insertions(+), 45 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/ctselect-edge-cases.ll
create mode 100644 llvm/test/CodeGen/X86/ctselect-i386-fp.ll
create mode 100644 llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
create mode 100644 llvm/test/CodeGen/X86/ctselect-i386.ll
create mode 100644 llvm/test/CodeGen/X86/ctselect-optimization.ll
create mode 100644 llvm/test/CodeGen/X86/ctselect-vector.ll
diff --git a/llvm/lib/Target/X86/X86.td b/llvm/lib/Target/X86/X86.td
index 50fb7204ebfa1..c4122e1ab7070 100644
--- a/llvm/lib/Target/X86/X86.td
+++ b/llvm/lib/Target/X86/X86.td
@@ -861,9 +861,10 @@ include "X86SchedSapphireRapids.td"
def ProcessorFeatures {
// x86-64 micro-architecture levels: x86-64 and x86-64-v[234]
- list<SubtargetFeature> X86_64V1Features = [
- FeatureX87, FeatureCX8, FeatureCMOV, FeatureMMX, FeatureSSE2,
- FeatureFXSR, FeatureNOPL, FeatureX86_64,
+ list<SubtargetFeature> X86_64V1Features = [FeatureX87, FeatureCX8,
+ FeatureCMOV, FeatureMMX,
+ FeatureSSE2, FeatureFXSR,
+ FeatureNOPL, FeatureX86_64,
];
list<SubtargetFeature> X86_64V1Tuning = [
TuningMacroFusion,
@@ -1199,6 +1200,7 @@ def ProcessorFeatures {
FeatureAVXNECONVERT,
FeatureAVXVNNIINT8,
FeatureAVXVNNIINT16,
+ FeatureUSERMSR,
FeatureSHA512,
FeatureSM3,
FeatureEGPR,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 6694441468bb4..13241b793ac19 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "X86ISelLowering.h"
+#include "MCTargetDesc/X86MCTargetDesc.h"
#include "MCTargetDesc/X86ShuffleDecode.h"
#include "X86.h"
#include "X86FrameLowering.h"
@@ -29,6 +30,8 @@
#include "llvm/Analysis/ProfileSummaryInfo.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/CodeGen/LivePhysRegs.h"
+#include "llvm/CodeGen/ISDOpcodes.h"
+#include "llvm/CodeGen/IntrinsicLowering.h"
#include "llvm/CodeGen/MachineFrameInfo.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineInstrBuilder.h"
@@ -48,6 +51,7 @@
#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
+#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/Intrinsics.h"
#include "llvm/IR/PatternMatch.h"
@@ -488,6 +492,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// X86 wants to expand cmov itself.
for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
@@ -496,6 +501,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
if (VT == MVT::i64 && !Subtarget.is64Bit())
continue;
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::SETCC, VT, Custom);
}
@@ -503,6 +509,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// Custom action for SELECT MMX and expand action for SELECT_CC MMX
setOperationAction(ISD::SELECT, MVT::x86mmx, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::x86mmx, Custom);
setOperationAction(ISD::SELECT_CC, MVT::x86mmx, Expand);
setOperationAction(ISD::EH_RETURN , MVT::Other, Custom);
@@ -632,6 +639,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::BR_CC, VT, Action);
setOperationAction(ISD::SETCC, VT, Action);
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::SELECT_CC, VT, Action);
setOperationAction(ISD::FROUND, VT, Action);
setOperationAction(ISD::FROUNDEVEN, VT, Action);
@@ -1077,7 +1085,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VSELECT, MVT::v4f32, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v4f32, Custom);
setOperationAction(ISD::SELECT, MVT::v4f32, Custom);
- setOperationAction(ISD::FCANONICALIZE, MVT::v4f32, Expand);
+ setOperationAction(ISD::CT_SELECT, MVT::v4f32, Custom);
+ setOperationAction(ISD::FCANONICALIZE, MVT::v4f32, Expand);
setOperationAction(ISD::LOAD, MVT::v2f32, Custom);
setOperationAction(ISD::STORE, MVT::v2f32, Custom);
@@ -1250,6 +1259,13 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SELECT, MVT::v8f16, Custom);
setOperationAction(ISD::SELECT, MVT::v16i8, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v2f64, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v2i64, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v4i32, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v8i16, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v8f16, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v16i8, Custom);
+
setOperationAction(ISD::FP_TO_SINT, MVT::v4i32, Custom);
setOperationAction(ISD::FP_TO_UINT, MVT::v4i32, Custom);
setOperationAction(ISD::FP_TO_SINT, MVT::v2i32, Custom);
@@ -1588,6 +1604,14 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::SELECT, MVT::v32i8, Custom);
setOperationAction(ISD::SELECT, MVT::v8f32, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v4f64, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v4i64, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v8i32, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v16i16, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v16f16, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v32i8, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v8f32, Custom);
+
for (auto VT : { MVT::v16i16, MVT::v8i32, MVT::v4i64 }) {
setOperationAction(ISD::SIGN_EXTEND, VT, Custom);
setOperationAction(ISD::ZERO_EXTEND, VT, Custom);
@@ -1787,6 +1811,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addRegisterClass(MVT::v16i1, &X86::VK16RegClass);
setOperationAction(ISD::SELECT, MVT::v1i1, Custom);
+ setOperationAction(ISD::CT_SELECT, MVT::v1i1, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, MVT::v1i1, Custom);
setOperationAction(ISD::BUILD_VECTOR, MVT::v1i1, Custom);
@@ -1832,6 +1857,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
for (auto VT : { MVT::v2i1, MVT::v4i1, MVT::v8i1, MVT::v16i1 }) {
setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::TRUNCATE, VT, Custom);
setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
@@ -2123,6 +2149,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
setOperationAction(ISD::INSERT_SUBVECTOR, VT, Legal);
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::VSELECT, VT, Custom);
setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
@@ -2319,6 +2346,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::EXTRACT_VECTOR_ELT, VT, Custom);
setOperationAction(ISD::INSERT_VECTOR_ELT, VT, Custom);
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
setOperationAction(ISD::VECTOR_SHUFFLE, VT, Custom);
setOperationAction(ISD::CONCAT_VECTORS, VT, Custom);
@@ -2385,6 +2413,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VSELECT, VT, Legal);
setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
setOperationAction(ISD::SELECT, VT, Custom);
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::FNEG, VT, Custom);
setOperationAction(ISD::FABS, VT, Custom);
@@ -2654,6 +2683,22 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
addRegisterClass(MVT::x86amx, &X86::TILERegClass);
}
+ // Handle 512-bit vector CT_SELECT without AVX512 by setting them to Expand
+ // This allows type legalization to split them into smaller vectors
+ for (auto VT : {MVT::v64i8, MVT::v32i16, MVT::v16i32, MVT::v8i64, MVT::v32f16,
+ MVT::v16f32, MVT::v8f64}) {
+ setOperationAction(ISD::CT_SELECT, VT, Expand);
+ }
+
+ // Handle 256-bit vector CT_SELECT without AVX by setting them to Expand
+ // This allows type legalization to split them into 128-bit vectors
+ if (!Subtarget.hasAVX()) {
+ for (auto VT : {MVT::v4f64, MVT::v4i64, MVT::v8i32, MVT::v16i16,
+ MVT::v16f16, MVT::v32i8, MVT::v8f32}) {
+ setOperationAction(ISD::CT_SELECT, VT, Expand);
+ }
+ }
+
// We want to custom lower some of our intrinsics.
setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
setOperationAction(ISD::INTRINSIC_W_CHAIN, MVT::Other, Custom);
@@ -2764,6 +2809,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
ISD::BITCAST,
ISD::VSELECT,
ISD::SELECT,
+ ISD::CT_SELECT,
ISD::SHL,
ISD::SRA,
ISD::SRL,
@@ -26076,6 +26122,174 @@ static SDValue LowerSIGN_EXTEND_Mask(SDValue Op, const SDLoc &dl,
return V;
}
+SDValue X86TargetLowering::LowerCT_SELECT(SDValue Op, SelectionDAG &DAG) const {
+ SDValue Cond = Op.getOperand(0); // condition
+ SDValue TrueOp = Op.getOperand(1); // true_value
+ SDValue FalseOp = Op.getOperand(2); // false_value
+ SDLoc DL(Op);
+ MVT VT = TrueOp.getSimpleValueType();
+
+ // Special handling for i386 targets (no CMOV) - route to post-RA expansion
+ // pseudos Let standard type legalization handle i64 automatically (splits
+ // into EDX:EAX)
+
+ // Handle soft float16 by converting to integer operations
+ if (isSoftF16(VT, Subtarget)) {
+ MVT NVT = VT.changeTypeToInteger();
+ SDValue CtSelect =
+ DAG.getNode(ISD::CT_SELECT, DL, NVT, Cond, DAG.getBitcast(NVT, FalseOp),
+ DAG.getBitcast(NVT, TrueOp));
+ return DAG.getBitcast(VT, CtSelect);
+ }
+
+ // Handle vector types
+ if (VT.isVector()) {
+ // Handle soft float16 vectors
+ if (isSoftF16(VT, Subtarget)) {
+ MVT NVT = VT.changeVectorElementTypeToInteger();
+ SDValue CtSelect = DAG.getNode(ISD::CT_SELECT, DL, NVT, Cond,
+ DAG.getBitcast(NVT, FalseOp),
+ DAG.getBitcast(NVT, TrueOp));
+ return DAG.getBitcast(VT, CtSelect);
+ }
+
+ unsigned VectorWidth = VT.getSizeInBits();
+ MVT EltVT = VT.getVectorElementType();
+
+ // 512-bit vectors without AVX512 are now handled by type legalization
+ // (Expand action) 256-bit vectors without AVX are now handled by type
+ // legalization (Expand action)
+
+ if (VectorWidth == 128 && !Subtarget.hasSSE1())
+ return SDValue();
+
+ // Handle special cases for floating point vectors
+ if (EltVT.isFloatingPoint()) {
+ // For vector floating point with AVX, use VBLENDV-style operations
+ if (Subtarget.hasAVX() && (VectorWidth == 256 || VectorWidth == 128)) {
+ // Convert to bitwise operations using the condition
+ MVT IntVT = VT.changeVectorElementTypeToInteger();
+ SDValue IntOp1 = DAG.getBitcast(IntVT, TrueOp);
+ SDValue IntOp2 = DAG.getBitcast(IntVT, FalseOp);
+
+ // Create the CT_SELECT node with integer types
+ SDValue IntResult =
+ DAG.getNode(X86ISD::CT_SELECT, DL, IntVT, IntOp2, IntOp1,
+ DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8),
+ EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget));
+ return DAG.getBitcast(VT, IntResult);
+ }
+ }
+
+ // For integer vectors or when we don't have advanced SIMD support,
+ // use the generic X86 CT_SELECT node which will be matched by the patterns
+ SDValue CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
+ SDValue EFLAGS = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget);
+ // Create the X86 CT_SELECT node - note operand order: true, false, cc, flags
+ return DAG.getNode(X86ISD::CT_SELECT, DL, VT, FalseOp, TrueOp, CC, EFLAGS);
+ }
+
+ // Look past (and (setcc_carry (cmp ...)), 1)
+ if (Cond.getOpcode() == ISD::AND &&
+ Cond.getOperand(0).getOpcode() == X86ISD::SETCC_CARRY &&
+ isOneConstant(Cond.getOperand(1)))
+ Cond = Cond.getOperand(0);
+
+ /// Process condition flags and prepare for CT_SELECT node creation
+ auto ProcessConditionFlags =
+ [&](SDValue Cond, MVT VT, SDLoc DL, SelectionDAG &DAG,
+ const X86Subtarget &Subtarget) -> std::pair<SDValue, SDValue> {
+ SDValue CC;
+ bool AddTest = true;
+
+ unsigned CondOpcode = Cond.getOpcode();
+ if (CondOpcode == X86ISD::SETCC || CondOpcode == X86ISD::SETCC_CARRY) {
+ CC = Cond.getOperand(0);
+ SDValue Cmp = Cond.getOperand(1);
+
+ if ((isX86LogicalCmp(Cmp)) || Cmp.getOpcode() == X86ISD::BT) {
+ Cond = Cmp;
+ AddTest = false;
+ }
+ } else if (CondOpcode == ISD::USUBO || CondOpcode == ISD::SSUBO ||
+ CondOpcode == ISD::UADDO || CondOpcode == ISD::SADDO ||
+ CondOpcode == ISD::UMULO || CondOpcode == ISD::SMULO) {
+ SDValue Value;
+ X86::CondCode X86Cond;
+ std::tie(Value, Cond) = getX86XALUOOp(X86Cond, Cond.getValue(0), DAG);
+ CC = DAG.getTargetConstant(X86Cond, DL, MVT::i8);
+ AddTest = false;
+ }
+
+ if (AddTest) {
+ // Look past the truncate if the high bits are known zero
+ if (isTruncWithZeroHighBitsInput(Cond, DAG))
+ Cond = Cond.getOperand(0);
+
+ // Try to match AND to BT instruction
+ if (Cond.getOpcode() == ISD::AND && Cond.hasOneUse()) {
+ X86::CondCode X86CondCode;
+ if (SDValue BT = LowerAndToBT(Cond, ISD::SETNE, DL, DAG, X86CondCode)) {
+ CC = DAG.getTargetConstant(X86CondCode, DL, MVT::i8);
+ Cond = BT;
+ AddTest = false;
+ }
+ }
+ }
+
+ if (AddTest) {
+ CC = DAG.getTargetConstant(X86::COND_NE, DL, MVT::i8);
+ Cond = EmitTest(Cond, X86::COND_NE, DL, DAG, Subtarget);
+ }
+
+ return {CC, Cond};
+ };
+
+ // Process condition flags and prepare for CT_SELECT
+ auto [CC, ProcessedCond] =
+ ProcessConditionFlags(Cond, VT, DL, DAG, Subtarget);
+
+ // Handle i8 CT_SELECT with truncate optimization
+ if (Op.getValueType() == MVT::i8 && TrueOp.getOpcode() == ISD::TRUNCATE &&
+ FalseOp.getOpcode() == ISD::TRUNCATE) {
+ SDValue T1 = TrueOp.getOperand(0), T2 = FalseOp.getOperand(0);
+ if (T1.getValueType() == T2.getValueType() &&
+ T1.getOpcode() != ISD::CopyFromReg &&
+ T2.getOpcode() != ISD::CopyFromReg) {
+ SDValue CtSelect = DAG.getNode(X86ISD::CT_SELECT, DL, T1.getValueType(),
+ T2, T1, CC, ProcessedCond);
+ return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), CtSelect);
+ }
+ }
+
+ // Promote small integer types to avoid partial register stalls
+ // Exception: For i8 without CMOV, we can generate a shorter instruction
+ // sequence without movzx so keep it as is.
+ if ((Op.getValueType() == MVT::i8 && Subtarget.hasCMOV()) ||
+ (Op.getValueType() == MVT::i16 && !X86::mayFoldLoad(TrueOp, Subtarget) &&
+ !X86::mayFoldLoad(FalseOp, Subtarget))) {
+ TrueOp = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, TrueOp);
+ FalseOp = DAG.getNode(ISD::ANY_EXTEND, DL, MVT::i32, FalseOp);
+ SDValue Ops[] = {FalseOp, TrueOp, CC, ProcessedCond};
+ SDValue CtSelect = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Ops);
+ return DAG.getNode(ISD::TRUNCATE, DL, Op.getValueType(), CtSelect);
+ }
+
+ if (isScalarFPTypeInSSEReg(VT)) {
+ MVT IntVT = (VT == MVT::f32) ? MVT::i32 : MVT::i64;
+ TrueOp = DAG.getBitcast(IntVT, TrueOp);
+ FalseOp = DAG.getBitcast(IntVT, FalseOp);
+ SDValue Ops[] = {FalseOp, TrueOp, CC, ProcessedCond};
+ SDValue CtSelect = DAG.getNode(X86ISD::CT_SELECT, DL, IntVT, Ops);
+ return DAG.getBitcast(VT, CtSelect);
+ }
+
+ // Create final CT_SELECT node
+ SDValue Ops[] = {FalseOp, TrueOp, CC, ProcessedCond};
+ return DAG.getNode(X86ISD::CT_SELECT, DL, Op.getValueType(), Ops,
+ Op->getFlags());
+}
+
static SDValue LowerANY_EXTEND(SDValue Op, const X86Subtarget &Subtarget,
SelectionDAG &DAG) {
SDValue In = Op->getOperand(0);
@@ -30482,30 +30696,65 @@ static SDValue LowervXi8MulWithUNPCK(SDValue A, SDValue B, const SDLoc &dl,
const X86Subtarget &Subtarget,
SelectionDAG &DAG,
SDValue *Low = nullptr) {
+ unsigned NumElts = VT.getVectorNumElements();
+
// For vXi8 we will unpack the low and high half of each 128 bit lane to widen
// to a vXi16 type. Do the multiplies, shift the results and pack the half
// lane results back together.
// We'll take different approaches for signed and unsigned.
- // For unsigned we'll use punpcklbw/punpckhbw to zero extend the bytes to
- // words and use pmullw to calculate the full 16-bit product.
+ // For unsigned we'll use punpcklbw/punpckhbw to put zero extend the bytes
+ // and use pmullw to calculate the full 16-bit product.
// For signed we'll use punpcklbw/punpckbw to extend the bytes to words and
// shift them left into the upper byte of each word. This allows us to use
// pmulhw to calculate the full 16-bit product. This trick means we don't
// need to sign extend the bytes to use pmullw.
- MVT ExVT = MVT::getVectorVT(MVT::i16, VT.getVectorNumElements() / 2);
+
+ MVT ExVT = MVT::getVectorVT(MVT::i16, NumElts / 2);
SDValue Zero = DAG.getConstant(0, dl, VT);
- SDValue ALo, AHi, BLo, BHi;
+ SDValue ALo, AHi;
if (IsSigned) {
ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, A));
- BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, B));
AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, A));
- BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, B));
} else {
ALo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, A, Zero));
- BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Zero));
AHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, A, Zero));
+ }
+
+ SDValue BLo, BHi;
+ if (ISD::isBuildVectorOfConstantSDNodes(B.getNode())) {
+ // If the RHS is a constant, manually unpackl/unpackh and extend.
+ SmallVector<SDValue, 16> LoOps, HiOps;
+ for (unsigned i = 0; i != NumElts; i += 16) {
+ for (unsigned j = 0; j != 8; ++j) {
+ SDValue LoOp = B.getOperand(i + j);
+ SDValue HiOp = B.getOperand(i + j + 8);
+
+ if (IsSigned) {
+ LoOp = DAG.getAnyExtOrTrunc(LoOp, dl, MVT::i16);
+ HiOp = DAG.getAnyExtOrTrunc(HiOp, dl, MVT::i16);
+ LoOp = DAG.getNode(ISD::SHL, dl, MVT::i16, LoOp,
+ DAG.getConstant(8, dl, MVT::i16));
+ HiOp = DAG.getNode(ISD::SHL, dl, MVT::i16, HiOp,
+ DAG.getConstant(8, dl, MVT::i16));
+ } else {
+ LoOp = DAG.getZExtOrTrunc(LoOp, dl, MVT::i16);
+ HiOp = DAG.getZExtOrTrunc(HiOp, dl, MVT::i16);
+ }
+
+ LoOps.push_back(LoOp);
+ HiOps.push_back(HiOp);
+ }
+ }
+
+ BLo = DAG.getBuildVector(ExVT, dl, LoOps);
+ BHi = DAG.getBuildVector(ExVT, dl, HiOps);
+ } else if (IsSigned) {
+ BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, Zero, B));
+ BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, Zero, B));
+ } else {
+ BLo = DAG.getBitcast(ExVT, getUnpackl(DAG, dl, VT, B, Zero));
BHi = DAG.getBitcast(ExVT, getUnpackh(DAG, dl, VT, B, Zero));
}
@@ -30518,7 +30767,7 @@ static SDValue LowervXi8MulWithUNPCK(SDValue A, SDValue B, const SDLoc &dl,
if (Low)
*Low = getPack(DAG, Subtarget, dl, VT, RLo, RHi);
- return getPack(DAG, Subtarget, dl, VT, RLo, RHi, /*PackHiHalf=*/true);
+ return getPack(DAG, Subtarget, dl, VT, RLo, RHi, /*PackHiHalf*/ true);
}
static SDValue LowerMULH(SDValue Op, const X86Subtarget &Subtarget,
@@ -34535,6 +34784,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::STRICT_FSETCCS: return LowerSETCC(Op, DAG);
case ISD::SETCCCARRY: return LowerSETCCCARRY(Op, DAG);
case ISD::SELECT: return LowerSELECT(Op, DAG);
+ case ISD::CT_SELECT: return LowerCT_SELECT(Op, DAG);
case ISD::COND_LOOP:
case ISD::BRCOND: return LowerConditionalBranch(Op, DAG);
case ISD::JumpTable: return LowerJumpTable(Op, DAG);
@@ -34626,6 +34876,12 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
}
}
+bool X86TargetLowering::isSelectSupported(SelectSupportKind Kind) const {
+ if (Kind == SelectSupportKind::CtSelect) {
+ return true;
+ }
+ return TargetLoweringBase::isSelectSupported(Kind);
+}
/// Replace a node with an illegal result type with a new node built out of
/// custom code.
void X86TargetLowering::ReplaceNodeResults(SDNode *N,
@@ -36242,12 +36498,6 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
}
}
-/// Return true if the addressing mode represented by AM is legal for this
-/// target, for a load/store of the specified type.
-bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
- const AddrMode &AM, Type *Ty,
- unsigned AS,
- Instruction *I) const {
// X86 supports extremely general addressing modes.
CodeModel::Model M = getTargetMachine().getCodeModel();
@@ -38581,6 +38831,480 @@ X86TargetLowering::emitPatchableEventCall(MachineInstr &MI,
return BB;
}
+/// Helper function to emit i386 CT_SELECT with condition materialization.
+/// This converts EFLAGS-based CT_SELECT into a condition byte that can be
+/// shared across multiple operations (critical for i64 type legalization).
+///
+/// Phase 1: Materialize condition byte from EFLAGS using SETCC
+/// Phase 2: Create internal pseudo with condition byte for post-RA expansion
+///
+/// This approach ensures that when i64 is type-legalized into two i32
+/// operations, both operations share the same condition byte rather than
+/// each independently reading (and destroying) EFLAGS.
+static MachineBasicBlock *
+emitCTSelectI386WithConditionMaterialization(MachineInstr &MI,
+ MachineBasicBlock *BB,
+ unsigned InternalPseudoOpcode) {
+ const TargetInstrInfo *TII = BB->getParent()->getSubtarget().getInstrInfo();
+ const MIMetadata MIMD(MI);
+ MachineFunction *MF = BB->getParent();
+ MachineRegisterInfo &MRI = MF->getRegInfo();
+
+ // Original pseudo operands: (outs dst), (ins src1, src2, cond)
+ Register Src1Reg = MI.getOperand(1).getReg();
+ Register Src2Reg = MI.getOperand(2).getReg();
+ X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(3).getImm());
+
+ // Get opposite condition (SETCC sets to 1 when condition is TRUE,
+ // but we want to select src1 when condition is FALSE for X86 semantics)
+ X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
+
+ // Step 1: Materialize condition byte from EFLAGS
+ // This is done OUTSIDE the constant-time bundle, before any EFLAGS corruption
+ Register CondByteReg = MRI.createVirtualRegister(&X86::GR8RegClass);
+ BuildMI(*BB, MI, MIMD, TII->get(X86::SETCCr), CondByteReg).addImm(OppCC);
+
+ // Step 2: Create internal pseudo that takes condition byte as input
+ // This pseudo will be expanded post-RA into the actual constant-time bundle
+ // The condition byte can now be safely shared between multiple pseudos
+
+ // Internal pseudo has operands: (outs dst, tmp_byte, tmp_mask), (ins src1,
+ // src2, cond_byte)
+ Register DstReg = MI.getOperand(0).getReg();
+
+ // Create virtual registers for the temporary outputs
+ Register TmpByteReg = MRI.createVirtualRegister(&X86::GR8RegClass);
+ Register TmpMaskReg;
+
+ // Determine the register class for tmp_mask based on the data type
+ if (InternalPseudoOpcode == X86::CT_SELECT_I386_INT_GR8rr) {
+ TmpMaskReg = MRI.createVirtualRegister(&X86::GR8RegClass);
+ } else if (InternalPseudoOpcode == X86::CT_SELECT_I386_INT_GR16rr) {
+ TmpMaskReg = MRI.createVirtualRegister(&X86::GR16RegClass);
+ } else if (InternalPseudoOpcode == X86::CT_SELECT_I386_INT_GR32rr) {
+ TmpMaskReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+ } else {
+ llvm_unreachable("Unknown internal pseudo opcode");
+ }
+
+ BuildMI(*BB, MI, MIMD, TII->get(InternalPseudoOpcode))
+ .addDef(DstReg) // dst (output)
+ .addDef(TmpByteReg) // tmp_byte (output)
+ .addDef(TmpMaskReg) // tmp_mask (output)
+ .addReg(Src1Reg) // src1 (input)
+ .addReg(Src2Reg) // src2 (input)
+ .addReg(CondByteReg); // pre-materialized condition byte (input)
+
+ MI.eraseFromParent();
+ return BB;
+}
+
+// Helper structure to hold memory operand information for FP loads
+struct FPLoadMemOperands {
+ bool IsValid = false;
+ unsigned BaseReg = 0;
+ int64_t ScaleVal = 1;
+ unsigned IndexReg = 0;
+ int64_t Disp = 0;
+ unsigned SegReg = 0;
+ int FrameIndex = -1;
+ bool IsFrameIndex = false;
+ int ConstantPoolIndex = -1;
+ bool IsConstantPool = false;
+ const GlobalValue *Global = nullptr;
+ int64_t GlobalOffset = 0;
+ bool IsGlobal = false;
+};
+
+// Check if a virtual register is defined by a simple FP load instruction
+// Returns the memory operands if it's a simple load, otherwise returns invalid
+static FPLoadMemOperands getFPLoadMemOperands(Register Reg,
+ MachineRegisterInfo &MRI,
+ unsigned ExpectedLoadOpcode) {
+ FPLoadMemOperands Result;
+
+ if (!Reg.isVirtual())
+ return Result;
+
+ MachineInstr *DefMI = MRI.getVRegDef(Reg);
+ if (!DefMI)
+ return Result;
+
+ // Check if it's the expected load opcode (e.g., LD_Fp32m, LD_Fp64m, LD_Fp80m)
+ if (DefMI->getOpcode() != ExpectedLoadOpcode)
+ return Result;
+
+ // Check that this is a simple load - not volatile, not atomic, etc.
+ // FP loads have hasSideEffects = 0 in their definition for simple loads
+ if (DefMI->hasOrderedMemoryRef())
+ return Result;
+
+ // The load should have a single def (the destination register) and memory operands
+ // Format: %reg = LD_Fpxxm <fi#N>, 1, %noreg, 0, %noreg
+ // or: %reg = LD_Fpxxm %base, scale, %index, disp, %segment
+ if (DefMI->getNumOperands() < 6)
+ return Result;
+
+ // Operand 0 is the destination, operands 1-5 are the memory reference
+ MachineOperand &BaseMO = DefMI->getOperand(1);
+ MachineOperand &ScaleMO = DefMI->getOperand(2);
+ MachineOperand &IndexMO = DefMI->getOperand(3);
+ MachineOperand &DispMO = DefMI->getOperand(4);
+ MachineOperand &SegMO = DefMI->getOperand(5);
+
+ // Check if this is a frame index load
+ if (BaseMO.isFI()) {
+ Result.IsValid = true;
+ Result.IsFrameIndex = true;
+ Result.FrameIndex = BaseMO.getIndex();
+ Result.ScaleVal = ScaleMO.getImm();
+ Result.IndexReg = IndexMO.getReg();
+ Result.Disp = DispMO.getImm();
+ Result.SegReg = SegMO.getReg();
+ return Result;
+ }
+
+ // Check if this is a constant pool load
+ // Format: %reg = LD_Fpxxm $noreg, 1, $noreg, %const.N, $noreg
+ if (BaseMO.isReg() && BaseMO.getReg() == X86::NoRegister &&
+ ScaleMO.isImm() && IndexMO.isReg() &&
+ IndexMO.getReg() == X86::NoRegister &&
+ DispMO.isCPI() && SegMO.isReg()) {
+ Result.IsValid = true;
+ Result.IsConstantPool = true;
+ Result.ConstantPoolIndex = DispMO.getIndex();
+ Result.ScaleVal = ScaleMO.getImm();
+ Result.IndexReg = IndexMO.getReg();
+ Result.Disp = 0;
+ Result.SegReg = SegMO.getReg();
+ return Result;
+ }
+
+ // Check if this is a global variable load
+ // Format: %reg = LD_Fpxxm $noreg, 1, $noreg, @global_name, $noreg
+ if (BaseMO.isReg() && BaseMO.getReg() == X86::NoRegister &&
+ ScaleMO.isImm() && IndexMO.isReg() &&
+ IndexMO.getReg() == X86::NoRegister &&
+ DispMO.isGlobal() && SegMO.isReg()) {
+ Result.IsValid = true;
+ Result.IsGlobal = true;
+ Result.Global = DispMO.getGlobal();
+ Result.GlobalOffset = DispMO.getOffset();
+ Result.ScaleVal = ScaleMO.getImm();
+ Result.IndexReg = IndexMO.getReg();
+ Result.Disp = 0;
+ Result.SegReg = SegMO.getReg();
+ return Result;
+ }
+
+ // Regular memory operands (e.g., pointer loads)
+ if (BaseMO.isReg() && ScaleMO.isImm() && IndexMO.isReg() &&
+ DispMO.isImm() && SegMO.isReg()) {
+ Result.IsValid = true;
+ Result.IsFrameIndex = false;
+ Result.IsConstantPool = false;
+ Result.BaseReg = BaseMO.getReg();
+ Result.ScaleVal = ScaleMO.getImm();
+ Result.IndexReg = IndexMO.getReg();
+ Result.Disp = DispMO.getImm();
+ Result.SegReg = SegMO.getReg();
+ return Result;
+ }
+
+ return Result;
+}
+
+static MachineBasicBlock *emitCTSelectI386WithFpType(MachineInstr &MI,
+ MachineBasicBlock *BB,
+ unsigned pseudoInstr) {
+ const TargetInstrInfo *TII = BB->getParent()->getSubtarget().getInstrInfo();
+ const MIMetadata MIMD(MI);
+ MachineFunction *MF = BB->getParent();
+ MachineRegisterInfo &MRI = MF->getRegInfo();
+ MachineFrameInfo &MFI = MF->getFrameInfo();
+ unsigned RegSizeInByte = 4;
+
+ // Get operands
+ // MI operands: %result:rfp80 = CT_SELECT_I386 %false:rfp80, %true:rfp80, %cond:i8imm
+ unsigned DestReg = MI.getOperand(0).getReg();
+ unsigned FalseReg = MI.getOperand(1).getReg();
+ unsigned TrueReg = MI.getOperand(2).getReg();
+ X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(3).getImm());
+ X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
+
+ // Materialize condition byte from EFLAGS
+ Register CondByteReg = MRI.createVirtualRegister(&X86::GR8RegClass);
+ BuildMI(*BB, MI, MIMD, TII->get(X86::SETCCr), CondByteReg).addImm(OppCC);
+
+ auto storeFpToSlot = [&](unsigned Opcode, int Slot, Register Reg) {
+ addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(Opcode)), Slot)
+ .addReg(Reg, RegState::Kill);
+ };
+
+ // Helper to load integer from memory operands
+ auto loadIntFromMemOperands = [&](const FPLoadMemOperands &MemOps,
+ unsigned Offset) -> unsigned {
+ unsigned IntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+ MachineInstrBuilder MIB =
+ BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), IntReg);
+
+ if (MemOps.IsFrameIndex) {
+ // Frame index: addFrameIndex + scale + index + disp + segment
+ MIB.addFrameIndex(MemOps.FrameIndex)
+ .addImm(MemOps.ScaleVal)
+ .addReg(MemOps.IndexReg)
+ .addImm(MemOps.Disp + Offset)
+ .addReg(MemOps.SegReg);
+ } else if (MemOps.IsConstantPool) {
+ // Constant pool: base_reg + scale + index + CP_index + segment
+ // MOV32rm format: base, scale, index, displacement, segment
+ MIB.addReg(X86::NoRegister) // Base register
+ .addImm(MemOps.ScaleVal) // Scale
+ .addReg(MemOps.IndexReg) // Index register
+ .addConstantPoolIndex(MemOps.ConstantPoolIndex, Offset) // Displacement (CP index)
+ .addReg(MemOps.SegReg); // Segment
+ } else if (MemOps.IsGlobal) {
+ // Global variable: base_reg + scale + index + global + segment
+ // MOV32rm format: base, scale, index, displacement, segment
+ MIB.addReg(X86::NoRegister) // Base register
+ .addImm(MemOps.ScaleVal) // Scale
+ .addReg(MemOps.IndexReg) // Index register
+ .addGlobalAddress(MemOps.Global, MemOps.GlobalOffset + Offset) // Displacement (global address)
+ .addReg(MemOps.SegReg); // Segment
+ } else {
+ // Regular memory: base_reg + scale + index + disp + segment
+ MIB.addReg(MemOps.BaseReg)
+ .addImm(MemOps.ScaleVal)
+ .addReg(MemOps.IndexReg)
+ .addImm(MemOps.Disp + Offset)
+ .addReg(MemOps.SegReg);
+ }
+
+ return IntReg;
+ };
+
+ // Optimized path: load integers directly from memory when both operands are
+ // memory loads, avoiding FP register round-trip
+ auto emitCtSelectFromMemory = [&](unsigned NumValues,
+ const FPLoadMemOperands &TrueMemOps,
+ const FPLoadMemOperands &FalseMemOps,
+ int ResultSlot) {
+ for (unsigned Val = 0; Val < NumValues; ++Val) {
+ unsigned Offset = Val * RegSizeInByte;
+
+ // Load true and false values directly from their memory locations as integers
+ unsigned TrueIntReg = loadIntFromMemOperands(TrueMemOps, Offset);
+ unsigned FalseIntReg = loadIntFromMemOperands(FalseMemOps, Offset);
+
+ // Use CT_SELECT_I386_INT_GR32 pseudo instruction for constant-time selection
+ unsigned ResultIntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+ unsigned TmpByteReg = MRI.createVirtualRegister(&X86::GR8RegClass);
+ unsigned TmpMaskReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+
+ BuildMI(*BB, MI, MIMD, TII->get(X86::CT_SELECT_I386_INT_GR32rr))
+ .addDef(ResultIntReg) // dst (output)
+ .addDef(TmpByteReg) // tmp_byte (output)
+ .addDef(TmpMaskReg) // tmp_mask (output)
+ .addReg(FalseIntReg) // src1 (input) - false value
+ .addReg(TrueIntReg) // src2 (input) - true value
+ .addReg(CondByteReg); // pre-materialized condition byte (input)
+
+ // Store result back to result slot
+ BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32mr))
+ .addFrameIndex(ResultSlot)
+ .addImm(1)
+ .addReg(0)
+ .addImm(Offset)
+ .addReg(0)
+ .addReg(ResultIntReg, RegState::Kill);
+ }
+ };
+
+ auto emitCtSelectWithPseudo = [&](unsigned NumValues, int TrueSlot, int FalseSlot, int ResultSlot) {
+ for (unsigned Val = 0; Val < NumValues; ++Val) {
+ unsigned Offset = Val * RegSizeInByte;
+
+ // Load true and false values from stack as 32-bit integers
+ unsigned TrueIntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+ BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), TrueIntReg)
+ .addFrameIndex(TrueSlot)
+ .addImm(1)
+ .addReg(0)
+ .addImm(Offset)
+ .addReg(0);
+
+ unsigned FalseIntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+ BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32rm), FalseIntReg)
+ .addFrameIndex(FalseSlot)
+ .addImm(1)
+ .addReg(0)
+ .addImm(Offset)
+ .addReg(0);
+
+ // Use CT_SELECT_I386_INT_GR32 pseudo instruction for constant-time selection
+ unsigned ResultIntReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+ unsigned TmpByteReg = MRI.createVirtualRegister(&X86::GR8RegClass);
+ unsigned TmpMaskReg = MRI.createVirtualRegister(&X86::GR32RegClass);
+
+ BuildMI(*BB, MI, MIMD, TII->get(X86::CT_SELECT_I386_INT_GR32rr))
+ .addDef(ResultIntReg) // dst (output)
+ .addDef(TmpByteReg) // tmp_byte (output)
+ .addDef(TmpMaskReg) // tmp_mask (output)
+ .addReg(FalseIntReg) // src1 (input) - false value
+ .addReg(TrueIntReg) // src2 (input) - true value
+ .addReg(CondByteReg); // pre-materialized condition byte (input)
+
+ // Store result back to result slot
+ BuildMI(*BB, MI, MIMD, TII->get(X86::MOV32mr))
+ .addFrameIndex(ResultSlot)
+ .addImm(1)
+ .addReg(0)
+ .addImm(Offset)
+ .addReg(0)
+ .addReg(ResultIntReg, RegState::Kill);
+ }
+ };
+
+ switch (pseudoInstr) {
+ case X86::CT_SELECT_I386_FP32rr: {
+ // Check if both operands are simple memory loads
+ FPLoadMemOperands TrueMemOps =
+ getFPLoadMemOperands(TrueReg, MRI, X86::LD_Fp32m);
+ FPLoadMemOperands FalseMemOps =
+ getFPLoadMemOperands(FalseReg, MRI, X86::LD_Fp32m);
+
+ int ResultSlot = MFI.CreateStackObject(RegSizeInByte, Align(4), false);
+
+ if (TrueMemOps.IsValid && FalseMemOps.IsValid) {
+ // Optimized path: load directly from memory as integers
+ // Works for both frame index loads (stack parameters) and
+ // constant pool loads (constants)
+ emitCtSelectFromMemory(1, TrueMemOps, FalseMemOps, ResultSlot);
+
+ // Erase the original FP load instructions since we're not using them
+ // and have loaded the data directly as integers instead
+ if (MRI.hasOneUse(TrueReg)) {
+ if (MachineInstr *TrueDefMI = MRI.getVRegDef(TrueReg))
+ TrueDefMI->eraseFromParent();
+ }
+ if (MRI.hasOneUse(FalseReg)) {
+ if (MachineInstr *FalseDefMI = MRI.getVRegDef(FalseReg))
+ FalseDefMI->eraseFromParent();
+ }
+ } else {
+ // General path: spill FP registers to stack first
+ int TrueSlot = MFI.CreateStackObject(RegSizeInByte, Align(4), false);
+ int FalseSlot = MFI.CreateStackObject(RegSizeInByte, Align(4), false);
+
+ storeFpToSlot(X86::ST_Fp32m, TrueSlot, TrueReg);
+ storeFpToSlot(X86::ST_Fp32m, FalseSlot, FalseReg);
+
+ emitCtSelectWithPseudo(1, TrueSlot, FalseSlot, ResultSlot);
+ }
+
+ // Load result back as f32
+ addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::LD_Fp32m), DestReg),
+ ResultSlot);
+ break;
+ }
+ case X86::CT_SELECT_I386_FP64rr: {
+ unsigned StackSlotSize = 8;
+
+ // Check if both operands are simple memory loads
+ FPLoadMemOperands TrueMemOps =
+ getFPLoadMemOperands(TrueReg, MRI, X86::LD_Fp64m);
+ FPLoadMemOperands FalseMemOps =
+ getFPLoadMemOperands(FalseReg, MRI, X86::LD_Fp64m);
+
+ int ResultSlot = MFI.CreateStackObject(StackSlotSize, Align(4), false);
+
+ if (TrueMemOps.IsValid && FalseMemOps.IsValid) {
+ // Optimized path: load directly from memory as integers
+ // Works for both frame index loads (stack parameters) and
+ // constant pool loads (constants)
+ emitCtSelectFromMemory(StackSlotSize / RegSizeInByte, TrueMemOps,
+ FalseMemOps, ResultSlot);
+
+ // Erase the original FP load instructions since we're not using them
+ if (MRI.hasOneUse(TrueReg)) {
+ if (MachineInstr *TrueDefMI = MRI.getVRegDef(TrueReg))
+ TrueDefMI->eraseFromParent();
+ }
+ if (MRI.hasOneUse(FalseReg)) {
+ if (MachineInstr *FalseDefMI = MRI.getVRegDef(FalseReg))
+ FalseDefMI->eraseFromParent();
+ }
+ } else {
+ // General path: spill FP registers to stack first
+ int TrueSlot = MFI.CreateStackObject(StackSlotSize, Align(4), false);
+ int FalseSlot = MFI.CreateStackObject(StackSlotSize, Align(4), false);
+
+ storeFpToSlot(X86::ST_Fp64m, TrueSlot, TrueReg);
+ storeFpToSlot(X86::ST_Fp64m, FalseSlot, FalseReg);
+
+ emitCtSelectWithPseudo(StackSlotSize / RegSizeInByte, TrueSlot, FalseSlot,
+ ResultSlot);
+ }
+
+ // Load result back as f64
+ addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::LD_Fp64m), DestReg),
+ ResultSlot);
+ break;
+ }
+ case X86::CT_SELECT_I386_FP80rr: {
+ // f80 is 80 bits (10 bytes), but stored with 12-byte alignment
+ unsigned StackObjectSize = 12;
+
+ // Check if both operands are simple memory loads
+ FPLoadMemOperands TrueMemOps =
+ getFPLoadMemOperands(TrueReg, MRI, X86::LD_Fp80m);
+ FPLoadMemOperands FalseMemOps =
+ getFPLoadMemOperands(FalseReg, MRI, X86::LD_Fp80m);
+
+ int ResultSlot = MFI.CreateStackObject(StackObjectSize, Align(4), false);
+
+ if (TrueMemOps.IsValid && FalseMemOps.IsValid) {
+ // Optimized path: load directly from memory as integers
+ // Works for both frame index loads (stack parameters) and
+ // constant pool loads (constants)
+ emitCtSelectFromMemory(StackObjectSize / RegSizeInByte, TrueMemOps,
+ FalseMemOps, ResultSlot);
+
+ // Erase the original FP load instructions since we're not using them
+ if (MRI.hasOneUse(TrueReg)) {
+ if (MachineInstr *TrueDefMI = MRI.getVRegDef(TrueReg))
+ TrueDefMI->eraseFromParent();
+ }
+ if (MRI.hasOneUse(FalseReg)) {
+ if (MachineInstr *FalseDefMI = MRI.getVRegDef(FalseReg))
+ FalseDefMI->eraseFromParent();
+ }
+ } else {
+ // General path: spill FP registers to stack first
+ int TrueSlot = MFI.CreateStackObject(StackObjectSize, Align(4), false);
+ int FalseSlot = MFI.CreateStackObject(StackObjectSize, Align(4), false);
+
+ storeFpToSlot(X86::ST_FpP80m, TrueSlot, TrueReg);
+ storeFpToSlot(X86::ST_FpP80m, FalseSlot, FalseReg);
+
+ emitCtSelectWithPseudo(StackObjectSize / RegSizeInByte, TrueSlot,
+ FalseSlot, ResultSlot);
+ }
+
+ // Load result back as f80
+ addFrameReference(BuildMI(*BB, MI, MIMD, TII->get(X86::LD_Fp80m), DestReg),
+ ResultSlot);
+ break;
+ }
+ default:
+ llvm_unreachable("Invalid CT_SELECT opcode");
+ }
+
+ MI.eraseFromParent();
+
+ return BB;
+}
+
MachineBasicBlock *
X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
MachineBasicBlock *BB) const {
@@ -38638,6 +39362,25 @@ X86TargetLowering::EmitInstrWithCustomInserter(MachineInstr &MI,
case X86::CMOV_VK64:
return EmitLoweredSelect(MI, BB);
+ case X86::CT_SELECT_I386_GR8rr:
+ return emitCTSelectI386WithConditionMaterialization(
+ MI, BB, X86::CT_SELECT_I386_INT_GR8rr);
+
+ case X86::CT_SELECT_I386_GR16rr:
+ return emitCTSelectI386WithConditionMaterialization(
+ MI, BB, X86::CT_SELECT_I386_INT_GR16rr);
+
+ case X86::CT_SELECT_I386_GR32rr:
+ return emitCTSelectI386WithConditionMaterialization(
+ MI, BB, X86::CT_SELECT_I386_INT_GR32rr);
+
+ case X86::CT_SELECT_I386_FP32rr:
+ return emitCTSelectI386WithFpType(MI, BB, X86::CT_SELECT_I386_FP32rr);
+ case X86::CT_SELECT_I386_FP64rr:
+ return emitCTSelectI386WithFpType(MI, BB, X86::CT_SELECT_I386_FP64rr);
+ case X86::CT_SELECT_I386_FP80rr:
+ return emitCTSelectI386WithFpType(MI, BB, X86::CT_SELECT_I386_FP80rr);
+
case X86::FP80_ADDr:
case X86::FP80_ADDm32: {
// Change the floating point control register to use double extended
@@ -42664,7 +43407,7 @@ static SDValue combineCommutableSHUFP(SDValue N, MVT VT, const SDLoc &DL,
if (!X86::mayFoldLoad(peekThroughOneUseBitcasts(N0), Subtarget) ||
X86::mayFoldLoad(peekThroughOneUseBitcasts(N1), Subtarget))
return SDValue();
- Imm = llvm::rotl<uint8_t>(Imm, 4);
+ Imm = ((Imm & 0x0F) << 4) | ((Imm & 0xF0) >> 4);
return DAG.getNode(X86ISD::SHUFP, DL, VT, N1, N0,
DAG.getTargetConstant(Imm, DL, MVT::i8));
};
@@ -45727,16 +46470,10 @@ bool X86TargetLowering::SimplifyDemandedBitsForTargetNode(
}
case X86ISD::PCMPGT:
// icmp sgt(0, R) == ashr(R, BitWidth-1).
- if (ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode())) {
- // iff we only need the signbit then we can use R directly.
- if (OriginalDemandedBits.isSignMask())
- return TLO.CombineTo(Op, Op.getOperand(1));
- // otherwise we just need R's signbit for the comparison.
- APInt SignMask = APInt::getSignMask(BitWidth);
- if (SimplifyDemandedBits(Op.getOperand(1), SignMask, OriginalDemandedElts,
- Known, TLO, Depth + 1))
- return true;
- }
+ // iff we only need the sign bit then we can use R directly.
+ if (OriginalDemandedBits.isSignMask() &&
+ ISD::isBuildVectorAllZeros(Op.getOperand(0).getNode()))
+ return TLO.CombineTo(Op, Op.getOperand(1));
break;
case X86ISD::MOVMSK: {
SDValue Src = Op.getOperand(0);
@@ -48781,15 +49518,6 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
DL, DAG, Subtarget))
return V;
- // If the sign bit is known then BLENDV can be folded away.
- if (N->getOpcode() == X86ISD::BLENDV) {
- KnownBits KnownCond = DAG.computeKnownBits(Cond);
- if (KnownCond.isNegative())
- return LHS;
- if (KnownCond.isNonNegative())
- return RHS;
- }
-
if (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::BLENDV) {
SmallVector<int, 64> CondMask;
if (createShuffleMaskFromVSELECT(CondMask, Cond,
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 9a958525057b6..fc8f13704d7d3 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -176,6 +176,8 @@ namespace llvm {
///
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
+ bool isSelectSupported(SelectSupportKind Kind) const override;
+
/// Replace the results of node with an illegal result
/// type with new values built out of custom code.
///
@@ -812,6 +814,7 @@ namespace llvm {
SDValue LowerSETCC(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSETCCCARRY(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerCT_SELECT(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerConditionalBranch(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerJumpTable(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerDYNAMIC_STACKALLOC(SDValue Op, SelectionDAG &DAG) const;
diff --git a/llvm/lib/Target/X86/X86InstrCMovSetCC.td b/llvm/lib/Target/X86/X86InstrCMovSetCC.td
index 39663049b92bf..5904ebcded423 100644
--- a/llvm/lib/Target/X86/X86InstrCMovSetCC.td
+++ b/llvm/lib/Target/X86/X86InstrCMovSetCC.td
@@ -106,6 +106,211 @@ let Predicates = [HasCMOV, HasNDD] in {
def : Pat<(X86cmov (loadi64 ndd_addr:$src1), GR64:$src2, timm:$cond, EFLAGS),
(CMOV64rm_ND GR64:$src2, ndd_addr:$src1, (inv_cond_XFORM timm:$cond))>;
}
+
+// Create pseudo instruction and do the pattern matching to them.
+// We use a machine pass to lower these pseudos into cmov, in order
+// to avoid backend optimizations
+let Uses = [EFLAGS], isNotDuplicable = 1, isPseudo = 1 in {
+
+ multiclass CT_SELECT<X86TypeInfo t> {
+ // register-only
+ let isCommutable = 0, SchedRW = [WriteCMOV], Predicates = [HasNativeCMOV],
+ AsmString = "ctselect\\t$dst, $src1, $src2, $cond" in {
+ def rr : PseudoI<(outs t.RegClass:$dst),
+ (ins t.RegClass:$src1, t.RegClass:$src2, i8imm:$cond),
+ [(set t.RegClass:$dst, (X86ct_select t.RegClass:$src1, t.RegClass:$src2, timm:$cond, EFLAGS))]>;
+ }
+
+ // register-memory
+ let SchedRW = [WriteCMOV.Folded, WriteCMOV.ReadAfterFold], Predicates = [HasNativeCMOV],
+ AsmString = "ctselect\\t$dst, $src1, $src2, $cond" in {
+ def rm : PseudoI<(outs t.RegClass:$dst),
+ (ins t.RegClass:$src1, t.MemOperand:$src2, i8imm:$cond),
+ [(set t.RegClass:$dst, (X86ct_select t.RegClass:$src1, (t.LoadNode addr:$src2), timm:$cond, EFLAGS))]>;
+ }
+ }
+}
+
+let isCodeGenOnly = 1, hasSideEffects = 1, ForceDisassemble = 1 in {
+ let Constraints = "$dst = $src1" in {
+ defm CT_SELECT16 : CT_SELECT<Xi16>;
+ defm CT_SELECT32 : CT_SELECT<Xi32>;
+ defm CT_SELECT64 : CT_SELECT<Xi64>;
+ }
+}
+
+// CT_SELECT_VEC base class
+class CT_SELECT_VEC<RegisterClass VRc, RegisterClass GRc>
+ : PseudoI<
+ (outs VRc:$dst, VRc:$tmpx, GRc:$tmpg),
+ (ins VRc:$t, VRc:$f, i8imm:$cond),
+ []
+ > {
+ let Uses = [EFLAGS];
+ let isPseudo = 1;
+ let isNotDuplicable = 1;
+ let hasSideEffects = 1;
+ let AsmString = "ctselect\t$dst, $f, $t, $cond";
+ let SchedRW = [];
+}
+
+// Width-specific class aliases
+class CT_SELECT_VEC128 : CT_SELECT_VEC<VR128, GR32>;
+class CT_SELECT_VEC128X : CT_SELECT_VEC<VR128X, GR32>;
+class CT_SELECT_VEC256 : CT_SELECT_VEC<VR256, GR32>;
+class CT_SELECT_VEC512 : CT_SELECT_VEC<VR512, GR32>;
+
+
+//===----------------------------------------------------------------------===//
+// 128-bit pseudos (SSE2 baseline; we use PXOR/PAND/MOVD/PSHUFD in the expander)
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasSSE1] in {
+
+ def CT_SELECT_V4F32 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+}
+
+let Predicates = [HasSSE2] in {
+
+ def CT_SELECT_V2F64 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V4I32 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V2I64 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V8I16 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V16I8 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+
+ // If your build has v8f16, keep this; otherwise comment it out.
+ def CT_SELECT_V8F16 : CT_SELECT_VEC128 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+}
+
+let Predicates = [HasAVX] in {
+
+ def CT_SELECT_V4F32X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V2F64X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V4I32X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V2I64X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V8I16X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V16I8X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+
+ // If your build has v8f16, keep this; otherwise comment it out.
+ def CT_SELECT_V8F16X : CT_SELECT_VEC128X {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// 256-bit pseudos
+//===----------------------------------------------------------------------===//
+let Predicates = [HasAVX] in {
+
+ def CT_SELECT_V8F32 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V4F64 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V8I32 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V4I64 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V16I16 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+ def CT_SELECT_V32I8 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+
+ // If your build has v16f16, keep this; otherwise comment it out.
+ def CT_SELECT_V16F16 : CT_SELECT_VEC256 {
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmpx, at earlyclobber $tmpg";
+ }
+}
+
+//===----------------------------------------------------------------------===//
+// Selection patterns: X86ct_select(...), EFLAGS -> CT_SELECT_V*
+//
+// NOTE:
+// * The SDNode carries Glue from CMP/TEST (due to SDNPInGlue).
+// * We list EFLAGS explicitly in the pattern (X86 style) to model the arch read.
+// * Temps (tmpx/tmpy,tmpg) are not in the pattern; they’re outs allocated by RA.
+//===----------------------------------------------------------------------===//
+
+let Predicates = [HasSSE1] in {
+
+ // 128-bit float (bitwise-equivalent ops in expander)
+ def : Pat<(v4f32 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V4F32 VR128:$t, VR128:$f, timm:$cc)>;
+}
+
+let Predicates = [HasSSE2] in {
+
+ // 128-bit integer
+ def : Pat<(v4i32 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V4I32 VR128:$t, VR128:$f, timm:$cc)>;
+ def : Pat<(v2i64 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V2I64 VR128:$t, VR128:$f, timm:$cc)>;
+ def : Pat<(v8i16 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V8I16 VR128:$t, VR128:$f, timm:$cc)>;
+ def : Pat<(v16i8 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V16I8 VR128:$t, VR128:$f, timm:$cc)>;
+ def : Pat<(v2f64 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V2F64 VR128:$t, VR128:$f, timm:$cc)>;
+
+ // 128-bit f16 (optional)
+ def : Pat<(v8f16 (X86ct_select VR128:$t, VR128:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V8F16 VR128:$t, VR128:$f, timm:$cc)>;
+}
+
+let Predicates = [HasAVX] in {
+
+ // 256-bit integer
+ def : Pat<(v8i32 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V8I32 VR256:$t, VR256:$f, timm:$cc)>;
+ def : Pat<(v4i64 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V4I64 VR256:$t, VR256:$f, timm:$cc)>;
+ def : Pat<(v16i16 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V16I16 VR256:$t, VR256:$f, timm:$cc)>;
+ def : Pat<(v32i8 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V32I8 VR256:$t, VR256:$f, timm:$cc)>;
+
+ // 256-bit float (bitwise-equivalent ops in expander)
+ def : Pat<(v8f32 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V8F32 VR256:$t, VR256:$f, timm:$cc)>;
+ def : Pat<(v4f64 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V4F64 VR256:$t, VR256:$f, timm:$cc)>;
+
+ // 256-bit f16 (optional)
+ def : Pat<(v16f16 (X86ct_select VR256:$t, VR256:$f, (i8 timm:$cc), EFLAGS)),
+ (CT_SELECT_V16F16 VR256:$t, VR256:$f, timm:$cc)>;
+}
+
let Predicates = [HasCMOV, HasCF] in {
def : Pat<(X86cmov GR16:$src1, 0, timm:$cond, EFLAGS),
(CFCMOV16rr GR16:$src1, (inv_cond_XFORM timm:$cond))>;
diff --git a/llvm/lib/Target/X86/X86InstrCompiler.td b/llvm/lib/Target/X86/X86InstrCompiler.td
index 6ab6f870f1bb8..df542c2ae0f55 100644
--- a/llvm/lib/Target/X86/X86InstrCompiler.td
+++ b/llvm/lib/Target/X86/X86InstrCompiler.td
@@ -699,6 +699,87 @@ def : Pat<(v16f32 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)),
def : Pat<(v8f64 (X86cmov VR512:$t, VR512:$f, timm:$cond, EFLAGS)),
(CMOV_VR512 VR512:$t, VR512:$f, timm:$cond)>;
+// CT_SELECT
+// Enhanced CT_SELECT pseudos for i386 with temporary register allocation
+// These use a two-phase approach:
+// 1. Custom inserter materializes condition byte from EFLAGS
+// 2. Post-RA expansion generates constant-time instruction bundles
+
+let isPseudo = 1, isNotDuplicable = 1 in {
+ // Phase 1: Initial pseudos that consume EFLAGS (via custom inserter)
+ // These are matched by patterns and convert EFLAGS to condition byte
+ class CT_SELECT_I386_INITIAL<RegisterClass RC, ValueType VT>
+ : PseudoI<(outs RC:$dst),
+ (ins RC:$src1, RC:$src2, i8imm:$cond),
+ [(set RC:$dst, (VT(X86ct_select RC:$src1, RC:$src2, timm:$cond,
+ EFLAGS)))]> {
+ let Uses = [EFLAGS];
+ let Defs = [EFLAGS];
+ let usesCustomInserter = 1;
+ let hasNoSchedulingInfo = 1;
+ }
+
+ // Phase 2: Internal pseudos with pre-materialized condition byte (post-RA expansion)
+ // These generate the actual constant-time instruction bundles
+ class CT_SELECT_I386_INTERNAL<RegisterClass RC, RegisterClass ByteRC>
+ : PseudoI<(outs RC:$dst, ByteRC:$tmp_byte, RC:$tmp_mask),
+ (ins RC:$src1, RC:$src2, ByteRC:$cond_byte), []> {
+ let hasNoSchedulingInfo = 1;
+ let Constraints = "@earlyclobber $dst, at earlyclobber $tmp_byte, at earlyclobber $tmp_mask";
+ let Defs = [EFLAGS]; // NEG instruction in post-RA expansion clobbers EFLAGS
+ }
+}
+
+// Phase 1 pseudos for non-CMOV targets (custom inserter materializes condition)
+let isCodeGenOnly = 1, hasSideEffects = 1, ForceDisassemble = 1 in {
+ let Predicates = [NoNativeCMOV] in {
+ def CT_SELECT_I386_GR8rr : CT_SELECT_I386_INITIAL<GR8, i8>;
+ def CT_SELECT_I386_GR16rr : CT_SELECT_I386_INITIAL<GR16, i16>;
+ def CT_SELECT_I386_GR32rr : CT_SELECT_I386_INITIAL<GR32, i32>;
+ }
+}
+
+// Phase 2 pseudos (post-RA expansion with pre-materialized condition byte)
+let isCodeGenOnly = 1, hasSideEffects = 1, ForceDisassemble = 1 in {
+ let Predicates = [NoNativeCMOV] in {
+ def CT_SELECT_I386_INT_GR8rr :
+ CT_SELECT_I386_INTERNAL<GR8, GR8>;
+ def CT_SELECT_I386_INT_GR16rr :
+ CT_SELECT_I386_INTERNAL<GR16, GR8>;
+ def CT_SELECT_I386_INT_GR32rr :
+ CT_SELECT_I386_INTERNAL<GR32, GR8>;
+ }
+}
+
+let hasSideEffects = 1,
+ ForceDisassemble = 1,
+ Constraints = "$dst = $src1" in {
+
+ let Predicates = [FPStackf32] in
+ def CT_SELECT_I386_FP32rr : CT_SELECT_I386_INITIAL<RFP32, f32>;
+
+ let Predicates = [FPStackf64] in
+ def CT_SELECT_I386_FP64rr : CT_SELECT_I386_INITIAL<RFP64, f64>;
+
+ def CT_SELECT_I386_FP80rr : CT_SELECT_I386_INITIAL<RFP80, f80>;
+}
+
+// Pattern matching for non-native-CMOV CT_SELECT (routes to custom inserter for condition materialization)
+// NoNativeCMOV ensures these patterns are used when actual CMOV instruction is not available
+// even if canUseCMOV() is true (e.g., i386 with SSE which can emulate CMOV)
+let Predicates = [NoNativeCMOV] in {
+ def : Pat<(i8(X86ct_select GR8:$src1, GR8:$src2, timm:$cond, EFLAGS)),
+ (CT_SELECT_I386_GR8rr GR8:$src1, GR8:$src2, timm:$cond)>;
+
+ def : Pat<(i16(X86ct_select GR16:$src1, GR16:$src2, timm:$cond, EFLAGS)),
+ (CT_SELECT_I386_GR16rr GR16:$src1, GR16:$src2, timm:$cond)>;
+
+ def : Pat<(i32(X86ct_select GR32:$src1, GR32:$src2, timm:$cond, EFLAGS)),
+ (CT_SELECT_I386_GR32rr GR32:$src1, GR32:$src2, timm:$cond)>;
+
+ // i64 patterns handled automatically by type legalization
+}
+
//===----------------------------------------------------------------------===//
// Normal-Instructions-With-Lock-Prefix Pseudo Instructions
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/X86/X86InstrFragments.td b/llvm/lib/Target/X86/X86InstrFragments.td
index 69f19ce08d314..da85a1a0366e1 100644
--- a/llvm/lib/Target/X86/X86InstrFragments.td
+++ b/llvm/lib/Target/X86/X86InstrFragments.td
@@ -28,6 +28,10 @@ def SDTX86Cmov : SDTypeProfile<1, 4,
[SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>,
SDTCisVT<3, i8>, SDTCisVT<4, i32>]>;
+def SDTX86CtSelect : SDTypeProfile<1, 4,
+ [SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>,
+ SDTCisVT<3, i8>, SDTCisVT<4, i32>]>;
+
// Unary and binary operator instructions that set EFLAGS as a side-effect.
def SDTUnaryArithWithFlags : SDTypeProfile<2, 1,
[SDTCisSameAs<0, 2>,
@@ -173,6 +177,8 @@ def X86ctest : SDNode<"X86ISD::CTEST", SDTX86Ccmp>;
def X86cload : SDNode<"X86ISD::CLOAD", SDTX86Cload, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
def X86cstore : SDNode<"X86ISD::CSTORE", SDTX86Cstore, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
+def X86ct_select: SDNode<"X86ISD::CT_SELECT", SDTX86CtSelect, [SDNPInGlue]>;
+
// X86 conditional moves. Operand 0 and operand 1 are the two values
// to select from. Operand 2 is the condition code, and operand 3 is the
// flag operand produced by a CMP or TEST instruction.
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 6998f913c6a52..0433e689254a4 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -474,6 +474,556 @@ bool X86InstrInfo::isFrameOperand(const MachineInstr &MI, unsigned int Op,
return false;
}
+struct CtSelectInstructions {
+ unsigned PAndOpc;
+ unsigned PAndnOpc;
+ unsigned POrOpc;
+ unsigned BroadcastOpc;
+ unsigned IntMoveOpc;
+ unsigned MoveOpc;
+ bool Use256;
+ bool UseBlendInstr;
+};
+
+static CtSelectInstructions
+getCtSelectInstructions(unsigned Opcode, const X86Subtarget &Subtarget) {
+ CtSelectInstructions Instructions = {};
+
+ switch (Opcode) {
+ case X86::CT_SELECT_V2F64:
+ if (Subtarget.hasSSE2()) {
+ Instructions.PAndOpc = X86::PANDrr;
+ Instructions.PAndnOpc = X86::PANDNrr;
+ Instructions.POrOpc = X86::PORrr;
+ Instructions.BroadcastOpc = X86::PSHUFDri;
+ Instructions.IntMoveOpc = X86::MOVDI2PDIrr;
+ Instructions.MoveOpc = X86::MOVAPDrr;
+ Instructions.UseBlendInstr = true;
+ } else {
+ llvm_unreachable("Double precision vectors require SSE2");
+ }
+ break;
+ case X86::CT_SELECT_V4F32:
+ if (Subtarget.hasSSE41()) {
+ Instructions.PAndOpc = X86::PANDrr;
+ Instructions.PAndnOpc = X86::PANDNrr;
+ Instructions.POrOpc = X86::PORrr;
+ Instructions.BroadcastOpc = X86::PSHUFDri;
+ Instructions.IntMoveOpc = X86::MOVDI2PDIrr;
+ Instructions.MoveOpc = X86::MOVAPSrr;
+ Instructions.UseBlendInstr = true;
+ } else if (Subtarget.hasSSE2()) {
+ Instructions.PAndOpc = X86::PANDrr;
+ Instructions.PAndnOpc = X86::PANDNrr;
+ Instructions.POrOpc = X86::PORrr;
+ Instructions.BroadcastOpc = X86::PSHUFDri;
+ Instructions.IntMoveOpc = X86::MOVDI2PDIrr;
+ Instructions.MoveOpc = X86::MOVAPSrr;
+ } else {
+ // fallback to SSE1, only support four 32-bit single precision
+ // floating-point values
+ Instructions.PAndOpc = X86::ANDPSrr;
+ Instructions.PAndnOpc = X86::ANDNPSrr;
+ Instructions.POrOpc = X86::ORPSrr;
+ Instructions.BroadcastOpc = X86::SHUFPSrri;
+ Instructions.IntMoveOpc = X86::MOVSS2DIrr;
+ Instructions.MoveOpc = X86::MOVAPSrr;
+ }
+ break;
+ case X86::CT_SELECT_V4I32:
+ case X86::CT_SELECT_V2I64:
+ case X86::CT_SELECT_V8I16:
+ case X86::CT_SELECT_V16I8:
+ if (Subtarget.hasSSE2()) {
+ Instructions.PAndOpc = X86::PANDrr;
+ Instructions.PAndnOpc = X86::PANDNrr;
+ Instructions.POrOpc = X86::PORrr;
+ Instructions.BroadcastOpc = X86::PSHUFDri;
+ Instructions.IntMoveOpc = X86::MOVDI2PDIrr;
+ Instructions.MoveOpc = X86::MOVDQArr;
+ } else {
+ llvm_unreachable("Integer vector operations require SSE2");
+ }
+ break;
+ case X86::CT_SELECT_V8F16:
+ if (Subtarget.hasSSE2()) {
+ Instructions.PAndOpc = X86::PANDrr;
+ Instructions.PAndnOpc = X86::PANDNrr;
+ Instructions.POrOpc = X86::PORrr;
+ Instructions.BroadcastOpc = X86::PSHUFDri;
+ Instructions.IntMoveOpc = X86::MOVDI2PDIrr;
+ Instructions.MoveOpc = X86::MOVDQArr;
+ } else {
+ llvm_unreachable("FP16 vector operations require SSE2");
+ }
+ break;
+ case X86::CT_SELECT_V4F32X:
+ case X86::CT_SELECT_V4I32X:
+ case X86::CT_SELECT_V2F64X:
+ case X86::CT_SELECT_V2I64X:
+ case X86::CT_SELECT_V8I16X:
+ case X86::CT_SELECT_V16I8X:
+ case X86::CT_SELECT_V8F16X:
+ if (Subtarget.hasAVX()) {
+ Instructions.PAndOpc = X86::VPANDrr;
+ Instructions.PAndnOpc = X86::VPANDNrr;
+ Instructions.POrOpc = X86::VPORrr;
+ Instructions.BroadcastOpc = X86::VPSHUFDri;
+ Instructions.IntMoveOpc = X86::VMOVDI2PDIrr;
+ Instructions.MoveOpc = (Opcode == X86::CT_SELECT_V4F32X) ? X86::VMOVAPSrr
+ : (Opcode == X86::CT_SELECT_V2F64X)
+ ? X86::VMOVAPDrr
+ : X86::VMOVDQArr;
+ } else {
+ llvm_unreachable("AVX variants require AVX support");
+ }
+ break;
+ case X86::CT_SELECT_V8F32:
+ case X86::CT_SELECT_V8I32:
+ if (Subtarget.hasAVX()) {
+ Instructions.PAndOpc = X86::VPANDYrr;
+ Instructions.PAndnOpc = X86::VPANDNYrr;
+ Instructions.POrOpc = X86::VPORYrr;
+ Instructions.BroadcastOpc = X86::VPERMILPSYri;
+ Instructions.IntMoveOpc = X86::VMOVDI2PDIrr;
+ Instructions.MoveOpc =
+ (Opcode == X86::CT_SELECT_V8F32) ? X86::VMOVAPSYrr : X86::VMOVDQAYrr;
+ Instructions.Use256 = true;
+ } else {
+ llvm_unreachable("256-bit vectors require AVX");
+ }
+ break;
+ case X86::CT_SELECT_V4F64:
+ case X86::CT_SELECT_V4I64:
+ if (Subtarget.hasAVX()) {
+ Instructions.PAndOpc = X86::VPANDYrr;
+ Instructions.PAndnOpc = X86::VPANDNYrr;
+ Instructions.POrOpc = X86::VPORYrr;
+ Instructions.BroadcastOpc = X86::VPERMILPDYri;
+ Instructions.IntMoveOpc = X86::VMOVDI2PDIrr;
+ Instructions.MoveOpc =
+ (Opcode == X86::CT_SELECT_V4F64) ? X86::VMOVAPDYrr : X86::VMOVDQAYrr;
+ Instructions.Use256 = true;
+ } else {
+ llvm_unreachable("256-bit vectors require AVX");
+ }
+ break;
+ case X86::CT_SELECT_V16I16:
+ case X86::CT_SELECT_V32I8:
+ case X86::CT_SELECT_V16F16:
+ if (Subtarget.hasAVX2()) {
+ Instructions.PAndOpc = X86::VPANDYrr;
+ Instructions.PAndnOpc = X86::VPANDNYrr;
+ Instructions.POrOpc = X86::VPORYrr;
+ Instructions.BroadcastOpc = X86::VPERMILPSYri;
+ Instructions.IntMoveOpc = X86::VMOVDI2PDIrr;
+ Instructions.MoveOpc = X86::VMOVDQAYrr;
+ Instructions.Use256 = true;
+ } else if (Subtarget.hasAVX()) {
+ Instructions.PAndOpc = X86::VPANDYrr;
+ Instructions.PAndnOpc = X86::VPANDNYrr;
+ Instructions.POrOpc = X86::VPORYrr;
+ Instructions.BroadcastOpc = X86::VPERMILPSYri;
+ Instructions.IntMoveOpc = X86::VMOVDI2PDIrr;
+ Instructions.MoveOpc = X86::VMOVDQAYrr;
+ Instructions.Use256 = true;
+ } else {
+ llvm_unreachable("256-bit integer vectors require AVX");
+ }
+ break;
+ default:
+ llvm_unreachable("Unexpected CT_SELECT opcode");
+ }
+
+ return Instructions;
+}
+
+bool X86InstrInfo::expandCtSelectVector(MachineInstr &MI) const {
+ unsigned Opcode = MI.getOpcode();
+ const DebugLoc &DL = MI.getDebugLoc();
+ auto Instruction = getCtSelectInstructions(Opcode, Subtarget);
+
+ MachineBasicBlock *MBB = MI.getParent();
+
+ // Operand layout matches the TableGen definition:
+ // (outs VR128:$dst, VR128:$tmpx, GR32:$tmpg),
+ // (ins VR128:$t, VR128:$f, i8imm:$cond)
+ Register Dst = MI.getOperand(0).getReg();
+ Register MaskReg = MI.getOperand(1).getReg(); // vector mask temp
+ Register TmpGPR = MI.getOperand(2).getReg(); // scalar mask temp (GPR32)
+ Register FalseVal = MI.getOperand(3).getReg(); // true_value
+ Register TrueVal = MI.getOperand(4).getReg(); // false_value
+ X86::CondCode CC = X86::CondCode(MI.getOperand(5).getImm()); // condition
+
+ MachineInstr *FirstInstr = nullptr;
+ MachineInstr *LastInstr = nullptr;
+ auto recordInstr = [&](MachineInstrBuilder MIB) {
+ MachineInstr *NewMI = MIB.getInstr();
+ LastInstr = NewMI;
+ if (!FirstInstr)
+ FirstInstr = NewMI;
+ };
+
+ // Create scalar mask in tempGPR and broadcast to vector mask
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::MOV32ri), TmpGPR)
+ .addImm(0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ const TargetRegisterInfo *TRI = &getRegisterInfo();
+ auto SubReg = TRI->getSubReg(TmpGPR, X86::sub_8bit);
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::SETCCr))
+ .addReg(SubReg)
+ .addImm(CC)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // Zero-extend byte to 32-bit register (movzbl %al, %eax)
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::MOVZX32rr8), TmpGPR)
+ .addReg(SubReg)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ if (Instruction.UseBlendInstr && Subtarget.hasSSE41()) {
+ // Shift left 31 bits to convert 1 -> 0x80000000, 0 -> 0x00000000 (shll $31,
+ // %eax)
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::SHL32ri), TmpGPR)
+ .addReg(TmpGPR)
+ .addImm(31));
+ } else {
+ // Negate to convert 1 -> 0xFFFFFFFF, 0 -> 0x00000000 (negl %eax)
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::NEG32r), TmpGPR)
+ .addReg(TmpGPR));
+ }
+
+ // Broadcast to TmpX (vector mask)
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::PXORrr), MaskReg)
+ .addReg(MaskReg)
+ .addReg(MaskReg)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // Move scalar mask to vector register
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.IntMoveOpc), MaskReg)
+ .addReg(TmpGPR)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ if (Instruction.Use256) {
+ // Broadcast to 256-bit vector register
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.BroadcastOpc), MaskReg)
+ .addReg(MaskReg)
+ .addImm(0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ } else {
+ if (Subtarget.hasSSE2() || Subtarget.hasAVX()) {
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.BroadcastOpc), MaskReg)
+ .addReg(MaskReg)
+ .addImm(0x00)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ } else {
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.BroadcastOpc), MaskReg)
+ .addReg(MaskReg)
+ .addReg(MaskReg)
+ .addImm(0x00)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ }
+ }
+
+ if (Instruction.UseBlendInstr && Subtarget.hasSSE41()) {
+ // Use dedicated blend instructions for SSE4.1+
+ unsigned BlendOpc;
+ switch (Opcode) {
+ case X86::CT_SELECT_V4F32:
+ BlendOpc = X86::BLENDVPSrr0;
+ break;
+ case X86::CT_SELECT_V2F64:
+ BlendOpc = X86::BLENDVPDrr0;
+ break;
+ default:
+ // alias for pblendvb that takes xmm0 as implicit mask register
+ BlendOpc = X86::PBLENDVBrr0;
+ break;
+ }
+
+ // Check if XMM0 is used as one of source registers, if yes then save it
+ // in Dst register and update FalseVal and TrueVal to Dst register
+ bool DidSaveXMM0 = false;
+ Register SavedXMM0 = X86::XMM0;
+ if (FalseVal == X86::XMM0 || TrueVal == X86::XMM0) {
+ Register SrcXMM0 = (FalseVal == X86::XMM0) ? FalseVal : TrueVal;
+
+ // if XMM0 is one of the source registers, it will not match with Dst
+ // registers, so we need to move it to Dst register
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), Dst)
+ .addReg(SrcXMM0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // update FalseVal and TrueVal to Dst register
+ if (FalseVal == X86::XMM0)
+ FalseVal = Dst;
+ if (TrueVal == X86::XMM0)
+ TrueVal = Dst;
+
+ // update SavedXMM0 to Dst register
+ SavedXMM0 = Dst;
+
+ // set DidSaveXMM0 to true to indicate that we saved XMM0 into Dst
+ // register
+ DidSaveXMM0 = true;
+ } else if (MaskReg != X86::XMM0 && Dst != X86::XMM0) {
+
+ // if XMM0 is not allocated for any of the register, we stil need to save
+ // and restore it after using as mask register
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), Dst)
+ .addReg(X86::XMM0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ SavedXMM0 = Dst;
+ DidSaveXMM0 = true;
+ }
+
+ if (MaskReg != X86::XMM0) {
+ // BLENDV uses XMM0 as implicit mask register
+ // https://www.felixcloutier.com/x86/pblendvb
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), X86::XMM0)
+ .addReg(MaskReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge));
+
+ // move FalseVal to mask (use MaskReg as the dst of the blend)
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::MOVAPSrr), MaskReg)
+ .addReg(FalseVal)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // MaskReg := blend(MaskReg /*false*/, TrueVal /*true*/) ; mask in
+ // xmm0
+ recordInstr(BuildMI(*MBB, MI, DL, get(BlendOpc), MaskReg)
+ .addReg(MaskReg)
+ .addReg(TrueVal)
+ .addReg(X86::XMM0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // restore XMM0 from SavedXMM0 if we saved it into Dst
+ if (DidSaveXMM0) {
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), X86::XMM0)
+ .addReg(SavedXMM0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ }
+ // dst = result (now in MaskReg)
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), Dst)
+ .addReg(MaskReg)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ } else {
+ // move FalseVal to Dst register since MaskReg is XMM0 and Dst is not
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), Dst)
+ .addReg(FalseVal)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // Dst := blend(Dst /*false*/, TrueVal /*true*/) ; mask in
+ // xmm0
+ recordInstr(BuildMI(*MBB, MI, DL, get(BlendOpc), Dst)
+ .addReg(Dst)
+ .addReg(TrueVal)
+ .addReg(X86::XMM0)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ }
+ } else {
+
+ // dst = mask
+ recordInstr(BuildMI(*MBB, MI, DL, get(Instruction.MoveOpc), Dst)
+ .addReg(MaskReg)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // mask &= true_val
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::PANDrr), MaskReg)
+ .addReg(MaskReg)
+ .addReg(TrueVal)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // dst = ~mask & false_val
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::PANDNrr), Dst)
+ .addReg(Dst)
+ .addReg(FalseVal)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+
+ // dst |= mask; (mask & t) | (~mask & f)
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::PORrr), Dst)
+ .addReg(Dst)
+ .addReg(MaskReg)
+ .setMIFlags(MachineInstr::MIFlag::NoMerge));
+ }
+
+ assert(FirstInstr && LastInstr && "Expected at least one expanded instruction");
+ auto BundleEnd = LastInstr->getIterator();
+ finalizeBundle(*MBB, FirstInstr->getIterator(), std::next(BundleEnd));
+
+ MI.eraseFromParent();
+
+ return true;
+}
+
+bool X86InstrInfo::expandCtSelectWithCMOV(MachineInstr &MI) const {
+ MachineBasicBlock *MBB = MI.getParent();
+ DebugLoc DL = MI.getDebugLoc();
+
+ // CT_SELECT pseudo has: (outs dst), (ins true_val, false_val, cond)
+ MachineOperand &OperandRes = MI.getOperand(0); // destination register
+ MachineOperand &OperandTrue = MI.getOperand(1); // true value
+ MachineOperand &OperandCond = MI.getOperand(3); // condition code
+
+ assert(OperandTrue.isReg() && OperandRes.isReg() && OperandCond.isImm() &&
+ "Invalid operand types");
+ assert(OperandTrue.getReg() == OperandRes.getReg() &&
+ "Result register different from True register");
+
+ assert(Subtarget.hasCMOV() && "target does not support CMOV instructions");
+
+ unsigned Opcode = 0;
+
+ switch (MI.getOpcode()) {
+ case X86::CT_SELECT16rr:
+ Opcode = X86::CMOV16rr;
+ break;
+ case X86::CT_SELECT32rr:
+ Opcode = X86::CMOV32rr;
+ break;
+ case X86::CT_SELECT64rr:
+ Opcode = X86::CMOV64rr;
+ break;
+ case X86::CT_SELECT16rm:
+ Opcode = X86::CMOV16rm;
+ break;
+ case X86::CT_SELECT32rm:
+ Opcode = X86::CMOV32rm;
+ break;
+ case X86::CT_SELECT64rm:
+ Opcode = X86::CMOV64rm;
+ break;
+ default:
+ llvm_unreachable("Invalid CT_SELECT opcode");
+ }
+
+ if (!Subtarget.hasCMOV()) {
+ llvm_unreachable("target does not support cmov");
+ }
+
+ // Build CMOV instruction: copy the first 3 operands (dst, true, false)
+ // and add condition code
+ MachineInstrBuilder CmovBuilder = BuildMI(*MBB, MI, DL, get(Opcode));
+ for (unsigned i = 0u; i < MI.getNumOperands(); ++i) { // Copy
+ CmovBuilder.add(MI.getOperand(i));
+ }
+
+ // Remove the original CT_SELECT instruction
+ MI.eraseFromParent();
+ return true;
+}
+
+/// Expand i386-specific CT_SELECT pseudo instructions (post-RA, constant-time)
+/// These internal pseudos receive a pre-materialized condition byte from the
+/// custom inserter, avoiding EFLAGS corruption issues during i64 type legalization.
+bool X86InstrInfo::expandCtSelectIntWithoutCMOV(MachineInstr &MI) const {
+ MachineBasicBlock *MBB = MI.getParent();
+ DebugLoc DL = MI.getDebugLoc();
+
+ // CT_SELECT_I386_INT_GRxxrr has operands: (outs dst, tmp_byte, tmp_mask),
+ // (ins src1, src2, cond_byte)
+ // Note: cond_byte is pre-materialized by custom inserter, not EFLAGS-dependent
+ Register DstReg = MI.getOperand(0).getReg();
+ Register TmpByteReg = MI.getOperand(1).getReg();
+ Register TmpMaskReg = MI.getOperand(2).getReg();
+ Register Src1Reg = MI.getOperand(3).getReg();
+ Register Src2Reg = MI.getOperand(4).getReg();
+ Register CondByteReg = MI.getOperand(5).getReg(); // Pre-materialized condition byte
+
+ // Determine instruction opcodes based on register width
+ unsigned MovZXOp, NegOp, MovOp, AndOp, NotOp, OrOp;
+ if (MI.getOpcode() == X86::CT_SELECT_I386_INT_GR8rr) {
+ MovZXOp = 0; // No zero-extend needed for GR8
+ NegOp = X86::NEG8r;
+ MovOp = X86::MOV8rr;
+ AndOp = X86::AND8rr;
+ NotOp = X86::NOT8r;
+ OrOp = X86::OR8rr;
+ } else if (MI.getOpcode() == X86::CT_SELECT_I386_INT_GR16rr) {
+ MovZXOp = X86::MOVZX16rr8;
+ NegOp = X86::NEG16r;
+ MovOp = X86::MOV16rr;
+ AndOp = X86::AND16rr;
+ NotOp = X86::NOT16r;
+ OrOp = X86::OR16rr;
+ } else { // X86::CT_SELECT_I386_INT_GR32rr
+ MovZXOp = X86::MOVZX32rr8;
+ NegOp = X86::NEG32r;
+ MovOp = X86::MOV32rr;
+ AndOp = X86::AND32rr;
+ NotOp = X86::NOT32r;
+ OrOp = X86::OR32rr;
+ }
+
+ // 7-instruction constant-time selection bundle (no SETCC inside):
+ // result = (true_val & mask) | (false_val & ~mask)
+ // The condition byte is already materialized, avoiding EFLAGS dependency
+
+ // Step 1: Copy pre-materialized condition byte to TmpByteReg
+ // This allows the bundle to work with allocated temporaries
+ auto I1 = BuildMI(*MBB, MI, DL, get(X86::MOV8rr), TmpByteReg)
+ .addReg(CondByteReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+ auto BundleStart = I1->getIterator();
+
+ // Step 2: Zero-extend condition byte to register width (0 or 1)
+ if (MI.getOpcode() != X86::CT_SELECT_I386_INT_GR8rr) {
+ BuildMI(*MBB, MI, DL, get(MovZXOp), TmpMaskReg)
+ .addReg(TmpByteReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+ }
+
+ // Step 3: Convert condition to bitmask (NEG: 1 -> 0xFFFF..., 0 -> 0x0000...)
+ Register MaskReg = (MI.getOpcode() == X86::CT_SELECT_I386_INT_GR8rr) ? TmpByteReg : TmpMaskReg;
+ BuildMI(*MBB, MI, DL, get(NegOp), MaskReg)
+ .addReg(MaskReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+
+ // Step 4,5: Apply mask to true value - copy src1 to dest, then AND with mask
+ BuildMI(*MBB, MI, DL, get(MovOp), DstReg)
+ .addReg(Src1Reg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+
+ BuildMI(*MBB, MI, DL, get(AndOp), DstReg)
+ .addReg(DstReg)
+ .addReg(MaskReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+
+ // Step 6: Create inverted mask inline (~mask)
+ BuildMI(*MBB, MI, DL, get(NotOp), MaskReg)
+ .addReg(MaskReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+
+ // Step 7: Apply inverted mask to false value - reuse mask register directly
+ BuildMI(*MBB, MI, DL, get(AndOp), MaskReg)
+ .addReg(MaskReg)
+ .addReg(Src2Reg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+
+ // Step 8: Final result: (src1 & mask) | (src2 & ~mask)
+ auto LI = BuildMI(*MBB, MI, DL, get(OrOp), DstReg)
+ .addReg(DstReg)
+ .addReg(MaskReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
+
+ // Bundle all generated instructions for atomic execution before removing MI
+ auto BundleEnd = std::next(LI->getIterator());
+ if (BundleStart != BundleEnd) {
+ // Only bundle if we have multiple instructions
+ finalizeBundle(*MBB, BundleStart, BundleEnd);
+ }
+
+ // TODO: Optimization opportunity - The register allocator may choose callee-saved
+ // registers (e.g., %ebx, %esi) for TmpByteReg/TmpMaskReg, causing unnecessary
+ // save/restore overhead. Consider constraining these to caller-saved register
+ // classes (e.g., GR8_AL, GR32_CallSaved) in the TableGen definitions to improve
+ // constant-time performance by eliminating prologue/epilogue instructions.
+
+ // Remove the original pseudo instruction
+ MI.eraseFromParent();
+ return true;
+}
+
static bool isFrameLoadOpcode(int Opcode, TypeSize &MemBytes) {
switch (Opcode) {
default:
@@ -6431,6 +6981,43 @@ bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
case X86::ADD64ri32_DB:
MIB->setDesc(get(X86::OR64ri32));
break;
+
+ case X86::CT_SELECT64rr:
+ case X86::CT_SELECT32rr:
+ case X86::CT_SELECT16rr:
+ case X86::CT_SELECT64rm:
+ case X86::CT_SELECT32rm:
+ case X86::CT_SELECT16rm:
+ // These CT_SELECT pseudos are only selected when CMOV is available
+ // Pattern matching ensures we use CT_SELECT_I386 when CMOV is not available
+ return expandCtSelectWithCMOV(MI);
+
+ // non-cmov CT_SELECT expansion (post-RA, constant-time)
+ // These are the internal pseudos with pre-materialized condition byte
+ case X86::CT_SELECT_I386_INT_GR8rr:
+ case X86::CT_SELECT_I386_INT_GR16rr:
+ case X86::CT_SELECT_I386_INT_GR32rr:
+ return expandCtSelectIntWithoutCMOV(MI);
+
+ case X86::CT_SELECT_V2F64:
+ case X86::CT_SELECT_V4F32:
+ case X86::CT_SELECT_V2I64:
+ case X86::CT_SELECT_V4I32:
+ case X86::CT_SELECT_V8I16:
+ case X86::CT_SELECT_V16I8:
+ case X86::CT_SELECT_V2F64X:
+ case X86::CT_SELECT_V4F32X:
+ case X86::CT_SELECT_V2I64X:
+ case X86::CT_SELECT_V4I32X:
+ case X86::CT_SELECT_V8I16X:
+ case X86::CT_SELECT_V16I8X:
+ case X86::CT_SELECT_V4I64:
+ case X86::CT_SELECT_V8I32:
+ case X86::CT_SELECT_V16I16:
+ case X86::CT_SELECT_V32I8:
+ case X86::CT_SELECT_V4F64:
+ case X86::CT_SELECT_V8F32:
+ return expandCtSelectVector(MI);
}
return false;
}
@@ -10679,27 +11266,39 @@ void X86InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
if (!ST.hasSSE1())
return;
- BuildMI(MBB, Iter, DL, get(X86::V_SET0), Reg);
+ // PXOR is safe to use because it doesn't affect flags.
+ BuildMI(MBB, Iter, DL, get(X86::PXORrr), Reg)
+ .addReg(Reg, RegState::Undef)
+ .addReg(Reg, RegState::Undef);
} else if (X86::VR256RegClass.contains(Reg)) {
// YMM#
if (!ST.hasAVX())
return;
- BuildMI(MBB, Iter, DL, get(X86::AVX_SET0), Reg);
+ // VPXOR is safe to use because it doesn't affect flags.
+ BuildMI(MBB, Iter, DL, get(X86::VPXORrr), Reg)
+ .addReg(Reg, RegState::Undef)
+ .addReg(Reg, RegState::Undef);
} else if (X86::VR512RegClass.contains(Reg)) {
// ZMM#
if (!ST.hasAVX512())
return;
- BuildMI(MBB, Iter, DL, get(X86::AVX512_512_SET0), Reg);
+ // VPXORY is safe to use because it doesn't affect flags.
+ BuildMI(MBB, Iter, DL, get(X86::VPXORYrr), Reg)
+ .addReg(Reg, RegState::Undef)
+ .addReg(Reg, RegState::Undef);
} else if (X86::VK1RegClass.contains(Reg) || X86::VK2RegClass.contains(Reg) ||
X86::VK4RegClass.contains(Reg) || X86::VK8RegClass.contains(Reg) ||
X86::VK16RegClass.contains(Reg)) {
if (!ST.hasVLX())
return;
- unsigned Op = ST.hasBWI() ? X86::KSET0Q : X86::KSET0W;
- BuildMI(MBB, Iter, DL, get(Op), Reg);
+ // KXOR is safe to use because it doesn't affect flags.
+ unsigned Op = ST.hasBWI() ? X86::KXORQkk : X86::KXORWkk;
+ BuildMI(MBB, Iter, DL, get(Op), Reg)
+ .addReg(Reg, RegState::Undef)
+ .addReg(Reg, RegState::Undef);
}
}
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index cab63623613a0..fc69d22b86c91 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -724,6 +724,12 @@ class X86InstrInfo final : public X86GenInstrInfo {
bool isFrameOperand(const MachineInstr &MI, unsigned int Op,
int &FrameIndex) const;
+ /// Expand the CT_SELECT pseudo-instructions.
+ bool expandCtSelectWithCMOV(MachineInstr &MI) const;
+ bool expandCtSelectIntWithoutCMOV(MachineInstr &MI) const;
+
+ bool expandCtSelectVector(MachineInstr &MI) const;
+
/// Returns true iff the routine could find two commutable operands in the
/// given machine instruction with 3 vector inputs.
/// The 'SrcOpIdx1' and 'SrcOpIdx2' are INPUT and OUTPUT arguments. Their
diff --git a/llvm/lib/Target/X86/X86InstrPredicates.td b/llvm/lib/Target/X86/X86InstrPredicates.td
index 352de4141ce71..ad08c2917b711 100644
--- a/llvm/lib/Target/X86/X86InstrPredicates.td
+++ b/llvm/lib/Target/X86/X86InstrPredicates.td
@@ -56,6 +56,11 @@ def HasCF : Predicate<"Subtarget->hasCF()">;
def HasJMPABS : Predicate<"Subtarget->hasJMPABS()">;
def HasCMOV : Predicate<"Subtarget->canUseCMOV()">;
def NoCMOV : Predicate<"!Subtarget->canUseCMOV()">;
+// Predicates for native CMOV instruction (checks hasCMOV(), not canUseCMOV())
+// HasCMOV may be true even without native CMOV (e.g., via SSE emulation)
+// Use HasNativeCMOV/NoNativeCMOV for constant-time code that requires actual CMOV
+def HasNativeCMOV : Predicate<"Subtarget->hasCMOV()">;
+def NoNativeCMOV : Predicate<"!Subtarget->hasCMOV()">;
def HasNOPL : Predicate<"Subtarget->hasNOPL()">;
def HasMMX : Predicate<"Subtarget->hasMMX()">;
def HasSSE1 : Predicate<"Subtarget->hasSSE1()">;
diff --git a/llvm/lib/Target/X86/X86TargetMachine.cpp b/llvm/lib/Target/X86/X86TargetMachine.cpp
index 5c8a508f37917..42e03671b4298 100644
--- a/llvm/lib/Target/X86/X86TargetMachine.cpp
+++ b/llvm/lib/Target/X86/X86TargetMachine.cpp
@@ -626,10 +626,11 @@ void X86PassConfig::addPreEmitPass2() {
// ObjC runtime functions present in the module.
const Function &F = MF.getFunction();
const Module *M = F.getParent();
- return M->getModuleFlag("kcfi") ||
+ return M->getModuleFlag("kcfi") || F.hasFnAttribute("ct-select") ||
(TT.isOSDarwin() &&
(M->getFunction("objc_retainAutoreleasedReturnValue") ||
- M->getFunction("objc_unsafeClaimAutoreleasedReturnValue")));
+ M->getFunction("objc_unsafeClaimAutoreleasedReturnValue"))) ||
+ F.hasFnAttribute("ct-select");
}));
// Analyzes and emits pseudos to support Win x64 Unwind V2. This pass must run
diff --git a/llvm/test/CodeGen/X86/ctselect-edge-cases.ll b/llvm/test/CodeGen/X86/ctselect-edge-cases.ll
new file mode 100644
index 0000000000000..0797265972a1f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ctselect-edge-cases.ll
@@ -0,0 +1,409 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+cmov | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+cmov | FileCheck %s --check-prefix=X32
+
+; Test ct.select edge cases and corner cases
+
+; Test with very large integers
+define i128 @test_ctselect_i128(i1 %cond, i128 %a, i128 %b) {
+; X64-LABEL: test_ctselect_i128:
+; X64: # %bb.0:
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rsi, %rax
+; X64-NEXT: cmovneq %rdx, %r8
+; X64-NEXT: movq %r8, %rdx
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_i128:
+; X32: # %bb.0:
+; X32-NEXT: pushl %edi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: pushl %eax
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: .cfi_offset %esi, -12
+; X32-NEXT: .cfi_offset %edi, -8
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edx
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl %ecx, 12(%eax)
+; X32-NEXT: movl %edx, 8(%eax)
+; X32-NEXT: movl %edi, 4(%eax)
+; X32-NEXT: movl %esi, (%eax)
+; X32-NEXT: addl $4, %esp
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: popl %edi
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl $4
+ %result = call i128 @llvm.ct.select.i128(i1 %cond, i128 %a, i128 %b)
+ ret i128 %result
+}
+
+; Test with small integer types
+define i1 @test_ctselect_i1(i1 %cond, i1 %a, i1 %b) {
+; X64-LABEL: test_ctselect_i1:
+; X64: # %bb.0:
+; X64-NEXT: movl %edx, %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %esi, %eax
+; X64-NEXT: # kill: def $al killed $al killed $eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_i1:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: # kill: def $al killed $al killed $eax
+; X32-NEXT: retl
+ %result = call i1 @llvm.ct.select.i1(i1 %cond, i1 %a, i1 %b)
+ ret i1 %result
+}
+
+; Test with extremal values
+define i32 @test_ctselect_extremal_values(i1 %cond) {
+; X64-LABEL: test_ctselect_extremal_values:
+; X64: # %bb.0:
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X64-NEXT: movl $-2147483648, %eax # imm = 0x80000000
+; X64-NEXT: cmovnel %ecx, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_extremal_values:
+; X32: # %bb.0:
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: movl $2147483647, %ecx # imm = 0x7FFFFFFF
+; X32-NEXT: movl $-2147483648, %eax # imm = 0x80000000
+; X32-NEXT: cmovnel %ecx, %eax
+; X32-NEXT: retl
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 2147483647, i32 -2147483648)
+ ret i32 %result
+}
+
+; Test with floating point special values
+define float @test_ctselect_f32_special_values(i1 %cond) {
+; X64-LABEL: test_ctselect_f32_special_values:
+; X64: # %bb.0:
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movl $2143289344, %eax # imm = 0x7FC00000
+; X64-NEXT: movl $2139095040, %ecx # imm = 0x7F800000
+; X64-NEXT: cmovnel %eax, %ecx
+; X64-NEXT: movd %ecx, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_f32_special_values:
+; X32: # %bb.0:
+; X32-NEXT: pushl %edi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: pushl %eax
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: .cfi_offset %esi, -12
+; X32-NEXT: .cfi_offset %edi, -8
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: sete %al
+; X32-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %ecx
+; X32-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %edx
+; X32-NEXT: movb %al, %ah
+; X32-NEXT: movzbl %ah, %edi
+; X32-NEXT: negl %edi
+; X32-NEXT: movl %edx, %esi
+; X32-NEXT: andl %edi, %esi
+; X32-NEXT: notl %edi
+; X32-NEXT: andl %ecx, %edi
+; X32-NEXT: orl %edi, %esi
+; X32-NEXT: movl %esi, (%esp)
+; X32-NEXT: flds (%esp)
+; X32-NEXT: addl $4, %esp
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: popl %edi
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl
+ %result = call float @llvm.ct.select.f32(i1 %cond, float 0x7FF8000000000000, float 0x7FF0000000000000)
+ ret float %result
+}
+
+define double @test_ctselect_f64_special_values(i1 %cond) {
+; X64-LABEL: test_ctselect_f64_special_values:
+; X64: # %bb.0:
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movabsq $9221120237041090560, %rax # imm = 0x7FF8000000000000
+; X64-NEXT: movabsq $9218868437227405312, %rcx # imm = 0x7FF0000000000000
+; X64-NEXT: cmovneq %rax, %rcx
+; X64-NEXT: movq %rcx, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_f64_special_values:
+; X32: # %bb.0:
+; X32-NEXT: pushl %edi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: subl $24, %esp
+; X32-NEXT: .cfi_def_cfa_offset 36
+; X32-NEXT: .cfi_offset %esi, -12
+; X32-NEXT: .cfi_offset %edi, -8
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X32-NEXT: sete %al
+; X32-NEXT: fxch %st(1)
+; X32-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NEXT: fstpl (%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl (%esp), %edx
+; X32-NEXT: movb %al, %ah
+; X32-NEXT: movzbl %ah, %edi
+; X32-NEXT: negl %edi
+; X32-NEXT: movl %edx, %esi
+; X32-NEXT: andl %edi, %esi
+; X32-NEXT: notl %edi
+; X32-NEXT: andl %ecx, %edi
+; X32-NEXT: orl %edi, %esi
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movb %al, %ah
+; X32-NEXT: movzbl %ah, %edi
+; X32-NEXT: negl %edi
+; X32-NEXT: movl %edx, %esi
+; X32-NEXT: andl %edi, %esi
+; X32-NEXT: notl %edi
+; X32-NEXT: andl %ecx, %edi
+; X32-NEXT: orl %edi, %esi
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT: fldl {{[0-9]+}}(%esp)
+; X32-NEXT: addl $24, %esp
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: popl %edi
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl
+ %result = call double @llvm.ct.select.f64(i1 %cond, double 0x7FF8000000000000, double 0x7FF0000000000000)
+ ret double %result
+}
+
+; Test with null pointers
+define ptr @test_ctselect_null_ptr(i1 %cond, ptr %ptr) {
+; X64-LABEL: test_ctselect_null_ptr:
+; X64: # %bb.0:
+; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rsi, %rax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_null_ptr:
+; X32: # %bb.0:
+; X32-NEXT: xorl %eax, %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: retl
+ %result = call ptr @llvm.ct.select.p0(i1 %cond, ptr %ptr, ptr null)
+ ret ptr %result
+}
+
+; Test with function pointers
+define ptr @test_ctselect_function_ptr(i1 %cond, ptr %func1, ptr %func2) {
+; X64-LABEL: test_ctselect_function_ptr:
+; X64: # %bb.0:
+; X64-NEXT: movq %rdx, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rsi, %rax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_function_ptr:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: retl
+ %result = call ptr @llvm.ct.select.p0(i1 %cond, ptr %func1, ptr %func2)
+ ret ptr %result
+}
+
+; Test with volatile loads
+define i32 @test_ctselect_volatile_load(i1 %cond, ptr %p1, ptr %p2) {
+; X64-LABEL: test_ctselect_volatile_load:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rsi), %ecx
+; X64-NEXT: movl (%rdx), %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %ecx, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_volatile_load:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl (%ecx), %ecx
+; X32-NEXT: movl (%eax), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %ecx, %eax
+; X32-NEXT: retl
+ %a = load volatile i32, ptr %p1
+ %b = load volatile i32, ptr %p2
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test with atomic loads
+define i32 @test_ctselect_atomic_load(i1 %cond, ptr %p1, ptr %p2) {
+; X64-LABEL: test_ctselect_atomic_load:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rsi), %ecx
+; X64-NEXT: movl (%rdx), %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %ecx, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_atomic_load:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl (%ecx), %ecx
+; X32-NEXT: movl (%eax), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %ecx, %eax
+; X32-NEXT: retl
+ %a = load atomic i32, ptr %p1 acquire, align 4
+ %b = load atomic i32, ptr %p2 acquire, align 4
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test with condition from icmp on pointers
+define ptr @test_ctselect_ptr_cmp(ptr %p1, ptr %p2, ptr %a, ptr %b) {
+; X64-LABEL: test_ctselect_ptr_cmp:
+; X64: # %bb.0:
+; X64-NEXT: movq %rcx, %rax
+; X64-NEXT: cmpq %rsi, %rdi
+; X64-NEXT: sete %cl
+; X64-NEXT: testb %cl, %cl
+; X64-NEXT: cmovneq %rdx, %rax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_ptr_cmp:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: sete %cl
+; X32-NEXT: testb %cl, %cl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: retl
+ %cmp = icmp eq ptr %p1, %p2
+ %result = call ptr @llvm.ct.select.p0(i1 %cmp, ptr %a, ptr %b)
+ ret ptr %result
+}
+
+; Test with struct pointer types (struct types themselves may not be directly supported)
+%struct.pair = type { i32, i32 }
+
+define ptr @test_ctselect_struct_ptr(i1 %cond, ptr %a, ptr %b) {
+; X64-LABEL: test_ctselect_struct_ptr:
+; X64: # %bb.0:
+; X64-NEXT: movq %rdx, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rsi, %rax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_struct_ptr:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: retl
+ %result = call ptr @llvm.ct.select.p0(i1 %cond, ptr %a, ptr %b)
+ ret ptr %result
+}
+
+; Test with deeply nested conditions (stress test for instruction selection)
+define i32 @test_ctselect_deeply_nested(i1 %c1, i1 %c2, i1 %c3, i1 %c4, i32 %a, i32 %b, i32 %c, i32 %d, i32 %e) {
+; X64-LABEL: test_ctselect_deeply_nested:
+; X64: # %bb.0:
+; X64-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; X64-NEXT: movl {{[0-9]+}}(%rsp), %r10d
+; X64-NEXT: movl {{[0-9]+}}(%rsp), %r11d
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %r8d, %r9d
+; X64-NEXT: testb $1, %sil
+; X64-NEXT: cmovnel %r9d, %r11d
+; X64-NEXT: testb $1, %dl
+; X64-NEXT: cmovnel %r11d, %r10d
+; X64-NEXT: testb $1, %cl
+; X64-NEXT: cmovnel %r10d, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_deeply_nested:
+; X32: # %bb.0:
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: .cfi_offset %esi, -8
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %esi, %edx
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %edx, %ecx
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %ecx, %eax
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl
+ %sel1 = call i32 @llvm.ct.select.i32(i1 %c1, i32 %a, i32 %b)
+ %sel2 = call i32 @llvm.ct.select.i32(i1 %c2, i32 %sel1, i32 %c)
+ %sel3 = call i32 @llvm.ct.select.i32(i1 %c3, i32 %sel2, i32 %d)
+ %sel4 = call i32 @llvm.ct.select.i32(i1 %c4, i32 %sel3, i32 %e)
+ ret i32 %sel4
+}
+
+; Test with misaligned loads
+define i32 @test_ctselect_misaligned_load(i1 %cond, ptr %p1, ptr %p2) {
+; X64-LABEL: test_ctselect_misaligned_load:
+; X64: # %bb.0:
+; X64-NEXT: movl (%rdx), %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel (%rsi), %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_misaligned_load:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl (%eax), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel (%ecx), %eax
+; X32-NEXT: retl
+ %a = load i32, ptr %p1, align 1
+ %b = load i32, ptr %p2, align 1
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Declare the intrinsics
+declare i1 @llvm.ct.select.i1(i1, i1, i1)
+declare i128 @llvm.ct.select.i128(i1, i128, i128)
+declare i32 @llvm.ct.select.i32(i1, i32, i32)
+declare float @llvm.ct.select.f32(i1, float, float)
+declare double @llvm.ct.select.f64(i1, double, double)
+declare ptr @llvm.ct.select.p0(i1, ptr, ptr)
diff --git a/llvm/test/CodeGen/X86/ctselect-i386-fp.ll b/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
new file mode 100644
index 0000000000000..b88ec72a37925
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
@@ -0,0 +1,722 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=-cmov | FileCheck %s --check-prefix=I386-NOCMOV
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+cmov | FileCheck %s --check-prefix=I386-CMOV
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=-cmov -verify-machineinstrs | FileCheck %s --check-prefix=I386-NOCMOV
+
+; Comprehensive CT_SELECT tests for i386 targets with floating-point types
+; - Without CMOV: constant-time implementation using FP->int conversion + existing post-RA CT_SELECT
+; - With CMOV: CMOV-based implementation
+; - Verifies security properties: no conditional branches, constant execution time
+; Strategy: FP values stored to memory, converted to integers, CT_SELECT on integers, converted back to FP
+
+; Test basic f32 functionality
+define float @test_ctselect_f32_basic(i1 %cond, float %a, float %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_basic:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %eax
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: flds (%esp)
+; I386-NOCMOV-NEXT: addl $4, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_basic:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: pushl %eax
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: flds (%esp)
+; I386-CMOV-NEXT: addl $4, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
+ ret float %result
+}
+
+; Test f32 with different condition codes
+define float @test_ctselect_f32_eq(float %x, float %y, float %a, float %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_eq:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %eax
+; I386-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fucompp
+; I386-NOCMOV-NEXT: fnstsw %ax
+; I386-NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
+; I386-NOCMOV-NEXT: sahf
+; I386-NOCMOV-NEXT: setnp %al
+; I386-NOCMOV-NEXT: sete %cl
+; I386-NOCMOV-NEXT: testb %al, %cl
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: flds (%esp)
+; I386-NOCMOV-NEXT: addl $4, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_eq:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: pushl %eax
+; I386-CMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fucompi %st(1), %st
+; I386-CMOV-NEXT: fstp %st(0)
+; I386-CMOV-NEXT: setnp %al
+; I386-CMOV-NEXT: sete %cl
+; I386-CMOV-NEXT: testb %al, %cl
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: flds (%esp)
+; I386-CMOV-NEXT: addl $4, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %cmp = fcmp oeq float %x, %y
+ %result = call float @llvm.ct.select.f32(i1 %cmp, float %a, float %b)
+ ret float %result
+}
+
+; Test basic f64 functionality
+define double @test_ctselect_f64_basic(i1 %cond, double %a, double %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f64_basic:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: subl $8, %esp
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldl (%esp)
+; I386-NOCMOV-NEXT: addl $8, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f64_basic:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: subl $8, %esp
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fldl (%esp)
+; I386-CMOV-NEXT: addl $8, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %result = call double @llvm.ct.select.f64(i1 %cond, double %a, double %b)
+ ret double %result
+}
+
+; Test basic x86_fp80 functionality
+define x86_fp80 @test_ctselect_f80_basic(i1 %cond, x86_fp80 %a, x86_fp80 %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f80_basic:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: subl $12, %esp
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldt (%esp)
+; I386-NOCMOV-NEXT: addl $12, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f80_basic:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: subl $12, %esp
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fldt (%esp)
+; I386-CMOV-NEXT: addl $12, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %result = call x86_fp80 @llvm.ct.select.f80(i1 %cond, x86_fp80 %a, x86_fp80 %b)
+ ret x86_fp80 %result
+}
+
+; Test f32 with complex conditions
+define float @test_ctselect_f32_gt(float %x, float %y, float %a, float %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_gt:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %eax
+; I386-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fucompp
+; I386-NOCMOV-NEXT: fnstsw %ax
+; I386-NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
+; I386-NOCMOV-NEXT: sahf
+; I386-NOCMOV-NEXT: seta %al
+; I386-NOCMOV-NEXT: testb %al, %al
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: flds (%esp)
+; I386-NOCMOV-NEXT: addl $4, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_gt:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: pushl %eax
+; I386-CMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fucompi %st(1), %st
+; I386-CMOV-NEXT: fstp %st(0)
+; I386-CMOV-NEXT: seta %al
+; I386-CMOV-NEXT: testb %al, %al
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: flds (%esp)
+; I386-CMOV-NEXT: addl $4, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %cmp = fcmp ogt float %x, %y
+ %result = call float @llvm.ct.select.f32(i1 %cmp, float %a, float %b)
+ ret float %result
+}
+
+; Test constant-time properties: verify no branches in generated code
+define float @test_ctselect_f32_no_branches(i1 %cond, float %a, float %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_no_branches:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %eax
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: flds (%esp)
+; I386-NOCMOV-NEXT: addl $4, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_no_branches:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: pushl %eax
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: flds (%esp)
+; I386-CMOV-NEXT: addl $4, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
+ ret float %result
+}
+
+; Test that BUNDLE directives are present for constant-time guarantees
+define float @test_ctselect_f32_bundled(i1 %cond, float %a, float %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_bundled:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %eax
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: flds (%esp)
+; I386-NOCMOV-NEXT: addl $4, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_bundled:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: pushl %eax
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: flds (%esp)
+; I386-CMOV-NEXT: addl $4, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
+ ret float %result
+}
+
+; Test edge case: NaN handling
+define float @test_ctselect_f32_nan(i1 %cond) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_nan:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: subl $12, %esp
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; I386-NOCMOV-NEXT: fldz
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: fxch %st(1)
+; I386-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fstps (%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl (%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: addl $12, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_nan:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: subl $12, %esp
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; I386-CMOV-NEXT: fldz
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: fxch %st(1)
+; I386-CMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fstps (%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl (%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: flds {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: addl $12, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %nan = bitcast i32 2139095040 to float ; 0x7F800000 = +inf
+ %zero = bitcast i32 0 to float
+ %result = call float @llvm.ct.select.f32(i1 %cond, float %nan, float %zero)
+ ret float %result
+}
+
+; Test memory alignment for f80
+define x86_fp80 @test_ctselect_f80_alignment(i1 %cond, x86_fp80 %a, x86_fp80 %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f80_alignment:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: subl $12, %esp
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldt (%esp)
+; I386-NOCMOV-NEXT: addl $12, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f80_alignment:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: subl $12, %esp
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fldt (%esp)
+; I386-CMOV-NEXT: addl $12, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %result = call x86_fp80 @llvm.ct.select.f80(i1 %cond, x86_fp80 %a, x86_fp80 %b)
+ ret x86_fp80 %result
+}
+
+; Stress test: multiple CT_SELECT operations
+define float @test_ctselect_f32_multiple(i1 %cond1, i1 %cond2, float %a, float %b, float %c, float %d) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_f32_multiple:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: subl $8, %esp
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %ecx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, (%esp)
+; I386-NOCMOV-NEXT: flds (%esp)
+; I386-NOCMOV-NEXT: addl $8, %esp
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_f32_multiple:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %edi
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: subl $8, %esp
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movb %al, %ah
+; I386-CMOV-NEXT: movzbl %ah, %edi
+; I386-CMOV-NEXT: negl %edi
+; I386-CMOV-NEXT: movl %edx, %esi
+; I386-CMOV-NEXT: andl %edi, %esi
+; I386-CMOV-NEXT: notl %edi
+; I386-CMOV-NEXT: andl %ecx, %edi
+; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, (%esp)
+; I386-CMOV-NEXT: flds (%esp)
+; I386-CMOV-NEXT: addl $8, %esp
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: retl
+ %sel1 = call float @llvm.ct.select.f32(i1 %cond1, float %a, float %b)
+ %sel2 = call float @llvm.ct.select.f32(i1 %cond2, float %sel1, float %c)
+ ret float %sel2
+}
+
+; Declare intrinsics
+declare float @llvm.ct.select.f32(i1, float, float)
+declare double @llvm.ct.select.f64(i1, double, double)
+declare x86_fp80 @llvm.ct.select.f80(i1, x86_fp80, x86_fp80)
diff --git a/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll b/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
new file mode 100644
index 0000000000000..6851c5babeb2d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
@@ -0,0 +1,428 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=i386-unknown-linux-gnu -mattr=-cmov,+mmx < %s | FileCheck %s --check-prefix=I386-NOCMOV
+; RUN: llc -mtriple=i386-unknown-linux-gnu -mattr=+cmov,+mmx < %s | FileCheck %s --check-prefix=I386-CMOV
+; RUN: llc -mtriple=i386-unknown-linux-gnu -mattr=-cmov,+mmx -verify-machineinstrs < %s | FileCheck %s --check-prefix=I386-NOCMOV
+
+; Test constant-time selection with MMX intrinsics to exercise VR64 CT_SELECT
+; These tests use MMX intrinsics to create <1 x i64> values that get allocated to VR64 registers
+
+; Test MMX ct.select using paddd intrinsic to force VR64 allocation
+define <1 x i64> @test_mmx_ctselect_with_paddd(i32 %cond, i64 %a, i64 %b) {
+; I386-NOCMOV-LABEL: test_mmx_ctselect_with_paddd:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: movb %bh, %al
+; I386-NOCMOV-NEXT: movzbl %al, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %esi, %edi
+; I386-NOCMOV-NEXT: andl %ebp, %edi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %ecx, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %ecx
+; I386-NOCMOV-NEXT: andl %esi, %ecx
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: orl %esi, %ecx
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-NOCMOV-NEXT: paddd %mm0, %mm0
+; I386-NOCMOV-NEXT: movq %mm0, (%esp)
+; I386-NOCMOV-NEXT: movl (%esp), %eax
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_mmx_ctselect_with_paddd:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %dl
+; I386-CMOV-NEXT: testb %dl, %dl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-CMOV-NEXT: paddd %mm0, %mm0
+; I386-CMOV-NEXT: movq %mm0, (%esp)
+; I386-CMOV-NEXT: movl (%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-CMOV-NEXT: retl
+ %mmx_a = bitcast i64 %a to <1 x i64>
+ %mmx_b = bitcast i64 %b to <1 x i64>
+ %cmp = icmp ne i32 %cond, 0
+ %sel = call <1 x i64> @llvm.ct.select.v1i64(i1 %cmp, <1 x i64> %mmx_a, <1 x i64> %mmx_b)
+ %result = call <1 x i64> @llvm.x86.mmx.padd.d(<1 x i64> %sel, <1 x i64> %sel)
+ ret <1 x i64> %result
+}
+
+; Test MMX ct.select using psllw intrinsic
+define <1 x i64> @test_mmx_ctselect_with_psllw(i32 %cond, i64 %a, i64 %b) {
+; I386-NOCMOV-LABEL: test_mmx_ctselect_with_psllw:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: movb %bh, %al
+; I386-NOCMOV-NEXT: movzbl %al, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %esi, %edi
+; I386-NOCMOV-NEXT: andl %ebp, %edi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %ecx, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %ecx
+; I386-NOCMOV-NEXT: andl %esi, %ecx
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: orl %esi, %ecx
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-NOCMOV-NEXT: psllw %mm0, %mm0
+; I386-NOCMOV-NEXT: movq %mm0, (%esp)
+; I386-NOCMOV-NEXT: movl (%esp), %eax
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_mmx_ctselect_with_psllw:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %dl
+; I386-CMOV-NEXT: testb %dl, %dl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-CMOV-NEXT: psllw %mm0, %mm0
+; I386-CMOV-NEXT: movq %mm0, (%esp)
+; I386-CMOV-NEXT: movl (%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-CMOV-NEXT: retl
+ %mmx_a = bitcast i64 %a to <1 x i64>
+ %mmx_b = bitcast i64 %b to <1 x i64>
+ %cmp = icmp ne i32 %cond, 0
+ %sel = call <1 x i64> @llvm.ct.select.v1i64(i1 %cmp, <1 x i64> %mmx_a, <1 x i64> %mmx_b)
+ %result = call <1 x i64> @llvm.x86.mmx.psll.w(<1 x i64> %sel, <1 x i64> %sel)
+ ret <1 x i64> %result
+}
+
+; Test nested MMX ct.selects with pand intrinsic
+define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64 %a, i64 %b, i64 %c) {
+; I386-NOCMOV-LABEL: test_mmx_nested_ctselect_with_pand:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: movb %bh, %cl
+; I386-NOCMOV-NEXT: movzbl %cl, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %edx, %edi
+; I386-NOCMOV-NEXT: andl %ebp, %edi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %eax, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: sete %dl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: movb %dl, %dh
+; I386-NOCMOV-NEXT: movzbl %dh, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %esi, %ebx
+; I386-NOCMOV-NEXT: andl %ebp, %ebx
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %eax, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %ebx
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %dl
+; I386-NOCMOV-NEXT: testb %dl, %dl
+; I386-NOCMOV-NEXT: sete %dh
+; I386-NOCMOV-NEXT: movb %dh, %al
+; I386-NOCMOV-NEXT: movzbl %al, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %ecx, %esi
+; I386-NOCMOV-NEXT: andl %ebp, %esi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %ebx, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %esi
+; I386-NOCMOV-NEXT: testb %dl, %dl
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; I386-NOCMOV-NEXT: movb %al, %dl
+; I386-NOCMOV-NEXT: movzbl %dl, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %ebx, %ecx
+; I386-NOCMOV-NEXT: andl %esi, %ecx
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: orl %esi, %ecx
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-NOCMOV-NEXT: pand %mm0, %mm0
+; I386-NOCMOV-NEXT: movq %mm0, (%esp)
+; I386-NOCMOV-NEXT: movl (%esp), %eax
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_mmx_nested_ctselect_with_pand:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: pushl %ebx
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 32
+; I386-CMOV-NEXT: .cfi_offset %esi, -12
+; I386-CMOV-NEXT: .cfi_offset %ebx, -8
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %bl
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %bh
+; I386-CMOV-NEXT: testb %bh, %bh
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: testb %bl, %bl
+; I386-CMOV-NEXT: cmovnel %esi, %edx
+; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel %ecx, %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-CMOV-NEXT: pand %mm0, %mm0
+; I386-CMOV-NEXT: movq %mm0, (%esp)
+; I386-CMOV-NEXT: movl (%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: popl %ebx
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-CMOV-NEXT: retl
+ %mmx_a = bitcast i64 %a to <1 x i64>
+ %mmx_b = bitcast i64 %b to <1 x i64>
+ %mmx_c = bitcast i64 %c to <1 x i64>
+ %cmp1 = icmp ne i32 %cond1, 0
+ %cmp2 = icmp ne i32 %cond2, 0
+ %sel1 = call <1 x i64> @llvm.ct.select.v1i64(i1 %cmp2, <1 x i64> %mmx_a, <1 x i64> %mmx_b)
+ %sel2 = call <1 x i64> @llvm.ct.select.v1i64(i1 %cmp1, <1 x i64> %sel1, <1 x i64> %mmx_c)
+ %result = call <1 x i64> @llvm.x86.mmx.pand(<1 x i64> %sel2, <1 x i64> %sel2)
+ ret <1 x i64> %result
+}
+
+; Test MMX ct.select with por intrinsic
+define <1 x i64> @test_mmx_ctselect_with_por(i32 %cond, i64 %a, i64 %b) {
+; I386-NOCMOV-LABEL: test_mmx_ctselect_with_por:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: movb %bh, %al
+; I386-NOCMOV-NEXT: movzbl %al, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %esi, %edi
+; I386-NOCMOV-NEXT: andl %ebp, %edi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %ecx, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: movb %al, %ah
+; I386-NOCMOV-NEXT: movzbl %ah, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %ecx
+; I386-NOCMOV-NEXT: andl %esi, %ecx
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: orl %esi, %ecx
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-NOCMOV-NEXT: por %mm0, %mm0
+; I386-NOCMOV-NEXT: movq %mm0, (%esp)
+; I386-NOCMOV-NEXT: movl (%esp), %eax
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_mmx_ctselect_with_por:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %dl
+; I386-CMOV-NEXT: testb %dl, %dl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
+; I386-CMOV-NEXT: por %mm0, %mm0
+; I386-CMOV-NEXT: movq %mm0, (%esp)
+; I386-CMOV-NEXT: movl (%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
+; I386-CMOV-NEXT: retl
+ %mmx_a = bitcast i64 %a to <1 x i64>
+ %mmx_b = bitcast i64 %b to <1 x i64>
+ %cmp = icmp ne i32 %cond, 0
+ %sel = call <1 x i64> @llvm.ct.select.v1i64(i1 %cmp, <1 x i64> %mmx_a, <1 x i64> %mmx_b)
+ %result = call <1 x i64> @llvm.x86.mmx.por(<1 x i64> %sel, <1 x i64> %sel)
+ ret <1 x i64> %result
+}
+
+; Declare MMX intrinsics
+declare <1 x i64> @llvm.x86.mmx.padd.d(<1 x i64>, <1 x i64>)
+declare <1 x i64> @llvm.x86.mmx.psll.w(<1 x i64>, <1 x i64>)
+declare <1 x i64> @llvm.x86.mmx.pand(<1 x i64>, <1 x i64>)
+declare <1 x i64> @llvm.x86.mmx.por(<1 x i64>, <1 x i64>)
+
+; Declare constant-time selection intrinsic
+declare <1 x i64> @llvm.ct.select.v1i64(i1, <1 x i64>, <1 x i64>)
diff --git a/llvm/test/CodeGen/X86/ctselect-i386.ll b/llvm/test/CodeGen/X86/ctselect-i386.ll
new file mode 100644
index 0000000000000..d1cc559f0c1c1
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ctselect-i386.ll
@@ -0,0 +1,267 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=-cmov | FileCheck %s --check-prefix=I386-NOCMOV
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+cmov | FileCheck %s --check-prefix=I386-CMOV
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=-cmov -verify-machineinstrs | FileCheck %s --check-prefix=I386-NOCMOV
+
+; Comprehensive CT_SELECT tests for i386 targets with scalar integer types
+; - Without CMOV: constant-time implementation using post-RA expansion with bundled instructions
+; - With CMOV: CMOV-based implementation
+; - Verifies security properties: no conditional branches, constant execution time
+; All expansion happens post-RA for better optimization control and constant-time guarantees
+
+; Test basic i32 functionality
+define i32 @test_ctselect_i32_basic(i1 %cond, i32 %a, i32 %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_i32_basic:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %bl
+; I386-NOCMOV-NEXT: movb %bl, %bh
+; I386-NOCMOV-NEXT: movzbl %bh, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %eax
+; I386-NOCMOV-NEXT: andl %esi, %eax
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_i32_basic:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: retl
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test i16 functionality
+define i16 @test_ctselect_i16_basic(i1 %cond, i16 %a, i16 %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_i16_basic:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %bl
+; I386-NOCMOV-NEXT: movb %bl, %bh
+; I386-NOCMOV-NEXT: movzbw %bh, %si
+; I386-NOCMOV-NEXT: negw %si
+; I386-NOCMOV-NEXT: movw %dx, %ax
+; I386-NOCMOV-NEXT: andw %si, %ax
+; I386-NOCMOV-NEXT: notw %si
+; I386-NOCMOV-NEXT: andw %cx, %si
+; I386-NOCMOV-NEXT: orw %si, %ax
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_i16_basic:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnew {{[0-9]+}}(%esp), %ax
+; I386-CMOV-NEXT: retl
+ %result = call i16 @llvm.ct.select.i16(i1 %cond, i16 %a, i16 %b)
+ ret i16 %result
+}
+
+; Test i8 functionality
+define i8 @test_ctselect_i8_basic(i1 %cond, i8 %a, i8 %b) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_i8_basic:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %ah
+; I386-NOCMOV-NEXT: movb %ah, %ch
+; I386-NOCMOV-NEXT: negb %ch
+; I386-NOCMOV-NEXT: movb %dl, %al
+; I386-NOCMOV-NEXT: andb %ch, %al
+; I386-NOCMOV-NEXT: notb %ch
+; I386-NOCMOV-NEXT: andb %cl, %ch
+; I386-NOCMOV-NEXT: orb %ch, %al
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_i8_basic:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: # kill: def $al killed $al killed $eax
+; I386-CMOV-NEXT: retl
+ %result = call i8 @llvm.ct.select.i8(i1 %cond, i8 %a, i8 %b)
+ ret i8 %result
+}
+
+; Test security property: constant-time execution for cryptographic use case
+define i32 @test_crypto_key_select(i32 %secret_bit, i32 %key1, i32 %key2) nounwind {
+; I386-NOCMOV-LABEL: test_crypto_key_select:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %al
+; I386-NOCMOV-NEXT: testb %al, %al
+; I386-NOCMOV-NEXT: sete %bl
+; I386-NOCMOV-NEXT: movb %bl, %bh
+; I386-NOCMOV-NEXT: movzbl %bh, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %eax
+; I386-NOCMOV-NEXT: andl %esi, %eax
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_crypto_key_select:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %cl
+; I386-CMOV-NEXT: testb %cl, %cl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: retl
+ %cond = icmp ne i32 %secret_bit, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %key1, i32 %key2)
+ ret i32 %result
+}
+
+; Test that no conditional branches appear in constant-time path
+define i32 @test_no_conditional_branches(i32 %secret, i32 %val1, i32 %val2) nounwind {
+; I386-NOCMOV-LABEL: test_no_conditional_branches:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %al
+; I386-NOCMOV-NEXT: testb %al, %al
+; I386-NOCMOV-NEXT: sete %bl
+; I386-NOCMOV-NEXT: movb %bl, %bh
+; I386-NOCMOV-NEXT: movzbl %bh, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %eax
+; I386-NOCMOV-NEXT: andl %esi, %eax
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_no_conditional_branches:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %cl
+; I386-CMOV-NEXT: testb %cl, %cl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: retl
+ %cond = icmp ne i32 %secret, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %val1, i32 %val2)
+ ret i32 %result
+}
+
+; Test with comparison condition
+define i32 @test_ctselect_i32_cmp(i32 %a, i32 %b, i32 %c) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_i32_cmp:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: cmpl %edx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: testb %al, %al
+; I386-NOCMOV-NEXT: sete %bl
+; I386-NOCMOV-NEXT: movb %bl, %bh
+; I386-NOCMOV-NEXT: movzbl %bh, %esi
+; I386-NOCMOV-NEXT: negl %esi
+; I386-NOCMOV-NEXT: movl %edx, %eax
+; I386-NOCMOV-NEXT: andl %esi, %eax
+; I386-NOCMOV-NEXT: notl %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_i32_cmp:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: cmpl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: sete %cl
+; I386-CMOV-NEXT: testb %cl, %cl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: retl
+ %cond = icmp eq i32 %a, %c
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %b, i32 %c)
+ ret i32 %result
+}
+
+; Test nested selects
+define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) nounwind {
+; I386-NOCMOV-LABEL: test_ctselect_nested:
+; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %bl
+; I386-NOCMOV-NEXT: movb %bl, %bh
+; I386-NOCMOV-NEXT: movzbl %bh, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %edx, %esi
+; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %eax, %edi
+; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %dl
+; I386-NOCMOV-NEXT: movb %dl, %dh
+; I386-NOCMOV-NEXT: movzbl %dh, %edi
+; I386-NOCMOV-NEXT: negl %edi
+; I386-NOCMOV-NEXT: movl %ecx, %eax
+; I386-NOCMOV-NEXT: andl %edi, %eax
+; I386-NOCMOV-NEXT: notl %edi
+; I386-NOCMOV-NEXT: andl %esi, %edi
+; I386-NOCMOV-NEXT: orl %edi, %eax
+; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: retl
+;
+; I386-CMOV-LABEL: test_ctselect_nested:
+; I386-CMOV: # %bb.0:
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel %ecx, %eax
+; I386-CMOV-NEXT: retl
+ %sel1 = call i32 @llvm.ct.select.i32(i1 %cond2, i32 %a, i32 %b)
+ %sel2 = call i32 @llvm.ct.select.i32(i1 %cond1, i32 %sel1, i32 %c)
+ ret i32 %sel2
+}
+
+; Declare ct.select intrinsics
+declare i8 @llvm.ct.select.i8(i1, i8, i8)
+declare i16 @llvm.ct.select.i16(i1, i16, i16)
+declare i32 @llvm.ct.select.i32(i1, i32, i32)
diff --git a/llvm/test/CodeGen/X86/ctselect-optimization.ll b/llvm/test/CodeGen/X86/ctselect-optimization.ll
new file mode 100644
index 0000000000000..481d49971a937
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ctselect-optimization.ll
@@ -0,0 +1,304 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+cmov | FileCheck %s
+
+; Test ct.select optimization patterns
+
+; Test smin(x, 0) pattern optimization
+define i32 @test_ctselect_smin_zero(i32 %x) {
+; CHECK-LABEL: test_ctselect_smin_zero:
+; CHECK: # %bb.0:
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: sets %cl
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp slt i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 0)
+ ret i32 %result
+}
+
+; Test smax(x, 0) pattern optimization
+define i32 @test_ctselect_smax_zero(i32 %x) {
+; CHECK-LABEL: test_ctselect_smax_zero:
+; CHECK: # %bb.0:
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: setg %cl
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp sgt i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 0)
+ ret i32 %result
+}
+
+; Test generic smin pattern
+define i32 @test_ctselect_smin_generic(i32 %x, i32 %y) {
+; CHECK-LABEL: test_ctselect_smin_generic:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: cmpl %esi, %edi
+; CHECK-NEXT: setl %cl
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp slt i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 %y)
+ ret i32 %result
+}
+
+; Test generic smax pattern
+define i32 @test_ctselect_smax_generic(i32 %x, i32 %y) {
+; CHECK-LABEL: test_ctselect_smax_generic:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: cmpl %esi, %edi
+; CHECK-NEXT: setg %cl
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp sgt i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 %y)
+ ret i32 %result
+}
+
+; Test umin pattern
+define i32 @test_ctselect_umin_generic(i32 %x, i32 %y) {
+; CHECK-LABEL: test_ctselect_umin_generic:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: cmpl %esi, %edi
+; CHECK-NEXT: setb %cl
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp ult i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 %y)
+ ret i32 %result
+}
+
+; Test umax pattern
+define i32 @test_ctselect_umax_generic(i32 %x, i32 %y) {
+; CHECK-LABEL: test_ctselect_umax_generic:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: cmpl %esi, %edi
+; CHECK-NEXT: seta %cl
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp ugt i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 %y)
+ ret i32 %result
+}
+
+; Test abs pattern
+define i32 @test_ctselect_abs(i32 %x) {
+; CHECK-LABEL: test_ctselect_abs:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: movl %edi, %ecx
+; CHECK-NEXT: negl %ecx
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: sets %dl
+; CHECK-NEXT: testb %dl, %dl
+; CHECK-NEXT: cmovnel %ecx, %eax
+; CHECK-NEXT: retq
+ %neg = sub i32 0, %x
+ %cmp = icmp slt i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %neg, i32 %x)
+ ret i32 %result
+}
+
+; Test nabs pattern (negative abs)
+define i32 @test_ctselect_nabs(i32 %x) {
+; CHECK-LABEL: test_ctselect_nabs:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edi, %eax
+; CHECK-NEXT: negl %eax
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: sets %cl
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %neg = sub i32 0, %x
+ %cmp = icmp slt i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 %x, i32 %neg)
+ ret i32 %result
+}
+
+; Test sign extension pattern
+define i32 @test_ctselect_sign_extend(i32 %x) {
+; CHECK-LABEL: test_ctselect_sign_extend:
+; CHECK: # %bb.0:
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: sets %cl
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: movl $-1, %ecx
+; CHECK-NEXT: cmovnel %ecx, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp slt i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 -1, i32 0)
+ ret i32 %result
+}
+
+; Test zero extension pattern
+define i32 @test_ctselect_zero_extend(i32 %x) {
+; CHECK-LABEL: test_ctselect_zero_extend:
+; CHECK: # %bb.0:
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: setne %cl
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: movl $1, %ecx
+; CHECK-NEXT: cmovnel %ecx, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp ne i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 1, i32 0)
+ ret i32 %result
+}
+
+; Test mask generation pattern
+define i32 @test_ctselect_mask_generation(i32 %x) {
+; CHECK-LABEL: test_ctselect_mask_generation:
+; CHECK: # %bb.0:
+; CHECK-NEXT: testl %edi, %edi
+; CHECK-NEXT: sets %cl
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: movl $-1, %ecx
+; CHECK-NEXT: cmovnel %ecx, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp slt i32 %x, 0
+ %result = call i32 @llvm.ct.select.i32(i1 %cmp, i32 -1, i32 0)
+ ret i32 %result
+}
+
+; Test constant folding with known condition
+define i32 @test_ctselect_constant_folding_true(i32 %a, i32 %b) {
+; CHECK-LABEL: test_ctselect_constant_folding_true:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: movb $1, %cl
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %result = call i32 @llvm.ct.select.i32(i1 true, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+define i32 @test_ctselect_constant_folding_false(i32 %a, i32 %b) {
+; CHECK-LABEL: test_ctselect_constant_folding_false:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: xorl %ecx, %ecx
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %edi, %eax
+; CHECK-NEXT: retq
+ %result = call i32 @llvm.ct.select.i32(i1 false, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test with identical operands
+define i32 @test_ctselect_identical_operands(i1 %cond, i32 %x) {
+; CHECK-LABEL: test_ctselect_identical_operands:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %esi, %eax
+; CHECK-NEXT: testb $1, %dil
+; CHECK-NEXT: cmovnel %esi, %eax
+; CHECK-NEXT: retq
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %x, i32 %x)
+ ret i32 %result
+}
+
+; Test with inverted condition
+define i32 @test_ctselect_inverted_condition(i32 %x, i32 %y, i32 %a, i32 %b) {
+; CHECK-LABEL: test_ctselect_inverted_condition:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl %edx, %eax
+; CHECK-NEXT: cmpl %esi, %edi
+; CHECK-NEXT: sete %dl
+; CHECK-NEXT: testb %dl, %dl
+; CHECK-NEXT: cmovnel %ecx, %eax
+; CHECK-NEXT: retq
+ %cmp = icmp eq i32 %x, %y
+ %not_cmp = xor i1 %cmp, true
+ %result = call i32 @llvm.ct.select.i32(i1 %not_cmp, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test for 64-bit specific optimizations
+define i64 @test_ctselect_i64_smin_zero(i64 %x) {
+; CHECK-LABEL: test_ctselect_i64_smin_zero:
+; CHECK: # %bb.0:
+; CHECK-NEXT: testq %rdi, %rdi
+; CHECK-NEXT: sets %cl
+; CHECK-NEXT: xorl %eax, %eax
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovneq %rdi, %rax
+; CHECK-NEXT: retq
+ %cmp = icmp slt i64 %x, 0
+ %result = call i64 @llvm.ct.select.i64(i1 %cmp, i64 %x, i64 0)
+ ret i64 %result
+}
+
+; Test for floating point optimizations
+define float @test_ctselect_f32_zero_positive(float %x) {
+; CHECK-LABEL: test_ctselect_f32_zero_positive:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movd %xmm0, %eax
+; CHECK-NEXT: xorps %xmm1, %xmm1
+; CHECK-NEXT: ucomiss %xmm1, %xmm0
+; CHECK-NEXT: seta %cl
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovnel %eax, %edx
+; CHECK-NEXT: movd %edx, %xmm0
+; CHECK-NEXT: retq
+ %cmp = fcmp ogt float %x, 0.0
+ %result = call float @llvm.ct.select.f32(i1 %cmp, float %x, float 0.0)
+ ret float %result
+}
+
+define double @test_ctselect_f64_zero_positive(double %x) {
+; CHECK-LABEL: test_ctselect_f64_zero_positive:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movq %xmm0, %rax
+; CHECK-NEXT: xorpd %xmm1, %xmm1
+; CHECK-NEXT: ucomisd %xmm1, %xmm0
+; CHECK-NEXT: seta %cl
+; CHECK-NEXT: xorl %edx, %edx
+; CHECK-NEXT: testb %cl, %cl
+; CHECK-NEXT: cmovneq %rax, %rdx
+; CHECK-NEXT: movq %rdx, %xmm0
+; CHECK-NEXT: retq
+ %cmp = fcmp ogt double %x, 0.0
+ %result = call double @llvm.ct.select.f64(i1 %cmp, double %x, double 0.0)
+ ret double %result
+}
+
+; Test chain of ct.select operations
+define i32 @test_ctselect_chain(i1 %c1, i1 %c2, i1 %c3, i32 %a, i32 %b, i32 %c, i32 %d) {
+; CHECK-LABEL: test_ctselect_chain:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT: testb $1, %dil
+; CHECK-NEXT: cmovnel %ecx, %r8d
+; CHECK-NEXT: testb $1, %sil
+; CHECK-NEXT: cmovnel %r8d, %r9d
+; CHECK-NEXT: testb $1, %dl
+; CHECK-NEXT: cmovnel %r9d, %eax
+; CHECK-NEXT: retq
+ %sel1 = call i32 @llvm.ct.select.i32(i1 %c1, i32 %a, i32 %b)
+ %sel2 = call i32 @llvm.ct.select.i32(i1 %c2, i32 %sel1, i32 %c)
+ %sel3 = call i32 @llvm.ct.select.i32(i1 %c3, i32 %sel2, i32 %d)
+ ret i32 %sel3
+}
+
+; Declare the intrinsics
+declare i32 @llvm.ct.select.i32(i1, i32, i32)
+declare i64 @llvm.ct.select.i64(i1, i64, i64)
+declare float @llvm.ct.select.f32(i1, float, float)
+declare double @llvm.ct.select.f64(i1, double, double)
diff --git a/llvm/test/CodeGen/X86/ctselect-vector.ll b/llvm/test/CodeGen/X86/ctselect-vector.ll
new file mode 100644
index 0000000000000..2206e32cd6d34
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ctselect-vector.ll
@@ -0,0 +1,1274 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+sse2 | FileCheck %s --check-prefix=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx | FileCheck %s --check-prefix=AVX
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
+
+; Test ct.select functionality for vector types
+
+; 128-bit vectors
+define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) {
+; SSE2-LABEL: test_ctselect_v4i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4i32:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB0_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %xmm0, %xmm1
+; AVX512-NEXT: .LBB0_2:
+; AVX512-NEXT: vmovaps %xmm1, %xmm0
+; AVX512-NEXT: retq
+ %result = call <4 x i32> @llvm.ct.select.v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %result
+}
+
+define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b) {
+; SSE2-LABEL: test_ctselect_v4f32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movaps %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4f32:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4f32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB1_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %xmm0, %xmm1
+; AVX512-NEXT: .LBB1_2:
+; AVX512-NEXT: vmovaps %xmm1, %xmm0
+; AVX512-NEXT: retq
+ %result = call <4 x float> @llvm.ct.select.v4f32(i1 %cond, <4 x float> %a, <4 x float> %b)
+ ret <4 x float> %result
+}
+
+define <2 x i64> @test_ctselect_v2i64(i1 %cond, <2 x i64> %a, <2 x i64> %b) {
+; SSE2-LABEL: test_ctselect_v2i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v2i64:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v2i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v2i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB2_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %xmm0, %xmm1
+; AVX512-NEXT: .LBB2_2:
+; AVX512-NEXT: vmovaps %xmm1, %xmm0
+; AVX512-NEXT: retq
+ %result = call <2 x i64> @llvm.ct.select.v2i64(i1 %cond, <2 x i64> %a, <2 x i64> %b)
+ ret <2 x i64> %result
+}
+
+define <2 x double> @test_ctselect_v2f64(i1 %cond, <2 x double> %a, <2 x double> %b) {
+; SSE2-LABEL: test_ctselect_v2f64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movapd %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v2f64:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v2f64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v2f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB3_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovapd %xmm0, %xmm1
+; AVX512-NEXT: .LBB3_2:
+; AVX512-NEXT: vmovapd %xmm1, %xmm0
+; AVX512-NEXT: retq
+ %result = call <2 x double> @llvm.ct.select.v2f64(i1 %cond, <2 x double> %a, <2 x double> %b)
+ ret <2 x double> %result
+}
+
+; 256-bit vectors
+define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
+; SSE2-LABEL: test_ctselect_v8i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: movd %eax, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm5, %xmm4
+; SSE2-NEXT: pand %xmm0, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm4
+; SSE2-NEXT: por %xmm5, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm3, %xmm2
+; SSE2-NEXT: por %xmm0, %xmm2
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v8i32:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm3, %ymm3
+; AVX-NEXT: vmovd %eax, %ymm3
+; AVX-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
+; AVX-NEXT: vmovdqa %ymm3, %ymm2
+; AVX-NEXT: pand %ymm0, %ymm3
+; AVX-NEXT: pandn %ymm1, %ymm2
+; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: vmovaps %ymm2, %ymm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v8i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm3, %ymm3
+; AVX2-NEXT: vmovd %eax, %ymm3
+; AVX2-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
+; AVX2-NEXT: vmovdqa %ymm3, %ymm2
+; AVX2-NEXT: pand %ymm0, %ymm3
+; AVX2-NEXT: pandn %ymm1, %ymm2
+; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: vmovaps %ymm2, %ymm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v8i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB4_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %ymm0, %ymm1
+; AVX512-NEXT: .LBB4_2:
+; AVX512-NEXT: vmovaps %ymm1, %ymm0
+; AVX512-NEXT: retq
+ %result = call <8 x i32> @llvm.ct.select.v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b)
+ ret <8 x i32> %result
+}
+
+define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b) {
+; SSE2-LABEL: test_ctselect_v8f32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: movd %eax, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
+; SSE2-NEXT: movaps %xmm5, %xmm4
+; SSE2-NEXT: pand %xmm0, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm4
+; SSE2-NEXT: por %xmm5, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movaps %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm3, %xmm2
+; SSE2-NEXT: por %xmm0, %xmm2
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v8f32:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm3, %ymm3
+; AVX-NEXT: vmovd %eax, %ymm3
+; AVX-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
+; AVX-NEXT: vmovdqa %ymm3, %ymm2
+; AVX-NEXT: pand %ymm0, %ymm3
+; AVX-NEXT: pandn %ymm1, %ymm2
+; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: vmovaps %ymm2, %ymm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v8f32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm3, %ymm3
+; AVX2-NEXT: vmovd %eax, %ymm3
+; AVX2-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
+; AVX2-NEXT: vmovdqa %ymm3, %ymm2
+; AVX2-NEXT: pand %ymm0, %ymm3
+; AVX2-NEXT: pandn %ymm1, %ymm2
+; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: vmovaps %ymm2, %ymm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v8f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB5_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %ymm0, %ymm1
+; AVX512-NEXT: .LBB5_2:
+; AVX512-NEXT: vmovaps %ymm1, %ymm0
+; AVX512-NEXT: retq
+ %result = call <8 x float> @llvm.ct.select.v8f32(i1 %cond, <8 x float> %a, <8 x float> %b)
+ ret <8 x float> %result
+}
+
+define <4 x i64> @test_ctselect_v4i64(i1 %cond, <4 x i64> %a, <4 x i64> %b) {
+; SSE2-LABEL: test_ctselect_v4i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: movd %eax, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm5, %xmm4
+; SSE2-NEXT: pand %xmm0, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm4
+; SSE2-NEXT: por %xmm5, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm3, %xmm2
+; SSE2-NEXT: por %xmm0, %xmm2
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4i64:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm3, %ymm3
+; AVX-NEXT: vmovd %eax, %ymm3
+; AVX-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
+; AVX-NEXT: vmovdqa %ymm3, %ymm2
+; AVX-NEXT: pand %ymm0, %ymm3
+; AVX-NEXT: pandn %ymm1, %ymm2
+; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: vmovaps %ymm2, %ymm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm3, %ymm3
+; AVX2-NEXT: vmovd %eax, %ymm3
+; AVX2-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
+; AVX2-NEXT: vmovdqa %ymm3, %ymm2
+; AVX2-NEXT: pand %ymm0, %ymm3
+; AVX2-NEXT: pandn %ymm1, %ymm2
+; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: vmovaps %ymm2, %ymm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB6_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %ymm0, %ymm1
+; AVX512-NEXT: .LBB6_2:
+; AVX512-NEXT: vmovaps %ymm1, %ymm0
+; AVX512-NEXT: retq
+ %result = call <4 x i64> @llvm.ct.select.v4i64(i1 %cond, <4 x i64> %a, <4 x i64> %b)
+ ret <4 x i64> %result
+}
+
+define <4 x double> @test_ctselect_v4f64(i1 %cond, <4 x double> %a, <4 x double> %b) {
+; SSE2-LABEL: test_ctselect_v4f64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: movd %eax, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
+; SSE2-NEXT: movapd %xmm5, %xmm4
+; SSE2-NEXT: pand %xmm0, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm4
+; SSE2-NEXT: por %xmm5, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movapd %xmm0, %xmm2
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm3, %xmm2
+; SSE2-NEXT: por %xmm0, %xmm2
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4f64:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm3, %ymm3
+; AVX-NEXT: vmovd %eax, %ymm3
+; AVX-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
+; AVX-NEXT: vmovdqa %ymm3, %ymm2
+; AVX-NEXT: pand %ymm0, %ymm3
+; AVX-NEXT: pandn %ymm1, %ymm2
+; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: vmovaps %ymm2, %ymm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4f64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm3, %ymm3
+; AVX2-NEXT: vmovd %eax, %ymm3
+; AVX2-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
+; AVX2-NEXT: vmovdqa %ymm3, %ymm2
+; AVX2-NEXT: pand %ymm0, %ymm3
+; AVX2-NEXT: pandn %ymm1, %ymm2
+; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: vmovaps %ymm2, %ymm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB7_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovapd %ymm0, %ymm1
+; AVX512-NEXT: .LBB7_2:
+; AVX512-NEXT: vmovapd %ymm1, %ymm0
+; AVX512-NEXT: retq
+ %result = call <4 x double> @llvm.ct.select.v4f64(i1 %cond, <4 x double> %a, <4 x double> %b)
+ ret <4 x double> %result
+}
+
+; 512-bit vectors (AVX512 only)
+define <16 x i32> @test_ctselect_v16i32(i1 %cond, <16 x i32> %a, <16 x i32> %b) {
+; SSE2-LABEL: test_ctselect_v16i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm9, %xmm9
+; SSE2-NEXT: movd %eax, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm9, %xmm8
+; SSE2-NEXT: pand %xmm0, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm8
+; SSE2-NEXT: por %xmm9, %xmm8
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm5, %xmm4
+; SSE2-NEXT: por %xmm0, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm6, %xmm5
+; SSE2-NEXT: por %xmm0, %xmm5
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm6
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: pandn %xmm7, %xmm6
+; SSE2-NEXT: por %xmm0, %xmm6
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v16i32:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm5, %ymm5
+; AVX-NEXT: vmovd %eax, %ymm5
+; AVX-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
+; AVX-NEXT: vmovdqa %ymm5, %ymm4
+; AVX-NEXT: pand %ymm0, %ymm5
+; AVX-NEXT: pandn %ymm2, %ymm4
+; AVX-NEXT: por %ymm5, %ymm4
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm0, %ymm0
+; AVX-NEXT: vmovd %eax, %ymm0
+; AVX-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
+; AVX-NEXT: vmovdqa %ymm0, %ymm2
+; AVX-NEXT: pand %ymm1, %ymm0
+; AVX-NEXT: pandn %ymm3, %ymm2
+; AVX-NEXT: por %ymm0, %ymm2
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v16i32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm5, %ymm5
+; AVX2-NEXT: vmovd %eax, %ymm5
+; AVX2-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
+; AVX2-NEXT: vmovdqa %ymm5, %ymm4
+; AVX2-NEXT: pand %ymm0, %ymm5
+; AVX2-NEXT: pandn %ymm2, %ymm4
+; AVX2-NEXT: por %ymm5, %ymm4
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm0, %ymm0
+; AVX2-NEXT: vmovd %eax, %ymm0
+; AVX2-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
+; AVX2-NEXT: vmovdqa %ymm0, %ymm2
+; AVX2-NEXT: pand %ymm1, %ymm0
+; AVX2-NEXT: pandn %ymm3, %ymm2
+; AVX2-NEXT: por %ymm0, %ymm2
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v16i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB8_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %zmm0, %zmm1
+; AVX512-NEXT: .LBB8_2:
+; AVX512-NEXT: vmovaps %zmm1, %zmm0
+; AVX512-NEXT: retq
+ %result = call <16 x i32> @llvm.ct.select.v16i32(i1 %cond, <16 x i32> %a, <16 x i32> %b)
+ ret <16 x i32> %result
+}
+
+define <16 x float> @test_ctselect_v16f32(i1 %cond, <16 x float> %a, <16 x float> %b) {
+; SSE2-LABEL: test_ctselect_v16f32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm9, %xmm9
+; SSE2-NEXT: movd %eax, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
+; SSE2-NEXT: movaps %xmm9, %xmm8
+; SSE2-NEXT: pand %xmm0, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm8
+; SSE2-NEXT: por %xmm9, %xmm8
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movaps %xmm0, %xmm4
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm5, %xmm4
+; SSE2-NEXT: por %xmm0, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movaps %xmm0, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm6, %xmm5
+; SSE2-NEXT: por %xmm0, %xmm5
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movaps %xmm0, %xmm6
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: pandn %xmm7, %xmm6
+; SSE2-NEXT: por %xmm0, %xmm6
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v16f32:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm5, %ymm5
+; AVX-NEXT: vmovd %eax, %ymm5
+; AVX-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
+; AVX-NEXT: vmovdqa %ymm5, %ymm4
+; AVX-NEXT: pand %ymm0, %ymm5
+; AVX-NEXT: pandn %ymm2, %ymm4
+; AVX-NEXT: por %ymm5, %ymm4
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm0, %ymm0
+; AVX-NEXT: vmovd %eax, %ymm0
+; AVX-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
+; AVX-NEXT: vmovdqa %ymm0, %ymm2
+; AVX-NEXT: pand %ymm1, %ymm0
+; AVX-NEXT: pandn %ymm3, %ymm2
+; AVX-NEXT: por %ymm0, %ymm2
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v16f32:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm5, %ymm5
+; AVX2-NEXT: vmovd %eax, %ymm5
+; AVX2-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
+; AVX2-NEXT: vmovdqa %ymm5, %ymm4
+; AVX2-NEXT: pand %ymm0, %ymm5
+; AVX2-NEXT: pandn %ymm2, %ymm4
+; AVX2-NEXT: por %ymm5, %ymm4
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm0, %ymm0
+; AVX2-NEXT: vmovd %eax, %ymm0
+; AVX2-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
+; AVX2-NEXT: vmovdqa %ymm0, %ymm2
+; AVX2-NEXT: pand %ymm1, %ymm0
+; AVX2-NEXT: pandn %ymm3, %ymm2
+; AVX2-NEXT: por %ymm0, %ymm2
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v16f32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB9_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %zmm0, %zmm1
+; AVX512-NEXT: .LBB9_2:
+; AVX512-NEXT: vmovaps %zmm1, %zmm0
+; AVX512-NEXT: retq
+ %result = call <16 x float> @llvm.ct.select.v16f32(i1 %cond, <16 x float> %a, <16 x float> %b)
+ ret <16 x float> %result
+}
+
+define <8 x i64> @test_ctselect_v8i64(i1 %cond, <8 x i64> %a, <8 x i64> %b) {
+; SSE2-LABEL: test_ctselect_v8i64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm9, %xmm9
+; SSE2-NEXT: movd %eax, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm9, %xmm8
+; SSE2-NEXT: pand %xmm0, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm8
+; SSE2-NEXT: por %xmm9, %xmm8
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm5, %xmm4
+; SSE2-NEXT: por %xmm0, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm6, %xmm5
+; SSE2-NEXT: por %xmm0, %xmm5
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm0, %xmm6
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: pandn %xmm7, %xmm6
+; SSE2-NEXT: por %xmm0, %xmm6
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v8i64:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm5, %ymm5
+; AVX-NEXT: vmovd %eax, %ymm5
+; AVX-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX-NEXT: vmovdqa %ymm5, %ymm4
+; AVX-NEXT: pand %ymm0, %ymm5
+; AVX-NEXT: pandn %ymm2, %ymm4
+; AVX-NEXT: por %ymm5, %ymm4
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm0, %ymm0
+; AVX-NEXT: vmovd %eax, %ymm0
+; AVX-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
+; AVX-NEXT: vmovdqa %ymm0, %ymm2
+; AVX-NEXT: pand %ymm1, %ymm0
+; AVX-NEXT: pandn %ymm3, %ymm2
+; AVX-NEXT: por %ymm0, %ymm2
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v8i64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm5, %ymm5
+; AVX2-NEXT: vmovd %eax, %ymm5
+; AVX2-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX2-NEXT: vmovdqa %ymm5, %ymm4
+; AVX2-NEXT: pand %ymm0, %ymm5
+; AVX2-NEXT: pandn %ymm2, %ymm4
+; AVX2-NEXT: por %ymm5, %ymm4
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm0, %ymm0
+; AVX2-NEXT: vmovd %eax, %ymm0
+; AVX2-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
+; AVX2-NEXT: vmovdqa %ymm0, %ymm2
+; AVX2-NEXT: pand %ymm1, %ymm0
+; AVX2-NEXT: pandn %ymm3, %ymm2
+; AVX2-NEXT: por %ymm0, %ymm2
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v8i64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB10_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %zmm0, %zmm1
+; AVX512-NEXT: .LBB10_2:
+; AVX512-NEXT: vmovaps %zmm1, %zmm0
+; AVX512-NEXT: retq
+ %result = call <8 x i64> @llvm.ct.select.v8i64(i1 %cond, <8 x i64> %a, <8 x i64> %b)
+ ret <8 x i64> %result
+}
+
+define <8 x double> @test_ctselect_v8f64(i1 %cond, <8 x double> %a, <8 x double> %b) {
+; SSE2-LABEL: test_ctselect_v8f64:
+; SSE2: # %bb.0:
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm9, %xmm9
+; SSE2-NEXT: movd %eax, %xmm9
+; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
+; SSE2-NEXT: movapd %xmm9, %xmm8
+; SSE2-NEXT: pand %xmm0, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm8
+; SSE2-NEXT: por %xmm9, %xmm8
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movapd %xmm0, %xmm4
+; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: pandn %xmm5, %xmm4
+; SSE2-NEXT: por %xmm0, %xmm4
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movapd %xmm0, %xmm5
+; SSE2-NEXT: pand %xmm2, %xmm0
+; SSE2-NEXT: pandn %xmm6, %xmm5
+; SSE2-NEXT: por %xmm0, %xmm5
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; SSE2-NEXT: movapd %xmm0, %xmm6
+; SSE2-NEXT: pand %xmm3, %xmm0
+; SSE2-NEXT: pandn %xmm7, %xmm6
+; SSE2-NEXT: por %xmm0, %xmm6
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v8f64:
+; AVX: # %bb.0:
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm5, %ymm5
+; AVX-NEXT: vmovd %eax, %ymm5
+; AVX-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX-NEXT: vmovdqa %ymm5, %ymm4
+; AVX-NEXT: pand %ymm0, %ymm5
+; AVX-NEXT: pandn %ymm2, %ymm4
+; AVX-NEXT: por %ymm5, %ymm4
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %ymm0, %ymm0
+; AVX-NEXT: vmovd %eax, %ymm0
+; AVX-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
+; AVX-NEXT: vmovdqa %ymm0, %ymm2
+; AVX-NEXT: pand %ymm1, %ymm0
+; AVX-NEXT: pandn %ymm3, %ymm2
+; AVX-NEXT: por %ymm0, %ymm2
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v8f64:
+; AVX2: # %bb.0:
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm5, %ymm5
+; AVX2-NEXT: vmovd %eax, %ymm5
+; AVX2-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
+; AVX2-NEXT: vmovdqa %ymm5, %ymm4
+; AVX2-NEXT: pand %ymm0, %ymm5
+; AVX2-NEXT: pandn %ymm2, %ymm4
+; AVX2-NEXT: por %ymm5, %ymm4
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %ymm0, %ymm0
+; AVX2-NEXT: vmovd %eax, %ymm0
+; AVX2-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
+; AVX2-NEXT: vmovdqa %ymm0, %ymm2
+; AVX2-NEXT: pand %ymm1, %ymm0
+; AVX2-NEXT: pandn %ymm3, %ymm2
+; AVX2-NEXT: por %ymm0, %ymm2
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v8f64:
+; AVX512: # %bb.0:
+; AVX512-NEXT: testb %dil, %dil
+; AVX512-NEXT: je .LBB11_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovapd %zmm0, %zmm1
+; AVX512-NEXT: .LBB11_2:
+; AVX512-NEXT: vmovapd %zmm1, %zmm0
+; AVX512-NEXT: retq
+ %result = call <8 x double> @llvm.ct.select.v8f64(i1 %cond, <8 x double> %a, <8 x double> %b)
+ ret <8 x double> %result
+}
+
+; Test with constant conditions for vector types
+define <4 x i32> @test_ctselect_v4i32_const_true(<4 x i32> %a, <4 x i32> %b) {
+; SSE2-LABEL: test_ctselect_v4i32_const_true:
+; SSE2: # %bb.0:
+; SSE2-NEXT: movb $1, %al
+; SSE2-NEXT: testb %al, %al
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4i32_const_true:
+; AVX: # %bb.0:
+; AVX-NEXT: movb $1, %al
+; AVX-NEXT: testb %al, %al
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4i32_const_true:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movb $1, %al
+; AVX2-NEXT: testb %al, %al
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4i32_const_true:
+; AVX512: # %bb.0:
+; AVX512-NEXT: retq
+ %result = call <4 x i32> @llvm.ct.select.v4i32(i1 true, <4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %result
+}
+
+define <4 x i32> @test_ctselect_v4i32_const_false(<4 x i32> %a, <4 x i32> %b) {
+; SSE2-LABEL: test_ctselect_v4i32_const_false:
+; SSE2: # %bb.0:
+; SSE2-NEXT: xorl %eax, %eax
+; SSE2-NEXT: testb %al, %al
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4i32_const_false:
+; AVX: # %bb.0:
+; AVX-NEXT: xorl %eax, %eax
+; AVX-NEXT: testb %al, %al
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4i32_const_false:
+; AVX2: # %bb.0:
+; AVX2-NEXT: xorl %eax, %eax
+; AVX2-NEXT: testb %al, %al
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4i32_const_false:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vmovaps %xmm1, %xmm0
+; AVX512-NEXT: retq
+ %result = call <4 x i32> @llvm.ct.select.v4i32(i1 false, <4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %result
+}
+
+; Test with comparison conditions for vector types
+define <4 x i32> @test_ctselect_v4i32_icmp(i32 %x, i32 %y, <4 x i32> %a, <4 x i32> %b) {
+; SSE2-LABEL: test_ctselect_v4i32_icmp:
+; SSE2: # %bb.0:
+; SSE2-NEXT: cmpl %esi, %edi
+; SSE2-NEXT: sete %al
+; SSE2-NEXT: testb %al, %al
+; SSE2-NEXT: movl $0, %eax
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: movzbl %al, %eax
+; SSE2-NEXT: negl %eax
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: movd %eax, %xmm3
+; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; SSE2-NEXT: movdqa %xmm3, %xmm2
+; SSE2-NEXT: pand %xmm0, %xmm3
+; SSE2-NEXT: pandn %xmm1, %xmm2
+; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm0
+; SSE2-NEXT: retq
+;
+; AVX-LABEL: test_ctselect_v4i32_icmp:
+; AVX: # %bb.0:
+; AVX-NEXT: cmpl %esi, %edi
+; AVX-NEXT: sete %al
+; AVX-NEXT: testb %al, %al
+; AVX-NEXT: movl $0, %eax
+; AVX-NEXT: setne %al
+; AVX-NEXT: movzbl %al, %eax
+; AVX-NEXT: negl %eax
+; AVX-NEXT: pxor %xmm3, %xmm3
+; AVX-NEXT: movd %eax, %xmm3
+; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX-NEXT: movdqa %xmm3, %xmm2
+; AVX-NEXT: pand %xmm0, %xmm3
+; AVX-NEXT: pandn %xmm1, %xmm2
+; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: vmovaps %xmm2, %xmm0
+; AVX-NEXT: retq
+;
+; AVX2-LABEL: test_ctselect_v4i32_icmp:
+; AVX2: # %bb.0:
+; AVX2-NEXT: cmpl %esi, %edi
+; AVX2-NEXT: sete %al
+; AVX2-NEXT: testb %al, %al
+; AVX2-NEXT: movl $0, %eax
+; AVX2-NEXT: setne %al
+; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: negl %eax
+; AVX2-NEXT: pxor %xmm3, %xmm3
+; AVX2-NEXT: movd %eax, %xmm3
+; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
+; AVX2-NEXT: movdqa %xmm3, %xmm2
+; AVX2-NEXT: pand %xmm0, %xmm3
+; AVX2-NEXT: pandn %xmm1, %xmm2
+; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: vmovaps %xmm2, %xmm0
+; AVX2-NEXT: retq
+; AVX512-LABEL: test_ctselect_v4i32_icmp:
+; AVX512: # %bb.0:
+; AVX512-NEXT: cmpl %esi, %edi
+; AVX512-NEXT: je .LBB14_2
+; AVX512-NEXT: # %bb.1:
+; AVX512-NEXT: vmovaps %xmm1, %xmm0
+; AVX512-NEXT: .LBB14_2:
+; AVX512-NEXT: retq
+ %cond = icmp eq i32 %x, %y
+ %result = call <4 x i32> @llvm.ct.select.v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %result
+}
+
+; Declare the intrinsics
+declare <4 x i32> @llvm.ct.select.v4i32(i1, <4 x i32>, <4 x i32>)
+declare <4 x float> @llvm.ct.select.v4f32(i1, <4 x float>, <4 x float>)
+declare <2 x i64> @llvm.ct.select.v2i64(i1, <2 x i64>, <2 x i64>)
+declare <2 x double> @llvm.ct.select.v2f64(i1, <2 x double>, <2 x double>)
+declare <8 x i32> @llvm.ct.select.v8i32(i1, <8 x i32>, <8 x i32>)
+declare <8 x float> @llvm.ct.select.v8f32(i1, <8 x float>, <8 x float>)
+declare <4 x i64> @llvm.ct.select.v4i64(i1, <4 x i64>, <4 x i64>)
+declare <4 x double> @llvm.ct.select.v4f64(i1, <4 x double>, <4 x double>)
+declare <16 x i32> @llvm.ct.select.v16i32(i1, <16 x i32>, <16 x i32>)
+declare <16 x float> @llvm.ct.select.v16f32(i1, <16 x float>, <16 x float>)
+declare <8 x i64> @llvm.ct.select.v8i64(i1, <8 x i64>, <8 x i64>)
+declare <8 x double> @llvm.ct.select.v8f64(i1, <8 x double>, <8 x double>)
>From e785e33a212b78cd49e0bce166ceb74efadd40e0 Mon Sep 17 00:00:00 2001
From: wizardengineer <juliuswoosebert at gmail.com>
Date: Wed, 5 Nov 2025 23:56:12 -0500
Subject: [PATCH 3/5] [LLVM][X86] Add f80 support for ct.select
Add special handling for x86_fp80 types in CTSELECT lowering by splitting
them into three 32-bit chunks, performing constant-time selection on each
chunk, and reassembling the result. This fixes crashes when compiling
tests with f80 types.
Also updated ctselect.ll to match current generic fallback implementation.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 75 +-
llvm/lib/Target/X86/X86ISelLowering.h | 2 -
llvm/lib/Target/X86/X86InstrFragments.td | 2 +-
llvm/lib/Target/X86/X86InstrInfo.cpp | 919 +++++++++++----------
llvm/lib/Target/X86/X86InstrInfo.h | 9 +-
llvm/test/CodeGen/X86/ctselect-i386-fp.ll | 272 +++---
llvm/test/CodeGen/X86/ctselect-i386-mmx.ll | 441 +++++-----
llvm/test/CodeGen/X86/ctselect-vector.ll | 703 +++++-----------
8 files changed, 1062 insertions(+), 1361 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 13241b793ac19..0bb638aa41700 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -26284,6 +26284,69 @@ SDValue X86TargetLowering::LowerCT_SELECT(SDValue Op, SelectionDAG &DAG) const {
return DAG.getBitcast(VT, CtSelect);
}
+ // Handle f80 types by splitting into three 32-bit chunks
+ if (VT == MVT::f80) {
+ SDValue Chain = DAG.getEntryNode();
+
+ // Create temporary stack slots for input f80 values
+ SDValue TrueSlot = DAG.CreateStackTemporary(MVT::f80);
+ SDValue FalseSlot = DAG.CreateStackTemporary(MVT::f80);
+
+ // Store f80 values to memory
+ SDValue StoreTrueF80 =
+ DAG.getStore(Chain, DL, TrueOp, TrueSlot, MachinePointerInfo());
+ SDValue StoreFalseF80 =
+ DAG.getStore(Chain, DL, FalseOp, FalseSlot, MachinePointerInfo());
+
+ // Load i32 parts from memory (3 chunks for 96-bit f80 storage)
+ SDValue TruePart0 =
+ DAG.getLoad(MVT::i32, DL, StoreTrueF80, TrueSlot, MachinePointerInfo());
+ SDValue TruePart1Ptr =
+ DAG.getMemBasePlusOffset(TrueSlot, TypeSize::getFixed(4), DL);
+ SDValue TruePart1 = DAG.getLoad(MVT::i32, DL, StoreTrueF80, TruePart1Ptr,
+ MachinePointerInfo());
+ SDValue TruePart2Ptr =
+ DAG.getMemBasePlusOffset(TrueSlot, TypeSize::getFixed(8), DL);
+ SDValue TruePart2 = DAG.getLoad(MVT::i32, DL, StoreTrueF80, TruePart2Ptr,
+ MachinePointerInfo());
+
+ SDValue FalsePart0 = DAG.getLoad(MVT::i32, DL, StoreFalseF80, FalseSlot,
+ MachinePointerInfo());
+ SDValue FalsePart1Ptr =
+ DAG.getMemBasePlusOffset(FalseSlot, TypeSize::getFixed(4), DL);
+ SDValue FalsePart1 = DAG.getLoad(MVT::i32, DL, StoreFalseF80, FalsePart1Ptr,
+ MachinePointerInfo());
+ SDValue FalsePart2Ptr =
+ DAG.getMemBasePlusOffset(FalseSlot, TypeSize::getFixed(8), DL);
+ SDValue FalsePart2 = DAG.getLoad(MVT::i32, DL, StoreFalseF80, FalsePart2Ptr,
+ MachinePointerInfo());
+
+ // Perform CT_SELECT on each 32-bit chunk
+ SDValue Part0Ops[] = {FalsePart0, TruePart0, CC, ProcessedCond};
+ SDValue Part0Select = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part0Ops);
+ SDValue Part1Ops[] = {FalsePart1, TruePart1, CC, ProcessedCond};
+ SDValue Part1Select = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part1Ops);
+ SDValue Part2Ops[] = {FalsePart2, TruePart2, CC, ProcessedCond};
+ SDValue Part2Select = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part2Ops);
+
+ // Create result stack slot and store the selected parts
+ SDValue ResultSlot = DAG.CreateStackTemporary(MVT::f80);
+ SDValue StorePart0 =
+ DAG.getStore(Chain, DL, Part0Select, ResultSlot, MachinePointerInfo());
+ SDValue ResPart1Ptr =
+ DAG.getMemBasePlusOffset(ResultSlot, TypeSize::getFixed(4), DL);
+ SDValue StorePart1 = DAG.getStore(StorePart0, DL, Part1Select, ResPart1Ptr,
+ MachinePointerInfo());
+ SDValue ResPart2Ptr =
+ DAG.getMemBasePlusOffset(ResultSlot, TypeSize::getFixed(8), DL);
+ SDValue StorePart2 = DAG.getStore(StorePart1, DL, Part2Select, ResPart2Ptr,
+ MachinePointerInfo());
+
+ // Load complete f80 result from memory
+ return DAG.getLoad(MVT::f80, DL, StorePart2, ResultSlot,
+ MachinePointerInfo());
+ }
+
// Create final CT_SELECT node
SDValue Ops[] = {FalseOp, TrueOp, CC, ProcessedCond};
return DAG.getNode(X86ISD::CT_SELECT, DL, Op.getValueType(), Ops,
@@ -34876,12 +34939,6 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
}
}
-bool X86TargetLowering::isSelectSupported(SelectSupportKind Kind) const {
- if (Kind == SelectSupportKind::CtSelect) {
- return true;
- }
- return TargetLoweringBase::isSelectSupported(Kind);
-}
/// Replace a node with an illegal result type with a new node built out of
/// custom code.
void X86TargetLowering::ReplaceNodeResults(SDNode *N,
@@ -36498,6 +36555,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
}
}
+/// Return true if the addressing mode represented by AM is legal for this
+/// target, for a load/store of the specified type.
+bool X86TargetLowering::isLegalAddressingMode(const DataLayout &DL,
+ const AddrMode &AM, Type *Ty,
+ unsigned AS,
+ Instruction *I) const {
// X86 supports extremely general addressing modes.
CodeModel::Model M = getTargetMachine().getCodeModel();
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index fc8f13704d7d3..6eb8b4e87f429 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -176,8 +176,6 @@ namespace llvm {
///
SDValue LowerOperation(SDValue Op, SelectionDAG &DAG) const override;
- bool isSelectSupported(SelectSupportKind Kind) const override;
-
/// Replace the results of node with an illegal result
/// type with new values built out of custom code.
///
diff --git a/llvm/lib/Target/X86/X86InstrFragments.td b/llvm/lib/Target/X86/X86InstrFragments.td
index da85a1a0366e1..cb59636af2831 100644
--- a/llvm/lib/Target/X86/X86InstrFragments.td
+++ b/llvm/lib/Target/X86/X86InstrFragments.td
@@ -177,7 +177,7 @@ def X86ctest : SDNode<"X86ISD::CTEST", SDTX86Ccmp>;
def X86cload : SDNode<"X86ISD::CLOAD", SDTX86Cload, [SDNPHasChain, SDNPMayLoad, SDNPMemOperand]>;
def X86cstore : SDNode<"X86ISD::CSTORE", SDTX86Cstore, [SDNPHasChain, SDNPMayStore, SDNPMemOperand]>;
-def X86ct_select: SDNode<"X86ISD::CT_SELECT", SDTX86CtSelect, [SDNPInGlue]>;
+def X86ct_select: SDNode<"X86ISD::CT_SELECT", SDTX86CtSelect>;
// X86 conditional moves. Operand 0 and operand 1 are the two values
// to select from. Operand 2 is the condition code, and operand 3 is the
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index 0433e689254a4..f584f730d3310 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -689,8 +689,7 @@ bool X86InstrInfo::expandCtSelectVector(MachineInstr &MI) const {
.addImm(31));
} else {
// Negate to convert 1 -> 0xFFFFFFFF, 0 -> 0x00000000 (negl %eax)
- recordInstr(BuildMI(*MBB, MI, DL, get(X86::NEG32r), TmpGPR)
- .addReg(TmpGPR));
+ recordInstr(BuildMI(*MBB, MI, DL, get(X86::NEG32r), TmpGPR).addReg(TmpGPR));
}
// Broadcast to TmpX (vector mask)
@@ -847,7 +846,8 @@ bool X86InstrInfo::expandCtSelectVector(MachineInstr &MI) const {
.setMIFlags(MachineInstr::MIFlag::NoMerge));
}
- assert(FirstInstr && LastInstr && "Expected at least one expanded instruction");
+ assert(FirstInstr && LastInstr &&
+ "Expected at least one expanded instruction");
auto BundleEnd = LastInstr->getIterator();
finalizeBundle(*MBB, FirstInstr->getIterator(), std::next(BundleEnd));
@@ -915,25 +915,28 @@ bool X86InstrInfo::expandCtSelectWithCMOV(MachineInstr &MI) const {
/// Expand i386-specific CT_SELECT pseudo instructions (post-RA, constant-time)
/// These internal pseudos receive a pre-materialized condition byte from the
-/// custom inserter, avoiding EFLAGS corruption issues during i64 type legalization.
+/// custom inserter, avoiding EFLAGS corruption issues during i64 type
+/// legalization.
bool X86InstrInfo::expandCtSelectIntWithoutCMOV(MachineInstr &MI) const {
MachineBasicBlock *MBB = MI.getParent();
DebugLoc DL = MI.getDebugLoc();
// CT_SELECT_I386_INT_GRxxrr has operands: (outs dst, tmp_byte, tmp_mask),
// (ins src1, src2, cond_byte)
- // Note: cond_byte is pre-materialized by custom inserter, not EFLAGS-dependent
+ // Note: cond_byte is pre-materialized by custom inserter, not
+ // EFLAGS-dependent
Register DstReg = MI.getOperand(0).getReg();
Register TmpByteReg = MI.getOperand(1).getReg();
Register TmpMaskReg = MI.getOperand(2).getReg();
Register Src1Reg = MI.getOperand(3).getReg();
Register Src2Reg = MI.getOperand(4).getReg();
- Register CondByteReg = MI.getOperand(5).getReg(); // Pre-materialized condition byte
+ Register CondByteReg =
+ MI.getOperand(5).getReg(); // Pre-materialized condition byte
// Determine instruction opcodes based on register width
unsigned MovZXOp, NegOp, MovOp, AndOp, NotOp, OrOp;
if (MI.getOpcode() == X86::CT_SELECT_I386_INT_GR8rr) {
- MovZXOp = 0; // No zero-extend needed for GR8
+ MovZXOp = 0; // No zero-extend needed for GR8
NegOp = X86::NEG8r;
MovOp = X86::MOV8rr;
AndOp = X86::AND8rr;
@@ -962,8 +965,8 @@ bool X86InstrInfo::expandCtSelectIntWithoutCMOV(MachineInstr &MI) const {
// Step 1: Copy pre-materialized condition byte to TmpByteReg
// This allows the bundle to work with allocated temporaries
auto I1 = BuildMI(*MBB, MI, DL, get(X86::MOV8rr), TmpByteReg)
- .addReg(CondByteReg)
- .setMIFlag(MachineInstr::MIFlag::NoMerge);
+ .addReg(CondByteReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
auto BundleStart = I1->getIterator();
// Step 2: Zero-extend condition byte to register width (0 or 1)
@@ -974,7 +977,9 @@ bool X86InstrInfo::expandCtSelectIntWithoutCMOV(MachineInstr &MI) const {
}
// Step 3: Convert condition to bitmask (NEG: 1 -> 0xFFFF..., 0 -> 0x0000...)
- Register MaskReg = (MI.getOpcode() == X86::CT_SELECT_I386_INT_GR8rr) ? TmpByteReg : TmpMaskReg;
+ Register MaskReg = (MI.getOpcode() == X86::CT_SELECT_I386_INT_GR8rr)
+ ? TmpByteReg
+ : TmpMaskReg;
BuildMI(*MBB, MI, DL, get(NegOp), MaskReg)
.addReg(MaskReg)
.setMIFlag(MachineInstr::MIFlag::NoMerge);
@@ -1002,9 +1007,9 @@ bool X86InstrInfo::expandCtSelectIntWithoutCMOV(MachineInstr &MI) const {
// Step 8: Final result: (src1 & mask) | (src2 & ~mask)
auto LI = BuildMI(*MBB, MI, DL, get(OrOp), DstReg)
- .addReg(DstReg)
- .addReg(MaskReg)
- .setMIFlag(MachineInstr::MIFlag::NoMerge);
+ .addReg(DstReg)
+ .addReg(MaskReg)
+ .setMIFlag(MachineInstr::MIFlag::NoMerge);
// Bundle all generated instructions for atomic execution before removing MI
auto BundleEnd = std::next(LI->getIterator());
@@ -1013,11 +1018,12 @@ bool X86InstrInfo::expandCtSelectIntWithoutCMOV(MachineInstr &MI) const {
finalizeBundle(*MBB, BundleStart, BundleEnd);
}
- // TODO: Optimization opportunity - The register allocator may choose callee-saved
- // registers (e.g., %ebx, %esi) for TmpByteReg/TmpMaskReg, causing unnecessary
- // save/restore overhead. Consider constraining these to caller-saved register
- // classes (e.g., GR8_AL, GR32_CallSaved) in the TableGen definitions to improve
- // constant-time performance by eliminating prologue/epilogue instructions.
+ // TODO: Optimization opportunity - The register allocator may choose
+ // callee-saved registers (e.g., %ebx, %esi) for TmpByteReg/TmpMaskReg,
+ // causing unnecessary save/restore overhead. Consider constraining these to
+ // caller-saved register classes (e.g., GR8_AL, GR32_CallSaved) in the
+ // TableGen definitions to improve constant-time performance by eliminating
+ // prologue/epilogue instructions.
// Remove the original pseudo instruction
MI.eraseFromParent();
@@ -1305,8 +1311,7 @@ static bool regIsPICBase(Register BaseReg, const MachineRegisterInfo &MRI) {
return isPICBase;
}
-bool X86InstrInfo::isReMaterializableImpl(
- const MachineInstr &MI) const {
+bool X86InstrInfo::isReMaterializableImpl(const MachineInstr &MI) const {
switch (MI.getOpcode()) {
default:
// This function should only be called for opcodes with the ReMaterializable
@@ -1824,32 +1829,32 @@ MachineInstr *X86InstrInfo::convertToThreeAddressWithLEA(unsigned MIOpc,
switch (MIOpc) {
default:
llvm_unreachable("Unreachable!");
- CASE_NF(SHL8ri)
- CASE_NF(SHL16ri) {
- unsigned ShAmt = MI.getOperand(2).getImm();
- MIB.addReg(0)
- .addImm(1LL << ShAmt)
- .addReg(InRegLEA, RegState::Kill)
- .addImm(0)
- .addReg(0);
- break;
- }
- CASE_NF(INC8r)
- CASE_NF(INC16r)
+ CASE_NF(SHL8ri)
+ CASE_NF(SHL16ri) {
+ unsigned ShAmt = MI.getOperand(2).getImm();
+ MIB.addReg(0)
+ .addImm(1LL << ShAmt)
+ .addReg(InRegLEA, RegState::Kill)
+ .addImm(0)
+ .addReg(0);
+ break;
+ }
+ CASE_NF(INC8r)
+ CASE_NF(INC16r)
addRegOffset(MIB, InRegLEA, true, 1);
break;
- CASE_NF(DEC8r)
- CASE_NF(DEC16r)
+ CASE_NF(DEC8r)
+ CASE_NF(DEC16r)
addRegOffset(MIB, InRegLEA, true, -1);
break;
- CASE_NF(ADD8ri)
- CASE_NF(ADD16ri)
+ CASE_NF(ADD8ri)
+ CASE_NF(ADD16ri)
case X86::ADD8ri_DB:
case X86::ADD16ri_DB:
addRegOffset(MIB, InRegLEA, true, MI.getOperand(2).getImm());
break;
- CASE_NF(ADD8rr)
- CASE_NF(ADD16rr)
+ CASE_NF(ADD8rr)
+ CASE_NF(ADD16rr)
case X86::ADD8rr_DB:
case X86::ADD16rr_DB: {
Src2 = MI.getOperand(2).getReg();
@@ -1987,128 +1992,129 @@ MachineInstr *X86InstrInfo::convertToThreeAddress(MachineInstr &MI,
switch (MIOpc) {
default:
llvm_unreachable("Unreachable!");
- CASE_NF(SHL64ri) {
- assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
- unsigned ShAmt = getTruncatedShiftCount(MI, 2);
- if (!isTruncatedShiftCountForLEA(ShAmt))
- return nullptr;
-
- // LEA can't handle RSP.
- if (Src.getReg().isVirtual() && !MF.getRegInfo().constrainRegClass(
- Src.getReg(), &X86::GR64_NOSPRegClass))
- return nullptr;
+ CASE_NF(SHL64ri) {
+ assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
+ unsigned ShAmt = getTruncatedShiftCount(MI, 2);
+ if (!isTruncatedShiftCountForLEA(ShAmt))
+ return nullptr;
- NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r))
- .add(Dest)
- .addReg(0)
- .addImm(1LL << ShAmt)
- .add(Src)
- .addImm(0)
- .addReg(0);
- break;
- }
- CASE_NF(SHL32ri) {
- assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
- unsigned ShAmt = getTruncatedShiftCount(MI, 2);
- if (!isTruncatedShiftCountForLEA(ShAmt))
- return nullptr;
+ // LEA can't handle RSP.
+ if (Src.getReg().isVirtual() &&
+ !MF.getRegInfo().constrainRegClass(Src.getReg(),
+ &X86::GR64_NOSPRegClass))
+ return nullptr;
- unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
+ NewMI = BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r))
+ .add(Dest)
+ .addReg(0)
+ .addImm(1LL << ShAmt)
+ .add(Src)
+ .addImm(0)
+ .addReg(0);
+ break;
+ }
+ CASE_NF(SHL32ri) {
+ assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
+ unsigned ShAmt = getTruncatedShiftCount(MI, 2);
+ if (!isTruncatedShiftCountForLEA(ShAmt))
+ return nullptr;
- // LEA can't handle ESP.
- bool isKill;
- MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
- if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
- isKill, ImplicitOp, LV, LIS))
- return nullptr;
+ unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
- MachineInstrBuilder MIB =
- BuildMI(MF, MI.getDebugLoc(), get(Opc))
- .add(Dest)
- .addReg(0)
- .addImm(1LL << ShAmt)
- .addReg(SrcReg, getKillRegState(isKill), SrcSubReg)
- .addImm(0)
- .addReg(0);
- if (ImplicitOp.getReg() != 0)
- MIB.add(ImplicitOp);
- NewMI = MIB;
+ // LEA can't handle ESP.
+ bool isKill;
+ MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
+ if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
+ isKill, ImplicitOp, LV, LIS))
+ return nullptr;
- // Add kills if classifyLEAReg created a new register.
- if (LV && SrcReg != Src.getReg())
- LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
- break;
- }
- CASE_NF(SHL8ri)
+ MachineInstrBuilder MIB =
+ BuildMI(MF, MI.getDebugLoc(), get(Opc))
+ .add(Dest)
+ .addReg(0)
+ .addImm(1LL << ShAmt)
+ .addReg(SrcReg, getKillRegState(isKill), SrcSubReg)
+ .addImm(0)
+ .addReg(0);
+ if (ImplicitOp.getReg() != 0)
+ MIB.add(ImplicitOp);
+ NewMI = MIB;
+
+ // Add kills if classifyLEAReg created a new register.
+ if (LV && SrcReg != Src.getReg())
+ LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
+ break;
+ }
+ CASE_NF(SHL8ri)
Is8BitOp = true;
[[fallthrough]];
- CASE_NF(SHL16ri) {
- assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
- unsigned ShAmt = getTruncatedShiftCount(MI, 2);
- if (!isTruncatedShiftCountForLEA(ShAmt))
- return nullptr;
- return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
- }
- CASE_NF(INC64r)
- CASE_NF(INC32r) {
- assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!");
- unsigned Opc = (MIOpc == X86::INC64r || MIOpc == X86::INC64r_NF)
- ? X86::LEA64r
- : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
- bool isKill;
- MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
- if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
- isKill, ImplicitOp, LV, LIS))
- return nullptr;
-
- MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
- .add(Dest)
- .addReg(SrcReg, getKillRegState(isKill));
- if (ImplicitOp.getReg() != 0)
- MIB.add(ImplicitOp);
+ CASE_NF(SHL16ri) {
+ assert(MI.getNumOperands() >= 3 && "Unknown shift instruction!");
+ unsigned ShAmt = getTruncatedShiftCount(MI, 2);
+ if (!isTruncatedShiftCountForLEA(ShAmt))
+ return nullptr;
+ return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
+ }
+ CASE_NF(INC64r)
+ CASE_NF(INC32r) {
+ assert(MI.getNumOperands() >= 2 && "Unknown inc instruction!");
+ unsigned Opc = (MIOpc == X86::INC64r || MIOpc == X86::INC64r_NF)
+ ? X86::LEA64r
+ : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
+ bool isKill;
+ MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
+ if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
+ isKill, ImplicitOp, LV, LIS))
+ return nullptr;
- NewMI = addOffset(MIB, 1);
+ MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
+ .add(Dest)
+ .addReg(SrcReg, getKillRegState(isKill));
+ if (ImplicitOp.getReg() != 0)
+ MIB.add(ImplicitOp);
- // Add kills if classifyLEAReg created a new register.
- if (LV && SrcReg != Src.getReg())
- LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
- break;
- }
- CASE_NF(DEC64r)
- CASE_NF(DEC32r) {
- assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!");
- unsigned Opc = (MIOpc == X86::DEC64r || MIOpc == X86::DEC64r_NF)
- ? X86::LEA64r
- : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
+ NewMI = addOffset(MIB, 1);
- bool isKill;
- MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
- if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
- isKill, ImplicitOp, LV, LIS))
- return nullptr;
+ // Add kills if classifyLEAReg created a new register.
+ if (LV && SrcReg != Src.getReg())
+ LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
+ break;
+ }
+ CASE_NF(DEC64r)
+ CASE_NF(DEC32r) {
+ assert(MI.getNumOperands() >= 2 && "Unknown dec instruction!");
+ unsigned Opc = (MIOpc == X86::DEC64r || MIOpc == X86::DEC64r_NF)
+ ? X86::LEA64r
+ : (Is64Bit ? X86::LEA64_32r : X86::LEA32r);
+
+ bool isKill;
+ MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
+ if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/false, SrcReg, SrcSubReg,
+ isKill, ImplicitOp, LV, LIS))
+ return nullptr;
- MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
- .add(Dest)
- .addReg(SrcReg, getKillRegState(isKill));
- if (ImplicitOp.getReg() != 0)
- MIB.add(ImplicitOp);
+ MachineInstrBuilder MIB = BuildMI(MF, MI.getDebugLoc(), get(Opc))
+ .add(Dest)
+ .addReg(SrcReg, getKillRegState(isKill));
+ if (ImplicitOp.getReg() != 0)
+ MIB.add(ImplicitOp);
- NewMI = addOffset(MIB, -1);
+ NewMI = addOffset(MIB, -1);
- // Add kills if classifyLEAReg created a new register.
- if (LV && SrcReg != Src.getReg())
- LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
- break;
- }
- CASE_NF(DEC8r)
- CASE_NF(INC8r)
+ // Add kills if classifyLEAReg created a new register.
+ if (LV && SrcReg != Src.getReg())
+ LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
+ break;
+ }
+ CASE_NF(DEC8r)
+ CASE_NF(INC8r)
Is8BitOp = true;
[[fallthrough]];
- CASE_NF(DEC16r)
- CASE_NF(INC16r)
+ CASE_NF(DEC16r)
+ CASE_NF(INC16r)
return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
- CASE_NF(ADD64rr)
- CASE_NF(ADD32rr)
+ CASE_NF(ADD64rr)
+ CASE_NF(ADD32rr)
case X86::ADD64rr_DB:
case X86::ADD32rr_DB: {
assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
@@ -2159,21 +2165,21 @@ MachineInstr *X86InstrInfo::convertToThreeAddress(MachineInstr &MI,
NumRegOperands = 3;
break;
}
- CASE_NF(ADD8rr)
+ CASE_NF(ADD8rr)
case X86::ADD8rr_DB:
Is8BitOp = true;
[[fallthrough]];
- CASE_NF(ADD16rr)
+ CASE_NF(ADD16rr)
case X86::ADD16rr_DB:
return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
- CASE_NF(ADD64ri32)
+ CASE_NF(ADD64ri32)
case X86::ADD64ri32_DB:
assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
NewMI = addOffset(
BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src),
MI.getOperand(2));
break;
- CASE_NF(ADD32ri)
+ CASE_NF(ADD32ri)
case X86::ADD32ri_DB: {
assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
@@ -2198,62 +2204,62 @@ MachineInstr *X86InstrInfo::convertToThreeAddress(MachineInstr &MI,
LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
break;
}
- CASE_NF(ADD8ri)
+ CASE_NF(ADD8ri)
case X86::ADD8ri_DB:
Is8BitOp = true;
[[fallthrough]];
- CASE_NF(ADD16ri)
+ CASE_NF(ADD16ri)
case X86::ADD16ri_DB:
return convertToThreeAddressWithLEA(MIOpc, MI, LV, LIS, Is8BitOp);
- CASE_NF(SUB8ri)
- CASE_NF(SUB16ri)
+ CASE_NF(SUB8ri)
+ CASE_NF(SUB16ri)
/// FIXME: Support these similar to ADD8ri/ADD16ri*.
return nullptr;
- CASE_NF(SUB32ri) {
- if (!MI.getOperand(2).isImm())
- return nullptr;
- int64_t Imm = MI.getOperand(2).getImm();
- if (!isInt<32>(-Imm))
- return nullptr;
+ CASE_NF(SUB32ri) {
+ if (!MI.getOperand(2).isImm())
+ return nullptr;
+ int64_t Imm = MI.getOperand(2).getImm();
+ if (!isInt<32>(-Imm))
+ return nullptr;
- assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
- unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
+ assert(MI.getNumOperands() >= 3 && "Unknown add instruction!");
+ unsigned Opc = Is64Bit ? X86::LEA64_32r : X86::LEA32r;
- bool isKill;
- MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
- if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, SrcSubReg,
- isKill, ImplicitOp, LV, LIS))
- return nullptr;
+ bool isKill;
+ MachineOperand ImplicitOp = MachineOperand::CreateReg(0, false);
+ if (!classifyLEAReg(MI, Src, Opc, /*AllowSP=*/true, SrcReg, SrcSubReg,
+ isKill, ImplicitOp, LV, LIS))
+ return nullptr;
- MachineInstrBuilder MIB =
- BuildMI(MF, MI.getDebugLoc(), get(Opc))
- .add(Dest)
- .addReg(SrcReg, getKillRegState(isKill), SrcSubReg);
- if (ImplicitOp.getReg() != 0)
- MIB.add(ImplicitOp);
+ MachineInstrBuilder MIB =
+ BuildMI(MF, MI.getDebugLoc(), get(Opc))
+ .add(Dest)
+ .addReg(SrcReg, getKillRegState(isKill), SrcSubReg);
+ if (ImplicitOp.getReg() != 0)
+ MIB.add(ImplicitOp);
- NewMI = addOffset(MIB, -Imm);
+ NewMI = addOffset(MIB, -Imm);
- // Add kills if classifyLEAReg created a new register.
- if (LV && SrcReg != Src.getReg())
- LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
- break;
- }
+ // Add kills if classifyLEAReg created a new register.
+ if (LV && SrcReg != Src.getReg())
+ LV->getVarInfo(SrcReg).Kills.push_back(NewMI);
+ break;
+ }
- CASE_NF(SUB64ri32) {
- if (!MI.getOperand(2).isImm())
- return nullptr;
- int64_t Imm = MI.getOperand(2).getImm();
- if (!isInt<32>(-Imm))
- return nullptr;
+ CASE_NF(SUB64ri32) {
+ if (!MI.getOperand(2).isImm())
+ return nullptr;
+ int64_t Imm = MI.getOperand(2).getImm();
+ if (!isInt<32>(-Imm))
+ return nullptr;
- assert(MI.getNumOperands() >= 3 && "Unknown sub instruction!");
+ assert(MI.getNumOperands() >= 3 && "Unknown sub instruction!");
- MachineInstrBuilder MIB =
- BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src);
- NewMI = addOffset(MIB, -Imm);
- break;
- }
+ MachineInstrBuilder MIB =
+ BuildMI(MF, MI.getDebugLoc(), get(X86::LEA64r)).add(Dest).add(Src);
+ NewMI = addOffset(MIB, -Imm);
+ break;
+ }
case X86::VMOVDQU8Z128rmk:
case X86::VMOVDQU8Z256rmk:
@@ -2853,17 +2859,17 @@ MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
case X86::OP##_ND:
switch (Opc) {
- // SHLD B, C, I <-> SHRD C, B, (BitWidth - I)
- CASE_ND(SHRD16rri8)
- CASE_ND(SHLD16rri8)
- CASE_ND(SHRD32rri8)
- CASE_ND(SHLD32rri8)
- CASE_ND(SHRD64rri8)
- CASE_ND(SHLD64rri8) {
- unsigned Size;
- switch (Opc) {
- default:
- llvm_unreachable("Unreachable!");
+ // SHLD B, C, I <-> SHRD C, B, (BitWidth - I)
+ CASE_ND(SHRD16rri8)
+ CASE_ND(SHLD16rri8)
+ CASE_ND(SHRD32rri8)
+ CASE_ND(SHLD32rri8)
+ CASE_ND(SHRD64rri8)
+ CASE_ND(SHLD64rri8) {
+ unsigned Size;
+ switch (Opc) {
+ default:
+ llvm_unreachable("Unreachable!");
#define FROM_TO_SIZE(A, B, S) \
case X86::A: \
Opc = X86::B; \
@@ -2882,16 +2888,16 @@ MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
Size = S; \
break;
- FROM_TO_SIZE(SHRD16rri8, SHLD16rri8, 16)
- FROM_TO_SIZE(SHRD32rri8, SHLD32rri8, 32)
- FROM_TO_SIZE(SHRD64rri8, SHLD64rri8, 64)
+ FROM_TO_SIZE(SHRD16rri8, SHLD16rri8, 16)
+ FROM_TO_SIZE(SHRD32rri8, SHLD32rri8, 32)
+ FROM_TO_SIZE(SHRD64rri8, SHLD64rri8, 64)
#undef FROM_TO_SIZE
+ }
+ WorkingMI = CloneIfNew(MI);
+ WorkingMI->setDesc(get(Opc));
+ WorkingMI->getOperand(3).setImm(Size - MI.getOperand(3).getImm());
+ break;
}
- WorkingMI = CloneIfNew(MI);
- WorkingMI->setDesc(get(Opc));
- WorkingMI->getOperand(3).setImm(Size - MI.getOperand(3).getImm());
- break;
- }
case X86::PFSUBrr:
case X86::PFSUBRrr:
// PFSUB x, y: x = x - y
@@ -3175,15 +3181,16 @@ MachineInstr *X86InstrInfo::commuteInstructionImpl(MachineInstr &MI, bool NewMI,
WorkingMI = CloneIfNew(MI);
WorkingMI->setDesc(get(Opc));
break;
- CASE_ND(CMOV16rr)
- CASE_ND(CMOV32rr)
- CASE_ND(CMOV64rr) {
- WorkingMI = CloneIfNew(MI);
- unsigned OpNo = MI.getDesc().getNumOperands() - 1;
- X86::CondCode CC = static_cast<X86::CondCode>(MI.getOperand(OpNo).getImm());
- WorkingMI->getOperand(OpNo).setImm(X86::GetOppositeBranchCondition(CC));
- break;
- }
+ CASE_ND(CMOV16rr)
+ CASE_ND(CMOV32rr)
+ CASE_ND(CMOV64rr) {
+ WorkingMI = CloneIfNew(MI);
+ unsigned OpNo = MI.getDesc().getNumOperands() - 1;
+ X86::CondCode CC =
+ static_cast<X86::CondCode>(MI.getOperand(OpNo).getImm());
+ WorkingMI->getOperand(OpNo).setImm(X86::GetOppositeBranchCondition(CC));
+ break;
+ }
case X86::VPTERNLOGDZrri:
case X86::VPTERNLOGDZrmi:
case X86::VPTERNLOGDZ128rri:
@@ -5392,29 +5399,29 @@ bool X86InstrInfo::analyzeCompare(const MachineInstr &MI, Register &SrcReg,
CmpMask = CmpValue = 0;
}
return true;
- // A SUB can be used to perform comparison.
- CASE_ND(SUB64rm)
- CASE_ND(SUB32rm)
- CASE_ND(SUB16rm)
- CASE_ND(SUB8rm)
+ // A SUB can be used to perform comparison.
+ CASE_ND(SUB64rm)
+ CASE_ND(SUB32rm)
+ CASE_ND(SUB16rm)
+ CASE_ND(SUB8rm)
SrcReg = MI.getOperand(1).getReg();
SrcReg2 = 0;
CmpMask = 0;
CmpValue = 0;
return true;
- CASE_ND(SUB64rr)
- CASE_ND(SUB32rr)
- CASE_ND(SUB16rr)
- CASE_ND(SUB8rr)
+ CASE_ND(SUB64rr)
+ CASE_ND(SUB32rr)
+ CASE_ND(SUB16rr)
+ CASE_ND(SUB8rr)
SrcReg = MI.getOperand(1).getReg();
SrcReg2 = MI.getOperand(2).getReg();
CmpMask = 0;
CmpValue = 0;
return true;
- CASE_ND(SUB64ri32)
- CASE_ND(SUB32ri)
- CASE_ND(SUB16ri)
- CASE_ND(SUB8ri)
+ CASE_ND(SUB64ri32)
+ CASE_ND(SUB32ri)
+ CASE_ND(SUB16ri)
+ CASE_ND(SUB8ri)
SrcReg = MI.getOperand(1).getReg();
SrcReg2 = 0;
if (MI.getOperand(2).isImm()) {
@@ -5469,27 +5476,27 @@ bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI,
case X86::CMP32rr:
case X86::CMP16rr:
case X86::CMP8rr:
- CASE_ND(SUB64rr)
- CASE_ND(SUB32rr)
- CASE_ND(SUB16rr)
- CASE_ND(SUB8rr) {
- Register OISrcReg;
- Register OISrcReg2;
- int64_t OIMask;
- int64_t OIValue;
- if (!analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) ||
- OIMask != ImmMask || OIValue != ImmValue)
+ CASE_ND(SUB64rr)
+ CASE_ND(SUB32rr)
+ CASE_ND(SUB16rr)
+ CASE_ND(SUB8rr) {
+ Register OISrcReg;
+ Register OISrcReg2;
+ int64_t OIMask;
+ int64_t OIValue;
+ if (!analyzeCompare(OI, OISrcReg, OISrcReg2, OIMask, OIValue) ||
+ OIMask != ImmMask || OIValue != ImmValue)
+ return false;
+ if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) {
+ *IsSwapped = false;
+ return true;
+ }
+ if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) {
+ *IsSwapped = true;
+ return true;
+ }
return false;
- if (SrcReg == OISrcReg && SrcReg2 == OISrcReg2) {
- *IsSwapped = false;
- return true;
}
- if (SrcReg == OISrcReg2 && SrcReg2 == OISrcReg) {
- *IsSwapped = true;
- return true;
- }
- return false;
- }
case X86::CMP64ri32:
case X86::CMP32ri:
case X86::CMP16ri:
@@ -5498,10 +5505,10 @@ bool X86InstrInfo::isRedundantFlagInstr(const MachineInstr &FlagI,
case X86::TEST32ri:
case X86::TEST16ri:
case X86::TEST8ri:
- CASE_ND(SUB64ri32)
- CASE_ND(SUB32ri)
- CASE_ND(SUB16ri)
- CASE_ND(SUB8ri)
+ CASE_ND(SUB64ri32)
+ CASE_ND(SUB32ri)
+ CASE_ND(SUB16ri)
+ CASE_ND(SUB8ri)
case X86::TEST64rr:
case X86::TEST32rr:
case X86::TEST16rr:
@@ -5558,98 +5565,98 @@ inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag,
default:
return false;
- // The shift instructions only modify ZF if their shift count is non-zero.
- // N.B.: The processor truncates the shift count depending on the encoding.
- CASE_ND(SAR8ri)
- CASE_ND(SAR16ri)
- CASE_ND(SAR32ri)
- CASE_ND(SAR64ri)
- CASE_ND(SHR8ri)
- CASE_ND(SHR16ri)
- CASE_ND(SHR32ri)
- CASE_ND(SHR64ri)
+ // The shift instructions only modify ZF if their shift count is non-zero.
+ // N.B.: The processor truncates the shift count depending on the encoding.
+ CASE_ND(SAR8ri)
+ CASE_ND(SAR16ri)
+ CASE_ND(SAR32ri)
+ CASE_ND(SAR64ri)
+ CASE_ND(SHR8ri)
+ CASE_ND(SHR16ri)
+ CASE_ND(SHR32ri)
+ CASE_ND(SHR64ri)
return getTruncatedShiftCount(MI, 2) != 0;
- // Some left shift instructions can be turned into LEA instructions but only
- // if their flags aren't used. Avoid transforming such instructions.
- CASE_ND(SHL8ri)
- CASE_ND(SHL16ri)
- CASE_ND(SHL32ri)
- CASE_ND(SHL64ri) {
- unsigned ShAmt = getTruncatedShiftCount(MI, 2);
- if (isTruncatedShiftCountForLEA(ShAmt))
- return false;
- return ShAmt != 0;
- }
+ // Some left shift instructions can be turned into LEA instructions but only
+ // if their flags aren't used. Avoid transforming such instructions.
+ CASE_ND(SHL8ri)
+ CASE_ND(SHL16ri)
+ CASE_ND(SHL32ri)
+ CASE_ND(SHL64ri) {
+ unsigned ShAmt = getTruncatedShiftCount(MI, 2);
+ if (isTruncatedShiftCountForLEA(ShAmt))
+ return false;
+ return ShAmt != 0;
+ }
- CASE_ND(SHRD16rri8)
- CASE_ND(SHRD32rri8)
- CASE_ND(SHRD64rri8)
- CASE_ND(SHLD16rri8)
- CASE_ND(SHLD32rri8)
- CASE_ND(SHLD64rri8)
+ CASE_ND(SHRD16rri8)
+ CASE_ND(SHRD32rri8)
+ CASE_ND(SHRD64rri8)
+ CASE_ND(SHLD16rri8)
+ CASE_ND(SHLD32rri8)
+ CASE_ND(SHLD64rri8)
return getTruncatedShiftCount(MI, 3) != 0;
- CASE_ND(SUB64ri32)
- CASE_ND(SUB32ri)
- CASE_ND(SUB16ri)
- CASE_ND(SUB8ri)
- CASE_ND(SUB64rr)
- CASE_ND(SUB32rr)
- CASE_ND(SUB16rr)
- CASE_ND(SUB8rr)
- CASE_ND(SUB64rm)
- CASE_ND(SUB32rm)
- CASE_ND(SUB16rm)
- CASE_ND(SUB8rm)
- CASE_ND(DEC64r)
- CASE_ND(DEC32r)
- CASE_ND(DEC16r)
- CASE_ND(DEC8r)
- CASE_ND(ADD64ri32)
- CASE_ND(ADD32ri)
- CASE_ND(ADD16ri)
- CASE_ND(ADD8ri)
- CASE_ND(ADD64rr)
- CASE_ND(ADD32rr)
- CASE_ND(ADD16rr)
- CASE_ND(ADD8rr)
- CASE_ND(ADD64rm)
- CASE_ND(ADD32rm)
- CASE_ND(ADD16rm)
- CASE_ND(ADD8rm)
- CASE_ND(INC64r)
- CASE_ND(INC32r)
- CASE_ND(INC16r)
- CASE_ND(INC8r)
- CASE_ND(ADC64ri32)
- CASE_ND(ADC32ri)
- CASE_ND(ADC16ri)
- CASE_ND(ADC8ri)
- CASE_ND(ADC64rr)
- CASE_ND(ADC32rr)
- CASE_ND(ADC16rr)
- CASE_ND(ADC8rr)
- CASE_ND(ADC64rm)
- CASE_ND(ADC32rm)
- CASE_ND(ADC16rm)
- CASE_ND(ADC8rm)
- CASE_ND(SBB64ri32)
- CASE_ND(SBB32ri)
- CASE_ND(SBB16ri)
- CASE_ND(SBB8ri)
- CASE_ND(SBB64rr)
- CASE_ND(SBB32rr)
- CASE_ND(SBB16rr)
- CASE_ND(SBB8rr)
- CASE_ND(SBB64rm)
- CASE_ND(SBB32rm)
- CASE_ND(SBB16rm)
- CASE_ND(SBB8rm)
- CASE_ND(NEG8r)
- CASE_ND(NEG16r)
- CASE_ND(NEG32r)
- CASE_ND(NEG64r)
+ CASE_ND(SUB64ri32)
+ CASE_ND(SUB32ri)
+ CASE_ND(SUB16ri)
+ CASE_ND(SUB8ri)
+ CASE_ND(SUB64rr)
+ CASE_ND(SUB32rr)
+ CASE_ND(SUB16rr)
+ CASE_ND(SUB8rr)
+ CASE_ND(SUB64rm)
+ CASE_ND(SUB32rm)
+ CASE_ND(SUB16rm)
+ CASE_ND(SUB8rm)
+ CASE_ND(DEC64r)
+ CASE_ND(DEC32r)
+ CASE_ND(DEC16r)
+ CASE_ND(DEC8r)
+ CASE_ND(ADD64ri32)
+ CASE_ND(ADD32ri)
+ CASE_ND(ADD16ri)
+ CASE_ND(ADD8ri)
+ CASE_ND(ADD64rr)
+ CASE_ND(ADD32rr)
+ CASE_ND(ADD16rr)
+ CASE_ND(ADD8rr)
+ CASE_ND(ADD64rm)
+ CASE_ND(ADD32rm)
+ CASE_ND(ADD16rm)
+ CASE_ND(ADD8rm)
+ CASE_ND(INC64r)
+ CASE_ND(INC32r)
+ CASE_ND(INC16r)
+ CASE_ND(INC8r)
+ CASE_ND(ADC64ri32)
+ CASE_ND(ADC32ri)
+ CASE_ND(ADC16ri)
+ CASE_ND(ADC8ri)
+ CASE_ND(ADC64rr)
+ CASE_ND(ADC32rr)
+ CASE_ND(ADC16rr)
+ CASE_ND(ADC8rr)
+ CASE_ND(ADC64rm)
+ CASE_ND(ADC32rm)
+ CASE_ND(ADC16rm)
+ CASE_ND(ADC8rm)
+ CASE_ND(SBB64ri32)
+ CASE_ND(SBB32ri)
+ CASE_ND(SBB16ri)
+ CASE_ND(SBB8ri)
+ CASE_ND(SBB64rr)
+ CASE_ND(SBB32rr)
+ CASE_ND(SBB16rr)
+ CASE_ND(SBB8rr)
+ CASE_ND(SBB64rm)
+ CASE_ND(SBB32rm)
+ CASE_ND(SBB16rm)
+ CASE_ND(SBB8rm)
+ CASE_ND(NEG8r)
+ CASE_ND(NEG16r)
+ CASE_ND(NEG32r)
+ CASE_ND(NEG64r)
case X86::LZCNT16rr:
case X86::LZCNT16rm:
case X86::LZCNT32rr:
@@ -5669,42 +5676,42 @@ inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag,
case X86::TZCNT64rr:
case X86::TZCNT64rm:
return true;
- CASE_ND(AND64ri32)
- CASE_ND(AND32ri)
- CASE_ND(AND16ri)
- CASE_ND(AND8ri)
- CASE_ND(AND64rr)
- CASE_ND(AND32rr)
- CASE_ND(AND16rr)
- CASE_ND(AND8rr)
- CASE_ND(AND64rm)
- CASE_ND(AND32rm)
- CASE_ND(AND16rm)
- CASE_ND(AND8rm)
- CASE_ND(XOR64ri32)
- CASE_ND(XOR32ri)
- CASE_ND(XOR16ri)
- CASE_ND(XOR8ri)
- CASE_ND(XOR64rr)
- CASE_ND(XOR32rr)
- CASE_ND(XOR16rr)
- CASE_ND(XOR8rr)
- CASE_ND(XOR64rm)
- CASE_ND(XOR32rm)
- CASE_ND(XOR16rm)
- CASE_ND(XOR8rm)
- CASE_ND(OR64ri32)
- CASE_ND(OR32ri)
- CASE_ND(OR16ri)
- CASE_ND(OR8ri)
- CASE_ND(OR64rr)
- CASE_ND(OR32rr)
- CASE_ND(OR16rr)
- CASE_ND(OR8rr)
- CASE_ND(OR64rm)
- CASE_ND(OR32rm)
- CASE_ND(OR16rm)
- CASE_ND(OR8rm)
+ CASE_ND(AND64ri32)
+ CASE_ND(AND32ri)
+ CASE_ND(AND16ri)
+ CASE_ND(AND8ri)
+ CASE_ND(AND64rr)
+ CASE_ND(AND32rr)
+ CASE_ND(AND16rr)
+ CASE_ND(AND8rr)
+ CASE_ND(AND64rm)
+ CASE_ND(AND32rm)
+ CASE_ND(AND16rm)
+ CASE_ND(AND8rm)
+ CASE_ND(XOR64ri32)
+ CASE_ND(XOR32ri)
+ CASE_ND(XOR16ri)
+ CASE_ND(XOR8ri)
+ CASE_ND(XOR64rr)
+ CASE_ND(XOR32rr)
+ CASE_ND(XOR16rr)
+ CASE_ND(XOR8rr)
+ CASE_ND(XOR64rm)
+ CASE_ND(XOR32rm)
+ CASE_ND(XOR16rm)
+ CASE_ND(XOR8rm)
+ CASE_ND(OR64ri32)
+ CASE_ND(OR32ri)
+ CASE_ND(OR16ri)
+ CASE_ND(OR8ri)
+ CASE_ND(OR64rr)
+ CASE_ND(OR32rr)
+ CASE_ND(OR16rr)
+ CASE_ND(OR8rr)
+ CASE_ND(OR64rm)
+ CASE_ND(OR32rm)
+ CASE_ND(OR16rm)
+ CASE_ND(OR8rm)
case X86::ANDN32rr:
case X86::ANDN32rm:
case X86::ANDN64rr:
@@ -5782,15 +5789,17 @@ inline static bool isDefConvertible(const MachineInstr &MI, bool &NoSignFlag,
}
/// Check whether the use can be converted to remove a comparison against zero.
-/// Returns the EFLAGS condition and the operand that we are comparing against zero.
-static std::pair<X86::CondCode, unsigned> isUseDefConvertible(const MachineInstr &MI) {
+/// Returns the EFLAGS condition and the operand that we are comparing against
+/// zero.
+static std::pair<X86::CondCode, unsigned>
+isUseDefConvertible(const MachineInstr &MI) {
switch (MI.getOpcode()) {
default:
return std::make_pair(X86::COND_INVALID, ~0U);
- CASE_ND(NEG8r)
- CASE_ND(NEG16r)
- CASE_ND(NEG32r)
- CASE_ND(NEG64r)
+ CASE_ND(NEG8r)
+ CASE_ND(NEG16r)
+ CASE_ND(NEG32r)
+ CASE_ND(NEG64r)
return std::make_pair(X86::COND_AE, 1U);
case X86::LZCNT16rr:
case X86::LZCNT32rr:
@@ -5834,51 +5843,53 @@ bool X86InstrInfo::optimizeCompareInstr(MachineInstr &CmpInstr, Register SrcReg,
switch (CmpInstr.getOpcode()) {
default:
break;
- CASE_ND(SUB64ri32)
- CASE_ND(SUB32ri)
- CASE_ND(SUB16ri)
- CASE_ND(SUB8ri)
- CASE_ND(SUB64rm)
- CASE_ND(SUB32rm)
- CASE_ND(SUB16rm)
- CASE_ND(SUB8rm)
- CASE_ND(SUB64rr)
- CASE_ND(SUB32rr)
- CASE_ND(SUB16rr)
- CASE_ND(SUB8rr) {
- if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
- return false;
- // There is no use of the destination register, we can replace SUB with CMP.
- unsigned NewOpcode = 0;
+ CASE_ND(SUB64ri32)
+ CASE_ND(SUB32ri)
+ CASE_ND(SUB16ri)
+ CASE_ND(SUB8ri)
+ CASE_ND(SUB64rm)
+ CASE_ND(SUB32rm)
+ CASE_ND(SUB16rm)
+ CASE_ND(SUB8rm)
+ CASE_ND(SUB64rr)
+ CASE_ND(SUB32rr)
+ CASE_ND(SUB16rr)
+ CASE_ND(SUB8rr) {
+ if (!MRI->use_nodbg_empty(CmpInstr.getOperand(0).getReg()))
+ return false;
+ // There is no use of the destination register, we can replace SUB with
+ // CMP.
+ unsigned NewOpcode = 0;
#define FROM_TO(A, B) \
CASE_ND(A) NewOpcode = X86::B; \
break;
- switch (CmpInstr.getOpcode()) {
- default:
- llvm_unreachable("Unreachable!");
- FROM_TO(SUB64rm, CMP64rm)
- FROM_TO(SUB32rm, CMP32rm)
- FROM_TO(SUB16rm, CMP16rm)
- FROM_TO(SUB8rm, CMP8rm)
- FROM_TO(SUB64rr, CMP64rr)
- FROM_TO(SUB32rr, CMP32rr)
- FROM_TO(SUB16rr, CMP16rr)
- FROM_TO(SUB8rr, CMP8rr)
- FROM_TO(SUB64ri32, CMP64ri32)
- FROM_TO(SUB32ri, CMP32ri)
- FROM_TO(SUB16ri, CMP16ri)
- FROM_TO(SUB8ri, CMP8ri)
- }
+ switch (CmpInstr.getOpcode()) {
+ default:
+ llvm_unreachable("Unreachable!");
+ FROM_TO(SUB64rm, CMP64rm)
+ FROM_TO(SUB32rm, CMP32rm)
+ FROM_TO(SUB16rm, CMP16rm)
+ FROM_TO(SUB8rm, CMP8rm)
+ FROM_TO(SUB64rr, CMP64rr)
+ FROM_TO(SUB32rr, CMP32rr)
+ FROM_TO(SUB16rr, CMP16rr)
+ FROM_TO(SUB8rr, CMP8rr)
+ FROM_TO(SUB64ri32, CMP64ri32)
+ FROM_TO(SUB32ri, CMP32ri)
+ FROM_TO(SUB16ri, CMP16ri)
+ FROM_TO(SUB8ri, CMP8ri)
+ }
#undef FROM_TO
- CmpInstr.setDesc(get(NewOpcode));
- CmpInstr.removeOperand(0);
- // Mutating this instruction invalidates any debug data associated with it.
- CmpInstr.dropDebugNumber();
- // Fall through to optimize Cmp if Cmp is CMPrr or CMPri.
- if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
- NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
- return false;
- }
+ CmpInstr.setDesc(get(NewOpcode));
+ CmpInstr.removeOperand(0);
+ // Mutating this instruction invalidates any debug data associated with
+ // it.
+ CmpInstr.dropDebugNumber();
+ // Fall through to optimize Cmp if Cmp is CMPrr or CMPri.
+ if (NewOpcode == X86::CMP64rm || NewOpcode == X86::CMP32rm ||
+ NewOpcode == X86::CMP16rm || NewOpcode == X86::CMP8rm)
+ return false;
+ }
}
// The following code tries to remove the comparison by re-using EFLAGS
@@ -6235,14 +6246,14 @@ static bool canConvert2Copy(unsigned Opc) {
switch (Opc) {
default:
return false;
- CASE_ND(ADD64ri32)
- CASE_ND(SUB64ri32)
- CASE_ND(OR64ri32)
- CASE_ND(XOR64ri32)
- CASE_ND(ADD32ri)
- CASE_ND(SUB32ri)
- CASE_ND(OR32ri)
- CASE_ND(XOR32ri)
+ CASE_ND(ADD64ri32)
+ CASE_ND(SUB64ri32)
+ CASE_ND(OR64ri32)
+ CASE_ND(XOR64ri32)
+ CASE_ND(ADD32ri)
+ CASE_ND(SUB32ri)
+ CASE_ND(OR32ri)
+ CASE_ND(XOR32ri)
return true;
}
}
@@ -9723,7 +9734,7 @@ Register X86InstrInfo::getGlobalBaseReg(MachineFunction *MF) const {
static const uint16_t *lookup(unsigned opcode, unsigned domain,
ArrayRef<uint16_t[3]> Table) {
- for (const uint16_t(&Row)[3] : Table)
+ for (const uint16_t (&Row)[3] : Table)
if (Row[domain - 1] == opcode)
return Row;
return nullptr;
@@ -9732,7 +9743,7 @@ static const uint16_t *lookup(unsigned opcode, unsigned domain,
static const uint16_t *lookupAVX512(unsigned opcode, unsigned domain,
ArrayRef<uint16_t[4]> Table) {
// If this is the integer domain make sure to check both integer columns.
- for (const uint16_t(&Row)[4] : Table)
+ for (const uint16_t (&Row)[4] : Table)
if (Row[domain - 1] == opcode || (domain == 3 && Row[3] == opcode))
return Row;
return nullptr;
@@ -10488,25 +10499,25 @@ bool X86InstrInfo::isAssociativeAndCommutative(const MachineInstr &Inst,
if (Invert)
return false;
switch (Inst.getOpcode()) {
- CASE_ND(ADD8rr)
- CASE_ND(ADD16rr)
- CASE_ND(ADD32rr)
- CASE_ND(ADD64rr)
- CASE_ND(AND8rr)
- CASE_ND(AND16rr)
- CASE_ND(AND32rr)
- CASE_ND(AND64rr)
- CASE_ND(OR8rr)
- CASE_ND(OR16rr)
- CASE_ND(OR32rr)
- CASE_ND(OR64rr)
- CASE_ND(XOR8rr)
- CASE_ND(XOR16rr)
- CASE_ND(XOR32rr)
- CASE_ND(XOR64rr)
- CASE_ND(IMUL16rr)
- CASE_ND(IMUL32rr)
- CASE_ND(IMUL64rr)
+ CASE_ND(ADD8rr)
+ CASE_ND(ADD16rr)
+ CASE_ND(ADD32rr)
+ CASE_ND(ADD64rr)
+ CASE_ND(AND8rr)
+ CASE_ND(AND16rr)
+ CASE_ND(AND32rr)
+ CASE_ND(AND64rr)
+ CASE_ND(OR8rr)
+ CASE_ND(OR16rr)
+ CASE_ND(OR32rr)
+ CASE_ND(OR64rr)
+ CASE_ND(XOR8rr)
+ CASE_ND(XOR16rr)
+ CASE_ND(XOR32rr)
+ CASE_ND(XOR64rr)
+ CASE_ND(IMUL16rr)
+ CASE_ND(IMUL32rr)
+ CASE_ND(IMUL64rr)
case X86::PANDrr:
case X86::PORrr:
case X86::PXORrr:
@@ -11330,8 +11341,8 @@ bool X86InstrInfo::getMachineCombinerPatterns(
break;
}
}
- return TargetInstrInfo::getMachineCombinerPatterns(Root,
- Patterns, DoRegPressureReduce);
+ return TargetInstrInfo::getMachineCombinerPatterns(Root, Patterns,
+ DoRegPressureReduce);
}
static void
diff --git a/llvm/lib/Target/X86/X86InstrInfo.h b/llvm/lib/Target/X86/X86InstrInfo.h
index fc69d22b86c91..15e62fb4b1ab8 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.h
+++ b/llvm/lib/Target/X86/X86InstrInfo.h
@@ -319,8 +319,7 @@ class X86InstrInfo final : public X86GenInstrInfo {
Register isLoadFromStackSlot(const MachineInstr &MI,
int &FrameIndex) const override;
- Register isLoadFromStackSlot(const MachineInstr &MI,
- int &FrameIndex,
+ Register isLoadFromStackSlot(const MachineInstr &MI, int &FrameIndex,
TypeSize &MemBytes) const override;
/// isLoadFromStackSlotPostFE - Check for post-frame ptr elimination
/// stack locations as well. This uses a heuristic so it isn't
@@ -330,8 +329,7 @@ class X86InstrInfo final : public X86GenInstrInfo {
Register isStoreToStackSlot(const MachineInstr &MI,
int &FrameIndex) const override;
- Register isStoreToStackSlot(const MachineInstr &MI,
- int &FrameIndex,
+ Register isStoreToStackSlot(const MachineInstr &MI, int &FrameIndex,
TypeSize &MemBytes) const override;
/// isStoreToStackSlotPostFE - Check for post-frame ptr elimination
/// stack locations as well. This uses a heuristic so it isn't
@@ -748,8 +746,7 @@ class X86InstrInfo final : public X86GenInstrInfo {
///
/// If IsIntrinsic is set, operand 1 will be ignored for commuting.
bool findThreeSrcCommutedOpIndices(const MachineInstr &MI,
- unsigned &SrcOpIdx1,
- unsigned &SrcOpIdx2,
+ unsigned &SrcOpIdx1, unsigned &SrcOpIdx2,
bool IsIntrinsic = false) const;
/// Returns true when instruction \p FlagI produces the same flags as \p OI.
diff --git a/llvm/test/CodeGen/X86/ctselect-i386-fp.ll b/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
index b88ec72a37925..4b5f31bad8313 100644
--- a/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
+++ b/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
@@ -209,94 +209,84 @@ define double @test_ctselect_f64_basic(i1 %cond, double %a, double %b) nounwind
define x86_fp80 @test_ctselect_f80_basic(i1 %cond, x86_fp80 %a, x86_fp80 %b) nounwind {
; I386-NOCMOV-LABEL: test_ctselect_f80_basic:
; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebp
+; I386-NOCMOV-NEXT: pushl %ebx
; I386-NOCMOV-NEXT: pushl %edi
; I386-NOCMOV-NEXT: pushl %esi
-; I386-NOCMOV-NEXT: subl $12, %esp
-; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: sete %al
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: subl $40, %esp
+; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: testb $1, %cl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, (%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: sete %ch
+; I386-NOCMOV-NEXT: movb %ch, %al
+; I386-NOCMOV-NEXT: movzbl %al, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %edi, %ebx
+; I386-NOCMOV-NEXT: andl %ebp, %ebx
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %edx, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %ebx
+; I386-NOCMOV-NEXT: testb $1, %cl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: sete %ch
+; I386-NOCMOV-NEXT: movb %ch, %cl
+; I386-NOCMOV-NEXT: movzbl %cl, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %edx, %edi
+; I386-NOCMOV-NEXT: andl %ebp, %edi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %eax, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl (%esp), %ebx # 4-byte Reload
+; I386-NOCMOV-NEXT: movb %al, %dl
+; I386-NOCMOV-NEXT: movzbl %dl, %edi
; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, %ecx
+; I386-NOCMOV-NEXT: andl %edi, %ecx
; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: fldt (%esp)
-; I386-NOCMOV-NEXT: addl $12, %esp
+; I386-NOCMOV-NEXT: andl %ebx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %ecx
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: addl $40, %esp
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: popl %ebp
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_ctselect_f80_basic:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: pushl %edi
-; I386-CMOV-NEXT: pushl %esi
-; I386-CMOV-NEXT: subl $12, %esp
+; I386-CMOV-NEXT: subl $36, %esp
+; I386-CMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, (%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, (%esp)
; I386-CMOV-NEXT: fldt (%esp)
-; I386-CMOV-NEXT: addl $12, %esp
-; I386-CMOV-NEXT: popl %esi
-; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: addl $36, %esp
; I386-CMOV-NEXT: retl
%result = call x86_fp80 @llvm.ct.select.f80(i1 %cond, x86_fp80 %a, x86_fp80 %b)
ret x86_fp80 %result
@@ -543,94 +533,84 @@ define float @test_ctselect_f32_nan(i1 %cond) nounwind {
define x86_fp80 @test_ctselect_f80_alignment(i1 %cond, x86_fp80 %a, x86_fp80 %b) nounwind {
; I386-NOCMOV-LABEL: test_ctselect_f80_alignment:
; I386-NOCMOV: # %bb.0:
+; I386-NOCMOV-NEXT: pushl %ebp
+; I386-NOCMOV-NEXT: pushl %ebx
; I386-NOCMOV-NEXT: pushl %edi
; I386-NOCMOV-NEXT: pushl %esi
-; I386-NOCMOV-NEXT: subl $12, %esp
-; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: sete %al
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: subl $40, %esp
+; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: testb $1, %cl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, (%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: sete %ch
+; I386-NOCMOV-NEXT: movb %ch, %al
+; I386-NOCMOV-NEXT: movzbl %al, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %edi, %ebx
+; I386-NOCMOV-NEXT: andl %ebp, %ebx
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %edx, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %ebx
+; I386-NOCMOV-NEXT: testb $1, %cl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
+; I386-NOCMOV-NEXT: sete %ch
+; I386-NOCMOV-NEXT: movb %ch, %cl
+; I386-NOCMOV-NEXT: movzbl %cl, %ebp
+; I386-NOCMOV-NEXT: negl %ebp
+; I386-NOCMOV-NEXT: movl %edx, %edi
+; I386-NOCMOV-NEXT: andl %ebp, %edi
+; I386-NOCMOV-NEXT: notl %ebp
+; I386-NOCMOV-NEXT: andl %eax, %ebp
+; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl (%esp), %ebx # 4-byte Reload
+; I386-NOCMOV-NEXT: movb %al, %dl
+; I386-NOCMOV-NEXT: movzbl %dl, %edi
; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, %ecx
+; I386-NOCMOV-NEXT: andl %edi, %ecx
; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: fldt (%esp)
-; I386-NOCMOV-NEXT: addl $12, %esp
+; I386-NOCMOV-NEXT: andl %ebx, %edi
+; I386-NOCMOV-NEXT: orl %edi, %ecx
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: addl $40, %esp
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: popl %ebp
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_ctselect_f80_alignment:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: pushl %edi
-; I386-CMOV-NEXT: pushl %esi
-; I386-CMOV-NEXT: subl $12, %esp
+; I386-CMOV-NEXT: subl $36, %esp
+; I386-CMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: sete %al
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, (%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, (%esp)
; I386-CMOV-NEXT: fldt (%esp)
-; I386-CMOV-NEXT: addl $12, %esp
-; I386-CMOV-NEXT: popl %esi
-; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: addl $36, %esp
; I386-CMOV-NEXT: retl
%result = call x86_fp80 @llvm.ct.select.f80(i1 %cond, x86_fp80 %a, x86_fp80 %b)
ret x86_fp80 %result
diff --git a/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll b/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
index 6851c5babeb2d..2e1f16c19ca8e 100644
--- a/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
+++ b/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
@@ -10,84 +10,67 @@
define <1 x i64> @test_mmx_ctselect_with_paddd(i32 %cond, i64 %a, i64 %b) {
; I386-NOCMOV-LABEL: test_mmx_ctselect_with_paddd:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %ebp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: pushl %ebx
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: pushl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; I386-NOCMOV-NEXT: pushl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: subl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
-; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
-; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
-; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: subl $16, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %ecx, %ecx
+; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: setne %bl
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: sete %bh
-; I386-NOCMOV-NEXT: movb %bh, %al
-; I386-NOCMOV-NEXT: movzbl %al, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %esi, %edi
-; I386-NOCMOV-NEXT: andl %ebp, %edi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %ecx, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %edi
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: sete %al
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %ecx
-; I386-NOCMOV-NEXT: andl %esi, %ecx
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: orl %esi, %ecx
-; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: xorl %edx, %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: xorl %edx, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: andl %ecx, %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: paddd %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: popl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-NOCMOV-NEXT: popl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: addl $16, %esp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_mmx_ctselect_with_paddd:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: subl $16, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-CMOV-NEXT: .cfi_offset %esi, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: setne %dl
-; I386-CMOV-NEXT: testb %dl, %dl
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: xorl %ecx, %ecx
+; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: sbbl %ecx, %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: xorl %edx, %esi
+; I386-CMOV-NEXT: andl %ecx, %esi
+; I386-CMOV-NEXT: xorl %edx, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: andl %ecx, %edx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: paddd %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: addl $16, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: popl %esi
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
; I386-CMOV-NEXT: retl
%mmx_a = bitcast i64 %a to <1 x i64>
@@ -102,84 +85,67 @@ define <1 x i64> @test_mmx_ctselect_with_paddd(i32 %cond, i64 %a, i64 %b) {
define <1 x i64> @test_mmx_ctselect_with_psllw(i32 %cond, i64 %a, i64 %b) {
; I386-NOCMOV-LABEL: test_mmx_ctselect_with_psllw:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %ebp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: pushl %ebx
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: pushl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; I386-NOCMOV-NEXT: pushl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: subl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
-; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
-; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
-; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: subl $16, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %ecx, %ecx
+; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: setne %bl
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: sete %bh
-; I386-NOCMOV-NEXT: movb %bh, %al
-; I386-NOCMOV-NEXT: movzbl %al, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %esi, %edi
-; I386-NOCMOV-NEXT: andl %ebp, %edi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %ecx, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %edi
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: sete %al
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %ecx
-; I386-NOCMOV-NEXT: andl %esi, %ecx
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: orl %esi, %ecx
-; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: xorl %edx, %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: xorl %edx, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: andl %ecx, %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: psllw %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: popl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-NOCMOV-NEXT: popl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: addl $16, %esp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_mmx_ctselect_with_psllw:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: subl $16, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-CMOV-NEXT: .cfi_offset %esi, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: setne %dl
-; I386-CMOV-NEXT: testb %dl, %dl
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: xorl %ecx, %ecx
+; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: sbbl %ecx, %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: xorl %edx, %esi
+; I386-CMOV-NEXT: andl %ecx, %esi
+; I386-CMOV-NEXT: xorl %edx, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: andl %ecx, %edx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: psllw %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: addl $16, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: popl %esi
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
; I386-CMOV-NEXT: retl
%mmx_a = bitcast i64 %a to <1 x i64>
@@ -194,86 +160,55 @@ define <1 x i64> @test_mmx_ctselect_with_psllw(i32 %cond, i64 %a, i64 %b) {
define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64 %a, i64 %b, i64 %c) {
; I386-NOCMOV-LABEL: test_mmx_nested_ctselect_with_pand:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %ebp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; I386-NOCMOV-NEXT: pushl %ebx
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; I386-NOCMOV-NEXT: pushl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
; I386-NOCMOV-NEXT: pushl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: subl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
-; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
-; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
-; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: subl $16, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 32
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -8
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: setne %bl
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: sete %bh
-; I386-NOCMOV-NEXT: movb %bh, %cl
-; I386-NOCMOV-NEXT: movzbl %cl, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %edx, %edi
-; I386-NOCMOV-NEXT: andl %ebp, %edi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %eax, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %edi
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: sete %dl
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-NOCMOV-NEXT: movb %dl, %dh
-; I386-NOCMOV-NEXT: movzbl %dh, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %esi, %ebx
-; I386-NOCMOV-NEXT: andl %ebp, %ebx
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %eax, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %ebx
-; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: setne %dl
-; I386-NOCMOV-NEXT: testb %dl, %dl
-; I386-NOCMOV-NEXT: sete %dh
-; I386-NOCMOV-NEXT: movb %dh, %al
-; I386-NOCMOV-NEXT: movzbl %al, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %ecx, %esi
-; I386-NOCMOV-NEXT: andl %ebp, %esi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %ebx, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %esi
-; I386-NOCMOV-NEXT: testb %dl, %dl
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: xorl %ecx, %ecx
+; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: movl $0, %edi
+; I386-NOCMOV-NEXT: sbbl %edi, %edi
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: andl %edi, %edx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; I386-NOCMOV-NEXT: movb %al, %dl
-; I386-NOCMOV-NEXT: movzbl %dl, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %ebx, %ecx
-; I386-NOCMOV-NEXT: andl %esi, %ecx
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: orl %esi, %ecx
-; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: xorl %esi, %ebx
+; I386-NOCMOV-NEXT: andl %edi, %ebx
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
+; I386-NOCMOV-NEXT: xorl %edi, %esi
+; I386-NOCMOV-NEXT: xorl %ebx, %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: xorl %edi, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-NOCMOV-NEXT: xorl %esi, %eax
+; I386-NOCMOV-NEXT: xorl %edx, %eax
+; I386-NOCMOV-NEXT: andl %ecx, %eax
+; I386-NOCMOV-NEXT: xorl %esi, %eax
+; I386-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: pand %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: addl $16, %esp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: popl %edi
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: popl %ebx
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
@@ -281,36 +216,51 @@ define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64
; I386-CMOV: # %bb.0:
; I386-CMOV-NEXT: pushl %ebx
; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: pushl %edi
; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-CMOV-NEXT: subl $16, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 32
-; I386-CMOV-NEXT: .cfi_offset %esi, -12
+; I386-CMOV-NEXT: .cfi_offset %esi, -16
+; I386-CMOV-NEXT: .cfi_offset %edi, -12
; I386-CMOV-NEXT: .cfi_offset %ebx, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: xorl %ecx, %ecx
+; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl $0, %edi
+; I386-CMOV-NEXT: sbbl %edi, %edi
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: andl %edi, %edx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; I386-CMOV-NEXT: xorl %esi, %ebx
+; I386-CMOV-NEXT: andl %edi, %ebx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: sbbl %ecx, %ecx
+; I386-CMOV-NEXT: xorl %edi, %esi
+; I386-CMOV-NEXT: xorl %ebx, %esi
+; I386-CMOV-NEXT: andl %ecx, %esi
+; I386-CMOV-NEXT: xorl %edi, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: setne %bl
-; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: setne %bh
-; I386-CMOV-NEXT: testb %bh, %bh
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: testb %bl, %bl
-; I386-CMOV-NEXT: cmovnel %esi, %edx
-; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: cmovnel %ecx, %eax
+; I386-CMOV-NEXT: xorl %esi, %eax
+; I386-CMOV-NEXT: xorl %edx, %eax
+; I386-CMOV-NEXT: andl %ecx, %eax
+; I386-CMOV-NEXT: xorl %esi, %eax
; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: pand %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $20, %esp
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-CMOV-NEXT: addl $16, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 16
; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-CMOV-NEXT: popl %edi
; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
; I386-CMOV-NEXT: popl %ebx
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
@@ -330,84 +280,67 @@ define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64
define <1 x i64> @test_mmx_ctselect_with_por(i32 %cond, i64 %a, i64 %b) {
; I386-NOCMOV-LABEL: test_mmx_ctselect_with_por:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %ebp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: pushl %ebx
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: pushl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; I386-NOCMOV-NEXT: pushl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: subl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
-; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
-; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
-; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: subl $16, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %ecx, %ecx
+; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: setne %bl
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: sete %bh
-; I386-NOCMOV-NEXT: movb %bh, %al
-; I386-NOCMOV-NEXT: movzbl %al, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %esi, %edi
-; I386-NOCMOV-NEXT: andl %ebp, %edi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %ecx, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %edi
-; I386-NOCMOV-NEXT: testb %bl, %bl
-; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: sete %al
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %ecx
-; I386-NOCMOV-NEXT: andl %esi, %ecx
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: orl %esi, %ecx
-; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: xorl %edx, %esi
+; I386-NOCMOV-NEXT: andl %ecx, %esi
+; I386-NOCMOV-NEXT: xorl %edx, %esi
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: andl %ecx, %edx
+; I386-NOCMOV-NEXT: xorl %eax, %edx
+; I386-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: por %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $20, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; I386-NOCMOV-NEXT: popl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-NOCMOV-NEXT: popl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: addl $16, %esp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: popl %ebp
+; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_mmx_ctselect_with_por:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: subl $20, %esp
+; I386-CMOV-NEXT: pushl %esi
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: subl $16, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
+; I386-CMOV-NEXT: .cfi_offset %esi, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: setne %dl
-; I386-CMOV-NEXT: testb %dl, %dl
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: xorl %ecx, %ecx
+; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: sbbl %ecx, %ecx
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: xorl %edx, %esi
+; I386-CMOV-NEXT: andl %ecx, %esi
+; I386-CMOV-NEXT: xorl %edx, %esi
+; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: andl %ecx, %edx
+; I386-CMOV-NEXT: xorl %eax, %edx
+; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: por %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $20, %esp
+; I386-CMOV-NEXT: addl $16, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-CMOV-NEXT: popl %esi
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
; I386-CMOV-NEXT: retl
%mmx_a = bitcast i64 %a to <1 x i64>
diff --git a/llvm/test/CodeGen/X86/ctselect-vector.ll b/llvm/test/CodeGen/X86/ctselect-vector.ll
index 2206e32cd6d34..79b76d1afca5c 100644
--- a/llvm/test/CodeGen/X86/ctselect-vector.ll
+++ b/llvm/test/CodeGen/X86/ctselect-vector.ll
@@ -266,31 +266,17 @@ define <2 x double> @test_ctselect_v2f64(i1 %cond, <2 x double> %a, <2 x double>
define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
; SSE2-LABEL: test_ctselect_v8i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: movd %eax, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm0, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm4
-; SSE2-NEXT: por %xmm5, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pand %xmm1, %xmm0
-; SSE2-NEXT: pandn %xmm3, %xmm2
-; SSE2-NEXT: por %xmm0, %xmm2
-; SSE2-NEXT: movaps %xmm4, %xmm0
-; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: movd %edi, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm4
+; SSE2-NEXT: psrad $31, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm5
+; SSE2-NEXT: pand %xmm4, %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
+; SSE2-NEXT: pand %xmm4, %xmm1
+; SSE2-NEXT: pandn %xmm3, %xmm4
+; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8i32:
@@ -342,31 +328,17 @@ define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b) {
; SSE2-LABEL: test_ctselect_v8f32:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: movd %eax, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
-; SSE2-NEXT: movaps %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm0, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm4
-; SSE2-NEXT: por %xmm5, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movaps %xmm0, %xmm2
-; SSE2-NEXT: pand %xmm1, %xmm0
-; SSE2-NEXT: pandn %xmm3, %xmm2
-; SSE2-NEXT: por %xmm0, %xmm2
-; SSE2-NEXT: movaps %xmm4, %xmm0
-; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: movd %edi, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm4
+; SSE2-NEXT: psrad $31, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm5
+; SSE2-NEXT: pand %xmm4, %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
+; SSE2-NEXT: pand %xmm4, %xmm1
+; SSE2-NEXT: pandn %xmm3, %xmm4
+; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8f32:
@@ -418,31 +390,17 @@ define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b
define <4 x i64> @test_ctselect_v4i64(i1 %cond, <4 x i64> %a, <4 x i64> %b) {
; SSE2-LABEL: test_ctselect_v4i64:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: movd %eax, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm0, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm4
-; SSE2-NEXT: por %xmm5, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm2
-; SSE2-NEXT: pand %xmm1, %xmm0
-; SSE2-NEXT: pandn %xmm3, %xmm2
-; SSE2-NEXT: por %xmm0, %xmm2
-; SSE2-NEXT: movaps %xmm4, %xmm0
-; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: movd %edi, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm4
+; SSE2-NEXT: psrad $31, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm5
+; SSE2-NEXT: pand %xmm4, %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
+; SSE2-NEXT: pand %xmm4, %xmm1
+; SSE2-NEXT: pandn %xmm3, %xmm4
+; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v4i64:
@@ -494,31 +452,17 @@ define <4 x i64> @test_ctselect_v4i64(i1 %cond, <4 x i64> %a, <4 x i64> %b) {
define <4 x double> @test_ctselect_v4f64(i1 %cond, <4 x double> %a, <4 x double> %b) {
; SSE2-LABEL: test_ctselect_v4f64:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: movd %eax, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,0,0]
-; SSE2-NEXT: movapd %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm0, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm4
-; SSE2-NEXT: por %xmm5, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movapd %xmm0, %xmm2
-; SSE2-NEXT: pand %xmm1, %xmm0
-; SSE2-NEXT: pandn %xmm3, %xmm2
-; SSE2-NEXT: por %xmm0, %xmm2
-; SSE2-NEXT: movaps %xmm4, %xmm0
-; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: movd %edi, %xmm4
+; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm4
+; SSE2-NEXT: psrad $31, %xmm4
+; SSE2-NEXT: movdqa %xmm4, %xmm5
+; SSE2-NEXT: pandn %xmm2, %xmm5
+; SSE2-NEXT: pand %xmm4, %xmm0
+; SSE2-NEXT: por %xmm5, %xmm0
+; SSE2-NEXT: pand %xmm4, %xmm1
+; SSE2-NEXT: pandn %xmm3, %xmm4
+; SSE2-NEXT: por %xmm4, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v4f64:
@@ -571,113 +515,63 @@ define <4 x double> @test_ctselect_v4f64(i1 %cond, <4 x double> %a, <4 x double>
define <16 x i32> @test_ctselect_v16i32(i1 %cond, <16 x i32> %a, <16 x i32> %b) {
; SSE2-LABEL: test_ctselect_v16i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm9, %xmm9
-; SSE2-NEXT: movd %eax, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm9, %xmm8
-; SSE2-NEXT: pand %xmm0, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm8
-; SSE2-NEXT: por %xmm9, %xmm8
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm4
-; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: movd %edi, %xmm8
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm8
+; SSE2-NEXT: psrad $31, %xmm8
+; SSE2-NEXT: movdqa %xmm8, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm9
+; SSE2-NEXT: pand %xmm8, %xmm0
+; SSE2-NEXT: por %xmm9, %xmm0
+; SSE2-NEXT: movdqa %xmm8, %xmm4
; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: por %xmm0, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm5
-; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: pandn %xmm6, %xmm5
-; SSE2-NEXT: por %xmm0, %xmm5
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm6
-; SSE2-NEXT: pand %xmm3, %xmm0
-; SSE2-NEXT: pandn %xmm7, %xmm6
-; SSE2-NEXT: por %xmm0, %xmm6
-; SSE2-NEXT: movaps %xmm8, %xmm0
-; SSE2-NEXT: movaps %xmm4, %xmm1
-; SSE2-NEXT: movaps %xmm5, %xmm2
-; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: pand %xmm8, %xmm1
+; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: movdqa %xmm8, %xmm4
+; SSE2-NEXT: pandn %xmm6, %xmm4
+; SSE2-NEXT: pand %xmm8, %xmm2
+; SSE2-NEXT: por %xmm4, %xmm2
+; SSE2-NEXT: pand %xmm8, %xmm3
+; SSE2-NEXT: pandn %xmm7, %xmm8
+; SSE2-NEXT: por %xmm8, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v16i32:
; AVX: # %bb.0:
-; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm5, %ymm5
-; AVX-NEXT: vmovd %eax, %ymm5
-; AVX-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
-; AVX-NEXT: vmovdqa %ymm5, %ymm4
-; AVX-NEXT: pand %ymm0, %ymm5
-; AVX-NEXT: pandn %ymm2, %ymm4
-; AVX-NEXT: por %ymm5, %ymm4
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm0, %ymm0
-; AVX-NEXT: vmovd %eax, %ymm0
-; AVX-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
-; AVX-NEXT: vmovdqa %ymm0, %ymm2
-; AVX-NEXT: pand %ymm1, %ymm0
-; AVX-NEXT: pandn %ymm3, %ymm2
-; AVX-NEXT: por %ymm0, %ymm2
-; AVX-NEXT: vmovaps %ymm4, %ymm0
-; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: vmovd %edi, %xmm4
+; AVX-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX-NEXT: vpshufb %xmm5, %xmm4, %xmm4
+; AVX-NEXT: vpsllw $7, %xmm4, %xmm4
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
+; AVX-NEXT: vpcmpgtb %xmm4, %xmm5, %xmm4
+; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
+; AVX-NEXT: vpmovsxbd %xmm5, %xmm5
+; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
+; AVX-NEXT: vpmovsxbd %xmm6, %xmm6
+; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5
+; AVX-NEXT: vpmovsxbd %xmm4, %xmm6
+; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
+; AVX-NEXT: vpmovsxbd %xmm4, %xmm4
+; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4
+; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
+; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
+; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
+; AVX-NEXT: vandnps %ymm3, %ymm5, %ymm2
+; AVX-NEXT: vandps %ymm5, %ymm1, %ymm1
+; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v16i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm5, %ymm5
-; AVX2-NEXT: vmovd %eax, %ymm5
-; AVX2-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
-; AVX2-NEXT: vmovdqa %ymm5, %ymm4
-; AVX2-NEXT: pand %ymm0, %ymm5
-; AVX2-NEXT: pandn %ymm2, %ymm4
-; AVX2-NEXT: por %ymm5, %ymm4
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm0, %ymm0
-; AVX2-NEXT: vmovd %eax, %ymm0
-; AVX2-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
-; AVX2-NEXT: vmovdqa %ymm0, %ymm2
-; AVX2-NEXT: pand %ymm1, %ymm0
-; AVX2-NEXT: pandn %ymm3, %ymm2
-; AVX2-NEXT: por %ymm0, %ymm2
-; AVX2-NEXT: vmovaps %ymm4, %ymm0
-; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: vmovd %edi, %xmm4
+; AVX2-NEXT: vpbroadcastb %xmm4, %xmm4
+; AVX2-NEXT: vpsllw $7, %xmm4, %xmm4
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm5
+; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
+; AVX2-NEXT: vpmovsxbd %xmm5, %ymm5
+; AVX2-NEXT: vpmovsxbd %xmm4, %ymm4
+; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpblendvb %ymm5, %ymm1, %ymm3, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v16i32:
; AVX512: # %bb.0:
@@ -695,113 +589,60 @@ define <16 x i32> @test_ctselect_v16i32(i1 %cond, <16 x i32> %a, <16 x i32> %b)
define <16 x float> @test_ctselect_v16f32(i1 %cond, <16 x float> %a, <16 x float> %b) {
; SSE2-LABEL: test_ctselect_v16f32:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm9, %xmm9
-; SSE2-NEXT: movd %eax, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
-; SSE2-NEXT: movaps %xmm9, %xmm8
-; SSE2-NEXT: pand %xmm0, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm8
-; SSE2-NEXT: por %xmm9, %xmm8
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movaps %xmm0, %xmm4
-; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: movd %edi, %xmm8
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm8
+; SSE2-NEXT: psrad $31, %xmm8
+; SSE2-NEXT: movdqa %xmm8, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm9
+; SSE2-NEXT: pand %xmm8, %xmm0
+; SSE2-NEXT: por %xmm9, %xmm0
+; SSE2-NEXT: movdqa %xmm8, %xmm4
; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: por %xmm0, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movaps %xmm0, %xmm5
-; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: pandn %xmm6, %xmm5
-; SSE2-NEXT: por %xmm0, %xmm5
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movaps %xmm0, %xmm6
-; SSE2-NEXT: pand %xmm3, %xmm0
-; SSE2-NEXT: pandn %xmm7, %xmm6
-; SSE2-NEXT: por %xmm0, %xmm6
-; SSE2-NEXT: movaps %xmm8, %xmm0
-; SSE2-NEXT: movaps %xmm4, %xmm1
-; SSE2-NEXT: movaps %xmm5, %xmm2
-; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: pand %xmm8, %xmm1
+; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: movdqa %xmm8, %xmm4
+; SSE2-NEXT: pandn %xmm6, %xmm4
+; SSE2-NEXT: pand %xmm8, %xmm2
+; SSE2-NEXT: por %xmm4, %xmm2
+; SSE2-NEXT: pand %xmm8, %xmm3
+; SSE2-NEXT: pandn %xmm7, %xmm8
+; SSE2-NEXT: por %xmm8, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v16f32:
; AVX: # %bb.0:
-; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm5, %ymm5
-; AVX-NEXT: vmovd %eax, %ymm5
-; AVX-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
-; AVX-NEXT: vmovdqa %ymm5, %ymm4
-; AVX-NEXT: pand %ymm0, %ymm5
-; AVX-NEXT: pandn %ymm2, %ymm4
-; AVX-NEXT: por %ymm5, %ymm4
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm0, %ymm0
-; AVX-NEXT: vmovd %eax, %ymm0
-; AVX-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
-; AVX-NEXT: vmovdqa %ymm0, %ymm2
-; AVX-NEXT: pand %ymm1, %ymm0
-; AVX-NEXT: pandn %ymm3, %ymm2
-; AVX-NEXT: por %ymm0, %ymm2
-; AVX-NEXT: vmovaps %ymm4, %ymm0
-; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: vmovd %edi, %xmm4
+; AVX-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX-NEXT: vpshufb %xmm5, %xmm4, %xmm4
+; AVX-NEXT: vpsllw $7, %xmm4, %xmm4
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
+; AVX-NEXT: vpcmpgtb %xmm4, %xmm5, %xmm4
+; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
+; AVX-NEXT: vpmovsxbd %xmm5, %xmm5
+; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
+; AVX-NEXT: vpmovsxbd %xmm6, %xmm6
+; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5
+; AVX-NEXT: vpmovsxbd %xmm4, %xmm6
+; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
+; AVX-NEXT: vpmovsxbd %xmm4, %xmm4
+; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4
+; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
+; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
+; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
+; AVX-NEXT: vandnps %ymm3, %ymm5, %ymm2
+; AVX-NEXT: vandps %ymm5, %ymm1, %ymm1
+; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v16f32:
; AVX2: # %bb.0:
-; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm5, %ymm5
-; AVX2-NEXT: vmovd %eax, %ymm5
-; AVX2-NEXT: vpermilps {{.*#+}} ymm5 = ymm5[0,0,0,0,4,4,4,4]
-; AVX2-NEXT: vmovdqa %ymm5, %ymm4
-; AVX2-NEXT: pand %ymm0, %ymm5
-; AVX2-NEXT: pandn %ymm2, %ymm4
-; AVX2-NEXT: por %ymm5, %ymm4
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm0, %ymm0
-; AVX2-NEXT: vmovd %eax, %ymm0
-; AVX2-NEXT: vpermilps {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4]
-; AVX2-NEXT: vmovdqa %ymm0, %ymm2
-; AVX2-NEXT: pand %ymm1, %ymm0
-; AVX2-NEXT: pandn %ymm3, %ymm2
-; AVX2-NEXT: por %ymm0, %ymm2
-; AVX2-NEXT: vmovaps %ymm4, %ymm0
-; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: vmovd %edi, %xmm4
+; AVX2-NEXT: vpbroadcastb %xmm4, %xmm4
+; AVX2-NEXT: vpsllw $7, %xmm4, %xmm4
+; AVX2-NEXT: vpmovsxbd %xmm4, %ymm4
+; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm3, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v16f32:
; AVX512: # %bb.0:
@@ -819,113 +660,52 @@ define <16 x float> @test_ctselect_v16f32(i1 %cond, <16 x float> %a, <16 x float
define <8 x i64> @test_ctselect_v8i64(i1 %cond, <8 x i64> %a, <8 x i64> %b) {
; SSE2-LABEL: test_ctselect_v8i64:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm9, %xmm9
-; SSE2-NEXT: movd %eax, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm9, %xmm8
-; SSE2-NEXT: pand %xmm0, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm8
-; SSE2-NEXT: por %xmm9, %xmm8
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm4
-; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: movd %edi, %xmm8
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm8
+; SSE2-NEXT: psrad $31, %xmm8
+; SSE2-NEXT: movdqa %xmm8, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm9
+; SSE2-NEXT: pand %xmm8, %xmm0
+; SSE2-NEXT: por %xmm9, %xmm0
+; SSE2-NEXT: movdqa %xmm8, %xmm4
; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: por %xmm0, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm5
-; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: pandn %xmm6, %xmm5
-; SSE2-NEXT: por %xmm0, %xmm5
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm0, %xmm6
-; SSE2-NEXT: pand %xmm3, %xmm0
-; SSE2-NEXT: pandn %xmm7, %xmm6
-; SSE2-NEXT: por %xmm0, %xmm6
-; SSE2-NEXT: movaps %xmm8, %xmm0
-; SSE2-NEXT: movaps %xmm4, %xmm1
-; SSE2-NEXT: movaps %xmm5, %xmm2
-; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: pand %xmm8, %xmm1
+; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: movdqa %xmm8, %xmm4
+; SSE2-NEXT: pandn %xmm6, %xmm4
+; SSE2-NEXT: pand %xmm8, %xmm2
+; SSE2-NEXT: por %xmm4, %xmm2
+; SSE2-NEXT: pand %xmm8, %xmm3
+; SSE2-NEXT: pandn %xmm7, %xmm8
+; SSE2-NEXT: por %xmm8, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8i64:
; AVX: # %bb.0:
-; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm5, %ymm5
-; AVX-NEXT: vmovd %eax, %ymm5
-; AVX-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
-; AVX-NEXT: vmovdqa %ymm5, %ymm4
-; AVX-NEXT: pand %ymm0, %ymm5
-; AVX-NEXT: pandn %ymm2, %ymm4
-; AVX-NEXT: por %ymm5, %ymm4
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm0, %ymm0
-; AVX-NEXT: vmovd %eax, %ymm0
-; AVX-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; AVX-NEXT: vmovdqa %ymm0, %ymm2
-; AVX-NEXT: pand %ymm1, %ymm0
-; AVX-NEXT: pandn %ymm3, %ymm2
-; AVX-NEXT: por %ymm0, %ymm2
-; AVX-NEXT: vmovaps %ymm4, %ymm0
-; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: vmovd %edi, %xmm4
+; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
+; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
+; AVX-NEXT: vpsllw $15, %xmm4, %xmm4
+; AVX-NEXT: vpsraw $15, %xmm4, %xmm4
+; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm4, %ymm4
+; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
+; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
+; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
+; AVX-NEXT: vandnps %ymm3, %ymm4, %ymm2
+; AVX-NEXT: vandps %ymm4, %ymm1, %ymm1
+; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v8i64:
; AVX2: # %bb.0:
-; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm5, %ymm5
-; AVX2-NEXT: vmovd %eax, %ymm5
-; AVX2-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
-; AVX2-NEXT: vmovdqa %ymm5, %ymm4
-; AVX2-NEXT: pand %ymm0, %ymm5
-; AVX2-NEXT: pandn %ymm2, %ymm4
-; AVX2-NEXT: por %ymm5, %ymm4
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm0, %ymm0
-; AVX2-NEXT: vmovd %eax, %ymm0
-; AVX2-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; AVX2-NEXT: vmovdqa %ymm0, %ymm2
-; AVX2-NEXT: pand %ymm1, %ymm0
-; AVX2-NEXT: pandn %ymm3, %ymm2
-; AVX2-NEXT: por %ymm0, %ymm2
-; AVX2-NEXT: vmovaps %ymm4, %ymm0
-; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: vmovd %edi, %xmm4
+; AVX2-NEXT: vpbroadcastw %xmm4, %xmm4
+; AVX2-NEXT: vpsllw $15, %xmm4, %xmm4
+; AVX2-NEXT: vpsraw $15, %xmm4, %xmm4
+; AVX2-NEXT: vpmovsxwq %xmm4, %ymm4
+; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm3, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v8i64:
; AVX512: # %bb.0:
@@ -943,113 +723,52 @@ define <8 x i64> @test_ctselect_v8i64(i1 %cond, <8 x i64> %a, <8 x i64> %b) {
define <8 x double> @test_ctselect_v8f64(i1 %cond, <8 x double> %a, <8 x double> %b) {
; SSE2-LABEL: test_ctselect_v8f64:
; SSE2: # %bb.0:
-; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm9, %xmm9
-; SSE2-NEXT: movd %eax, %xmm9
-; SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm9[0,0,0,0]
-; SSE2-NEXT: movapd %xmm9, %xmm8
-; SSE2-NEXT: pand %xmm0, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm8
-; SSE2-NEXT: por %xmm9, %xmm8
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movapd %xmm0, %xmm4
-; SSE2-NEXT: pand %xmm1, %xmm0
+; SSE2-NEXT: movd %edi, %xmm8
+; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
+; SSE2-NEXT: pslld $31, %xmm8
+; SSE2-NEXT: psrad $31, %xmm8
+; SSE2-NEXT: movdqa %xmm8, %xmm9
+; SSE2-NEXT: pandn %xmm4, %xmm9
+; SSE2-NEXT: pand %xmm8, %xmm0
+; SSE2-NEXT: por %xmm9, %xmm0
+; SSE2-NEXT: movdqa %xmm8, %xmm4
; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: por %xmm0, %xmm4
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movapd %xmm0, %xmm5
-; SSE2-NEXT: pand %xmm2, %xmm0
-; SSE2-NEXT: pandn %xmm6, %xmm5
-; SSE2-NEXT: por %xmm0, %xmm5
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movd %eax, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; SSE2-NEXT: movapd %xmm0, %xmm6
-; SSE2-NEXT: pand %xmm3, %xmm0
-; SSE2-NEXT: pandn %xmm7, %xmm6
-; SSE2-NEXT: por %xmm0, %xmm6
-; SSE2-NEXT: movaps %xmm8, %xmm0
-; SSE2-NEXT: movaps %xmm4, %xmm1
-; SSE2-NEXT: movaps %xmm5, %xmm2
-; SSE2-NEXT: movaps %xmm6, %xmm3
+; SSE2-NEXT: pand %xmm8, %xmm1
+; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: movdqa %xmm8, %xmm4
+; SSE2-NEXT: pandn %xmm6, %xmm4
+; SSE2-NEXT: pand %xmm8, %xmm2
+; SSE2-NEXT: por %xmm4, %xmm2
+; SSE2-NEXT: pand %xmm8, %xmm3
+; SSE2-NEXT: pandn %xmm7, %xmm8
+; SSE2-NEXT: por %xmm8, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8f64:
; AVX: # %bb.0:
-; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm5, %ymm5
-; AVX-NEXT: vmovd %eax, %ymm5
-; AVX-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
-; AVX-NEXT: vmovdqa %ymm5, %ymm4
-; AVX-NEXT: pand %ymm0, %ymm5
-; AVX-NEXT: pandn %ymm2, %ymm4
-; AVX-NEXT: por %ymm5, %ymm4
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm0, %ymm0
-; AVX-NEXT: vmovd %eax, %ymm0
-; AVX-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; AVX-NEXT: vmovdqa %ymm0, %ymm2
-; AVX-NEXT: pand %ymm1, %ymm0
-; AVX-NEXT: pandn %ymm3, %ymm2
-; AVX-NEXT: por %ymm0, %ymm2
-; AVX-NEXT: vmovaps %ymm4, %ymm0
-; AVX-NEXT: vmovaps %ymm2, %ymm1
+; AVX-NEXT: vmovd %edi, %xmm4
+; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
+; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
+; AVX-NEXT: vpsllw $15, %xmm4, %xmm4
+; AVX-NEXT: vpsraw $15, %xmm4, %xmm4
+; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm4, %ymm4
+; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
+; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
+; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
+; AVX-NEXT: vandnps %ymm3, %ymm4, %ymm2
+; AVX-NEXT: vandps %ymm4, %ymm1, %ymm1
+; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v8f64:
; AVX2: # %bb.0:
-; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm5, %ymm5
-; AVX2-NEXT: vmovd %eax, %ymm5
-; AVX2-NEXT: vpermilpd {{.*#+}} ymm5 = ymm5[0,0,2,2]
-; AVX2-NEXT: vmovdqa %ymm5, %ymm4
-; AVX2-NEXT: pand %ymm0, %ymm5
-; AVX2-NEXT: pandn %ymm2, %ymm4
-; AVX2-NEXT: por %ymm5, %ymm4
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm0, %ymm0
-; AVX2-NEXT: vmovd %eax, %ymm0
-; AVX2-NEXT: vpermilpd {{.*#+}} ymm0 = ymm0[0,0,2,2]
-; AVX2-NEXT: vmovdqa %ymm0, %ymm2
-; AVX2-NEXT: pand %ymm1, %ymm0
-; AVX2-NEXT: pandn %ymm3, %ymm2
-; AVX2-NEXT: por %ymm0, %ymm2
-; AVX2-NEXT: vmovaps %ymm4, %ymm0
-; AVX2-NEXT: vmovaps %ymm2, %ymm1
+; AVX2-NEXT: vmovd %edi, %xmm4
+; AVX2-NEXT: vpbroadcastw %xmm4, %xmm4
+; AVX2-NEXT: vpsllw $15, %xmm4, %xmm4
+; AVX2-NEXT: vpsraw $15, %xmm4, %xmm4
+; AVX2-NEXT: vpmovsxwq %xmm4, %ymm4
+; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm3, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v8f64:
; AVX512: # %bb.0:
>From 88fff4ba339fa171aa091b3cec20b96cf0d74a47 Mon Sep 17 00:00:00 2001
From: wizardengineer <juliuswoosebert at gmail.com>
Date: Fri, 22 May 2026 12:00:06 -0400
Subject: [PATCH 4/5] [X86] Restore combineSelect BLENDV sign-bit fold
The optimization that folds X86ISD::BLENDV away when the condition's
sign bit is statically known was inadvertently dropped during a prior
rebase conflict resolution. Its absence caused unrelated CHECK lines
in combine-sdiv.ll, vector-compress.ll, vector-idiv-{u,s}div-*.ll to
fail because main expects the post-optimization codegen.
Restores the block at llvm/lib/Target/X86/X86ISelLowering.cpp line
~49587 (after combineLogicBlendIntoConditionalNegate, before the
VSELECT/BLENDV shuffle handling), matching origin/main verbatim.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 18 +++++++++++++++---
1 file changed, 15 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 0bb638aa41700..be38a4b305668 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -26323,11 +26323,14 @@ SDValue X86TargetLowering::LowerCT_SELECT(SDValue Op, SelectionDAG &DAG) const {
// Perform CT_SELECT on each 32-bit chunk
SDValue Part0Ops[] = {FalsePart0, TruePart0, CC, ProcessedCond};
- SDValue Part0Select = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part0Ops);
+ SDValue Part0Select =
+ DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part0Ops);
SDValue Part1Ops[] = {FalsePart1, TruePart1, CC, ProcessedCond};
- SDValue Part1Select = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part1Ops);
+ SDValue Part1Select =
+ DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part1Ops);
SDValue Part2Ops[] = {FalsePart2, TruePart2, CC, ProcessedCond};
- SDValue Part2Select = DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part2Ops);
+ SDValue Part2Select =
+ DAG.getNode(X86ISD::CT_SELECT, DL, MVT::i32, Part2Ops);
// Create result stack slot and store the selected parts
SDValue ResultSlot = DAG.CreateStackTemporary(MVT::f80);
@@ -49581,6 +49584,15 @@ static SDValue combineSelect(SDNode *N, SelectionDAG &DAG,
DL, DAG, Subtarget))
return V;
+ // If the sign bit is known then BLENDV can be folded away.
+ if (N->getOpcode() == X86ISD::BLENDV) {
+ KnownBits KnownCond = DAG.computeKnownBits(Cond);
+ if (KnownCond.isNegative())
+ return LHS;
+ if (KnownCond.isNonNegative())
+ return RHS;
+ }
+
if (N->getOpcode() == ISD::VSELECT || N->getOpcode() == X86ISD::BLENDV) {
SmallVector<int, 64> CondMask;
if (createShuffleMaskFromVSELECT(CondMask, Cond,
>From f76446b66f0e56aa226738bb762622067970cb99 Mon Sep 17 00:00:00 2001
From: wizardengineer <juliuswoosebert at gmail.com>
Date: Fri, 22 May 2026 12:11:42 -0400
Subject: [PATCH 5/5] [X86] Stop claiming f128 CT_SELECT Custom; regen ctselect
tests
X86's LowerCT_SELECT falls through to X86ISD::CT_SELECT for any FP
type it claims Custom but doesn't special-case (only f80 has the
three-i32-chunk path, and scalar f32/f64 the SSE register path).
For f128 the fall-through produces an X86ISD::CT_SELECT with f128
result type, for which no isel pattern exists, and selection
ICEs with "Cannot select".
Drop f128 from the CT_SELECT Custom loop so the generic legalizer's
memory-blend path (recently added in the core PR) handles it.
Regenerate CHECK lines for all X86 ctselect tests to reflect the
expanded coverage (half/bfloat/fp80/fp128) introduced upstream.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 7 +-
llvm/test/CodeGen/X86/ctselect-edge-cases.ll | 27 +-
llvm/test/CodeGen/X86/ctselect-i386-fp.ll | 234 +-
llvm/test/CodeGen/X86/ctselect-i386-mmx.ll | 371 ++-
llvm/test/CodeGen/X86/ctselect-i386.ll | 71 +-
llvm/test/CodeGen/X86/ctselect-vector.ll | 659 +----
llvm/test/CodeGen/X86/ctselect.ll | 2538 ++++++++----------
7 files changed, 1408 insertions(+), 2499 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index be38a4b305668..85f76c8e904fb 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -492,11 +492,16 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// X86 wants to expand cmov itself.
for (auto VT : { MVT::f32, MVT::f64, MVT::f80, MVT::f128 }) {
setOperationAction(ISD::SELECT, VT, Custom);
- setOperationAction(ISD::CT_SELECT, VT, Custom);
setOperationAction(ISD::SETCC, VT, Custom);
setOperationAction(ISD::STRICT_FSETCC, VT, Custom);
setOperationAction(ISD::STRICT_FSETCCS, VT, Custom);
}
+ // CT_SELECT for f32/f64/f80: native lowering via LowerCT_SELECT.
+ // f128 has no isel pattern for X86ISD::CT_SELECT; fall through to the
+ // generic legalizer's memory-blend expansion.
+ for (auto VT : { MVT::f32, MVT::f64, MVT::f80 }) {
+ setOperationAction(ISD::CT_SELECT, VT, Custom);
+ }
for (auto VT : { MVT::i8, MVT::i16, MVT::i32, MVT::i64 }) {
if (VT == MVT::i64 && !Subtarget.is64Bit())
continue;
diff --git a/llvm/test/CodeGen/X86/ctselect-edge-cases.ll b/llvm/test/CodeGen/X86/ctselect-edge-cases.ll
index 0797265972a1f..33375747f9637 100644
--- a/llvm/test/CodeGen/X86/ctselect-edge-cases.ll
+++ b/llvm/test/CodeGen/X86/ctselect-edge-cases.ll
@@ -117,14 +117,7 @@ define float @test_ctselect_f32_special_values(i1 %cond) {
; X32-NEXT: sete %al
; X32-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %ecx
; X32-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %edx
-; X32-NEXT: movb %al, %ah
-; X32-NEXT: movzbl %ah, %edi
-; X32-NEXT: negl %edi
-; X32-NEXT: movl %edx, %esi
-; X32-NEXT: andl %edi, %esi
-; X32-NEXT: notl %edi
-; X32-NEXT: andl %ecx, %edi
-; X32-NEXT: orl %edi, %esi
+; X32-NEXT: BUNDLE
; X32-NEXT: movl %esi, (%esp)
; X32-NEXT: flds (%esp)
; X32-NEXT: addl $4, %esp
@@ -167,25 +160,11 @@ define double @test_ctselect_f64_special_values(i1 %cond) {
; X32-NEXT: fstpl (%esp)
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl (%esp), %edx
-; X32-NEXT: movb %al, %ah
-; X32-NEXT: movzbl %ah, %edi
-; X32-NEXT: negl %edi
-; X32-NEXT: movl %edx, %esi
-; X32-NEXT: andl %edi, %esi
-; X32-NEXT: notl %edi
-; X32-NEXT: andl %ecx, %edi
-; X32-NEXT: orl %edi, %esi
+; X32-NEXT: BUNDLE
; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movb %al, %ah
-; X32-NEXT: movzbl %ah, %edi
-; X32-NEXT: negl %edi
-; X32-NEXT: movl %edx, %esi
-; X32-NEXT: andl %edi, %esi
-; X32-NEXT: notl %edi
-; X32-NEXT: andl %ecx, %edi
-; X32-NEXT: orl %edi, %esi
+; X32-NEXT: BUNDLE
; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X32-NEXT: fldl {{[0-9]+}}(%esp)
; X32-NEXT: addl $24, %esp
diff --git a/llvm/test/CodeGen/X86/ctselect-i386-fp.ll b/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
index 4b5f31bad8313..c322c5a091695 100644
--- a/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
+++ b/llvm/test/CodeGen/X86/ctselect-i386-fp.ll
@@ -20,14 +20,7 @@ define float @test_ctselect_f32_basic(i1 %cond, float %a, float %b) nounwind {
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: flds (%esp)
; I386-NOCMOV-NEXT: addl $4, %esp
@@ -44,14 +37,7 @@ define float @test_ctselect_f32_basic(i1 %cond, float %a, float %b) nounwind {
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: flds (%esp)
; I386-CMOV-NEXT: addl $4, %esp
@@ -81,14 +67,7 @@ define float @test_ctselect_f32_eq(float %x, float %y, float %a, float %b) nounw
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: flds (%esp)
; I386-NOCMOV-NEXT: addl $4, %esp
@@ -111,14 +90,7 @@ define float @test_ctselect_f32_eq(float %x, float %y, float %a, float %b) nounw
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: flds (%esp)
; I386-CMOV-NEXT: addl $4, %esp
@@ -141,25 +113,11 @@ define double @test_ctselect_f64_basic(i1 %cond, double %a, double %b) nounwind
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: fldl (%esp)
; I386-NOCMOV-NEXT: addl $8, %esp
@@ -176,25 +134,11 @@ define double @test_ctselect_f64_basic(i1 %cond, double %a, double %b) nounwind
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: fldl (%esp)
; I386-CMOV-NEXT: addl $8, %esp
@@ -226,39 +170,18 @@ define x86_fp80 @test_ctselect_f80_basic(i1 %cond, x86_fp80 %a, x86_fp80 %b) nou
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
; I386-NOCMOV-NEXT: sete %ch
-; I386-NOCMOV-NEXT: movb %ch, %al
-; I386-NOCMOV-NEXT: movzbl %al, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %edi, %ebx
-; I386-NOCMOV-NEXT: andl %ebp, %ebx
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %edx, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %ebx
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: testb $1, %cl
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: sete %ch
-; I386-NOCMOV-NEXT: movb %ch, %cl
-; I386-NOCMOV-NEXT: movzbl %cl, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %edx, %edi
-; I386-NOCMOV-NEXT: andl %ebp, %edi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %eax, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl (%esp), %ebx # 4-byte Reload
-; I386-NOCMOV-NEXT: movb %al, %dl
-; I386-NOCMOV-NEXT: movzbl %dl, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %esi, %ecx
-; I386-NOCMOV-NEXT: andl %edi, %ecx
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ebx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %ecx
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: addl $40, %esp
@@ -310,14 +233,7 @@ define float @test_ctselect_f32_gt(float %x, float %y, float %a, float %b) nounw
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: flds (%esp)
; I386-NOCMOV-NEXT: addl $4, %esp
@@ -339,14 +255,7 @@ define float @test_ctselect_f32_gt(float %x, float %y, float %a, float %b) nounw
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: flds (%esp)
; I386-CMOV-NEXT: addl $4, %esp
@@ -369,14 +278,7 @@ define float @test_ctselect_f32_no_branches(i1 %cond, float %a, float %b) nounwi
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: flds (%esp)
; I386-NOCMOV-NEXT: addl $4, %esp
@@ -393,14 +295,7 @@ define float @test_ctselect_f32_no_branches(i1 %cond, float %a, float %b) nounwi
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: flds (%esp)
; I386-CMOV-NEXT: addl $4, %esp
@@ -422,14 +317,7 @@ define float @test_ctselect_f32_bundled(i1 %cond, float %a, float %b) nounwind {
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: flds (%esp)
; I386-NOCMOV-NEXT: addl $4, %esp
@@ -446,14 +334,7 @@ define float @test_ctselect_f32_bundled(i1 %cond, float %a, float %b) nounwind {
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: flds (%esp)
; I386-CMOV-NEXT: addl $4, %esp
@@ -480,14 +361,7 @@ define float @test_ctselect_f32_nan(i1 %cond) nounwind {
; I386-NOCMOV-NEXT: fstps (%esp)
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl (%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: addl $12, %esp
@@ -509,14 +383,7 @@ define float @test_ctselect_f32_nan(i1 %cond) nounwind {
; I386-CMOV-NEXT: fstps (%esp)
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl (%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: flds {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: addl $12, %esp
@@ -550,39 +417,18 @@ define x86_fp80 @test_ctselect_f80_alignment(i1 %cond, x86_fp80 %a, x86_fp80 %b)
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
; I386-NOCMOV-NEXT: sete %ch
-; I386-NOCMOV-NEXT: movb %ch, %al
-; I386-NOCMOV-NEXT: movzbl %al, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %edi, %ebx
-; I386-NOCMOV-NEXT: andl %ebp, %ebx
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %edx, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %ebx
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: testb $1, %cl
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: sete %ch
-; I386-NOCMOV-NEXT: movb %ch, %cl
-; I386-NOCMOV-NEXT: movzbl %cl, %ebp
-; I386-NOCMOV-NEXT: negl %ebp
-; I386-NOCMOV-NEXT: movl %edx, %edi
-; I386-NOCMOV-NEXT: andl %ebp, %edi
-; I386-NOCMOV-NEXT: notl %ebp
-; I386-NOCMOV-NEXT: andl %eax, %ebp
-; I386-NOCMOV-NEXT: orl %ebp, %edi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movl %ebx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl (%esp), %ebx # 4-byte Reload
-; I386-NOCMOV-NEXT: movb %al, %dl
-; I386-NOCMOV-NEXT: movzbl %dl, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %esi, %ecx
-; I386-NOCMOV-NEXT: andl %edi, %ecx
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ebx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %ecx
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: addl $40, %esp
@@ -627,27 +473,13 @@ define float @test_ctselect_f32_multiple(i1 %cond1, i1 %cond2, float %a, float %
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: movb %al, %ah
-; I386-NOCMOV-NEXT: movzbl %ah, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %ecx, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: movl %esi, (%esp)
; I386-NOCMOV-NEXT: flds (%esp)
; I386-NOCMOV-NEXT: addl $8, %esp
@@ -664,27 +496,13 @@ define float @test_ctselect_f32_multiple(i1 %cond1, i1 %cond2, float %a, float %
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: sete %al
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: movb %al, %ah
-; I386-CMOV-NEXT: movzbl %ah, %edi
-; I386-CMOV-NEXT: negl %edi
-; I386-CMOV-NEXT: movl %edx, %esi
-; I386-CMOV-NEXT: andl %edi, %esi
-; I386-CMOV-NEXT: notl %edi
-; I386-CMOV-NEXT: andl %ecx, %edi
-; I386-CMOV-NEXT: orl %edi, %esi
+; I386-CMOV-NEXT: BUNDLE
; I386-CMOV-NEXT: movl %esi, (%esp)
; I386-CMOV-NEXT: flds (%esp)
; I386-CMOV-NEXT: addl $8, %esp
diff --git a/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll b/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
index 2e1f16c19ca8e..ea258070ff633 100644
--- a/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
+++ b/llvm/test/CodeGen/X86/ctselect-i386-mmx.ll
@@ -10,67 +10,70 @@
define <1 x i64> @test_mmx_ctselect_with_paddd(i32 %cond, i64 %a, i64 %b) {
; I386-NOCMOV-LABEL: test_mmx_ctselect_with_paddd:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: subl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 24
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -8
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %ecx, %ecx
-; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: xorl %edx, %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: xorl %edx, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: andl %ecx, %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: paddd %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_mmx_ctselect_with_paddd:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: pushl %esi
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: subl $16, %esp
+; I386-CMOV-NEXT: subl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
-; I386-CMOV-NEXT: .cfi_offset %esi, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %ecx, %ecx
-; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: sbbl %ecx, %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: xorl %edx, %esi
-; I386-CMOV-NEXT: andl %ecx, %esi
-; I386-CMOV-NEXT: xorl %edx, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: andl %ecx, %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %dl
+; I386-CMOV-NEXT: testb %dl, %dl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: paddd %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $16, %esp
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: addl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
; I386-CMOV-NEXT: retl
%mmx_a = bitcast i64 %a to <1 x i64>
@@ -85,67 +88,70 @@ define <1 x i64> @test_mmx_ctselect_with_paddd(i32 %cond, i64 %a, i64 %b) {
define <1 x i64> @test_mmx_ctselect_with_psllw(i32 %cond, i64 %a, i64 %b) {
; I386-NOCMOV-LABEL: test_mmx_ctselect_with_psllw:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: subl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 24
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -8
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %ecx, %ecx
-; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: xorl %edx, %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: xorl %edx, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: andl %ecx, %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: psllw %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_mmx_ctselect_with_psllw:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: pushl %esi
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: subl $16, %esp
+; I386-CMOV-NEXT: subl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
-; I386-CMOV-NEXT: .cfi_offset %esi, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %ecx, %ecx
-; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: sbbl %ecx, %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: xorl %edx, %esi
-; I386-CMOV-NEXT: andl %ecx, %esi
-; I386-CMOV-NEXT: xorl %edx, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: andl %ecx, %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %dl
+; I386-CMOV-NEXT: testb %dl, %dl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: psllw %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $16, %esp
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: addl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
; I386-CMOV-NEXT: retl
%mmx_a = bitcast i64 %a to <1 x i64>
@@ -160,55 +166,58 @@ define <1 x i64> @test_mmx_ctselect_with_psllw(i32 %cond, i64 %a, i64 %b) {
define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64 %a, i64 %b, i64 %c) {
; I386-NOCMOV-LABEL: test_mmx_nested_ctselect_with_pand:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: pushl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: pushl %ebx
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %edi
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-NOCMOV-NEXT: subl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 32
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -16
-; I386-NOCMOV-NEXT: .cfi_offset %edi, -12
-; I386-NOCMOV-NEXT: .cfi_offset %ebx, -8
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: xorl %ecx, %ecx
-; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: movl $0, %edi
-; I386-NOCMOV-NEXT: sbbl %edi, %edi
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: andl %edi, %edx
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; I386-NOCMOV-NEXT: xorl %esi, %ebx
-; I386-NOCMOV-NEXT: andl %edi, %ebx
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
-; I386-NOCMOV-NEXT: xorl %edi, %esi
-; I386-NOCMOV-NEXT: xorl %ebx, %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: xorl %edi, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: xorl %esi, %eax
-; I386-NOCMOV-NEXT: xorl %edx, %eax
-; I386-NOCMOV-NEXT: andl %ecx, %eax
-; I386-NOCMOV-NEXT: xorl %esi, %eax
-; I386-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-NOCMOV-NEXT: sete %dl
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %dl
+; I386-NOCMOV-NEXT: testb %dl, %dl
+; I386-NOCMOV-NEXT: sete %dh
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: testb %dl, %dl
+; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: pand %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
; I386-NOCMOV-NEXT: popl %esi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; I386-NOCMOV-NEXT: popl %edi
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
@@ -216,51 +225,36 @@ define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64
; I386-CMOV: # %bb.0:
; I386-CMOV-NEXT: pushl %ebx
; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: pushl %edi
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
; I386-CMOV-NEXT: pushl %esi
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-CMOV-NEXT: subl $16, %esp
+; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-CMOV-NEXT: subl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 32
-; I386-CMOV-NEXT: .cfi_offset %esi, -16
-; I386-CMOV-NEXT: .cfi_offset %edi, -12
+; I386-CMOV-NEXT: .cfi_offset %esi, -12
; I386-CMOV-NEXT: .cfi_offset %ebx, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: xorl %ecx, %ecx
-; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: movl $0, %edi
-; I386-CMOV-NEXT: sbbl %edi, %edi
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: andl %edi, %edx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; I386-CMOV-NEXT: xorl %esi, %ebx
-; I386-CMOV-NEXT: andl %edi, %ebx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: sbbl %ecx, %ecx
-; I386-CMOV-NEXT: xorl %edi, %esi
-; I386-CMOV-NEXT: xorl %ebx, %esi
-; I386-CMOV-NEXT: andl %ecx, %esi
-; I386-CMOV-NEXT: xorl %edi, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: xorl %esi, %eax
-; I386-CMOV-NEXT: xorl %edx, %eax
-; I386-CMOV-NEXT: andl %ecx, %eax
-; I386-CMOV-NEXT: xorl %esi, %eax
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %bl
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %bh
+; I386-CMOV-NEXT: testb %bh, %bh
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; I386-CMOV-NEXT: testb %bl, %bl
+; I386-CMOV-NEXT: cmovnel %esi, %edx
+; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel %ecx, %eax
; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: pand %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $16, %esp
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 16
-; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: addl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 12
-; I386-CMOV-NEXT: popl %edi
+; I386-CMOV-NEXT: popl %esi
; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
; I386-CMOV-NEXT: popl %ebx
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
@@ -280,67 +274,70 @@ define <1 x i64> @test_mmx_nested_ctselect_with_pand(i32 %cond1, i32 %cond2, i64
define <1 x i64> @test_mmx_ctselect_with_por(i32 %cond, i64 %a, i64 %b) {
; I386-NOCMOV-LABEL: test_mmx_ctselect_with_por:
; I386-NOCMOV: # %bb.0:
-; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: pushl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-NOCMOV-NEXT: subl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 24
-; I386-NOCMOV-NEXT: .cfi_offset %esi, -8
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; I386-NOCMOV-NEXT: pushl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: pushl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: pushl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; I386-NOCMOV-NEXT: subl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 40
+; I386-NOCMOV-NEXT: .cfi_offset %esi, -20
+; I386-NOCMOV-NEXT: .cfi_offset %edi, -16
+; I386-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; I386-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %ecx, %ecx
-; I386-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-NOCMOV-NEXT: sbbl %ecx, %ecx
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-NOCMOV-NEXT: xorl %edx, %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: xorl %edx, %esi
-; I386-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: andl %ecx, %edx
-; I386-NOCMOV-NEXT: xorl %eax, %edx
-; I386-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: setne %bl
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: sete %bh
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: testb %bl, %bl
+; I386-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; I386-NOCMOV-NEXT: sete %al
+; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; I386-NOCMOV-NEXT: BUNDLE
+; I386-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-NOCMOV-NEXT: por %mm0, %mm0
; I386-NOCMOV-NEXT: movq %mm0, (%esp)
; I386-NOCMOV-NEXT: movl (%esp), %eax
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-NOCMOV-NEXT: addl $16, %esp
-; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: addl $20, %esp
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 20
; I386-NOCMOV-NEXT: popl %esi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; I386-NOCMOV-NEXT: popl %edi
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; I386-NOCMOV-NEXT: popl %ebx
+; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; I386-NOCMOV-NEXT: popl %ebp
; I386-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_mmx_ctselect_with_por:
; I386-CMOV: # %bb.0:
-; I386-CMOV-NEXT: pushl %esi
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: subl $16, %esp
+; I386-CMOV-NEXT: subl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 24
-; I386-CMOV-NEXT: .cfi_offset %esi, -8
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %ecx, %ecx
-; I386-CMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; I386-CMOV-NEXT: sbbl %ecx, %ecx
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; I386-CMOV-NEXT: xorl %edx, %esi
-; I386-CMOV-NEXT: andl %ecx, %esi
-; I386-CMOV-NEXT: xorl %edx, %esi
-; I386-CMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: andl %ecx, %edx
-; I386-CMOV-NEXT: xorl %eax, %edx
-; I386-CMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: cmpl $0, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: setne %dl
+; I386-CMOV-NEXT: testb %dl, %dl
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; I386-CMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; I386-CMOV-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; I386-CMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
; I386-CMOV-NEXT: movq {{[0-9]+}}(%esp), %mm0
; I386-CMOV-NEXT: por %mm0, %mm0
; I386-CMOV-NEXT: movq %mm0, (%esp)
; I386-CMOV-NEXT: movl (%esp), %eax
; I386-CMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; I386-CMOV-NEXT: addl $16, %esp
-; I386-CMOV-NEXT: .cfi_def_cfa_offset 8
-; I386-CMOV-NEXT: popl %esi
+; I386-CMOV-NEXT: addl $20, %esp
; I386-CMOV-NEXT: .cfi_def_cfa_offset 4
; I386-CMOV-NEXT: retl
%mmx_a = bitcast i64 %a to <1 x i64>
diff --git a/llvm/test/CodeGen/X86/ctselect-i386.ll b/llvm/test/CodeGen/X86/ctselect-i386.ll
index d1cc559f0c1c1..d5a28e8e578f4 100644
--- a/llvm/test/CodeGen/X86/ctselect-i386.ll
+++ b/llvm/test/CodeGen/X86/ctselect-i386.ll
@@ -19,14 +19,7 @@ define i32 @test_ctselect_i32_basic(i1 %cond, i32 %a, i32 %b) nounwind {
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %bl
-; I386-NOCMOV-NEXT: movb %bl, %bh
-; I386-NOCMOV-NEXT: movzbl %bh, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %eax
-; I386-NOCMOV-NEXT: andl %esi, %eax
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %ebx
; I386-NOCMOV-NEXT: retl
@@ -51,14 +44,7 @@ define i16 @test_ctselect_i16_basic(i1 %cond, i16 %a, i16 %b) nounwind {
; I386-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %bl
-; I386-NOCMOV-NEXT: movb %bl, %bh
-; I386-NOCMOV-NEXT: movzbw %bh, %si
-; I386-NOCMOV-NEXT: negw %si
-; I386-NOCMOV-NEXT: movw %dx, %ax
-; I386-NOCMOV-NEXT: andw %si, %ax
-; I386-NOCMOV-NEXT: notw %si
-; I386-NOCMOV-NEXT: andw %cx, %si
-; I386-NOCMOV-NEXT: orw %si, %ax
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %ebx
; I386-NOCMOV-NEXT: retl
@@ -81,13 +67,7 @@ define i8 @test_ctselect_i8_basic(i1 %cond, i8 %a, i8 %b) nounwind {
; I386-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %ah
-; I386-NOCMOV-NEXT: movb %ah, %ch
-; I386-NOCMOV-NEXT: negb %ch
-; I386-NOCMOV-NEXT: movb %dl, %al
-; I386-NOCMOV-NEXT: andb %ch, %al
-; I386-NOCMOV-NEXT: notb %ch
-; I386-NOCMOV-NEXT: andb %cl, %ch
-; I386-NOCMOV-NEXT: orb %ch, %al
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: retl
;
; I386-CMOV-LABEL: test_ctselect_i8_basic:
@@ -113,14 +93,7 @@ define i32 @test_crypto_key_select(i32 %secret_bit, i32 %key1, i32 %key2) nounwi
; I386-NOCMOV-NEXT: setne %al
; I386-NOCMOV-NEXT: testb %al, %al
; I386-NOCMOV-NEXT: sete %bl
-; I386-NOCMOV-NEXT: movb %bl, %bh
-; I386-NOCMOV-NEXT: movzbl %bh, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %eax
-; I386-NOCMOV-NEXT: andl %esi, %eax
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %ebx
; I386-NOCMOV-NEXT: retl
@@ -150,14 +123,7 @@ define i32 @test_no_conditional_branches(i32 %secret, i32 %val1, i32 %val2) noun
; I386-NOCMOV-NEXT: setne %al
; I386-NOCMOV-NEXT: testb %al, %al
; I386-NOCMOV-NEXT: sete %bl
-; I386-NOCMOV-NEXT: movb %bl, %bh
-; I386-NOCMOV-NEXT: movzbl %bh, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %eax
-; I386-NOCMOV-NEXT: andl %esi, %eax
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %ebx
; I386-NOCMOV-NEXT: retl
@@ -187,14 +153,7 @@ define i32 @test_ctselect_i32_cmp(i32 %a, i32 %b, i32 %c) nounwind {
; I386-NOCMOV-NEXT: sete %al
; I386-NOCMOV-NEXT: testb %al, %al
; I386-NOCMOV-NEXT: sete %bl
-; I386-NOCMOV-NEXT: movb %bl, %bh
-; I386-NOCMOV-NEXT: movzbl %bh, %esi
-; I386-NOCMOV-NEXT: negl %esi
-; I386-NOCMOV-NEXT: movl %edx, %eax
-; I386-NOCMOV-NEXT: andl %esi, %eax
-; I386-NOCMOV-NEXT: notl %esi
-; I386-NOCMOV-NEXT: andl %ecx, %esi
-; I386-NOCMOV-NEXT: orl %esi, %eax
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %ebx
; I386-NOCMOV-NEXT: retl
@@ -224,24 +183,10 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) n
; I386-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %bl
-; I386-NOCMOV-NEXT: movb %bl, %bh
-; I386-NOCMOV-NEXT: movzbl %bh, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %edx, %esi
-; I386-NOCMOV-NEXT: andl %edi, %esi
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %eax, %edi
-; I386-NOCMOV-NEXT: orl %edi, %esi
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; I386-NOCMOV-NEXT: sete %dl
-; I386-NOCMOV-NEXT: movb %dl, %dh
-; I386-NOCMOV-NEXT: movzbl %dh, %edi
-; I386-NOCMOV-NEXT: negl %edi
-; I386-NOCMOV-NEXT: movl %ecx, %eax
-; I386-NOCMOV-NEXT: andl %edi, %eax
-; I386-NOCMOV-NEXT: notl %edi
-; I386-NOCMOV-NEXT: andl %esi, %edi
-; I386-NOCMOV-NEXT: orl %edi, %eax
+; I386-NOCMOV-NEXT: BUNDLE
; I386-NOCMOV-NEXT: popl %esi
; I386-NOCMOV-NEXT: popl %edi
; I386-NOCMOV-NEXT: popl %ebx
diff --git a/llvm/test/CodeGen/X86/ctselect-vector.ll b/llvm/test/CodeGen/X86/ctselect-vector.ll
index 79b76d1afca5c..dd59d16a04280 100644
--- a/llvm/test/CodeGen/X86/ctselect-vector.ll
+++ b/llvm/test/CodeGen/X86/ctselect-vector.ll
@@ -10,51 +10,21 @@ define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) {
; SSE2-LABEL: test_ctselect_v4i32:
; SSE2: # %bb.0:
; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v4i32:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v4i32:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4i32:
@@ -74,51 +44,21 @@ define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b
; SSE2-LABEL: test_ctselect_v4f32:
; SSE2: # %bb.0:
; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movaps %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v4f32:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v4f32:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4f32:
@@ -138,51 +78,21 @@ define <2 x i64> @test_ctselect_v2i64(i1 %cond, <2 x i64> %a, <2 x i64> %b) {
; SSE2-LABEL: test_ctselect_v2i64:
; SSE2: # %bb.0:
; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v2i64:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v2i64:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v2i64:
@@ -202,51 +112,21 @@ define <2 x double> @test_ctselect_v2f64(i1 %cond, <2 x double> %a, <2 x double>
; SSE2-LABEL: test_ctselect_v2f64:
; SSE2: # %bb.0:
; SSE2-NEXT: testb $1, %dil
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movapd %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v2f64:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v2f64:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v2f64:
@@ -266,50 +146,24 @@ define <2 x double> @test_ctselect_v2f64(i1 %cond, <2 x double> %a, <2 x double>
define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
; SSE2-LABEL: test_ctselect_v8i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm4
-; SSE2-NEXT: psrad $31, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm5
-; SSE2-NEXT: pand %xmm4, %xmm0
-; SSE2-NEXT: por %xmm5, %xmm0
-; SSE2-NEXT: pand %xmm4, %xmm1
-; SSE2-NEXT: pandn %xmm3, %xmm4
-; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8i32:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm3, %ymm3
-; AVX-NEXT: vmovd %eax, %ymm3
-; AVX-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
-; AVX-NEXT: vmovdqa %ymm3, %ymm2
-; AVX-NEXT: pand %ymm0, %ymm3
-; AVX-NEXT: pandn %ymm1, %ymm2
-; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %ymm2, %ymm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v8i32:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm3, %ymm3
-; AVX2-NEXT: vmovd %eax, %ymm3
-; AVX2-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
-; AVX2-NEXT: vmovdqa %ymm3, %ymm2
-; AVX2-NEXT: pand %ymm0, %ymm3
-; AVX2-NEXT: pandn %ymm1, %ymm2
-; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %ymm2, %ymm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v8i32:
@@ -328,50 +182,24 @@ define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b) {
; SSE2-LABEL: test_ctselect_v8f32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm4
-; SSE2-NEXT: psrad $31, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm5
-; SSE2-NEXT: pand %xmm4, %xmm0
-; SSE2-NEXT: por %xmm5, %xmm0
-; SSE2-NEXT: pand %xmm4, %xmm1
-; SSE2-NEXT: pandn %xmm3, %xmm4
-; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8f32:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm3, %ymm3
-; AVX-NEXT: vmovd %eax, %ymm3
-; AVX-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
-; AVX-NEXT: vmovdqa %ymm3, %ymm2
-; AVX-NEXT: pand %ymm0, %ymm3
-; AVX-NEXT: pandn %ymm1, %ymm2
-; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %ymm2, %ymm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v8f32:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm3, %ymm3
-; AVX2-NEXT: vmovd %eax, %ymm3
-; AVX2-NEXT: vpermilps {{.*#+}} ymm3 = ymm3[0,0,0,0,4,4,4,4]
-; AVX2-NEXT: vmovdqa %ymm3, %ymm2
-; AVX2-NEXT: pand %ymm0, %ymm3
-; AVX2-NEXT: pandn %ymm1, %ymm2
-; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %ymm2, %ymm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v8f32:
@@ -390,50 +218,24 @@ define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b
define <4 x i64> @test_ctselect_v4i64(i1 %cond, <4 x i64> %a, <4 x i64> %b) {
; SSE2-LABEL: test_ctselect_v4i64:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm4
-; SSE2-NEXT: psrad $31, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm5
-; SSE2-NEXT: pand %xmm4, %xmm0
-; SSE2-NEXT: por %xmm5, %xmm0
-; SSE2-NEXT: pand %xmm4, %xmm1
-; SSE2-NEXT: pandn %xmm3, %xmm4
-; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v4i64:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm3, %ymm3
-; AVX-NEXT: vmovd %eax, %ymm3
-; AVX-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
-; AVX-NEXT: vmovdqa %ymm3, %ymm2
-; AVX-NEXT: pand %ymm0, %ymm3
-; AVX-NEXT: pandn %ymm1, %ymm2
-; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %ymm2, %ymm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v4i64:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm3, %ymm3
-; AVX2-NEXT: vmovd %eax, %ymm3
-; AVX2-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
-; AVX2-NEXT: vmovdqa %ymm3, %ymm2
-; AVX2-NEXT: pand %ymm0, %ymm3
-; AVX2-NEXT: pandn %ymm1, %ymm2
-; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %ymm2, %ymm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4i64:
@@ -452,50 +254,24 @@ define <4 x i64> @test_ctselect_v4i64(i1 %cond, <4 x i64> %a, <4 x i64> %b) {
define <4 x double> @test_ctselect_v4f64(i1 %cond, <4 x double> %a, <4 x double> %b) {
; SSE2-LABEL: test_ctselect_v4f64:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm4
-; SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm4
-; SSE2-NEXT: psrad $31, %xmm4
-; SSE2-NEXT: movdqa %xmm4, %xmm5
-; SSE2-NEXT: pandn %xmm2, %xmm5
-; SSE2-NEXT: pand %xmm4, %xmm0
-; SSE2-NEXT: por %xmm5, %xmm0
-; SSE2-NEXT: pand %xmm4, %xmm1
-; SSE2-NEXT: pandn %xmm3, %xmm4
-; SSE2-NEXT: por %xmm4, %xmm1
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm4, %xmm0
+; SSE2-NEXT: movaps %xmm2, %xmm1
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v4f64:
; AVX: # %bb.0:
; AVX-NEXT: testb $1, %dil
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %ymm3, %ymm3
-; AVX-NEXT: vmovd %eax, %ymm3
-; AVX-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
-; AVX-NEXT: vmovdqa %ymm3, %ymm2
-; AVX-NEXT: pand %ymm0, %ymm3
-; AVX-NEXT: pandn %ymm1, %ymm2
-; AVX-NEXT: por %ymm3, %ymm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %ymm2, %ymm0
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v4f64:
; AVX2: # %bb.0:
; AVX2-NEXT: testb $1, %dil
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %ymm3, %ymm3
-; AVX2-NEXT: vmovd %eax, %ymm3
-; AVX2-NEXT: vpermilpd {{.*#+}} ymm3 = ymm3[0,0,2,2]
-; AVX2-NEXT: vmovdqa %ymm3, %ymm2
-; AVX2-NEXT: pand %ymm0, %ymm3
-; AVX2-NEXT: pandn %ymm1, %ymm2
-; AVX2-NEXT: por %ymm3, %ymm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %ymm2, %ymm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4f64:
@@ -515,63 +291,33 @@ define <4 x double> @test_ctselect_v4f64(i1 %cond, <4 x double> %a, <4 x double>
define <16 x i32> @test_ctselect_v16i32(i1 %cond, <16 x i32> %a, <16 x i32> %b) {
; SSE2-LABEL: test_ctselect_v16i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm8
-; SSE2-NEXT: psrad $31, %xmm8
-; SSE2-NEXT: movdqa %xmm8, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm9
-; SSE2-NEXT: pand %xmm8, %xmm0
-; SSE2-NEXT: por %xmm9, %xmm0
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm1
-; SSE2-NEXT: por %xmm4, %xmm1
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm6, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm2
-; SSE2-NEXT: por %xmm4, %xmm2
-; SSE2-NEXT: pand %xmm8, %xmm3
-; SSE2-NEXT: pandn %xmm7, %xmm8
-; SSE2-NEXT: por %xmm8, %xmm3
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v16i32:
; AVX: # %bb.0:
-; AVX-NEXT: vmovd %edi, %xmm4
-; AVX-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX-NEXT: vpshufb %xmm5, %xmm4, %xmm4
-; AVX-NEXT: vpsllw $7, %xmm4, %xmm4
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
-; AVX-NEXT: vpcmpgtb %xmm4, %xmm5, %xmm4
-; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; AVX-NEXT: vpmovsxbd %xmm5, %xmm5
-; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
-; AVX-NEXT: vpmovsxbd %xmm6, %xmm6
-; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5
-; AVX-NEXT: vpmovsxbd %xmm4, %xmm6
-; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
-; AVX-NEXT: vpmovsxbd %xmm4, %xmm4
-; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4
-; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
-; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
-; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
-; AVX-NEXT: vandnps %ymm3, %ymm5, %ymm2
-; AVX-NEXT: vandps %ymm5, %ymm1, %ymm1
-; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v16i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovd %edi, %xmm4
-; AVX2-NEXT: vpbroadcastb %xmm4, %xmm4
-; AVX2-NEXT: vpsllw $7, %xmm4, %xmm4
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm5
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[2,3,2,3]
-; AVX2-NEXT: vpmovsxbd %xmm5, %ymm5
-; AVX2-NEXT: vpmovsxbd %xmm4, %ymm4
-; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT: vpblendvb %ymm5, %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v16i32:
; AVX512: # %bb.0:
@@ -589,60 +335,33 @@ define <16 x i32> @test_ctselect_v16i32(i1 %cond, <16 x i32> %a, <16 x i32> %b)
define <16 x float> @test_ctselect_v16f32(i1 %cond, <16 x float> %a, <16 x float> %b) {
; SSE2-LABEL: test_ctselect_v16f32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm8
-; SSE2-NEXT: psrad $31, %xmm8
-; SSE2-NEXT: movdqa %xmm8, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm9
-; SSE2-NEXT: pand %xmm8, %xmm0
-; SSE2-NEXT: por %xmm9, %xmm0
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm1
-; SSE2-NEXT: por %xmm4, %xmm1
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm6, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm2
-; SSE2-NEXT: por %xmm4, %xmm2
-; SSE2-NEXT: pand %xmm8, %xmm3
-; SSE2-NEXT: pandn %xmm7, %xmm8
-; SSE2-NEXT: por %xmm8, %xmm3
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v16f32:
; AVX: # %bb.0:
-; AVX-NEXT: vmovd %edi, %xmm4
-; AVX-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX-NEXT: vpshufb %xmm5, %xmm4, %xmm4
-; AVX-NEXT: vpsllw $7, %xmm4, %xmm4
-; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm4, %xmm4
-; AVX-NEXT: vpcmpgtb %xmm4, %xmm5, %xmm4
-; AVX-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; AVX-NEXT: vpmovsxbd %xmm5, %xmm5
-; AVX-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[3,3,3,3]
-; AVX-NEXT: vpmovsxbd %xmm6, %xmm6
-; AVX-NEXT: vinsertf128 $1, %xmm6, %ymm5, %ymm5
-; AVX-NEXT: vpmovsxbd %xmm4, %xmm6
-; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[1,1,1,1]
-; AVX-NEXT: vpmovsxbd %xmm4, %xmm4
-; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm6, %ymm4
-; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
-; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
-; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
-; AVX-NEXT: vandnps %ymm3, %ymm5, %ymm2
-; AVX-NEXT: vandps %ymm5, %ymm1, %ymm1
-; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v16f32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovd %edi, %xmm4
-; AVX2-NEXT: vpbroadcastb %xmm4, %xmm4
-; AVX2-NEXT: vpsllw $7, %xmm4, %xmm4
-; AVX2-NEXT: vpmovsxbd %xmm4, %ymm4
-; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v16f32:
; AVX512: # %bb.0:
@@ -660,52 +379,33 @@ define <16 x float> @test_ctselect_v16f32(i1 %cond, <16 x float> %a, <16 x float
define <8 x i64> @test_ctselect_v8i64(i1 %cond, <8 x i64> %a, <8 x i64> %b) {
; SSE2-LABEL: test_ctselect_v8i64:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm8
-; SSE2-NEXT: psrad $31, %xmm8
-; SSE2-NEXT: movdqa %xmm8, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm9
-; SSE2-NEXT: pand %xmm8, %xmm0
-; SSE2-NEXT: por %xmm9, %xmm0
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm1
-; SSE2-NEXT: por %xmm4, %xmm1
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm6, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm2
-; SSE2-NEXT: por %xmm4, %xmm2
-; SSE2-NEXT: pand %xmm8, %xmm3
-; SSE2-NEXT: pandn %xmm7, %xmm8
-; SSE2-NEXT: por %xmm8, %xmm3
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8i64:
; AVX: # %bb.0:
-; AVX-NEXT: vmovd %edi, %xmm4
-; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
-; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
-; AVX-NEXT: vpsllw $15, %xmm4, %xmm4
-; AVX-NEXT: vpsraw $15, %xmm4, %xmm4
-; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm4, %ymm4
-; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
-; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
-; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
-; AVX-NEXT: vandnps %ymm3, %ymm4, %ymm2
-; AVX-NEXT: vandps %ymm4, %ymm1, %ymm1
-; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v8i64:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovd %edi, %xmm4
-; AVX2-NEXT: vpbroadcastw %xmm4, %xmm4
-; AVX2-NEXT: vpsllw $15, %xmm4, %xmm4
-; AVX2-NEXT: vpsraw $15, %xmm4, %xmm4
-; AVX2-NEXT: vpmovsxwq %xmm4, %ymm4
-; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v8i64:
; AVX512: # %bb.0:
@@ -723,52 +423,33 @@ define <8 x i64> @test_ctselect_v8i64(i1 %cond, <8 x i64> %a, <8 x i64> %b) {
define <8 x double> @test_ctselect_v8f64(i1 %cond, <8 x double> %a, <8 x double> %b) {
; SSE2-LABEL: test_ctselect_v8f64:
; SSE2: # %bb.0:
-; SSE2-NEXT: movd %edi, %xmm8
-; SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm8[0,0,0,0]
-; SSE2-NEXT: pslld $31, %xmm8
-; SSE2-NEXT: psrad $31, %xmm8
-; SSE2-NEXT: movdqa %xmm8, %xmm9
-; SSE2-NEXT: pandn %xmm4, %xmm9
-; SSE2-NEXT: pand %xmm8, %xmm0
-; SSE2-NEXT: por %xmm9, %xmm0
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm5, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm1
-; SSE2-NEXT: por %xmm4, %xmm1
-; SSE2-NEXT: movdqa %xmm8, %xmm4
-; SSE2-NEXT: pandn %xmm6, %xmm4
-; SSE2-NEXT: pand %xmm8, %xmm2
-; SSE2-NEXT: por %xmm4, %xmm2
-; SSE2-NEXT: pand %xmm8, %xmm3
-; SSE2-NEXT: pandn %xmm7, %xmm8
-; SSE2-NEXT: por %xmm8, %xmm3
+; SSE2-NEXT: testb $1, %dil
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: BUNDLE
+; SSE2-NEXT: movaps %xmm8, %xmm0
+; SSE2-NEXT: movaps %xmm4, %xmm1
+; SSE2-NEXT: movaps %xmm5, %xmm2
+; SSE2-NEXT: movaps %xmm6, %xmm3
; SSE2-NEXT: retq
;
; AVX-LABEL: test_ctselect_v8f64:
; AVX: # %bb.0:
-; AVX-NEXT: vmovd %edi, %xmm4
-; AVX-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
-; AVX-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
-; AVX-NEXT: vpsllw $15, %xmm4, %xmm4
-; AVX-NEXT: vpsraw $15, %xmm4, %xmm4
-; AVX-NEXT: vinsertf128 $1, %xmm4, %ymm4, %ymm4
-; AVX-NEXT: vandnps %ymm2, %ymm4, %ymm2
-; AVX-NEXT: vandps %ymm4, %ymm0, %ymm0
-; AVX-NEXT: vorps %ymm2, %ymm0, %ymm0
-; AVX-NEXT: vandnps %ymm3, %ymm4, %ymm2
-; AVX-NEXT: vandps %ymm4, %ymm1, %ymm1
-; AVX-NEXT: vorps %ymm2, %ymm1, %ymm1
+; AVX-NEXT: testb $1, %dil
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: BUNDLE
+; AVX-NEXT: vmovaps %ymm4, %ymm0
+; AVX-NEXT: vmovaps %ymm2, %ymm1
; AVX-NEXT: retq
;
; AVX2-LABEL: test_ctselect_v8f64:
; AVX2: # %bb.0:
-; AVX2-NEXT: vmovd %edi, %xmm4
-; AVX2-NEXT: vpbroadcastw %xmm4, %xmm4
-; AVX2-NEXT: vpsllw $15, %xmm4, %xmm4
-; AVX2-NEXT: vpsraw $15, %xmm4, %xmm4
-; AVX2-NEXT: vpmovsxwq %xmm4, %ymm4
-; AVX2-NEXT: vpblendvb %ymm4, %ymm0, %ymm2, %ymm0
-; AVX2-NEXT: vpblendvb %ymm4, %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: testb $1, %dil
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: BUNDLE
+; AVX2-NEXT: vmovaps %ymm4, %ymm0
+; AVX2-NEXT: vmovaps %ymm2, %ymm1
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v8f64:
; AVX512: # %bb.0:
@@ -789,17 +470,7 @@ define <4 x i32> @test_ctselect_v4i32_const_true(<4 x i32> %a, <4 x i32> %b) {
; SSE2: # %bb.0:
; SSE2-NEXT: movb $1, %al
; SSE2-NEXT: testb %al, %al
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
@@ -807,17 +478,7 @@ define <4 x i32> @test_ctselect_v4i32_const_true(<4 x i32> %a, <4 x i32> %b) {
; AVX: # %bb.0:
; AVX-NEXT: movb $1, %al
; AVX-NEXT: testb %al, %al
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
@@ -825,17 +486,7 @@ define <4 x i32> @test_ctselect_v4i32_const_true(<4 x i32> %a, <4 x i32> %b) {
; AVX2: # %bb.0:
; AVX2-NEXT: movb $1, %al
; AVX2-NEXT: testb %al, %al
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4i32_const_true:
@@ -850,17 +501,7 @@ define <4 x i32> @test_ctselect_v4i32_const_false(<4 x i32> %a, <4 x i32> %b) {
; SSE2: # %bb.0:
; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: testb %al, %al
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
@@ -868,17 +509,7 @@ define <4 x i32> @test_ctselect_v4i32_const_false(<4 x i32> %a, <4 x i32> %b) {
; AVX: # %bb.0:
; AVX-NEXT: xorl %eax, %eax
; AVX-NEXT: testb %al, %al
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
@@ -886,17 +517,7 @@ define <4 x i32> @test_ctselect_v4i32_const_false(<4 x i32> %a, <4 x i32> %b) {
; AVX2: # %bb.0:
; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: testb %al, %al
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4i32_const_false:
@@ -914,17 +535,7 @@ define <4 x i32> @test_ctselect_v4i32_icmp(i32 %x, i32 %y, <4 x i32> %a, <4 x i3
; SSE2-NEXT: cmpl %esi, %edi
; SSE2-NEXT: sete %al
; SSE2-NEXT: testb %al, %al
-; SSE2-NEXT: movl $0, %eax
-; SSE2-NEXT: setne %al
-; SSE2-NEXT: movzbl %al, %eax
-; SSE2-NEXT: negl %eax
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movd %eax, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; SSE2-NEXT: movdqa %xmm3, %xmm2
-; SSE2-NEXT: pand %xmm0, %xmm3
-; SSE2-NEXT: pandn %xmm1, %xmm2
-; SSE2-NEXT: por %xmm3, %xmm2
+; SSE2-NEXT: BUNDLE
; SSE2-NEXT: movaps %xmm2, %xmm0
; SSE2-NEXT: retq
;
@@ -933,17 +544,7 @@ define <4 x i32> @test_ctselect_v4i32_icmp(i32 %x, i32 %y, <4 x i32> %a, <4 x i3
; AVX-NEXT: cmpl %esi, %edi
; AVX-NEXT: sete %al
; AVX-NEXT: testb %al, %al
-; AVX-NEXT: movl $0, %eax
-; AVX-NEXT: setne %al
-; AVX-NEXT: movzbl %al, %eax
-; AVX-NEXT: negl %eax
-; AVX-NEXT: pxor %xmm3, %xmm3
-; AVX-NEXT: movd %eax, %xmm3
-; AVX-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX-NEXT: movdqa %xmm3, %xmm2
-; AVX-NEXT: pand %xmm0, %xmm3
-; AVX-NEXT: pandn %xmm1, %xmm2
-; AVX-NEXT: por %xmm3, %xmm2
+; AVX-NEXT: BUNDLE
; AVX-NEXT: vmovaps %xmm2, %xmm0
; AVX-NEXT: retq
;
@@ -952,17 +553,7 @@ define <4 x i32> @test_ctselect_v4i32_icmp(i32 %x, i32 %y, <4 x i32> %a, <4 x i3
; AVX2-NEXT: cmpl %esi, %edi
; AVX2-NEXT: sete %al
; AVX2-NEXT: testb %al, %al
-; AVX2-NEXT: movl $0, %eax
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: pxor %xmm3, %xmm3
-; AVX2-NEXT: movd %eax, %xmm3
-; AVX2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,0,0,0]
-; AVX2-NEXT: movdqa %xmm3, %xmm2
-; AVX2-NEXT: pand %xmm0, %xmm3
-; AVX2-NEXT: pandn %xmm1, %xmm2
-; AVX2-NEXT: por %xmm3, %xmm2
+; AVX2-NEXT: BUNDLE
; AVX2-NEXT: vmovaps %xmm2, %xmm0
; AVX2-NEXT: retq
; AVX512-LABEL: test_ctselect_v4i32_icmp:
diff --git a/llvm/test/CodeGen/X86/ctselect.ll b/llvm/test/CodeGen/X86/ctselect.ll
index 9d54c29db9683..029eacc72b856 100644
--- a/llvm/test/CodeGen/X86/ctselect.ll
+++ b/llvm/test/CodeGen/X86/ctselect.ll
@@ -8,37 +8,27 @@
define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) #0 {
; X64-LABEL: test_ctselect_i8:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: andb $1, %al
-; X64-NEXT: xorl %edx, %esi
-; X64-NEXT: negb %al
-; X64-NEXT: andb %sil, %al
-; X64-NEXT: xorb %dl, %al
+; X64-NEXT: movl %edx, %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %esi, %eax
; X64-NEXT: # kill: def $al killed $al killed $eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_i8:
; X32: # %bb.0:
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorb %cl, %dl
-; X32-NEXT: negb %al
-; X32-NEXT: andb %dl, %al
-; X32-NEXT: xorb %cl, %al
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: # kill: def $al killed $al killed $eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_i8:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorb %cl, %dl
-; X32-NOCMOV-NEXT: negb %al
-; X32-NOCMOV-NEXT: andb %dl, %al
-; X32-NOCMOV-NEXT: xorb %cl, %al
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: retl
%result = call i8 @llvm.ct.select.i8(i1 %cond, i8 %a, i8 %b)
ret i8 %result
@@ -47,38 +37,29 @@ define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) #0 {
define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_i32:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: xorl %edx, %esi
-; X64-NEXT: andl $1, %eax
-; X64-NEXT: negl %eax
-; X64-NEXT: andl %esi, %eax
-; X64-NEXT: xorl %edx, %eax
+; X64-NEXT: movl %edx, %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %esi, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_i32:
; X32: # %bb.0:
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_i32:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
ret i32 %result
@@ -87,56 +68,41 @@ define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) #0 {
define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) #0 {
; X64-LABEL: test_ctselect_i64:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: xorq %rdx, %rsi
-; X64-NEXT: andl $1, %eax
-; X64-NEXT: negq %rax
-; X64-NEXT: andq %rsi, %rax
-; X64-NEXT: xorq %rdx, %rax
+; X64-NEXT: movq %rdx, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rsi, %rax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_i64:
; X32: # %bb.0:
-; X32-NEXT: pushl %edi
-; X32-NEXT: pushl %esi
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: andb $1, %dl
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %esi, %eax
-; X32-NEXT: movzbl %dl, %edi
-; X32-NEXT: negl %edi
-; X32-NEXT: andl %edi, %eax
-; X32-NEXT: xorl %esi, %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl %edi, %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: popl %esi
-; X32-NEXT: popl %edi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edx
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_i64:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebp
+; X32-NOCMOV-NEXT: pushl %ebx
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: andb $1, %dl
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %esi, %eax
-; X32-NOCMOV-NEXT: movzbl %dl, %edi
-; X32-NOCMOV-NEXT: negl %edi
-; X32-NOCMOV-NEXT: andl %edi, %eax
-; X32-NOCMOV-NEXT: xorl %esi, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl %edi, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: testb $1, %bl
+; X32-NOCMOV-NEXT: sete %bh
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %bl
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
+; X32-NOCMOV-NEXT: popl %ebp
; X32-NOCMOV-NEXT: retl
%result = call i64 @llvm.ct.select.i64(i1 %cond, i64 %a, i64 %b)
ret i64 %result
@@ -145,56 +111,45 @@ define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) #0 {
define float @test_ctselect_f32(i1 %cond, float %a, float %b) #0 {
; X64-LABEL: test_ctselect_f32:
; X64: # %bb.0:
-; X64-NEXT: movd %xmm1, %eax
-; X64-NEXT: pxor %xmm1, %xmm0
-; X64-NEXT: movd %xmm0, %ecx
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl %ecx, %edi
-; X64-NEXT: xorl %eax, %edi
-; X64-NEXT: movd %edi, %xmm0
+; X64-NEXT: movd %xmm0, %eax
+; X64-NEXT: movd %xmm1, %ecx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %eax, %ecx
+; X64-NEXT: movd %ecx, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f32:
; X32: # %bb.0:
-; X32-NEXT: subl $12, %esp
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps (%esp)
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
+; X32-NEXT: pushl %edi
+; X32-NEXT: pushl %esi
+; X32-NEXT: pushl %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: sete %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: movl (%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl %eax, %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: addl $12, %esp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %esi, (%esp)
+; X32-NEXT: flds (%esp)
+; X32-NEXT: addl $4, %esp
+; X32-NEXT: popl %esi
+; X32-NEXT: popl %edi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f32:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: subl $12, %esp
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps (%esp)
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: pushl %eax
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: movl (%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl %eax, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: addl $12, %esp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, (%esp)
+; X32-NOCMOV-NEXT: flds (%esp)
+; X32-NOCMOV-NEXT: addl $4, %esp
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: retl
%result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
ret float %result
@@ -203,73 +158,53 @@ define float @test_ctselect_f32(i1 %cond, float %a, float %b) #0 {
define double @test_ctselect_f64(i1 %cond, double %a, double %b) #0 {
; X64-LABEL: test_ctselect_f64:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-NEXT: movq %xmm1, %rax
-; X64-NEXT: pxor %xmm1, %xmm0
-; X64-NEXT: movq %xmm0, %rcx
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negq %rdi
-; X64-NEXT: andq %rcx, %rdi
-; X64-NEXT: xorq %rax, %rdi
-; X64-NEXT: movq %rdi, %xmm0
+; X64-NEXT: movq %xmm0, %rax
+; X64-NEXT: movq %xmm1, %rcx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rax, %rcx
+; X64-NEXT: movq %rcx, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f64:
; X32: # %bb.0:
+; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $24, %esp
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: fldl {{[0-9]+}}(%esp)
-; X32-NEXT: fldl {{[0-9]+}}(%esp)
-; X32-NEXT: fstpl {{[0-9]+}}(%esp)
-; X32-NEXT: fstpl {{[0-9]+}}(%esp)
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: subl $8, %esp
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: sete %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: andl %eax, %esi
-; X32-NEXT: xorl %edx, %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: BUNDLE
; X32-NEXT: movl %esi, (%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl %eax, %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X32-NEXT: fldl (%esp)
-; X32-NEXT: addl $24, %esp
+; X32-NEXT: addl $8, %esp
; X32-NEXT: popl %esi
+; X32-NEXT: popl %edi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f64:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $24, %esp
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: fldl {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fldl {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: subl $8, %esp
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: andl %eax, %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: movl %esi, (%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl %eax, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fldl (%esp)
-; X32-NOCMOV-NEXT: addl $24, %esp
+; X32-NOCMOV-NEXT: addl $8, %esp
; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: retl
%result = call double @llvm.ct.select.f64(i1 %cond, double %a, double %b)
ret double %result
@@ -278,42 +213,31 @@ define double @test_ctselect_f64(i1 %cond, double %a, double %b) #0 {
define half @test_ctselect_f16(i1 %cond, half %a, half %b) #0 {
; X64-LABEL: test_ctselect_f16:
; X64: # %bb.0:
-; X64-NEXT: pextrw $0, %xmm1, %eax
-; X64-NEXT: pextrw $0, %xmm0, %ecx
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl %ecx, %edi
-; X64-NEXT: xorl %eax, %edi
-; X64-NEXT: pinsrw $0, %edi, %xmm0
+; X64-NEXT: pextrw $0, %xmm0, %eax
+; X64-NEXT: pextrw $0, %xmm1, %ecx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %ecx, %eax
+; X64-NEXT: pinsrw $0, %eax, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f16:
; X32: # %bb.0:
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorw %cx, %dx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: # kill: def $ax killed $ax killed $eax
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %ax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f16:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorw %cx, %dx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: # kill: def $ax killed $ax killed $eax
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%result = call half @llvm.ct.select.f16(i1 %cond, half %a, half %b)
ret half %result
@@ -324,12 +248,9 @@ define bfloat @test_ctselect_bf16(i1 %cond, bfloat %a, bfloat %b) #0 {
; X64: # %bb.0:
; X64-NEXT: pextrw $0, %xmm1, %eax
; X64-NEXT: pextrw $0, %xmm0, %ecx
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl %ecx, %edi
-; X64-NEXT: xorl %eax, %edi
-; X64-NEXT: pinsrw $0, %edi, %xmm0
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %ecx, %eax
+; X64-NEXT: pinsrw $0, %eax, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_bf16:
@@ -344,15 +265,10 @@ define bfloat @test_ctselect_bf16(i1 %cond, bfloat %a, bfloat %b) #0 {
; X32-NEXT: fstps (%esp)
; X32-NEXT: calll __truncsfbf2
; X32-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: andb $1, %cl
-; X32-NEXT: xorl %esi, %eax
-; X32-NEXT: movzbl %cl, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: andl %eax, %ecx
-; X32-NEXT: xorl %esi, %ecx
-; X32-NEXT: shll $16, %ecx
-; X32-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %eax, %esi
+; X32-NEXT: shll $16, %esi
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: addl $8, %esp
; X32-NEXT: popl %esi
@@ -360,8 +276,9 @@ define bfloat @test_ctselect_bf16(i1 %cond, bfloat %a, bfloat %b) #0 {
;
; X32-NOCMOV-LABEL: test_ctselect_bf16:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $8, %esp
+; X32-NOCMOV-NEXT: subl $20, %esp
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
@@ -370,18 +287,15 @@ define bfloat @test_ctselect_bf16(i1 %cond, bfloat %a, bfloat %b) #0 {
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: andb $1, %cl
-; X32-NOCMOV-NEXT: xorl %esi, %eax
-; X32-NOCMOV-NEXT: movzbl %cl, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl %eax, %ecx
-; X32-NOCMOV-NEXT: xorl %esi, %ecx
-; X32-NOCMOV-NEXT: shll $16, %ecx
-; X32-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: shll $16, %edx
+; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: addl $8, %esp
+; X32-NOCMOV-NEXT: addl $20, %esp
; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: retl
%result = call bfloat @llvm.ct.select.bf16(i1 %cond, bfloat %a, bfloat %b)
ret bfloat %result
@@ -390,66 +304,40 @@ define bfloat @test_ctselect_bf16(i1 %cond, bfloat %a, bfloat %b) #0 {
define fp128 @test_ctselect_f128(i1 %cond, fp128 %a, fp128 %b) #0 {
; X64-LABEL: test_ctselect_f128:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
; X64-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
; X64-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negq %rdi
; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rax
; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rcx
-; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rdx
-; X64-NEXT: xorq %rax, %rdx
-; X64-NEXT: andq %rdi, %rdx
-; X64-NEXT: xorq %rax, %rdx
-; X64-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rax
-; X64-NEXT: xorq %rcx, %rax
-; X64-NEXT: andq %rdi, %rax
-; X64-NEXT: xorq %rcx, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq -{{[0-9]+}}(%rsp), %rax
; X64-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NEXT: cmovneq -{{[0-9]+}}(%rsp), %rcx
+; X64-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NEXT: movaps -{{[0-9]+}}(%rsp), %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f128:
; X32: # %bb.0:
-; X32-NEXT: pushl %ebp
-; X32-NEXT: pushl %ebx
; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $12, %esp
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: andb $1, %bl
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %edi, %ecx
-; X32-NEXT: movzbl %bl, %ebp
-; X32-NEXT: negl %ebp
-; X32-NEXT: andl %ebp, %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: andl %ebp, %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: andl %ebp, %ebx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: pushl %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %edx, %eax
-; X32-NEXT: andl %ebp, %eax
-; X32-NEXT: xorl %edi, %ecx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl %edx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl %eax, 12(%edx)
-; X32-NEXT: movl %ebx, 8(%edx)
-; X32-NEXT: movl %esi, 4(%edx)
-; X32-NEXT: movl %ecx, (%edx)
-; X32-NEXT: movl %edx, %eax
-; X32-NEXT: addl $12, %esp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edx
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl %ecx, 12(%eax)
+; X32-NEXT: movl %edx, 8(%eax)
+; X32-NEXT: movl %edi, 4(%eax)
+; X32-NEXT: movl %esi, (%eax)
+; X32-NEXT: addl $4, %esp
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
-; X32-NEXT: popl %ebx
-; X32-NEXT: popl %ebp
; X32-NEXT: retl $4
;
; X32-NOCMOV-LABEL: test_ctselect_f128:
@@ -459,34 +347,36 @@ define fp128 @test_ctselect_f128(i1 %cond, fp128 %a, fp128 %b) #0 {
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: subl $12, %esp
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: andb $1, %bl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: testb $1, %bl
+; X32-NOCMOV-NEXT: sete %bh
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %bl
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %bl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %edi, %ecx
-; X32-NOCMOV-NEXT: movzbl %bl, %ebp
-; X32-NOCMOV-NEXT: negl %ebp
-; X32-NOCMOV-NEXT: andl %ebp, %ecx
+; X32-NOCMOV-NEXT: sete %bh
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: andl %ebp, %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: andl %ebp, %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %bl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %edx, %eax
-; X32-NOCMOV-NEXT: andl %ebp, %eax
-; X32-NOCMOV-NEXT: xorl %edi, %ecx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl %edx, %eax
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl %eax, 12(%edx)
-; X32-NOCMOV-NEXT: movl %ebx, 8(%edx)
-; X32-NOCMOV-NEXT: movl %esi, 4(%edx)
-; X32-NOCMOV-NEXT: movl %ecx, (%edx)
-; X32-NOCMOV-NEXT: movl %edx, %eax
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, 12(%eax)
+; X32-NOCMOV-NEXT: movl %edi, 8(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, 4(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, (%eax)
; X32-NOCMOV-NEXT: addl $12, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
@@ -504,133 +394,78 @@ define x86_fp80 @test_ctselect_f80(i1 %cond, x86_fp80 %a, x86_fp80 %b) #0 {
; X64-NEXT: fldt {{[0-9]+}}(%rsp)
; X64-NEXT: fstpt -{{[0-9]+}}(%rsp)
; X64-NEXT: fstpt -{{[0-9]+}}(%rsp)
-; X64-NEXT: movl -{{[0-9]+}}(%rsp), %ecx
; X64-NEXT: movl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %edx
-; X64-NEXT: xorw %cx, %dx
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl %edi, %edx
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: movw %dx, -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %ecx
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %edx
-; X64-NEXT: xorw %cx, %dx
-; X64-NEXT: andl %edi, %edx
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: movw %dx, -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %ecx
-; X64-NEXT: xorw %ax, %cx
-; X64-NEXT: andl %edi, %ecx
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: movw %cx, -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %ecx
-; X64-NEXT: xorw %ax, %cx
-; X64-NEXT: andl %edi, %ecx
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: movw %cx, -{{[0-9]+}}(%rsp)
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT: movl %eax, -{{[0-9]+}}(%rsp)
; X64-NEXT: movl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %ecx
-; X64-NEXT: xorw %ax, %cx
-; X64-NEXT: andl %edi, %ecx
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: movw %cx, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movl -{{[0-9]+}}(%rsp), %ecx
+; X64-NEXT: cmovnel -{{[0-9]+}}(%rsp), %ecx
+; X64-NEXT: movl %ecx, -{{[0-9]+}}(%rsp)
+; X64-NEXT: cmovnel -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT: movl %eax, -{{[0-9]+}}(%rsp)
; X64-NEXT: fldt -{{[0-9]+}}(%rsp)
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f80:
; X32: # %bb.0:
-; X32-NEXT: pushl %esi
; X32-NEXT: subl $36, %esp
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
; X32-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NEXT: fstpt {{[0-9]+}}(%esp)
; X32-NEXT: fstpt {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorw %dx, %si
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %eax, %esi
-; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: movw %si, (%esp)
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorw %dx, %si
-; X32-NEXT: andl %eax, %esi
-; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: movw %si, {{[0-9]+}}(%esp)
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorw %cx, %dx
-; X32-NEXT: andl %eax, %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movw %dx, {{[0-9]+}}(%esp)
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorw %cx, %dx
-; X32-NEXT: andl %eax, %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movw %dx, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorw %cx, %dx
-; X32-NEXT: andl %eax, %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movw %dx, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl %eax, (%esp)
; X32-NEXT: fldt (%esp)
; X32-NEXT: addl $36, %esp
-; X32-NEXT: popl %esi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f80:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebp
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $36, %esp
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: subl $40, %esp
; X32-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorw %dx, %si
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %eax, %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: movw %si, (%esp)
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorw %dx, %si
-; X32-NOCMOV-NEXT: andl %eax, %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: movw %si, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorw %cx, %dx
-; X32-NOCMOV-NEXT: andl %eax, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movw %dx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorw %cx, %dx
-; X32-NOCMOV-NEXT: andl %eax, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movw %dx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorw %cx, %dx
-; X32-NOCMOV-NEXT: andl %eax, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movw %dx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fldt (%esp)
-; X32-NOCMOV-NEXT: addl $36, %esp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: sete %ch
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: sete %ch
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl (%esp), %ebx # 4-byte Reload
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: addl $40, %esp
; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
+; X32-NOCMOV-NEXT: popl %ebp
; X32-NOCMOV-NEXT: retl
%result = call x86_fp80 @llvm.ct.select.f80(i1 %cond, x86_fp80 %a, x86_fp80 %b)
ret x86_fp80 %result
@@ -639,38 +474,29 @@ define x86_fp80 @test_ctselect_f80(i1 %cond, x86_fp80 %a, x86_fp80 %b) #0 {
define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) #0 {
; X64-LABEL: test_ctselect_ptr:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: xorq %rdx, %rsi
-; X64-NEXT: andl $1, %eax
-; X64-NEXT: negq %rax
-; X64-NEXT: andq %rsi, %rax
-; X64-NEXT: xorq %rdx, %rax
+; X64-NEXT: movq %rdx, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovneq %rsi, %rax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_ptr:
; X32: # %bb.0:
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_ptr:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%result = call ptr @llvm.ct.select.p0(i1 %cond, ptr %a, ptr %b)
ret ptr %result
@@ -680,25 +506,32 @@ define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) #0 {
define i32 @test_ctselect_const_true(i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_const_true:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: xorl %esi, %eax
-; X64-NEXT: xorl %esi, %eax
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: movb $1, %cl
+; X64-NEXT: testb %cl, %cl
+; X64-NEXT: cmovnel %edi, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_const_true:
; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movb $1, %cl
+; X32-NEXT: testb %cl, %cl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_const_true:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movb $1, %al
+; X32-NOCMOV-NEXT: testb %al, %al
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%result = call i32 @llvm.ct.select.i32(i1 true, i32 %a, i32 %b)
ret i32 %result
@@ -708,18 +541,31 @@ define i32 @test_ctselect_const_false(i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_const_false:
; X64: # %bb.0:
; X64-NEXT: movl %esi, %eax
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: testb %cl, %cl
+; X64-NEXT: cmovnel %edi, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_const_false:
; X32: # %bb.0:
-; X32-NEXT: xorl %eax, %eax
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %ecx, %ecx
+; X32-NEXT: testb %cl, %cl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_const_false:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: xorl %eax, %eax
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: testb %al, %al
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%result = call i32 @llvm.ct.select.i32(i1 false, i32 %a, i32 %b)
ret i32 %result
@@ -729,41 +575,37 @@ define i32 @test_ctselect_const_false(i32 %a, i32 %b) #0 {
define i32 @test_ctselect_icmp_eq(i32 %x, i32 %y, i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_icmp_eq:
; X64: # %bb.0:
-; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: movl %ecx, %eax
; X64-NEXT: cmpl %esi, %edi
-; X64-NEXT: sete %al
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: negl %eax
-; X64-NEXT: andl %edx, %eax
-; X64-NEXT: xorl %ecx, %eax
+; X64-NEXT: sete %cl
+; X64-NEXT: testb %cl, %cl
+; X64-NEXT: cmovnel %edx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_icmp_eq:
; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %eax, %eax
-; X32-NEXT: cmpl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: sete %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: sete %cl
+; X32-NEXT: testb %cl, %cl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_icmp_eq:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %eax, %eax
-; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: sete %al
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: testb %al, %al
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%cond = icmp eq i32 %x, %y
%result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
@@ -773,38 +615,37 @@ define i32 @test_ctselect_icmp_eq(i32 %x, i32 %y, i32 %a, i32 %b) #0 {
define i32 @test_ctselect_icmp_ult(i32 %x, i32 %y, i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_icmp_ult:
; X64: # %bb.0:
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: movl %ecx, %eax
; X64-NEXT: cmpl %esi, %edi
-; X64-NEXT: sbbl %eax, %eax
-; X64-NEXT: andl %edx, %eax
-; X64-NEXT: xorl %ecx, %eax
+; X64-NEXT: setb %cl
+; X64-NEXT: testb %cl, %cl
+; X64-NEXT: cmovnel %edx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_icmp_ult:
; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %edx, %edx
-; X32-NEXT: cmpl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: sbbl %edx, %edx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: setb %cl
+; X32-NEXT: testb %cl, %cl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_icmp_ult:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %edx, %edx
; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: sbbl %edx, %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: setb %al
+; X32-NOCMOV-NEXT: testb %al, %al
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%cond = icmp ult i32 %x, %y
%result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
@@ -814,49 +655,44 @@ define i32 @test_ctselect_icmp_ult(i32 %x, i32 %y, i32 %a, i32 %b) #0 {
define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) #0 {
; X64-LABEL: test_ctselect_fcmp_oeq:
; X64: # %bb.0:
-; X64-NEXT: movd %xmm3, %eax
-; X64-NEXT: cmpeqss %xmm1, %xmm0
-; X64-NEXT: pxor %xmm3, %xmm2
-; X64-NEXT: pand %xmm0, %xmm2
-; X64-NEXT: movd %xmm2, %ecx
-; X64-NEXT: xorl %eax, %ecx
+; X64-NEXT: movd %xmm2, %eax
+; X64-NEXT: movd %xmm3, %ecx
+; X64-NEXT: ucomiss %xmm1, %xmm0
+; X64-NEXT: setnp %dl
+; X64-NEXT: sete %sil
+; X64-NEXT: testb %dl, %sil
+; X64-NEXT: cmovnel %eax, %ecx
; X64-NEXT: movd %ecx, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_fcmp_oeq:
; X32: # %bb.0:
-; X32-NEXT: subl $12, %esp
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps (%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: pushl %edi
+; X32-NEXT: pushl %esi
+; X32-NEXT: pushl %eax
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fucompi %st(1), %st
; X32-NEXT: fstp %st(0)
-; X32-NEXT: setnp %cl
-; X32-NEXT: sete %dl
-; X32-NEXT: andb %cl, %dl
-; X32-NEXT: movzbl %dl, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: movl (%esp), %edx
-; X32-NEXT: xorl %eax, %edx
-; X32-NEXT: andl %ecx, %edx
-; X32-NEXT: xorl %eax, %edx
-; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: addl $12, %esp
-; X32-NEXT: retl
-;
+; X32-NEXT: setnp %al
+; X32-NEXT: sete %cl
+; X32-NEXT: testb %al, %cl
+; X32-NEXT: sete %al
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %esi, (%esp)
+; X32-NEXT: flds (%esp)
+; X32-NEXT: addl $4, %esp
+; X32-NEXT: popl %esi
+; X32-NEXT: popl %edi
+; X32-NEXT: retl
+;
; X32-NOCMOV-LABEL: test_ctselect_fcmp_oeq:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: subl $12, %esp
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps (%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: pushl %eax
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fucompp
@@ -864,17 +700,17 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) #0
; X32-NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
; X32-NOCMOV-NEXT: sahf
; X32-NOCMOV-NEXT: setnp %al
-; X32-NOCMOV-NEXT: sete %dl
-; X32-NOCMOV-NEXT: andb %al, %dl
-; X32-NOCMOV-NEXT: movzbl %dl, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: movl (%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl %eax, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: addl $12, %esp
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: testb %al, %cl
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, (%esp)
+; X32-NOCMOV-NEXT: flds (%esp)
+; X32-NOCMOV-NEXT: addl $4, %esp
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: retl
%cond = fcmp oeq float %x, %y
%result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
@@ -885,43 +721,33 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) #0
define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) #0 {
; X64-LABEL: test_ctselect_load:
; X64: # %bb.0:
-; X64-NEXT: movl (%rdx), %ecx
-; X64-NEXT: movl (%rsi), %eax
-; X64-NEXT: xorl %ecx, %eax
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl %edi, %eax
-; X64-NEXT: xorl %ecx, %eax
+; X64-NEXT: movl (%rdx), %eax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel (%rsi), %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_load:
; X32: # %bb.0:
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl (%edx), %edx
-; X32-NEXT: movl (%ecx), %ecx
-; X32-NEXT: xorl %edx, %ecx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %ecx, %eax
-; X32-NEXT: xorl %edx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl (%eax), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel (%ecx), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_load:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl (%edx), %edx
; X32-NOCMOV-NEXT: movl (%ecx), %ecx
-; X32-NOCMOV-NEXT: xorl %edx, %ecx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %ecx, %eax
-; X32-NOCMOV-NEXT: xorl %edx, %eax
+; X32-NOCMOV-NEXT: movl (%eax), %edx
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%a = load i32, ptr %p1
%b = load i32, ptr %p2
@@ -933,67 +759,40 @@ define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) #0 {
define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) #0 {
; X64-LABEL: test_ctselect_nested:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: andl $1, %esi
-; X64-NEXT: negl %esi
-; X64-NEXT: andl %edx, %esi
-; X64-NEXT: xorl %r8d, %ecx
-; X64-NEXT: xorl %esi, %ecx
-; X64-NEXT: andl $1, %eax
-; X64-NEXT: negl %eax
-; X64-NEXT: andl %ecx, %eax
-; X64-NEXT: xorl %r8d, %eax
+; X64-NEXT: movl %r8d, %eax
+; X64-NEXT: testb $1, %sil
+; X64-NEXT: cmovnel %edx, %ecx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: cmovnel %ecx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_nested:
; X32: # %bb.0:
-; X32-NEXT: pushl %edi
-; X32-NEXT: pushl %esi
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movb {{[0-9]+}}(%esp), %ah
-; X32-NEXT: andb $1, %ah
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: movzbl %ah, %edi
-; X32-NEXT: negl %edi
-; X32-NEXT: andl %esi, %edi
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: xorl %edi, %edx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: popl %esi
-; X32-NEXT: popl %edi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %ecx, %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_nested:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebx
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %ah
-; X32-NOCMOV-NEXT: andb $1, %ah
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: movzbl %ah, %edi
-; X32-NOCMOV-NEXT: negl %edi
-; X32-NOCMOV-NEXT: andl %esi, %edi
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: xorl %edi, %edx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %dl
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%inner = call i32 @llvm.ct.select.i32(i1 %cond2, i32 %a, i32 %b)
%result = call i32 @llvm.ct.select.i32(i1 %cond1, i32 %inner, i32 %c)
@@ -1006,44 +805,49 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) #
define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) #0 {
; X64-LABEL: test_ctselect_nested_and_i1_to_i32:
; X64: # %bb.0:
+; X64-NEXT: movl %ecx, %eax
; X64-NEXT: andl %esi, %edi
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: movzbl %dil, %eax
-; X64-NEXT: negl %eax
-; X64-NEXT: andl %edx, %eax
-; X64-NEXT: xorl %ecx, %eax
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movl $1, %esi
+; X64-NEXT: cmovnel %esi, %ecx
+; X64-NEXT: testb $1, %cl
+; X64-NEXT: cmovnel %edx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_nested_and_i1_to_i32:
; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NEXT: andb $1, %al
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: andb {{[0-9]+}}(%esp), %cl
+; X32-NEXT: xorl %edx, %edx
+; X32-NEXT: testb $1, %cl
+; X32-NEXT: movl $1, %ecx
+; X32-NEXT: cmovnel %ecx, %edx
+; X32-NEXT: testb $1, %dl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_nested_and_i1_to_i32:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: xorl %ebx, %ebx
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movb $1, %al
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%inner = call i1 @llvm.ct.select.i1(i1 %c1, i1 true, i1 false)
%cond = call i1 @llvm.ct.select.i1(i1 %c0, i1 %inner, i1 false)
@@ -1057,44 +861,49 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) #
define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) #0 {
; X64-LABEL: test_ctselect_nested_or_i1_to_i32:
; X64: # %bb.0:
+; X64-NEXT: movl %ecx, %eax
; X64-NEXT: orl %esi, %edi
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: movzbl %dil, %eax
-; X64-NEXT: negl %eax
-; X64-NEXT: andl %edx, %eax
-; X64-NEXT: xorl %ecx, %eax
+; X64-NEXT: xorl %ecx, %ecx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movl $1, %esi
+; X64-NEXT: cmovnel %esi, %ecx
+; X64-NEXT: testb $1, %cl
+; X64-NEXT: cmovnel %edx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_nested_or_i1_to_i32:
; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: orb {{[0-9]+}}(%esp), %al
-; X32-NEXT: andb $1, %al
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: orb {{[0-9]+}}(%esp), %cl
+; X32-NEXT: xorl %edx, %edx
+; X32-NEXT: testb $1, %cl
+; X32-NEXT: movl $1, %ecx
+; X32-NEXT: cmovnel %ecx, %edx
+; X32-NEXT: testb $1, %dl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_nested_or_i1_to_i32:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: orb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: xorl %ebx, %ebx
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movb $1, %al
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%inner = call i1 @llvm.ct.select.i1(i1 %c1, i1 true, i1 false)
%cond = call i1 @llvm.ct.select.i1(i1 %c0, i1 true, i1 %inner)
@@ -1110,47 +919,52 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) #0
define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i32 %y) #0 {
; X64-LABEL: test_ctselect_double_nested_and_i1:
; X64: # %bb.0:
+; X64-NEXT: movl %r8d, %eax
; X64-NEXT: andl %esi, %edi
; X64-NEXT: andl %edx, %edi
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: xorl %r8d, %ecx
-; X64-NEXT: movzbl %dil, %eax
-; X64-NEXT: negl %eax
-; X64-NEXT: andl %ecx, %eax
-; X64-NEXT: xorl %r8d, %eax
+; X64-NEXT: xorl %edx, %edx
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movl $1, %esi
+; X64-NEXT: cmovnel %esi, %edx
+; X64-NEXT: testb $1, %dl
+; X64-NEXT: cmovnel %ecx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_double_nested_and_i1:
; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NEXT: andb $1, %al
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: andb {{[0-9]+}}(%esp), %cl
+; X32-NEXT: andb {{[0-9]+}}(%esp), %cl
+; X32-NEXT: xorl %edx, %edx
+; X32-NEXT: testb $1, %cl
+; X32-NEXT: movl $1, %ecx
+; X32-NEXT: cmovnel %ecx, %edx
+; X32-NEXT: testb $1, %dl
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_double_nested_and_i1:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: xorl %ebx, %ebx
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movb $1, %al
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
; X32-NOCMOV-NEXT: retl
%inner2 = call i1 @llvm.ct.select.i1(i1 %c2, i1 true, i1 false)
%inner1 = call i1 @llvm.ct.select.i1(i1 %c1, i1 %inner2, i1 false)
@@ -1167,53 +981,31 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) #0 {
; X64-LABEL: test_ctselect_v4i32:
; X64: # %bb.0:
-; X64-NEXT: pxor %xmm1, %xmm0
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: movd %edi, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; X64-NEXT: pand %xmm2, %xmm0
-; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: BUNDLE
+; X64-NEXT: movaps %xmm2, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_v4i32:
; X32: # %bb.0:
-; X32-NEXT: pushl %ebp
-; X32-NEXT: pushl %ebx
; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: andb $1, %bl
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %edi, %ecx
-; X32-NEXT: movzbl %bl, %ebp
-; X32-NEXT: negl %ebp
-; X32-NEXT: andl %ebp, %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: andl %ebp, %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: andl %ebp, %ebx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %edx, %eax
-; X32-NEXT: andl %ebp, %eax
-; X32-NEXT: xorl %edi, %ecx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl %edx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl %eax, 12(%edx)
-; X32-NEXT: movl %ebx, 8(%edx)
-; X32-NEXT: movl %esi, 4(%edx)
-; X32-NEXT: movl %ecx, (%edx)
-; X32-NEXT: movl %edx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edx
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl %ecx, 12(%eax)
+; X32-NEXT: movl %edx, 8(%eax)
+; X32-NEXT: movl %esi, 4(%eax)
+; X32-NEXT: movl %edi, (%eax)
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
-; X32-NEXT: popl %ebx
-; X32-NEXT: popl %ebp
; X32-NEXT: retl $4
;
; X32-NOCMOV-LABEL: test_ctselect_v4i32:
@@ -1222,34 +1014,38 @@ define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) #0 {
; X32-NOCMOV-NEXT: pushl %ebx
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: subl $8, %esp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: andb $1, %bl
+; X32-NOCMOV-NEXT: testb $1, %bl
+; X32-NOCMOV-NEXT: sete %bh
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %bl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: sete %bh
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %bl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %edi, %ecx
-; X32-NOCMOV-NEXT: movzbl %bl, %ebp
-; X32-NOCMOV-NEXT: negl %ebp
-; X32-NOCMOV-NEXT: andl %ebp, %ecx
+; X32-NOCMOV-NEXT: sete %bh
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: andl %ebp, %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: andl %ebp, %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %bl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %edx, %eax
-; X32-NOCMOV-NEXT: andl %ebp, %eax
-; X32-NOCMOV-NEXT: xorl %edi, %ecx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl %edx, %eax
+; X32-NOCMOV-NEXT: sete %bl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl %eax, 12(%edx)
-; X32-NOCMOV-NEXT: movl %ebx, 8(%edx)
-; X32-NOCMOV-NEXT: movl %esi, 4(%edx)
-; X32-NOCMOV-NEXT: movl %ecx, (%edx)
-; X32-NOCMOV-NEXT: movl %edx, %eax
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, 12(%eax)
+; X32-NOCMOV-NEXT: movl %edi, 8(%eax)
+; X32-NOCMOV-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, 4(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, (%eax)
+; X32-NOCMOV-NEXT: addl $8, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: popl %ebx
@@ -1261,147 +1057,101 @@ define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) #0 {
define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b) #0 {
; X64-LABEL: test_ctselect_v4f32:
; X64: # %bb.0:
-; X64-NEXT: pxor %xmm1, %xmm0
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: movd %edi, %xmm2
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; X64-NEXT: pand %xmm2, %xmm0
-; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: BUNDLE
+; X64-NEXT: movaps %xmm2, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_v4f32:
; X32: # %bb.0:
-; X32-NEXT: pushl %ebp
; X32-NEXT: pushl %ebx
; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $48, %esp
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps (%esp)
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: andb $1, %dl
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: subl $16, %esp
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: movzbl %dl, %edx
-; X32-NEXT: negl %edx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorl %ebx, %ebp
-; X32-NEXT: andl %edx, %ebp
-; X32-NEXT: xorl %ebx, %ebp
-; X32-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl %edi, %ebx
-; X32-NEXT: andl %edx, %ebx
-; X32-NEXT: xorl %edi, %ebx
-; X32-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: xorl %esi, %edi
-; X32-NEXT: andl %edx, %edi
-; X32-NEXT: xorl %esi, %edi
+; X32-NEXT: BUNDLE
; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X32-NEXT: movl (%esp), %esi
-; X32-NEXT: xorl %ecx, %esi
-; X32-NEXT: andl %edx, %esi
-; X32-NEXT: xorl %ecx, %esi
-; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, (%esp)
+; X32-NEXT: flds (%esp)
; X32-NEXT: fstps 12(%eax)
; X32-NEXT: fstps 8(%eax)
; X32-NEXT: fstps 4(%eax)
; X32-NEXT: fstps (%eax)
-; X32-NEXT: addl $48, %esp
+; X32-NEXT: addl $16, %esp
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
; X32-NEXT: popl %ebx
-; X32-NEXT: popl %ebp
; X32-NEXT: retl $4
;
; X32-NOCMOV-LABEL: test_ctselect_v4f32:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: pushl %ebp
; X32-NOCMOV-NEXT: pushl %ebx
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $48, %esp
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps (%esp)
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: andb $1, %dl
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: subl $16, %esp
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: movzbl %dl, %edx
-; X32-NOCMOV-NEXT: negl %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorl %ebx, %ebp
-; X32-NOCMOV-NEXT: andl %edx, %ebp
-; X32-NOCMOV-NEXT: xorl %ebx, %ebp
-; X32-NOCMOV-NEXT: movl %ebp, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl %edi, %ebx
-; X32-NOCMOV-NEXT: andl %edx, %ebx
-; X32-NOCMOV-NEXT: xorl %edi, %ebx
-; X32-NOCMOV-NEXT: movl %ebx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: xorl %esi, %edi
-; X32-NOCMOV-NEXT: andl %edx, %edi
-; X32-NOCMOV-NEXT: xorl %esi, %edi
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl (%esp), %esi
-; X32-NOCMOV-NEXT: xorl %ecx, %esi
-; X32-NOCMOV-NEXT: andl %edx, %esi
-; X32-NOCMOV-NEXT: xorl %ecx, %esi
-; X32-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, (%esp)
+; X32-NOCMOV-NEXT: flds (%esp)
; X32-NOCMOV-NEXT: fstps 12(%eax)
; X32-NOCMOV-NEXT: fstps 8(%eax)
; X32-NOCMOV-NEXT: fstps 4(%eax)
; X32-NOCMOV-NEXT: fstps (%eax)
-; X32-NOCMOV-NEXT: addl $48, %esp
+; X32-NOCMOV-NEXT: addl $16, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: popl %ebx
-; X32-NOCMOV-NEXT: popl %ebp
; X32-NOCMOV-NEXT: retl $4
%result = call <4 x float> @llvm.ct.select.v4f32(i1 %cond, <4 x float> %a, <4 x float> %b)
ret <4 x float> %result
@@ -1410,16 +1160,11 @@ define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b
define <8 x i32> @test_ctselect_v8i32_avx(i1 %cond, <8 x i32> %a, <8 x i32> %b) #0 {
; X64-LABEL: test_ctselect_v8i32_avx:
; X64: # %bb.0:
-; X64-NEXT: pxor %xmm2, %xmm0
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: movd %edi, %xmm4
-; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; X64-NEXT: pand %xmm4, %xmm0
-; X64-NEXT: pxor %xmm2, %xmm0
-; X64-NEXT: pxor %xmm3, %xmm1
-; X64-NEXT: pand %xmm4, %xmm1
-; X64-NEXT: pxor %xmm3, %xmm1
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: BUNDLE
+; X64-NEXT: BUNDLE
+; X64-NEXT: movaps %xmm4, %xmm0
+; X64-NEXT: movaps %xmm2, %xmm1
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_v8i32_avx:
@@ -1429,58 +1174,36 @@ define <8 x i32> @test_ctselect_v8i32_avx(i1 %cond, <8 x i32> %a, <8 x i32> %b)
; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
; X32-NEXT: subl $8, %esp
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movzbl %al, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: andl %ecx, %edx
-; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andl %ecx, %eax
-; X32-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: andl %ecx, %ebp
; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: andl %ecx, %ebx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: andl %ecx, %edi
; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: andl %ecx, %esi
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: andl %ecx, %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andl %ecx, %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %ecx, (%esp) # 4-byte Folded Spill
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edx
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %esi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %edi
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl %ecx, (%esp) # 4-byte Spill
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl %eax, 28(%ecx)
-; X32-NEXT: movl %edx, 24(%ecx)
-; X32-NEXT: movl %esi, 20(%ecx)
-; X32-NEXT: movl %edi, 16(%ecx)
-; X32-NEXT: movl %ebx, 12(%ecx)
-; X32-NEXT: movl %ebp, 8(%ecx)
-; X32-NEXT: movl (%esp), %eax # 4-byte Reload
-; X32-NEXT: movl %eax, 4(%ecx)
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT: movl %eax, (%ecx)
-; X32-NEXT: movl %ecx, %eax
+; X32-NEXT: cmovnel {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl %ecx, 28(%eax)
+; X32-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X32-NEXT: movl %ecx, 24(%eax)
+; X32-NEXT: movl %ebp, 20(%eax)
+; X32-NEXT: movl %ebx, 16(%eax)
+; X32-NEXT: movl %edi, 12(%eax)
+; X32-NEXT: movl %esi, 8(%eax)
+; X32-NEXT: movl %edx, 4(%eax)
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT: movl %ecx, (%eax)
; X32-NEXT: addl $8, %esp
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
@@ -1494,60 +1217,68 @@ define <8 x i32> @test_ctselect_v8i32_avx(i1 %cond, <8 x i32> %a, <8 x i32> %b)
; X32-NOCMOV-NEXT: pushl %ebx
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $8, %esp
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: subl $20, %esp
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movzbl %al, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl %ecx, %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andl %ecx, %eax
-; X32-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: andl %ecx, %ebp
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: andl %ecx, %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: andl %ecx, %edi
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, (%esp) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: sete %dl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: andl %ecx, %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: andl %ecx, %edx
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andl %ecx, %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %ecx, (%esp) # 4-byte Folded Spill
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl %eax, 28(%ecx)
-; X32-NOCMOV-NEXT: movl %edx, 24(%ecx)
-; X32-NOCMOV-NEXT: movl %esi, 20(%ecx)
-; X32-NOCMOV-NEXT: movl %edi, 16(%ecx)
-; X32-NOCMOV-NEXT: movl %ebx, 12(%ecx)
-; X32-NOCMOV-NEXT: movl %ebp, 8(%ecx)
-; X32-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: movl %eax, 4(%ecx)
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: movl %eax, (%ecx)
-; X32-NOCMOV-NEXT: movl %ecx, %eax
-; X32-NOCMOV-NEXT: addl $8, %esp
+; X32-NOCMOV-NEXT: movl %esi, 28(%eax)
+; X32-NOCMOV-NEXT: movl %edi, 24(%eax)
+; X32-NOCMOV-NEXT: movl %ebp, 20(%eax)
+; X32-NOCMOV-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, 16(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, 12(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, 8(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, 4(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %ecx, (%eax)
+; X32-NOCMOV-NEXT: addl $20, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: popl %ebx
@@ -1560,122 +1291,77 @@ define <8 x i32> @test_ctselect_v8i32_avx(i1 %cond, <8 x i32> %a, <8 x i32> %b)
define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b) #0 {
; X64-LABEL: test_ctselect_v8f32:
; X64: # %bb.0:
-; X64-NEXT: pxor %xmm2, %xmm0
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: movd %edi, %xmm4
-; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; X64-NEXT: pand %xmm4, %xmm0
-; X64-NEXT: pxor %xmm2, %xmm0
-; X64-NEXT: pxor %xmm3, %xmm1
-; X64-NEXT: pand %xmm4, %xmm1
-; X64-NEXT: pxor %xmm3, %xmm1
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: BUNDLE
+; X64-NEXT: BUNDLE
+; X64-NEXT: movaps %xmm4, %xmm0
+; X64-NEXT: movaps %xmm2, %xmm1
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_v8f32:
; X32: # %bb.0:
-; X32-NEXT: pushl %ebp
; X32-NEXT: pushl %ebx
; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $104, %esp
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: andb $1, %dl
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: subl $36, %esp
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: movzbl %dl, %edx
-; X32-NEXT: negl %edx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %ebx, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ebx, %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %ebp, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ebp, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %ebx, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ebx, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %edi, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %edi, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %esi, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %esi, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps (%esp) # 4-byte Folded Spill
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: testb $1, %al
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: sete %cl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps 28(%eax)
; X32-NEXT: fstps 24(%eax)
@@ -1686,117 +1372,76 @@ define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b
; X32-NEXT: fstps 4(%eax)
; X32-NEXT: flds (%esp) # 4-byte Folded Reload
; X32-NEXT: fstps (%eax)
-; X32-NEXT: addl $104, %esp
+; X32-NEXT: addl $36, %esp
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
; X32-NEXT: popl %ebx
-; X32-NEXT: popl %ebp
; X32-NEXT: retl $4
;
; X32-NOCMOV-LABEL: test_ctselect_v8f32:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: pushl %ebp
; X32-NOCMOV-NEXT: pushl %ebx
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $104, %esp
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: andb $1, %dl
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: subl $36, %esp
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: movzbl %dl, %edx
-; X32-NOCMOV-NEXT: negl %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %ebx, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ebx, %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %ebp, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ebp, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %ebx, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ebx, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %edi, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %edi, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %esi, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %esi, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp) # 4-byte Folded Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps 28(%eax)
; X32-NOCMOV-NEXT: fstps 24(%eax)
@@ -1807,11 +1452,10 @@ define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b
; X32-NOCMOV-NEXT: fstps 4(%eax)
; X32-NOCMOV-NEXT: flds (%esp) # 4-byte Folded Reload
; X32-NOCMOV-NEXT: fstps (%eax)
-; X32-NOCMOV-NEXT: addl $104, %esp
+; X32-NOCMOV-NEXT: addl $36, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: popl %ebx
-; X32-NOCMOV-NEXT: popl %ebp
; X32-NOCMOV-NEXT: retl $4
%result = call <8 x float> @llvm.ct.select.v8f32(i1 %cond, <8 x float> %a, <8 x float> %b)
ret <8 x float> %result
@@ -1820,14 +1464,9 @@ define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b
define <8 x half> @test_ctselect_v8f16(i1 %cond, <8 x half> %a, <8 x half> %b) #0 {
; X64-LABEL: test_ctselect_v8f16:
; X64: # %bb.0:
-; X64-NEXT: pxor %xmm1, %xmm0
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: movd %edi, %xmm2
-; X64-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
-; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
-; X64-NEXT: pand %xmm2, %xmm0
-; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: BUNDLE
+; X64-NEXT: movaps %xmm2, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_v8f16:
@@ -1836,69 +1475,38 @@ define <8 x half> @test_ctselect_v8f16(i1 %cond, <8 x half> %a, <8 x half> %b) #
; X32-NEXT: pushl %ebx
; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $12, %esp
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorw %di, %cx
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl %eax, %ecx
-; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorw %si, %cx
-; X32-NEXT: andl %eax, %ecx
-; X32-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X32-NEXT: pushl %eax
; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorw %dx, %cx
-; X32-NEXT: andl %eax, %ecx
-; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorw %cx, %bx
-; X32-NEXT: andl %eax, %ebx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorw %cx, %bp
-; X32-NEXT: andl %eax, %ebp
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edi
; X32-NEXT: movzwl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorw %cx, %si
-; X32-NEXT: andl %eax, %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorw %cx, %dx
-; X32-NEXT: andl %eax, %edx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %ax
+; X32-NEXT: movw %ax, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %dx
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %si
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %di
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %bx
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %bp
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %cx
+; X32-NEXT: movw %cx, (%esp) # 2-byte Spill
; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: xorw %di, %cx
-; X32-NEXT: andl %eax, %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %eax, (%esp) # 4-byte Folded Spill
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: cmovnew {{[0-9]+}}(%esp), %cx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NEXT: movw %cx, 14(%eax)
-; X32-NEXT: movw %dx, 12(%eax)
-; X32-NEXT: movw %si, 10(%eax)
-; X32-NEXT: movw %bp, 8(%eax)
-; X32-NEXT: movw %bx, 6(%eax)
-; X32-NEXT: movw %di, 4(%eax)
-; X32-NEXT: movl (%esp), %ecx # 4-byte Reload
-; X32-NEXT: movw %cx, 2(%eax)
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT: movzwl (%esp), %ecx # 2-byte Folded Reload
+; X32-NEXT: movw %cx, 12(%eax)
+; X32-NEXT: movw %bp, 10(%eax)
+; X32-NEXT: movw %bx, 8(%eax)
+; X32-NEXT: movw %di, 6(%eax)
+; X32-NEXT: movw %si, 4(%eax)
+; X32-NEXT: movw %dx, 2(%eax)
+; X32-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 2-byte Folded Reload
; X32-NEXT: movw %cx, (%eax)
-; X32-NEXT: addl $12, %esp
+; X32-NEXT: addl $4, %esp
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
; X32-NEXT: popl %ebx
@@ -1912,66 +1520,65 @@ define <8 x half> @test_ctselect_v8f16(i1 %cond, <8 x half> %a, <8 x half> %b) #
; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: subl $12, %esp
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorw %di, %cx
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %eax, %ecx
-; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorw %si, %cx
-; X32-NOCMOV-NEXT: andl %eax, %ecx
-; X32-NOCMOV-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movw %cx, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movw %cx, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movw %dx, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorw %dx, %cx
-; X32-NOCMOV-NEXT: andl %eax, %ecx
-; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorw %cx, %bx
-; X32-NOCMOV-NEXT: andl %eax, %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movw %si, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorw %cx, %bp
-; X32-NOCMOV-NEXT: andl %eax, %ebp
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorw %cx, %si
-; X32-NOCMOV-NEXT: andl %eax, %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorw %cx, %dx
-; X32-NOCMOV-NEXT: andl %eax, %edx
+; X32-NOCMOV-NEXT: sete %ah
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movw %di, {{[-0-9]+}}(%e{{[sb]}}p) # 2-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %al
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: xorw %di, %cx
-; X32-NOCMOV-NEXT: andl %eax, %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %eax, (%esp) # 4-byte Folded Spill
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: sete %dl
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %al
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movw %cx, 14(%eax)
-; X32-NOCMOV-NEXT: movw %dx, 12(%eax)
-; X32-NOCMOV-NEXT: movw %si, 10(%eax)
-; X32-NOCMOV-NEXT: movw %bp, 8(%eax)
-; X32-NOCMOV-NEXT: movw %bx, 6(%eax)
-; X32-NOCMOV-NEXT: movw %di, 4(%eax)
-; X32-NOCMOV-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %si, 14(%eax)
+; X32-NOCMOV-NEXT: movw %di, 12(%eax)
+; X32-NOCMOV-NEXT: movw %bp, 10(%eax)
+; X32-NOCMOV-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 2-byte Folded Reload
+; X32-NOCMOV-NEXT: movw %cx, 8(%eax)
+; X32-NOCMOV-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 2-byte Folded Reload
+; X32-NOCMOV-NEXT: movw %cx, 6(%eax)
+; X32-NOCMOV-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 2-byte Folded Reload
+; X32-NOCMOV-NEXT: movw %cx, 4(%eax)
+; X32-NOCMOV-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 2-byte Folded Reload
; X32-NOCMOV-NEXT: movw %cx, 2(%eax)
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movzwl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 2-byte Folded Reload
; X32-NOCMOV-NEXT: movw %cx, (%eax)
; X32-NOCMOV-NEXT: addl $12, %esp
; X32-NOCMOV-NEXT: popl %esi
@@ -1986,79 +1593,79 @@ define <8 x half> @test_ctselect_v8f16(i1 %cond, <8 x half> %a, <8 x half> %b) #
define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat> %b) #0 {
; X64-LABEL: test_ctselect_v8bf16:
; X64: # %bb.0:
-; X64-NEXT: movq %xmm1, %rax
-; X64-NEXT: movq %xmm0, %rcx
-; X64-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1,1]
-; X64-NEXT: movq %xmm1, %rsi
+; X64-NEXT: pushq %rbp
+; X64-NEXT: pushq %r15
+; X64-NEXT: pushq %r14
+; X64-NEXT: pushq %r13
+; X64-NEXT: pushq %r12
+; X64-NEXT: pushq %rbx
+; X64-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; X64-NEXT: movq %xmm0, %rdi
; X64-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
-; X64-NEXT: movq %xmm0, %r9
-; X64-NEXT: movq %r9, %rdx
-; X64-NEXT: movl %esi, %r8d
-; X64-NEXT: shrl $16, %r8d
-; X64-NEXT: movl %r9d, %r10d
-; X64-NEXT: shrl $16, %r10d
-; X64-NEXT: xorl %r8d, %r10d
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl %edi, %r10d
-; X64-NEXT: xorl %r8d, %r10d
-; X64-NEXT: movq %r9, %r8
-; X64-NEXT: xorl %esi, %r9d
-; X64-NEXT: andl %edi, %r9d
-; X64-NEXT: shll $16, %r10d
-; X64-NEXT: xorl %esi, %r9d
-; X64-NEXT: movzwl %r9w, %r9d
-; X64-NEXT: orl %r10d, %r9d
-; X64-NEXT: movq %rsi, %r10
-; X64-NEXT: shrq $48, %r10
-; X64-NEXT: shrq $48, %r8
-; X64-NEXT: xorl %r10d, %r8d
-; X64-NEXT: andl %edi, %r8d
-; X64-NEXT: xorl %r10d, %r8d
-; X64-NEXT: shrq $32, %rsi
-; X64-NEXT: shrq $32, %rdx
-; X64-NEXT: xorl %esi, %edx
-; X64-NEXT: andl %edi, %edx
-; X64-NEXT: xorl %esi, %edx
-; X64-NEXT: movq %rcx, %rsi
-; X64-NEXT: shll $16, %r8d
-; X64-NEXT: movzwl %dx, %edx
-; X64-NEXT: orl %r8d, %edx
-; X64-NEXT: movl %eax, %r8d
-; X64-NEXT: shrl $16, %r8d
-; X64-NEXT: shlq $32, %rdx
-; X64-NEXT: orq %r9, %rdx
-; X64-NEXT: movl %ecx, %r9d
+; X64-NEXT: movq %xmm0, %r14
+; X64-NEXT: movl %r14d, %eax
+; X64-NEXT: shrl $16, %eax
+; X64-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; X64-NEXT: movq %xmm1, %r10
+; X64-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1,1]
+; X64-NEXT: movq %xmm1, %r11
+; X64-NEXT: movl %r11d, %r9d
; X64-NEXT: shrl $16, %r9d
-; X64-NEXT: xorl %r8d, %r9d
-; X64-NEXT: andl %edi, %r9d
-; X64-NEXT: xorl %r8d, %r9d
-; X64-NEXT: movq %rcx, %r8
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: andl %edi, %ecx
-; X64-NEXT: shll $16, %r9d
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: movzwl %cx, %ecx
-; X64-NEXT: orl %r9d, %ecx
-; X64-NEXT: movq %rax, %r9
+; X64-NEXT: movq %r11, %rdx
+; X64-NEXT: shrq $48, %rdx
+; X64-NEXT: movq %r14, %r12
+; X64-NEXT: shrq $48, %r12
+; X64-NEXT: movq %r11, %rbx
+; X64-NEXT: shrq $32, %rbx
+; X64-NEXT: movq %r14, %r13
+; X64-NEXT: shrq $32, %r13
+; X64-NEXT: movl %edi, %ebp
+; X64-NEXT: shrl $16, %ebp
+; X64-NEXT: movl %r10d, %esi
+; X64-NEXT: shrl $16, %esi
+; X64-NEXT: movq %r10, %rcx
+; X64-NEXT: shrq $48, %rcx
+; X64-NEXT: movq %rdi, %r15
+; X64-NEXT: shrq $48, %r15
+; X64-NEXT: movq %r10, %r8
+; X64-NEXT: shrq $32, %r8
+; X64-NEXT: movq %rdi, %rax
; X64-NEXT: shrq $32, %rax
-; X64-NEXT: shrq $32, %rsi
-; X64-NEXT: shrq $48, %r9
-; X64-NEXT: shrq $48, %r8
-; X64-NEXT: xorl %r9d, %r8d
-; X64-NEXT: andl %edi, %r8d
-; X64-NEXT: xorl %eax, %esi
-; X64-NEXT: andl %edi, %esi
-; X64-NEXT: xorl %r9d, %r8d
-; X64-NEXT: xorl %eax, %esi
-; X64-NEXT: shll $16, %r8d
-; X64-NEXT: movzwl %si, %eax
-; X64-NEXT: orl %r8d, %eax
-; X64-NEXT: shlq $32, %rax
-; X64-NEXT: orq %rcx, %rax
-; X64-NEXT: movq %rax, %xmm0
-; X64-NEXT: movq %rdx, %xmm1
+; X64-NEXT: testb $1, {{[-0-9]+}}(%r{{[sb]}}p) # 1-byte Folded Reload
+; X64-NEXT: cmovnel %eax, %r8d
+; X64-NEXT: cmovnel %r15d, %ecx
+; X64-NEXT: cmovnel %edi, %r10d
+; X64-NEXT: cmovnel %ebp, %esi
+; X64-NEXT: cmovnel %r13d, %ebx
+; X64-NEXT: cmovnel %r12d, %edx
+; X64-NEXT: cmovnel %r14d, %r11d
+; X64-NEXT: movl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Reload
+; X64-NEXT: cmovnel %eax, %r9d
+; X64-NEXT: shll $16, %r9d
+; X64-NEXT: movzwl %r11w, %eax
+; X64-NEXT: orl %r9d, %eax
+; X64-NEXT: shll $16, %edx
+; X64-NEXT: movzwl %bx, %edi
+; X64-NEXT: orl %edx, %edi
+; X64-NEXT: shlq $32, %rdi
+; X64-NEXT: orq %rax, %rdi
+; X64-NEXT: shll $16, %esi
+; X64-NEXT: movzwl %r10w, %eax
+; X64-NEXT: orl %esi, %eax
+; X64-NEXT: shll $16, %ecx
+; X64-NEXT: movzwl %r8w, %edx
+; X64-NEXT: orl %ecx, %edx
+; X64-NEXT: shlq $32, %rdx
+; X64-NEXT: orq %rax, %rdx
+; X64-NEXT: movq %rdx, %xmm0
+; X64-NEXT: movq %rdi, %xmm1
; X64-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X64-NEXT: popq %rbx
+; X64-NEXT: popq %r12
+; X64-NEXT: popq %r13
+; X64-NEXT: popq %r14
+; X64-NEXT: popq %r15
+; X64-NEXT: popq %rbp
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_v8bf16:
@@ -2076,7 +1683,8 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps (%esp)
; X32-NEXT: calll __truncsfbf2
-; X32-NEXT: movl %eax, %ebp
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps (%esp)
; X32-NEXT: calll __truncsfbf2
@@ -2110,6 +1718,10 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps (%esp)
; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: movl %eax, %esi
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
; X32-NEXT: # kill: def $ax killed $ax def $eax
; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NEXT: flds {{[0-9]+}}(%esp)
@@ -2134,73 +1746,48 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps (%esp)
; X32-NEXT: calll __truncsfbf2
-; X32-NEXT: movl %eax, %esi
-; X32-NEXT: flds {{[0-9]+}}(%esp)
-; X32-NEXT: fstps (%esp)
-; X32-NEXT: calll __truncsfbf2
-; X32-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: movl %eax, %ebx
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps (%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
; X32-NEXT: calll __truncsfbf2
; X32-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: andb $1, %cl
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT: movzbl %cl, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: andl %ecx, %eax
-; X32-NEXT: movl %eax, %ebx
-; X32-NEXT: movl %ebp, %eax
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %eax
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %ebp
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %edx
-; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %edx
-; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %edx
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %edi
-; X32-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT: andl %ecx, %esi
-; X32-NEXT: movl %esi, %ecx
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: cmovnel %eax, %ebx
; X32-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: cmovnel %eax, %edi
+; X32-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: cmovnel %eax, %edi
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: cmovnel %eax, %esi
+; X32-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: cmovnel %eax, %esi
; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NEXT: movw %bx, 14(%ecx)
-; X32-NEXT: movw %ax, 12(%ecx)
-; X32-NEXT: movw %dx, 10(%ecx)
-; X32-NEXT: movw %si, 8(%ecx)
-; X32-NEXT: movw %di, 6(%ecx)
-; X32-NEXT: movw %bp, 4(%ecx)
; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT: movw %ax, 2(%ecx)
+; X32-NEXT: cmovnel %eax, %ebx
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: cmovnel %eax, %edx
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT: cmovnel %ecx, %eax
+; X32-NEXT: movw %ax, 14(%ebp)
+; X32-NEXT: movw %dx, 12(%ebp)
+; X32-NEXT: movw %bx, 10(%ebp)
+; X32-NEXT: movw %si, 8(%ebp)
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: movw %ax, 6(%ebp)
+; X32-NEXT: movw %di, 4(%ebp)
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: movw %ax, 2(%ebp)
; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT: movw %ax, (%ecx)
-; X32-NEXT: movl %ecx, %eax
+; X32-NEXT: movw %ax, (%ebp)
+; X32-NEXT: movl %ebp, %eax
; X32-NEXT: addl $60, %esp
; X32-NEXT: popl %esi
; X32-NEXT: popl %edi
@@ -2223,17 +1810,25 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: movl %eax, %ebp
+; X32-NOCMOV-NEXT: movl %eax, %edi
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl %eax, %ebx
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %ch
+; X32-NOCMOV-NEXT: testb $1, %ch
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, %ch
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
@@ -2242,13 +1837,15 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl %eax, %esi
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
@@ -2257,13 +1854,15 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl %eax, %esi
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
@@ -2272,7 +1871,8 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: movl %eax, %edi
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
@@ -2281,73 +1881,50 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: movl %eax, %esi
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
-; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl %eax, %ebx
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: calll __truncsfbf2
; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: andb $1, %cl
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: movzbl %cl, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl %ecx, %eax
-; X32-NOCMOV-NEXT: movl %eax, %ebx
-; X32-NOCMOV-NEXT: movl %ebp, %eax
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %eax
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %dl
; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %ebp
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %edx
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %edi
+; X32-NOCMOV-NEXT: BUNDLE
; X32-NOCMOV-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: andl %ecx, %esi
-; X32-NOCMOV-NEXT: movl %esi, %ecx
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: testb $1, %cl
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movw %si, 14(%eax)
; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
-; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
-; X32-NOCMOV-NEXT: movw %bx, 14(%ecx)
-; X32-NOCMOV-NEXT: movw %ax, 12(%ecx)
-; X32-NOCMOV-NEXT: movw %dx, 10(%ecx)
-; X32-NOCMOV-NEXT: movw %si, 8(%ecx)
-; X32-NOCMOV-NEXT: movw %di, 6(%ecx)
-; X32-NOCMOV-NEXT: movw %bp, 4(%ecx)
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: movw %ax, 2(%ecx)
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: movw %ax, (%ecx)
-; X32-NOCMOV-NEXT: movl %ecx, %eax
+; X32-NOCMOV-NEXT: movw %si, 12(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %cx, 10(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %cx, 8(%eax)
+; X32-NOCMOV-NEXT: movw %dx, 6(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %cx, 4(%eax)
+; X32-NOCMOV-NEXT: movw %bx, 2(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %cx, (%eax)
; X32-NOCMOV-NEXT: addl $60, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: popl %edi
@@ -2361,39 +1938,45 @@ define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat
define float @test_ctselect_f32_nan_inf(i1 %cond) #0 {
; X64-LABEL: test_ctselect_f32_nan_inf:
; X64: # %bb.0:
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl $4194304, %edi # imm = 0x400000
-; X64-NEXT: xorl $2139095040, %edi # imm = 0x7F800000
-; X64-NEXT: movd %edi, %xmm0
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movl $2143289344, %eax # imm = 0x7FC00000
+; X64-NEXT: movl $2139095040, %ecx # imm = 0x7F800000
+; X64-NEXT: cmovnel %eax, %ecx
+; X64-NEXT: movd %ecx, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f32_nan_inf:
; X32: # %bb.0:
+; X32-NEXT: pushl %edi
+; X32-NEXT: pushl %esi
; X32-NEXT: pushl %eax
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: andl $4194304, %eax # imm = 0x400000
-; X32-NEXT: xorl $2139095040, %eax # imm = 0x7F800000
-; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NEXT: sete %al
+; X32-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %ecx
+; X32-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %edx
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %esi, (%esp)
; X32-NEXT: flds (%esp)
-; X32-NEXT: popl %eax
+; X32-NEXT: addl $4, %esp
+; X32-NEXT: popl %esi
+; X32-NEXT: popl %edi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f32_nan_inf:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: pushl %eax
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl $4194304, %eax # imm = 0x400000
-; X32-NOCMOV-NEXT: xorl $2139095040, %eax # imm = 0x7F800000
-; X32-NOCMOV-NEXT: movl %eax, (%esp)
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %ecx
+; X32-NOCMOV-NEXT: movl {{\.?LCPI[0-9]+_[0-9]+}}, %edx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, (%esp)
; X32-NOCMOV-NEXT: flds (%esp)
-; X32-NOCMOV-NEXT: popl %eax
+; X32-NOCMOV-NEXT: addl $4, %esp
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: retl
%result = call float @llvm.ct.select.f32(i1 %cond, float 0x7FF8000000000000, float 0x7FF0000000000000)
ret float %result
@@ -2402,72 +1985,63 @@ define float @test_ctselect_f32_nan_inf(i1 %cond) #0 {
define double @test_ctselect_f64_nan_inf(i1 %cond) #0 {
; X64-LABEL: test_ctselect_f64_nan_inf:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: negq %rdi
-; X64-NEXT: movabsq $2251799813685248, %rax # imm = 0x8000000000000
-; X64-NEXT: andq %rdi, %rax
+; X64-NEXT: testb $1, %dil
+; X64-NEXT: movabsq $9221120237041090560, %rax # imm = 0x7FF8000000000000
; X64-NEXT: movabsq $9218868437227405312, %rcx # imm = 0x7FF0000000000000
-; X64-NEXT: xorq %rax, %rcx
+; X64-NEXT: cmovneq %rax, %rcx
; X64-NEXT: movq %rcx, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f64_nan_inf:
; X32: # %bb.0:
+; X32-NEXT: pushl %edi
; X32-NEXT: pushl %esi
; X32-NEXT: subl $24, %esp
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andb $1, %al
+; X32-NEXT: testb $1, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
-; X32-NEXT: fstpl {{[0-9]+}}(%esp)
; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X32-NEXT: sete %al
+; X32-NEXT: fxch %st(1)
; X32-NEXT: fstpl {{[0-9]+}}(%esp)
-; X32-NEXT: movzbl %al, %eax
-; X32-NEXT: negl %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: fstpl (%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl (%esp), %edx
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: andl %eax, %esi
-; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: movl %esi, (%esp)
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl %eax, %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NEXT: fldl (%esp)
+; X32-NEXT: BUNDLE
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT: fldl {{[0-9]+}}(%esp)
; X32-NEXT: addl $24, %esp
; X32-NEXT: popl %esi
+; X32-NEXT: popl %edi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f64_nan_inf:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: pushl %esi
; X32-NOCMOV-NEXT: subl $24, %esp
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: testb $1, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
-; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: fxch %st(1)
; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movzbl %al, %eax
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: fstpl (%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl (%esp), %edx
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: andl %eax, %esi
-; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: movl %esi, (%esp)
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl %eax, %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fldl (%esp)
+; X32-NOCMOV-NEXT: BUNDLE
+; X32-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fldl {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: addl $24, %esp
; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: retl
%result = call double @llvm.ct.select.f64(i1 %cond, double 0x7FF8000000000000, double 0x7FF0000000000000)
ret double %result
More information about the llvm-branch-commits
mailing list