[llvm] [ConstantTime][LLVM] Add llvm.ct.select intrinsic with generic SelectionDAG lowering (PR #166702)
Julius Alexandre via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 19 19:53:08 PDT 2026
https://github.com/wizardengineer updated https://github.com/llvm/llvm-project/pull/166702
>From 4e77664e222435c8444aa315b1a2345598f2f081 Mon Sep 17 00:00:00 2001
From: wizardengineer <juliuswoosebert at gmail.com>
Date: Wed, 5 Nov 2025 10:51:08 -0500
Subject: [PATCH 01/10] [ConstantTime][LLVM] Add llvm.ct.select intrinsic with
generic SelectionDAG lowering
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 4 +
llvm/include/llvm/CodeGen/SelectionDAG.h | 7 +
llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 3 +
llvm/include/llvm/CodeGen/TargetLowering.h | 18 +-
llvm/include/llvm/IR/Intrinsics.td | 10 +-
.../include/llvm/Target/TargetSelectionDAG.td | 6 +
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 112 ++-
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 46 +-
.../SelectionDAG/LegalizeFloatTypes.cpp | 12 +
.../SelectionDAG/LegalizeIntegerTypes.cpp | 20 +
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 6 +-
.../SelectionDAG/LegalizeTypesGeneric.cpp | 14 +
.../SelectionDAG/LegalizeVectorTypes.cpp | 13 +
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 1 +
.../SelectionDAG/SelectionDAGBuilder.cpp | 131 +++
.../SelectionDAG/SelectionDAGBuilder.h | 3 +
.../SelectionDAG/SelectionDAGDumper.cpp | 1 +
llvm/test/CodeGen/RISCV/ctselect-fallback.ll | 330 ++++++++
llvm/test/CodeGen/X86/ctselect.ll | 779 ++++++++++++++++++
19 files changed, 1503 insertions(+), 13 deletions(-)
create mode 100644 llvm/test/CodeGen/RISCV/ctselect-fallback.ll
create mode 100644 llvm/test/CodeGen/X86/ctselect.ll
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 23ef5d22963ae..366892fe05e8c 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -805,6 +805,10 @@ enum NodeType {
/// i1 then the high bits must conform to getBooleanContents.
SELECT,
+ /// Constant-time Select, implemented with CMOV instruction. This is used to
+ /// implement constant-time select.
+ CTSELECT,
+
/// Select with a vector condition (op #0) and two vector operands (ops #1
/// and #2), returning a vector result. All vectors have the same length.
/// Much like the scalar select and setcc, each bit in the condition selects
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index abc2e1f0ebe4f..00c5998a2f973 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1415,6 +1415,13 @@ class SelectionDAG {
return getNode(Opcode, DL, VT, Cond, LHS, RHS, Flags);
}
+ SDValue getCTSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS,
+ SDValue RHS, SDNodeFlags Flags = SDNodeFlags()) {
+ assert(LHS.getValueType() == VT && RHS.getValueType() == VT &&
+ "Cannot use select on differing types");
+ return getNode(ISD::CTSELECT, DL, VT, Cond, LHS, RHS, Flags);
+ }
+
/// Helper function to make it easier to build SelectCC's if you just have an
/// ISD::CondCode instead of an SDValue.
SDValue getSelectCC(const SDLoc &DL, SDValue LHS, SDValue RHS, SDValue True,
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index 990b649fafe19..ec53ea02b06e6 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -431,6 +431,9 @@ struct SDNodeFlags {
NonNeg | NoNaNs | NoInfs | SameSign | InBounds,
FastMathFlags = NoNaNs | NoInfs | NoSignedZeros | AllowReciprocal |
AllowContract | ApproximateFuncs | AllowReassociation,
+
+ // Flag for disabling optimization
+ NoMerge = 1 << 15,
};
/// Default constructor turns off all optimization flags.
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 9a6dd7735421e..be332c060ee79 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -260,11 +260,15 @@ class LLVM_ABI TargetLoweringBase {
/// Enum that describes what type of support for selects the target has.
enum SelectSupportKind {
- ScalarValSelect, // The target supports scalar selects (ex: cmov).
- ScalarCondVectorVal, // The target supports selects with a scalar condition
- // and vector values (ex: cmov).
- VectorMaskSelect // The target supports vector selects with a vector
- // mask (ex: x86 blends).
+ ScalarValSelect, // The target supports scalar selects (ex: cmov).
+ ScalarCondVectorVal, // The target supports selects with a scalar condition
+ // and vector values (ex: cmov).
+ VectorMaskSelect, // The target supports vector selects with a vector
+ // mask (ex: x86 blends).
+ CtSelect, // The target implements a custom constant-time select.
+ ScalarCondVectorValCtSelect, // The target supports selects with a scalar
+ // condition and vector values.
+ VectorMaskValCtSelect, // The target supports vector selects with a vector
};
/// Enum that specifies what an atomic load/AtomicRMWInst is expanded
@@ -495,8 +499,8 @@ class LLVM_ABI TargetLoweringBase {
MachineMemOperand::Flags
getVPIntrinsicMemOperandFlags(const VPIntrinsic &VPIntrin) const;
- virtual bool isSelectSupported(SelectSupportKind /*kind*/) const {
- return true;
+ virtual bool isSelectSupported(SelectSupportKind kind) const {
+ return kind != CtSelect;
}
/// Return true if the @llvm.get.active.lane.mask intrinsic should be expanded
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 37c9c783465d6..e878a039b3b6a 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2043,8 +2043,16 @@ def int_coro_subfn_addr : DefaultAttrsIntrinsic<
[IntrReadMem, IntrArgMemOnly, ReadOnly<ArgIndex<0>>,
NoCapture<ArgIndex<0>>]>;
-///===------------------------- Other Intrinsics --------------------------===//
+///===-------------------------- Constant Time Intrinsics ----------------------===//
//
+// Intrinsic to support constant time select
+def int_ct_select
+ : DefaultAttrsIntrinsic<[llvm_any_ty],
+ [llvm_i1_ty, LLVMMatchType<0>, LLVMMatchType<0>],
+ [IntrWriteMem, IntrWillReturn, NoUndef<RetIndex>]>;
+
+
+////===-------------------------- Other Intrinsics --------------------------===//
// TODO: We should introduce a new memory kind fo traps (and other side effects
// we only model to keep things alive).
def int_trap : Intrinsic<[], [],
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 69be608107359..24cc74fed90e0 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -214,6 +214,11 @@ def SDTSelect : SDTypeProfile<1, 3, [ // select
SDTCisInt<1>, SDTCisSameAs<0, 2>, SDTCisSameAs<2, 3>
]>;
+def SDTCtSelect
+ : SDTypeProfile<1, 3,
+ [ // ctselect
+ SDTCisInt<1>, SDTCisSameAs<0, 2>, SDTCisSameAs<2, 3>]>;
+
def SDTVSelect : SDTypeProfile<1, 3, [ // vselect
SDTCisVec<0>, SDTCisInt<1>, SDTCisSameAs<0, 2>, SDTCisSameAs<2, 3>, SDTCisSameNumEltsAs<0, 1>
]>;
@@ -767,6 +772,7 @@ def reset_fpmode : SDNode<"ISD::RESET_FPMODE", SDTNone, [SDNPHasChain]>;
def setcc : SDNode<"ISD::SETCC" , SDTSetCC>;
def select : SDNode<"ISD::SELECT" , SDTSelect>;
+def ctselect : SDNode<"ISD::CTSELECT", SDTCtSelect>;
def vselect : SDNode<"ISD::VSELECT" , SDTVSelect>;
def selectcc : SDNode<"ISD::SELECT_CC" , SDTSelectCC>;
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 32e6d65159b5e..8e32fe90498e6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -484,6 +484,7 @@ namespace {
SDValue visitCTTZ_ZERO_POISON(SDNode *N);
SDValue visitCTPOP(SDNode *N);
SDValue visitSELECT(SDNode *N);
+ SDValue visitCTSELECT(SDNode *N);
SDValue visitVSELECT(SDNode *N);
SDValue visitVP_SELECT(SDNode *N);
SDValue visitSELECT_CC(SDNode *N);
@@ -1963,6 +1964,7 @@ void DAGCombiner::Run(CombineLevel AtLevel) {
}
SDValue DAGCombiner::visit(SDNode *N) {
+
// clang-format off
switch (N->getOpcode()) {
default: break;
@@ -2039,6 +2041,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
case ISD::CTTZ_ZERO_POISON: return visitCTTZ_ZERO_POISON(N);
case ISD::CTPOP: return visitCTPOP(N);
case ISD::SELECT: return visitSELECT(N);
+ case ISD::CTSELECT: return visitCTSELECT(N);
case ISD::VSELECT: return visitVSELECT(N);
case ISD::SELECT_CC: return visitSELECT_CC(N);
case ISD::SETCC: return visitSETCC(N);
@@ -6216,6 +6219,7 @@ static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2,
N0CC = cast<CondCodeSDNode>(N0.getOperand(4))->get();
break;
case ISD::SELECT:
+ case ISD::CTSELECT:
case ISD::VSELECT:
if (N0.getOperand(0).getOpcode() != ISD::SETCC)
return SDValue();
@@ -12926,8 +12930,9 @@ template <class MatchContextClass>
static SDValue foldBoolSelectToLogic(SDNode *N, const SDLoc &DL,
SelectionDAG &DAG) {
assert((N->getOpcode() == ISD::SELECT || N->getOpcode() == ISD::VSELECT ||
- N->getOpcode() == ISD::VP_SELECT) &&
- "Expected a (v)(vp.)select");
+ N->getOpcode() == ISD::VP_SELECT ||
+ N->getOpcode() == ISD::CTSELECT) &&
+ "Expected a (v)(vp.)(ct) select");
SDValue Cond = N->getOperand(0);
SDValue T = N->getOperand(1), F = N->getOperand(2);
EVT VT = N->getValueType(0);
@@ -13431,6 +13436,109 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
return SDValue();
}
+SDValue DAGCombiner::visitCTSELECT(SDNode *N) {
+ SDValue N0 = N->getOperand(0);
+ SDValue N1 = N->getOperand(1);
+ SDValue N2 = N->getOperand(2);
+ EVT VT = N->getValueType(0);
+ EVT VT0 = N0.getValueType();
+ SDLoc DL(N);
+ SDNodeFlags Flags = N->getFlags();
+
+ if (SDValue V = foldBoolSelectToLogic<EmptyMatchContext>(N, DL, DAG))
+ return V;
+
+ // ctselect (not Cond), N1, N2 -> ctselect Cond, N2, N1
+ if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) {
+ SDValue SelectOp = DAG.getNode(ISD::CTSELECT, DL, VT, F, N2, N1);
+ SelectOp->setFlags(Flags);
+ return SelectOp;
+ }
+
+ if (VT0 == MVT::i1) {
+ // The code in this block deals with the following 2 equivalences:
+ // select(C0|C1, x, y) <=> select(C0, x, select(C1, x, y))
+ // select(C0&C1, x, y) <=> select(C0, select(C1, x, y), y)
+ // The target can specify its preferred form with the
+ // shouldNormalizeToSelectSequence() callback. However we always transform
+ // to the right anyway if we find the inner select exists in the DAG anyway
+ // and we always transform to the left side if we know that we can further
+ // optimize the combination of the conditions.
+ bool normalizeToSequence =
+ TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT);
+ // ctselect (and Cond0, Cond1), X, Y
+ // -> ctselect Cond0, (ctselect Cond1, X, Y), Y
+ if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) {
+ SDValue Cond0 = N0->getOperand(0);
+ SDValue Cond1 = N0->getOperand(1);
+ SDValue InnerSelect = DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(),
+ Cond1, N1, N2, Flags);
+ if (normalizeToSequence || !InnerSelect.use_empty())
+ return DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), Cond0,
+ InnerSelect, N2, Flags);
+ // Cleanup on failure.
+ if (InnerSelect.use_empty())
+ recursivelyDeleteUnusedNodes(InnerSelect.getNode());
+ }
+ // ctselect (or Cond0, Cond1), X, Y -> ctselect Cond0, X, (ctselect Cond1,
+ // X, Y)
+ if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) {
+ SDValue Cond0 = N0->getOperand(0);
+ SDValue Cond1 = N0->getOperand(1);
+ SDValue InnerSelect = DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(),
+ Cond1, N1, N2, Flags);
+ if (normalizeToSequence || !InnerSelect.use_empty())
+ return DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), Cond0, N1,
+ InnerSelect, Flags);
+ // Cleanup on failure.
+ if (InnerSelect.use_empty())
+ recursivelyDeleteUnusedNodes(InnerSelect.getNode());
+ }
+
+ // ctselect Cond0, (ctselect Cond1, X, Y), Y -> ctselect (and Cond0, Cond1),
+ // X, Y
+ if (N1->getOpcode() == ISD::CTSELECT && N1->hasOneUse()) {
+ SDValue N1_0 = N1->getOperand(0);
+ SDValue N1_1 = N1->getOperand(1);
+ SDValue N1_2 = N1->getOperand(2);
+ if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) {
+ // Create the actual and node if we can generate good code for it.
+ if (!normalizeToSequence) {
+ SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
+ return DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), And, N1_1,
+ N2, Flags);
+ }
+ // Otherwise see if we can optimize the "and" to a better pattern.
+ if (SDValue Combined = visitANDLike(N0, N1_0, N)) {
+ return DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), Combined,
+ N1_1, N2, Flags);
+ }
+ }
+ }
+ // ctselect Cond0, X, (ctselect Cond1, X, Y) -> ctselect (or Cond0, Cond1),
+ // X, Y
+ if (N2->getOpcode() == ISD::CTSELECT && N2->hasOneUse()) {
+ SDValue N2_0 = N2->getOperand(0);
+ SDValue N2_1 = N2->getOperand(1);
+ SDValue N2_2 = N2->getOperand(2);
+ if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) {
+ // Create the actual or node if we can generate good code for it.
+ if (!normalizeToSequence) {
+ SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
+ return DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), Or, N1, N2_2,
+ Flags);
+ }
+ // Otherwise see if we can optimize to a better pattern.
+ if (SDValue Combined = visitORLike(N0, N2_0, DL))
+ return DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), Combined, N1,
+ N2_2, Flags);
+ }
+ }
+ }
+
+ return SDValue();
+}
+
// This function assumes all the vselect's arguments are CONCAT_VECTOR
// nodes and that the condition is a BV of ConstantSDNodes (or undefs).
static SDValue ConvertSelectToConcatVector(SDNode *N, SelectionDAG &DAG) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 65b8212755e6e..0a60dcc8a19b4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4350,6 +4350,46 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
}
Results.push_back(Tmp1);
break;
+ case ISD::CTSELECT: {
+ Tmp1 = Node->getOperand(0);
+ Tmp2 = Node->getOperand(1);
+ Tmp3 = Node->getOperand(2);
+ EVT VT = Tmp2.getValueType();
+ if (VT.isVector()) {
+ SmallVector<SDValue> Elements;
+ unsigned NumElements = VT.getVectorNumElements();
+ EVT ScalarVT = VT.getScalarType();
+ for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
+ SDValue IdxVal = DAG.getConstant(Idx, dl, MVT::i64);
+ SDValue TVal =
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Tmp2, IdxVal);
+ SDValue FVal =
+ DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Tmp3, IdxVal);
+ Elements.push_back(
+ DAG.getCTSelect(dl, ScalarVT, Tmp1, TVal, FVal, Node->getFlags()));
+ }
+ Tmp1 = DAG.getBuildVector(VT, dl, Elements);
+ } else if (VT.isFloatingPoint()) {
+ EVT IntegerVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
+ Tmp2 = DAG.getBitcast(IntegerVT, Tmp2);
+ Tmp3 = DAG.getBitcast(IntegerVT, Tmp3);
+ Tmp1 = DAG.getBitcast(VT, DAG.getCTSelect(dl, IntegerVT, Tmp1, Tmp2, Tmp3,
+ Node->getFlags()));
+ } else {
+ assert(VT.isInteger());
+ EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
+ auto [Tmp2Lo, Tmp2Hi] = DAG.SplitScalar(Tmp2, dl, HalfVT, HalfVT);
+ auto [Tmp3Lo, Tmp3Hi] = DAG.SplitScalar(Tmp3, dl, HalfVT, HalfVT);
+ SDValue ResLo =
+ DAG.getCTSelect(dl, HalfVT, Tmp1, Tmp2Lo, Tmp3Lo, Node->getFlags());
+ SDValue ResHi =
+ DAG.getCTSelect(dl, HalfVT, Tmp1, Tmp2Hi, Tmp3Hi, Node->getFlags());
+ Tmp1 = DAG.getNode(ISD::BUILD_PAIR, dl, VT, ResLo, ResHi);
+ Tmp1->setFlags(Node->getFlags());
+ }
+ Results.push_back(Tmp1);
+ break;
+ }
case ISD::BR_JT: {
SDValue Chain = Node->getOperand(0);
SDValue Table = Node->getOperand(1);
@@ -5726,7 +5766,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
Results.push_back(DAG.getNode(ISD::TRUNCATE, dl, OVT, Tmp2));
break;
}
- case ISD::SELECT: {
+ case ISD::SELECT:
+ case ISD::CTSELECT: {
unsigned ExtOp, TruncOp;
if (Node->getValueType(0).isVector() ||
Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
@@ -5744,7 +5785,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
// Perform the larger operation, then round down.
- Tmp1 = DAG.getSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
+ Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3);
+ Tmp1->setFlags(Node->getFlags());
if (TruncOp != ISD::FP_ROUND)
Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
else
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 25cc420c42482..2265b01fd3cd5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -161,6 +161,7 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
+ case ISD::CTSELECT: R = SoftenFloatRes_CTSELECT(N); break;
case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
case ISD::STRICT_SINT_TO_FP:
@@ -1065,6 +1066,13 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
LHS.getValueType(), N->getOperand(0), LHS, RHS);
}
+SDValue DAGTypeLegalizer::SoftenFloatRes_CTSELECT(SDNode *N) {
+ SDValue LHS = GetSoftenedFloat(N->getOperand(1));
+ SDValue RHS = GetSoftenedFloat(N->getOperand(2));
+ return DAG.getCTSelect(SDLoc(N), LHS.getValueType(), N->getOperand(0), LHS,
+ RHS);
+}
+
SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
SDValue LHS = GetSoftenedFloat(N->getOperand(2));
SDValue RHS = GetSoftenedFloat(N->getOperand(3));
@@ -1585,6 +1593,7 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
case ISD::POISON:
case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
+ case ISD::CTSELECT: SplitRes_Select(N, Lo, Hi); break;
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
@@ -2763,6 +2772,9 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
break;
case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
+ case ISD::CTSELECT:
+ R = SoftPromoteHalfRes_SELECT(N);
+ break;
case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
case ISD::STRICT_SINT_TO_FP:
case ISD::STRICT_UINT_TO_FP:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index 2fee0b280ed9b..d532d688b760a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -98,6 +98,7 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
Res = PromoteIntRes_VECTOR_COMPRESS(N);
break;
case ISD::SELECT:
+ case ISD::CTSELECT:
case ISD::VSELECT:
case ISD::VP_SELECT:
case ISD::VP_MERGE:
@@ -2173,6 +2174,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
break;
case ISD::VSELECT:
case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
+ case ISD::CTSELECT:
+ Res = PromoteIntOp_CTSELECT(N, OpNo);
+ break;
case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
case ISD::VP_SETCC:
case ISD::SETCC: Res = PromoteIntOp_SETCC(N, OpNo); break;
@@ -2586,6 +2590,19 @@ SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
N->getOperand(2)), 0);
}
+SDValue DAGTypeLegalizer::PromoteIntOp_CTSELECT(SDNode *N, unsigned OpNo) {
+ assert(OpNo == 0 && "Only know how to promote the condition!");
+ SDValue Cond = N->getOperand(0);
+ EVT OpTy = N->getOperand(1).getValueType();
+
+ // Promote all the way up to the canonical SetCC type.
+ EVT OpVT = N->getOpcode() == ISD::CTSELECT ? OpTy.getScalarType() : OpTy;
+ Cond = PromoteTargetBoolean(Cond, OpVT);
+
+ return SDValue(
+ DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), N->getOperand(2)), 0);
+}
+
SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
assert(OpNo == 0 && "Don't know how to promote this operand!");
@@ -3232,6 +3249,9 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break;
case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
+ case ISD::CTSELECT:
+ SplitRes_Select(N, Lo, Hi);
+ break;
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
case ISD::POISON:
case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index d4d56a9563f71..fb1ce23b44637 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -389,6 +389,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntOp_CONCAT_VECTORS(SDNode *N);
SDValue PromoteIntOp_ScalarOp(SDNode *N);
SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo);
+ SDValue PromoteIntOp_CTSELECT(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_SETCC(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_Shift(SDNode *N);
@@ -629,6 +630,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftenFloatRes_LOAD(SDNode *N);
SDValue SoftenFloatRes_ATOMIC_LOAD(SDNode *N);
SDValue SoftenFloatRes_SELECT(SDNode *N);
+ SDValue SoftenFloatRes_CTSELECT(SDNode *N);
SDValue SoftenFloatRes_SELECT_CC(SDNode *N);
SDValue SoftenFloatRes_UNDEF(SDNode *N);
SDValue SoftenFloatRes_VAARG(SDNode *N);
@@ -861,6 +863,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N);
SDValue ScalarizeVecRes_VSELECT(SDNode *N);
SDValue ScalarizeVecRes_SELECT(SDNode *N);
+ SDValue ScalarizeVecRes_CTSELECT(SDNode *N);
SDValue ScalarizeVecRes_SELECT_CC(SDNode *N);
SDValue ScalarizeVecRes_SETCC(SDNode *N);
SDValue ScalarizeVecRes_UNDEF(SDNode *N);
@@ -1203,7 +1206,8 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void SplitVecRes_AssertZext(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitVecRes_AssertSext(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_ARITH_FENCE (SDNode *N, SDValue &Lo, SDValue &Hi);
- void SplitRes_Select (SDNode *N, SDValue &Lo, SDValue &Hi);
+ void SplitRes_Select(SDNode *N, SDValue &Lo, SDValue &Hi);
+ void SplitRes_CTSELECT(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_SELECT_CC (SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_UNDEF (SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_FREEZE (SDNode *N, SDValue &Lo, SDValue &Hi);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
index 0cff883f83186..d942362dbc2dc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
@@ -569,6 +569,20 @@ void DAGTypeLegalizer::SplitRes_Select(SDNode *N, SDValue &Lo, SDValue &Hi) {
Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH, EVLHi);
}
+void DAGTypeLegalizer::SplitRes_CTSELECT(SDNode *N, SDValue &Lo, SDValue &Hi) {
+ SDValue LL, LH, RL, RH, CL, CH;
+ SDLoc dl(N);
+ GetSplitOp(N->getOperand(1), LL, LH);
+ GetSplitOp(N->getOperand(2), RL, RH);
+
+ SDValue Cond = N->getOperand(0);
+ CL = CH = Cond;
+ assert(!Cond.getValueType().isVector() && "Unsupported vector type");
+
+ Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), CL, LL, RL);
+ Hi = DAG.getNode(N->getOpcode(), dl, LH.getValueType(), CH, LH, RH);
+}
+
void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo,
SDValue &Hi) {
SDValue LL, LH, RL, RH;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 3ac1ca367abc8..d98d001be1003 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -87,6 +87,9 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
+ case ISD::CTSELECT:
+ R = ScalarizeVecRes_CTSELECT(N);
+ break;
case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
case ISD::POISON:
@@ -764,6 +767,12 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
GetScalarizedVector(N->getOperand(2)));
}
+SDValue DAGTypeLegalizer::ScalarizeVecRes_CTSELECT(SDNode *N) {
+ SDValue LHS = GetScalarizedVector(N->getOperand(1));
+ return DAG.getCTSelect(SDLoc(N), LHS.getValueType(), N->getOperand(0), LHS,
+ GetScalarizedVector(N->getOperand(2)));
+}
+
SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
SDValue LHS = GetScalarizedVector(N->getOperand(2));
return DAG.getNode(ISD::SELECT_CC, SDLoc(N), LHS.getValueType(),
@@ -1381,6 +1390,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
case ISD::SELECT:
case ISD::VP_MERGE:
case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
+ case ISD::CTSELECT:
+ SplitRes_CTSELECT(N, Lo, Hi);
+ break;
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
case ISD::POISON:
case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
@@ -5289,6 +5301,7 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
case ISD::VSELECT:
case ISD::SELECT:
+ case ISD::CTSELECT:
case ISD::VP_SELECT:
case ISD::VP_MERGE:
Res = WidenVecRes_Select(N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index c61a2edd45255..2da390e0c4989 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8980,6 +8980,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
return V;
break;
}
+
case ISD::SELECT:
case ISD::VSELECT:
if (SDValue V = simplifySelect(N1, N2, N3))
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 0f6bd53cafdd8..1f2789c8c22e4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6692,6 +6692,105 @@ void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
setValue(&I, Result);
}
+/// Fallback implementation for constant-time select using DAG chaining.
+/// This implementation uses data dependencies through virtual registers to
+/// prevent optimizations from breaking the constant-time property.
+/// It handles scalars, vectors (fixed and scalable), and floating-point types.
+SDValue SelectionDAGBuilder::createProtectedCtSelectFallback(
+ SelectionDAG &DAG, const SDLoc &DL, SDValue Cond, SDValue T, SDValue F,
+ EVT VT) {
+
+ SDValue WorkingT = T;
+ SDValue WorkingF = F;
+ EVT WorkingVT = VT;
+
+ SDValue Chain = DAG.getEntryNode();
+ MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
+
+ // Handle vector condition: splat scalar condition to vector
+ if (VT.isVector() && !Cond.getValueType().isVector()) {
+ ElementCount NumElems = VT.getVectorElementCount();
+ EVT CondVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElems);
+
+ if (VT.isScalableVector()) {
+ Cond = DAG.getSplatVector(CondVT, DL, Cond);
+ } else {
+ Cond = DAG.getSplatBuildVector(CondVT, DL, Cond);
+ }
+ }
+
+ // Handle floating-point types: bitcast to integer for bitwise operations
+ if (VT.isFloatingPoint()) {
+ if (VT.isVector()) {
+ // float vector -> int vector
+ EVT ElemVT = VT.getVectorElementType();
+ unsigned int ElemBitWidth = ElemVT.getScalarSizeInBits();
+ EVT IntElemVT = EVT::getIntegerVT(*DAG.getContext(), ElemBitWidth);
+
+ WorkingVT = EVT::getVectorVT(*DAG.getContext(), IntElemVT,
+ VT.getVectorElementCount());
+ } else {
+ WorkingVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
+ }
+
+ WorkingT = DAG.getBitcast(WorkingVT, T);
+ WorkingF = DAG.getBitcast(WorkingVT, F);
+ }
+
+ // Create mask: sign-extend condition to all bits
+ SDValue Mask = DAG.getSExtOrTrunc(Cond, DL, WorkingVT);
+
+ // Create all-ones constant for inversion
+ SDValue AllOnes;
+ if (WorkingVT.isScalableVector()) {
+ unsigned BitWidth = WorkingVT.getScalarSizeInBits();
+ APInt AllOnesVal = APInt::getAllOnes(BitWidth);
+ SDValue ScalarAllOnes =
+ DAG.getConstant(AllOnesVal, DL, WorkingVT.getScalarType());
+ AllOnes = DAG.getSplatVector(WorkingVT, DL, ScalarAllOnes);
+ } else {
+ AllOnes = DAG.getAllOnesConstant(DL, WorkingVT);
+ }
+
+ // Invert mask for false value
+ SDValue Invert = DAG.getNode(ISD::XOR, DL, WorkingVT, Mask, AllOnes);
+
+ // Compute: (T & Mask) | (F & ~Mask)
+ // This is constant-time because both branches are always computed
+ SDValue TM = DAG.getNode(ISD::AND, DL, WorkingVT, Mask, WorkingT);
+
+ // DAG chaining: create data dependency through virtual register
+ // This prevents optimizations from reordering or eliminating operations
+ const TargetLowering &TLI = DAG.getTargetLoweringInfo();
+ bool CanUseChaining = false;
+
+ if (!WorkingVT.isScalableVector()) {
+ // For fixed-size vectors and scalars, check if type is legal
+ CanUseChaining = TLI.isTypeLegal(WorkingVT.getSimpleVT());
+ } else {
+ // For scalable vectors, disable chaining (conservative approach)
+ CanUseChaining = false;
+ }
+
+ if (CanUseChaining) {
+ // Apply chaining through registers for additional protection
+ const TargetRegisterClass *RC = TLI.getRegClassFor(WorkingVT.getSimpleVT());
+ Register TMReg = MRI.createVirtualRegister(RC);
+ Chain = DAG.getCopyToReg(Chain, DL, TMReg, TM);
+ TM = DAG.getCopyFromReg(Chain, DL, TMReg, WorkingVT);
+ }
+
+ SDValue FM = DAG.getNode(ISD::AND, DL, WorkingVT, Invert, WorkingF);
+ SDValue Result = DAG.getNode(ISD::OR, DL, WorkingVT, TM, FM);
+
+ // Convert back to original type if needed
+ if (WorkingVT != VT) {
+ Result = DAG.getBitcast(VT, Result);
+ }
+
+ return Result;
+}
+
/// Lower the call to the specified intrinsic function.
void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
unsigned Intrinsic) {
@@ -6893,6 +6992,38 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
updateDAGForMaybeTailCall(MC);
return;
}
+ case Intrinsic::ct_select: {
+ // Set function attribute to indicate ct.select usage
+ Function &F = DAG.getMachineFunction().getFunction();
+ F.addFnAttr("ct-select");
+
+ SDLoc DL = getCurSDLoc();
+
+ SDValue Cond = getValue(I.getArgOperand(0)); // i1
+ SDValue A = getValue(I.getArgOperand(1)); // T
+ SDValue B = getValue(I.getArgOperand(2)); // T
+
+ assert((A.getValueType() == B.getValueType()) &&
+ "Operands are of different types");
+
+ EVT VT = A.getValueType();
+ EVT CondVT = Cond.getValueType();
+
+ // assert if Cond type is Vector
+ assert(!CondVT.isVector() && "Vector type cond not supported yet");
+
+ // Handle scalar types
+ if (TLI.isSelectSupported(
+ TargetLoweringBase::SelectSupportKind::CtSelect) &&
+ !CondVT.isVector()) {
+ SDValue Result = DAG.getNode(ISD::CTSELECT, DL, VT, Cond, A, B);
+ setValue(&I, Result);
+ return;
+ }
+
+ setValue(&I, createProtectedCtSelectFallback(DAG, DL, Cond, A, B, VT));
+ return;
+ }
case Intrinsic::call_preallocated_setup: {
const CallBase *PreallocatedCall = FindPreallocatedCall(&I);
SDValue SrcValue = DAG.getSrcValue(PreallocatedCall);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 6c7711af078f0..1775d1d5588d3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -219,6 +219,9 @@ class SelectionDAGBuilder {
peelDominantCaseCluster(const SwitchInst &SI,
SwitchCG::CaseClusterVector &Clusters,
BranchProbability &PeeledCaseProb);
+ SDValue createProtectedCtSelectFallback(SelectionDAG &DAG, const SDLoc &DL,
+ SDValue Cond, SDValue T, SDValue F,
+ EVT VT);
private:
const TargetMachine &TM;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 028f0b9486c01..789424191395a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -346,6 +346,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::FPOWI: return "fpowi";
case ISD::STRICT_FPOWI: return "strict_fpowi";
case ISD::SETCC: return "setcc";
+ case ISD::CTSELECT: return "ctselect";
case ISD::SETCCCARRY: return "setcccarry";
case ISD::STRICT_FSETCC: return "strict_fsetcc";
case ISD::STRICT_FSETCCS: return "strict_fsetccs";
diff --git a/llvm/test/CodeGen/RISCV/ctselect-fallback.ll b/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
new file mode 100644
index 0000000000000..f46bde0a05b8b
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
@@ -0,0 +1,330 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=riscv64 -O3 | FileCheck %s --check-prefix=RV64
+; RUN: llc < %s -mtriple=riscv32 -O3 | FileCheck %s --check-prefix=RV32
+
+; Test basic ct.select functionality for scalar types
+define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) {
+; RV64-LABEL: test_ctselect_i8:
+; RV64: # %bb.0:
+; RV64-NEXT: xor a1, a1, a2
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: xor a0, a0, a2
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_i8:
+; RV32: # %bb.0:
+; RV32-NEXT: xor a1, a1, a2
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a0, a1, a0
+; RV32-NEXT: xor a0, a0, a2
+; RV32-NEXT: ret
+ %result = call i8 @llvm.ct.select.i8(i1 %cond, i8 %a, i8 %b)
+ ret i8 %result
+}
+
+define i16 @test_ctselect_i16(i1 %cond, i16 %a, i16 %b) {
+; RV64-LABEL: test_ctselect_i16:
+; RV64: # %bb.0:
+; RV64-NEXT: xor a1, a1, a2
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: xor a0, a0, a2
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_i16:
+; RV32: # %bb.0:
+; RV32-NEXT: xor a1, a1, a2
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a0, a1, a0
+; RV32-NEXT: xor a0, a0, a2
+; RV32-NEXT: ret
+ %result = call i16 @llvm.ct.select.i16(i1 %cond, i16 %a, i16 %b)
+ ret i16 %result
+}
+
+define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) {
+; RV64-LABEL: test_ctselect_i32:
+; RV64: # %bb.0:
+; RV64-NEXT: xor a1, a1, a2
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: xor a0, a0, a2
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_i32:
+; RV32: # %bb.0:
+; RV32-NEXT: andi a0, a0, 1
+; RV32-NEXT: neg a3, a0
+; RV32-NEXT: addi a0, a0, -1
+; RV32-NEXT: and a1, a3, a1
+; RV32-NEXT: and a0, a0, a2
+; RV32-NEXT: or a0, a1, a0
+; RV32-NEXT: ret
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) {
+; RV64-LABEL: test_ctselect_i64:
+; RV64: # %bb.0:
+; RV64-NEXT: andi a0, a0, 1
+; RV64-NEXT: neg a3, a0
+; RV64-NEXT: addi a0, a0, -1
+; RV64-NEXT: and a1, a3, a1
+; RV64-NEXT: and a0, a0, a2
+; RV64-NEXT: or a0, a1, a0
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_i64:
+; RV32: # %bb.0:
+; RV32-NEXT: xor a1, a1, a3
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: xor a2, a2, a4
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a1, a1, a0
+; RV32-NEXT: and a2, a2, a0
+; RV32-NEXT: xor a0, a1, a3
+; RV32-NEXT: xor a1, a2, a4
+; RV32-NEXT: ret
+ %result = call i64 @llvm.ct.select.i64(i1 %cond, i64 %a, i64 %b)
+ ret i64 %result
+}
+
+define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) {
+; RV64-LABEL: test_ctselect_ptr:
+; RV64: # %bb.0:
+; RV64-NEXT: andi a0, a0, 1
+; RV64-NEXT: neg a3, a0
+; RV64-NEXT: addi a0, a0, -1
+; RV64-NEXT: and a1, a3, a1
+; RV64-NEXT: and a0, a0, a2
+; RV64-NEXT: or a0, a1, a0
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_ptr:
+; RV32: # %bb.0:
+; RV32-NEXT: andi a0, a0, 1
+; RV32-NEXT: neg a3, a0
+; RV32-NEXT: addi a0, a0, -1
+; RV32-NEXT: and a1, a3, a1
+; RV32-NEXT: and a0, a0, a2
+; RV32-NEXT: or a0, a1, a0
+; RV32-NEXT: ret
+ %result = call ptr @llvm.ct.select.p0(i1 %cond, ptr %a, ptr %b)
+ ret ptr %result
+}
+
+; Test with constant conditions
+define i32 @test_ctselect_const_true(i32 %a, i32 %b) {
+; RV64-LABEL: test_ctselect_const_true:
+; RV64: # %bb.0:
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_const_true:
+; RV32: # %bb.0:
+; RV32-NEXT: ret
+ %result = call i32 @llvm.ct.select.i32(i1 true, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+define i32 @test_ctselect_const_false(i32 %a, i32 %b) {
+; RV64-LABEL: test_ctselect_const_false:
+; RV64: # %bb.0:
+; RV64-NEXT: mv a0, a1
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_const_false:
+; RV32: # %bb.0:
+; RV32-NEXT: mv a0, a1
+; RV32-NEXT: ret
+ %result = call i32 @llvm.ct.select.i32(i1 false, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test with comparison conditions
+define i32 @test_ctselect_icmp_eq(i32 %x, i32 %y, i32 %a, i32 %b) {
+; RV64-LABEL: test_ctselect_icmp_eq:
+; RV64: # %bb.0:
+; RV64-NEXT: sext.w a1, a1
+; RV64-NEXT: sext.w a0, a0
+; RV64-NEXT: xor a0, a0, a1
+; RV64-NEXT: snez a0, a0
+; RV64-NEXT: xor a2, a2, a3
+; RV64-NEXT: addi a0, a0, -1
+; RV64-NEXT: and a0, a2, a0
+; RV64-NEXT: xor a0, a0, a3
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_icmp_eq:
+; RV32: # %bb.0:
+; RV32-NEXT: xor a0, a0, a1
+; RV32-NEXT: snez a0, a0
+; RV32-NEXT: addi a0, a0, -1
+; RV32-NEXT: and a2, a0, a2
+; RV32-NEXT: not a0, a0
+; RV32-NEXT: and a0, a0, a3
+; RV32-NEXT: or a0, a2, a0
+; RV32-NEXT: ret
+ %cond = icmp eq i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+define i32 @test_ctselect_icmp_ne(i32 %x, i32 %y, i32 %a, i32 %b) {
+; RV64-LABEL: test_ctselect_icmp_ne:
+; RV64: # %bb.0:
+; RV64-NEXT: sext.w a1, a1
+; RV64-NEXT: sext.w a0, a0
+; RV64-NEXT: xor a0, a0, a1
+; RV64-NEXT: seqz a0, a0
+; RV64-NEXT: xor a2, a2, a3
+; RV64-NEXT: addi a0, a0, -1
+; RV64-NEXT: and a0, a2, a0
+; RV64-NEXT: xor a0, a0, a3
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_icmp_ne:
+; RV32: # %bb.0:
+; RV32-NEXT: xor a0, a0, a1
+; RV32-NEXT: seqz a0, a0
+; RV32-NEXT: addi a0, a0, -1
+; RV32-NEXT: and a2, a0, a2
+; RV32-NEXT: not a0, a0
+; RV32-NEXT: and a0, a0, a3
+; RV32-NEXT: or a0, a2, a0
+; RV32-NEXT: ret
+ %cond = icmp ne i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+define i32 @test_ctselect_icmp_slt(i32 %x, i32 %y, i32 %a, i32 %b) {
+; RV64-LABEL: test_ctselect_icmp_slt:
+; RV64: # %bb.0:
+; RV64-NEXT: sext.w a1, a1
+; RV64-NEXT: sext.w a0, a0
+; RV64-NEXT: slt a0, a0, a1
+; RV64-NEXT: xor a2, a2, a3
+; RV64-NEXT: neg a0, a0
+; RV64-NEXT: and a0, a2, a0
+; RV64-NEXT: xor a0, a0, a3
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_icmp_slt:
+; RV32: # %bb.0:
+; RV32-NEXT: slt a0, a0, a1
+; RV32-NEXT: neg a1, a0
+; RV32-NEXT: addi a0, a0, -1
+; RV32-NEXT: and a1, a1, a2
+; RV32-NEXT: and a0, a0, a3
+; RV32-NEXT: or a0, a1, a0
+; RV32-NEXT: ret
+ %cond = icmp slt i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+define i32 @test_ctselect_icmp_ult(i32 %x, i32 %y, i32 %a, i32 %b) {
+; RV64-LABEL: test_ctselect_icmp_ult:
+; RV64: # %bb.0:
+; RV64-NEXT: sext.w a1, a1
+; RV64-NEXT: sext.w a0, a0
+; RV64-NEXT: sltu a0, a0, a1
+; RV64-NEXT: xor a2, a2, a3
+; RV64-NEXT: neg a0, a0
+; RV64-NEXT: and a0, a2, a0
+; RV64-NEXT: xor a0, a0, a3
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_icmp_ult:
+; RV32: # %bb.0:
+; RV32-NEXT: sltu a0, a0, a1
+; RV32-NEXT: neg a1, a0
+; RV32-NEXT: addi a0, a0, -1
+; RV32-NEXT: and a1, a1, a2
+; RV32-NEXT: and a0, a0, a3
+; RV32-NEXT: or a0, a1, a0
+; RV32-NEXT: ret
+ %cond = icmp ult i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test with memory operands
+define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) {
+; RV64-LABEL: test_ctselect_load:
+; RV64: # %bb.0:
+; RV64-NEXT: lw a1, 0(a1)
+; RV64-NEXT: lw a2, 0(a2)
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: xor a1, a1, a2
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: xor a0, a0, a2
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_load:
+; RV32: # %bb.0:
+; RV32-NEXT: lw a1, 0(a1)
+; RV32-NEXT: lw a2, 0(a2)
+; RV32-NEXT: andi a0, a0, 1
+; RV32-NEXT: neg a3, a0
+; RV32-NEXT: addi a0, a0, -1
+; RV32-NEXT: and a1, a3, a1
+; RV32-NEXT: and a0, a0, a2
+; RV32-NEXT: or a0, a1, a0
+; RV32-NEXT: ret
+ %a = load i32, ptr %p1
+ %b = load i32, ptr %p2
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test nested ctselect calls
+define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
+; RV64-LABEL: test_ctselect_nested:
+; RV64: # %bb.0:
+; RV64-NEXT: xor a2, a2, a3
+; RV64-NEXT: slli a1, a1, 63
+; RV64-NEXT: xor a3, a3, a4
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: srai a1, a1, 63
+; RV64-NEXT: and a1, a2, a1
+; RV64-NEXT: xor a1, a1, a3
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: xor a0, a0, a4
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_nested:
+; RV32: # %bb.0:
+; RV32-NEXT: andi a1, a1, 1
+; RV32-NEXT: andi a0, a0, 1
+; RV32-NEXT: neg a5, a1
+; RV32-NEXT: addi a1, a1, -1
+; RV32-NEXT: and a2, a5, a2
+; RV32-NEXT: neg a5, a0
+; RV32-NEXT: addi a0, a0, -1
+; RV32-NEXT: and a1, a1, a3
+; RV32-NEXT: or a1, a2, a1
+; RV32-NEXT: and a1, a5, a1
+; RV32-NEXT: and a0, a0, a4
+; RV32-NEXT: or a0, a1, a0
+; RV32-NEXT: ret
+ %inner = call i32 @llvm.ct.select.i32(i1 %cond2, i32 %a, i32 %b)
+ %result = call i32 @llvm.ct.select.i32(i1 %cond1, i32 %inner, i32 %c)
+ ret i32 %result
+}
+
+; Declare the intrinsics
+declare i8 @llvm.ct.select.i8(i1, i8, i8)
+declare i16 @llvm.ct.select.i16(i1, i16, i16)
+declare i32 @llvm.ct.select.i32(i1, i32, i32)
+declare i64 @llvm.ct.select.i64(i1, i64, i64)
+declare ptr @llvm.ct.select.p0(i1, ptr, ptr)
diff --git a/llvm/test/CodeGen/X86/ctselect.ll b/llvm/test/CodeGen/X86/ctselect.ll
new file mode 100644
index 0000000000000..095787a5e2a4b
--- /dev/null
+++ b/llvm/test/CodeGen/X86/ctselect.ll
@@ -0,0 +1,779 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+cmov | FileCheck %s --check-prefix=X64
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=+cmov | FileCheck %s --check-prefix=X32
+; RUN: llc < %s -mtriple=i386-unknown-linux-gnu -mattr=-cmov | FileCheck %s --check-prefix=X32-NOCMOV
+
+; Test basic ct.select functionality for scalar types
+
+define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) {
+; X64-LABEL: test_ctselect_i8:
+; X64: # %bb.0:
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: andb $1, %dil
+; X64-NEXT: leal -1(%rdi), %eax
+; X64-NEXT: movl %edi, %ecx
+; X64-NEXT: negb %cl
+; X64-NEXT: andb %sil, %cl
+; X64-NEXT: andb %dl, %al
+; X64-NEXT: orb %cl, %al
+; X64-NEXT: # kill: def $al killed $al killed $eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_i8:
+; X32: # %bb.0:
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb $1, %al
+; X32-NEXT: movl %eax, %ecx
+; X32-NEXT: negb %cl
+; X32-NEXT: andb {{[0-9]+}}(%esp), %cl
+; X32-NEXT: decb %al
+; X32-NEXT: andb {{[0-9]+}}(%esp), %al
+; X32-NEXT: orb %cl, %al
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_i8:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: movl %eax, %ecx
+; X32-NOCMOV-NEXT: negb %cl
+; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %cl
+; X32-NOCMOV-NEXT: decb %al
+; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
+; X32-NOCMOV-NEXT: orb %cl, %al
+; X32-NOCMOV-NEXT: retl
+ %result = call i8 @llvm.ct.select.i8(i1 %cond, i8 %a, i8 %b)
+ ret i8 %result
+}
+
+define i16 @test_ctselect_i16(i1 %cond, i16 %a, i16 %b) {
+; X64-LABEL: test_ctselect_i16:
+; X64: # %bb.0:
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: leal -1(%rdi), %ecx
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: negl %eax
+; X64-NEXT: andl %esi, %eax
+; X64-NEXT: andl %edx, %ecx
+; X64-NEXT: orl %ecx, %eax
+; X64-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_i16:
+; X32: # %bb.0:
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andl $1, %eax
+; X32-NEXT: leal -1(%eax), %ecx
+; X32-NEXT: andw {{[0-9]+}}(%esp), %cx
+; X32-NEXT: negl %eax
+; X32-NEXT: andw {{[0-9]+}}(%esp), %ax
+; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: # kill: def $ax killed $ax killed $eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_i16:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: leal -1(%eax), %ecx
+; X32-NOCMOV-NEXT: andw {{[0-9]+}}(%esp), %cx
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andw {{[0-9]+}}(%esp), %ax
+; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax killed $eax
+; X32-NOCMOV-NEXT: retl
+ %result = call i16 @llvm.ct.select.i16(i1 %cond, i16 %a, i16 %b)
+ ret i16 %result
+}
+
+define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) {
+; X64-LABEL: test_ctselect_i32:
+; X64: # %bb.0:
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: leal -1(%rdi), %eax
+; X64-NEXT: movl %edi, %ecx
+; X64-NEXT: negl %ecx
+; X64-NEXT: andl %esi, %ecx
+; X64-NEXT: andl %edx, %eax
+; X64-NEXT: orl %ecx, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_i32:
+; X32: # %bb.0:
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andl $1, %eax
+; X32-NEXT: movl %eax, %ecx
+; X32-NEXT: negl %ecx
+; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: decl %eax
+; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_i32:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: movl %eax, %ecx
+; X32-NOCMOV-NEXT: negl %ecx
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: decl %eax
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: retl
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) {
+; X64-LABEL: test_ctselect_i64:
+; X64: # %bb.0:
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: leaq -1(%rdi), %rax
+; X64-NEXT: negq %rdi
+; X64-NEXT: andq %rsi, %rdi
+; X64-NEXT: andq %rdx, %rax
+; X64-NEXT: orq %rdi, %rax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_i64:
+; X32: # %bb.0:
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: .cfi_offset %esi, -8
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %edx, %eax
+; X32-NEXT: andl $1, %esi
+; X32-NEXT: negl %esi
+; X32-NEXT: andl %esi, %eax
+; X32-NEXT: xorl %edx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: andl %esi, %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_i64:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: .cfi_offset %esi, -8
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %edx, %eax
+; X32-NOCMOV-NEXT: andl $1, %esi
+; X32-NOCMOV-NEXT: negl %esi
+; X32-NOCMOV-NEXT: andl %esi, %eax
+; X32-NOCMOV-NEXT: xorl %edx, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: andl %esi, %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; X32-NOCMOV-NEXT: retl
+ %result = call i64 @llvm.ct.select.i64(i1 %cond, i64 %a, i64 %b)
+ ret i64 %result
+}
+
+define float @test_ctselect_f32(i1 %cond, float %a, float %b) {
+; X64-LABEL: test_ctselect_f32:
+; X64: # %bb.0:
+; X64-NEXT: movd %xmm1, %eax
+; X64-NEXT: movd %xmm0, %ecx
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: movl %edi, %edx
+; X64-NEXT: negl %edx
+; X64-NEXT: andl %ecx, %edx
+; X64-NEXT: decl %edi
+; X64-NEXT: andl %eax, %edi
+; X64-NEXT: orl %edx, %edi
+; X64-NEXT: movd %edi, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_f32:
+; X32: # %bb.0:
+; X32-NEXT: pushl %eax
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andl $1, %eax
+; X32-NEXT: movl %eax, %ecx
+; X32-NEXT: negl %ecx
+; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: decl %eax
+; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: flds (%esp)
+; X32-NEXT: popl %eax
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_f32:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %eax
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: movl %eax, %ecx
+; X32-NOCMOV-NEXT: negl %ecx
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: decl %eax
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: movl %eax, (%esp)
+; X32-NOCMOV-NEXT: flds (%esp)
+; X32-NOCMOV-NEXT: popl %eax
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; X32-NOCMOV-NEXT: retl
+ %result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
+ ret float %result
+}
+
+define double @test_ctselect_f64(i1 %cond, double %a, double %b) {
+; X64-LABEL: test_ctselect_f64:
+; X64: # %bb.0:
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: movq %xmm1, %rax
+; X64-NEXT: movq %xmm0, %rcx
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: movq %rdi, %rdx
+; X64-NEXT: negq %rdx
+; X64-NEXT: andq %rcx, %rdx
+; X64-NEXT: decq %rdi
+; X64-NEXT: andq %rax, %rdi
+; X64-NEXT: orq %rdx, %rdi
+; X64-NEXT: movq %rdi, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_f64:
+; X32: # %bb.0:
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: subl $8, %esp
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: .cfi_offset %esi, -8
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl %edx, %esi
+; X32-NEXT: andl $1, %ecx
+; X32-NEXT: negl %ecx
+; X32-NEXT: andl %ecx, %esi
+; X32-NEXT: xorl %edx, %esi
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %eax, %edx
+; X32-NEXT: andl %ecx, %edx
+; X32-NEXT: xorl %eax, %edx
+; X32-NEXT: movl %edx, (%esp)
+; X32-NEXT: fldl (%esp)
+; X32-NEXT: addl $8, %esp
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_f64:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: subl $8, %esp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; X32-NOCMOV-NEXT: .cfi_offset %esi, -8
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl %edx, %esi
+; X32-NOCMOV-NEXT: andl $1, %ecx
+; X32-NOCMOV-NEXT: negl %ecx
+; X32-NOCMOV-NEXT: andl %ecx, %esi
+; X32-NOCMOV-NEXT: xorl %edx, %esi
+; X32-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %eax, %edx
+; X32-NOCMOV-NEXT: andl %ecx, %edx
+; X32-NOCMOV-NEXT: xorl %eax, %edx
+; X32-NOCMOV-NEXT: movl %edx, (%esp)
+; X32-NOCMOV-NEXT: fldl (%esp)
+; X32-NOCMOV-NEXT: addl $8, %esp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; X32-NOCMOV-NEXT: retl
+ %result = call double @llvm.ct.select.f64(i1 %cond, double %a, double %b)
+ ret double %result
+}
+
+define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) {
+; X64-LABEL: test_ctselect_ptr:
+; X64: # %bb.0:
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: leaq -1(%rdi), %rax
+; X64-NEXT: negq %rdi
+; X64-NEXT: andq %rsi, %rdi
+; X64-NEXT: andq %rdx, %rax
+; X64-NEXT: orq %rdi, %rax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_ptr:
+; X32: # %bb.0:
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andl $1, %eax
+; X32-NEXT: movl %eax, %ecx
+; X32-NEXT: negl %ecx
+; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: decl %eax
+; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_ptr:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: movl %eax, %ecx
+; X32-NOCMOV-NEXT: negl %ecx
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: decl %eax
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: retl
+ %result = call ptr @llvm.ct.select.p0(i1 %cond, ptr %a, ptr %b)
+ ret ptr %result
+}
+
+; Test with constant conditions
+define i32 @test_ctselect_const_true(i32 %a, i32 %b) {
+; X64-LABEL: test_ctselect_const_true:
+; X64: # %bb.0:
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_const_true:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_const_true:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: retl
+ %result = call i32 @llvm.ct.select.i32(i1 true, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+define i32 @test_ctselect_const_false(i32 %a, i32 %b) {
+; X64-LABEL: test_ctselect_const_false:
+; X64: # %bb.0:
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_const_false:
+; X32: # %bb.0:
+; X32-NEXT: xorl %eax, %eax
+; X32-NEXT: orl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_const_false:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: xorl %eax, %eax
+; X32-NOCMOV-NEXT: orl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: retl
+ %result = call i32 @llvm.ct.select.i32(i1 false, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test with comparison conditions
+define i32 @test_ctselect_icmp_eq(i32 %x, i32 %y, i32 %a, i32 %b) {
+; X64-LABEL: test_ctselect_icmp_eq:
+; X64: # %bb.0:
+; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: cmpl %esi, %edi
+; X64-NEXT: sete %al
+; X64-NEXT: movl %eax, %esi
+; X64-NEXT: negl %esi
+; X64-NEXT: andl %edx, %esi
+; X64-NEXT: decl %eax
+; X64-NEXT: andl %ecx, %eax
+; X64-NEXT: orl %esi, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_icmp_eq:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorl %eax, %eax
+; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: sete %al
+; X32-NEXT: movl %eax, %ecx
+; X32-NEXT: negl %ecx
+; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: decl %eax
+; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_icmp_eq:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorl %eax, %eax
+; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: sete %al
+; X32-NOCMOV-NEXT: movl %eax, %ecx
+; X32-NOCMOV-NEXT: negl %ecx
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: decl %eax
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: retl
+ %cond = icmp eq i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+define i32 @test_ctselect_icmp_ne(i32 %x, i32 %y, i32 %a, i32 %b) {
+; X64-LABEL: test_ctselect_icmp_ne:
+; X64: # %bb.0:
+; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: cmpl %esi, %edi
+; X64-NEXT: setne %al
+; X64-NEXT: movl %eax, %esi
+; X64-NEXT: negl %esi
+; X64-NEXT: andl %edx, %esi
+; X64-NEXT: decl %eax
+; X64-NEXT: andl %ecx, %eax
+; X64-NEXT: orl %esi, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_icmp_ne:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorl %eax, %eax
+; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: setne %al
+; X32-NEXT: movl %eax, %ecx
+; X32-NEXT: negl %ecx
+; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: decl %eax
+; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_icmp_ne:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorl %eax, %eax
+; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: setne %al
+; X32-NOCMOV-NEXT: movl %eax, %ecx
+; X32-NOCMOV-NEXT: negl %ecx
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: decl %eax
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: retl
+ %cond = icmp ne i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+define i32 @test_ctselect_icmp_slt(i32 %x, i32 %y, i32 %a, i32 %b) {
+; X64-LABEL: test_ctselect_icmp_slt:
+; X64: # %bb.0:
+; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: cmpl %esi, %edi
+; X64-NEXT: setl %al
+; X64-NEXT: movl %eax, %esi
+; X64-NEXT: negl %esi
+; X64-NEXT: andl %edx, %esi
+; X64-NEXT: decl %eax
+; X64-NEXT: andl %ecx, %eax
+; X64-NEXT: orl %esi, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_icmp_slt:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorl %eax, %eax
+; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: setl %al
+; X32-NEXT: movl %eax, %ecx
+; X32-NEXT: negl %ecx
+; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: decl %eax
+; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_icmp_slt:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorl %eax, %eax
+; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: setl %al
+; X32-NOCMOV-NEXT: movl %eax, %ecx
+; X32-NOCMOV-NEXT: negl %ecx
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: decl %eax
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: retl
+ %cond = icmp slt i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+define i32 @test_ctselect_icmp_ult(i32 %x, i32 %y, i32 %a, i32 %b) {
+; X64-LABEL: test_ctselect_icmp_ult:
+; X64: # %bb.0:
+; X64-NEXT: xorl %eax, %eax
+; X64-NEXT: cmpl %esi, %edi
+; X64-NEXT: sbbl %eax, %eax
+; X64-NEXT: andl %eax, %edx
+; X64-NEXT: notl %eax
+; X64-NEXT: andl %ecx, %eax
+; X64-NEXT: orl %edx, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_icmp_ult:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorl %eax, %eax
+; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: sbbl %eax, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: andl %eax, %ecx
+; X32-NEXT: notl %eax
+; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_icmp_ult:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorl %eax, %eax
+; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: sbbl %eax, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: andl %eax, %ecx
+; X32-NOCMOV-NEXT: notl %eax
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: retl
+ %cond = icmp ult i32 %x, %y
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) {
+; X64-LABEL: test_ctselect_fcmp_oeq:
+; X64: # %bb.0:
+; X64-NEXT: movd %xmm3, %eax
+; X64-NEXT: cmpeqss %xmm1, %xmm0
+; X64-NEXT: movd %xmm0, %ecx
+; X64-NEXT: pand %xmm2, %xmm0
+; X64-NEXT: movd %xmm0, %edx
+; X64-NEXT: notl %ecx
+; X64-NEXT: andl %eax, %ecx
+; X64-NEXT: orl %edx, %ecx
+; X64-NEXT: movd %ecx, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_fcmp_oeq:
+; X32: # %bb.0:
+; X32-NEXT: pushl %eax
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fucompi %st(1), %st
+; X32-NEXT: fstp %st(0)
+; X32-NEXT: setnp %al
+; X32-NEXT: sete %cl
+; X32-NEXT: andb %al, %cl
+; X32-NEXT: movzbl %cl, %eax
+; X32-NEXT: movl %eax, %ecx
+; X32-NEXT: negl %ecx
+; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: decl %eax
+; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: flds (%esp)
+; X32-NEXT: popl %eax
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_fcmp_oeq:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %eax
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fucompp
+; X32-NOCMOV-NEXT: fnstsw %ax
+; X32-NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
+; X32-NOCMOV-NEXT: sahf
+; X32-NOCMOV-NEXT: setnp %al
+; X32-NOCMOV-NEXT: sete %cl
+; X32-NOCMOV-NEXT: andb %al, %cl
+; X32-NOCMOV-NEXT: movzbl %cl, %eax
+; X32-NOCMOV-NEXT: movl %eax, %ecx
+; X32-NOCMOV-NEXT: negl %ecx
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: decl %eax
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: movl %eax, (%esp)
+; X32-NOCMOV-NEXT: flds (%esp)
+; X32-NOCMOV-NEXT: popl %eax
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; X32-NOCMOV-NEXT: retl
+ %cond = fcmp oeq float %x, %y
+ %result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
+ ret float %result
+}
+
+; Test with memory operands
+define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) {
+; X64-LABEL: test_ctselect_load:
+; X64: # %bb.0:
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: leal -1(%rdi), %eax
+; X64-NEXT: movl %edi, %ecx
+; X64-NEXT: negl %ecx
+; X64-NEXT: andl (%rsi), %ecx
+; X64-NEXT: andl (%rdx), %eax
+; X64-NEXT: orl %ecx, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_load:
+; X32: # %bb.0:
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: .cfi_offset %esi, -8
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andl $1, %eax
+; X32-NEXT: movl %eax, %esi
+; X32-NEXT: negl %esi
+; X32-NEXT: andl (%edx), %esi
+; X32-NEXT: decl %eax
+; X32-NEXT: andl (%ecx), %eax
+; X32-NEXT: orl %esi, %eax
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_load:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: .cfi_offset %esi, -8
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: movl %eax, %esi
+; X32-NOCMOV-NEXT: negl %esi
+; X32-NOCMOV-NEXT: andl (%edx), %esi
+; X32-NOCMOV-NEXT: decl %eax
+; X32-NOCMOV-NEXT: andl (%ecx), %eax
+; X32-NOCMOV-NEXT: orl %esi, %eax
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; X32-NOCMOV-NEXT: retl
+ %a = load i32, ptr %p1
+ %b = load i32, ptr %p2
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test nested ctselect calls
+define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
+; X64-LABEL: test_ctselect_nested:
+; X64: # %bb.0:
+; X64-NEXT: # kill: def $esi killed $esi def $rsi
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: andl $1, %esi
+; X64-NEXT: leal -1(%rsi), %r9d
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: negl %eax
+; X64-NEXT: andl %edx, %eax
+; X64-NEXT: andl %ecx, %r9d
+; X64-NEXT: orl %eax, %r9d
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: leal -1(%rdi), %eax
+; X64-NEXT: movl %edi, %ecx
+; X64-NEXT: negl %ecx
+; X64-NEXT: andl %r9d, %ecx
+; X64-NEXT: andl %r8d, %eax
+; X64-NEXT: orl %ecx, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_nested:
+; X32: # %bb.0:
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: andl $1, %ecx
+; X32-NEXT: movl %ecx, %edx
+; X32-NEXT: negl %edx
+; X32-NEXT: andl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: decl %ecx
+; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: orl %edx, %ecx
+; X32-NEXT: andl $1, %eax
+; X32-NEXT: movl %eax, %edx
+; X32-NEXT: negl %edx
+; X32-NEXT: andl %ecx, %edx
+; X32-NEXT: decl %eax
+; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: orl %edx, %eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_nested:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: andl $1, %ecx
+; X32-NOCMOV-NEXT: movl %ecx, %edx
+; X32-NOCMOV-NEXT: negl %edx
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: decl %ecx
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: orl %edx, %ecx
+; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: movl %eax, %edx
+; X32-NOCMOV-NEXT: negl %edx
+; X32-NOCMOV-NEXT: andl %ecx, %edx
+; X32-NOCMOV-NEXT: decl %eax
+; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: orl %edx, %eax
+; X32-NOCMOV-NEXT: retl
+ %inner = call i32 @llvm.ct.select.i32(i1 %cond2, i32 %a, i32 %b)
+ %result = call i32 @llvm.ct.select.i32(i1 %cond1, i32 %inner, i32 %c)
+ ret i32 %result
+}
+
+; Declare the intrinsics
+declare i8 @llvm.ct.select.i8(i1, i8, i8)
+declare i16 @llvm.ct.select.i16(i1, i16, i16)
+declare i32 @llvm.ct.select.i32(i1, i32, i32)
+declare i64 @llvm.ct.select.i64(i1, i64, i64)
+declare float @llvm.ct.select.f32(i1, float, float)
+declare double @llvm.ct.select.f64(i1, double, double)
+declare ptr @llvm.ct.select.p0(i1, ptr, ptr)
>From 49f0d2a60fc05f86c22a8960fc5e8a808fa8511d Mon Sep 17 00:00:00 2001
From: Akshay K <iit.akshay at gmail.com>
Date: Tue, 10 Feb 2026 09:43:36 -0500
Subject: [PATCH 02/10] [LLVM][CodeGen] Improve CTSELECT fallback lowering and
target support modeling (#179395)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
This pull request refactors and improves the **fallback handling** of
constant-time select (CTSELECT) in LLVM’s code generation
infrastructure. The changes clarify semantics, simplify
target-capability checks, and improve the correctness and
maintainability of fallback lowering, without changing the intended
constant-time guarantees.
- **CTSELECT semantics**
- Clarified documentation for the `CTSELECT` node to explicitly describe
its operands and its role as the lowering target for the constant-time
select intrinsic.
- **TargetLowering cleanup**
- Removed CTSELECT-specific entries from `SelectSupportKind`.
- Introduced a dedicated `isCtSelectSupported(EVT)` hook to cleanly
separate ISA select support from constant-time guarantees.
- Updated intrinsic lowering to use this new capability check.
- **Fallback implementation improvements**
- Updated the generic SelectionDAG fallback lowering to use the
canonical bitwise formulation:
```
F ^ ((T ^ F) & Mask)
```
- Simplified handling of vector splats and bitcasting in the fallback
path.
- **DAGCombiner refactor**
- Renamed and clarified CTSELECT-related DAGCombiner code paths for
improved readability.
- **Miscellaneous cleanups**
- Improved documentation headers for constant-time intrinsics.
- Removed unused flags and minor nits uncovered during review.
These changes primarily affect the **generic fallback path** used when
targets do not provide specialized CTSELECT lowering. Target-specific
implementations are handled in follow-up PRs
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 4 +-
llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 2 -
llvm/include/llvm/CodeGen/TargetLowering.h | 11 ++--
llvm/include/llvm/IR/Intrinsics.td | 2 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 8 +--
.../SelectionDAG/SelectionDAGBuilder.cpp | 61 ++++++-------------
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 2 +
llvm/lib/Target/ARM/ARMISelLowering.h | 2 +
llvm/lib/Target/X86/X86ISelLowering.h | 2 +
9 files changed, 35 insertions(+), 59 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 366892fe05e8c..f9573f797b6a8 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -805,8 +805,8 @@ enum NodeType {
/// i1 then the high bits must conform to getBooleanContents.
SELECT,
- /// Constant-time Select, implemented with CMOV instruction. This is used to
- /// implement constant-time select.
+ /// CTSELECT(Cond, TrueVal, FalseVal). Cond is i1 and the value operands must
+ /// have the same type. Used to lower the constant-time select intrinsic.
CTSELECT,
/// Select with a vector condition (op #0) and two vector operands (ops #1
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index ec53ea02b06e6..fa8e7c2e612ee 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -432,8 +432,6 @@ struct SDNodeFlags {
FastMathFlags = NoNaNs | NoInfs | NoSignedZeros | AllowReciprocal |
AllowContract | ApproximateFuncs | AllowReassociation,
- // Flag for disabling optimization
- NoMerge = 1 << 15,
};
/// Default constructor turns off all optimization flags.
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index be332c060ee79..73b9291b646be 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -265,10 +265,6 @@ class LLVM_ABI TargetLoweringBase {
// and vector values (ex: cmov).
VectorMaskSelect, // The target supports vector selects with a vector
// mask (ex: x86 blends).
- CtSelect, // The target implements a custom constant-time select.
- ScalarCondVectorValCtSelect, // The target supports selects with a scalar
- // condition and vector values.
- VectorMaskValCtSelect, // The target supports vector selects with a vector
};
/// Enum that specifies what an atomic load/AtomicRMWInst is expanded
@@ -499,9 +495,10 @@ class LLVM_ABI TargetLoweringBase {
MachineMemOperand::Flags
getVPIntrinsicMemOperandFlags(const VPIntrinsic &VPIntrin) const;
- virtual bool isSelectSupported(SelectSupportKind kind) const {
- return kind != CtSelect;
- }
+ virtual bool isSelectSupported(SelectSupportKind kind) const { return true; }
+
+ /// Return true if the target has custom lowering for constant-time select.
+ virtual bool isCtSelectSupported(EVT VT) const { return false; }
/// Return true if the @llvm.get.active.lane.mask intrinsic should be expanded
/// using generic code in SelectionDAGBuilder.
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index e878a039b3b6a..96bca07cbb6dd 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2043,7 +2043,7 @@ def int_coro_subfn_addr : DefaultAttrsIntrinsic<
[IntrReadMem, IntrArgMemOnly, ReadOnly<ArgIndex<0>>,
NoCapture<ArgIndex<0>>]>;
-///===-------------------------- Constant Time Intrinsics ----------------------===//
+///===---------------------- Constant Time Intrinsics ----------------------===//
//
// Intrinsic to support constant time select
def int_ct_select
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 8e32fe90498e6..6b9e418de53cb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -484,7 +484,8 @@ namespace {
SDValue visitCTTZ_ZERO_POISON(SDNode *N);
SDValue visitCTPOP(SDNode *N);
SDValue visitSELECT(SDNode *N);
- SDValue visitCTSELECT(SDNode *N);
+ // visit CTSELECT Node
+ SDValue visitConstantTimeSelect(SDNode *N);
SDValue visitVSELECT(SDNode *N);
SDValue visitVP_SELECT(SDNode *N);
SDValue visitSELECT_CC(SDNode *N);
@@ -1964,7 +1965,6 @@ void DAGCombiner::Run(CombineLevel AtLevel) {
}
SDValue DAGCombiner::visit(SDNode *N) {
-
// clang-format off
switch (N->getOpcode()) {
default: break;
@@ -2041,7 +2041,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
case ISD::CTTZ_ZERO_POISON: return visitCTTZ_ZERO_POISON(N);
case ISD::CTPOP: return visitCTPOP(N);
case ISD::SELECT: return visitSELECT(N);
- case ISD::CTSELECT: return visitCTSELECT(N);
+ case ISD::CTSELECT: return visitConstantTimeSelect(N);
case ISD::VSELECT: return visitVSELECT(N);
case ISD::SELECT_CC: return visitSELECT_CC(N);
case ISD::SETCC: return visitSETCC(N);
@@ -13436,7 +13436,7 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
return SDValue();
}
-SDValue DAGCombiner::visitCTSELECT(SDNode *N) {
+SDValue DAGCombiner::visitConstantTimeSelect(SDNode *N) {
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
SDValue N2 = N->getOperand(2);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 1f2789c8c22e4..c18199b625865 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6694,7 +6694,10 @@ void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
/// Fallback implementation for constant-time select using DAG chaining.
/// This implementation uses data dependencies through virtual registers to
-/// prevent optimizations from breaking the constant-time property.
+/// prevent optimizations from breaking the constant-time property. It is a
+/// best-effort safeguard; for stronger guarantees we prefer target-specific
+/// lowering pipelines that preserve the select pattern by construction.
+///
/// It handles scalars, vectors (fixed and scalable), and floating-point types.
SDValue SelectionDAGBuilder::createProtectedCtSelectFallback(
SelectionDAG &DAG, const SDLoc &DL, SDValue Cond, SDValue T, SDValue F,
@@ -6711,28 +6714,12 @@ SDValue SelectionDAGBuilder::createProtectedCtSelectFallback(
if (VT.isVector() && !Cond.getValueType().isVector()) {
ElementCount NumElems = VT.getVectorElementCount();
EVT CondVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElems);
-
- if (VT.isScalableVector()) {
- Cond = DAG.getSplatVector(CondVT, DL, Cond);
- } else {
- Cond = DAG.getSplatBuildVector(CondVT, DL, Cond);
- }
+ Cond = DAG.getSplat(CondVT, DL, Cond);
}
// Handle floating-point types: bitcast to integer for bitwise operations
if (VT.isFloatingPoint()) {
- if (VT.isVector()) {
- // float vector -> int vector
- EVT ElemVT = VT.getVectorElementType();
- unsigned int ElemBitWidth = ElemVT.getScalarSizeInBits();
- EVT IntElemVT = EVT::getIntegerVT(*DAG.getContext(), ElemBitWidth);
-
- WorkingVT = EVT::getVectorVT(*DAG.getContext(), IntElemVT,
- VT.getVectorElementCount());
- } else {
- WorkingVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
- }
-
+ WorkingVT = VT.changeTypeToInteger();
WorkingT = DAG.getBitcast(WorkingVT, T);
WorkingF = DAG.getBitcast(WorkingVT, F);
}
@@ -6740,24 +6727,10 @@ SDValue SelectionDAGBuilder::createProtectedCtSelectFallback(
// Create mask: sign-extend condition to all bits
SDValue Mask = DAG.getSExtOrTrunc(Cond, DL, WorkingVT);
- // Create all-ones constant for inversion
- SDValue AllOnes;
- if (WorkingVT.isScalableVector()) {
- unsigned BitWidth = WorkingVT.getScalarSizeInBits();
- APInt AllOnesVal = APInt::getAllOnes(BitWidth);
- SDValue ScalarAllOnes =
- DAG.getConstant(AllOnesVal, DL, WorkingVT.getScalarType());
- AllOnes = DAG.getSplatVector(WorkingVT, DL, ScalarAllOnes);
- } else {
- AllOnes = DAG.getAllOnesConstant(DL, WorkingVT);
- }
-
- // Invert mask for false value
- SDValue Invert = DAG.getNode(ISD::XOR, DL, WorkingVT, Mask, AllOnes);
-
- // Compute: (T & Mask) | (F & ~Mask)
+ // Compute: F ^ ((T ^ F) & Mask)
// This is constant-time because both branches are always computed
- SDValue TM = DAG.getNode(ISD::AND, DL, WorkingVT, Mask, WorkingT);
+ SDValue XorTF = DAG.getNode(ISD::XOR, DL, WorkingVT, WorkingT, WorkingF);
+ SDValue TM = DAG.getNode(ISD::AND, DL, WorkingVT, XorTF, Mask);
// DAG chaining: create data dependency through virtual register
// This prevents optimizations from reordering or eliminating operations
@@ -6765,10 +6738,15 @@ SDValue SelectionDAGBuilder::createProtectedCtSelectFallback(
bool CanUseChaining = false;
if (!WorkingVT.isScalableVector()) {
- // For fixed-size vectors and scalars, check if type is legal
+ // For fixed-size vectors and scalars, chaining is a best-effort hardening
+ // step. The CT guarantee comes from the dataflow-only select
+ // pattern (both sides computed, no control-flow). Chaining only adds an
+ // extra dependency to discourage later combines.
CanUseChaining = TLI.isTypeLegal(WorkingVT.getSimpleVT());
} else {
- // For scalable vectors, disable chaining (conservative approach)
+ // For scalable vectors, skip chaining because there is no stable register
+ // class to copy through. CT behavior still relies on the masking/select
+ // pattern above.
CanUseChaining = false;
}
@@ -6780,8 +6758,7 @@ SDValue SelectionDAGBuilder::createProtectedCtSelectFallback(
TM = DAG.getCopyFromReg(Chain, DL, TMReg, WorkingVT);
}
- SDValue FM = DAG.getNode(ISD::AND, DL, WorkingVT, Invert, WorkingF);
- SDValue Result = DAG.getNode(ISD::OR, DL, WorkingVT, TM, FM);
+ SDValue Result = DAG.getNode(ISD::XOR, DL, WorkingVT, WorkingF, TM);
// Convert back to original type if needed
if (WorkingVT != VT) {
@@ -7013,9 +6990,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
assert(!CondVT.isVector() && "Vector type cond not supported yet");
// Handle scalar types
- if (TLI.isSelectSupported(
- TargetLoweringBase::SelectSupportKind::CtSelect) &&
- !CondVT.isVector()) {
+ if (TLI.isCtSelectSupported(VT) && !CondVT.isVector()) {
SDValue Result = DAG.getNode(ISD::CTSELECT, DL, VT, Cond, A, B);
setValue(&I, Result);
return;
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 5cc4e9fa1b063..6a453ba0df038 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -167,6 +167,8 @@ class AArch64TargetLowering : public TargetLowering {
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
EVT VT) const override;
+ bool isCtSelectSupported(EVT VT) const override { return false; }
+
SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const;
MachineBasicBlock *EmitF128CSEL(MachineInstr &MI,
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 10f5442d7429b..72d0acf2dc7c1 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -119,6 +119,8 @@ class VectorType;
return (Kind != ScalarCondVectorVal);
}
+ bool isCtSelectSupported(EVT VT) const override { return false; }
+
bool isReadOnly(const GlobalValue *GV) const;
/// getSetCCResultType - Return the value type to use for ISD::SETCC.
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 798050028c15a..7d041dee6e345 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -111,6 +111,8 @@ namespace llvm {
unsigned getJumpTableEncoding() const override;
bool useSoftFloat() const override;
+ bool isCtSelectSupported(EVT VT) const override { return false; }
+
void markLibCallAttributes(MachineFunction *MF, unsigned CC,
ArgListTy &Args) const override;
>From 4214b01a612e4743f089d684a71d51789bd677dc Mon Sep 17 00:00:00 2001
From: Akshay K <iit.akshay at gmail.com>
Date: Wed, 11 Feb 2026 14:08:35 -0500
Subject: [PATCH 03/10] [LLVM][ConstantTime] Strengthen constant-time handling
of CTSELECT in DAG combine and legalization (#180883)
This PR improves the handling of CTSELECT in SelectionDAG to strengthen
constant-time guarantees and ensure correctness during DAG combining and
type legalization.
Key changes:
- Introduce a dedicated visitConstantTimeSelect routine that performs
only CT-safe optimizations (e.g., negated condition canonicalization and
i1 AND/OR nesting merges) while avoiding unsafe generic rewrites.
- Remove aggressive generic combinations that could violate
constant-time intent.
- Add specialized type legalization routines (SplitRes_CTSELECT,
PromoteFloatRes_CTSELECT, SoftPromoteHalfRes_CTSELECT) to preserve
constant-time semantics during result splitting and promotion.
- Handling of vector expansion with constant-time bitwise blend
lowering.
- Remove redundant X86/RISCV tests and add vector coverage while
maintaining critical code path validation.
These changes improve correctness, maintainability, and the end-to-end
constant-time behavior of the fallback implementation across the
optimization and legalization stages.
---
llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 1 -
llvm/include/llvm/CodeGen/TargetLowering.h | 2 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 105 +-
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 56 +-
.../SelectionDAG/LegalizeFloatTypes.cpp | 11 +-
.../SelectionDAG/LegalizeIntegerTypes.cpp | 2 +-
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 1 +
.../SelectionDAG/LegalizeTypesGeneric.cpp | 14 +-
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 1 -
llvm/test/CodeGen/RISCV/ctselect-fallback.ll | 698 +++++++--
llvm/test/CodeGen/X86/ctselect.ll | 1268 ++++++++++++-----
11 files changed, 1560 insertions(+), 599 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
index fa8e7c2e612ee..990b649fafe19 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h
@@ -431,7 +431,6 @@ struct SDNodeFlags {
NonNeg | NoNaNs | NoInfs | SameSign | InBounds,
FastMathFlags = NoNaNs | NoInfs | NoSignedZeros | AllowReciprocal |
AllowContract | ApproximateFuncs | AllowReassociation,
-
};
/// Default constructor turns off all optimization flags.
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 73b9291b646be..760f06ff1442b 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -263,7 +263,7 @@ class LLVM_ABI TargetLoweringBase {
ScalarValSelect, // The target supports scalar selects (ex: cmov).
ScalarCondVectorVal, // The target supports selects with a scalar condition
// and vector values (ex: cmov).
- VectorMaskSelect, // The target supports vector selects with a vector
+ VectorMaskSelect // The target supports vector selects with a vector
// mask (ex: x86 blends).
};
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 6b9e418de53cb..0b15066cef57e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -6219,7 +6219,6 @@ static SDValue isSaturatingMinMax(SDValue N0, SDValue N1, SDValue N2,
N0CC = cast<CondCodeSDNode>(N0.getOperand(4))->get();
break;
case ISD::SELECT:
- case ISD::CTSELECT:
case ISD::VSELECT:
if (N0.getOperand(0).getOpcode() != ISD::SETCC)
return SDValue();
@@ -12930,8 +12929,7 @@ template <class MatchContextClass>
static SDValue foldBoolSelectToLogic(SDNode *N, const SDLoc &DL,
SelectionDAG &DAG) {
assert((N->getOpcode() == ISD::SELECT || N->getOpcode() == ISD::VSELECT ||
- N->getOpcode() == ISD::VP_SELECT ||
- N->getOpcode() == ISD::CTSELECT) &&
+ N->getOpcode() == ISD::VP_SELECT) &&
"Expected a (v)(vp.)(ct) select");
SDValue Cond = N->getOperand(0);
SDValue T = N->getOperand(1), F = N->getOperand(2);
@@ -13436,6 +13434,12 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
return SDValue();
}
+// Keep CTSELECT combines deliberately conservative to preserve constant-time
+// intent across generic DAG combines. We only accept:
+// - canonicalization of negated conditions (flip true/false operands), and
+// - i1 CTSELECT nesting merges via AND/OR that keep the result as CTSELECT.
+// Broader rewrites should be done in target-specific lowering when stronger
+// guarantees about legality and constant-time preservation are available.
SDValue DAGCombiner::visitConstantTimeSelect(SDNode *N) {
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
@@ -13445,10 +13449,10 @@ SDValue DAGCombiner::visitConstantTimeSelect(SDNode *N) {
SDLoc DL(N);
SDNodeFlags Flags = N->getFlags();
- if (SDValue V = foldBoolSelectToLogic<EmptyMatchContext>(N, DL, DAG))
- return V;
-
// ctselect (not Cond), N1, N2 -> ctselect Cond, N2, N1
+ // This is a CT-safe canonicalization: flip negated condition by swapping
+ // arms. extractBooleanFlip only matches boolean xor-with-1, so this preserves
+ // dataflow semantics and does not introduce data-dependent control flow.
if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) {
SDValue SelectOp = DAG.getNode(ISD::CTSELECT, DL, VT, F, N2, N1);
SelectOp->setFlags(Flags);
@@ -13456,82 +13460,43 @@ SDValue DAGCombiner::visitConstantTimeSelect(SDNode *N) {
}
if (VT0 == MVT::i1) {
- // The code in this block deals with the following 2 equivalences:
- // select(C0|C1, x, y) <=> select(C0, x, select(C1, x, y))
- // select(C0&C1, x, y) <=> select(C0, select(C1, x, y), y)
- // The target can specify its preferred form with the
- // shouldNormalizeToSelectSequence() callback. However we always transform
- // to the right anyway if we find the inner select exists in the DAG anyway
- // and we always transform to the left side if we know that we can further
- // optimize the combination of the conditions.
- bool normalizeToSequence =
- TLI.shouldNormalizeToSelectSequence(*DAG.getContext(), VT);
- // ctselect (and Cond0, Cond1), X, Y
- // -> ctselect Cond0, (ctselect Cond1, X, Y), Y
- if (N0->getOpcode() == ISD::AND && N0->hasOneUse()) {
- SDValue Cond0 = N0->getOperand(0);
- SDValue Cond1 = N0->getOperand(1);
- SDValue InnerSelect = DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(),
- Cond1, N1, N2, Flags);
- if (normalizeToSequence || !InnerSelect.use_empty())
- return DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), Cond0,
- InnerSelect, N2, Flags);
- // Cleanup on failure.
- if (InnerSelect.use_empty())
- recursivelyDeleteUnusedNodes(InnerSelect.getNode());
- }
- // ctselect (or Cond0, Cond1), X, Y -> ctselect Cond0, X, (ctselect Cond1,
- // X, Y)
- if (N0->getOpcode() == ISD::OR && N0->hasOneUse()) {
- SDValue Cond0 = N0->getOperand(0);
- SDValue Cond1 = N0->getOperand(1);
- SDValue InnerSelect = DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(),
- Cond1, N1, N2, Flags);
- if (normalizeToSequence || !InnerSelect.use_empty())
- return DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), Cond0, N1,
- InnerSelect, Flags);
- // Cleanup on failure.
- if (InnerSelect.use_empty())
- recursivelyDeleteUnusedNodes(InnerSelect.getNode());
- }
-
- // ctselect Cond0, (ctselect Cond1, X, Y), Y -> ctselect (and Cond0, Cond1),
- // X, Y
+ // Nested CTSELECT merging optimizations for i1 conditions.
+ // These are CT-safe because:
+ // 1. AND/OR are bitwise operations that execute in constant time
+ // 2. The optimization combines two sequential CTSELECTs into one,
+ // reducing the total number of constant-time operations without
+ // changing semantics
+ // 3. No data-dependent branches or memory accesses are introduced
+ //
+ // ctselect C0, (ctselect C1, X, Y), Y -> ctselect (C0 & C1), X, Y
+ // Semantic equivalence: If C0 is true, evaluate inner select (C1 ? X :
+ // Y). If C0 is false, choose Y. This is equivalent to (C0 && C1) ? X : Y.
if (N1->getOpcode() == ISD::CTSELECT && N1->hasOneUse()) {
SDValue N1_0 = N1->getOperand(0);
SDValue N1_1 = N1->getOperand(1);
SDValue N1_2 = N1->getOperand(2);
if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) {
- // Create the actual and node if we can generate good code for it.
- if (!normalizeToSequence) {
- SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
- return DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), And, N1_1,
- N2, Flags);
- }
- // Otherwise see if we can optimize the "and" to a better pattern.
- if (SDValue Combined = visitANDLike(N0, N1_0, N)) {
- return DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), Combined,
- N1_1, N2, Flags);
- }
+ SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
+ SDValue SelectOp =
+ DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), And, N1_1, N2);
+ SelectOp->setFlags(Flags);
+ return SelectOp;
}
}
- // ctselect Cond0, X, (ctselect Cond1, X, Y) -> ctselect (or Cond0, Cond1),
- // X, Y
+
+ // ctselect C0, X, (ctselect C1, X, Y) -> ctselect (C0 | C1), X, Y
+ // Semantic equivalence: If C0 is true, choose X. If C0 is false, evaluate
+ // inner select (C1 ? X : Y). This is equivalent to (C0 || C1) ? X : Y.
if (N2->getOpcode() == ISD::CTSELECT && N2->hasOneUse()) {
SDValue N2_0 = N2->getOperand(0);
SDValue N2_1 = N2->getOperand(1);
SDValue N2_2 = N2->getOperand(2);
if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) {
- // Create the actual or node if we can generate good code for it.
- if (!normalizeToSequence) {
- SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
- return DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), Or, N1, N2_2,
- Flags);
- }
- // Otherwise see if we can optimize to a better pattern.
- if (SDValue Combined = visitORLike(N0, N2_0, DL))
- return DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), Combined, N1,
- N2_2, Flags);
+ SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
+ SDValue SelectOp =
+ DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), Or, N1, N2_2);
+ SelectOp->setFlags(Flags);
+ return SelectOp;
}
}
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 0a60dcc8a19b4..b2aa925bb55ae 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4356,19 +4356,51 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Tmp3 = Node->getOperand(2);
EVT VT = Tmp2.getValueType();
if (VT.isVector()) {
- SmallVector<SDValue> Elements;
- unsigned NumElements = VT.getVectorNumElements();
- EVT ScalarVT = VT.getScalarType();
- for (unsigned Idx = 0; Idx < NumElements; ++Idx) {
- SDValue IdxVal = DAG.getConstant(Idx, dl, MVT::i64);
- SDValue TVal =
- DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Tmp2, IdxVal);
- SDValue FVal =
- DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, ScalarVT, Tmp3, IdxVal);
- Elements.push_back(
- DAG.getCTSelect(dl, ScalarVT, Tmp1, TVal, FVal, Node->getFlags()));
+ // Constant-time vector blending using pattern F ^ ((T ^ F) & Mask)
+ // where Mask = broadcast(i1 ? -1 : 0) to match vector element width.
+ //
+ // This formulation uses only XOR and AND operations, avoiding branches
+ // that would leak timing information. It's equivalent to:
+ // Mask==0xFF: F ^ ((T ^ F) & 0xFF) = F ^ (T ^ F) = T
+ // Mask==0x00: F ^ ((T ^ F) & 0x00) = F ^ 0 = F
+
+ EVT IntVT = VT;
+ SDValue T = Tmp2; // True value
+ SDValue F = Tmp3; // False value
+
+ // Step 1: Handle floating-point vectors by bitcasting to integer
+ if (VT.isFloatingPoint()) {
+ IntVT = EVT::getVectorVT(
+ *DAG.getContext(),
+ EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits()),
+ VT.getVectorElementCount());
+ T = DAG.getNode(ISD::BITCAST, dl, IntVT, T);
+ F = DAG.getNode(ISD::BITCAST, dl, IntVT, F);
}
- Tmp1 = DAG.getBuildVector(VT, dl, Elements);
+
+ // Step 2: Broadcast the i1 condition to a vector of i1s
+ // Creates [cond, cond, cond, ...] with i1 elements
+ EVT VecI1Ty = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
+ VT.getVectorNumElements());
+ SDValue VecCond = DAG.getSplatBuildVector(VecI1Ty, dl, Tmp1);
+
+ // Step 3: Sign-extend i1 vector to get all-bits mask
+ // true (i1=1) -> 0xFFFFFFFF..., false (i1=0) -> 0x00000000
+ // Sign extension is constant-time: pure arithmetic, no branches
+ SDValue Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, IntVT, VecCond);
+
+ // Step 4: Compute constant-time blend: F ^ ((T ^ F) & Mask)
+ // All operations (XOR, AND) execute in constant time
+ SDValue TXorF = DAG.getNode(ISD::XOR, dl, IntVT, T, F);
+ SDValue MaskedDiff = DAG.getNode(ISD::AND, dl, IntVT, TXorF, Mask);
+ Tmp1 = DAG.getNode(ISD::XOR, dl, IntVT, F, MaskedDiff);
+
+ // Step 5: Bitcast back to original floating-point type if needed
+ if (VT.isFloatingPoint()) {
+ Tmp1 = DAG.getNode(ISD::BITCAST, dl, VT, Tmp1);
+ }
+
+ Tmp1->setFlags(Node->getFlags());
} else if (VT.isFloatingPoint()) {
EVT IntegerVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
Tmp2 = DAG.getBitcast(IntegerVT, Tmp2);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 2265b01fd3cd5..0189d69381208 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -1593,7 +1593,7 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
case ISD::POISON:
case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
- case ISD::CTSELECT: SplitRes_Select(N, Lo, Hi); break;
+ case ISD::CTSELECT: SplitRes_CTSELECT(N, Lo, Hi); break;
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
@@ -2773,7 +2773,7 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
break;
case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
case ISD::CTSELECT:
- R = SoftPromoteHalfRes_SELECT(N);
+ R = SoftPromoteHalfRes_CTSELECT(N);
break;
case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
case ISD::STRICT_SINT_TO_FP:
@@ -3042,6 +3042,13 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
Op2);
}
+SDValue DAGTypeLegalizer::SoftPromoteHalfRes_CTSELECT(SDNode *N) {
+ SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
+ SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
+ return DAG.getCTSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
+ Op2);
+}
+
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
SDValue Op3 = GetSoftPromotedHalf(N->getOperand(3));
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index d532d688b760a..ddbba309c1e96 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -3250,7 +3250,7 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
case ISD::CTSELECT:
- SplitRes_Select(N, Lo, Hi);
+ SplitRes_CTSELECT(N, Lo, Hi);
break;
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
case ISD::POISON:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index fb1ce23b44637..535d1420fd620 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -793,6 +793,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftPromoteHalfRes_LOAD(SDNode *N);
SDValue SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N);
SDValue SoftPromoteHalfRes_SELECT(SDNode *N);
+ SDValue SoftPromoteHalfRes_CTSELECT(SDNode *N);
SDValue SoftPromoteHalfRes_SELECT_CC(SDNode *N);
SDValue SoftPromoteHalfRes_UnaryOp(SDNode *N);
SDValue SoftPromoteHalfRes_FABS(SDNode *N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
index d942362dbc2dc..48bedbb9112ad 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
@@ -570,17 +570,9 @@ void DAGTypeLegalizer::SplitRes_Select(SDNode *N, SDValue &Lo, SDValue &Hi) {
}
void DAGTypeLegalizer::SplitRes_CTSELECT(SDNode *N, SDValue &Lo, SDValue &Hi) {
- SDValue LL, LH, RL, RH, CL, CH;
- SDLoc dl(N);
- GetSplitOp(N->getOperand(1), LL, LH);
- GetSplitOp(N->getOperand(2), RL, RH);
-
- SDValue Cond = N->getOperand(0);
- CL = CH = Cond;
- assert(!Cond.getValueType().isVector() && "Unsupported vector type");
-
- Lo = DAG.getNode(N->getOpcode(), dl, LL.getValueType(), CL, LL, RL);
- Hi = DAG.getNode(N->getOpcode(), dl, LH.getValueType(), CH, LH, RH);
+ // Reuse generic select splitting to support scalar and vector conditions.
+ // SplitRes_Select rebuilds with N->getOpcode(), so CTSELECT is preserved.
+ SplitRes_Select(N, Lo, Hi);
}
void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo,
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 2da390e0c4989..c61a2edd45255 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -8980,7 +8980,6 @@ SDValue SelectionDAG::getNode(unsigned Opcode, const SDLoc &DL, EVT VT,
return V;
break;
}
-
case ISD::SELECT:
case ISD::VSELECT:
if (SDValue V = simplifySelect(N1, N2, N3))
diff --git a/llvm/test/CodeGen/RISCV/ctselect-fallback.ll b/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
index f46bde0a05b8b..c624c17d7e33e 100644
--- a/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
+++ b/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
@@ -10,7 +10,7 @@ define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) {
; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: and a0, a1, a0
-; RV64-NEXT: xor a0, a0, a2
+; RV64-NEXT: xor a0, a2, a0
; RV64-NEXT: ret
;
; RV32-LABEL: test_ctselect_i8:
@@ -19,52 +19,28 @@ define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) {
; RV32-NEXT: slli a0, a0, 31
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a0, a1, a0
-; RV32-NEXT: xor a0, a0, a2
+; RV32-NEXT: xor a0, a2, a0
; RV32-NEXT: ret
%result = call i8 @llvm.ct.select.i8(i1 %cond, i8 %a, i8 %b)
ret i8 %result
}
-
-define i16 @test_ctselect_i16(i1 %cond, i16 %a, i16 %b) {
-; RV64-LABEL: test_ctselect_i16:
+define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) {
+; RV64-LABEL: test_ctselect_i32:
; RV64: # %bb.0:
; RV64-NEXT: xor a1, a1, a2
; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: and a0, a1, a0
-; RV64-NEXT: xor a0, a0, a2
+; RV64-NEXT: xor a0, a2, a0
; RV64-NEXT: ret
;
-; RV32-LABEL: test_ctselect_i16:
+; RV32-LABEL: test_ctselect_i32:
; RV32: # %bb.0:
; RV32-NEXT: xor a1, a1, a2
; RV32-NEXT: slli a0, a0, 31
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a0, a1, a0
-; RV32-NEXT: xor a0, a0, a2
-; RV32-NEXT: ret
- %result = call i16 @llvm.ct.select.i16(i1 %cond, i16 %a, i16 %b)
- ret i16 %result
-}
-
-define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) {
-; RV64-LABEL: test_ctselect_i32:
-; RV64: # %bb.0:
-; RV64-NEXT: xor a1, a1, a2
-; RV64-NEXT: slli a0, a0, 63
-; RV64-NEXT: srai a0, a0, 63
-; RV64-NEXT: and a0, a1, a0
-; RV64-NEXT: xor a0, a0, a2
-; RV64-NEXT: ret
-;
-; RV32-LABEL: test_ctselect_i32:
-; RV32: # %bb.0:
-; RV32-NEXT: andi a0, a0, 1
-; RV32-NEXT: neg a3, a0
-; RV32-NEXT: addi a0, a0, -1
-; RV32-NEXT: and a1, a3, a1
-; RV32-NEXT: and a0, a0, a2
-; RV32-NEXT: or a0, a1, a0
+; RV32-NEXT: xor a0, a2, a0
; RV32-NEXT: ret
%result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
ret i32 %result
@@ -73,12 +49,11 @@ define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) {
define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) {
; RV64-LABEL: test_ctselect_i64:
; RV64: # %bb.0:
-; RV64-NEXT: andi a0, a0, 1
-; RV64-NEXT: neg a3, a0
-; RV64-NEXT: addi a0, a0, -1
-; RV64-NEXT: and a1, a3, a1
-; RV64-NEXT: and a0, a0, a2
-; RV64-NEXT: or a0, a1, a0
+; RV64-NEXT: xor a1, a1, a2
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: xor a0, a2, a0
; RV64-NEXT: ret
;
; RV32-LABEL: test_ctselect_i64:
@@ -89,8 +64,8 @@ define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) {
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a1, a1, a0
; RV32-NEXT: and a2, a2, a0
-; RV32-NEXT: xor a0, a1, a3
-; RV32-NEXT: xor a1, a2, a4
+; RV32-NEXT: xor a0, a3, a1
+; RV32-NEXT: xor a1, a4, a2
; RV32-NEXT: ret
%result = call i64 @llvm.ct.select.i64(i1 %cond, i64 %a, i64 %b)
ret i64 %result
@@ -99,22 +74,20 @@ define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) {
define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) {
; RV64-LABEL: test_ctselect_ptr:
; RV64: # %bb.0:
-; RV64-NEXT: andi a0, a0, 1
-; RV64-NEXT: neg a3, a0
-; RV64-NEXT: addi a0, a0, -1
-; RV64-NEXT: and a1, a3, a1
-; RV64-NEXT: and a0, a0, a2
-; RV64-NEXT: or a0, a1, a0
+; RV64-NEXT: xor a1, a1, a2
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: xor a0, a2, a0
; RV64-NEXT: ret
;
; RV32-LABEL: test_ctselect_ptr:
; RV32: # %bb.0:
-; RV32-NEXT: andi a0, a0, 1
-; RV32-NEXT: neg a3, a0
-; RV32-NEXT: addi a0, a0, -1
-; RV32-NEXT: and a1, a3, a1
-; RV32-NEXT: and a0, a0, a2
-; RV32-NEXT: or a0, a1, a0
+; RV32-NEXT: xor a1, a1, a2
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a0, a1, a0
+; RV32-NEXT: xor a0, a2, a0
; RV32-NEXT: ret
%result = call ptr @llvm.ct.select.p0(i1 %cond, ptr %a, ptr %b)
ret ptr %result
@@ -128,6 +101,8 @@ define i32 @test_ctselect_const_true(i32 %a, i32 %b) {
;
; RV32-LABEL: test_ctselect_const_true:
; RV32: # %bb.0:
+; RV32-NEXT: xor a0, a0, a1
+; RV32-NEXT: xor a0, a1, a0
; RV32-NEXT: ret
%result = call i32 @llvm.ct.select.i32(i1 true, i32 %a, i32 %b)
ret i32 %result
@@ -158,131 +133,199 @@ define i32 @test_ctselect_icmp_eq(i32 %x, i32 %y, i32 %a, i32 %b) {
; RV64-NEXT: xor a2, a2, a3
; RV64-NEXT: addi a0, a0, -1
; RV64-NEXT: and a0, a2, a0
-; RV64-NEXT: xor a0, a0, a3
+; RV64-NEXT: xor a0, a3, a0
; RV64-NEXT: ret
;
; RV32-LABEL: test_ctselect_icmp_eq:
; RV32: # %bb.0:
; RV32-NEXT: xor a0, a0, a1
; RV32-NEXT: snez a0, a0
+; RV32-NEXT: xor a2, a2, a3
; RV32-NEXT: addi a0, a0, -1
-; RV32-NEXT: and a2, a0, a2
-; RV32-NEXT: not a0, a0
-; RV32-NEXT: and a0, a0, a3
-; RV32-NEXT: or a0, a2, a0
+; RV32-NEXT: and a0, a2, a0
+; RV32-NEXT: xor a0, a3, a0
; RV32-NEXT: ret
%cond = icmp eq i32 %x, %y
%result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
ret i32 %result
}
-
-define i32 @test_ctselect_icmp_ne(i32 %x, i32 %y, i32 %a, i32 %b) {
-; RV64-LABEL: test_ctselect_icmp_ne:
+define i32 @test_ctselect_icmp_ult(i32 %x, i32 %y, i32 %a, i32 %b) {
+; RV64-LABEL: test_ctselect_icmp_ult:
; RV64: # %bb.0:
; RV64-NEXT: sext.w a1, a1
; RV64-NEXT: sext.w a0, a0
-; RV64-NEXT: xor a0, a0, a1
-; RV64-NEXT: seqz a0, a0
+; RV64-NEXT: sltu a0, a0, a1
; RV64-NEXT: xor a2, a2, a3
-; RV64-NEXT: addi a0, a0, -1
+; RV64-NEXT: neg a0, a0
; RV64-NEXT: and a0, a2, a0
-; RV64-NEXT: xor a0, a0, a3
+; RV64-NEXT: xor a0, a3, a0
; RV64-NEXT: ret
;
-; RV32-LABEL: test_ctselect_icmp_ne:
+; RV32-LABEL: test_ctselect_icmp_ult:
; RV32: # %bb.0:
-; RV32-NEXT: xor a0, a0, a1
-; RV32-NEXT: seqz a0, a0
-; RV32-NEXT: addi a0, a0, -1
-; RV32-NEXT: and a2, a0, a2
-; RV32-NEXT: not a0, a0
-; RV32-NEXT: and a0, a0, a3
-; RV32-NEXT: or a0, a2, a0
+; RV32-NEXT: sltu a0, a0, a1
+; RV32-NEXT: xor a2, a2, a3
+; RV32-NEXT: neg a0, a0
+; RV32-NEXT: and a0, a2, a0
+; RV32-NEXT: xor a0, a3, a0
; RV32-NEXT: ret
- %cond = icmp ne i32 %x, %y
+ %cond = icmp ult i32 %x, %y
%result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
ret i32 %result
}
-define i32 @test_ctselect_icmp_slt(i32 %x, i32 %y, i32 %a, i32 %b) {
-; RV64-LABEL: test_ctselect_icmp_slt:
+; Test with memory operands
+define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) {
+; RV64-LABEL: test_ctselect_load:
; RV64: # %bb.0:
-; RV64-NEXT: sext.w a1, a1
-; RV64-NEXT: sext.w a0, a0
-; RV64-NEXT: slt a0, a0, a1
+; RV64-NEXT: lw a1, 0(a1)
+; RV64-NEXT: lw a2, 0(a2)
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: xor a1, a1, a2
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: xor a0, a2, a0
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_load:
+; RV32: # %bb.0:
+; RV32-NEXT: lw a1, 0(a1)
+; RV32-NEXT: lw a2, 0(a2)
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: xor a1, a1, a2
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a0, a1, a0
+; RV32-NEXT: xor a0, a2, a0
+; RV32-NEXT: ret
+ %a = load i32, ptr %p1
+ %b = load i32, ptr %p2
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ ret i32 %result
+}
+
+; Test nested CTSELECT pattern with AND merging on i1 values
+; Pattern: ctselect C0, (ctselect C1, X, Y), Y -> ctselect (C0 & C1), X, Y
+define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
+; RV64-LABEL: test_ctselect_nested_and_i1_to_i32:
+; RV64: # %bb.0:
+; RV64-NEXT: and a0, a1, a0
; RV64-NEXT: xor a2, a2, a3
-; RV64-NEXT: neg a0, a0
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: and a0, a2, a0
-; RV64-NEXT: xor a0, a0, a3
+; RV64-NEXT: xor a0, a3, a0
; RV64-NEXT: ret
;
-; RV32-LABEL: test_ctselect_icmp_slt:
+; RV32-LABEL: test_ctselect_nested_and_i1_to_i32:
; RV32: # %bb.0:
-; RV32-NEXT: slt a0, a0, a1
-; RV32-NEXT: neg a1, a0
-; RV32-NEXT: addi a0, a0, -1
-; RV32-NEXT: and a1, a1, a2
-; RV32-NEXT: and a0, a0, a3
-; RV32-NEXT: or a0, a1, a0
+; RV32-NEXT: and a0, a1, a0
+; RV32-NEXT: xor a2, a2, a3
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a0, a2, a0
+; RV32-NEXT: xor a0, a3, a0
; RV32-NEXT: ret
- %cond = icmp slt i32 %x, %y
- %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ %inner = call i1 @llvm.ct.select.i1(i1 %c1, i1 true, i1 false)
+ %cond = call i1 @llvm.ct.select.i1(i1 %c0, i1 %inner, i1 false)
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %x, i32 %y)
ret i32 %result
}
-define i32 @test_ctselect_icmp_ult(i32 %x, i32 %y, i32 %a, i32 %b) {
-; RV64-LABEL: test_ctselect_icmp_ult:
+; Test nested CTSELECT pattern with OR merging on i1 values
+; Pattern: ctselect C0, X, (ctselect C1, X, Y) -> ctselect (C0 | C1), X, Y
+define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
+; RV64-LABEL: test_ctselect_nested_or_i1_to_i32:
; RV64: # %bb.0:
-; RV64-NEXT: sext.w a1, a1
-; RV64-NEXT: sext.w a0, a0
-; RV64-NEXT: sltu a0, a0, a1
+; RV64-NEXT: or a0, a0, a1
; RV64-NEXT: xor a2, a2, a3
-; RV64-NEXT: neg a0, a0
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: and a0, a2, a0
-; RV64-NEXT: xor a0, a0, a3
+; RV64-NEXT: xor a0, a3, a0
; RV64-NEXT: ret
;
-; RV32-LABEL: test_ctselect_icmp_ult:
+; RV32-LABEL: test_ctselect_nested_or_i1_to_i32:
; RV32: # %bb.0:
-; RV32-NEXT: sltu a0, a0, a1
-; RV32-NEXT: neg a1, a0
-; RV32-NEXT: addi a0, a0, -1
-; RV32-NEXT: and a1, a1, a2
-; RV32-NEXT: and a0, a0, a3
-; RV32-NEXT: or a0, a1, a0
+; RV32-NEXT: or a0, a0, a1
+; RV32-NEXT: xor a2, a2, a3
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a0, a2, a0
+; RV32-NEXT: xor a0, a3, a0
; RV32-NEXT: ret
- %cond = icmp ult i32 %x, %y
- %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ %inner = call i1 @llvm.ct.select.i1(i1 %c1, i1 true, i1 false)
+ %cond = call i1 @llvm.ct.select.i1(i1 %c0, i1 true, i1 %inner)
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %x, i32 %y)
ret i32 %result
}
-; Test with memory operands
-define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) {
-; RV64-LABEL: test_ctselect_load:
+; Test double nested CTSELECT with recursive AND merging
+; Pattern: ctselect C0, (ctselect C1, (ctselect C2, X, Y), Y), Y
+; -> ctselect (C0 & C1 & C2), X, Y
+define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i32 %y) {
+; RV64-LABEL: test_ctselect_double_nested_and_i1:
; RV64: # %bb.0:
-; RV64-NEXT: lw a1, 0(a1)
-; RV64-NEXT: lw a2, 0(a2)
+; RV64-NEXT: and a1, a2, a1
+; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: xor a3, a3, a4
; RV64-NEXT: slli a0, a0, 63
-; RV64-NEXT: xor a1, a1, a2
; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: and a0, a3, a0
+; RV64-NEXT: xor a0, a4, a0
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_double_nested_and_i1:
+; RV32: # %bb.0:
+; RV32-NEXT: and a1, a2, a1
+; RV32-NEXT: and a0, a1, a0
+; RV32-NEXT: xor a3, a3, a4
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a0, a3, a0
+; RV32-NEXT: xor a0, a4, a0
+; RV32-NEXT: ret
+ %inner2 = call i1 @llvm.ct.select.i1(i1 %c2, i1 true, i1 false)
+ %inner1 = call i1 @llvm.ct.select.i1(i1 %c1, i1 %inner2, i1 false)
+ %cond = call i1 @llvm.ct.select.i1(i1 %c0, i1 %inner1, i1 false)
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %x, i32 %y)
+ ret i32 %result
+}
+
+; Test double nested CTSELECT with mixed AND/OR patterns
+define i32 @test_ctselect_double_nested_mixed_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i32 %y, i32 %z) {
+; RV64-LABEL: test_ctselect_double_nested_mixed_i1:
+; RV64: # %bb.0:
; RV64-NEXT: and a0, a1, a0
-; RV64-NEXT: xor a0, a0, a2
+; RV64-NEXT: xor a3, a3, a4
+; RV64-NEXT: or a0, a0, a2
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: and a3, a3, a0
+; RV64-NEXT: xor a4, a4, a5
+; RV64-NEXT: xor a3, a4, a3
+; RV64-NEXT: and a0, a3, a0
+; RV64-NEXT: xor a0, a5, a0
; RV64-NEXT: ret
;
-; RV32-LABEL: test_ctselect_load:
+; RV32-LABEL: test_ctselect_double_nested_mixed_i1:
; RV32: # %bb.0:
-; RV32-NEXT: lw a1, 0(a1)
-; RV32-NEXT: lw a2, 0(a2)
-; RV32-NEXT: andi a0, a0, 1
-; RV32-NEXT: neg a3, a0
-; RV32-NEXT: addi a0, a0, -1
-; RV32-NEXT: and a1, a3, a1
-; RV32-NEXT: and a0, a0, a2
-; RV32-NEXT: or a0, a1, a0
+; RV32-NEXT: and a0, a1, a0
+; RV32-NEXT: xor a3, a3, a4
+; RV32-NEXT: or a0, a0, a2
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a3, a3, a0
+; RV32-NEXT: xor a4, a4, a5
+; RV32-NEXT: xor a3, a4, a3
+; RV32-NEXT: and a0, a3, a0
+; RV32-NEXT: xor a0, a5, a0
; RV32-NEXT: ret
- %a = load i32, ptr %p1
- %b = load i32, ptr %p2
- %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
+ %inner1 = call i1 @llvm.ct.select.i1(i1 %c1, i1 true, i1 false)
+ %and_cond = call i1 @llvm.ct.select.i1(i1 %c0, i1 %inner1, i1 false)
+ %inner2 = call i1 @llvm.ct.select.i1(i1 %c2, i1 true, i1 false)
+ %or_cond = call i1 @llvm.ct.select.i1(i1 %and_cond, i1 true, i1 %inner2)
+ %inner_result = call i32 @llvm.ct.select.i32(i1 %or_cond, i32 %x, i32 %y)
+ %result = call i32 @llvm.ct.select.i32(i1 %or_cond, i32 %inner_result, i32 %z)
ret i32 %result
}
@@ -296,35 +339,416 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: srai a1, a1, 63
; RV64-NEXT: and a1, a2, a1
-; RV64-NEXT: xor a1, a1, a3
+; RV64-NEXT: xor a1, a3, a1
; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: and a0, a1, a0
-; RV64-NEXT: xor a0, a0, a4
+; RV64-NEXT: xor a0, a4, a0
; RV64-NEXT: ret
;
; RV32-LABEL: test_ctselect_nested:
; RV32: # %bb.0:
-; RV32-NEXT: andi a1, a1, 1
-; RV32-NEXT: andi a0, a0, 1
-; RV32-NEXT: neg a5, a1
-; RV32-NEXT: addi a1, a1, -1
-; RV32-NEXT: and a2, a5, a2
-; RV32-NEXT: neg a5, a0
-; RV32-NEXT: addi a0, a0, -1
-; RV32-NEXT: and a1, a1, a3
-; RV32-NEXT: or a1, a2, a1
-; RV32-NEXT: and a1, a5, a1
-; RV32-NEXT: and a0, a0, a4
-; RV32-NEXT: or a0, a1, a0
+; RV32-NEXT: xor a2, a2, a3
+; RV32-NEXT: slli a1, a1, 31
+; RV32-NEXT: xor a3, a3, a4
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: srai a1, a1, 31
+; RV32-NEXT: and a1, a2, a1
+; RV32-NEXT: xor a1, a3, a1
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a0, a1, a0
+; RV32-NEXT: xor a0, a4, a0
; RV32-NEXT: ret
%inner = call i32 @llvm.ct.select.i32(i1 %cond2, i32 %a, i32 %b)
%result = call i32 @llvm.ct.select.i32(i1 %cond1, i32 %inner, i32 %c)
ret i32 %result
}
+; Test floating-point ct.select selecting between NaN and Inf
+define float @test_ctselect_f32_nan_inf(i1 %cond) {
+; RV64-LABEL: test_ctselect_f32_nan_inf:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: lui a1, 1024
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: and a0, a0, a1
+; RV64-NEXT: lui a1, 522240
+; RV64-NEXT: or a0, a0, a1
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_f32_nan_inf:
+; RV32: # %bb.0:
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: lui a1, 1024
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a0, a0, a1
+; RV32-NEXT: lui a1, 522240
+; RV32-NEXT: xor a0, a0, a1
+; RV32-NEXT: ret
+ %result = call float @llvm.ct.select.f32(i1 %cond, float 0x7FF8000000000000, float 0x7FF0000000000000)
+ ret float %result
+}
+
+define double @test_ctselect_f64_nan_inf(i1 %cond) {
+; RV64-LABEL: test_ctselect_f64_nan_inf:
+; RV64: # %bb.0:
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: li a1, 1
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: slli a1, a1, 51
+; RV64-NEXT: and a0, a0, a1
+; RV64-NEXT: li a1, 2047
+; RV64-NEXT: slli a1, a1, 52
+; RV64-NEXT: xor a0, a0, a1
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_f64_nan_inf:
+; RV32: # %bb.0:
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: lui a1, 128
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a0, a0, a1
+; RV32-NEXT: lui a1, 524032
+; RV32-NEXT: or a1, a0, a1
+; RV32-NEXT: li a0, 0
+; RV32-NEXT: ret
+ %result = call double @llvm.ct.select.f64(i1 %cond, double 0x7FF8000000000000, double 0x7FF0000000000000)
+ ret double %result
+}
+
+; Test basic floating-point ct.select
+define float @test_ctselect_f32(i1 %cond, float %a, float %b) {
+; RV64-LABEL: test_ctselect_f32:
+; RV64: # %bb.0:
+; RV64-NEXT: xor a1, a1, a2
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: xor a0, a2, a0
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_f32:
+; RV32: # %bb.0:
+; RV32-NEXT: xor a1, a1, a2
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a0, a1, a0
+; RV32-NEXT: xor a0, a2, a0
+; RV32-NEXT: ret
+ %result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
+ ret float %result
+}
+
+define double @test_ctselect_f64(i1 %cond, double %a, double %b) {
+; RV64-LABEL: test_ctselect_f64:
+; RV64: # %bb.0:
+; RV64-NEXT: xor a1, a1, a2
+; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: srai a0, a0, 63
+; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: xor a0, a2, a0
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_f64:
+; RV32: # %bb.0:
+; RV32-NEXT: xor a1, a1, a3
+; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: xor a2, a2, a4
+; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: and a1, a1, a0
+; RV32-NEXT: and a2, a2, a0
+; RV32-NEXT: xor a0, a3, a1
+; RV32-NEXT: xor a1, a4, a2
+; RV32-NEXT: ret
+ %result = call double @llvm.ct.select.f64(i1 %cond, double %a, double %b)
+ ret double %result
+}
+
+; Test vector ct.select with integer vectors
+define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) {
+; RV64-LABEL: test_ctselect_v4i32:
+; RV64: # %bb.0:
+; RV64-NEXT: lw a4, 0(a3)
+; RV64-NEXT: lw a5, 8(a3)
+; RV64-NEXT: lw a6, 16(a3)
+; RV64-NEXT: lw a3, 24(a3)
+; RV64-NEXT: lw a7, 0(a2)
+; RV64-NEXT: lw t0, 8(a2)
+; RV64-NEXT: lw t1, 16(a2)
+; RV64-NEXT: lw a2, 24(a2)
+; RV64-NEXT: slli a1, a1, 63
+; RV64-NEXT: srai a1, a1, 63
+; RV64-NEXT: xor a7, a7, a4
+; RV64-NEXT: xor t0, t0, a5
+; RV64-NEXT: xor t1, t1, a6
+; RV64-NEXT: xor a2, a2, a3
+; RV64-NEXT: and a7, a7, a1
+; RV64-NEXT: and t0, t0, a1
+; RV64-NEXT: and t1, t1, a1
+; RV64-NEXT: and a1, a2, a1
+; RV64-NEXT: xor a2, a4, a7
+; RV64-NEXT: xor a4, a5, t0
+; RV64-NEXT: xor a5, a6, t1
+; RV64-NEXT: xor a1, a3, a1
+; RV64-NEXT: sw a2, 0(a0)
+; RV64-NEXT: sw a4, 4(a0)
+; RV64-NEXT: sw a5, 8(a0)
+; RV64-NEXT: sw a1, 12(a0)
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_v4i32:
+; RV32: # %bb.0:
+; RV32-NEXT: lw a4, 0(a3)
+; RV32-NEXT: lw a5, 4(a3)
+; RV32-NEXT: lw a6, 8(a3)
+; RV32-NEXT: lw a3, 12(a3)
+; RV32-NEXT: lw a7, 0(a2)
+; RV32-NEXT: lw t0, 4(a2)
+; RV32-NEXT: lw t1, 8(a2)
+; RV32-NEXT: lw a2, 12(a2)
+; RV32-NEXT: slli a1, a1, 31
+; RV32-NEXT: srai a1, a1, 31
+; RV32-NEXT: xor a7, a7, a4
+; RV32-NEXT: xor t0, t0, a5
+; RV32-NEXT: xor t1, t1, a6
+; RV32-NEXT: xor a2, a2, a3
+; RV32-NEXT: and a7, a7, a1
+; RV32-NEXT: and t0, t0, a1
+; RV32-NEXT: and t1, t1, a1
+; RV32-NEXT: and a1, a2, a1
+; RV32-NEXT: xor a2, a4, a7
+; RV32-NEXT: xor a4, a5, t0
+; RV32-NEXT: xor a5, a6, t1
+; RV32-NEXT: xor a1, a3, a1
+; RV32-NEXT: sw a2, 0(a0)
+; RV32-NEXT: sw a4, 4(a0)
+; RV32-NEXT: sw a5, 8(a0)
+; RV32-NEXT: sw a1, 12(a0)
+; RV32-NEXT: ret
+ %result = call <4 x i32> @llvm.ct.select.v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %result
+}
+define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b) {
+; RV64-LABEL: test_ctselect_v4f32:
+; RV64: # %bb.0:
+; RV64-NEXT: lw a4, 0(a3)
+; RV64-NEXT: lw a5, 8(a3)
+; RV64-NEXT: lw a6, 16(a3)
+; RV64-NEXT: lw a3, 24(a3)
+; RV64-NEXT: lw a7, 0(a2)
+; RV64-NEXT: lw t0, 8(a2)
+; RV64-NEXT: lw t1, 16(a2)
+; RV64-NEXT: lw a2, 24(a2)
+; RV64-NEXT: slli a1, a1, 63
+; RV64-NEXT: srai a1, a1, 63
+; RV64-NEXT: xor a7, a7, a4
+; RV64-NEXT: xor t0, t0, a5
+; RV64-NEXT: xor t1, t1, a6
+; RV64-NEXT: xor a2, a2, a3
+; RV64-NEXT: and a7, a7, a1
+; RV64-NEXT: and t0, t0, a1
+; RV64-NEXT: and t1, t1, a1
+; RV64-NEXT: and a1, a2, a1
+; RV64-NEXT: xor a2, a4, a7
+; RV64-NEXT: xor a4, a5, t0
+; RV64-NEXT: xor a5, a6, t1
+; RV64-NEXT: xor a1, a3, a1
+; RV64-NEXT: sw a2, 0(a0)
+; RV64-NEXT: sw a4, 4(a0)
+; RV64-NEXT: sw a5, 8(a0)
+; RV64-NEXT: sw a1, 12(a0)
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_v4f32:
+; RV32: # %bb.0:
+; RV32-NEXT: lw a4, 0(a3)
+; RV32-NEXT: lw a5, 4(a3)
+; RV32-NEXT: lw a6, 8(a3)
+; RV32-NEXT: lw a3, 12(a3)
+; RV32-NEXT: lw a7, 0(a2)
+; RV32-NEXT: lw t0, 4(a2)
+; RV32-NEXT: lw t1, 8(a2)
+; RV32-NEXT: lw a2, 12(a2)
+; RV32-NEXT: slli a1, a1, 31
+; RV32-NEXT: srai a1, a1, 31
+; RV32-NEXT: xor a7, a7, a4
+; RV32-NEXT: xor t0, t0, a5
+; RV32-NEXT: xor t1, t1, a6
+; RV32-NEXT: xor a2, a2, a3
+; RV32-NEXT: and a7, a7, a1
+; RV32-NEXT: and t0, t0, a1
+; RV32-NEXT: and t1, t1, a1
+; RV32-NEXT: and a1, a2, a1
+; RV32-NEXT: xor a2, a4, a7
+; RV32-NEXT: xor a4, a5, t0
+; RV32-NEXT: xor a5, a6, t1
+; RV32-NEXT: xor a1, a3, a1
+; RV32-NEXT: sw a2, 0(a0)
+; RV32-NEXT: sw a4, 4(a0)
+; RV32-NEXT: sw a5, 8(a0)
+; RV32-NEXT: sw a1, 12(a0)
+; RV32-NEXT: ret
+ %result = call <4 x float> @llvm.ct.select.v4f32(i1 %cond, <4 x float> %a, <4 x float> %b)
+ ret <4 x float> %result
+}
+define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
+; RV64-LABEL: test_ctselect_v8i32:
+; RV64: # %bb.0:
+; RV64-NEXT: addi sp, sp, -32
+; RV64-NEXT: .cfi_def_cfa_offset 32
+; RV64-NEXT: sd s0, 24(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s1, 16(sp) # 8-byte Folded Spill
+; RV64-NEXT: sd s2, 8(sp) # 8-byte Folded Spill
+; RV64-NEXT: .cfi_offset s0, -8
+; RV64-NEXT: .cfi_offset s1, -16
+; RV64-NEXT: .cfi_offset s2, -24
+; RV64-NEXT: lw a7, 32(a3)
+; RV64-NEXT: lw a6, 40(a3)
+; RV64-NEXT: lw a5, 48(a3)
+; RV64-NEXT: lw a4, 56(a3)
+; RV64-NEXT: lw t0, 32(a2)
+; RV64-NEXT: lw t1, 40(a2)
+; RV64-NEXT: lw t2, 48(a2)
+; RV64-NEXT: lw t3, 56(a2)
+; RV64-NEXT: lw t4, 0(a3)
+; RV64-NEXT: lw t5, 8(a3)
+; RV64-NEXT: lw t6, 16(a3)
+; RV64-NEXT: lw a3, 24(a3)
+; RV64-NEXT: lw s0, 0(a2)
+; RV64-NEXT: lw s1, 8(a2)
+; RV64-NEXT: lw s2, 16(a2)
+; RV64-NEXT: lw a2, 24(a2)
+; RV64-NEXT: slli a1, a1, 63
+; RV64-NEXT: srai a1, a1, 63
+; RV64-NEXT: xor s0, s0, t4
+; RV64-NEXT: xor s1, s1, t5
+; RV64-NEXT: xor s2, s2, t6
+; RV64-NEXT: xor a2, a2, a3
+; RV64-NEXT: xor t0, t0, a7
+; RV64-NEXT: xor t1, t1, a6
+; RV64-NEXT: xor t2, t2, a5
+; RV64-NEXT: xor t3, t3, a4
+; RV64-NEXT: and s0, s0, a1
+; RV64-NEXT: and s1, s1, a1
+; RV64-NEXT: and s2, s2, a1
+; RV64-NEXT: and a2, a2, a1
+; RV64-NEXT: and t0, t0, a1
+; RV64-NEXT: and t1, t1, a1
+; RV64-NEXT: and t2, t2, a1
+; RV64-NEXT: and a1, t3, a1
+; RV64-NEXT: xor t3, t4, s0
+; RV64-NEXT: xor t4, t5, s1
+; RV64-NEXT: xor t5, t6, s2
+; RV64-NEXT: xor a2, a3, a2
+; RV64-NEXT: xor a3, a7, t0
+; RV64-NEXT: xor a6, a6, t1
+; RV64-NEXT: xor a5, a5, t2
+; RV64-NEXT: xor a1, a4, a1
+; RV64-NEXT: sw a3, 16(a0)
+; RV64-NEXT: sw a6, 20(a0)
+; RV64-NEXT: sw a5, 24(a0)
+; RV64-NEXT: sw a1, 28(a0)
+; RV64-NEXT: sw t3, 0(a0)
+; RV64-NEXT: sw t4, 4(a0)
+; RV64-NEXT: sw t5, 8(a0)
+; RV64-NEXT: sw a2, 12(a0)
+; RV64-NEXT: ld s0, 24(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s1, 16(sp) # 8-byte Folded Reload
+; RV64-NEXT: ld s2, 8(sp) # 8-byte Folded Reload
+; RV64-NEXT: .cfi_restore s0
+; RV64-NEXT: .cfi_restore s1
+; RV64-NEXT: .cfi_restore s2
+; RV64-NEXT: addi sp, sp, 32
+; RV64-NEXT: .cfi_def_cfa_offset 0
+; RV64-NEXT: ret
+;
+; RV32-LABEL: test_ctselect_v8i32:
+; RV32: # %bb.0:
+; RV32-NEXT: addi sp, sp, -16
+; RV32-NEXT: .cfi_def_cfa_offset 16
+; RV32-NEXT: sw s0, 12(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s1, 8(sp) # 4-byte Folded Spill
+; RV32-NEXT: sw s2, 4(sp) # 4-byte Folded Spill
+; RV32-NEXT: .cfi_offset s0, -4
+; RV32-NEXT: .cfi_offset s1, -8
+; RV32-NEXT: .cfi_offset s2, -12
+; RV32-NEXT: lw a7, 16(a3)
+; RV32-NEXT: lw a6, 20(a3)
+; RV32-NEXT: lw a5, 24(a3)
+; RV32-NEXT: lw a4, 28(a3)
+; RV32-NEXT: lw t0, 16(a2)
+; RV32-NEXT: lw t1, 20(a2)
+; RV32-NEXT: lw t2, 24(a2)
+; RV32-NEXT: lw t3, 28(a2)
+; RV32-NEXT: lw t4, 0(a3)
+; RV32-NEXT: lw t5, 4(a3)
+; RV32-NEXT: lw t6, 8(a3)
+; RV32-NEXT: lw a3, 12(a3)
+; RV32-NEXT: lw s0, 0(a2)
+; RV32-NEXT: lw s1, 4(a2)
+; RV32-NEXT: lw s2, 8(a2)
+; RV32-NEXT: lw a2, 12(a2)
+; RV32-NEXT: slli a1, a1, 31
+; RV32-NEXT: srai a1, a1, 31
+; RV32-NEXT: xor s0, s0, t4
+; RV32-NEXT: xor s1, s1, t5
+; RV32-NEXT: xor s2, s2, t6
+; RV32-NEXT: xor a2, a2, a3
+; RV32-NEXT: xor t0, t0, a7
+; RV32-NEXT: xor t1, t1, a6
+; RV32-NEXT: xor t2, t2, a5
+; RV32-NEXT: xor t3, t3, a4
+; RV32-NEXT: and s0, s0, a1
+; RV32-NEXT: and s1, s1, a1
+; RV32-NEXT: and s2, s2, a1
+; RV32-NEXT: and a2, a2, a1
+; RV32-NEXT: and t0, t0, a1
+; RV32-NEXT: and t1, t1, a1
+; RV32-NEXT: and t2, t2, a1
+; RV32-NEXT: and a1, t3, a1
+; RV32-NEXT: xor t3, t4, s0
+; RV32-NEXT: xor t4, t5, s1
+; RV32-NEXT: xor t5, t6, s2
+; RV32-NEXT: xor a2, a3, a2
+; RV32-NEXT: xor a3, a7, t0
+; RV32-NEXT: xor a6, a6, t1
+; RV32-NEXT: xor a5, a5, t2
+; RV32-NEXT: xor a1, a4, a1
+; RV32-NEXT: sw a3, 16(a0)
+; RV32-NEXT: sw a6, 20(a0)
+; RV32-NEXT: sw a5, 24(a0)
+; RV32-NEXT: sw a1, 28(a0)
+; RV32-NEXT: sw t3, 0(a0)
+; RV32-NEXT: sw t4, 4(a0)
+; RV32-NEXT: sw t5, 8(a0)
+; RV32-NEXT: sw a2, 12(a0)
+; RV32-NEXT: lw s0, 12(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s1, 8(sp) # 4-byte Folded Reload
+; RV32-NEXT: lw s2, 4(sp) # 4-byte Folded Reload
+; RV32-NEXT: .cfi_restore s0
+; RV32-NEXT: .cfi_restore s1
+; RV32-NEXT: .cfi_restore s2
+; RV32-NEXT: addi sp, sp, 16
+; RV32-NEXT: .cfi_def_cfa_offset 0
+; RV32-NEXT: ret
+ %result = call <8 x i32> @llvm.ct.select.v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b)
+ ret <8 x i32> %result
+}
+
; Declare the intrinsics
+declare i1 @llvm.ct.select.i1(i1, i1, i1)
declare i8 @llvm.ct.select.i8(i1, i8, i8)
declare i16 @llvm.ct.select.i16(i1, i16, i16)
declare i32 @llvm.ct.select.i32(i1, i32, i32)
declare i64 @llvm.ct.select.i64(i1, i64, i64)
declare ptr @llvm.ct.select.p0(i1, ptr, ptr)
+declare float @llvm.ct.select.f32(i1, float, float)
+declare double @llvm.ct.select.f64(i1, double, double)
+
+; Vector intrinsics
+declare <4 x i32> @llvm.ct.select.v4i32(i1, <4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.ct.select.v2i64(i1, <2 x i64>, <2 x i64>)
+declare <8 x i16> @llvm.ct.select.v8i16(i1, <8 x i16>, <8 x i16>)
+declare <16 x i8> @llvm.ct.select.v16i8(i1, <16 x i8>, <16 x i8>)
+declare <4 x float> @llvm.ct.select.v4f32(i1, <4 x float>, <4 x float>)
+declare <2 x double> @llvm.ct.select.v2f64(i1, <2 x double>, <2 x double>)
+declare <8 x i32> @llvm.ct.select.v8i32(i1, <8 x i32>, <8 x i32>)
diff --git a/llvm/test/CodeGen/X86/ctselect.ll b/llvm/test/CodeGen/X86/ctselect.ll
index 095787a5e2a4b..2b6091c880637 100644
--- a/llvm/test/CodeGen/X86/ctselect.ll
+++ b/llvm/test/CodeGen/X86/ctselect.ll
@@ -8,120 +8,75 @@
define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) {
; X64-LABEL: test_ctselect_i8:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-NEXT: andb $1, %dil
-; X64-NEXT: leal -1(%rdi), %eax
-; X64-NEXT: movl %edi, %ecx
-; X64-NEXT: negb %cl
-; X64-NEXT: andb %sil, %cl
-; X64-NEXT: andb %dl, %al
-; X64-NEXT: orb %cl, %al
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: xorl %edx, %esi
+; X64-NEXT: andb $1, %al
+; X64-NEXT: negb %al
+; X64-NEXT: andb %sil, %al
+; X64-NEXT: xorb %dl, %al
; X64-NEXT: # kill: def $al killed $al killed $eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_i8:
; X32: # %bb.0:
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorb %cl, %dl
; X32-NEXT: andb $1, %al
-; X32-NEXT: movl %eax, %ecx
-; X32-NEXT: negb %cl
-; X32-NEXT: andb {{[0-9]+}}(%esp), %cl
-; X32-NEXT: decb %al
-; X32-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NEXT: orb %cl, %al
+; X32-NEXT: negb %al
+; X32-NEXT: andb %dl, %al
+; X32-NEXT: xorb %cl, %al
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_i8:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorb %cl, %dl
; X32-NOCMOV-NEXT: andb $1, %al
-; X32-NOCMOV-NEXT: movl %eax, %ecx
-; X32-NOCMOV-NEXT: negb %cl
-; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %cl
-; X32-NOCMOV-NEXT: decb %al
-; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: orb %cl, %al
+; X32-NOCMOV-NEXT: negb %al
+; X32-NOCMOV-NEXT: andb %dl, %al
+; X32-NOCMOV-NEXT: xorb %cl, %al
; X32-NOCMOV-NEXT: retl
%result = call i8 @llvm.ct.select.i8(i1 %cond, i8 %a, i8 %b)
ret i8 %result
}
-define i16 @test_ctselect_i16(i1 %cond, i16 %a, i16 %b) {
-; X64-LABEL: test_ctselect_i16:
+define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) {
+; X64-LABEL: test_ctselect_i32:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: leal -1(%rdi), %ecx
; X64-NEXT: movl %edi, %eax
+; X64-NEXT: xorl %edx, %esi
+; X64-NEXT: andl $1, %eax
; X64-NEXT: negl %eax
; X64-NEXT: andl %esi, %eax
-; X64-NEXT: andl %edx, %ecx
-; X64-NEXT: orl %ecx, %eax
-; X64-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-NEXT: retq
-;
-; X32-LABEL: test_ctselect_i16:
-; X32: # %bb.0:
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andl $1, %eax
-; X32-NEXT: leal -1(%eax), %ecx
-; X32-NEXT: andw {{[0-9]+}}(%esp), %cx
-; X32-NEXT: negl %eax
-; X32-NEXT: andw {{[0-9]+}}(%esp), %ax
-; X32-NEXT: orl %ecx, %eax
-; X32-NEXT: # kill: def $ax killed $ax killed $eax
-; X32-NEXT: retl
-;
-; X32-NOCMOV-LABEL: test_ctselect_i16:
-; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andl $1, %eax
-; X32-NOCMOV-NEXT: leal -1(%eax), %ecx
-; X32-NOCMOV-NEXT: andw {{[0-9]+}}(%esp), %cx
-; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andw {{[0-9]+}}(%esp), %ax
-; X32-NOCMOV-NEXT: orl %ecx, %eax
-; X32-NOCMOV-NEXT: # kill: def $ax killed $ax killed $eax
-; X32-NOCMOV-NEXT: retl
- %result = call i16 @llvm.ct.select.i16(i1 %cond, i16 %a, i16 %b)
- ret i16 %result
-}
-
-define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) {
-; X64-LABEL: test_ctselect_i32:
-; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: leal -1(%rdi), %eax
-; X64-NEXT: movl %edi, %ecx
-; X64-NEXT: negl %ecx
-; X64-NEXT: andl %esi, %ecx
-; X64-NEXT: andl %edx, %eax
-; X64-NEXT: orl %ecx, %eax
+; X64-NEXT: xorl %edx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_i32:
; X32: # %bb.0:
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %ecx, %edx
; X32-NEXT: andl $1, %eax
-; X32-NEXT: movl %eax, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: decl %eax
-; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: negl %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ecx, %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_i32:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: andl $1, %eax
-; X32-NOCMOV-NEXT: movl %eax, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: decl %eax
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
; X32-NOCMOV-NEXT: retl
%result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
ret i32 %result
@@ -130,13 +85,12 @@ define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) {
define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) {
; X64-LABEL: test_ctselect_i64:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: leaq -1(%rdi), %rax
-; X64-NEXT: negq %rdi
-; X64-NEXT: andq %rsi, %rdi
-; X64-NEXT: andq %rdx, %rax
-; X64-NEXT: orq %rdi, %rax
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: xorq %rdx, %rsi
+; X64-NEXT: andl $1, %eax
+; X64-NEXT: negq %rax
+; X64-NEXT: andq %rsi, %rax
+; X64-NEXT: xorq %rdx, %rax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_i64:
@@ -190,14 +144,12 @@ define float @test_ctselect_f32(i1 %cond, float %a, float %b) {
; X64-LABEL: test_ctselect_f32:
; X64: # %bb.0:
; X64-NEXT: movd %xmm1, %eax
+; X64-NEXT: pxor %xmm1, %xmm0
; X64-NEXT: movd %xmm0, %ecx
; X64-NEXT: andl $1, %edi
-; X64-NEXT: movl %edi, %edx
-; X64-NEXT: negl %edx
-; X64-NEXT: andl %ecx, %edx
-; X64-NEXT: decl %edi
-; X64-NEXT: andl %eax, %edi
-; X64-NEXT: orl %edx, %edi
+; X64-NEXT: negl %edi
+; X64-NEXT: andl %ecx, %edi
+; X64-NEXT: xorl %eax, %edi
; X64-NEXT: movd %edi, %xmm0
; X64-NEXT: retq
;
@@ -206,13 +158,13 @@ define float @test_ctselect_f32(i1 %cond, float %a, float %b) {
; X32-NEXT: pushl %eax
; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %ecx, %edx
; X32-NEXT: andl $1, %eax
-; X32-NEXT: movl %eax, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: decl %eax
-; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: negl %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ecx, %eax
; X32-NEXT: movl %eax, (%esp)
; X32-NEXT: flds (%esp)
; X32-NEXT: popl %eax
@@ -224,13 +176,13 @@ define float @test_ctselect_f32(i1 %cond, float %a, float %b) {
; X32-NOCMOV-NEXT: pushl %eax
; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: andl $1, %eax
-; X32-NOCMOV-NEXT: movl %eax, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: decl %eax
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
; X32-NOCMOV-NEXT: movl %eax, (%esp)
; X32-NOCMOV-NEXT: flds (%esp)
; X32-NOCMOV-NEXT: popl %eax
@@ -245,14 +197,12 @@ define double @test_ctselect_f64(i1 %cond, double %a, double %b) {
; X64: # %bb.0:
; X64-NEXT: # kill: def $edi killed $edi def $rdi
; X64-NEXT: movq %xmm1, %rax
+; X64-NEXT: pxor %xmm1, %xmm0
; X64-NEXT: movq %xmm0, %rcx
; X64-NEXT: andl $1, %edi
-; X64-NEXT: movq %rdi, %rdx
-; X64-NEXT: negq %rdx
-; X64-NEXT: andq %rcx, %rdx
-; X64-NEXT: decq %rdi
-; X64-NEXT: andq %rax, %rdi
-; X64-NEXT: orq %rdx, %rdi
+; X64-NEXT: negq %rdi
+; X64-NEXT: andq %rcx, %rdi
+; X64-NEXT: xorq %rax, %rdi
; X64-NEXT: movq %rdi, %xmm0
; X64-NEXT: retq
;
@@ -320,37 +270,36 @@ define double @test_ctselect_f64(i1 %cond, double %a, double %b) {
define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) {
; X64-LABEL: test_ctselect_ptr:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: leaq -1(%rdi), %rax
-; X64-NEXT: negq %rdi
-; X64-NEXT: andq %rsi, %rdi
-; X64-NEXT: andq %rdx, %rax
-; X64-NEXT: orq %rdi, %rax
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: xorq %rdx, %rsi
+; X64-NEXT: andl $1, %eax
+; X64-NEXT: negq %rax
+; X64-NEXT: andq %rsi, %rax
+; X64-NEXT: xorq %rdx, %rax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_ptr:
; X32: # %bb.0:
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %ecx, %edx
; X32-NEXT: andl $1, %eax
-; X32-NEXT: movl %eax, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: decl %eax
-; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: negl %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ecx, %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_ptr:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: andl $1, %eax
-; X32-NOCMOV-NEXT: movl %eax, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: decl %eax
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
; X32-NOCMOV-NEXT: retl
%result = call ptr @llvm.ct.select.p0(i1 %cond, ptr %a, ptr %b)
ret ptr %result
@@ -361,16 +310,24 @@ define i32 @test_ctselect_const_true(i32 %a, i32 %b) {
; X64-LABEL: test_ctselect_const_true:
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
+; X64-NEXT: xorl %esi, %eax
+; X64-NEXT: xorl %esi, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_const_true:
; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: xorl %ecx, %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_const_true:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
; X32-NOCMOV-NEXT: retl
%result = call i32 @llvm.ct.select.i32(i1 true, i32 %a, i32 %b)
ret i32 %result
@@ -385,13 +342,13 @@ define i32 @test_ctselect_const_false(i32 %a, i32 %b) {
; X32-LABEL: test_ctselect_const_false:
; X32: # %bb.0:
; X32-NEXT: xorl %eax, %eax
-; X32-NEXT: orl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_const_false:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: xorl %eax, %eax
-; X32-NOCMOV-NEXT: orl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: retl
%result = call i32 @llvm.ct.select.i32(i1 false, i32 %a, i32 %b)
ret i32 %result
@@ -404,174 +361,79 @@ define i32 @test_ctselect_icmp_eq(i32 %x, i32 %y, i32 %a, i32 %b) {
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpl %esi, %edi
; X64-NEXT: sete %al
-; X64-NEXT: movl %eax, %esi
-; X64-NEXT: negl %esi
-; X64-NEXT: andl %edx, %esi
-; X64-NEXT: decl %eax
-; X64-NEXT: andl %ecx, %eax
-; X64-NEXT: orl %esi, %eax
+; X64-NEXT: xorl %ecx, %edx
+; X64-NEXT: negl %eax
+; X64-NEXT: andl %edx, %eax
+; X64-NEXT: xorl %ecx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_icmp_eq:
; X32: # %bb.0:
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: xorl %eax, %eax
-; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: cmpl {{[0-9]+}}(%esp), %edx
; X32-NEXT: sete %al
-; X32-NEXT: movl %eax, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: decl %eax
-; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: negl %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ecx, %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_icmp_eq:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: xorl %eax, %eax
-; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: sete %al
-; X32-NOCMOV-NEXT: movl %eax, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: decl %eax
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
; X32-NOCMOV-NEXT: retl
%cond = icmp eq i32 %x, %y
%result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
ret i32 %result
}
-define i32 @test_ctselect_icmp_ne(i32 %x, i32 %y, i32 %a, i32 %b) {
-; X64-LABEL: test_ctselect_icmp_ne:
-; X64: # %bb.0:
-; X64-NEXT: xorl %eax, %eax
-; X64-NEXT: cmpl %esi, %edi
-; X64-NEXT: setne %al
-; X64-NEXT: movl %eax, %esi
-; X64-NEXT: negl %esi
-; X64-NEXT: andl %edx, %esi
-; X64-NEXT: decl %eax
-; X64-NEXT: andl %ecx, %eax
-; X64-NEXT: orl %esi, %eax
-; X64-NEXT: retq
-;
-; X32-LABEL: test_ctselect_icmp_ne:
-; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %eax, %eax
-; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: setne %al
-; X32-NEXT: movl %eax, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: decl %eax
-; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: orl %ecx, %eax
-; X32-NEXT: retl
-;
-; X32-NOCMOV-LABEL: test_ctselect_icmp_ne:
-; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %eax, %eax
-; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: setne %al
-; X32-NOCMOV-NEXT: movl %eax, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: decl %eax
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: orl %ecx, %eax
-; X32-NOCMOV-NEXT: retl
- %cond = icmp ne i32 %x, %y
- %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
- ret i32 %result
-}
-
-define i32 @test_ctselect_icmp_slt(i32 %x, i32 %y, i32 %a, i32 %b) {
-; X64-LABEL: test_ctselect_icmp_slt:
-; X64: # %bb.0:
-; X64-NEXT: xorl %eax, %eax
-; X64-NEXT: cmpl %esi, %edi
-; X64-NEXT: setl %al
-; X64-NEXT: movl %eax, %esi
-; X64-NEXT: negl %esi
-; X64-NEXT: andl %edx, %esi
-; X64-NEXT: decl %eax
-; X64-NEXT: andl %ecx, %eax
-; X64-NEXT: orl %esi, %eax
-; X64-NEXT: retq
-;
-; X32-LABEL: test_ctselect_icmp_slt:
-; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %eax, %eax
-; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: setl %al
-; X32-NEXT: movl %eax, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: decl %eax
-; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: orl %ecx, %eax
-; X32-NEXT: retl
-;
-; X32-NOCMOV-LABEL: test_ctselect_icmp_slt:
-; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %eax, %eax
-; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: setl %al
-; X32-NOCMOV-NEXT: movl %eax, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: decl %eax
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: orl %ecx, %eax
-; X32-NOCMOV-NEXT: retl
- %cond = icmp slt i32 %x, %y
- %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
- ret i32 %result
-}
-
define i32 @test_ctselect_icmp_ult(i32 %x, i32 %y, i32 %a, i32 %b) {
; X64-LABEL: test_ctselect_icmp_ult:
; X64: # %bb.0:
+; X64-NEXT: xorl %ecx, %edx
; X64-NEXT: xorl %eax, %eax
; X64-NEXT: cmpl %esi, %edi
; X64-NEXT: sbbl %eax, %eax
-; X64-NEXT: andl %eax, %edx
-; X64-NEXT: notl %eax
-; X64-NEXT: andl %ecx, %eax
-; X64-NEXT: orl %edx, %eax
+; X64-NEXT: andl %edx, %eax
+; X64-NEXT: xorl %ecx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_icmp_ult:
; X32: # %bb.0:
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %eax, %eax
-; X32-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: sbbl %eax, %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: andl %eax, %ecx
-; X32-NEXT: notl %eax
-; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: orl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %edx, %edx
+; X32-NEXT: cmpl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: sbbl %edx, %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ecx, %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_icmp_ult:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %eax, %eax
-; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: sbbl %eax, %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: andl %eax, %ecx
-; X32-NOCMOV-NEXT: notl %eax
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: orl %ecx, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %edx, %edx
+; X32-NOCMOV-NEXT: cmpl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: sbbl %edx, %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
; X32-NOCMOV-NEXT: retl
%cond = icmp ult i32 %x, %y
%result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %a, i32 %b)
@@ -583,12 +445,10 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) {
; X64: # %bb.0:
; X64-NEXT: movd %xmm3, %eax
; X64-NEXT: cmpeqss %xmm1, %xmm0
-; X64-NEXT: movd %xmm0, %ecx
-; X64-NEXT: pand %xmm2, %xmm0
-; X64-NEXT: movd %xmm0, %edx
-; X64-NEXT: notl %ecx
-; X64-NEXT: andl %eax, %ecx
-; X64-NEXT: orl %edx, %ecx
+; X64-NEXT: pxor %xmm3, %xmm2
+; X64-NEXT: pand %xmm0, %xmm2
+; X64-NEXT: movd %xmm2, %ecx
+; X64-NEXT: xorl %eax, %ecx
; X64-NEXT: movd %ecx, %xmm0
; X64-NEXT: retq
;
@@ -596,21 +456,21 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) {
; X32: # %bb.0:
; X32-NEXT: pushl %eax
; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fucompi %st(1), %st
; X32-NEXT: fstp %st(0)
-; X32-NEXT: setnp %al
-; X32-NEXT: sete %cl
-; X32-NEXT: andb %al, %cl
-; X32-NEXT: movzbl %cl, %eax
-; X32-NEXT: movl %eax, %ecx
+; X32-NEXT: setnp %cl
+; X32-NEXT: sete %dl
+; X32-NEXT: andb %cl, %dl
+; X32-NEXT: movzbl %dl, %ecx
; X32-NEXT: negl %ecx
-; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: decl %eax
-; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: orl %ecx, %eax
-; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %eax, %edx
+; X32-NEXT: andl %ecx, %edx
+; X32-NEXT: xorl %eax, %edx
+; X32-NEXT: movl %edx, (%esp)
; X32-NEXT: flds (%esp)
; X32-NEXT: popl %eax
; X32-NEXT: .cfi_def_cfa_offset 4
@@ -620,6 +480,7 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) {
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: pushl %eax
; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fucompp
@@ -627,16 +488,15 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) {
; X32-NOCMOV-NEXT: # kill: def $ah killed $ah killed $ax
; X32-NOCMOV-NEXT: sahf
; X32-NOCMOV-NEXT: setnp %al
-; X32-NOCMOV-NEXT: sete %cl
-; X32-NOCMOV-NEXT: andb %al, %cl
-; X32-NOCMOV-NEXT: movzbl %cl, %eax
-; X32-NOCMOV-NEXT: movl %eax, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: decl %eax
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: orl %ecx, %eax
-; X32-NOCMOV-NEXT: movl %eax, (%esp)
+; X32-NOCMOV-NEXT: sete %dl
+; X32-NOCMOV-NEXT: andb %al, %dl
+; X32-NOCMOV-NEXT: movzbl %dl, %eax
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: andl %eax, %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: movl %edx, (%esp)
; X32-NOCMOV-NEXT: flds (%esp)
; X32-NOCMOV-NEXT: popl %eax
; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
@@ -650,52 +510,41 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) {
define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) {
; X64-LABEL: test_ctselect_load:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: movl (%rdx), %ecx
+; X64-NEXT: movl (%rsi), %eax
+; X64-NEXT: xorl %ecx, %eax
; X64-NEXT: andl $1, %edi
-; X64-NEXT: leal -1(%rdi), %eax
-; X64-NEXT: movl %edi, %ecx
-; X64-NEXT: negl %ecx
-; X64-NEXT: andl (%rsi), %ecx
-; X64-NEXT: andl (%rdx), %eax
-; X64-NEXT: orl %ecx, %eax
+; X64-NEXT: negl %edi
+; X64-NEXT: andl %edi, %eax
+; X64-NEXT: xorl %ecx, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_load:
; X32: # %bb.0:
-; X32-NEXT: pushl %esi
-; X32-NEXT: .cfi_def_cfa_offset 8
-; X32-NEXT: .cfi_offset %esi, -8
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl (%edx), %edx
+; X32-NEXT: movl (%ecx), %ecx
+; X32-NEXT: xorl %edx, %ecx
; X32-NEXT: andl $1, %eax
-; X32-NEXT: movl %eax, %esi
-; X32-NEXT: negl %esi
-; X32-NEXT: andl (%edx), %esi
-; X32-NEXT: decl %eax
-; X32-NEXT: andl (%ecx), %eax
-; X32-NEXT: orl %esi, %eax
-; X32-NEXT: popl %esi
-; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: negl %eax
+; X32-NEXT: andl %ecx, %eax
+; X32-NEXT: xorl %edx, %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_load:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; X32-NOCMOV-NEXT: .cfi_offset %esi, -8
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl (%edx), %edx
+; X32-NOCMOV-NEXT: movl (%ecx), %ecx
+; X32-NOCMOV-NEXT: xorl %edx, %ecx
; X32-NOCMOV-NEXT: andl $1, %eax
-; X32-NOCMOV-NEXT: movl %eax, %esi
-; X32-NOCMOV-NEXT: negl %esi
-; X32-NOCMOV-NEXT: andl (%edx), %esi
-; X32-NOCMOV-NEXT: decl %eax
-; X32-NOCMOV-NEXT: andl (%ecx), %eax
-; X32-NOCMOV-NEXT: orl %esi, %eax
-; X32-NOCMOV-NEXT: popl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl %ecx, %eax
+; X32-NOCMOV-NEXT: xorl %edx, %eax
; X32-NOCMOV-NEXT: retl
%a = load i32, ptr %p1
%b = load i32, ptr %p2
@@ -707,69 +556,753 @@ define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) {
define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
; X64-LABEL: test_ctselect_nested:
; X64: # %bb.0:
-; X64-NEXT: # kill: def $esi killed $esi def $rsi
-; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: xorl %ecx, %edx
; X64-NEXT: andl $1, %esi
-; X64-NEXT: leal -1(%rsi), %r9d
-; X64-NEXT: movl %esi, %eax
+; X64-NEXT: negl %esi
+; X64-NEXT: andl %edx, %esi
+; X64-NEXT: xorl %r8d, %ecx
+; X64-NEXT: xorl %esi, %ecx
+; X64-NEXT: andl $1, %eax
; X64-NEXT: negl %eax
-; X64-NEXT: andl %edx, %eax
-; X64-NEXT: andl %ecx, %r9d
-; X64-NEXT: orl %eax, %r9d
-; X64-NEXT: andl $1, %edi
-; X64-NEXT: leal -1(%rdi), %eax
-; X64-NEXT: movl %edi, %ecx
-; X64-NEXT: negl %ecx
-; X64-NEXT: andl %r9d, %ecx
-; X64-NEXT: andl %r8d, %eax
-; X64-NEXT: orl %ecx, %eax
+; X64-NEXT: andl %ecx, %eax
+; X64-NEXT: xorl %r8d, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_nested:
; X32: # %bb.0:
+; X32-NEXT: pushl %edi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: .cfi_offset %esi, -12
+; X32-NEXT: .cfi_offset %edi, -8
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: andl $1, %ecx
-; X32-NEXT: movl %ecx, %edx
-; X32-NEXT: negl %edx
-; X32-NEXT: andl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: decl %ecx
-; X32-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: orl %edx, %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: xorl %edx, %edi
+; X32-NEXT: andl $1, %esi
+; X32-NEXT: negl %esi
+; X32-NEXT: andl %edi, %esi
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: xorl %esi, %edx
; X32-NEXT: andl $1, %eax
-; X32-NEXT: movl %eax, %edx
-; X32-NEXT: negl %edx
-; X32-NEXT: andl %ecx, %edx
-; X32-NEXT: decl %eax
-; X32-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: orl %edx, %eax
+; X32-NEXT: negl %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: popl %edi
+; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_nested:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; X32-NOCMOV-NEXT: .cfi_offset %esi, -12
+; X32-NOCMOV-NEXT: .cfi_offset %edi, -8
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: andl $1, %ecx
-; X32-NOCMOV-NEXT: movl %ecx, %edx
-; X32-NOCMOV-NEXT: negl %edx
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: decl %ecx
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: orl %edx, %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: xorl %edx, %edi
+; X32-NOCMOV-NEXT: andl $1, %esi
+; X32-NOCMOV-NEXT: negl %esi
+; X32-NOCMOV-NEXT: andl %edi, %esi
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: xorl %esi, %edx
; X32-NOCMOV-NEXT: andl $1, %eax
-; X32-NOCMOV-NEXT: movl %eax, %edx
-; X32-NOCMOV-NEXT: negl %edx
-; X32-NOCMOV-NEXT: andl %ecx, %edx
-; X32-NOCMOV-NEXT: decl %eax
-; X32-NOCMOV-NEXT: andl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: orl %edx, %eax
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl
%inner = call i32 @llvm.ct.select.i32(i1 %cond2, i32 %a, i32 %b)
%result = call i32 @llvm.ct.select.i32(i1 %cond1, i32 %inner, i32 %c)
ret i32 %result
}
+; Test nested CTSELECT pattern with AND merging on i1 values
+; Pattern: ctselect C0, (ctselect C1, X, Y), Y -> ctselect (C0 & C1), X, Y
+; This optimization only applies when selecting between i1 values (boolean logic)
+define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
+; X64-LABEL: test_ctselect_nested_and_i1_to_i32:
+; X64: # %bb.0:
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: andl %esi, %eax
+; X64-NEXT: xorl %ecx, %edx
+; X64-NEXT: andl $1, %eax
+; X64-NEXT: negl %eax
+; X64-NEXT: andl %edx, %eax
+; X64-NEXT: xorl %ecx, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_nested_and_i1_to_i32:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb {{[0-9]+}}(%esp), %al
+; X32-NEXT: movzbl %al, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: andl $1, %eax
+; X32-NEXT: negl %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_nested_and_i1_to_i32:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
+; X32-NOCMOV-NEXT: movzbl %al, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: retl
+ %inner = call i1 @llvm.ct.select.i1(i1 %c1, i1 true, i1 false)
+ %cond = call i1 @llvm.ct.select.i1(i1 %c0, i1 %inner, i1 false)
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %x, i32 %y)
+ ret i32 %result
+}
+
+; Test nested CTSELECT pattern with OR merging on i1 values
+; Pattern: ctselect C0, X, (ctselect C1, X, Y) -> ctselect (C0 | C1), X, Y
+; This optimization only applies when selecting between i1 values (boolean logic)
+define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
+; X64-LABEL: test_ctselect_nested_or_i1_to_i32:
+; X64: # %bb.0:
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: orl %esi, %eax
+; X64-NEXT: xorl %ecx, %edx
+; X64-NEXT: andl $1, %eax
+; X64-NEXT: negl %eax
+; X64-NEXT: andl %edx, %eax
+; X64-NEXT: xorl %ecx, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_nested_or_i1_to_i32:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: orb {{[0-9]+}}(%esp), %al
+; X32-NEXT: movzbl %al, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: andl $1, %eax
+; X32-NEXT: negl %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_nested_or_i1_to_i32:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: orb {{[0-9]+}}(%esp), %al
+; X32-NOCMOV-NEXT: movzbl %al, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: retl
+ %inner = call i1 @llvm.ct.select.i1(i1 %c1, i1 true, i1 false)
+ %cond = call i1 @llvm.ct.select.i1(i1 %c0, i1 true, i1 %inner)
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %x, i32 %y)
+ ret i32 %result
+}
+
+; Test double nested CTSELECT with recursive AND merging
+; Pattern: ctselect C0, (ctselect C1, (ctselect C2, X, Y), Y), Y
+; -> ctselect C0, (ctselect (C1 & C2), X, Y), Y
+; -> ctselect (C0 & (C1 & C2)), X, Y
+; This tests that the optimization can be applied recursively
+define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i32 %y) {
+; X64-LABEL: test_ctselect_double_nested_and_i1:
+; X64: # %bb.0:
+; X64-NEXT: movl %esi, %eax
+; X64-NEXT: andl %edx, %eax
+; X64-NEXT: andl %edi, %eax
+; X64-NEXT: xorl %r8d, %ecx
+; X64-NEXT: andl $1, %eax
+; X64-NEXT: negl %eax
+; X64-NEXT: andl %ecx, %eax
+; X64-NEXT: xorl %r8d, %eax
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_double_nested_and_i1:
+; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb {{[0-9]+}}(%esp), %al
+; X32-NEXT: andb {{[0-9]+}}(%esp), %al
+; X32-NEXT: movzbl %al, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: andl $1, %eax
+; X32-NEXT: negl %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_double_nested_and_i1:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
+; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
+; X32-NOCMOV-NEXT: movzbl %al, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: retl
+ %inner2 = call i1 @llvm.ct.select.i1(i1 %c2, i1 true, i1 false)
+ %inner1 = call i1 @llvm.ct.select.i1(i1 %c1, i1 %inner2, i1 false)
+ %cond = call i1 @llvm.ct.select.i1(i1 %c0, i1 %inner1, i1 false)
+ %result = call i32 @llvm.ct.select.i32(i1 %cond, i32 %x, i32 %y)
+ ret i32 %result
+}
+
+; Vector CTSELECT Tests
+; ============================================================================
+
+; Test vector CTSELECT with v4i32 (128-bit vector with single i1 mask)
+; NOW CONSTANT-TIME: Uses bitwise XOR/AND operations instead of branches!
+define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) {
+; X64-LABEL: test_ctselect_v4i32:
+; X64: # %bb.0:
+; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: movd %edi, %xmm2
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
+; X64-NEXT: pslld $31, %xmm2
+; X64-NEXT: psrad $31, %xmm2
+; X64-NEXT: pand %xmm2, %xmm0
+; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_v4i32:
+; X32: # %bb.0:
+; X32-NEXT: pushl %ebp
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: pushl %ebx
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: pushl %edi
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 20
+; X32-NEXT: .cfi_offset %esi, -20
+; X32-NEXT: .cfi_offset %edi, -16
+; X32-NEXT: .cfi_offset %ebx, -12
+; X32-NEXT: .cfi_offset %ebp, -8
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %ebx, %edx
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: andl $1, %edi
+; X32-NEXT: negl %edi
+; X32-NEXT: andl %edi, %edx
+; X32-NEXT: xorl %ebx, %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: xorl %ebp, %ebx
+; X32-NEXT: andl %edi, %ebx
+; X32-NEXT: xorl %ebp, %ebx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: xorl %esi, %ebp
+; X32-NEXT: andl %edi, %ebp
+; X32-NEXT: xorl %esi, %ebp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl %ecx, %esi
+; X32-NEXT: andl %edi, %esi
+; X32-NEXT: xorl %ecx, %esi
+; X32-NEXT: movl %esi, 12(%eax)
+; X32-NEXT: movl %ebp, 8(%eax)
+; X32-NEXT: movl %ebx, 4(%eax)
+; X32-NEXT: movl %edx, (%eax)
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: popl %edi
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: popl %ebx
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: popl %ebp
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl $4
+;
+; X32-NOCMOV-LABEL: test_ctselect_v4i32:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; X32-NOCMOV-NEXT: .cfi_offset %esi, -20
+; X32-NOCMOV-NEXT: .cfi_offset %edi, -16
+; X32-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; X32-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ebx, %edx
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: andl $1, %edi
+; X32-NOCMOV-NEXT: negl %edi
+; X32-NOCMOV-NEXT: andl %edi, %edx
+; X32-NOCMOV-NEXT: xorl %ebx, %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: xorl %ebp, %ebx
+; X32-NOCMOV-NEXT: andl %edi, %ebx
+; X32-NOCMOV-NEXT: xorl %ebp, %ebx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: xorl %esi, %ebp
+; X32-NOCMOV-NEXT: andl %edi, %ebp
+; X32-NOCMOV-NEXT: xorl %esi, %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl %ecx, %esi
+; X32-NOCMOV-NEXT: andl %edi, %esi
+; X32-NOCMOV-NEXT: xorl %ecx, %esi
+; X32-NOCMOV-NEXT: movl %esi, 12(%eax)
+; X32-NOCMOV-NEXT: movl %ebp, 8(%eax)
+; X32-NOCMOV-NEXT: movl %ebx, 4(%eax)
+; X32-NOCMOV-NEXT: movl %edx, (%eax)
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; X32-NOCMOV-NEXT: popl %ebx
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: popl %ebp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; X32-NOCMOV-NEXT: retl $4
+ %result = call <4 x i32> @llvm.ct.select.v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b)
+ ret <4 x i32> %result
+}
+define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b) {
+; X64-LABEL: test_ctselect_v4f32:
+; X64: # %bb.0:
+; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: movd %edi, %xmm2
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
+; X64-NEXT: pslld $31, %xmm2
+; X64-NEXT: psrad $31, %xmm2
+; X64-NEXT: pand %xmm2, %xmm0
+; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_v4f32:
+; X32: # %bb.0:
+; X32-NEXT: pushl %ebp
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: pushl %ebx
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: pushl %edi
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 20
+; X32-NEXT: .cfi_offset %esi, -20
+; X32-NEXT: .cfi_offset %edi, -16
+; X32-NEXT: .cfi_offset %ebx, -12
+; X32-NEXT: .cfi_offset %ebp, -8
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %ebx, %edx
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: andl $1, %edi
+; X32-NEXT: negl %edi
+; X32-NEXT: andl %edi, %edx
+; X32-NEXT: xorl %ebx, %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: xorl %ebp, %ebx
+; X32-NEXT: andl %edi, %ebx
+; X32-NEXT: xorl %ebp, %ebx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: xorl %esi, %ebp
+; X32-NEXT: andl %edi, %ebp
+; X32-NEXT: xorl %esi, %ebp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl %ecx, %esi
+; X32-NEXT: andl %edi, %esi
+; X32-NEXT: xorl %ecx, %esi
+; X32-NEXT: movl %esi, 12(%eax)
+; X32-NEXT: movl %ebp, 8(%eax)
+; X32-NEXT: movl %ebx, 4(%eax)
+; X32-NEXT: movl %edx, (%eax)
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: popl %edi
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: popl %ebx
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: popl %ebp
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl $4
+;
+; X32-NOCMOV-LABEL: test_ctselect_v4f32:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; X32-NOCMOV-NEXT: .cfi_offset %esi, -20
+; X32-NOCMOV-NEXT: .cfi_offset %edi, -16
+; X32-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; X32-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ebx, %edx
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: andl $1, %edi
+; X32-NOCMOV-NEXT: negl %edi
+; X32-NOCMOV-NEXT: andl %edi, %edx
+; X32-NOCMOV-NEXT: xorl %ebx, %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: xorl %ebp, %ebx
+; X32-NOCMOV-NEXT: andl %edi, %ebx
+; X32-NOCMOV-NEXT: xorl %ebp, %ebx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: xorl %esi, %ebp
+; X32-NOCMOV-NEXT: andl %edi, %ebp
+; X32-NOCMOV-NEXT: xorl %esi, %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl %ecx, %esi
+; X32-NOCMOV-NEXT: andl %edi, %esi
+; X32-NOCMOV-NEXT: xorl %ecx, %esi
+; X32-NOCMOV-NEXT: movl %esi, 12(%eax)
+; X32-NOCMOV-NEXT: movl %ebp, 8(%eax)
+; X32-NOCMOV-NEXT: movl %ebx, 4(%eax)
+; X32-NOCMOV-NEXT: movl %edx, (%eax)
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; X32-NOCMOV-NEXT: popl %ebx
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: popl %ebp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; X32-NOCMOV-NEXT: retl $4
+ %result = call <4 x float> @llvm.ct.select.v4f32(i1 %cond, <4 x float> %a, <4 x float> %b)
+ ret <4 x float> %result
+}
+
+define <8 x i32> @test_ctselect_v8i32_avx(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
+; X64-LABEL: test_ctselect_v8i32_avx:
+; X64: # %bb.0:
+; X64-NEXT: movd %edi, %xmm4
+; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
+; X64-NEXT: pslld $31, %xmm4
+; X64-NEXT: psrad $31, %xmm4
+; X64-NEXT: movdqa %xmm4, %xmm5
+; X64-NEXT: pandn %xmm2, %xmm5
+; X64-NEXT: pand %xmm4, %xmm0
+; X64-NEXT: por %xmm5, %xmm0
+; X64-NEXT: pand %xmm4, %xmm1
+; X64-NEXT: pandn %xmm3, %xmm4
+; X64-NEXT: por %xmm4, %xmm1
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_v8i32_avx:
+; X32: # %bb.0:
+; X32-NEXT: pushl %ebp
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: pushl %ebx
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: pushl %edi
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 20
+; X32-NEXT: subl $8, %esp
+; X32-NEXT: .cfi_def_cfa_offset 28
+; X32-NEXT: .cfi_offset %esi, -20
+; X32-NEXT: .cfi_offset %edi, -16
+; X32-NEXT: .cfi_offset %ebx, -12
+; X32-NEXT: .cfi_offset %ebp, -8
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorl %eax, %ecx
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: andl $1, %edx
+; X32-NEXT: negl %edx
+; X32-NEXT: andl %edx, %ecx
+; X32-NEXT: xorl %eax, %ecx
+; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %esi, %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %esi, %eax
+; X32-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl %ebx, %esi
+; X32-NEXT: andl %edx, %esi
+; X32-NEXT: xorl %ebx, %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: xorl %ebp, %ebx
+; X32-NEXT: andl %edx, %ebx
+; X32-NEXT: xorl %ebp, %ebx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: xorl %edi, %ebp
+; X32-NEXT: andl %edx, %ebp
+; X32-NEXT: xorl %edi, %ebp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: xorl %eax, %edi
+; X32-NEXT: andl %edx, %edi
+; X32-NEXT: xorl %eax, %edi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorl %eax, %ecx
+; X32-NEXT: andl %edx, %ecx
+; X32-NEXT: xorl %eax, %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl %eax, 28(%edx)
+; X32-NEXT: movl %ecx, 24(%edx)
+; X32-NEXT: movl %edi, 20(%edx)
+; X32-NEXT: movl %ebp, 16(%edx)
+; X32-NEXT: movl %ebx, 12(%edx)
+; X32-NEXT: movl %esi, 8(%edx)
+; X32-NEXT: movl (%esp), %eax # 4-byte Reload
+; X32-NEXT: movl %eax, 4(%edx)
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: movl %eax, (%edx)
+; X32-NEXT: movl %edx, %eax
+; X32-NEXT: addl $8, %esp
+; X32-NEXT: .cfi_def_cfa_offset 20
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: popl %edi
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: popl %ebx
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: popl %ebp
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl $4
+;
+; X32-NOCMOV-LABEL: test_ctselect_v8i32_avx:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; X32-NOCMOV-NEXT: subl $8, %esp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 28
+; X32-NOCMOV-NEXT: .cfi_offset %esi, -20
+; X32-NOCMOV-NEXT: .cfi_offset %edi, -16
+; X32-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; X32-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorl %eax, %ecx
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: andl $1, %edx
+; X32-NOCMOV-NEXT: negl %edx
+; X32-NOCMOV-NEXT: andl %edx, %ecx
+; X32-NOCMOV-NEXT: xorl %eax, %ecx
+; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %esi, %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %esi, %eax
+; X32-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl %ebx, %esi
+; X32-NOCMOV-NEXT: andl %edx, %esi
+; X32-NOCMOV-NEXT: xorl %ebx, %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: xorl %ebp, %ebx
+; X32-NOCMOV-NEXT: andl %edx, %ebx
+; X32-NOCMOV-NEXT: xorl %ebp, %ebx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: xorl %edi, %ebp
+; X32-NOCMOV-NEXT: andl %edx, %ebp
+; X32-NOCMOV-NEXT: xorl %edi, %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: xorl %eax, %edi
+; X32-NOCMOV-NEXT: andl %edx, %edi
+; X32-NOCMOV-NEXT: xorl %eax, %edi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorl %eax, %ecx
+; X32-NOCMOV-NEXT: andl %edx, %ecx
+; X32-NOCMOV-NEXT: xorl %eax, %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl %eax, 28(%edx)
+; X32-NOCMOV-NEXT: movl %ecx, 24(%edx)
+; X32-NOCMOV-NEXT: movl %edi, 20(%edx)
+; X32-NOCMOV-NEXT: movl %ebp, 16(%edx)
+; X32-NOCMOV-NEXT: movl %ebx, 12(%edx)
+; X32-NOCMOV-NEXT: movl %esi, 8(%edx)
+; X32-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %eax, 4(%edx)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %eax, (%edx)
+; X32-NOCMOV-NEXT: movl %edx, %eax
+; X32-NOCMOV-NEXT: addl $8, %esp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; X32-NOCMOV-NEXT: popl %ebx
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: popl %ebp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; X32-NOCMOV-NEXT: retl $4
+ %result = call <8 x i32> @llvm.ct.select.v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b)
+ ret <8 x i32> %result
+}
+
+define float @test_ctselect_f32_nan_inf(i1 %cond) {
+; X64-LABEL: test_ctselect_f32_nan_inf:
+; X64: # %bb.0:
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: negl %edi
+; X64-NEXT: andl $4194304, %edi # imm = 0x400000
+; X64-NEXT: xorl $2139095040, %edi # imm = 0x7F800000
+; X64-NEXT: movd %edi, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_f32_nan_inf:
+; X32: # %bb.0:
+; X32-NEXT: pushl %eax
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andl $1, %eax
+; X32-NEXT: negl %eax
+; X32-NEXT: andl $4194304, %eax # imm = 0x400000
+; X32-NEXT: xorl $2139095040, %eax # imm = 0x7F800000
+; X32-NEXT: movl %eax, (%esp)
+; X32-NEXT: flds (%esp)
+; X32-NEXT: popl %eax
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_f32_nan_inf:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %eax
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl $4194304, %eax # imm = 0x400000
+; X32-NOCMOV-NEXT: xorl $2139095040, %eax # imm = 0x7F800000
+; X32-NOCMOV-NEXT: movl %eax, (%esp)
+; X32-NOCMOV-NEXT: flds (%esp)
+; X32-NOCMOV-NEXT: popl %eax
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; X32-NOCMOV-NEXT: retl
+ %result = call float @llvm.ct.select.f32(i1 %cond, float 0x7FF8000000000000, float 0x7FF0000000000000)
+ ret float %result
+}
+
+define double @test_ctselect_f64_nan_inf(i1 %cond) {
+; X64-LABEL: test_ctselect_f64_nan_inf:
+; X64: # %bb.0:
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: negq %rdi
+; X64-NEXT: movabsq $2251799813685248, %rax # imm = 0x8000000000000
+; X64-NEXT: andq %rdi, %rax
+; X64-NEXT: movabsq $9218868437227405312, %rcx # imm = 0x7FF0000000000000
+; X64-NEXT: xorq %rax, %rcx
+; X64-NEXT: movq %rcx, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_f64_nan_inf:
+; X32: # %bb.0:
+; X32-NEXT: subl $12, %esp
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andl $1, %eax
+; X32-NEXT: negl %eax
+; X32-NEXT: andl $524288, %eax # imm = 0x80000
+; X32-NEXT: orl $2146435072, %eax # imm = 0x7FF00000
+; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT: movl $0, (%esp)
+; X32-NEXT: fldl (%esp)
+; X32-NEXT: addl $12, %esp
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_f64_nan_inf:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: subl $12, %esp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl $524288, %eax # imm = 0x80000
+; X32-NOCMOV-NEXT: orl $2146435072, %eax # imm = 0x7FF00000
+; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl $0, (%esp)
+; X32-NOCMOV-NEXT: fldl (%esp)
+; X32-NOCMOV-NEXT: addl $12, %esp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; X32-NOCMOV-NEXT: retl
+ %result = call double @llvm.ct.select.f64(i1 %cond, double 0x7FF8000000000000, double 0x7FF0000000000000)
+ ret double %result
+}
+
; Declare the intrinsics
+declare i1 @llvm.ct.select.i1(i1, i1, i1)
declare i8 @llvm.ct.select.i8(i1, i8, i8)
declare i16 @llvm.ct.select.i16(i1, i16, i16)
declare i32 @llvm.ct.select.i32(i1, i32, i32)
@@ -777,3 +1310,12 @@ declare i64 @llvm.ct.select.i64(i1, i64, i64)
declare float @llvm.ct.select.f32(i1, float, float)
declare double @llvm.ct.select.f64(i1, double, double)
declare ptr @llvm.ct.select.p0(i1, ptr, ptr)
+
+; Vector intrinsics
+declare <4 x i32> @llvm.ct.select.v4i32(i1, <4 x i32>, <4 x i32>)
+declare <2 x i64> @llvm.ct.select.v2i64(i1, <2 x i64>, <2 x i64>)
+declare <8 x i16> @llvm.ct.select.v8i16(i1, <8 x i16>, <8 x i16>)
+declare <16 x i8> @llvm.ct.select.v16i8(i1, <16 x i8>, <16 x i8>)
+declare <4 x float> @llvm.ct.select.v4f32(i1, <4 x float>, <4 x float>)
+declare <2 x double> @llvm.ct.select.v2f64(i1, <2 x double>, <2 x double>)
+declare <8 x i32> @llvm.ct.select.v8i32(i1, <8 x i32>, <8 x i32>)
>From 8cb284eea92e420b82b73513f99fa70c0ff25f2d Mon Sep 17 00:00:00 2001
From: Akshay K <iit.akshay at gmail.com>
Date: Wed, 11 Feb 2026 19:23:51 -0500
Subject: [PATCH 04/10] [LangRef][ConstantTime] Add documentation for
llvm.ct.select.* constant-time intrinsics (#181042)
This PR introduces and documents the llvm.ct.select.* constant-time
intrinsics, providing timing-independent selection operations for
security-sensitive code. The LangRef is updated with syntax, semantics,
supported types, and usage guidance.
Additionally, test coverage is extended with a new <8 x float> variant
(llvm.ct.select.v8f32) and corresponding X86 codegen tests to ensure
correct lowering on both x64 and x32 targets.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 10 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 11 +-
llvm/test/CodeGen/X86/ctselect.ll | 188 ++++++++++++++++++
3 files changed, 199 insertions(+), 10 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 760f06ff1442b..6b019a57bf709 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -260,11 +260,11 @@ class LLVM_ABI TargetLoweringBase {
/// Enum that describes what type of support for selects the target has.
enum SelectSupportKind {
- ScalarValSelect, // The target supports scalar selects (ex: cmov).
- ScalarCondVectorVal, // The target supports selects with a scalar condition
- // and vector values (ex: cmov).
- VectorMaskSelect // The target supports vector selects with a vector
- // mask (ex: x86 blends).
+ ScalarValSelect, // The target supports scalar selects (ex: cmov).
+ ScalarCondVectorVal, // The target supports selects with a scalar condition
+ // and vector values (ex: cmov).
+ VectorMaskSelect // The target supports vector selects with a vector
+ // mask (ex: x86 blends).
};
/// Enum that specifies what an atomic load/AtomicRMWInst is expanded
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 0b15066cef57e..bd88ac67046fc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -484,8 +484,9 @@ namespace {
SDValue visitCTTZ_ZERO_POISON(SDNode *N);
SDValue visitCTPOP(SDNode *N);
SDValue visitSELECT(SDNode *N);
- // visit CTSELECT Node
- SDValue visitConstantTimeSelect(SDNode *N);
+ // ISD::CTSELECT - Constant-Time SELECT (not related to CT in
+ // CTPOP/CTLZ/CTTZ where CT means "count").
+ SDValue visitCT_SELECT(SDNode *N);
SDValue visitVSELECT(SDNode *N);
SDValue visitVP_SELECT(SDNode *N);
SDValue visitSELECT_CC(SDNode *N);
@@ -2041,7 +2042,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
case ISD::CTTZ_ZERO_POISON: return visitCTTZ_ZERO_POISON(N);
case ISD::CTPOP: return visitCTPOP(N);
case ISD::SELECT: return visitSELECT(N);
- case ISD::CTSELECT: return visitConstantTimeSelect(N);
+ case ISD::CTSELECT: return visitCT_SELECT(N);
case ISD::VSELECT: return visitVSELECT(N);
case ISD::SELECT_CC: return visitSELECT_CC(N);
case ISD::SETCC: return visitSETCC(N);
@@ -12930,7 +12931,7 @@ static SDValue foldBoolSelectToLogic(SDNode *N, const SDLoc &DL,
SelectionDAG &DAG) {
assert((N->getOpcode() == ISD::SELECT || N->getOpcode() == ISD::VSELECT ||
N->getOpcode() == ISD::VP_SELECT) &&
- "Expected a (v)(vp.)(ct) select");
+ "Expected a (v)(vp.)select");
SDValue Cond = N->getOperand(0);
SDValue T = N->getOperand(1), F = N->getOperand(2);
EVT VT = N->getValueType(0);
@@ -13440,7 +13441,7 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
// - i1 CTSELECT nesting merges via AND/OR that keep the result as CTSELECT.
// Broader rewrites should be done in target-specific lowering when stronger
// guarantees about legality and constant-time preservation are available.
-SDValue DAGCombiner::visitConstantTimeSelect(SDNode *N) {
+SDValue DAGCombiner::visitCT_SELECT(SDNode *N) {
SDValue N0 = N->getOperand(0);
SDValue N1 = N->getOperand(1);
SDValue N2 = N->getOperand(2);
diff --git a/llvm/test/CodeGen/X86/ctselect.ll b/llvm/test/CodeGen/X86/ctselect.ll
index 2b6091c880637..a970fd8933b9b 100644
--- a/llvm/test/CodeGen/X86/ctselect.ll
+++ b/llvm/test/CodeGen/X86/ctselect.ll
@@ -1210,6 +1210,193 @@ define <8 x i32> @test_ctselect_v8i32_avx(i1 %cond, <8 x i32> %a, <8 x i32> %b)
ret <8 x i32> %result
}
+define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b) {
+; X64-LABEL: test_ctselect_v8f32:
+; X64: # %bb.0:
+; X64-NEXT: movd %edi, %xmm4
+; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
+; X64-NEXT: pslld $31, %xmm4
+; X64-NEXT: psrad $31, %xmm4
+; X64-NEXT: movdqa %xmm4, %xmm5
+; X64-NEXT: pandn %xmm2, %xmm5
+; X64-NEXT: pand %xmm4, %xmm0
+; X64-NEXT: por %xmm5, %xmm0
+; X64-NEXT: pand %xmm4, %xmm1
+; X64-NEXT: pandn %xmm3, %xmm4
+; X64-NEXT: por %xmm4, %xmm1
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_v8f32:
+; X32: # %bb.0:
+; X32-NEXT: pushl %ebp
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: pushl %ebx
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: pushl %edi
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 20
+; X32-NEXT: subl $8, %esp
+; X32-NEXT: .cfi_def_cfa_offset 28
+; X32-NEXT: .cfi_offset %esi, -20
+; X32-NEXT: .cfi_offset %edi, -16
+; X32-NEXT: .cfi_offset %ebx, -12
+; X32-NEXT: .cfi_offset %ebp, -8
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorl %eax, %ecx
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: andl $1, %edx
+; X32-NEXT: negl %edx
+; X32-NEXT: andl %edx, %ecx
+; X32-NEXT: xorl %eax, %ecx
+; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %esi, %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %esi, %eax
+; X32-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl %ebx, %esi
+; X32-NEXT: andl %edx, %esi
+; X32-NEXT: xorl %ebx, %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: xorl %ebp, %ebx
+; X32-NEXT: andl %edx, %ebx
+; X32-NEXT: xorl %ebp, %ebx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: xorl %edi, %ebp
+; X32-NEXT: andl %edx, %ebp
+; X32-NEXT: xorl %edi, %ebp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: xorl %eax, %edi
+; X32-NEXT: andl %edx, %edi
+; X32-NEXT: xorl %eax, %edi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorl %eax, %ecx
+; X32-NEXT: andl %edx, %ecx
+; X32-NEXT: xorl %eax, %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl %eax, 28(%edx)
+; X32-NEXT: movl %ecx, 24(%edx)
+; X32-NEXT: movl %edi, 20(%edx)
+; X32-NEXT: movl %ebp, 16(%edx)
+; X32-NEXT: movl %ebx, 12(%edx)
+; X32-NEXT: movl %esi, 8(%edx)
+; X32-NEXT: movl (%esp), %eax # 4-byte Reload
+; X32-NEXT: movl %eax, 4(%edx)
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: movl %eax, (%edx)
+; X32-NEXT: movl %edx, %eax
+; X32-NEXT: addl $8, %esp
+; X32-NEXT: .cfi_def_cfa_offset 20
+; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: popl %edi
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: popl %ebx
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: popl %ebp
+; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: retl $4
+;
+; X32-NOCMOV-LABEL: test_ctselect_v8f32:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; X32-NOCMOV-NEXT: subl $8, %esp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 28
+; X32-NOCMOV-NEXT: .cfi_offset %esi, -20
+; X32-NOCMOV-NEXT: .cfi_offset %edi, -16
+; X32-NOCMOV-NEXT: .cfi_offset %ebx, -12
+; X32-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorl %eax, %ecx
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: andl $1, %edx
+; X32-NOCMOV-NEXT: negl %edx
+; X32-NOCMOV-NEXT: andl %edx, %ecx
+; X32-NOCMOV-NEXT: xorl %eax, %ecx
+; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %esi, %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %esi, %eax
+; X32-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl %ebx, %esi
+; X32-NOCMOV-NEXT: andl %edx, %esi
+; X32-NOCMOV-NEXT: xorl %ebx, %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: xorl %ebp, %ebx
+; X32-NOCMOV-NEXT: andl %edx, %ebx
+; X32-NOCMOV-NEXT: xorl %ebp, %ebx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: xorl %edi, %ebp
+; X32-NOCMOV-NEXT: andl %edx, %ebp
+; X32-NOCMOV-NEXT: xorl %edi, %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: xorl %eax, %edi
+; X32-NOCMOV-NEXT: andl %edx, %edi
+; X32-NOCMOV-NEXT: xorl %eax, %edi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorl %eax, %ecx
+; X32-NOCMOV-NEXT: andl %edx, %ecx
+; X32-NOCMOV-NEXT: xorl %eax, %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl %eax, 28(%edx)
+; X32-NOCMOV-NEXT: movl %ecx, 24(%edx)
+; X32-NOCMOV-NEXT: movl %edi, 20(%edx)
+; X32-NOCMOV-NEXT: movl %ebp, 16(%edx)
+; X32-NOCMOV-NEXT: movl %ebx, 12(%edx)
+; X32-NOCMOV-NEXT: movl %esi, 8(%edx)
+; X32-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %eax, 4(%edx)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %eax, (%edx)
+; X32-NOCMOV-NEXT: movl %edx, %eax
+; X32-NOCMOV-NEXT: addl $8, %esp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; X32-NOCMOV-NEXT: popl %ebx
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: popl %ebp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; X32-NOCMOV-NEXT: retl $4
+ %result = call <8 x float> @llvm.ct.select.v8f32(i1 %cond, <8 x float> %a, <8 x float> %b)
+ ret <8 x float> %result
+}
+
define float @test_ctselect_f32_nan_inf(i1 %cond) {
; X64-LABEL: test_ctselect_f32_nan_inf:
; X64: # %bb.0:
@@ -1319,3 +1506,4 @@ declare <16 x i8> @llvm.ct.select.v16i8(i1, <16 x i8>, <16 x i8>)
declare <4 x float> @llvm.ct.select.v4f32(i1, <4 x float>, <4 x float>)
declare <2 x double> @llvm.ct.select.v2f64(i1, <2 x double>, <2 x double>)
declare <8 x i32> @llvm.ct.select.v8i32(i1, <8 x i32>, <8 x i32>)
+declare <8 x float> @llvm.ct.select.v8f32(i1, <8 x float>, <8 x float>)
>From d855f76af0f16911634aab2b96c23e47b679c3b8 Mon Sep 17 00:00:00 2001
From: wizardengineer <juliuswoosebert at gmail.com>
Date: Thu, 19 Feb 2026 23:13:21 -0500
Subject: [PATCH 05/10] [ConstantTime] Changed CTSELECT instances to CT_SELECT
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 4 +--
llvm/include/llvm/CodeGen/SelectionDAG.h | 2 +-
llvm/include/llvm/CodeGen/TargetLowering.h | 7 ++---
.../include/llvm/Target/TargetSelectionDAG.td | 2 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 28 +++++++++----------
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 4 +--
.../SelectionDAG/LegalizeFloatTypes.cpp | 12 ++++----
.../SelectionDAG/LegalizeIntegerTypes.cpp | 14 +++++-----
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 10 +++----
.../SelectionDAG/LegalizeTypesGeneric.cpp | 4 +--
.../SelectionDAG/LegalizeVectorTypes.cpp | 12 ++++----
.../SelectionDAG/SelectionDAGBuilder.cpp | 5 ++--
.../SelectionDAG/SelectionDAGDumper.cpp | 2 +-
llvm/lib/CodeGen/TargetLoweringBase.cpp | 3 +-
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 2 --
llvm/lib/Target/ARM/ARMISelLowering.h | 2 --
llvm/lib/Target/X86/X86ISelLowering.h | 2 --
llvm/test/CodeGen/RISCV/ctselect-fallback.ll | 18 ++++++------
llvm/test/CodeGen/X86/ctselect.ll | 22 +++++++--------
19 files changed, 75 insertions(+), 80 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index f9573f797b6a8..0c953dd176ed1 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -805,9 +805,9 @@ enum NodeType {
/// i1 then the high bits must conform to getBooleanContents.
SELECT,
- /// CTSELECT(Cond, TrueVal, FalseVal). Cond is i1 and the value operands must
+ /// CT_SELECT(Cond, TrueVal, FalseVal). Cond is i1 and the value operands must
/// have the same type. Used to lower the constant-time select intrinsic.
- CTSELECT,
+ CT_SELECT,
/// Select with a vector condition (op #0) and two vector operands (ops #1
/// and #2), returning a vector result. All vectors have the same length.
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index 00c5998a2f973..c5ea42fd4b81d 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1419,7 +1419,7 @@ class SelectionDAG {
SDValue RHS, SDNodeFlags Flags = SDNodeFlags()) {
assert(LHS.getValueType() == VT && RHS.getValueType() == VT &&
"Cannot use select on differing types");
- return getNode(ISD::CTSELECT, DL, VT, Cond, LHS, RHS, Flags);
+ return getNode(ISD::CT_SELECT, DL, VT, Cond, LHS, RHS, Flags);
}
/// Helper function to make it easier to build SelectCC's if you just have an
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 6b019a57bf709..9a6dd7735421e 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -495,10 +495,9 @@ class LLVM_ABI TargetLoweringBase {
MachineMemOperand::Flags
getVPIntrinsicMemOperandFlags(const VPIntrinsic &VPIntrin) const;
- virtual bool isSelectSupported(SelectSupportKind kind) const { return true; }
-
- /// Return true if the target has custom lowering for constant-time select.
- virtual bool isCtSelectSupported(EVT VT) const { return false; }
+ virtual bool isSelectSupported(SelectSupportKind /*kind*/) const {
+ return true;
+ }
/// Return true if the @llvm.get.active.lane.mask intrinsic should be expanded
/// using generic code in SelectionDAGBuilder.
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 24cc74fed90e0..f599a80058dd4 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -772,7 +772,7 @@ def reset_fpmode : SDNode<"ISD::RESET_FPMODE", SDTNone, [SDNPHasChain]>;
def setcc : SDNode<"ISD::SETCC" , SDTSetCC>;
def select : SDNode<"ISD::SELECT" , SDTSelect>;
-def ctselect : SDNode<"ISD::CTSELECT", SDTCtSelect>;
+def ct_select : SDNode<"ISD::CT_SELECT", SDTCtSelect>;
def vselect : SDNode<"ISD::VSELECT" , SDTVSelect>;
def selectcc : SDNode<"ISD::SELECT_CC" , SDTSelectCC>;
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index bd88ac67046fc..1293518113eeb 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -484,7 +484,7 @@ namespace {
SDValue visitCTTZ_ZERO_POISON(SDNode *N);
SDValue visitCTPOP(SDNode *N);
SDValue visitSELECT(SDNode *N);
- // ISD::CTSELECT - Constant-Time SELECT (not related to CT in
+ // ISD::CT_SELECT - Constant-Time SELECT (not related to CT in
// CTPOP/CTLZ/CTTZ where CT means "count").
SDValue visitCT_SELECT(SDNode *N);
SDValue visitVSELECT(SDNode *N);
@@ -2042,7 +2042,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
case ISD::CTTZ_ZERO_POISON: return visitCTTZ_ZERO_POISON(N);
case ISD::CTPOP: return visitCTPOP(N);
case ISD::SELECT: return visitSELECT(N);
- case ISD::CTSELECT: return visitCT_SELECT(N);
+ case ISD::CT_SELECT: return visitCT_SELECT(N);
case ISD::VSELECT: return visitVSELECT(N);
case ISD::SELECT_CC: return visitSELECT_CC(N);
case ISD::SETCC: return visitSETCC(N);
@@ -13435,10 +13435,10 @@ SDValue DAGCombiner::visitSELECT(SDNode *N) {
return SDValue();
}
-// Keep CTSELECT combines deliberately conservative to preserve constant-time
+// Keep CT_SELECT combines deliberately conservative to preserve constant-time
// intent across generic DAG combines. We only accept:
// - canonicalization of negated conditions (flip true/false operands), and
-// - i1 CTSELECT nesting merges via AND/OR that keep the result as CTSELECT.
+// - i1 CT_SELECT nesting merges via AND/OR that keep the result as CT_SELECT.
// Broader rewrites should be done in target-specific lowering when stronger
// guarantees about legality and constant-time preservation are available.
SDValue DAGCombiner::visitCT_SELECT(SDNode *N) {
@@ -13450,52 +13450,52 @@ SDValue DAGCombiner::visitCT_SELECT(SDNode *N) {
SDLoc DL(N);
SDNodeFlags Flags = N->getFlags();
- // ctselect (not Cond), N1, N2 -> ctselect Cond, N2, N1
+ // ct_select (not Cond), N1, N2 -> ct_select Cond, N2, N1
// This is a CT-safe canonicalization: flip negated condition by swapping
// arms. extractBooleanFlip only matches boolean xor-with-1, so this preserves
// dataflow semantics and does not introduce data-dependent control flow.
if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) {
- SDValue SelectOp = DAG.getNode(ISD::CTSELECT, DL, VT, F, N2, N1);
+ SDValue SelectOp = DAG.getNode(ISD::CT_SELECT, DL, VT, F, N2, N1);
SelectOp->setFlags(Flags);
return SelectOp;
}
if (VT0 == MVT::i1) {
- // Nested CTSELECT merging optimizations for i1 conditions.
+ // Nested CT_SELECT merging optimizations for i1 conditions.
// These are CT-safe because:
// 1. AND/OR are bitwise operations that execute in constant time
- // 2. The optimization combines two sequential CTSELECTs into one,
+ // 2. The optimization combines two sequential CT_SELECTs into one,
// reducing the total number of constant-time operations without
// changing semantics
// 3. No data-dependent branches or memory accesses are introduced
//
- // ctselect C0, (ctselect C1, X, Y), Y -> ctselect (C0 & C1), X, Y
+ // ct_select C0, (ct_select C1, X, Y), Y -> ct_select (C0 & C1), X, Y
// Semantic equivalence: If C0 is true, evaluate inner select (C1 ? X :
// Y). If C0 is false, choose Y. This is equivalent to (C0 && C1) ? X : Y.
- if (N1->getOpcode() == ISD::CTSELECT && N1->hasOneUse()) {
+ if (N1->getOpcode() == ISD::CT_SELECT && N1->hasOneUse()) {
SDValue N1_0 = N1->getOperand(0);
SDValue N1_1 = N1->getOperand(1);
SDValue N1_2 = N1->getOperand(2);
if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) {
SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
SDValue SelectOp =
- DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), And, N1_1, N2);
+ DAG.getNode(ISD::CT_SELECT, DL, N1.getValueType(), And, N1_1, N2);
SelectOp->setFlags(Flags);
return SelectOp;
}
}
- // ctselect C0, X, (ctselect C1, X, Y) -> ctselect (C0 | C1), X, Y
+ // ct_select C0, X, (ct_select C1, X, Y) -> ct_select (C0 | C1), X, Y
// Semantic equivalence: If C0 is true, choose X. If C0 is false, evaluate
// inner select (C1 ? X : Y). This is equivalent to (C0 || C1) ? X : Y.
- if (N2->getOpcode() == ISD::CTSELECT && N2->hasOneUse()) {
+ if (N2->getOpcode() == ISD::CT_SELECT && N2->hasOneUse()) {
SDValue N2_0 = N2->getOperand(0);
SDValue N2_1 = N2->getOperand(1);
SDValue N2_2 = N2->getOperand(2);
if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) {
SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
SDValue SelectOp =
- DAG.getNode(ISD::CTSELECT, DL, N1.getValueType(), Or, N1, N2_2);
+ DAG.getNode(ISD::CT_SELECT, DL, N1.getValueType(), Or, N1, N2_2);
SelectOp->setFlags(Flags);
return SelectOp;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index b2aa925bb55ae..0e4df41131889 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4350,7 +4350,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
}
Results.push_back(Tmp1);
break;
- case ISD::CTSELECT: {
+ case ISD::CT_SELECT: {
Tmp1 = Node->getOperand(0);
Tmp2 = Node->getOperand(1);
Tmp3 = Node->getOperand(2);
@@ -5799,7 +5799,7 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
break;
}
case ISD::SELECT:
- case ISD::CTSELECT: {
+ case ISD::CT_SELECT: {
unsigned ExtOp, TruncOp;
if (Node->getValueType(0).isVector() ||
Node->getValueType(0).getSizeInBits() == NVT.getSizeInBits()) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 0189d69381208..4dfca9b273418 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -161,7 +161,7 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
- case ISD::CTSELECT: R = SoftenFloatRes_CTSELECT(N); break;
+ case ISD::CT_SELECT: R = SoftenFloatRes_CT_SELECT(N); break;
case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
case ISD::STRICT_SINT_TO_FP:
@@ -1066,7 +1066,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
LHS.getValueType(), N->getOperand(0), LHS, RHS);
}
-SDValue DAGTypeLegalizer::SoftenFloatRes_CTSELECT(SDNode *N) {
+SDValue DAGTypeLegalizer::SoftenFloatRes_CT_SELECT(SDNode *N) {
SDValue LHS = GetSoftenedFloat(N->getOperand(1));
SDValue RHS = GetSoftenedFloat(N->getOperand(2));
return DAG.getCTSelect(SDLoc(N), LHS.getValueType(), N->getOperand(0), LHS,
@@ -1593,7 +1593,7 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
case ISD::POISON:
case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
- case ISD::CTSELECT: SplitRes_CTSELECT(N, Lo, Hi); break;
+ case ISD::CT_SELECT: SplitRes_CT_SELECT(N, Lo, Hi); break;
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
@@ -2772,8 +2772,8 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
break;
case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
- case ISD::CTSELECT:
- R = SoftPromoteHalfRes_CTSELECT(N);
+ case ISD::CT_SELECT:
+ R = SoftPromoteHalfRes_CT_SELECT(N);
break;
case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
case ISD::STRICT_SINT_TO_FP:
@@ -3042,7 +3042,7 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
Op2);
}
-SDValue DAGTypeLegalizer::SoftPromoteHalfRes_CTSELECT(SDNode *N) {
+SDValue DAGTypeLegalizer::SoftPromoteHalfRes_CT_SELECT(SDNode *N) {
SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
return DAG.getCTSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index ddbba309c1e96..c90703fa54278 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -98,7 +98,7 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
Res = PromoteIntRes_VECTOR_COMPRESS(N);
break;
case ISD::SELECT:
- case ISD::CTSELECT:
+ case ISD::CT_SELECT:
case ISD::VSELECT:
case ISD::VP_SELECT:
case ISD::VP_MERGE:
@@ -2174,8 +2174,8 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
break;
case ISD::VSELECT:
case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
- case ISD::CTSELECT:
- Res = PromoteIntOp_CTSELECT(N, OpNo);
+ case ISD::CT_SELECT:
+ Res = PromoteIntOp_CT_SELECT(N, OpNo);
break;
case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
case ISD::VP_SETCC:
@@ -2590,13 +2590,13 @@ SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
N->getOperand(2)), 0);
}
-SDValue DAGTypeLegalizer::PromoteIntOp_CTSELECT(SDNode *N, unsigned OpNo) {
+SDValue DAGTypeLegalizer::PromoteIntOp_CT_SELECT(SDNode *N, unsigned OpNo) {
assert(OpNo == 0 && "Only know how to promote the condition!");
SDValue Cond = N->getOperand(0);
EVT OpTy = N->getOperand(1).getValueType();
// Promote all the way up to the canonical SetCC type.
- EVT OpVT = N->getOpcode() == ISD::CTSELECT ? OpTy.getScalarType() : OpTy;
+ EVT OpVT = N->getOpcode() == ISD::CT_SELECT ? OpTy.getScalarType() : OpTy;
Cond = PromoteTargetBoolean(Cond, OpVT);
return SDValue(
@@ -3249,8 +3249,8 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break;
case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
- case ISD::CTSELECT:
- SplitRes_CTSELECT(N, Lo, Hi);
+ case ISD::CT_SELECT:
+ SplitRes_CT_SELECT(N, Lo, Hi);
break;
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
case ISD::POISON:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 535d1420fd620..6a6966603f355 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -389,7 +389,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntOp_CONCAT_VECTORS(SDNode *N);
SDValue PromoteIntOp_ScalarOp(SDNode *N);
SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo);
- SDValue PromoteIntOp_CTSELECT(SDNode *N, unsigned OpNo);
+ SDValue PromoteIntOp_CT_SELECT(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_SETCC(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_Shift(SDNode *N);
@@ -630,7 +630,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftenFloatRes_LOAD(SDNode *N);
SDValue SoftenFloatRes_ATOMIC_LOAD(SDNode *N);
SDValue SoftenFloatRes_SELECT(SDNode *N);
- SDValue SoftenFloatRes_CTSELECT(SDNode *N);
+ SDValue SoftenFloatRes_CT_SELECT(SDNode *N);
SDValue SoftenFloatRes_SELECT_CC(SDNode *N);
SDValue SoftenFloatRes_UNDEF(SDNode *N);
SDValue SoftenFloatRes_VAARG(SDNode *N);
@@ -793,7 +793,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftPromoteHalfRes_LOAD(SDNode *N);
SDValue SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N);
SDValue SoftPromoteHalfRes_SELECT(SDNode *N);
- SDValue SoftPromoteHalfRes_CTSELECT(SDNode *N);
+ SDValue SoftPromoteHalfRes_CT_SELECT(SDNode *N);
SDValue SoftPromoteHalfRes_SELECT_CC(SDNode *N);
SDValue SoftPromoteHalfRes_UnaryOp(SDNode *N);
SDValue SoftPromoteHalfRes_FABS(SDNode *N);
@@ -864,7 +864,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N);
SDValue ScalarizeVecRes_VSELECT(SDNode *N);
SDValue ScalarizeVecRes_SELECT(SDNode *N);
- SDValue ScalarizeVecRes_CTSELECT(SDNode *N);
+ SDValue ScalarizeVecRes_CT_SELECT(SDNode *N);
SDValue ScalarizeVecRes_SELECT_CC(SDNode *N);
SDValue ScalarizeVecRes_SETCC(SDNode *N);
SDValue ScalarizeVecRes_UNDEF(SDNode *N);
@@ -1208,7 +1208,7 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void SplitVecRes_AssertSext(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_ARITH_FENCE (SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_Select(SDNode *N, SDValue &Lo, SDValue &Hi);
- void SplitRes_CTSELECT(SDNode *N, SDValue &Lo, SDValue &Hi);
+ void SplitRes_CT_SELECT(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_SELECT_CC (SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_UNDEF (SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_FREEZE (SDNode *N, SDValue &Lo, SDValue &Hi);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
index 48bedbb9112ad..6e0fde28ae51b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
@@ -569,9 +569,9 @@ void DAGTypeLegalizer::SplitRes_Select(SDNode *N, SDValue &Lo, SDValue &Hi) {
Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH, EVLHi);
}
-void DAGTypeLegalizer::SplitRes_CTSELECT(SDNode *N, SDValue &Lo, SDValue &Hi) {
+void DAGTypeLegalizer::SplitRes_CT_SELECT(SDNode *N, SDValue &Lo, SDValue &Hi) {
// Reuse generic select splitting to support scalar and vector conditions.
- // SplitRes_Select rebuilds with N->getOpcode(), so CTSELECT is preserved.
+ // SplitRes_Select rebuilds with N->getOpcode(), so CT_SELECT is preserved.
SplitRes_Select(N, Lo, Hi);
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index d98d001be1003..49b38c1c7cf4a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -87,8 +87,8 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
- case ISD::CTSELECT:
- R = ScalarizeVecRes_CTSELECT(N);
+ case ISD::CT_SELECT:
+ R = ScalarizeVecRes_CT_SELECT(N);
break;
case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
@@ -767,7 +767,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
GetScalarizedVector(N->getOperand(2)));
}
-SDValue DAGTypeLegalizer::ScalarizeVecRes_CTSELECT(SDNode *N) {
+SDValue DAGTypeLegalizer::ScalarizeVecRes_CT_SELECT(SDNode *N) {
SDValue LHS = GetScalarizedVector(N->getOperand(1));
return DAG.getCTSelect(SDLoc(N), LHS.getValueType(), N->getOperand(0), LHS,
GetScalarizedVector(N->getOperand(2)));
@@ -1390,8 +1390,8 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
case ISD::SELECT:
case ISD::VP_MERGE:
case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
- case ISD::CTSELECT:
- SplitRes_CTSELECT(N, Lo, Hi);
+ case ISD::CT_SELECT:
+ SplitRes_CT_SELECT(N, Lo, Hi);
break;
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
case ISD::POISON:
@@ -5301,7 +5301,7 @@ void DAGTypeLegalizer::WidenVectorResult(SDNode *N, unsigned ResNo) {
case ISD::SIGN_EXTEND_INREG: Res = WidenVecRes_InregOp(N); break;
case ISD::VSELECT:
case ISD::SELECT:
- case ISD::CTSELECT:
+ case ISD::CT_SELECT:
case ISD::VP_SELECT:
case ISD::VP_MERGE:
Res = WidenVecRes_Select(N);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index c18199b625865..b763d0fd2bb69 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6990,8 +6990,9 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
assert(!CondVT.isVector() && "Vector type cond not supported yet");
// Handle scalar types
- if (TLI.isCtSelectSupported(VT) && !CondVT.isVector()) {
- SDValue Result = DAG.getNode(ISD::CTSELECT, DL, VT, Cond, A, B);
+ if (TLI.isOperationLegalOrCustom(ISD::CT_SELECT, VT) &&
+ !CondVT.isVector()) {
+ SDValue Result = DAG.getNode(ISD::CT_SELECT, DL, VT, Cond, A, B);
setValue(&I, Result);
return;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 789424191395a..7dc94fa1455cc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -346,7 +346,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::FPOWI: return "fpowi";
case ISD::STRICT_FPOWI: return "strict_fpowi";
case ISD::SETCC: return "setcc";
- case ISD::CTSELECT: return "ctselect";
+ case ISD::CT_SELECT: return "ct_select";
case ISD::SETCCCARRY: return "setcccarry";
case ISD::STRICT_FSETCC: return "strict_fsetcc";
case ISD::STRICT_FSETCCS: return "strict_fsetccs";
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 44ce04e6c4dac..fcd6e7b3051dd 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1160,7 +1160,8 @@ void TargetLoweringBase::initActions() {
ISD::FTANH, ISD::FATAN2,
ISD::FMULADD, ISD::CONVERT_FROM_ARBITRARY_FP,
ISD::CONVERT_TO_ARBITRARY_FP,
- ISD::PSEUDO_FMIN, ISD::PSEUDO_FMAX},
+ ISD::PSEUDO_FMIN, ISD::PSEUDO_FMAX,
+ ISD::CT_SELECT},
VT, Expand);
// clang-format on
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 6a453ba0df038..5cc4e9fa1b063 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -167,8 +167,6 @@ class AArch64TargetLowering : public TargetLowering {
EVT getSetCCResultType(const DataLayout &DL, LLVMContext &Context,
EVT VT) const override;
- bool isCtSelectSupported(EVT VT) const override { return false; }
-
SDValue ReconstructShuffle(SDValue Op, SelectionDAG &DAG) const;
MachineBasicBlock *EmitF128CSEL(MachineInstr &MI,
diff --git a/llvm/lib/Target/ARM/ARMISelLowering.h b/llvm/lib/Target/ARM/ARMISelLowering.h
index 72d0acf2dc7c1..10f5442d7429b 100644
--- a/llvm/lib/Target/ARM/ARMISelLowering.h
+++ b/llvm/lib/Target/ARM/ARMISelLowering.h
@@ -119,8 +119,6 @@ class VectorType;
return (Kind != ScalarCondVectorVal);
}
- bool isCtSelectSupported(EVT VT) const override { return false; }
-
bool isReadOnly(const GlobalValue *GV) const;
/// getSetCCResultType - Return the value type to use for ISD::SETCC.
diff --git a/llvm/lib/Target/X86/X86ISelLowering.h b/llvm/lib/Target/X86/X86ISelLowering.h
index 7d041dee6e345..798050028c15a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.h
+++ b/llvm/lib/Target/X86/X86ISelLowering.h
@@ -111,8 +111,6 @@ namespace llvm {
unsigned getJumpTableEncoding() const override;
bool useSoftFloat() const override;
- bool isCtSelectSupported(EVT VT) const override { return false; }
-
void markLibCallAttributes(MachineFunction *MF, unsigned CC,
ArgListTy &Args) const override;
diff --git a/llvm/test/CodeGen/RISCV/ctselect-fallback.ll b/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
index c624c17d7e33e..d4617c7e75da7 100644
--- a/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
+++ b/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
@@ -203,8 +203,8 @@ define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) {
ret i32 %result
}
-; Test nested CTSELECT pattern with AND merging on i1 values
-; Pattern: ctselect C0, (ctselect C1, X, Y), Y -> ctselect (C0 & C1), X, Y
+; Test nested CT_SELECT pattern with AND merging on i1 values
+; Pattern: ct_select C0, (ct_select C1, X, Y), Y -> ct_select (C0 & C1), X, Y
define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; RV64-LABEL: test_ctselect_nested_and_i1_to_i32:
; RV64: # %bb.0:
@@ -231,8 +231,8 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
ret i32 %result
}
-; Test nested CTSELECT pattern with OR merging on i1 values
-; Pattern: ctselect C0, X, (ctselect C1, X, Y) -> ctselect (C0 | C1), X, Y
+; Test nested CT_SELECT pattern with OR merging on i1 values
+; Pattern: ct_select C0, X, (ct_select C1, X, Y) -> ct_select (C0 | C1), X, Y
define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; RV64-LABEL: test_ctselect_nested_or_i1_to_i32:
; RV64: # %bb.0:
@@ -259,9 +259,9 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
ret i32 %result
}
-; Test double nested CTSELECT with recursive AND merging
-; Pattern: ctselect C0, (ctselect C1, (ctselect C2, X, Y), Y), Y
-; -> ctselect (C0 & C1 & C2), X, Y
+; Test double nested CT_SELECT with recursive AND merging
+; Pattern: ct_select C0, (ct_select C1, (ct_select C2, X, Y), Y), Y
+; -> ct_select (C0 & C1 & C2), X, Y
define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i32 %y) {
; RV64-LABEL: test_ctselect_double_nested_and_i1:
; RV64: # %bb.0:
@@ -291,7 +291,7 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
ret i32 %result
}
-; Test double nested CTSELECT with mixed AND/OR patterns
+; Test double nested CT_SELECT with mixed AND/OR patterns
define i32 @test_ctselect_double_nested_mixed_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i32 %y, i32 %z) {
; RV64-LABEL: test_ctselect_double_nested_mixed_i1:
; RV64: # %bb.0:
@@ -329,7 +329,7 @@ define i32 @test_ctselect_double_nested_mixed_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x,
ret i32 %result
}
-; Test nested ctselect calls
+; Test nested ct_select calls
define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
; RV64-LABEL: test_ctselect_nested:
; RV64: # %bb.0:
diff --git a/llvm/test/CodeGen/X86/ctselect.ll b/llvm/test/CodeGen/X86/ctselect.ll
index a970fd8933b9b..bf65e04721df1 100644
--- a/llvm/test/CodeGen/X86/ctselect.ll
+++ b/llvm/test/CodeGen/X86/ctselect.ll
@@ -552,7 +552,7 @@ define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) {
ret i32 %result
}
-; Test nested ctselect calls
+; Test nested ct_select calls
define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
; X64-LABEL: test_ctselect_nested:
; X64: # %bb.0:
@@ -631,8 +631,8 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
ret i32 %result
}
-; Test nested CTSELECT pattern with AND merging on i1 values
-; Pattern: ctselect C0, (ctselect C1, X, Y), Y -> ctselect (C0 & C1), X, Y
+; Test nested CT_SELECT pattern with AND merging on i1 values
+; Pattern: ct_select C0, (ct_select C1, X, Y), Y -> ct_select (C0 & C1), X, Y
; This optimization only applies when selecting between i1 values (boolean logic)
define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; X64-LABEL: test_ctselect_nested_and_i1_to_i32:
@@ -679,8 +679,8 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
ret i32 %result
}
-; Test nested CTSELECT pattern with OR merging on i1 values
-; Pattern: ctselect C0, X, (ctselect C1, X, Y) -> ctselect (C0 | C1), X, Y
+; Test nested CT_SELECT pattern with OR merging on i1 values
+; Pattern: ct_select C0, X, (ct_select C1, X, Y) -> ct_select (C0 | C1), X, Y
; This optimization only applies when selecting between i1 values (boolean logic)
define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; X64-LABEL: test_ctselect_nested_or_i1_to_i32:
@@ -727,10 +727,10 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
ret i32 %result
}
-; Test double nested CTSELECT with recursive AND merging
-; Pattern: ctselect C0, (ctselect C1, (ctselect C2, X, Y), Y), Y
-; -> ctselect C0, (ctselect (C1 & C2), X, Y), Y
-; -> ctselect (C0 & (C1 & C2)), X, Y
+; Test double nested CT_SELECT with recursive AND merging
+; Pattern: ct_select C0, (ct_select C1, (ct_select C2, X, Y), Y), Y
+; -> ct_select C0, (ct_select (C1 & C2), X, Y), Y
+; -> ct_select (C0 & (C1 & C2)), X, Y
; This tests that the optimization can be applied recursively
define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i32 %y) {
; X64-LABEL: test_ctselect_double_nested_and_i1:
@@ -781,10 +781,10 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
ret i32 %result
}
-; Vector CTSELECT Tests
+; Vector CT_SELECT Tests
; ============================================================================
-; Test vector CTSELECT with v4i32 (128-bit vector with single i1 mask)
+; Test vector CT_SELECT with v4i32 (128-bit vector with single i1 mask)
; NOW CONSTANT-TIME: Uses bitwise XOR/AND operations instead of branches!
define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) {
; X64-LABEL: test_ctselect_v4i32:
>From 904766a700e87dfa8f747997d9656b2e596fcf9f Mon Sep 17 00:00:00 2001
From: wizardengineer <juliuswoosebert at gmail.com>
Date: Sat, 7 Mar 2026 15:36:09 -0500
Subject: [PATCH 06/10] [ConstantTime] Fix CT_SELECT expansion to preserve
constant-time guarantees
Create CT_SELECT nodes for scalar types regardless of target support, so
they survive DAGCombiner (visitCT_SELECT is conservative). Expand to
AND/OR/XOR during operation legalization after SETCC is lowered, preventing
the sext(setcc)->select fold chain that converts constant-time patterns
into data-dependent conditional moves (e.g. movn/movz on MIPS).
The mask uses SUB(0, AND(Cond, 1)) instead of SIGN_EXTEND because type
legalization already promoted i1 to the SetCC result type, making
SIGN_EXTEND a no-op for same-width types.
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 20 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 16 +-
llvm/test/CodeGen/RISCV/ctselect-fallback.ll | 22 +-
llvm/test/CodeGen/X86/ctselect.ll | 259 ++++++++++--------
4 files changed, 179 insertions(+), 138 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 0e4df41131889..ca7a8bf6951c7 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4409,14 +4409,18 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Node->getFlags()));
} else {
assert(VT.isInteger());
- EVT HalfVT = VT.getHalfSizedIntegerVT(*DAG.getContext());
- auto [Tmp2Lo, Tmp2Hi] = DAG.SplitScalar(Tmp2, dl, HalfVT, HalfVT);
- auto [Tmp3Lo, Tmp3Hi] = DAG.SplitScalar(Tmp3, dl, HalfVT, HalfVT);
- SDValue ResLo =
- DAG.getCTSelect(dl, HalfVT, Tmp1, Tmp2Lo, Tmp3Lo, Node->getFlags());
- SDValue ResHi =
- DAG.getCTSelect(dl, HalfVT, Tmp1, Tmp2Hi, Tmp3Hi, Node->getFlags());
- Tmp1 = DAG.getNode(ISD::BUILD_PAIR, dl, VT, ResLo, ResHi);
+ // Expand: Result = F ^ ((T ^ F) & Mask), Mask = 0 - (Cond & 1).
+ // SUB+AND creates the mask because i1 is already type-promoted;
+ // SIGN_EXTEND(i32, i32) would be a no-op leaving mask as 0/1.
+ SDValue Cond = Tmp1;
+ if (Cond.getValueType() != VT)
+ Cond = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Cond);
+ SDValue Mask = DAG.getNode(
+ ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT),
+ DAG.getNode(ISD::AND, dl, VT, Cond, DAG.getConstant(1, dl, VT)));
+ SDValue Diff = DAG.getNode(ISD::XOR, dl, VT, Tmp2, Tmp3);
+ Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Tmp3,
+ DAG.getNode(ISD::AND, dl, VT, Diff, Mask));
Tmp1->setFlags(Node->getFlags());
}
Results.push_back(Tmp1);
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index b763d0fd2bb69..f40cc5d0c202a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6989,9 +6989,19 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
// assert if Cond type is Vector
assert(!CondVT.isVector() && "Vector type cond not supported yet");
- // Handle scalar types
- if (TLI.isOperationLegalOrCustom(ISD::CT_SELECT, VT) &&
- !CondVT.isVector()) {
+ // Create a CT_SELECT node for scalar types so it survives DAGCombiner
+ // (visitCT_SELECT is conservative) and expands to AND/OR/XOR during
+ // operation legalization, after SETCC is lowered. Unsupported vectors
+ // and floats with illegal integer equivalents (e.g. f64 on i386) use
+ // the inline fallback which runs before type legalization.
+ bool CreateNode =
+ TLI.isOperationLegalOrCustom(ISD::CT_SELECT, VT) ||
+ (!VT.isVector() &&
+ (!VT.isFloatingPoint() ||
+ TLI.isTypeLegal(
+ EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()))));
+
+ if (CreateNode) {
SDValue Result = DAG.getNode(ISD::CT_SELECT, DL, VT, Cond, A, B);
setValue(&I, Result);
return;
diff --git a/llvm/test/CodeGen/RISCV/ctselect-fallback.ll b/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
index d4617c7e75da7..ee8072703ee31 100644
--- a/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
+++ b/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
@@ -101,8 +101,6 @@ define i32 @test_ctselect_const_true(i32 %a, i32 %b) {
;
; RV32-LABEL: test_ctselect_const_true:
; RV32: # %bb.0:
-; RV32-NEXT: xor a0, a0, a1
-; RV32-NEXT: xor a0, a1, a0
; RV32-NEXT: ret
%result = call i32 @llvm.ct.select.i32(i1 true, i32 %a, i32 %b)
ret i32 %result
@@ -208,7 +206,7 @@ define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) {
define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; RV64-LABEL: test_ctselect_nested_and_i1_to_i32:
; RV64: # %bb.0:
-; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: and a0, a0, a1
; RV64-NEXT: xor a2, a2, a3
; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: srai a0, a0, 63
@@ -218,7 +216,7 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
;
; RV32-LABEL: test_ctselect_nested_and_i1_to_i32:
; RV32: # %bb.0:
-; RV32-NEXT: and a0, a1, a0
+; RV32-NEXT: and a0, a0, a1
; RV32-NEXT: xor a2, a2, a3
; RV32-NEXT: slli a0, a0, 31
; RV32-NEXT: srai a0, a0, 31
@@ -265,8 +263,8 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i32 %y) {
; RV64-LABEL: test_ctselect_double_nested_and_i1:
; RV64: # %bb.0:
-; RV64-NEXT: and a1, a2, a1
-; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: and a0, a0, a1
+; RV64-NEXT: and a0, a0, a2
; RV64-NEXT: xor a3, a3, a4
; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: srai a0, a0, 63
@@ -276,8 +274,8 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
;
; RV32-LABEL: test_ctselect_double_nested_and_i1:
; RV32: # %bb.0:
-; RV32-NEXT: and a1, a2, a1
-; RV32-NEXT: and a0, a1, a0
+; RV32-NEXT: and a0, a0, a1
+; RV32-NEXT: and a0, a0, a2
; RV32-NEXT: xor a3, a3, a4
; RV32-NEXT: slli a0, a0, 31
; RV32-NEXT: srai a0, a0, 31
@@ -295,7 +293,7 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
define i32 @test_ctselect_double_nested_mixed_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i32 %y, i32 %z) {
; RV64-LABEL: test_ctselect_double_nested_mixed_i1:
; RV64: # %bb.0:
-; RV64-NEXT: and a0, a1, a0
+; RV64-NEXT: and a0, a0, a1
; RV64-NEXT: xor a3, a3, a4
; RV64-NEXT: or a0, a0, a2
; RV64-NEXT: slli a0, a0, 63
@@ -309,7 +307,7 @@ define i32 @test_ctselect_double_nested_mixed_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x,
;
; RV32-LABEL: test_ctselect_double_nested_mixed_i1:
; RV32: # %bb.0:
-; RV32-NEXT: and a0, a1, a0
+; RV32-NEXT: and a0, a0, a1
; RV32-NEXT: xor a3, a3, a4
; RV32-NEXT: or a0, a0, a2
; RV32-NEXT: slli a0, a0, 31
@@ -382,7 +380,7 @@ define float @test_ctselect_f32_nan_inf(i1 %cond) {
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a0, a0, a1
; RV32-NEXT: lui a1, 522240
-; RV32-NEXT: xor a0, a0, a1
+; RV32-NEXT: or a0, a0, a1
; RV32-NEXT: ret
%result = call float @llvm.ct.select.f32(i1 %cond, float 0x7FF8000000000000, float 0x7FF0000000000000)
ret float %result
@@ -398,7 +396,7 @@ define double @test_ctselect_f64_nan_inf(i1 %cond) {
; RV64-NEXT: and a0, a0, a1
; RV64-NEXT: li a1, 2047
; RV64-NEXT: slli a1, a1, 52
-; RV64-NEXT: xor a0, a0, a1
+; RV64-NEXT: or a0, a0, a1
; RV64-NEXT: ret
;
; RV32-LABEL: test_ctselect_f64_nan_inf:
diff --git a/llvm/test/CodeGen/X86/ctselect.ll b/llvm/test/CodeGen/X86/ctselect.ll
index bf65e04721df1..e1abae80cef4f 100644
--- a/llvm/test/CodeGen/X86/ctselect.ll
+++ b/llvm/test/CodeGen/X86/ctselect.ll
@@ -9,8 +9,8 @@ define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) {
; X64-LABEL: test_ctselect_i8:
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
-; X64-NEXT: xorl %edx, %esi
; X64-NEXT: andb $1, %al
+; X64-NEXT: xorl %edx, %esi
; X64-NEXT: negb %al
; X64-NEXT: andb %sil, %al
; X64-NEXT: xorb %dl, %al
@@ -20,10 +20,10 @@ define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) {
; X32-LABEL: test_ctselect_i8:
; X32: # %bb.0:
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb $1, %al
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
; X32-NEXT: xorb %cl, %dl
-; X32-NEXT: andb $1, %al
; X32-NEXT: negb %al
; X32-NEXT: andb %dl, %al
; X32-NEXT: xorb %cl, %al
@@ -32,10 +32,10 @@ define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) {
; X32-NOCMOV-LABEL: test_ctselect_i8:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: xorb %cl, %dl
-; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: negb %al
; X32-NOCMOV-NEXT: andb %dl, %al
; X32-NOCMOV-NEXT: xorb %cl, %al
@@ -58,10 +58,11 @@ define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) {
; X32-LABEL: test_ctselect_i32:
; X32: # %bb.0:
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb $1, %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl $1, %eax
+; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
; X32-NEXT: andl %edx, %eax
; X32-NEXT: xorl %ecx, %eax
@@ -70,10 +71,11 @@ define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) {
; X32-NOCMOV-LABEL: test_ctselect_i32:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
; X32-NOCMOV-NEXT: andl %edx, %eax
; X32-NOCMOV-NEXT: xorl %ecx, %eax
@@ -95,45 +97,57 @@ define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) {
;
; X32-LABEL: test_ctselect_i64:
; X32: # %bb.0:
-; X32-NEXT: pushl %esi
+; X32-NEXT: pushl %edi
; X32-NEXT: .cfi_def_cfa_offset 8
-; X32-NEXT: .cfi_offset %esi, -8
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: pushl %esi
+; X32-NEXT: .cfi_def_cfa_offset 12
+; X32-NEXT: .cfi_offset %esi, -12
+; X32-NEXT: .cfi_offset %edi, -8
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: andb $1, %dl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %edx, %eax
-; X32-NEXT: andl $1, %esi
-; X32-NEXT: negl %esi
-; X32-NEXT: andl %esi, %eax
-; X32-NEXT: xorl %edx, %eax
+; X32-NEXT: xorl %esi, %eax
+; X32-NEXT: movzbl %dl, %edi
+; X32-NEXT: negl %edi
+; X32-NEXT: andl %edi, %eax
+; X32-NEXT: xorl %esi, %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl %esi, %edx
+; X32-NEXT: andl %edi, %edx
; X32-NEXT: xorl %ecx, %edx
; X32-NEXT: popl %esi
+; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: popl %edi
; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_i64:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: pushl %edi
; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; X32-NOCMOV-NEXT: .cfi_offset %esi, -8
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
+; X32-NOCMOV-NEXT: .cfi_offset %esi, -12
+; X32-NOCMOV-NEXT: .cfi_offset %edi, -8
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: andb $1, %dl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %edx, %eax
-; X32-NOCMOV-NEXT: andl $1, %esi
-; X32-NOCMOV-NEXT: negl %esi
-; X32-NOCMOV-NEXT: andl %esi, %eax
-; X32-NOCMOV-NEXT: xorl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %esi, %eax
+; X32-NOCMOV-NEXT: movzbl %dl, %edi
+; X32-NOCMOV-NEXT: negl %edi
+; X32-NOCMOV-NEXT: andl %edi, %eax
+; X32-NOCMOV-NEXT: xorl %esi, %eax
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl %esi, %edx
+; X32-NOCMOV-NEXT: andl %edi, %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: popl %edi
; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl
%result = call i64 @llvm.ct.select.i64(i1 %cond, i64 %a, i64 %b)
@@ -155,37 +169,47 @@ define float @test_ctselect_f32(i1 %cond, float %a, float %b) {
;
; X32-LABEL: test_ctselect_f32:
; X32: # %bb.0:
-; X32-NEXT: pushl %eax
-; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: subl $12, %esp
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb $1, %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl $1, %eax
+; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: movl %eax, (%esp)
-; X32-NEXT: flds (%esp)
-; X32-NEXT: popl %eax
+; X32-NEXT: movl (%esp), %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: andl %eax, %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: addl $12, %esp
; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f32:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: pushl %eax
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: subl $12, %esp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: movl %eax, (%esp)
-; X32-NOCMOV-NEXT: flds (%esp)
-; X32-NOCMOV-NEXT: popl %eax
+; X32-NOCMOV-NEXT: movl (%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: andl %eax, %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: addl $12, %esp
; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl
%result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
@@ -281,10 +305,11 @@ define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) {
; X32-LABEL: test_ctselect_ptr:
; X32: # %bb.0:
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb $1, %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl $1, %eax
+; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
; X32-NEXT: andl %edx, %eax
; X32-NEXT: xorl %ecx, %eax
@@ -293,10 +318,11 @@ define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) {
; X32-NOCMOV-LABEL: test_ctselect_ptr:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
; X32-NOCMOV-NEXT: andl %edx, %eax
; X32-NOCMOV-NEXT: xorl %ecx, %eax
@@ -310,24 +336,16 @@ define i32 @test_ctselect_const_true(i32 %a, i32 %b) {
; X64-LABEL: test_ctselect_const_true:
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
-; X64-NEXT: xorl %esi, %eax
-; X64-NEXT: xorl %esi, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_const_true:
; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %ecx, %eax
-; X32-NEXT: xorl %ecx, %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_const_true:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
-; X32-NOCMOV-NEXT: xorl %ecx, %eax
; X32-NOCMOV-NEXT: retl
%result = call i32 @llvm.ct.select.i32(i1 true, i32 %a, i32 %b)
ret i32 %result
@@ -341,14 +359,12 @@ define i32 @test_ctselect_const_false(i32 %a, i32 %b) {
;
; X32-LABEL: test_ctselect_const_false:
; X32: # %bb.0:
-; X32-NEXT: xorl %eax, %eax
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_const_false:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: xorl %eax, %eax
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: retl
%result = call i32 @llvm.ct.select.i32(i1 false, i32 %a, i32 %b)
ret i32 %result
@@ -443,19 +459,20 @@ define i32 @test_ctselect_icmp_ult(i32 %x, i32 %y, i32 %a, i32 %b) {
define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) {
; X64-LABEL: test_ctselect_fcmp_oeq:
; X64: # %bb.0:
-; X64-NEXT: movd %xmm3, %eax
; X64-NEXT: cmpeqss %xmm1, %xmm0
-; X64-NEXT: pxor %xmm3, %xmm2
-; X64-NEXT: pand %xmm0, %xmm2
-; X64-NEXT: movd %xmm2, %ecx
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: movd %ecx, %xmm0
+; X64-NEXT: xorps %xmm3, %xmm2
+; X64-NEXT: andps %xmm2, %xmm0
+; X64-NEXT: xorps %xmm3, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_fcmp_oeq:
; X32: # %bb.0:
-; X32-NEXT: pushl %eax
-; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: subl $12, %esp
+; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
@@ -466,20 +483,24 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) {
; X32-NEXT: andb %cl, %dl
; X32-NEXT: movzbl %dl, %ecx
; X32-NEXT: negl %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl (%esp), %edx
; X32-NEXT: xorl %eax, %edx
; X32-NEXT: andl %ecx, %edx
; X32-NEXT: xorl %eax, %edx
-; X32-NEXT: movl %edx, (%esp)
-; X32-NEXT: flds (%esp)
-; X32-NEXT: popl %eax
+; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: addl $12, %esp
; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_fcmp_oeq:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: pushl %eax
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: subl $12, %esp
+; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
@@ -492,13 +513,13 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) {
; X32-NOCMOV-NEXT: andb %al, %dl
; X32-NOCMOV-NEXT: movzbl %dl, %eax
; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl (%esp), %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: andl %eax, %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movl %edx, (%esp)
-; X32-NOCMOV-NEXT: flds (%esp)
-; X32-NOCMOV-NEXT: popl %eax
+; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: addl $12, %esp
; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl
%cond = fcmp oeq float %x, %y
@@ -522,12 +543,13 @@ define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) {
; X32-LABEL: test_ctselect_load:
; X32: # %bb.0:
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb $1, %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: movl (%edx), %edx
; X32-NEXT: movl (%ecx), %ecx
; X32-NEXT: xorl %edx, %ecx
-; X32-NEXT: andl $1, %eax
+; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
; X32-NEXT: andl %ecx, %eax
; X32-NEXT: xorl %edx, %eax
@@ -536,12 +558,13 @@ define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) {
; X32-NOCMOV-LABEL: test_ctselect_load:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl (%edx), %edx
; X32-NOCMOV-NEXT: movl (%ecx), %ecx
; X32-NOCMOV-NEXT: xorl %edx, %ecx
-; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
; X32-NOCMOV-NEXT: andl %ecx, %eax
; X32-NOCMOV-NEXT: xorl %edx, %eax
@@ -578,17 +601,19 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
; X32-NEXT: .cfi_offset %esi, -12
; X32-NEXT: .cfi_offset %edi, -8
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb $1, %al
+; X32-NEXT: movb {{[0-9]+}}(%esp), %ah
+; X32-NEXT: andb $1, %ah
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %esi
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: xorl %edx, %edi
-; X32-NEXT: andl $1, %esi
-; X32-NEXT: negl %esi
-; X32-NEXT: andl %edi, %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl %edx, %esi
+; X32-NEXT: movzbl %ah, %edi
+; X32-NEXT: negl %edi
+; X32-NEXT: andl %esi, %edi
; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: xorl %esi, %edx
-; X32-NEXT: andl $1, %eax
+; X32-NEXT: xorl %edi, %edx
+; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
; X32-NEXT: andl %edx, %eax
; X32-NEXT: xorl %ecx, %eax
@@ -607,17 +632,19 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
; X32-NOCMOV-NEXT: .cfi_offset %esi, -12
; X32-NOCMOV-NEXT: .cfi_offset %edi, -8
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %ah
+; X32-NOCMOV-NEXT: andb $1, %ah
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %esi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: xorl %edx, %edi
-; X32-NOCMOV-NEXT: andl $1, %esi
-; X32-NOCMOV-NEXT: negl %esi
-; X32-NOCMOV-NEXT: andl %edi, %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl %edx, %esi
+; X32-NOCMOV-NEXT: movzbl %ah, %edi
+; X32-NOCMOV-NEXT: negl %edi
+; X32-NOCMOV-NEXT: andl %esi, %edi
; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: xorl %esi, %edx
-; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: xorl %edi, %edx
+; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
; X32-NOCMOV-NEXT: andl %edx, %eax
; X32-NOCMOV-NEXT: xorl %ecx, %eax
@@ -651,10 +678,10 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NEXT: movzbl %al, %eax
+; X32-NEXT: andb $1, %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl $1, %eax
+; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
; X32-NEXT: andl %edx, %eax
; X32-NEXT: xorl %ecx, %eax
@@ -665,10 +692,10 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: movzbl %al, %eax
+; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
; X32-NOCMOV-NEXT: andl %edx, %eax
; X32-NOCMOV-NEXT: xorl %ecx, %eax
@@ -699,10 +726,10 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NEXT: orb {{[0-9]+}}(%esp), %al
-; X32-NEXT: movzbl %al, %eax
+; X32-NEXT: andb $1, %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl $1, %eax
+; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
; X32-NEXT: andl %edx, %eax
; X32-NEXT: xorl %ecx, %eax
@@ -713,10 +740,10 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: orb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: movzbl %al, %eax
+; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
; X32-NOCMOV-NEXT: andl %edx, %eax
; X32-NOCMOV-NEXT: xorl %ecx, %eax
@@ -735,9 +762,9 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i32 %y) {
; X64-LABEL: test_ctselect_double_nested_and_i1:
; X64: # %bb.0:
-; X64-NEXT: movl %esi, %eax
+; X64-NEXT: movl %edi, %eax
+; X64-NEXT: andl %esi, %eax
; X64-NEXT: andl %edx, %eax
-; X64-NEXT: andl %edi, %eax
; X64-NEXT: xorl %r8d, %ecx
; X64-NEXT: andl $1, %eax
; X64-NEXT: negl %eax
@@ -751,10 +778,10 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NEXT: andb {{[0-9]+}}(%esp), %al
; X32-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NEXT: movzbl %al, %eax
+; X32-NEXT: andb $1, %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: andl $1, %eax
+; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
; X32-NEXT: andl %edx, %eax
; X32-NEXT: xorl %ecx, %eax
@@ -766,10 +793,10 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
-; X32-NOCMOV-NEXT: movzbl %al, %eax
+; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
; X32-NOCMOV-NEXT: andl %edx, %eax
; X32-NOCMOV-NEXT: xorl %ecx, %eax
@@ -1403,7 +1430,7 @@ define float @test_ctselect_f32_nan_inf(i1 %cond) {
; X64-NEXT: andl $1, %edi
; X64-NEXT: negl %edi
; X64-NEXT: andl $4194304, %edi # imm = 0x400000
-; X64-NEXT: xorl $2139095040, %edi # imm = 0x7F800000
+; X64-NEXT: orl $2139095040, %edi # imm = 0x7F800000
; X64-NEXT: movd %edi, %xmm0
; X64-NEXT: retq
;
@@ -1412,10 +1439,11 @@ define float @test_ctselect_f32_nan_inf(i1 %cond) {
; X32-NEXT: pushl %eax
; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andl $1, %eax
+; X32-NEXT: andb $1, %al
+; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
; X32-NEXT: andl $4194304, %eax # imm = 0x400000
-; X32-NEXT: xorl $2139095040, %eax # imm = 0x7F800000
+; X32-NEXT: orl $2139095040, %eax # imm = 0x7F800000
; X32-NEXT: movl %eax, (%esp)
; X32-NEXT: flds (%esp)
; X32-NEXT: popl %eax
@@ -1427,10 +1455,11 @@ define float @test_ctselect_f32_nan_inf(i1 %cond) {
; X32-NOCMOV-NEXT: pushl %eax
; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
; X32-NOCMOV-NEXT: andl $4194304, %eax # imm = 0x400000
-; X32-NOCMOV-NEXT: xorl $2139095040, %eax # imm = 0x7F800000
+; X32-NOCMOV-NEXT: orl $2139095040, %eax # imm = 0x7F800000
; X32-NOCMOV-NEXT: movl %eax, (%esp)
; X32-NOCMOV-NEXT: flds (%esp)
; X32-NOCMOV-NEXT: popl %eax
@@ -1449,7 +1478,7 @@ define double @test_ctselect_f64_nan_inf(i1 %cond) {
; X64-NEXT: movabsq $2251799813685248, %rax # imm = 0x8000000000000
; X64-NEXT: andq %rdi, %rax
; X64-NEXT: movabsq $9218868437227405312, %rcx # imm = 0x7FF0000000000000
-; X64-NEXT: xorq %rax, %rcx
+; X64-NEXT: orq %rax, %rcx
; X64-NEXT: movq %rcx, %xmm0
; X64-NEXT: retq
;
>From 630980874b81ed8368763101adf9e1be829316d4 Mon Sep 17 00:00:00 2001
From: AkshayK <iit.akshay at gmail.com>
Date: Thu, 21 May 2026 17:19:01 -0400
Subject: [PATCH 07/10] [ConstantTime] Move ct.select lowering to legalizer;
misc fixes
- LegalizeDAG owns CT_SELECT lowering: scalar-mask+splat for vectors
(avoids illegal vNi1), memory-blend for FP types lacking a legal
same-size integer (f64 on i386 no-SSE, fp128, x86_fp80)
- Add InstSimplify folds for constant cond and identical arms
- Extend X86 tests to half/bfloat/fp128/x86_fp80; nounwind cleanup
- Preserve flags through legalizer; reuse SDTSelect
---
.../include/llvm/Target/TargetSelectionDAG.td | 7 +-
llvm/lib/Analysis/InstructionSimplify.cpp | 16 +
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 17 +-
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 208 +-
.../SelectionDAG/LegalizeFloatTypes.cpp | 4 +-
.../SelectionDAG/LegalizeVectorTypes.cpp | 2 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 123 +-
.../SelectionDAG/SelectionDAGBuilder.h | 3 -
llvm/test/CodeGen/RISCV/ctselect-fallback.ll | 74 +-
llvm/test/CodeGen/X86/ctselect.ll | 2080 ++++++++++++-----
.../test/Transforms/InstSimplify/ct-select.ll | 102 +
11 files changed, 1847 insertions(+), 789 deletions(-)
create mode 100644 llvm/test/Transforms/InstSimplify/ct-select.ll
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index f599a80058dd4..4a44fbe7c1480 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -214,11 +214,6 @@ def SDTSelect : SDTypeProfile<1, 3, [ // select
SDTCisInt<1>, SDTCisSameAs<0, 2>, SDTCisSameAs<2, 3>
]>;
-def SDTCtSelect
- : SDTypeProfile<1, 3,
- [ // ctselect
- SDTCisInt<1>, SDTCisSameAs<0, 2>, SDTCisSameAs<2, 3>]>;
-
def SDTVSelect : SDTypeProfile<1, 3, [ // vselect
SDTCisVec<0>, SDTCisInt<1>, SDTCisSameAs<0, 2>, SDTCisSameAs<2, 3>, SDTCisSameNumEltsAs<0, 1>
]>;
@@ -772,7 +767,7 @@ def reset_fpmode : SDNode<"ISD::RESET_FPMODE", SDTNone, [SDNPHasChain]>;
def setcc : SDNode<"ISD::SETCC" , SDTSetCC>;
def select : SDNode<"ISD::SELECT" , SDTSelect>;
-def ct_select : SDNode<"ISD::CT_SELECT", SDTCtSelect>;
+def ct_select : SDNode<"ISD::CT_SELECT", SDTSelect>;
def vselect : SDNode<"ISD::VSELECT" , SDTVSelect>;
def selectcc : SDNode<"ISD::SELECT_CC" , SDTSelectCC>;
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index ddaa44f43e631..6275592155b99 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -7552,6 +7552,22 @@ static Value *simplifyIntrinsic(CallBase *Call, ArrayRef<Value *> Args,
}
return nullptr;
}
+ case Intrinsic::ct_select: {
+ // Only fold on a literal IR-constant condition or identical arms. Folding
+ // through ValueTracking-derived known bits would defeat the constant-time
+ // contract on conditions the user wants kept opaque.
+ Value *Cond = Args[0], *TrueVal = Args[1], *FalseVal = Args[2];
+ if (auto *CI = dyn_cast<ConstantInt>(Cond)) {
+ if (CI->isOne())
+ return TrueVal;
+ if (CI->isZero())
+ return FalseVal;
+ }
+ // ct.select C, X, X -> X: the condition is unused, so no secret is exposed.
+ if (TrueVal == FalseVal)
+ return TrueVal;
+ return nullptr;
+ }
default: {
// Use the default FP environment if none is found.
fp::ExceptionBehavior ExBehavior = fp::ebIgnore;
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 1293518113eeb..aa1d0bd485cc4 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -13454,11 +13454,8 @@ SDValue DAGCombiner::visitCT_SELECT(SDNode *N) {
// This is a CT-safe canonicalization: flip negated condition by swapping
// arms. extractBooleanFlip only matches boolean xor-with-1, so this preserves
// dataflow semantics and does not introduce data-dependent control flow.
- if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false)) {
- SDValue SelectOp = DAG.getNode(ISD::CT_SELECT, DL, VT, F, N2, N1);
- SelectOp->setFlags(Flags);
- return SelectOp;
- }
+ if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false))
+ return DAG.getCTSelect(DL, VT, F, N2, N1, Flags);
if (VT0 == MVT::i1) {
// Nested CT_SELECT merging optimizations for i1 conditions.
@@ -13478,10 +13475,7 @@ SDValue DAGCombiner::visitCT_SELECT(SDNode *N) {
SDValue N1_2 = N1->getOperand(2);
if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) {
SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
- SDValue SelectOp =
- DAG.getNode(ISD::CT_SELECT, DL, N1.getValueType(), And, N1_1, N2);
- SelectOp->setFlags(Flags);
- return SelectOp;
+ return DAG.getCTSelect(DL, N1.getValueType(), And, N1_1, N2, Flags);
}
}
@@ -13494,10 +13488,7 @@ SDValue DAGCombiner::visitCT_SELECT(SDNode *N) {
SDValue N2_2 = N2->getOperand(2);
if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) {
SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
- SDValue SelectOp =
- DAG.getNode(ISD::CT_SELECT, DL, N1.getValueType(), Or, N1, N2_2);
- SelectOp->setFlags(Flags);
- return SelectOp;
+ return DAG.getCTSelect(DL, N1.getValueType(), Or, N1, N2_2, Flags);
}
}
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index ca7a8bf6951c7..de06ed1015e96 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -26,6 +26,7 @@
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineMemOperand.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/RuntimeLibcallUtil.h"
#include "llvm/CodeGen/SelectionDAG.h"
#include "llvm/CodeGen/SelectionDAGNodes.h"
@@ -4351,78 +4352,155 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Results.push_back(Tmp1);
break;
case ISD::CT_SELECT: {
- Tmp1 = Node->getOperand(0);
- Tmp2 = Node->getOperand(1);
- Tmp3 = Node->getOperand(2);
+ // Constant-time select: F ^ ((T ^ F) & Mask), Mask = 0 - (cond & 1).
+ // Bitwise-only — no select/cmov SDNode is constructed, so the CT property
+ // holds against any combiner that targets those opcodes. FP types operate
+ // on the same-size integer; vectors build the mask as a scalar then splat
+ // (avoids illegal vNi1).
+ //
+ // The masked-diff is routed through a virtual register (CopyToReg /
+ // CopyFromReg) below as a forward-looking DAGCombine barrier. This is
+ // *not* required for correctness against any combiner in tree today —
+ // DAGCombiner has no rewrite that recognizes XOR/AND/XOR-with-sext-mask
+ // and reconstructs a SELECT. The chain edge is defense-in-depth against
+ // a hypothetical future fold of that form: the dependency partitions the
+ // bitwise sequence into a region the combiner can't see through. Cost is
+ // at most a coalesce-able MOV per call. Swap for ARITH_FENCE when its
+ // int/vector extension lands.
+ Tmp1 = Node->getOperand(0); // cond
+ Tmp2 = Node->getOperand(1); // T
+ Tmp3 = Node->getOperand(2); // F
EVT VT = Tmp2.getValueType();
- if (VT.isVector()) {
- // Constant-time vector blending using pattern F ^ ((T ^ F) & Mask)
- // where Mask = broadcast(i1 ? -1 : 0) to match vector element width.
- //
- // This formulation uses only XOR and AND operations, avoiding branches
- // that would leak timing information. It's equivalent to:
- // Mask==0xFF: F ^ ((T ^ F) & 0xFF) = F ^ (T ^ F) = T
- // Mask==0x00: F ^ ((T ^ F) & 0x00) = F ^ 0 = F
-
- EVT IntVT = VT;
- SDValue T = Tmp2; // True value
- SDValue F = Tmp3; // False value
-
- // Step 1: Handle floating-point vectors by bitcasting to integer
- if (VT.isFloatingPoint()) {
- IntVT = EVT::getVectorVT(
+
+ // Memory-blend for FP scalars whose same-size integer isn't legal (f64 on
+ // i386 no-SSE, x86_fp80, fp128). The bitcast-to-int expansion below can't
+ // run since LegalizeDAG is the last legalization stage. Spill T/F, blend
+ // chunk-by-chunk at a legal int width, reload as the FP type. Fixed and
+ // scalable FP vectors fall through to the unified path below.
+ if (VT.isFloatingPoint() && !VT.isVector() &&
+ !TLI.isTypeLegal(VT.changeTypeToInteger())) {
+ const DataLayout &DL = DAG.getDataLayout();
+ Type *VTTy = VT.getTypeForEVT(*DAG.getContext());
+ unsigned StorageBytes = DL.getTypeStoreSize(VTTy);
+ assert(StorageBytes > 0 && "FP type with zero storage size");
+
+ // Pick the largest legal scalar integer chunk that divides StorageBytes
+ // evenly. i8 is the universal fallback (legal on every target).
+ MVT ChunkVT = MVT::i8;
+ for (MVT MV : {MVT::i64, MVT::i32, MVT::i16}) {
+ unsigned MVBytes = MV.getSizeInBits() / 8;
+ if (TLI.isTypeLegal(MV) && MVBytes <= StorageBytes &&
+ StorageBytes % MVBytes == 0) {
+ ChunkVT = MV;
+ break;
+ }
+ }
+ unsigned ChunkBytes = ChunkVT.getSizeInBits() / 8;
+ unsigned NumChunks = StorageBytes / ChunkBytes;
+
+ MachineFunction &MF = DAG.getMachineFunction();
+ SDValue StackT = DAG.CreateStackTemporary(VT);
+ SDValue StackF = DAG.CreateStackTemporary(VT);
+ SDValue StackR = DAG.CreateStackTemporary(VT);
+ int FIT = cast<FrameIndexSDNode>(StackT.getNode())->getIndex();
+ int FIF = cast<FrameIndexSDNode>(StackF.getNode())->getIndex();
+ int FIR = cast<FrameIndexSDNode>(StackR.getNode())->getIndex();
+ MachinePointerInfo PIT = MachinePointerInfo::getFixedStack(MF, FIT);
+ MachinePointerInfo PIF = MachinePointerInfo::getFixedStack(MF, FIF);
+ MachinePointerInfo PIR = MachinePointerInfo::getFixedStack(MF, FIR);
+
+ SDValue Chain = DAG.getEntryNode();
+ Chain = DAG.getStore(Chain, dl, Tmp2, StackT, PIT);
+ Chain = DAG.getStore(Chain, dl, Tmp3, StackF, PIF);
+
+ for (unsigned i = 0; i < NumChunks; ++i) {
+ TypeSize Off = TypeSize::getFixed(i * ChunkBytes);
+ SDValue TPtr = DAG.getMemBasePlusOffset(StackT, Off, dl);
+ SDValue FPtr = DAG.getMemBasePlusOffset(StackF, Off, dl);
+ SDValue RPtr = DAG.getMemBasePlusOffset(StackR, Off, dl);
+
+ SDValue Ti = DAG.getLoad(ChunkVT, dl, Chain, TPtr,
+ PIT.getWithOffset(i * ChunkBytes));
+ Chain = Ti.getValue(1);
+ SDValue Fi = DAG.getLoad(ChunkVT, dl, Chain, FPtr,
+ PIF.getWithOffset(i * ChunkBytes));
+ Chain = Fi.getValue(1);
+
+ // Blend this chunk via CT_SELECT on a legal integer type. The
+ // recursive node will be Expand'd by the scalar-int branch below.
+ SDValue Ri =
+ DAG.getCTSelect(dl, ChunkVT, Tmp1, Ti, Fi, Node->getFlags());
+
+ Chain = DAG.getStore(Chain, dl, Ri, RPtr,
+ PIR.getWithOffset(i * ChunkBytes));
+ }
+
+ Tmp1 = DAG.getLoad(VT, dl, Chain, StackR, PIR);
+ Tmp1->setFlags(Node->getFlags());
+ Results.push_back(Tmp1);
+ break;
+ }
+
+ SDValue WorkingT = Tmp2;
+ SDValue WorkingF = Tmp3;
+ EVT WorkingVT = VT;
+
+ bool IsFP = VT.isVector() ? VT.getVectorElementType().isFloatingPoint()
+ : VT.isFloatingPoint();
+ if (IsFP) {
+ if (VT.isVector())
+ WorkingVT = EVT::getVectorVT(
*DAG.getContext(),
EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits()),
VT.getVectorElementCount());
- T = DAG.getNode(ISD::BITCAST, dl, IntVT, T);
- F = DAG.getNode(ISD::BITCAST, dl, IntVT, F);
- }
+ else
+ WorkingVT = VT.changeTypeToInteger();
+ WorkingT = DAG.getBitcast(WorkingVT, Tmp2);
+ WorkingF = DAG.getBitcast(WorkingVT, Tmp3);
+ }
- // Step 2: Broadcast the i1 condition to a vector of i1s
- // Creates [cond, cond, cond, ...] with i1 elements
- EVT VecI1Ty = EVT::getVectorVT(*DAG.getContext(), MVT::i1,
- VT.getVectorNumElements());
- SDValue VecCond = DAG.getSplatBuildVector(VecI1Ty, dl, Tmp1);
-
- // Step 3: Sign-extend i1 vector to get all-bits mask
- // true (i1=1) -> 0xFFFFFFFF..., false (i1=0) -> 0x00000000
- // Sign extension is constant-time: pure arithmetic, no branches
- SDValue Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, IntVT, VecCond);
-
- // Step 4: Compute constant-time blend: F ^ ((T ^ F) & Mask)
- // All operations (XOR, AND) execute in constant time
- SDValue TXorF = DAG.getNode(ISD::XOR, dl, IntVT, T, F);
- SDValue MaskedDiff = DAG.getNode(ISD::AND, dl, IntVT, TXorF, Mask);
- Tmp1 = DAG.getNode(ISD::XOR, dl, IntVT, F, MaskedDiff);
-
- // Step 5: Bitcast back to original floating-point type if needed
- if (VT.isFloatingPoint()) {
- Tmp1 = DAG.getNode(ISD::BITCAST, dl, VT, Tmp1);
+ // Compute the all-ones/all-zeros mask as a scalar, then splat for vectors.
+ EVT MaskEltVT =
+ WorkingVT.isVector() ? WorkingVT.getVectorElementType() : WorkingVT;
+ SDValue ScalarCond = Tmp1;
+ if (ScalarCond.getValueType() != MaskEltVT)
+ ScalarCond = DAG.getAnyExtOrTrunc(ScalarCond, dl, MaskEltVT);
+ SDValue ScalarMask =
+ DAG.getNode(ISD::SUB, dl, MaskEltVT, DAG.getConstant(0, dl, MaskEltVT),
+ DAG.getNode(ISD::AND, dl, MaskEltVT, ScalarCond,
+ DAG.getConstant(1, dl, MaskEltVT)));
+ SDValue Mask = WorkingVT.isVector()
+ ? DAG.getSplat(WorkingVT, dl, ScalarMask)
+ : ScalarMask;
+
+ // F ^ ((T ^ F) & Mask)
+ SDValue XorTF = DAG.getNode(ISD::XOR, dl, WorkingVT, WorkingT, WorkingF);
+ SDValue TM = DAG.getNode(ISD::AND, dl, WorkingVT, XorTF, Mask);
+
+ // Forward-looking DAGCombine barrier (see header): route the masked-diff
+ // through a vreg so the chain edge partitions it from the surrounding
+ // bitwise ops, guarding against a future combiner that might fold
+ // XOR/AND/XOR-with-sext-mask back to SELECT. Skipped where no register
+ // class is available (scalable vectors, non-simple or illegal types, or
+ // a legal type the target has no register class for).
+ if (WorkingVT.isSimple() && !WorkingVT.isScalableVector() &&
+ TLI.isTypeLegal(WorkingVT.getSimpleVT())) {
+ if (const TargetRegisterClass *RC =
+ TLI.getRegClassFor(WorkingVT.getSimpleVT())) {
+ MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
+ Register TMReg = MRI.createVirtualRegister(RC);
+ SDValue Chain = DAG.getEntryNode();
+ Chain = DAG.getCopyToReg(Chain, dl, TMReg, TM);
+ TM = DAG.getCopyFromReg(Chain, dl, TMReg, WorkingVT);
}
-
- Tmp1->setFlags(Node->getFlags());
- } else if (VT.isFloatingPoint()) {
- EVT IntegerVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits());
- Tmp2 = DAG.getBitcast(IntegerVT, Tmp2);
- Tmp3 = DAG.getBitcast(IntegerVT, Tmp3);
- Tmp1 = DAG.getBitcast(VT, DAG.getCTSelect(dl, IntegerVT, Tmp1, Tmp2, Tmp3,
- Node->getFlags()));
- } else {
- assert(VT.isInteger());
- // Expand: Result = F ^ ((T ^ F) & Mask), Mask = 0 - (Cond & 1).
- // SUB+AND creates the mask because i1 is already type-promoted;
- // SIGN_EXTEND(i32, i32) would be a no-op leaving mask as 0/1.
- SDValue Cond = Tmp1;
- if (Cond.getValueType() != VT)
- Cond = DAG.getNode(ISD::ANY_EXTEND, dl, VT, Cond);
- SDValue Mask = DAG.getNode(
- ISD::SUB, dl, VT, DAG.getConstant(0, dl, VT),
- DAG.getNode(ISD::AND, dl, VT, Cond, DAG.getConstant(1, dl, VT)));
- SDValue Diff = DAG.getNode(ISD::XOR, dl, VT, Tmp2, Tmp3);
- Tmp1 = DAG.getNode(ISD::XOR, dl, VT, Tmp3,
- DAG.getNode(ISD::AND, dl, VT, Diff, Mask));
- Tmp1->setFlags(Node->getFlags());
}
+
+ Tmp1 = DAG.getNode(ISD::XOR, dl, WorkingVT, WorkingF, TM);
+
+ if (WorkingVT != VT)
+ Tmp1 = DAG.getBitcast(VT, Tmp1);
+
+ Tmp1->setFlags(Node->getFlags());
Results.push_back(Tmp1);
break;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 4dfca9b273418..8278973fe4b87 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -1070,7 +1070,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_CT_SELECT(SDNode *N) {
SDValue LHS = GetSoftenedFloat(N->getOperand(1));
SDValue RHS = GetSoftenedFloat(N->getOperand(2));
return DAG.getCTSelect(SDLoc(N), LHS.getValueType(), N->getOperand(0), LHS,
- RHS);
+ RHS, N->getFlags());
}
SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
@@ -3046,7 +3046,7 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_CT_SELECT(SDNode *N) {
SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
return DAG.getCTSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
- Op2);
+ Op2, N->getFlags());
}
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 49b38c1c7cf4a..06f0e7269fbcc 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -770,7 +770,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
SDValue DAGTypeLegalizer::ScalarizeVecRes_CT_SELECT(SDNode *N) {
SDValue LHS = GetScalarizedVector(N->getOperand(1));
return DAG.getCTSelect(SDLoc(N), LHS.getValueType(), N->getOperand(0), LHS,
- GetScalarizedVector(N->getOperand(2)));
+ GetScalarizedVector(N->getOperand(2)), N->getFlags());
}
SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index f40cc5d0c202a..1ee878d93922a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6692,82 +6692,6 @@ void SelectionDAGBuilder::visitVectorExtractLastActive(const CallInst &I,
setValue(&I, Result);
}
-/// Fallback implementation for constant-time select using DAG chaining.
-/// This implementation uses data dependencies through virtual registers to
-/// prevent optimizations from breaking the constant-time property. It is a
-/// best-effort safeguard; for stronger guarantees we prefer target-specific
-/// lowering pipelines that preserve the select pattern by construction.
-///
-/// It handles scalars, vectors (fixed and scalable), and floating-point types.
-SDValue SelectionDAGBuilder::createProtectedCtSelectFallback(
- SelectionDAG &DAG, const SDLoc &DL, SDValue Cond, SDValue T, SDValue F,
- EVT VT) {
-
- SDValue WorkingT = T;
- SDValue WorkingF = F;
- EVT WorkingVT = VT;
-
- SDValue Chain = DAG.getEntryNode();
- MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
-
- // Handle vector condition: splat scalar condition to vector
- if (VT.isVector() && !Cond.getValueType().isVector()) {
- ElementCount NumElems = VT.getVectorElementCount();
- EVT CondVT = EVT::getVectorVT(*DAG.getContext(), MVT::i1, NumElems);
- Cond = DAG.getSplat(CondVT, DL, Cond);
- }
-
- // Handle floating-point types: bitcast to integer for bitwise operations
- if (VT.isFloatingPoint()) {
- WorkingVT = VT.changeTypeToInteger();
- WorkingT = DAG.getBitcast(WorkingVT, T);
- WorkingF = DAG.getBitcast(WorkingVT, F);
- }
-
- // Create mask: sign-extend condition to all bits
- SDValue Mask = DAG.getSExtOrTrunc(Cond, DL, WorkingVT);
-
- // Compute: F ^ ((T ^ F) & Mask)
- // This is constant-time because both branches are always computed
- SDValue XorTF = DAG.getNode(ISD::XOR, DL, WorkingVT, WorkingT, WorkingF);
- SDValue TM = DAG.getNode(ISD::AND, DL, WorkingVT, XorTF, Mask);
-
- // DAG chaining: create data dependency through virtual register
- // This prevents optimizations from reordering or eliminating operations
- const TargetLowering &TLI = DAG.getTargetLoweringInfo();
- bool CanUseChaining = false;
-
- if (!WorkingVT.isScalableVector()) {
- // For fixed-size vectors and scalars, chaining is a best-effort hardening
- // step. The CT guarantee comes from the dataflow-only select
- // pattern (both sides computed, no control-flow). Chaining only adds an
- // extra dependency to discourage later combines.
- CanUseChaining = TLI.isTypeLegal(WorkingVT.getSimpleVT());
- } else {
- // For scalable vectors, skip chaining because there is no stable register
- // class to copy through. CT behavior still relies on the masking/select
- // pattern above.
- CanUseChaining = false;
- }
-
- if (CanUseChaining) {
- // Apply chaining through registers for additional protection
- const TargetRegisterClass *RC = TLI.getRegClassFor(WorkingVT.getSimpleVT());
- Register TMReg = MRI.createVirtualRegister(RC);
- Chain = DAG.getCopyToReg(Chain, DL, TMReg, TM);
- TM = DAG.getCopyFromReg(Chain, DL, TMReg, WorkingVT);
- }
-
- SDValue Result = DAG.getNode(ISD::XOR, DL, WorkingVT, WorkingF, TM);
-
- // Convert back to original type if needed
- if (WorkingVT != VT) {
- Result = DAG.getBitcast(VT, Result);
- }
-
- return Result;
-}
-
/// Lower the call to the specified intrinsic function.
void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
unsigned Intrinsic) {
@@ -6970,44 +6894,21 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
return;
}
case Intrinsic::ct_select: {
- // Set function attribute to indicate ct.select usage
- Function &F = DAG.getMachineFunction().getFunction();
- F.addFnAttr("ct-select");
-
- SDLoc DL = getCurSDLoc();
-
- SDValue Cond = getValue(I.getArgOperand(0)); // i1
- SDValue A = getValue(I.getArgOperand(1)); // T
- SDValue B = getValue(I.getArgOperand(2)); // T
-
- assert((A.getValueType() == B.getValueType()) &&
+ SDValue Cond = getValue(I.getArgOperand(0));
+ SDValue A = getValue(I.getArgOperand(1));
+ SDValue B = getValue(I.getArgOperand(2));
+ assert(A.getValueType() == B.getValueType() &&
"Operands are of different types");
+ assert(!Cond.getValueType().isVector() && "Vector condition not supported");
- EVT VT = A.getValueType();
- EVT CondVT = Cond.getValueType();
-
- // assert if Cond type is Vector
- assert(!CondVT.isVector() && "Vector type cond not supported yet");
-
- // Create a CT_SELECT node for scalar types so it survives DAGCombiner
- // (visitCT_SELECT is conservative) and expands to AND/OR/XOR during
- // operation legalization, after SETCC is lowered. Unsupported vectors
- // and floats with illegal integer equivalents (e.g. f64 on i386) use
- // the inline fallback which runs before type legalization.
- bool CreateNode =
- TLI.isOperationLegalOrCustom(ISD::CT_SELECT, VT) ||
- (!VT.isVector() &&
- (!VT.isFloatingPoint() ||
- TLI.isTypeLegal(
- EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()))));
-
- if (CreateNode) {
- SDValue Result = DAG.getNode(ISD::CT_SELECT, DL, VT, Cond, A, B);
- setValue(&I, Result);
- return;
- }
+ // TODO: a follow-up target-specific bundling pass needs to know the
+ // function uses ct.select. When that consumer lands, record it in the
+ // target's MachineFunctionInfo instead of mutating the IR from codegen.
+ // Kept commented for reference until then:
+ // DAG.getMachineFunction().getFunction().addFnAttr("ct-select");
- setValue(&I, createProtectedCtSelectFallback(DAG, DL, Cond, A, B, VT));
+ setValue(&I, DAG.getNode(ISD::CT_SELECT, getCurSDLoc(), A.getValueType(),
+ Cond, A, B));
return;
}
case Intrinsic::call_preallocated_setup: {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
index 1775d1d5588d3..6c7711af078f0 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h
@@ -219,9 +219,6 @@ class SelectionDAGBuilder {
peelDominantCaseCluster(const SwitchInst &SI,
SwitchCG::CaseClusterVector &Clusters,
BranchProbability &PeeledCaseProb);
- SDValue createProtectedCtSelectFallback(SelectionDAG &DAG, const SDLoc &DL,
- SDValue Cond, SDValue T, SDValue F,
- EVT VT);
private:
const TargetMachine &TM;
diff --git a/llvm/test/CodeGen/RISCV/ctselect-fallback.ll b/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
index ee8072703ee31..0df231b7a98d9 100644
--- a/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
+++ b/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
@@ -97,10 +97,14 @@ define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) {
define i32 @test_ctselect_const_true(i32 %a, i32 %b) {
; RV64-LABEL: test_ctselect_const_true:
; RV64: # %bb.0:
+; RV64-NEXT: xor a0, a0, a1
+; RV64-NEXT: xor a0, a1, a0
; RV64-NEXT: ret
;
; RV32-LABEL: test_ctselect_const_true:
; RV32: # %bb.0:
+; RV32-NEXT: xor a0, a0, a1
+; RV32-NEXT: xor a0, a1, a0
; RV32-NEXT: ret
%result = call i32 @llvm.ct.select.i32(i1 true, i32 %a, i32 %b)
ret i32 %result
@@ -207,6 +211,7 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; RV64-LABEL: test_ctselect_nested_and_i1_to_i32:
; RV64: # %bb.0:
; RV64-NEXT: and a0, a0, a1
+; RV64-NEXT: andi a0, a0, 1
; RV64-NEXT: xor a2, a2, a3
; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: srai a0, a0, 63
@@ -217,6 +222,7 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; RV32-LABEL: test_ctselect_nested_and_i1_to_i32:
; RV32: # %bb.0:
; RV32-NEXT: and a0, a0, a1
+; RV32-NEXT: andi a0, a0, 1
; RV32-NEXT: xor a2, a2, a3
; RV32-NEXT: slli a0, a0, 31
; RV32-NEXT: srai a0, a0, 31
@@ -235,6 +241,7 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; RV64-LABEL: test_ctselect_nested_or_i1_to_i32:
; RV64: # %bb.0:
; RV64-NEXT: or a0, a0, a1
+; RV64-NEXT: andi a0, a0, 1
; RV64-NEXT: xor a2, a2, a3
; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: srai a0, a0, 63
@@ -245,6 +252,7 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; RV32-LABEL: test_ctselect_nested_or_i1_to_i32:
; RV32: # %bb.0:
; RV32-NEXT: or a0, a0, a1
+; RV32-NEXT: andi a0, a0, 1
; RV32-NEXT: xor a2, a2, a3
; RV32-NEXT: slli a0, a0, 31
; RV32-NEXT: srai a0, a0, 31
@@ -265,6 +273,7 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
; RV64: # %bb.0:
; RV64-NEXT: and a0, a0, a1
; RV64-NEXT: and a0, a0, a2
+; RV64-NEXT: andi a0, a0, 1
; RV64-NEXT: xor a3, a3, a4
; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: srai a0, a0, 63
@@ -276,6 +285,7 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
; RV32: # %bb.0:
; RV32-NEXT: and a0, a0, a1
; RV32-NEXT: and a0, a0, a2
+; RV32-NEXT: andi a0, a0, 1
; RV32-NEXT: xor a3, a3, a4
; RV32-NEXT: slli a0, a0, 31
; RV32-NEXT: srai a0, a0, 31
@@ -295,7 +305,9 @@ define i32 @test_ctselect_double_nested_mixed_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x,
; RV64: # %bb.0:
; RV64-NEXT: and a0, a0, a1
; RV64-NEXT: xor a3, a3, a4
+; RV64-NEXT: andi a0, a0, 1
; RV64-NEXT: or a0, a0, a2
+; RV64-NEXT: andi a0, a0, 1
; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: and a3, a3, a0
@@ -309,7 +321,9 @@ define i32 @test_ctselect_double_nested_mixed_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x,
; RV32: # %bb.0:
; RV32-NEXT: and a0, a0, a1
; RV32-NEXT: xor a3, a3, a4
+; RV32-NEXT: andi a0, a0, 1
; RV32-NEXT: or a0, a0, a2
+; RV32-NEXT: andi a0, a0, 1
; RV32-NEXT: slli a0, a0, 31
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a3, a3, a0
@@ -370,7 +384,7 @@ define float @test_ctselect_f32_nan_inf(i1 %cond) {
; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: and a0, a0, a1
; RV64-NEXT: lui a1, 522240
-; RV64-NEXT: or a0, a0, a1
+; RV64-NEXT: xor a0, a0, a1
; RV64-NEXT: ret
;
; RV32-LABEL: test_ctselect_f32_nan_inf:
@@ -380,7 +394,7 @@ define float @test_ctselect_f32_nan_inf(i1 %cond) {
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a0, a0, a1
; RV32-NEXT: lui a1, 522240
-; RV32-NEXT: or a0, a0, a1
+; RV32-NEXT: xor a0, a0, a1
; RV32-NEXT: ret
%result = call float @llvm.ct.select.f32(i1 %cond, float 0x7FF8000000000000, float 0x7FF0000000000000)
ret float %result
@@ -396,7 +410,7 @@ define double @test_ctselect_f64_nan_inf(i1 %cond) {
; RV64-NEXT: and a0, a0, a1
; RV64-NEXT: li a1, 2047
; RV64-NEXT: slli a1, a1, 52
-; RV64-NEXT: or a0, a0, a1
+; RV64-NEXT: xor a0, a0, a1
; RV64-NEXT: ret
;
; RV32-LABEL: test_ctselect_f64_nan_inf:
@@ -406,7 +420,7 @@ define double @test_ctselect_f64_nan_inf(i1 %cond) {
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a0, a0, a1
; RV32-NEXT: lui a1, 524032
-; RV32-NEXT: or a1, a0, a1
+; RV32-NEXT: xor a1, a0, a1
; RV32-NEXT: li a0, 0
; RV32-NEXT: ret
%result = call double @llvm.ct.select.f64(i1 %cond, double 0x7FF8000000000000, double 0x7FF0000000000000)
@@ -599,10 +613,10 @@ define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
; RV64-NEXT: .cfi_offset s0, -8
; RV64-NEXT: .cfi_offset s1, -16
; RV64-NEXT: .cfi_offset s2, -24
-; RV64-NEXT: lw a7, 32(a3)
-; RV64-NEXT: lw a6, 40(a3)
-; RV64-NEXT: lw a5, 48(a3)
-; RV64-NEXT: lw a4, 56(a3)
+; RV64-NEXT: lw a4, 32(a3)
+; RV64-NEXT: lw a5, 40(a3)
+; RV64-NEXT: lw a6, 48(a3)
+; RV64-NEXT: lw a7, 56(a3)
; RV64-NEXT: lw t0, 32(a2)
; RV64-NEXT: lw t1, 40(a2)
; RV64-NEXT: lw t2, 48(a2)
@@ -621,10 +635,10 @@ define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
; RV64-NEXT: xor s1, s1, t5
; RV64-NEXT: xor s2, s2, t6
; RV64-NEXT: xor a2, a2, a3
-; RV64-NEXT: xor t0, t0, a7
-; RV64-NEXT: xor t1, t1, a6
-; RV64-NEXT: xor t2, t2, a5
-; RV64-NEXT: xor t3, t3, a4
+; RV64-NEXT: xor t0, t0, a4
+; RV64-NEXT: xor t1, t1, a5
+; RV64-NEXT: xor t2, t2, a6
+; RV64-NEXT: xor t3, t3, a7
; RV64-NEXT: and s0, s0, a1
; RV64-NEXT: and s1, s1, a1
; RV64-NEXT: and s2, s2, a1
@@ -637,12 +651,12 @@ define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
; RV64-NEXT: xor t4, t5, s1
; RV64-NEXT: xor t5, t6, s2
; RV64-NEXT: xor a2, a3, a2
-; RV64-NEXT: xor a3, a7, t0
-; RV64-NEXT: xor a6, a6, t1
-; RV64-NEXT: xor a5, a5, t2
-; RV64-NEXT: xor a1, a4, a1
+; RV64-NEXT: xor a3, a4, t0
+; RV64-NEXT: xor a4, a5, t1
+; RV64-NEXT: xor a5, a6, t2
+; RV64-NEXT: xor a1, a7, a1
; RV64-NEXT: sw a3, 16(a0)
-; RV64-NEXT: sw a6, 20(a0)
+; RV64-NEXT: sw a4, 20(a0)
; RV64-NEXT: sw a5, 24(a0)
; RV64-NEXT: sw a1, 28(a0)
; RV64-NEXT: sw t3, 0(a0)
@@ -669,10 +683,10 @@ define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
; RV32-NEXT: .cfi_offset s0, -4
; RV32-NEXT: .cfi_offset s1, -8
; RV32-NEXT: .cfi_offset s2, -12
-; RV32-NEXT: lw a7, 16(a3)
-; RV32-NEXT: lw a6, 20(a3)
-; RV32-NEXT: lw a5, 24(a3)
-; RV32-NEXT: lw a4, 28(a3)
+; RV32-NEXT: lw a4, 16(a3)
+; RV32-NEXT: lw a5, 20(a3)
+; RV32-NEXT: lw a6, 24(a3)
+; RV32-NEXT: lw a7, 28(a3)
; RV32-NEXT: lw t0, 16(a2)
; RV32-NEXT: lw t1, 20(a2)
; RV32-NEXT: lw t2, 24(a2)
@@ -691,10 +705,10 @@ define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
; RV32-NEXT: xor s1, s1, t5
; RV32-NEXT: xor s2, s2, t6
; RV32-NEXT: xor a2, a2, a3
-; RV32-NEXT: xor t0, t0, a7
-; RV32-NEXT: xor t1, t1, a6
-; RV32-NEXT: xor t2, t2, a5
-; RV32-NEXT: xor t3, t3, a4
+; RV32-NEXT: xor t0, t0, a4
+; RV32-NEXT: xor t1, t1, a5
+; RV32-NEXT: xor t2, t2, a6
+; RV32-NEXT: xor t3, t3, a7
; RV32-NEXT: and s0, s0, a1
; RV32-NEXT: and s1, s1, a1
; RV32-NEXT: and s2, s2, a1
@@ -707,12 +721,12 @@ define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
; RV32-NEXT: xor t4, t5, s1
; RV32-NEXT: xor t5, t6, s2
; RV32-NEXT: xor a2, a3, a2
-; RV32-NEXT: xor a3, a7, t0
-; RV32-NEXT: xor a6, a6, t1
-; RV32-NEXT: xor a5, a5, t2
-; RV32-NEXT: xor a1, a4, a1
+; RV32-NEXT: xor a3, a4, t0
+; RV32-NEXT: xor a4, a5, t1
+; RV32-NEXT: xor a5, a6, t2
+; RV32-NEXT: xor a1, a7, a1
; RV32-NEXT: sw a3, 16(a0)
-; RV32-NEXT: sw a6, 20(a0)
+; RV32-NEXT: sw a4, 20(a0)
; RV32-NEXT: sw a5, 24(a0)
; RV32-NEXT: sw a1, 28(a0)
; RV32-NEXT: sw t3, 0(a0)
diff --git a/llvm/test/CodeGen/X86/ctselect.ll b/llvm/test/CodeGen/X86/ctselect.ll
index e1abae80cef4f..9d54c29db9683 100644
--- a/llvm/test/CodeGen/X86/ctselect.ll
+++ b/llvm/test/CodeGen/X86/ctselect.ll
@@ -5,7 +5,7 @@
; Test basic ct.select functionality for scalar types
-define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) {
+define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) #0 {
; X64-LABEL: test_ctselect_i8:
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
@@ -44,7 +44,7 @@ define i8 @test_ctselect_i8(i1 %cond, i8 %a, i8 %b) {
ret i8 %result
}
-define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) {
+define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_i32:
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
@@ -84,7 +84,7 @@ define i32 @test_ctselect_i32(i1 %cond, i32 %a, i32 %b) {
ret i32 %result
}
-define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) {
+define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) #0 {
; X64-LABEL: test_ctselect_i64:
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
@@ -98,11 +98,7 @@ define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) {
; X32-LABEL: test_ctselect_i64:
; X32: # %bb.0:
; X32-NEXT: pushl %edi
-; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: pushl %esi
-; X32-NEXT: .cfi_def_cfa_offset 12
-; X32-NEXT: .cfi_offset %esi, -12
-; X32-NEXT: .cfi_offset %edi, -8
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
; X32-NEXT: andb $1, %dl
; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
@@ -118,19 +114,13 @@ define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) {
; X32-NEXT: andl %edi, %edx
; X32-NEXT: xorl %ecx, %edx
; X32-NEXT: popl %esi
-; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: popl %edi
-; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_i64:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: pushl %edi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; X32-NOCMOV-NEXT: .cfi_offset %esi, -12
-; X32-NOCMOV-NEXT: .cfi_offset %edi, -8
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: andb $1, %dl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
@@ -146,15 +136,13 @@ define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) {
; X32-NOCMOV-NEXT: andl %edi, %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: popl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: popl %edi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl
%result = call i64 @llvm.ct.select.i64(i1 %cond, i64 %a, i64 %b)
ret i64 %result
}
-define float @test_ctselect_f32(i1 %cond, float %a, float %b) {
+define float @test_ctselect_f32(i1 %cond, float %a, float %b) #0 {
; X64-LABEL: test_ctselect_f32:
; X64: # %bb.0:
; X64-NEXT: movd %xmm1, %eax
@@ -170,7 +158,6 @@ define float @test_ctselect_f32(i1 %cond, float %a, float %b) {
; X32-LABEL: test_ctselect_f32:
; X32: # %bb.0:
; X32-NEXT: subl $12, %esp
-; X32-NEXT: .cfi_def_cfa_offset 16
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
@@ -187,13 +174,11 @@ define float @test_ctselect_f32(i1 %cond, float %a, float %b) {
; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: addl $12, %esp
-; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f32:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: subl $12, %esp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
@@ -210,13 +195,12 @@ define float @test_ctselect_f32(i1 %cond, float %a, float %b) {
; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: addl $12, %esp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl
%result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
ret float %result
}
-define double @test_ctselect_f64(i1 %cond, double %a, double %b) {
+define double @test_ctselect_f64(i1 %cond, double %a, double %b) #0 {
; X64-LABEL: test_ctselect_f64:
; X64: # %bb.0:
; X64-NEXT: # kill: def $edi killed $edi def $rdi
@@ -233,65 +217,426 @@ define double @test_ctselect_f64(i1 %cond, double %a, double %b) {
; X32-LABEL: test_ctselect_f64:
; X32: # %bb.0:
; X32-NEXT: pushl %esi
-; X32-NEXT: .cfi_def_cfa_offset 8
-; X32-NEXT: subl $8, %esp
-; X32-NEXT: .cfi_def_cfa_offset 16
-; X32-NEXT: .cfi_offset %esi, -8
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: subl $24, %esp
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb $1, %al
+; X32-NEXT: fldl {{[0-9]+}}(%esp)
+; X32-NEXT: fldl {{[0-9]+}}(%esp)
+; X32-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NEXT: movzbl %al, %eax
+; X32-NEXT: negl %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: andl $1, %ecx
-; X32-NEXT: negl %ecx
-; X32-NEXT: andl %ecx, %esi
+; X32-NEXT: andl %eax, %esi
; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT: movl %esi, (%esp)
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %eax, %edx
-; X32-NEXT: andl %ecx, %edx
-; X32-NEXT: xorl %eax, %edx
-; X32-NEXT: movl %edx, (%esp)
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: andl %eax, %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NEXT: fldl (%esp)
-; X32-NEXT: addl $8, %esp
-; X32-NEXT: .cfi_def_cfa_offset 8
+; X32-NEXT: addl $24, %esp
; X32-NEXT: popl %esi
-; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f64:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
-; X32-NOCMOV-NEXT: subl $8, %esp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
-; X32-NOCMOV-NEXT: .cfi_offset %esi, -8
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: subl $24, %esp
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: fldl {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fldl {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movzbl %al, %eax
+; X32-NOCMOV-NEXT: negl %eax
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: andl $1, %ecx
-; X32-NOCMOV-NEXT: negl %ecx
-; X32-NOCMOV-NEXT: andl %ecx, %esi
+; X32-NOCMOV-NEXT: andl %eax, %esi
; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl %esi, (%esp)
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %eax, %edx
-; X32-NOCMOV-NEXT: andl %ecx, %edx
-; X32-NOCMOV-NEXT: xorl %eax, %edx
-; X32-NOCMOV-NEXT: movl %edx, (%esp)
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: andl %eax, %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fldl (%esp)
-; X32-NOCMOV-NEXT: addl $8, %esp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
+; X32-NOCMOV-NEXT: addl $24, %esp
; X32-NOCMOV-NEXT: popl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl
%result = call double @llvm.ct.select.f64(i1 %cond, double %a, double %b)
ret double %result
}
-define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) {
+define half @test_ctselect_f16(i1 %cond, half %a, half %b) #0 {
+; X64-LABEL: test_ctselect_f16:
+; X64: # %bb.0:
+; X64-NEXT: pextrw $0, %xmm1, %eax
+; X64-NEXT: pextrw $0, %xmm0, %ecx
+; X64-NEXT: xorl %eax, %ecx
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: negl %edi
+; X64-NEXT: andl %ecx, %edi
+; X64-NEXT: xorl %eax, %edi
+; X64-NEXT: pinsrw $0, %edi, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_f16:
+; X32: # %bb.0:
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb $1, %al
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorw %cx, %dx
+; X32-NEXT: movzbl %al, %eax
+; X32-NEXT: negl %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: # kill: def $ax killed $ax killed $eax
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_f16:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorw %cx, %dx
+; X32-NOCMOV-NEXT: movzbl %al, %eax
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax killed $eax
+; X32-NOCMOV-NEXT: retl
+ %result = call half @llvm.ct.select.f16(i1 %cond, half %a, half %b)
+ ret half %result
+}
+
+define bfloat @test_ctselect_bf16(i1 %cond, bfloat %a, bfloat %b) #0 {
+; X64-LABEL: test_ctselect_bf16:
+; X64: # %bb.0:
+; X64-NEXT: pextrw $0, %xmm1, %eax
+; X64-NEXT: pextrw $0, %xmm0, %ecx
+; X64-NEXT: xorl %eax, %ecx
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: negl %edi
+; X64-NEXT: andl %ecx, %edi
+; X64-NEXT: xorl %eax, %edi
+; X64-NEXT: pinsrw $0, %edi, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_bf16:
+; X32: # %bb.0:
+; X32-NEXT: pushl %esi
+; X32-NEXT: subl $8, %esp
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: movl %eax, %esi
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: andb $1, %cl
+; X32-NEXT: xorl %esi, %eax
+; X32-NEXT: movzbl %cl, %ecx
+; X32-NEXT: negl %ecx
+; X32-NEXT: andl %eax, %ecx
+; X32-NEXT: xorl %esi, %ecx
+; X32-NEXT: shll $16, %ecx
+; X32-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: addl $8, %esp
+; X32-NEXT: popl %esi
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_bf16:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: subl $8, %esp
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: movl %eax, %esi
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: andb $1, %cl
+; X32-NOCMOV-NEXT: xorl %esi, %eax
+; X32-NOCMOV-NEXT: movzbl %cl, %ecx
+; X32-NOCMOV-NEXT: negl %ecx
+; X32-NOCMOV-NEXT: andl %eax, %ecx
+; X32-NOCMOV-NEXT: xorl %esi, %ecx
+; X32-NOCMOV-NEXT: shll $16, %ecx
+; X32-NOCMOV-NEXT: movl %ecx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: addl $8, %esp
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: retl
+ %result = call bfloat @llvm.ct.select.bf16(i1 %cond, bfloat %a, bfloat %b)
+ ret bfloat %result
+}
+
+define fp128 @test_ctselect_f128(i1 %cond, fp128 %a, fp128 %b) #0 {
+; X64-LABEL: test_ctselect_f128:
+; X64: # %bb.0:
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
+; X64-NEXT: movaps %xmm1, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps %xmm0, -{{[0-9]+}}(%rsp)
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: negq %rdi
+; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rax
+; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rcx
+; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rdx
+; X64-NEXT: xorq %rax, %rdx
+; X64-NEXT: andq %rdi, %rdx
+; X64-NEXT: xorq %rax, %rdx
+; X64-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rax
+; X64-NEXT: xorq %rcx, %rax
+; X64-NEXT: andq %rdi, %rax
+; X64-NEXT: xorq %rcx, %rax
+; X64-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movaps -{{[0-9]+}}(%rsp), %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_f128:
+; X32: # %bb.0:
+; X32-NEXT: pushl %ebp
+; X32-NEXT: pushl %ebx
+; X32-NEXT: pushl %edi
+; X32-NEXT: pushl %esi
+; X32-NEXT: subl $12, %esp
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: andb $1, %bl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorl %edi, %ecx
+; X32-NEXT: movzbl %bl, %ebp
+; X32-NEXT: negl %ebp
+; X32-NEXT: andl %ebp, %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: andl %ebp, %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: andl %ebp, %ebx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %edx, %eax
+; X32-NEXT: andl %ebp, %eax
+; X32-NEXT: xorl %edi, %ecx
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: xorl %edx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl %eax, 12(%edx)
+; X32-NEXT: movl %ebx, 8(%edx)
+; X32-NEXT: movl %esi, 4(%edx)
+; X32-NEXT: movl %ecx, (%edx)
+; X32-NEXT: movl %edx, %eax
+; X32-NEXT: addl $12, %esp
+; X32-NEXT: popl %esi
+; X32-NEXT: popl %edi
+; X32-NEXT: popl %ebx
+; X32-NEXT: popl %ebp
+; X32-NEXT: retl $4
+;
+; X32-NOCMOV-LABEL: test_ctselect_f128:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebp
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: subl $12, %esp
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: andb $1, %bl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorl %edi, %ecx
+; X32-NOCMOV-NEXT: movzbl %bl, %ebp
+; X32-NOCMOV-NEXT: negl %ebp
+; X32-NOCMOV-NEXT: andl %ebp, %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: andl %ebp, %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: andl %ebp, %ebx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %edx, %eax
+; X32-NOCMOV-NEXT: andl %ebp, %eax
+; X32-NOCMOV-NEXT: xorl %edi, %ecx
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: xorl %edx, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl %eax, 12(%edx)
+; X32-NOCMOV-NEXT: movl %ebx, 8(%edx)
+; X32-NOCMOV-NEXT: movl %esi, 4(%edx)
+; X32-NOCMOV-NEXT: movl %ecx, (%edx)
+; X32-NOCMOV-NEXT: movl %edx, %eax
+; X32-NOCMOV-NEXT: addl $12, %esp
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
+; X32-NOCMOV-NEXT: popl %ebp
+; X32-NOCMOV-NEXT: retl $4
+ %result = call fp128 @llvm.ct.select.f128(i1 %cond, fp128 %a, fp128 %b)
+ ret fp128 %result
+}
+
+define x86_fp80 @test_ctselect_f80(i1 %cond, x86_fp80 %a, x86_fp80 %b) #0 {
+; X64-LABEL: test_ctselect_f80:
+; X64: # %bb.0:
+; X64-NEXT: fldt {{[0-9]+}}(%rsp)
+; X64-NEXT: fldt {{[0-9]+}}(%rsp)
+; X64-NEXT: fstpt -{{[0-9]+}}(%rsp)
+; X64-NEXT: fstpt -{{[0-9]+}}(%rsp)
+; X64-NEXT: movl -{{[0-9]+}}(%rsp), %ecx
+; X64-NEXT: movl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %edx
+; X64-NEXT: xorw %cx, %dx
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: negl %edi
+; X64-NEXT: andl %edi, %edx
+; X64-NEXT: xorl %ecx, %edx
+; X64-NEXT: movw %dx, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %ecx
+; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %edx
+; X64-NEXT: xorw %cx, %dx
+; X64-NEXT: andl %edi, %edx
+; X64-NEXT: xorl %ecx, %edx
+; X64-NEXT: movw %dx, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %ecx
+; X64-NEXT: xorw %ax, %cx
+; X64-NEXT: andl %edi, %ecx
+; X64-NEXT: xorl %eax, %ecx
+; X64-NEXT: movw %cx, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %ecx
+; X64-NEXT: xorw %ax, %cx
+; X64-NEXT: andl %edi, %ecx
+; X64-NEXT: xorl %eax, %ecx
+; X64-NEXT: movw %cx, -{{[0-9]+}}(%rsp)
+; X64-NEXT: movl -{{[0-9]+}}(%rsp), %eax
+; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %ecx
+; X64-NEXT: xorw %ax, %cx
+; X64-NEXT: andl %edi, %ecx
+; X64-NEXT: xorl %eax, %ecx
+; X64-NEXT: movw %cx, -{{[0-9]+}}(%rsp)
+; X64-NEXT: fldt -{{[0-9]+}}(%rsp)
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_f80:
+; X32: # %bb.0:
+; X32-NEXT: pushl %esi
+; X32-NEXT: subl $36, %esp
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb $1, %al
+; X32-NEXT: fldt {{[0-9]+}}(%esp)
+; X32-NEXT: fldt {{[0-9]+}}(%esp)
+; X32-NEXT: fstpt {{[0-9]+}}(%esp)
+; X32-NEXT: fstpt {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorw %dx, %si
+; X32-NEXT: movzbl %al, %eax
+; X32-NEXT: negl %eax
+; X32-NEXT: andl %eax, %esi
+; X32-NEXT: xorl %edx, %esi
+; X32-NEXT: movw %si, (%esp)
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorw %dx, %si
+; X32-NEXT: andl %eax, %esi
+; X32-NEXT: xorl %edx, %esi
+; X32-NEXT: movw %si, {{[0-9]+}}(%esp)
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorw %cx, %dx
+; X32-NEXT: andl %eax, %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: movw %dx, {{[0-9]+}}(%esp)
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorw %cx, %dx
+; X32-NEXT: andl %eax, %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: movw %dx, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorw %cx, %dx
+; X32-NEXT: andl %eax, %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: movw %dx, {{[0-9]+}}(%esp)
+; X32-NEXT: fldt (%esp)
+; X32-NEXT: addl $36, %esp
+; X32-NEXT: popl %esi
+; X32-NEXT: retl
+;
+; X32-NOCMOV-LABEL: test_ctselect_f80:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: subl $36, %esp
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorw %dx, %si
+; X32-NOCMOV-NEXT: movzbl %al, %eax
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl %eax, %esi
+; X32-NOCMOV-NEXT: xorl %edx, %esi
+; X32-NOCMOV-NEXT: movw %si, (%esp)
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorw %dx, %si
+; X32-NOCMOV-NEXT: andl %eax, %esi
+; X32-NOCMOV-NEXT: xorl %edx, %esi
+; X32-NOCMOV-NEXT: movw %si, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorw %cx, %dx
+; X32-NOCMOV-NEXT: andl %eax, %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: movw %dx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorw %cx, %dx
+; X32-NOCMOV-NEXT: andl %eax, %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: movw %dx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorw %cx, %dx
+; X32-NOCMOV-NEXT: andl %eax, %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: movw %dx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fldt (%esp)
+; X32-NOCMOV-NEXT: addl $36, %esp
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: retl
+ %result = call x86_fp80 @llvm.ct.select.f80(i1 %cond, x86_fp80 %a, x86_fp80 %b)
+ ret x86_fp80 %result
+}
+
+define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) #0 {
; X64-LABEL: test_ctselect_ptr:
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
@@ -332,26 +677,34 @@ define ptr @test_ctselect_ptr(i1 %cond, ptr %a, ptr %b) {
}
; Test with constant conditions
-define i32 @test_ctselect_const_true(i32 %a, i32 %b) {
+define i32 @test_ctselect_const_true(i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_const_true:
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
+; X64-NEXT: xorl %esi, %eax
+; X64-NEXT: xorl %esi, %eax
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_const_true:
; X32: # %bb.0:
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: xorl %ecx, %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_const_true:
; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
; X32-NOCMOV-NEXT: retl
%result = call i32 @llvm.ct.select.i32(i1 true, i32 %a, i32 %b)
ret i32 %result
}
-define i32 @test_ctselect_const_false(i32 %a, i32 %b) {
+define i32 @test_ctselect_const_false(i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_const_false:
; X64: # %bb.0:
; X64-NEXT: movl %esi, %eax
@@ -359,19 +712,21 @@ define i32 @test_ctselect_const_false(i32 %a, i32 %b) {
;
; X32-LABEL: test_ctselect_const_false:
; X32: # %bb.0:
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %eax, %eax
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_const_false:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %eax, %eax
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: retl
%result = call i32 @llvm.ct.select.i32(i1 false, i32 %a, i32 %b)
ret i32 %result
}
; Test with comparison conditions
-define i32 @test_ctselect_icmp_eq(i32 %x, i32 %y, i32 %a, i32 %b) {
+define i32 @test_ctselect_icmp_eq(i32 %x, i32 %y, i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_icmp_eq:
; X64: # %bb.0:
; X64-NEXT: xorl %eax, %eax
@@ -415,7 +770,7 @@ define i32 @test_ctselect_icmp_eq(i32 %x, i32 %y, i32 %a, i32 %b) {
ret i32 %result
}
-define i32 @test_ctselect_icmp_ult(i32 %x, i32 %y, i32 %a, i32 %b) {
+define i32 @test_ctselect_icmp_ult(i32 %x, i32 %y, i32 %a, i32 %b) #0 {
; X64-LABEL: test_ctselect_icmp_ult:
; X64: # %bb.0:
; X64-NEXT: xorl %ecx, %edx
@@ -456,19 +811,21 @@ define i32 @test_ctselect_icmp_ult(i32 %x, i32 %y, i32 %a, i32 %b) {
ret i32 %result
}
-define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) {
+define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) #0 {
; X64-LABEL: test_ctselect_fcmp_oeq:
; X64: # %bb.0:
+; X64-NEXT: movd %xmm3, %eax
; X64-NEXT: cmpeqss %xmm1, %xmm0
-; X64-NEXT: xorps %xmm3, %xmm2
-; X64-NEXT: andps %xmm2, %xmm0
-; X64-NEXT: xorps %xmm3, %xmm0
+; X64-NEXT: pxor %xmm3, %xmm2
+; X64-NEXT: pand %xmm0, %xmm2
+; X64-NEXT: movd %xmm2, %ecx
+; X64-NEXT: xorl %eax, %ecx
+; X64-NEXT: movd %ecx, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_fcmp_oeq:
; X32: # %bb.0:
; X32-NEXT: subl $12, %esp
-; X32-NEXT: .cfi_def_cfa_offset 16
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: fstps {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
@@ -490,13 +847,11 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) {
; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NEXT: flds {{[0-9]+}}(%esp)
; X32-NEXT: addl $12, %esp
-; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_fcmp_oeq:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: subl $12, %esp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
@@ -520,7 +875,6 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) {
; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: addl $12, %esp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl
%cond = fcmp oeq float %x, %y
%result = call float @llvm.ct.select.f32(i1 %cond, float %a, float %b)
@@ -528,7 +882,7 @@ define float @test_ctselect_fcmp_oeq(float %x, float %y, float %a, float %b) {
}
; Test with memory operands
-define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) {
+define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) #0 {
; X64-LABEL: test_ctselect_load:
; X64: # %bb.0:
; X64-NEXT: movl (%rdx), %ecx
@@ -576,7 +930,7 @@ define i32 @test_ctselect_load(i1 %cond, ptr %p1, ptr %p2) {
}
; Test nested ct_select calls
-define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
+define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) #0 {
; X64-LABEL: test_ctselect_nested:
; X64: # %bb.0:
; X64-NEXT: movl %edi, %eax
@@ -595,11 +949,7 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
; X32-LABEL: test_ctselect_nested:
; X32: # %bb.0:
; X32-NEXT: pushl %edi
-; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: pushl %esi
-; X32-NEXT: .cfi_def_cfa_offset 12
-; X32-NEXT: .cfi_offset %esi, -12
-; X32-NEXT: .cfi_offset %edi, -8
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NEXT: andb $1, %al
; X32-NEXT: movb {{[0-9]+}}(%esp), %ah
@@ -618,19 +968,13 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
; X32-NEXT: andl %edx, %eax
; X32-NEXT: xorl %ecx, %eax
; X32-NEXT: popl %esi
-; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: popl %edi
-; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_nested:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: pushl %edi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
-; X32-NOCMOV-NEXT: .cfi_offset %esi, -12
-; X32-NOCMOV-NEXT: .cfi_offset %edi, -8
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movb {{[0-9]+}}(%esp), %ah
@@ -649,9 +993,7 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
; X32-NOCMOV-NEXT: andl %edx, %eax
; X32-NOCMOV-NEXT: xorl %ecx, %eax
; X32-NOCMOV-NEXT: popl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: popl %edi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl
%inner = call i32 @llvm.ct.select.i32(i1 %cond2, i32 %a, i32 %b)
%result = call i32 @llvm.ct.select.i32(i1 %cond1, i32 %inner, i32 %c)
@@ -661,13 +1003,14 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
; Test nested CT_SELECT pattern with AND merging on i1 values
; Pattern: ct_select C0, (ct_select C1, X, Y), Y -> ct_select (C0 & C1), X, Y
; This optimization only applies when selecting between i1 values (boolean logic)
-define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
+define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) #0 {
; X64-LABEL: test_ctselect_nested_and_i1_to_i32:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: andl %esi, %eax
+; X64-NEXT: andl %esi, %edi
+; X64-NEXT: andb $1, %dil
+; X64-NEXT: andb $1, %dil
; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: andl $1, %eax
+; X64-NEXT: movzbl %dil, %eax
; X64-NEXT: negl %eax
; X64-NEXT: andl %edx, %eax
; X64-NEXT: xorl %ecx, %eax
@@ -679,6 +1022,7 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NEXT: andb {{[0-9]+}}(%esp), %al
; X32-NEXT: andb $1, %al
+; X32-NEXT: andb $1, %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: xorl %ecx, %edx
; X32-NEXT: movzbl %al, %eax
@@ -693,6 +1037,7 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: movzbl %al, %eax
@@ -709,13 +1054,14 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; Test nested CT_SELECT pattern with OR merging on i1 values
; Pattern: ct_select C0, X, (ct_select C1, X, Y) -> ct_select (C0 | C1), X, Y
; This optimization only applies when selecting between i1 values (boolean logic)
-define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
+define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) #0 {
; X64-LABEL: test_ctselect_nested_or_i1_to_i32:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: orl %esi, %eax
+; X64-NEXT: orl %esi, %edi
+; X64-NEXT: andb $1, %dil
+; X64-NEXT: andb $1, %dil
; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: andl $1, %eax
+; X64-NEXT: movzbl %dil, %eax
; X64-NEXT: negl %eax
; X64-NEXT: andl %edx, %eax
; X64-NEXT: xorl %ecx, %eax
@@ -727,6 +1073,7 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NEXT: orb {{[0-9]+}}(%esp), %al
; X32-NEXT: andb $1, %al
+; X32-NEXT: andb $1, %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: xorl %ecx, %edx
; X32-NEXT: movzbl %al, %eax
@@ -741,6 +1088,7 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: orb {{[0-9]+}}(%esp), %al
; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: movzbl %al, %eax
@@ -759,14 +1107,15 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; -> ct_select C0, (ct_select (C1 & C2), X, Y), Y
; -> ct_select (C0 & (C1 & C2)), X, Y
; This tests that the optimization can be applied recursively
-define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i32 %y) {
+define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i32 %y) #0 {
; X64-LABEL: test_ctselect_double_nested_and_i1:
; X64: # %bb.0:
-; X64-NEXT: movl %edi, %eax
-; X64-NEXT: andl %esi, %eax
-; X64-NEXT: andl %edx, %eax
+; X64-NEXT: andl %esi, %edi
+; X64-NEXT: andl %edx, %edi
+; X64-NEXT: andb $1, %dil
+; X64-NEXT: andb $1, %dil
; X64-NEXT: xorl %r8d, %ecx
-; X64-NEXT: andl $1, %eax
+; X64-NEXT: movzbl %dil, %eax
; X64-NEXT: negl %eax
; X64-NEXT: andl %ecx, %eax
; X64-NEXT: xorl %r8d, %eax
@@ -779,6 +1128,7 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
; X32-NEXT: andb {{[0-9]+}}(%esp), %al
; X32-NEXT: andb {{[0-9]+}}(%esp), %al
; X32-NEXT: andb $1, %al
+; X32-NEXT: andb $1, %al
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: xorl %ecx, %edx
; X32-NEXT: movzbl %al, %eax
@@ -794,6 +1144,7 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
; X32-NOCMOV-NEXT: andb {{[0-9]+}}(%esp), %al
; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: movzbl %al, %eax
@@ -813,14 +1164,14 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
; Test vector CT_SELECT with v4i32 (128-bit vector with single i1 mask)
; NOW CONSTANT-TIME: Uses bitwise XOR/AND operations instead of branches!
-define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) {
+define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) #0 {
; X64-LABEL: test_ctselect_v4i32:
; X64: # %bb.0:
; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: negl %edi
; X64-NEXT: movd %edi, %xmm2
; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; X64-NEXT: pslld $31, %xmm2
-; X64-NEXT: psrad $31, %xmm2
; X64-NEXT: pand %xmm2, %xmm0
; X64-NEXT: pxor %xmm1, %xmm0
; X64-NEXT: retq
@@ -828,117 +1179,93 @@ define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) {
; X32-LABEL: test_ctselect_v4i32:
; X32: # %bb.0:
; X32-NEXT: pushl %ebp
-; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: pushl %ebx
-; X32-NEXT: .cfi_def_cfa_offset 12
; X32-NEXT: pushl %edi
-; X32-NEXT: .cfi_def_cfa_offset 16
; X32-NEXT: pushl %esi
-; X32-NEXT: .cfi_def_cfa_offset 20
-; X32-NEXT: .cfi_offset %esi, -20
-; X32-NEXT: .cfi_offset %edi, -16
-; X32-NEXT: .cfi_offset %ebx, -12
-; X32-NEXT: .cfi_offset %ebp, -8
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: andb $1, %bl
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorl %edi, %ecx
+; X32-NEXT: movzbl %bl, %ebp
+; X32-NEXT: negl %ebp
+; X32-NEXT: andl %ebp, %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: andl %ebp, %esi
; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: andl %ebp, %ebx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ebx, %edx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: andl $1, %edi
-; X32-NEXT: negl %edi
-; X32-NEXT: andl %edi, %edx
-; X32-NEXT: xorl %ebx, %edx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl %ebp, %ebx
-; X32-NEXT: andl %edi, %ebx
-; X32-NEXT: xorl %ebp, %ebx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorl %esi, %ebp
-; X32-NEXT: andl %edi, %ebp
-; X32-NEXT: xorl %esi, %ebp
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl %ecx, %esi
-; X32-NEXT: andl %edi, %esi
-; X32-NEXT: xorl %ecx, %esi
-; X32-NEXT: movl %esi, 12(%eax)
-; X32-NEXT: movl %ebp, 8(%eax)
-; X32-NEXT: movl %ebx, 4(%eax)
-; X32-NEXT: movl %edx, (%eax)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %edx, %eax
+; X32-NEXT: andl %ebp, %eax
+; X32-NEXT: xorl %edi, %ecx
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: xorl %edx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl %eax, 12(%edx)
+; X32-NEXT: movl %ebx, 8(%edx)
+; X32-NEXT: movl %esi, 4(%edx)
+; X32-NEXT: movl %ecx, (%edx)
+; X32-NEXT: movl %edx, %eax
; X32-NEXT: popl %esi
-; X32-NEXT: .cfi_def_cfa_offset 16
; X32-NEXT: popl %edi
-; X32-NEXT: .cfi_def_cfa_offset 12
; X32-NEXT: popl %ebx
-; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: popl %ebp
-; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl $4
;
; X32-NOCMOV-LABEL: test_ctselect_v4i32:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: pushl %ebp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: pushl %ebx
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
; X32-NOCMOV-NEXT: pushl %edi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; X32-NOCMOV-NEXT: .cfi_offset %esi, -20
-; X32-NOCMOV-NEXT: .cfi_offset %edi, -16
-; X32-NOCMOV-NEXT: .cfi_offset %ebx, -12
-; X32-NOCMOV-NEXT: .cfi_offset %ebp, -8
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: andb $1, %bl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorl %edi, %ecx
+; X32-NOCMOV-NEXT: movzbl %bl, %ebp
+; X32-NOCMOV-NEXT: negl %ebp
+; X32-NOCMOV-NEXT: andl %ebp, %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: andl %ebp, %esi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: andl %ebp, %ebx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ebx, %edx
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: andl $1, %edi
-; X32-NOCMOV-NEXT: negl %edi
-; X32-NOCMOV-NEXT: andl %edi, %edx
-; X32-NOCMOV-NEXT: xorl %ebx, %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl %ebp, %ebx
-; X32-NOCMOV-NEXT: andl %edi, %ebx
-; X32-NOCMOV-NEXT: xorl %ebp, %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorl %esi, %ebp
-; X32-NOCMOV-NEXT: andl %edi, %ebp
-; X32-NOCMOV-NEXT: xorl %esi, %ebp
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl %ecx, %esi
-; X32-NOCMOV-NEXT: andl %edi, %esi
-; X32-NOCMOV-NEXT: xorl %ecx, %esi
-; X32-NOCMOV-NEXT: movl %esi, 12(%eax)
-; X32-NOCMOV-NEXT: movl %ebp, 8(%eax)
-; X32-NOCMOV-NEXT: movl %ebx, 4(%eax)
-; X32-NOCMOV-NEXT: movl %edx, (%eax)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %edx, %eax
+; X32-NOCMOV-NEXT: andl %ebp, %eax
+; X32-NOCMOV-NEXT: xorl %edi, %ecx
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: xorl %edx, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl %eax, 12(%edx)
+; X32-NOCMOV-NEXT: movl %ebx, 8(%edx)
+; X32-NOCMOV-NEXT: movl %esi, 4(%edx)
+; X32-NOCMOV-NEXT: movl %ecx, (%edx)
+; X32-NOCMOV-NEXT: movl %edx, %eax
; X32-NOCMOV-NEXT: popl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; X32-NOCMOV-NEXT: popl %edi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
; X32-NOCMOV-NEXT: popl %ebx
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: popl %ebp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl $4
%result = call <4 x i32> @llvm.ct.select.v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b)
ret <4 x i32> %result
}
-define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b) {
+define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b) #0 {
; X64-LABEL: test_ctselect_v4f32:
; X64: # %bb.0:
; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: negl %edi
; X64-NEXT: movd %edi, %xmm2
; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,0,0]
-; X64-NEXT: pslld $31, %xmm2
-; X64-NEXT: psrad $31, %xmm2
; X64-NEXT: pand %xmm2, %xmm0
; X64-NEXT: pxor %xmm1, %xmm0
; X64-NEXT: retq
@@ -946,530 +1273,1133 @@ define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b
; X32-LABEL: test_ctselect_v4f32:
; X32: # %bb.0:
; X32-NEXT: pushl %ebp
-; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: pushl %ebx
-; X32-NEXT: .cfi_def_cfa_offset 12
; X32-NEXT: pushl %edi
-; X32-NEXT: .cfi_def_cfa_offset 16
; X32-NEXT: pushl %esi
-; X32-NEXT: .cfi_def_cfa_offset 20
-; X32-NEXT: .cfi_offset %esi, -20
-; X32-NEXT: .cfi_offset %edi, -16
-; X32-NEXT: .cfi_offset %ebx, -12
-; X32-NEXT: .cfi_offset %ebp, -8
+; X32-NEXT: subl $48, %esp
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: andb $1, %dl
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorl %ebx, %edx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: andl $1, %edi
-; X32-NEXT: negl %edi
-; X32-NEXT: andl %edi, %edx
-; X32-NEXT: xorl %ebx, %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl %ebp, %ebx
-; X32-NEXT: andl %edi, %ebx
-; X32-NEXT: xorl %ebp, %ebx
+; X32-NEXT: movzbl %dl, %edx
+; X32-NEXT: negl %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorl %esi, %ebp
-; X32-NEXT: andl %edi, %ebp
-; X32-NEXT: xorl %esi, %ebp
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl %ebx, %ebp
+; X32-NEXT: andl %edx, %ebp
+; X32-NEXT: xorl %ebx, %ebp
+; X32-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: xorl %edi, %ebx
+; X32-NEXT: andl %edx, %ebx
+; X32-NEXT: xorl %edi, %ebx
+; X32-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: xorl %esi, %edi
+; X32-NEXT: andl %edx, %edi
+; X32-NEXT: xorl %esi, %edi
+; X32-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X32-NEXT: movl (%esp), %esi
; X32-NEXT: xorl %ecx, %esi
-; X32-NEXT: andl %edi, %esi
+; X32-NEXT: andl %edx, %esi
; X32-NEXT: xorl %ecx, %esi
-; X32-NEXT: movl %esi, 12(%eax)
-; X32-NEXT: movl %ebp, 8(%eax)
-; X32-NEXT: movl %ebx, 4(%eax)
-; X32-NEXT: movl %edx, (%eax)
+; X32-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps 12(%eax)
+; X32-NEXT: fstps 8(%eax)
+; X32-NEXT: fstps 4(%eax)
+; X32-NEXT: fstps (%eax)
+; X32-NEXT: addl $48, %esp
; X32-NEXT: popl %esi
-; X32-NEXT: .cfi_def_cfa_offset 16
; X32-NEXT: popl %edi
-; X32-NEXT: .cfi_def_cfa_offset 12
; X32-NEXT: popl %ebx
-; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: popl %ebp
-; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl $4
;
; X32-NOCMOV-LABEL: test_ctselect_v4f32:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: pushl %ebp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: pushl %ebx
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
; X32-NOCMOV-NEXT: pushl %edi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; X32-NOCMOV-NEXT: .cfi_offset %esi, -20
-; X32-NOCMOV-NEXT: .cfi_offset %edi, -16
-; X32-NOCMOV-NEXT: .cfi_offset %ebx, -12
-; X32-NOCMOV-NEXT: .cfi_offset %ebp, -8
+; X32-NOCMOV-NEXT: subl $48, %esp
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: andb $1, %dl
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorl %ebx, %edx
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: andl $1, %edi
-; X32-NOCMOV-NEXT: negl %edi
-; X32-NOCMOV-NEXT: andl %edi, %edx
-; X32-NOCMOV-NEXT: xorl %ebx, %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl %ebp, %ebx
-; X32-NOCMOV-NEXT: andl %edi, %ebx
-; X32-NOCMOV-NEXT: xorl %ebp, %ebx
+; X32-NOCMOV-NEXT: movzbl %dl, %edx
+; X32-NOCMOV-NEXT: negl %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorl %esi, %ebp
-; X32-NOCMOV-NEXT: andl %edi, %ebp
-; X32-NOCMOV-NEXT: xorl %esi, %ebp
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl %ebx, %ebp
+; X32-NOCMOV-NEXT: andl %edx, %ebp
+; X32-NOCMOV-NEXT: xorl %ebx, %ebp
+; X32-NOCMOV-NEXT: movl %ebp, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: xorl %edi, %ebx
+; X32-NOCMOV-NEXT: andl %edx, %ebx
+; X32-NOCMOV-NEXT: xorl %edi, %ebx
+; X32-NOCMOV-NEXT: movl %ebx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: xorl %esi, %edi
+; X32-NOCMOV-NEXT: andl %edx, %edi
+; X32-NOCMOV-NEXT: xorl %esi, %edi
+; X32-NOCMOV-NEXT: movl %edi, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl (%esp), %esi
; X32-NOCMOV-NEXT: xorl %ecx, %esi
-; X32-NOCMOV-NEXT: andl %edi, %esi
+; X32-NOCMOV-NEXT: andl %edx, %esi
; X32-NOCMOV-NEXT: xorl %ecx, %esi
-; X32-NOCMOV-NEXT: movl %esi, 12(%eax)
-; X32-NOCMOV-NEXT: movl %ebp, 8(%eax)
-; X32-NOCMOV-NEXT: movl %ebx, 4(%eax)
-; X32-NOCMOV-NEXT: movl %edx, (%eax)
+; X32-NOCMOV-NEXT: movl %esi, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps 12(%eax)
+; X32-NOCMOV-NEXT: fstps 8(%eax)
+; X32-NOCMOV-NEXT: fstps 4(%eax)
+; X32-NOCMOV-NEXT: fstps (%eax)
+; X32-NOCMOV-NEXT: addl $48, %esp
; X32-NOCMOV-NEXT: popl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; X32-NOCMOV-NEXT: popl %edi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
; X32-NOCMOV-NEXT: popl %ebx
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: popl %ebp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl $4
%result = call <4 x float> @llvm.ct.select.v4f32(i1 %cond, <4 x float> %a, <4 x float> %b)
ret <4 x float> %result
}
-define <8 x i32> @test_ctselect_v8i32_avx(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
+define <8 x i32> @test_ctselect_v8i32_avx(i1 %cond, <8 x i32> %a, <8 x i32> %b) #0 {
; X64-LABEL: test_ctselect_v8i32_avx:
; X64: # %bb.0:
+; X64-NEXT: pxor %xmm2, %xmm0
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: negl %edi
; X64-NEXT: movd %edi, %xmm4
; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; X64-NEXT: pslld $31, %xmm4
-; X64-NEXT: psrad $31, %xmm4
-; X64-NEXT: movdqa %xmm4, %xmm5
-; X64-NEXT: pandn %xmm2, %xmm5
; X64-NEXT: pand %xmm4, %xmm0
-; X64-NEXT: por %xmm5, %xmm0
+; X64-NEXT: pxor %xmm2, %xmm0
+; X64-NEXT: pxor %xmm3, %xmm1
; X64-NEXT: pand %xmm4, %xmm1
-; X64-NEXT: pandn %xmm3, %xmm4
-; X64-NEXT: por %xmm4, %xmm1
+; X64-NEXT: pxor %xmm3, %xmm1
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_v8i32_avx:
; X32: # %bb.0:
; X32-NEXT: pushl %ebp
-; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: pushl %ebx
-; X32-NEXT: .cfi_def_cfa_offset 12
; X32-NEXT: pushl %edi
-; X32-NEXT: .cfi_def_cfa_offset 16
; X32-NEXT: pushl %esi
-; X32-NEXT: .cfi_def_cfa_offset 20
; X32-NEXT: subl $8, %esp
-; X32-NEXT: .cfi_def_cfa_offset 28
-; X32-NEXT: .cfi_offset %esi, -20
-; X32-NEXT: .cfi_offset %edi, -16
-; X32-NEXT: .cfi_offset %ebx, -12
-; X32-NEXT: .cfi_offset %ebp, -8
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb $1, %al
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movzbl %al, %ecx
+; X32-NEXT: negl %ecx
+; X32-NEXT: andl %ecx, %edx
+; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andl %ecx, %eax
+; X32-NEXT: movl %eax, (%esp) # 4-byte Spill
; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: andl %ecx, %ebp
; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: andl %ecx, %ebx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: andl %ecx, %edi
; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: andl %ecx, %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: andl %ecx, %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andl %ecx, %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %eax, %ecx
+; X32-NEXT: xorl %ecx, (%esp) # 4-byte Folded Spill
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl %eax, 28(%ecx)
+; X32-NEXT: movl %edx, 24(%ecx)
+; X32-NEXT: movl %esi, 20(%ecx)
+; X32-NEXT: movl %edi, 16(%ecx)
+; X32-NEXT: movl %ebx, 12(%ecx)
+; X32-NEXT: movl %ebp, 8(%ecx)
+; X32-NEXT: movl (%esp), %eax # 4-byte Reload
+; X32-NEXT: movl %eax, 4(%ecx)
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: movl %eax, (%ecx)
+; X32-NEXT: movl %ecx, %eax
+; X32-NEXT: addl $8, %esp
+; X32-NEXT: popl %esi
+; X32-NEXT: popl %edi
+; X32-NEXT: popl %ebx
+; X32-NEXT: popl %ebp
+; X32-NEXT: retl $4
+;
+; X32-NOCMOV-LABEL: test_ctselect_v8i32_avx:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebp
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: subl $8, %esp
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movzbl %al, %ecx
+; X32-NOCMOV-NEXT: negl %ecx
+; X32-NOCMOV-NEXT: andl %ecx, %edx
+; X32-NOCMOV-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andl %ecx, %eax
+; X32-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: andl %ecx, %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: andl %ecx, %ebx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: andl %ecx, %edi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: andl %ecx, %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: andl %ecx, %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andl %ecx, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorl %ecx, (%esp) # 4-byte Folded Spill
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl %eax, 28(%ecx)
+; X32-NOCMOV-NEXT: movl %edx, 24(%ecx)
+; X32-NOCMOV-NEXT: movl %esi, 20(%ecx)
+; X32-NOCMOV-NEXT: movl %edi, 16(%ecx)
+; X32-NOCMOV-NEXT: movl %ebx, 12(%ecx)
+; X32-NOCMOV-NEXT: movl %ebp, 8(%ecx)
+; X32-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %eax, 4(%ecx)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NOCMOV-NEXT: movl %eax, (%ecx)
+; X32-NOCMOV-NEXT: movl %ecx, %eax
+; X32-NOCMOV-NEXT: addl $8, %esp
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
+; X32-NOCMOV-NEXT: popl %ebp
+; X32-NOCMOV-NEXT: retl $4
+ %result = call <8 x i32> @llvm.ct.select.v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b)
+ ret <8 x i32> %result
+}
+
+define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b) #0 {
+; X64-LABEL: test_ctselect_v8f32:
+; X64: # %bb.0:
+; X64-NEXT: pxor %xmm2, %xmm0
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: negl %edi
+; X64-NEXT: movd %edi, %xmm4
+; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
+; X64-NEXT: pand %xmm4, %xmm0
+; X64-NEXT: pxor %xmm2, %xmm0
+; X64-NEXT: pxor %xmm3, %xmm1
+; X64-NEXT: pand %xmm4, %xmm1
+; X64-NEXT: pxor %xmm3, %xmm1
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_v8f32:
+; X32: # %bb.0:
+; X32-NEXT: pushl %ebp
+; X32-NEXT: pushl %ebx
+; X32-NEXT: pushl %edi
+; X32-NEXT: pushl %esi
+; X32-NEXT: subl $104, %esp
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps {{[0-9]+}}(%esp)
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: andl $1, %edx
-; X32-NEXT: negl %edx
-; X32-NEXT: andl %edx, %ecx
-; X32-NEXT: xorl %eax, %ecx
-; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: andb $1, %dl
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %esi, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %esi, %eax
; X32-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl %ebx, %esi
-; X32-NEXT: andl %edx, %esi
-; X32-NEXT: xorl %ebx, %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: movzbl %dl, %edx
+; X32-NEXT: negl %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %ebx, %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ebx, %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl %ebp, %ebx
-; X32-NEXT: andl %edx, %ebx
-; X32-NEXT: xorl %ebp, %ebx
; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorl %edi, %ebp
-; X32-NEXT: andl %edx, %ebp
-; X32-NEXT: xorl %edi, %ebp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %ebp, %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ebp, %eax
+; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %ebx, %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ebx, %eax
+; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %edi, %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %edi, %eax
+; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: xorl %esi, %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %esi, %eax
+; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: andl %edx, %eax
+; X32-NEXT: xorl %ecx, %eax
+; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp) # 4-byte Folded Spill
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps 28(%eax)
+; X32-NEXT: fstps 24(%eax)
+; X32-NEXT: fstps 20(%eax)
+; X32-NEXT: fstps 16(%eax)
+; X32-NEXT: fstps 12(%eax)
+; X32-NEXT: fstps 8(%eax)
+; X32-NEXT: fstps 4(%eax)
+; X32-NEXT: flds (%esp) # 4-byte Folded Reload
+; X32-NEXT: fstps (%eax)
+; X32-NEXT: addl $104, %esp
+; X32-NEXT: popl %esi
+; X32-NEXT: popl %edi
+; X32-NEXT: popl %ebx
+; X32-NEXT: popl %ebp
+; X32-NEXT: retl $4
+;
+; X32-NOCMOV-LABEL: test_ctselect_v8f32:
+; X32-NOCMOV: # %bb.0:
+; X32-NOCMOV-NEXT: pushl %ebp
+; X32-NOCMOV-NEXT: pushl %ebx
+; X32-NOCMOV-NEXT: pushl %edi
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: subl $104, %esp
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: andb $1, %dl
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: movzbl %dl, %edx
+; X32-NOCMOV-NEXT: negl %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %ebx, %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ebx, %eax
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %ebp, %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ebp, %eax
+; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %ebx, %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ebx, %eax
+; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %edi, %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %edi, %eax
+; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: xorl %esi, %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %esi, %eax
+; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: andl %edx, %eax
+; X32-NOCMOV-NEXT: xorl %ecx, %eax
+; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp) # 4-byte Folded Spill
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps 28(%eax)
+; X32-NOCMOV-NEXT: fstps 24(%eax)
+; X32-NOCMOV-NEXT: fstps 20(%eax)
+; X32-NOCMOV-NEXT: fstps 16(%eax)
+; X32-NOCMOV-NEXT: fstps 12(%eax)
+; X32-NOCMOV-NEXT: fstps 8(%eax)
+; X32-NOCMOV-NEXT: fstps 4(%eax)
+; X32-NOCMOV-NEXT: flds (%esp) # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: fstps (%eax)
+; X32-NOCMOV-NEXT: addl $104, %esp
+; X32-NOCMOV-NEXT: popl %esi
+; X32-NOCMOV-NEXT: popl %edi
+; X32-NOCMOV-NEXT: popl %ebx
+; X32-NOCMOV-NEXT: popl %ebp
+; X32-NOCMOV-NEXT: retl $4
+ %result = call <8 x float> @llvm.ct.select.v8f32(i1 %cond, <8 x float> %a, <8 x float> %b)
+ ret <8 x float> %result
+}
+
+define <8 x half> @test_ctselect_v8f16(i1 %cond, <8 x half> %a, <8 x half> %b) #0 {
+; X64-LABEL: test_ctselect_v8f16:
+; X64: # %bb.0:
+; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: negl %edi
+; X64-NEXT: movd %edi, %xmm2
+; X64-NEXT: pshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7]
+; X64-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,1,0,1]
+; X64-NEXT: pand %xmm2, %xmm0
+; X64-NEXT: pxor %xmm1, %xmm0
+; X64-NEXT: retq
+;
+; X32-LABEL: test_ctselect_v8f16:
+; X32: # %bb.0:
+; X32-NEXT: pushl %ebp
+; X32-NEXT: pushl %ebx
+; X32-NEXT: pushl %edi
+; X32-NEXT: pushl %esi
+; X32-NEXT: subl $12, %esp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: andb $1, %al
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorw %di, %cx
+; X32-NEXT: movzbl %al, %eax
+; X32-NEXT: negl %eax
+; X32-NEXT: andl %eax, %ecx
+; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorw %si, %cx
+; X32-NEXT: andl %eax, %ecx
+; X32-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: xorw %dx, %cx
+; X32-NEXT: andl %eax, %ecx
+; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: xorw %cx, %bx
+; X32-NEXT: andl %eax, %ebx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: xorw %cx, %bp
+; X32-NEXT: andl %eax, %ebp
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorw %cx, %si
+; X32-NEXT: andl %eax, %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorw %cx, %dx
+; X32-NEXT: andl %eax, %edx
+; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: xorl %eax, %edi
-; X32-NEXT: andl %edx, %edi
-; X32-NEXT: xorl %eax, %edi
+; X32-NEXT: xorw %di, %cx
+; X32-NEXT: andl %eax, %ecx
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %eax, %ecx
-; X32-NEXT: andl %edx, %ecx
-; X32-NEXT: xorl %eax, %ecx
+; X32-NEXT: xorl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl %eax, 28(%edx)
-; X32-NEXT: movl %ecx, 24(%edx)
-; X32-NEXT: movl %edi, 20(%edx)
-; X32-NEXT: movl %ebp, 16(%edx)
-; X32-NEXT: movl %ebx, 12(%edx)
-; X32-NEXT: movl %esi, 8(%edx)
-; X32-NEXT: movl (%esp), %eax # 4-byte Reload
-; X32-NEXT: movl %eax, 4(%edx)
-; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT: movl %eax, (%edx)
-; X32-NEXT: movl %edx, %eax
-; X32-NEXT: addl $8, %esp
-; X32-NEXT: .cfi_def_cfa_offset 20
+; X32-NEXT: xorl %eax, (%esp) # 4-byte Folded Spill
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %edi
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %ebp
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT: movw %cx, 14(%eax)
+; X32-NEXT: movw %dx, 12(%eax)
+; X32-NEXT: movw %si, 10(%eax)
+; X32-NEXT: movw %bp, 8(%eax)
+; X32-NEXT: movw %bx, 6(%eax)
+; X32-NEXT: movw %di, 4(%eax)
+; X32-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X32-NEXT: movw %cx, 2(%eax)
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NEXT: movw %cx, (%eax)
+; X32-NEXT: addl $12, %esp
; X32-NEXT: popl %esi
-; X32-NEXT: .cfi_def_cfa_offset 16
; X32-NEXT: popl %edi
-; X32-NEXT: .cfi_def_cfa_offset 12
; X32-NEXT: popl %ebx
-; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: popl %ebp
-; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl $4
;
-; X32-NOCMOV-LABEL: test_ctselect_v8i32_avx:
+; X32-NOCMOV-LABEL: test_ctselect_v8f16:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: pushl %ebp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: pushl %ebx
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
; X32-NOCMOV-NEXT: pushl %edi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; X32-NOCMOV-NEXT: subl $8, %esp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 28
-; X32-NOCMOV-NEXT: .cfi_offset %esi, -20
-; X32-NOCMOV-NEXT: .cfi_offset %edi, -16
-; X32-NOCMOV-NEXT: .cfi_offset %ebx, -12
-; X32-NOCMOV-NEXT: .cfi_offset %ebp, -8
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: subl $12, %esp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %eax, %ecx
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: andl $1, %edx
-; X32-NOCMOV-NEXT: negl %edx
-; X32-NOCMOV-NEXT: andl %edx, %ecx
-; X32-NOCMOV-NEXT: xorl %eax, %ecx
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorw %di, %cx
+; X32-NOCMOV-NEXT: movzbl %al, %eax
+; X32-NOCMOV-NEXT: negl %eax
+; X32-NOCMOV-NEXT: andl %eax, %ecx
; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %esi, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %esi, %eax
-; X32-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl %ebx, %esi
-; X32-NOCMOV-NEXT: andl %edx, %esi
-; X32-NOCMOV-NEXT: xorl %ebx, %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl %ebp, %ebx
-; X32-NOCMOV-NEXT: andl %edx, %ebx
-; X32-NOCMOV-NEXT: xorl %ebp, %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorl %edi, %ebp
-; X32-NOCMOV-NEXT: andl %edx, %ebp
-; X32-NOCMOV-NEXT: xorl %edi, %ebp
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorw %si, %cx
+; X32-NOCMOV-NEXT: andl %eax, %ecx
+; X32-NOCMOV-NEXT: movl %ecx, (%esp) # 4-byte Spill
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: xorw %dx, %cx
+; X32-NOCMOV-NEXT: andl %eax, %ecx
+; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: xorw %cx, %bx
+; X32-NOCMOV-NEXT: andl %eax, %ebx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: xorw %cx, %bp
+; X32-NOCMOV-NEXT: andl %eax, %ebp
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorw %cx, %si
+; X32-NOCMOV-NEXT: andl %eax, %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorw %cx, %dx
+; X32-NOCMOV-NEXT: andl %eax, %edx
+; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: xorl %eax, %edi
-; X32-NOCMOV-NEXT: andl %edx, %edi
-; X32-NOCMOV-NEXT: xorl %eax, %edi
+; X32-NOCMOV-NEXT: xorw %di, %cx
+; X32-NOCMOV-NEXT: andl %eax, %ecx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %eax, %ecx
-; X32-NOCMOV-NEXT: andl %edx, %ecx
-; X32-NOCMOV-NEXT: xorl %eax, %ecx
+; X32-NOCMOV-NEXT: xorl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Folded Spill
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl %eax, 28(%edx)
-; X32-NOCMOV-NEXT: movl %ecx, 24(%edx)
-; X32-NOCMOV-NEXT: movl %edi, 20(%edx)
-; X32-NOCMOV-NEXT: movl %ebp, 16(%edx)
-; X32-NOCMOV-NEXT: movl %ebx, 12(%edx)
-; X32-NOCMOV-NEXT: movl %esi, 8(%edx)
-; X32-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: movl %eax, 4(%edx)
-; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: movl %eax, (%edx)
-; X32-NOCMOV-NEXT: movl %edx, %eax
-; X32-NOCMOV-NEXT: addl $8, %esp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; X32-NOCMOV-NEXT: xorl %eax, (%esp) # 4-byte Folded Spill
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edi
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebx
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ebp
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X32-NOCMOV-NEXT: movw %cx, 14(%eax)
+; X32-NOCMOV-NEXT: movw %dx, 12(%eax)
+; X32-NOCMOV-NEXT: movw %si, 10(%eax)
+; X32-NOCMOV-NEXT: movw %bp, 8(%eax)
+; X32-NOCMOV-NEXT: movw %bx, 6(%eax)
+; X32-NOCMOV-NEXT: movw %di, 4(%eax)
+; X32-NOCMOV-NEXT: movl (%esp), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %cx, 2(%eax)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %cx, (%eax)
+; X32-NOCMOV-NEXT: addl $12, %esp
; X32-NOCMOV-NEXT: popl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; X32-NOCMOV-NEXT: popl %edi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
; X32-NOCMOV-NEXT: popl %ebx
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: popl %ebp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl $4
- %result = call <8 x i32> @llvm.ct.select.v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b)
- ret <8 x i32> %result
+ %result = call <8 x half> @llvm.ct.select.v8f16(i1 %cond, <8 x half> %a, <8 x half> %b)
+ ret <8 x half> %result
}
-define <8 x float> @test_ctselect_v8f32(i1 %cond, <8 x float> %a, <8 x float> %b) {
-; X64-LABEL: test_ctselect_v8f32:
+define <8 x bfloat> @test_ctselect_v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat> %b) #0 {
+; X64-LABEL: test_ctselect_v8bf16:
; X64: # %bb.0:
-; X64-NEXT: movd %edi, %xmm4
-; X64-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,0,0]
-; X64-NEXT: pslld $31, %xmm4
-; X64-NEXT: psrad $31, %xmm4
-; X64-NEXT: movdqa %xmm4, %xmm5
-; X64-NEXT: pandn %xmm2, %xmm5
-; X64-NEXT: pand %xmm4, %xmm0
-; X64-NEXT: por %xmm5, %xmm0
-; X64-NEXT: pand %xmm4, %xmm1
-; X64-NEXT: pandn %xmm3, %xmm4
-; X64-NEXT: por %xmm4, %xmm1
+; X64-NEXT: movq %xmm1, %rax
+; X64-NEXT: movq %xmm0, %rcx
+; X64-NEXT: punpckhqdq {{.*#+}} xmm1 = xmm1[1,1]
+; X64-NEXT: movq %xmm1, %rsi
+; X64-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1,1]
+; X64-NEXT: movq %xmm0, %r9
+; X64-NEXT: movq %r9, %rdx
+; X64-NEXT: movl %esi, %r8d
+; X64-NEXT: shrl $16, %r8d
+; X64-NEXT: movl %r9d, %r10d
+; X64-NEXT: shrl $16, %r10d
+; X64-NEXT: xorl %r8d, %r10d
+; X64-NEXT: andl $1, %edi
+; X64-NEXT: negl %edi
+; X64-NEXT: andl %edi, %r10d
+; X64-NEXT: xorl %r8d, %r10d
+; X64-NEXT: movq %r9, %r8
+; X64-NEXT: xorl %esi, %r9d
+; X64-NEXT: andl %edi, %r9d
+; X64-NEXT: shll $16, %r10d
+; X64-NEXT: xorl %esi, %r9d
+; X64-NEXT: movzwl %r9w, %r9d
+; X64-NEXT: orl %r10d, %r9d
+; X64-NEXT: movq %rsi, %r10
+; X64-NEXT: shrq $48, %r10
+; X64-NEXT: shrq $48, %r8
+; X64-NEXT: xorl %r10d, %r8d
+; X64-NEXT: andl %edi, %r8d
+; X64-NEXT: xorl %r10d, %r8d
+; X64-NEXT: shrq $32, %rsi
+; X64-NEXT: shrq $32, %rdx
+; X64-NEXT: xorl %esi, %edx
+; X64-NEXT: andl %edi, %edx
+; X64-NEXT: xorl %esi, %edx
+; X64-NEXT: movq %rcx, %rsi
+; X64-NEXT: shll $16, %r8d
+; X64-NEXT: movzwl %dx, %edx
+; X64-NEXT: orl %r8d, %edx
+; X64-NEXT: movl %eax, %r8d
+; X64-NEXT: shrl $16, %r8d
+; X64-NEXT: shlq $32, %rdx
+; X64-NEXT: orq %r9, %rdx
+; X64-NEXT: movl %ecx, %r9d
+; X64-NEXT: shrl $16, %r9d
+; X64-NEXT: xorl %r8d, %r9d
+; X64-NEXT: andl %edi, %r9d
+; X64-NEXT: xorl %r8d, %r9d
+; X64-NEXT: movq %rcx, %r8
+; X64-NEXT: xorl %eax, %ecx
+; X64-NEXT: andl %edi, %ecx
+; X64-NEXT: shll $16, %r9d
+; X64-NEXT: xorl %eax, %ecx
+; X64-NEXT: movzwl %cx, %ecx
+; X64-NEXT: orl %r9d, %ecx
+; X64-NEXT: movq %rax, %r9
+; X64-NEXT: shrq $32, %rax
+; X64-NEXT: shrq $32, %rsi
+; X64-NEXT: shrq $48, %r9
+; X64-NEXT: shrq $48, %r8
+; X64-NEXT: xorl %r9d, %r8d
+; X64-NEXT: andl %edi, %r8d
+; X64-NEXT: xorl %eax, %esi
+; X64-NEXT: andl %edi, %esi
+; X64-NEXT: xorl %r9d, %r8d
+; X64-NEXT: xorl %eax, %esi
+; X64-NEXT: shll $16, %r8d
+; X64-NEXT: movzwl %si, %eax
+; X64-NEXT: orl %r8d, %eax
+; X64-NEXT: shlq $32, %rax
+; X64-NEXT: orq %rcx, %rax
+; X64-NEXT: movq %rax, %xmm0
+; X64-NEXT: movq %rdx, %xmm1
+; X64-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; X64-NEXT: retq
;
-; X32-LABEL: test_ctselect_v8f32:
+; X32-LABEL: test_ctselect_v8bf16:
; X32: # %bb.0:
; X32-NEXT: pushl %ebp
-; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: pushl %ebx
-; X32-NEXT: .cfi_def_cfa_offset 12
; X32-NEXT: pushl %edi
-; X32-NEXT: .cfi_def_cfa_offset 16
; X32-NEXT: pushl %esi
-; X32-NEXT: .cfi_def_cfa_offset 20
-; X32-NEXT: subl $8, %esp
-; X32-NEXT: .cfi_def_cfa_offset 28
-; X32-NEXT: .cfi_offset %esi, -20
-; X32-NEXT: .cfi_offset %edi, -16
-; X32-NEXT: .cfi_offset %ebx, -12
-; X32-NEXT: .cfi_offset %ebp, -8
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %eax, %ecx
-; X32-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: andl $1, %edx
-; X32-NEXT: negl %edx
-; X32-NEXT: andl %edx, %ecx
-; X32-NEXT: xorl %eax, %ecx
+; X32-NEXT: subl $60, %esp
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: movl %eax, %ebp
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: movl %eax, %edi
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: movl %eax, %esi
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NEXT: fstps (%esp)
+; X32-NEXT: calll __truncsfbf2
+; X32-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: andb $1, %cl
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT: movzbl %cl, %ecx
+; X32-NEXT: negl %ecx
+; X32-NEXT: andl %ecx, %eax
+; X32-NEXT: movl %eax, %ebx
+; X32-NEXT: movl %ebp, %eax
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT: andl %ecx, %eax
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT: andl %ecx, %ebp
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT: andl %ecx, %edx
+; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT: andl %ecx, %edx
+; X32-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT: andl %ecx, %edx
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT: andl %ecx, %edi
+; X32-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT: andl %ecx, %esi
+; X32-NEXT: movl %esi, %ecx
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X32-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl %esi, %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl %esi, %eax
-; X32-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorl %ebx, %esi
-; X32-NEXT: andl %edx, %esi
-; X32-NEXT: xorl %ebx, %esi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NEXT: xorl %ebp, %ebx
-; X32-NEXT: andl %edx, %ebx
-; X32-NEXT: xorl %ebp, %ebx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NEXT: xorl %edi, %ebp
-; X32-NEXT: andl %edx, %ebp
-; X32-NEXT: xorl %edi, %ebp
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NEXT: xorl %eax, %edi
-; X32-NEXT: andl %edx, %edi
-; X32-NEXT: xorl %eax, %edi
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: xorl %eax, %ecx
-; X32-NEXT: andl %edx, %ecx
-; X32-NEXT: xorl %eax, %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andl %edx, %eax
-; X32-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl %eax, 28(%edx)
-; X32-NEXT: movl %ecx, 24(%edx)
-; X32-NEXT: movl %edi, 20(%edx)
-; X32-NEXT: movl %ebp, 16(%edx)
-; X32-NEXT: movl %ebx, 12(%edx)
-; X32-NEXT: movl %esi, 8(%edx)
-; X32-NEXT: movl (%esp), %eax # 4-byte Reload
-; X32-NEXT: movl %eax, 4(%edx)
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NEXT: movw %bx, 14(%ecx)
+; X32-NEXT: movw %ax, 12(%ecx)
+; X32-NEXT: movw %dx, 10(%ecx)
+; X32-NEXT: movw %si, 8(%ecx)
+; X32-NEXT: movw %di, 6(%ecx)
+; X32-NEXT: movw %bp, 4(%ecx)
; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NEXT: movl %eax, (%edx)
-; X32-NEXT: movl %edx, %eax
-; X32-NEXT: addl $8, %esp
-; X32-NEXT: .cfi_def_cfa_offset 20
+; X32-NEXT: movw %ax, 2(%ecx)
+; X32-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NEXT: movw %ax, (%ecx)
+; X32-NEXT: movl %ecx, %eax
+; X32-NEXT: addl $60, %esp
; X32-NEXT: popl %esi
-; X32-NEXT: .cfi_def_cfa_offset 16
; X32-NEXT: popl %edi
-; X32-NEXT: .cfi_def_cfa_offset 12
; X32-NEXT: popl %ebx
-; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: popl %ebp
-; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl $4
;
-; X32-NOCMOV-LABEL: test_ctselect_v8f32:
+; X32-NOCMOV-LABEL: test_ctselect_v8bf16:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: pushl %ebp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: pushl %ebx
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
; X32-NOCMOV-NEXT: pushl %edi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 20
-; X32-NOCMOV-NEXT: subl $8, %esp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 28
-; X32-NOCMOV-NEXT: .cfi_offset %esi, -20
-; X32-NOCMOV-NEXT: .cfi_offset %edi, -16
-; X32-NOCMOV-NEXT: .cfi_offset %ebx, -12
-; X32-NOCMOV-NEXT: .cfi_offset %ebp, -8
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %eax, %ecx
-; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: andl $1, %edx
-; X32-NOCMOV-NEXT: negl %edx
-; X32-NOCMOV-NEXT: andl %edx, %ecx
-; X32-NOCMOV-NEXT: xorl %eax, %ecx
+; X32-NOCMOV-NEXT: subl $60, %esp
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: movl %eax, %ebp
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: movl %eax, %edi
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: movl %eax, %esi
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: flds {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstps (%esp)
+; X32-NOCMOV-NEXT: calll __truncsfbf2
+; X32-NOCMOV-NEXT: # kill: def $ax killed $ax def $eax
+; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: andb $1, %cl
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: movzbl %cl, %ecx
+; X32-NOCMOV-NEXT: negl %ecx
+; X32-NOCMOV-NEXT: andl %ecx, %eax
+; X32-NOCMOV-NEXT: movl %eax, %ebx
+; X32-NOCMOV-NEXT: movl %ebp, %eax
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: andl %ecx, %eax
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Reload
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: andl %ecx, %ebp
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: andl %ecx, %edx
+; X32-NOCMOV-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: andl %ecx, %edx
+; X32-NOCMOV-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Reload
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: andl %ecx, %edx
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: andl %ecx, %edi
+; X32-NOCMOV-NEXT: movl %edi, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: andl %ecx, %esi
+; X32-NOCMOV-NEXT: movl %esi, %ecx
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ebp # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Reload
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edi # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Reload
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %esi # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %edx # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Folded Reload
+; X32-NOCMOV-NEXT: xorl {{[-0-9]+}}(%e{{[sb]}}p), %ecx # 4-byte Folded Reload
; X32-NOCMOV-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl %esi, %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl %esi, %eax
-; X32-NOCMOV-NEXT: movl %eax, (%esp) # 4-byte Spill
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorl %ebx, %esi
-; X32-NOCMOV-NEXT: andl %edx, %esi
-; X32-NOCMOV-NEXT: xorl %ebx, %esi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebx
-; X32-NOCMOV-NEXT: xorl %ebp, %ebx
-; X32-NOCMOV-NEXT: andl %edx, %ebx
-; X32-NOCMOV-NEXT: xorl %ebp, %ebx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ebp
-; X32-NOCMOV-NEXT: xorl %edi, %ebp
-; X32-NOCMOV-NEXT: andl %edx, %ebp
-; X32-NOCMOV-NEXT: xorl %edi, %ebp
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edi
-; X32-NOCMOV-NEXT: xorl %eax, %edi
-; X32-NOCMOV-NEXT: andl %edx, %edi
-; X32-NOCMOV-NEXT: xorl %eax, %edi
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: xorl %eax, %ecx
-; X32-NOCMOV-NEXT: andl %edx, %ecx
-; X32-NOCMOV-NEXT: xorl %eax, %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andl %edx, %eax
-; X32-NOCMOV-NEXT: xorl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl %eax, 28(%edx)
-; X32-NOCMOV-NEXT: movl %ecx, 24(%edx)
-; X32-NOCMOV-NEXT: movl %edi, 20(%edx)
-; X32-NOCMOV-NEXT: movl %ebp, 16(%edx)
-; X32-NOCMOV-NEXT: movl %ebx, 12(%edx)
-; X32-NOCMOV-NEXT: movl %esi, 8(%edx)
-; X32-NOCMOV-NEXT: movl (%esp), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: movl %eax, 4(%edx)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %ebx # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %bx, 14(%ecx)
+; X32-NOCMOV-NEXT: movw %ax, 12(%ecx)
+; X32-NOCMOV-NEXT: movw %dx, 10(%ecx)
+; X32-NOCMOV-NEXT: movw %si, 8(%ecx)
+; X32-NOCMOV-NEXT: movw %di, 6(%ecx)
+; X32-NOCMOV-NEXT: movw %bp, 4(%ecx)
; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
-; X32-NOCMOV-NEXT: movl %eax, (%edx)
-; X32-NOCMOV-NEXT: movl %edx, %eax
-; X32-NOCMOV-NEXT: addl $8, %esp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 20
+; X32-NOCMOV-NEXT: movw %ax, 2(%ecx)
+; X32-NOCMOV-NEXT: movl {{[-0-9]+}}(%e{{[sb]}}p), %eax # 4-byte Reload
+; X32-NOCMOV-NEXT: movw %ax, (%ecx)
+; X32-NOCMOV-NEXT: movl %ecx, %eax
+; X32-NOCMOV-NEXT: addl $60, %esp
; X32-NOCMOV-NEXT: popl %esi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
; X32-NOCMOV-NEXT: popl %edi
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 12
; X32-NOCMOV-NEXT: popl %ebx
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: popl %ebp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl $4
- %result = call <8 x float> @llvm.ct.select.v8f32(i1 %cond, <8 x float> %a, <8 x float> %b)
- ret <8 x float> %result
+ %result = call <8 x bfloat> @llvm.ct.select.v8bf16(i1 %cond, <8 x bfloat> %a, <8 x bfloat> %b)
+ ret <8 x bfloat> %result
}
-define float @test_ctselect_f32_nan_inf(i1 %cond) {
+define float @test_ctselect_f32_nan_inf(i1 %cond) #0 {
; X64-LABEL: test_ctselect_f32_nan_inf:
; X64: # %bb.0:
; X64-NEXT: andl $1, %edi
; X64-NEXT: negl %edi
; X64-NEXT: andl $4194304, %edi # imm = 0x400000
-; X64-NEXT: orl $2139095040, %edi # imm = 0x7F800000
+; X64-NEXT: xorl $2139095040, %edi # imm = 0x7F800000
; X64-NEXT: movd %edi, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f32_nan_inf:
; X32: # %bb.0:
; X32-NEXT: pushl %eax
-; X32-NEXT: .cfi_def_cfa_offset 8
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NEXT: andb $1, %al
; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
; X32-NEXT: andl $4194304, %eax # imm = 0x400000
-; X32-NEXT: orl $2139095040, %eax # imm = 0x7F800000
+; X32-NEXT: xorl $2139095040, %eax # imm = 0x7F800000
; X32-NEXT: movl %eax, (%esp)
; X32-NEXT: flds (%esp)
; X32-NEXT: popl %eax
-; X32-NEXT: .cfi_def_cfa_offset 4
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f32_nan_inf:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: pushl %eax
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 8
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
; X32-NOCMOV-NEXT: andl $4194304, %eax # imm = 0x400000
-; X32-NOCMOV-NEXT: orl $2139095040, %eax # imm = 0x7F800000
+; X32-NOCMOV-NEXT: xorl $2139095040, %eax # imm = 0x7F800000
; X32-NOCMOV-NEXT: movl %eax, (%esp)
; X32-NOCMOV-NEXT: flds (%esp)
; X32-NOCMOV-NEXT: popl %eax
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
; X32-NOCMOV-NEXT: retl
%result = call float @llvm.ct.select.f32(i1 %cond, float 0x7FF8000000000000, float 0x7FF0000000000000)
ret float %result
}
-define double @test_ctselect_f64_nan_inf(i1 %cond) {
+define double @test_ctselect_f64_nan_inf(i1 %cond) #0 {
; X64-LABEL: test_ctselect_f64_nan_inf:
; X64: # %bb.0:
; X64-NEXT: # kill: def $edi killed $edi def $rdi
@@ -1478,40 +2408,66 @@ define double @test_ctselect_f64_nan_inf(i1 %cond) {
; X64-NEXT: movabsq $2251799813685248, %rax # imm = 0x8000000000000
; X64-NEXT: andq %rdi, %rax
; X64-NEXT: movabsq $9218868437227405312, %rcx # imm = 0x7FF0000000000000
-; X64-NEXT: orq %rax, %rcx
+; X64-NEXT: xorq %rax, %rcx
; X64-NEXT: movq %rcx, %xmm0
; X64-NEXT: retq
;
; X32-LABEL: test_ctselect_f64_nan_inf:
; X32: # %bb.0:
-; X32-NEXT: subl $12, %esp
-; X32-NEXT: .cfi_def_cfa_offset 16
+; X32-NEXT: pushl %esi
+; X32-NEXT: subl $24, %esp
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NEXT: andl $1, %eax
+; X32-NEXT: andb $1, %al
+; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X32-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X32-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
-; X32-NEXT: andl $524288, %eax # imm = 0x80000
-; X32-NEXT: orl $2146435072, %eax # imm = 0x7FF00000
-; X32-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NEXT: movl $0, (%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: xorl %edx, %esi
+; X32-NEXT: andl %eax, %esi
+; X32-NEXT: xorl %edx, %esi
+; X32-NEXT: movl %esi, (%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: andl %eax, %edx
+; X32-NEXT: xorl %ecx, %edx
+; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NEXT: fldl (%esp)
-; X32-NEXT: addl $12, %esp
-; X32-NEXT: .cfi_def_cfa_offset 4
+; X32-NEXT: addl $24, %esp
+; X32-NEXT: popl %esi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f64_nan_inf:
; X32-NOCMOV: # %bb.0:
-; X32-NOCMOV-NEXT: subl $12, %esp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 16
+; X32-NOCMOV-NEXT: pushl %esi
+; X32-NOCMOV-NEXT: subl $24, %esp
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
-; X32-NOCMOV-NEXT: andl $1, %eax
+; X32-NOCMOV-NEXT: andb $1, %al
+; X32-NOCMOV-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
+; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl $524288, %eax # imm = 0x80000
-; X32-NOCMOV-NEXT: orl $2146435072, %eax # imm = 0x7FF00000
-; X32-NOCMOV-NEXT: movl %eax, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl $0, (%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: xorl %edx, %esi
+; X32-NOCMOV-NEXT: andl %eax, %esi
+; X32-NOCMOV-NEXT: xorl %edx, %esi
+; X32-NOCMOV-NEXT: movl %esi, (%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: andl %eax, %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
+; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fldl (%esp)
-; X32-NOCMOV-NEXT: addl $12, %esp
-; X32-NOCMOV-NEXT: .cfi_def_cfa_offset 4
+; X32-NOCMOV-NEXT: addl $24, %esp
+; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: retl
%result = call double @llvm.ct.select.f64(i1 %cond, double 0x7FF8000000000000, double 0x7FF0000000000000)
ret double %result
@@ -1525,6 +2481,10 @@ declare i32 @llvm.ct.select.i32(i1, i32, i32)
declare i64 @llvm.ct.select.i64(i1, i64, i64)
declare float @llvm.ct.select.f32(i1, float, float)
declare double @llvm.ct.select.f64(i1, double, double)
+declare half @llvm.ct.select.f16(i1, half, half)
+declare bfloat @llvm.ct.select.bf16(i1, bfloat, bfloat)
+declare fp128 @llvm.ct.select.f128(i1, fp128, fp128)
+declare x86_fp80 @llvm.ct.select.f80(i1, x86_fp80, x86_fp80)
declare ptr @llvm.ct.select.p0(i1, ptr, ptr)
; Vector intrinsics
@@ -1536,3 +2496,7 @@ declare <4 x float> @llvm.ct.select.v4f32(i1, <4 x float>, <4 x float>)
declare <2 x double> @llvm.ct.select.v2f64(i1, <2 x double>, <2 x double>)
declare <8 x i32> @llvm.ct.select.v8i32(i1, <8 x i32>, <8 x i32>)
declare <8 x float> @llvm.ct.select.v8f32(i1, <8 x float>, <8 x float>)
+declare <8 x half> @llvm.ct.select.v8f16(i1, <8 x half>, <8 x half>)
+declare <8 x bfloat> @llvm.ct.select.v8bf16(i1, <8 x bfloat>, <8 x bfloat>)
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/Transforms/InstSimplify/ct-select.ll b/llvm/test/Transforms/InstSimplify/ct-select.ll
new file mode 100644
index 0000000000000..3a89009be8480
--- /dev/null
+++ b/llvm/test/Transforms/InstSimplify/ct-select.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
+; RUN: opt < %s -passes=instsimplify -S | FileCheck %s
+
+declare i32 @llvm.ct.select.i32(i1, i32, i32)
+declare i64 @llvm.ct.select.i64(i1, i64, i64)
+declare float @llvm.ct.select.f32(i1, float, float)
+declare ptr @llvm.ct.select.p0(i1, ptr, ptr)
+declare <4 x i32> @llvm.ct.select.v4i32(i1, <4 x i32>, <4 x i32>)
+
+define i32 @ct_select_true(i32 %x, i32 %y) {
+; CHECK-LABEL: @ct_select_true(
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.ct.select.i32(i1 true, i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT: ret i32 [[X]]
+;
+ %r = call i32 @llvm.ct.select.i32(i1 true, i32 %x, i32 %y)
+ ret i32 %r
+}
+
+define i32 @ct_select_false(i32 %x, i32 %y) {
+; CHECK-LABEL: @ct_select_false(
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.ct.select.i32(i1 false, i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT: ret i32 [[Y]]
+;
+ %r = call i32 @llvm.ct.select.i32(i1 false, i32 %x, i32 %y)
+ ret i32 %r
+}
+
+define i64 @ct_select_true_i64(i64 %x, i64 %y) {
+; CHECK-LABEL: @ct_select_true_i64(
+; CHECK-NEXT: [[R:%.*]] = call i64 @llvm.ct.select.i64(i1 true, i64 [[X:%.*]], i64 [[Y:%.*]])
+; CHECK-NEXT: ret i64 [[X]]
+;
+ %r = call i64 @llvm.ct.select.i64(i1 true, i64 %x, i64 %y)
+ ret i64 %r
+}
+
+define float @ct_select_false_f32(float %x, float %y) {
+; CHECK-LABEL: @ct_select_false_f32(
+; CHECK-NEXT: [[R:%.*]] = call float @llvm.ct.select.f32(i1 false, float [[X:%.*]], float [[Y:%.*]])
+; CHECK-NEXT: ret float [[Y]]
+;
+ %r = call float @llvm.ct.select.f32(i1 false, float %x, float %y)
+ ret float %r
+}
+
+define ptr @ct_select_true_ptr(ptr %x, ptr %y) {
+; CHECK-LABEL: @ct_select_true_ptr(
+; CHECK-NEXT: [[R:%.*]] = call ptr @llvm.ct.select.p0(i1 true, ptr [[X:%.*]], ptr [[Y:%.*]])
+; CHECK-NEXT: ret ptr [[X]]
+;
+ %r = call ptr @llvm.ct.select.p0(i1 true, ptr %x, ptr %y)
+ ret ptr %r
+}
+
+define <4 x i32> @ct_select_true_v4i32(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: @ct_select_true_v4i32(
+; CHECK-NEXT: [[R:%.*]] = call <4 x i32> @llvm.ct.select.v4i32(i1 true, <4 x i32> [[X:%.*]], <4 x i32> [[Y:%.*]])
+; CHECK-NEXT: ret <4 x i32> [[X]]
+;
+ %r = call <4 x i32> @llvm.ct.select.v4i32(i1 true, <4 x i32> %x, <4 x i32> %y)
+ ret <4 x i32> %r
+}
+
+define i32 @ct_select_same_arms(i1 %c, i32 %x) {
+; CHECK-LABEL: @ct_select_same_arms(
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.ct.select.i32(i1 [[C:%.*]], i32 [[X:%.*]], i32 [[X]])
+; CHECK-NEXT: ret i32 [[X]]
+;
+ %r = call i32 @llvm.ct.select.i32(i1 %c, i32 %x, i32 %x)
+ ret i32 %r
+}
+
+define <4 x i32> @ct_select_same_arms_vec(i1 %c, <4 x i32> %x) {
+; CHECK-LABEL: @ct_select_same_arms_vec(
+; CHECK-NEXT: [[R:%.*]] = call <4 x i32> @llvm.ct.select.v4i32(i1 [[C:%.*]], <4 x i32> [[X:%.*]], <4 x i32> [[X]])
+; CHECK-NEXT: ret <4 x i32> [[X]]
+;
+ %r = call <4 x i32> @llvm.ct.select.v4i32(i1 %c, <4 x i32> %x, <4 x i32> %x)
+ ret <4 x i32> %r
+}
+
+; Negative test: must NOT fold when condition is a non-literal value, even if
+; analyses could prove it constant. The whole point of ct.select is to keep
+; the lowering even when the cond looks statically derivable.
+define i32 @ct_select_runtime_cond_no_fold(i1 %c, i32 %x, i32 %y) {
+; CHECK-LABEL: @ct_select_runtime_cond_no_fold(
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.ct.select.i32(i1 [[C:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = call i32 @llvm.ct.select.i32(i1 %c, i32 %x, i32 %y)
+ ret i32 %r
+}
+
+; Negative test: distinct arms with a runtime condition must not fold.
+define i32 @ct_select_distinct_arms_no_fold(i1 %c, i32 %x, i32 %y) {
+; CHECK-LABEL: @ct_select_distinct_arms_no_fold(
+; CHECK-NEXT: [[R:%.*]] = call i32 @llvm.ct.select.i32(i1 [[C:%.*]], i32 [[X:%.*]], i32 [[Y:%.*]])
+; CHECK-NEXT: ret i32 [[R]]
+;
+ %r = call i32 @llvm.ct.select.i32(i1 %c, i32 %x, i32 %y)
+ ret i32 %r
+}
>From 61defe7c71f3fad0431de9f8b5736f618fbe7bf8 Mon Sep 17 00:00:00 2001
From: AkshayK <iit.akshay at gmail.com>
Date: Fri, 22 May 2026 15:12:51 -0400
Subject: [PATCH 08/10] [ConstantTime] Reword comments and LangRef doc
---
llvm/include/llvm/CodeGen/ISDOpcodes.h | 7 +++++--
llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp | 7 -------
2 files changed, 5 insertions(+), 9 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index 0c953dd176ed1..c8d964fdb5053 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -805,8 +805,11 @@ enum NodeType {
/// i1 then the high bits must conform to getBooleanContents.
SELECT,
- /// CT_SELECT(Cond, TrueVal, FalseVal). Cond is i1 and the value operands must
- /// have the same type. Used to lower the constant-time select intrinsic.
+ /// CT_SELECT(COND, TRUEVAL, FALSEVAL) - Constant-time select: returns TRUEVAL
+ /// if COND (an i1) is true, else FALSEVAL; the value operands and result
+ /// share one type. Unlike SELECT, it must lower to code whose timing is
+ /// independent of COND -- no data-dependent branches, both arms always
+ /// evaluated -- and combines must preserve that. Node for llvm.ct.select.*.
CT_SELECT,
/// Select with a vector condition (op #0) and two vector operands (ops #1
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 1ee878d93922a..377499198e4b3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6900,13 +6900,6 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
assert(A.getValueType() == B.getValueType() &&
"Operands are of different types");
assert(!Cond.getValueType().isVector() && "Vector condition not supported");
-
- // TODO: a follow-up target-specific bundling pass needs to know the
- // function uses ct.select. When that consumer lands, record it in the
- // target's MachineFunctionInfo instead of mutating the IR from codegen.
- // Kept commented for reference until then:
- // DAG.getMachineFunction().getFunction().addFnAttr("ct-select");
-
setValue(&I, DAG.getNode(ISD::CT_SELECT, getCurSDLoc(), A.getValueType(),
Cond, A, B));
return;
>From 64613ed31ef6ec1523ad78f1a0e08404998e44e7 Mon Sep 17 00:00:00 2001
From: AkshayK <iit.akshay at gmail.com>
Date: Tue, 14 Jul 2026 19:39:41 -0400
Subject: [PATCH 09/10] [ConstantTime] Address reviewer feedback for
llvm.ct.select core
Model llvm.ct.select as IntrInaccessibleMemOnly so the call stays pinned
without pessimizing alias analysis. Make CT_SELECT flagless (drop the
getCTSelect flags parameter, dead flag propagation, and
setFlags-after-getNode) and assert its condition is scalar. Blend the FP
memory fallback at the widest legal integer width with ext-load and
trunc-store tails instead of illegal i8 chunks. Restore the LangRef
section dropped in the rebase onto the Markdown docs migration and note
RISC-V Zkt/Zvkt. Apply review style fixes and regenerate affected tests.
---
llvm/docs/LangRef.md | 125 ++++++++++
llvm/include/llvm/CodeGen/ISDOpcodes.h | 4 +-
llvm/include/llvm/CodeGen/SelectionDAG.h | 9 +-
llvm/include/llvm/IR/Intrinsics.td | 10 +-
.../include/llvm/Target/TargetSelectionDAG.td | 2 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 35 ++-
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 66 ++---
.../SelectionDAG/LegalizeFloatTypes.cpp | 8 +-
.../SelectionDAG/LegalizeIntegerTypes.cpp | 6 +-
.../SelectionDAG/LegalizeVectorTypes.cpp | 2 +-
.../SelectionDAG/SelectionDAGBuilder.cpp | 6 +-
.../SelectionDAG/SelectionDAGDumper.cpp | 2 +-
llvm/test/CodeGen/RISCV/ctselect-fallback.ll | 226 +++++++++---------
llvm/test/CodeGen/X86/ctselect.ll | 91 ++-----
14 files changed, 347 insertions(+), 245 deletions(-)
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index 9526cd020d724..c2196cc74e0fb 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -15756,6 +15756,131 @@ catch:
ret void
```
+### Constant-Time Intrinsics
+
+These intrinsics are provided to support constant-time operations for
+security-sensitive code. Constant-time operations execute in time independent
+of secret data values, preventing timing side-channel leaks.
+
+(int_ct_select)=
+
+#### '`llvm.ct.select.*`' Intrinsic
+
+##### Syntax:
+
+This is an overloaded intrinsic. You can use `llvm.ct.select` on any
+integer or floating-point type, pointer types, or vectors types.
+
+```
+declare i32 @llvm.ct.select.i32(i1 <cond>, i32 <val1>, i32 <val2>)
+declare i64 @llvm.ct.select.i64(i1 <cond>, i64 <val1>, i64 <val2>)
+declare float @llvm.ct.select.f32(i1 <cond>, float <val1>, float <val2>)
+declare double @llvm.ct.select.f64(i1 <cond>, double <val1>, double <val2>)
+declare ptr @llvm.ct.select.p0(i1 <cond>, ptr <val1>, ptr <val2>)
+
+; 128-bit vectors
+declare <4 x i32> @llvm.ct.select.v4i32(i1 <cond>, <4 x i32> <val1>, <4 x i32> <val2>)
+declare <2 x i64> @llvm.ct.select.v2i64(i1 <cond>, <2 x i64> <val1>, <2 x i64> <val2>)
+declare <4 x float> @llvm.ct.select.v4f32(i1 <cond>, <4 x float> <val1>, <4 x float> <val2>)
+declare <2 x double> @llvm.ct.select.v2f64(i1 <cond>, <2 x double> <val1>, <2 x double> <val2>)
+
+; 256-bit vectors
+declare <8 x i32> @llvm.ct.select.v8i32(i1 <cond>, <8 x i32> <val1>, <8 x i32> <val2>)
+declare <8 x float> @llvm.ct.select.v8f32(i1 <cond>, <8 x float> <val1>, <8 x float> <val2>)
+declare <4 x double> @llvm.ct.select.v4f64(i1 <cond>, <4 x double> <val1>, <4 x double> <val2>)
+```
+
+##### Overview:
+
+The '`llvm.ct.select`' family of intrinsic functions selects one of two
+values based on a condition, with the guarantee that the operation executes
+in constant time. Unlike the standard {ref}`select <i_select>` instruction,
+`llvm.ct.select` ensures that the execution time and observable behavior
+do not depend on the condition value, preventing timing-based side-channel
+leaks.
+
+##### Arguments:
+
+The '`llvm.ct.select`' intrinsic requires three arguments:
+
+1. The condition, which must be a scalar value of type 'i1'. Unlike
+ {ref}`select <i_select>` which accepts both scalar 'i1' and vector
+ '`<N x i1>`' conditions, `llvm.ct.select` only accepts a scalar 'i1'
+ condition. Vector conditions are not supported.
+2. The first value argument of any {ref}`first class <t_firstclass>` type.
+ This can be a scalar or vector type.
+3. The second value argument, which must have the same type as the first
+ value argument.
+
+##### Semantics:
+
+If the condition evaluates to true, the intrinsic returns the first value
+argument; otherwise, it returns the second value argument.
+
+The key semantic difference from {ref}`select <i_select>` is the constant-time
+code generation guarantee: the intrinsic must be lowered to machine code that:
+
+- Does not introduce data-dependent control flow based on the condition value
+- Executes the same sequence of instructions regardless of the condition value
+- Computes both value arguments before performing the selection
+
+**Platform Requirements:** The constant-time guarantee is conditional on
+hardware support for data-independent execution timing. This may be a
+processor mode that the platform enables, such as Arm DIT (Data Independent
+Timing) or x86 DOIT (Data Operand Independent Timing), or a static
+architectural guarantee, such as the RISC-V `Zkt` and `Zvkt` extensions,
+which certify data-independent latency for the scalar and vector instructions
+emitted by the lowering. Without such hardware support, the generated code
+will still be free from data-dependent control flow, but microarchitectural
+timing variations may still occur.
+
+The typical implementation uses bitwise operations to blend the two values
+based on a mask derived from the condition:
+
+```
+mask = sext(cond) ; sign-extend condition to all 1s or all 0s
+result = val2 ^ ((val1 ^ val2) & mask)
+```
+
+Targets with native constant-time select support use target-specific
+instructions to generate optimized bitwise operations with stronger guarantees.
+Targets without native support lower the intrinsic to a sequence of generic
+bitwise operations as shown above, structured to resist pattern recognition
+and preserve the constant-time property through optimization passes.
+
+Optimizations must preserve the constant-time code generation semantics.
+Transforms that would introduce data-dependent control flow are not permitted.
+This includes converting to conditional branches, using predicated instructions
+with data-dependent timing, or optimizing away either value argument before the
+selection completes (both paths must be computed).
+
+##### Examples:
+
+```llvm
+; Constant-time integer selection
+%x = call i32 @llvm.ct.select.i32(i1 %cond, i32 42, i32 17)
+%key = call i64 @llvm.ct.select.i64(i1 %cond, i64 %k_a, i64 %k_b)
+
+; Constant-time 128-bit integer vector selection (scalar condition broadcast to all lanes)
+%v4 = call <4 x i32> @llvm.ct.select.v4i32(i1 %cond,
+ <4 x i32> <i32 1, i32 2, i32 3, i32 4>,
+ <4 x i32> <i32 5, i32 6, i32 7, i32 8>)
+
+; Constant-time 256-bit integer vector selection
+%v8 = call <8 x i32> @llvm.ct.select.v8i32(i1 %cond,
+ <8 x i32> %vec_a, <8 x i32> %vec_b)
+
+; Constant-time 256-bit float vector selection
+%v8f = call <8 x float> @llvm.ct.select.v8f32(i1 %cond,
+ <8 x float> %fvec_a, <8 x float> %fvec_b)
+
+; Constant-time float selection
+%f = call float @llvm.ct.select.f32(i1 %cond, float 1.0, float 0.0)
+
+; Constant-time pointer selection
+%ptr = call ptr @llvm.ct.select.p0(i1 %cond, ptr %ptr_a, ptr %ptr_b)
+```
+
### Standard C/C++ Library Intrinsics
LLVM provides intrinsics for a few important standard C/C++ library
diff --git a/llvm/include/llvm/CodeGen/ISDOpcodes.h b/llvm/include/llvm/CodeGen/ISDOpcodes.h
index c8d964fdb5053..ab8471ad009c0 100644
--- a/llvm/include/llvm/CodeGen/ISDOpcodes.h
+++ b/llvm/include/llvm/CodeGen/ISDOpcodes.h
@@ -809,7 +809,9 @@ enum NodeType {
/// if COND (an i1) is true, else FALSEVAL; the value operands and result
/// share one type. Unlike SELECT, it must lower to code whose timing is
/// independent of COND -- no data-dependent branches, both arms always
- /// evaluated -- and combines must preserve that. Node for llvm.ct.select.*.
+ /// evaluated -- and combines must preserve that. Carries no SDNodeFlags;
+ /// FMF-driven select combines (e.g. select -> fminnum) must not be applied.
+ /// Node for llvm.ct.select.*.
CT_SELECT,
/// Select with a vector condition (op #0) and two vector operands (ops #1
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index c5ea42fd4b81d..b3bd33d7edc38 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1415,11 +1415,16 @@ class SelectionDAG {
return getNode(Opcode, DL, VT, Cond, LHS, RHS, Flags);
}
+ /// Helper function to build CT_SELECT nodes. Unlike select, ct.select only
+ /// accepts a scalar condition, shared by all lanes of vector operands, and
+ /// carries no SDNodeFlags (see the ISD::CT_SELECT documentation).
SDValue getCTSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS,
- SDValue RHS, SDNodeFlags Flags = SDNodeFlags()) {
+ SDValue RHS) {
assert(LHS.getValueType() == VT && RHS.getValueType() == VT &&
"Cannot use select on differing types");
- return getNode(ISD::CT_SELECT, DL, VT, Cond, LHS, RHS, Flags);
+ assert(!Cond.getValueType().isVector() &&
+ "ct.select condition must be a scalar");
+ return getNode(ISD::CT_SELECT, DL, VT, Cond, LHS, RHS);
}
/// Helper function to make it easier to build SelectCC's if you just have an
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 96bca07cbb6dd..24761ec020605 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2045,11 +2045,17 @@ def int_coro_subfn_addr : DefaultAttrsIntrinsic<
///===---------------------- Constant Time Intrinsics ----------------------===//
//
-// Intrinsic to support constant time select
+// Intrinsic to support constant-time select. The selected value is pure, but
+// the intrinsic is intentionally modeled like llvm.sideeffect: it only touches
+// inaccessible memory so it remains an opaque barrier to generic transforms.
+// This prevents passes from deleting it when unused, speculating it, or
+// rewriting it into an ordinary select, while avoiding claims about writes to
+// accessible program memory. The constant-time lowering contract is described
+// in LangRef.
def int_ct_select
: DefaultAttrsIntrinsic<[llvm_any_ty],
[llvm_i1_ty, LLVMMatchType<0>, LLVMMatchType<0>],
- [IntrWriteMem, IntrWillReturn, NoUndef<RetIndex>]>;
+ [IntrInaccessibleMemOnly, NoUndef<RetIndex>]>;
////===-------------------------- Other Intrinsics --------------------------===//
diff --git a/llvm/include/llvm/Target/TargetSelectionDAG.td b/llvm/include/llvm/Target/TargetSelectionDAG.td
index 4a44fbe7c1480..d1111572ac8d7 100644
--- a/llvm/include/llvm/Target/TargetSelectionDAG.td
+++ b/llvm/include/llvm/Target/TargetSelectionDAG.td
@@ -767,7 +767,7 @@ def reset_fpmode : SDNode<"ISD::RESET_FPMODE", SDTNone, [SDNPHasChain]>;
def setcc : SDNode<"ISD::SETCC" , SDTSetCC>;
def select : SDNode<"ISD::SELECT" , SDTSelect>;
-def ct_select : SDNode<"ISD::CT_SELECT", SDTSelect>;
+def ct_select : SDNode<"ISD::CT_SELECT" , SDTSelect>;
def vselect : SDNode<"ISD::VSELECT" , SDTVSelect>;
def selectcc : SDNode<"ISD::SELECT_CC" , SDTSelectCC>;
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index aa1d0bd485cc4..1caa54458cdb3 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -484,8 +484,6 @@ namespace {
SDValue visitCTTZ_ZERO_POISON(SDNode *N);
SDValue visitCTPOP(SDNode *N);
SDValue visitSELECT(SDNode *N);
- // ISD::CT_SELECT - Constant-Time SELECT (not related to CT in
- // CTPOP/CTLZ/CTTZ where CT means "count").
SDValue visitCT_SELECT(SDNode *N);
SDValue visitVSELECT(SDNode *N);
SDValue visitVP_SELECT(SDNode *N);
@@ -2042,7 +2040,7 @@ SDValue DAGCombiner::visit(SDNode *N) {
case ISD::CTTZ_ZERO_POISON: return visitCTTZ_ZERO_POISON(N);
case ISD::CTPOP: return visitCTPOP(N);
case ISD::SELECT: return visitSELECT(N);
- case ISD::CT_SELECT: return visitCT_SELECT(N);
+ case ISD::CT_SELECT: return visitCT_SELECT(N);
case ISD::VSELECT: return visitVSELECT(N);
case ISD::SELECT_CC: return visitSELECT_CC(N);
case ISD::SETCC: return visitSETCC(N);
@@ -13448,14 +13446,13 @@ SDValue DAGCombiner::visitCT_SELECT(SDNode *N) {
EVT VT = N->getValueType(0);
EVT VT0 = N0.getValueType();
SDLoc DL(N);
- SDNodeFlags Flags = N->getFlags();
// ct_select (not Cond), N1, N2 -> ct_select Cond, N2, N1
// This is a CT-safe canonicalization: flip negated condition by swapping
// arms. extractBooleanFlip only matches boolean xor-with-1, so this preserves
// dataflow semantics and does not introduce data-dependent control flow.
if (SDValue F = extractBooleanFlip(N0, DAG, TLI, false))
- return DAG.getCTSelect(DL, VT, F, N2, N1, Flags);
+ return DAG.getCTSelect(DL, VT, F, N2, N1);
if (VT0 == MVT::i1) {
// Nested CT_SELECT merging optimizations for i1 conditions.
@@ -13469,26 +13466,26 @@ SDValue DAGCombiner::visitCT_SELECT(SDNode *N) {
// ct_select C0, (ct_select C1, X, Y), Y -> ct_select (C0 & C1), X, Y
// Semantic equivalence: If C0 is true, evaluate inner select (C1 ? X :
// Y). If C0 is false, choose Y. This is equivalent to (C0 && C1) ? X : Y.
- if (N1->getOpcode() == ISD::CT_SELECT && N1->hasOneUse()) {
- SDValue N1_0 = N1->getOperand(0);
- SDValue N1_1 = N1->getOperand(1);
- SDValue N1_2 = N1->getOperand(2);
- if (N1_2 == N2 && N0.getValueType() == N1_0.getValueType()) {
- SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N1_0);
- return DAG.getCTSelect(DL, N1.getValueType(), And, N1_1, N2, Flags);
+ if (N1.getOpcode() == ISD::CT_SELECT && N1.hasOneUse()) {
+ SDValue N10 = N1.getOperand(0);
+ SDValue N11 = N1.getOperand(1);
+ SDValue N12 = N1.getOperand(2);
+ if (N12 == N2 && N0.getValueType() == N10.getValueType()) {
+ SDValue And = DAG.getNode(ISD::AND, DL, N0.getValueType(), N0, N10);
+ return DAG.getCTSelect(DL, N1.getValueType(), And, N11, N2);
}
}
// ct_select C0, X, (ct_select C1, X, Y) -> ct_select (C0 | C1), X, Y
// Semantic equivalence: If C0 is true, choose X. If C0 is false, evaluate
// inner select (C1 ? X : Y). This is equivalent to (C0 || C1) ? X : Y.
- if (N2->getOpcode() == ISD::CT_SELECT && N2->hasOneUse()) {
- SDValue N2_0 = N2->getOperand(0);
- SDValue N2_1 = N2->getOperand(1);
- SDValue N2_2 = N2->getOperand(2);
- if (N2_1 == N1 && N0.getValueType() == N2_0.getValueType()) {
- SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N2_0);
- return DAG.getCTSelect(DL, N1.getValueType(), Or, N1, N2_2, Flags);
+ if (N2.getOpcode() == ISD::CT_SELECT && N2.hasOneUse()) {
+ SDValue N20 = N2.getOperand(0);
+ SDValue N21 = N2.getOperand(1);
+ SDValue N22 = N2.getOperand(2);
+ if (N21 == N1 && N0.getValueType() == N20.getValueType()) {
+ SDValue Or = DAG.getNode(ISD::OR, DL, N0.getValueType(), N0, N20);
+ return DAG.getCTSelect(DL, N1.getValueType(), Or, N1, N22);
}
}
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index de06ed1015e96..073288a879daa 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4384,19 +4384,17 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
unsigned StorageBytes = DL.getTypeStoreSize(VTTy);
assert(StorageBytes > 0 && "FP type with zero storage size");
- // Pick the largest legal scalar integer chunk that divides StorageBytes
- // evenly. i8 is the universal fallback (legal on every target).
- MVT ChunkVT = MVT::i8;
- for (MVT MV : {MVT::i64, MVT::i32, MVT::i16}) {
- unsigned MVBytes = MV.getSizeInBits() / 8;
- if (TLI.isTypeLegal(MV) && MVBytes <= StorageBytes &&
- StorageBytes % MVBytes == 0) {
- ChunkVT = MV;
+ // Blend at the widest legal scalar integer width. Chunks narrower than
+ // that (e.g. the 2-byte tail of x86_fp80) are zero-extended on load and
+ // truncated on store, so every value in the DAG has a legal type.
+ MVT BlendVT;
+ for (MVT MV : {MVT::i64, MVT::i32, MVT::i16, MVT::i8})
+ if (TLI.isTypeLegal(MV)) {
+ BlendVT = MV;
break;
}
- }
- unsigned ChunkBytes = ChunkVT.getSizeInBits() / 8;
- unsigned NumChunks = StorageBytes / ChunkBytes;
+ assert(BlendVT.isValid() && "no legal scalar integer type");
+ unsigned BlendBytes = BlendVT.getSizeInBits() / 8;
MachineFunction &MF = DAG.getMachineFunction();
SDValue StackT = DAG.CreateStackTemporary(VT);
@@ -4413,30 +4411,45 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Chain = DAG.getStore(Chain, dl, Tmp2, StackT, PIT);
Chain = DAG.getStore(Chain, dl, Tmp3, StackF, PIF);
- for (unsigned i = 0; i < NumChunks; ++i) {
- TypeSize Off = TypeSize::getFixed(i * ChunkBytes);
+ // Walk the storage in power-of-2 chunks, widest first, so every access
+ // stays naturally aligned within the max-aligned stack temporaries.
+ unsigned Offset = 0;
+ while (Offset < StorageBytes) {
+ unsigned ChunkBytes =
+ std::min(BlendBytes, llvm::bit_floor(StorageBytes - Offset));
+ MVT MemVT = MVT::getIntegerVT(ChunkBytes * 8);
+ TypeSize Off = TypeSize::getFixed(Offset);
SDValue TPtr = DAG.getMemBasePlusOffset(StackT, Off, dl);
SDValue FPtr = DAG.getMemBasePlusOffset(StackF, Off, dl);
SDValue RPtr = DAG.getMemBasePlusOffset(StackR, Off, dl);
- SDValue Ti = DAG.getLoad(ChunkVT, dl, Chain, TPtr,
- PIT.getWithOffset(i * ChunkBytes));
- Chain = Ti.getValue(1);
- SDValue Fi = DAG.getLoad(ChunkVT, dl, Chain, FPtr,
- PIF.getWithOffset(i * ChunkBytes));
+ SDValue Ti, Fi;
+ if (MemVT == BlendVT) {
+ Ti = DAG.getLoad(BlendVT, dl, Chain, TPtr, PIT.getWithOffset(Offset));
+ Chain = Ti.getValue(1);
+ Fi = DAG.getLoad(BlendVT, dl, Chain, FPtr, PIF.getWithOffset(Offset));
+ } else {
+ Ti = DAG.getExtLoad(ISD::ZEXTLOAD, dl, BlendVT, Chain, TPtr,
+ PIT.getWithOffset(Offset), MemVT);
+ Chain = Ti.getValue(1);
+ Fi = DAG.getExtLoad(ISD::ZEXTLOAD, dl, BlendVT, Chain, FPtr,
+ PIF.getWithOffset(Offset), MemVT);
+ }
Chain = Fi.getValue(1);
- // Blend this chunk via CT_SELECT on a legal integer type. The
+ // Blend this chunk via CT_SELECT on the legal integer type. The
// recursive node will be Expand'd by the scalar-int branch below.
- SDValue Ri =
- DAG.getCTSelect(dl, ChunkVT, Tmp1, Ti, Fi, Node->getFlags());
+ SDValue Ri = DAG.getCTSelect(dl, BlendVT, Tmp1, Ti, Fi);
- Chain = DAG.getStore(Chain, dl, Ri, RPtr,
- PIR.getWithOffset(i * ChunkBytes));
+ if (MemVT == BlendVT)
+ Chain = DAG.getStore(Chain, dl, Ri, RPtr, PIR.getWithOffset(Offset));
+ else
+ Chain = DAG.getTruncStore(Chain, dl, Ri, RPtr,
+ PIR.getWithOffset(Offset), MemVT);
+ Offset += ChunkBytes;
}
Tmp1 = DAG.getLoad(VT, dl, Chain, StackR, PIR);
- Tmp1->setFlags(Node->getFlags());
Results.push_back(Tmp1);
break;
}
@@ -4500,7 +4513,6 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
if (WorkingVT != VT)
Tmp1 = DAG.getBitcast(VT, Tmp1);
- Tmp1->setFlags(Node->getFlags());
Results.push_back(Tmp1);
break;
}
@@ -5899,8 +5911,8 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
// Perform the larger operation, then round down.
- Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3);
- Tmp1->setFlags(Node->getFlags());
+ Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3,
+ Node->getFlags());
if (TruncOp != ISD::FP_ROUND)
Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
else
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 8278973fe4b87..63d92907b6582 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -161,7 +161,7 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
- case ISD::CT_SELECT: R = SoftenFloatRes_CT_SELECT(N); break;
+ case ISD::CT_SELECT: R = SoftenFloatRes_CT_SELECT(N); break;
case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
case ISD::STRICT_SINT_TO_FP:
@@ -1070,7 +1070,7 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_CT_SELECT(SDNode *N) {
SDValue LHS = GetSoftenedFloat(N->getOperand(1));
SDValue RHS = GetSoftenedFloat(N->getOperand(2));
return DAG.getCTSelect(SDLoc(N), LHS.getValueType(), N->getOperand(0), LHS,
- RHS, N->getFlags());
+ RHS);
}
SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
@@ -1593,7 +1593,7 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
case ISD::POISON:
case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
- case ISD::CT_SELECT: SplitRes_CT_SELECT(N, Lo, Hi); break;
+ case ISD::CT_SELECT: SplitRes_CT_SELECT(N, Lo, Hi); break;
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
@@ -3046,7 +3046,7 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_CT_SELECT(SDNode *N) {
SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
return DAG.getCTSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
- Op2, N->getFlags());
+ Op2);
}
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index c90703fa54278..ede25ab017b29 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -2595,9 +2595,9 @@ SDValue DAGTypeLegalizer::PromoteIntOp_CT_SELECT(SDNode *N, unsigned OpNo) {
SDValue Cond = N->getOperand(0);
EVT OpTy = N->getOperand(1).getValueType();
- // Promote all the way up to the canonical SetCC type.
- EVT OpVT = N->getOpcode() == ISD::CT_SELECT ? OpTy.getScalarType() : OpTy;
- Cond = PromoteTargetBoolean(Cond, OpVT);
+ // Promote all the way up to the canonical SetCC type. The condition is
+ // always scalar, so derive the boolean type from the operands' scalar type.
+ Cond = PromoteTargetBoolean(Cond, OpTy.getScalarType());
return SDValue(
DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), N->getOperand(2)), 0);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 06f0e7269fbcc..49b38c1c7cf4a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -770,7 +770,7 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
SDValue DAGTypeLegalizer::ScalarizeVecRes_CT_SELECT(SDNode *N) {
SDValue LHS = GetScalarizedVector(N->getOperand(1));
return DAG.getCTSelect(SDLoc(N), LHS.getValueType(), N->getOperand(0), LHS,
- GetScalarizedVector(N->getOperand(2)), N->getFlags());
+ GetScalarizedVector(N->getOperand(2)));
}
SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index 377499198e4b3..a6f9652889a31 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6897,11 +6897,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
SDValue Cond = getValue(I.getArgOperand(0));
SDValue A = getValue(I.getArgOperand(1));
SDValue B = getValue(I.getArgOperand(2));
- assert(A.getValueType() == B.getValueType() &&
- "Operands are of different types");
- assert(!Cond.getValueType().isVector() && "Vector condition not supported");
- setValue(&I, DAG.getNode(ISD::CT_SELECT, getCurSDLoc(), A.getValueType(),
- Cond, A, B));
+ setValue(&I, DAG.getCTSelect(getCurSDLoc(), A.getValueType(), Cond, A, B));
return;
}
case Intrinsic::call_preallocated_setup: {
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 7dc94fa1455cc..feb00b86de91a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -346,7 +346,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
case ISD::FPOWI: return "fpowi";
case ISD::STRICT_FPOWI: return "strict_fpowi";
case ISD::SETCC: return "setcc";
- case ISD::CT_SELECT: return "ct_select";
+ case ISD::CT_SELECT: return "ct_select";
case ISD::SETCCCARRY: return "setcccarry";
case ISD::STRICT_FSETCC: return "strict_fsetcc";
case ISD::STRICT_FSETCCS: return "strict_fsetccs";
diff --git a/llvm/test/CodeGen/RISCV/ctselect-fallback.ll b/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
index 0df231b7a98d9..68d5489895a6a 100644
--- a/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
+++ b/llvm/test/CodeGen/RISCV/ctselect-fallback.ll
@@ -58,10 +58,10 @@ define i64 @test_ctselect_i64(i1 %cond, i64 %a, i64 %b) {
;
; RV32-LABEL: test_ctselect_i64:
; RV32: # %bb.0:
-; RV32-NEXT: xor a1, a1, a3
; RV32-NEXT: slli a0, a0, 31
-; RV32-NEXT: xor a2, a2, a4
+; RV32-NEXT: xor a1, a1, a3
; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: xor a2, a2, a4
; RV32-NEXT: and a1, a1, a0
; RV32-NEXT: and a2, a2, a0
; RV32-NEXT: xor a0, a3, a1
@@ -212,8 +212,8 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; RV64: # %bb.0:
; RV64-NEXT: and a0, a0, a1
; RV64-NEXT: andi a0, a0, 1
-; RV64-NEXT: xor a2, a2, a3
; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: xor a2, a2, a3
; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: and a0, a2, a0
; RV64-NEXT: xor a0, a3, a0
@@ -223,8 +223,8 @@ define i32 @test_ctselect_nested_and_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; RV32: # %bb.0:
; RV32-NEXT: and a0, a0, a1
; RV32-NEXT: andi a0, a0, 1
-; RV32-NEXT: xor a2, a2, a3
; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: xor a2, a2, a3
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a0, a2, a0
; RV32-NEXT: xor a0, a3, a0
@@ -242,8 +242,8 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; RV64: # %bb.0:
; RV64-NEXT: or a0, a0, a1
; RV64-NEXT: andi a0, a0, 1
-; RV64-NEXT: xor a2, a2, a3
; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: xor a2, a2, a3
; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: and a0, a2, a0
; RV64-NEXT: xor a0, a3, a0
@@ -253,8 +253,8 @@ define i32 @test_ctselect_nested_or_i1_to_i32(i1 %c0, i1 %c1, i32 %x, i32 %y) {
; RV32: # %bb.0:
; RV32-NEXT: or a0, a0, a1
; RV32-NEXT: andi a0, a0, 1
-; RV32-NEXT: xor a2, a2, a3
; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: xor a2, a2, a3
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a0, a2, a0
; RV32-NEXT: xor a0, a3, a0
@@ -274,8 +274,8 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
; RV64-NEXT: and a0, a0, a1
; RV64-NEXT: and a0, a0, a2
; RV64-NEXT: andi a0, a0, 1
-; RV64-NEXT: xor a3, a3, a4
; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: xor a3, a3, a4
; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: and a0, a3, a0
; RV64-NEXT: xor a0, a4, a0
@@ -286,8 +286,8 @@ define i32 @test_ctselect_double_nested_and_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x, i
; RV32-NEXT: and a0, a0, a1
; RV32-NEXT: and a0, a0, a2
; RV32-NEXT: andi a0, a0, 1
-; RV32-NEXT: xor a3, a3, a4
; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: xor a3, a3, a4
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a0, a3, a0
; RV32-NEXT: xor a0, a4, a0
@@ -304,11 +304,11 @@ define i32 @test_ctselect_double_nested_mixed_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x,
; RV64-LABEL: test_ctselect_double_nested_mixed_i1:
; RV64: # %bb.0:
; RV64-NEXT: and a0, a0, a1
-; RV64-NEXT: xor a3, a3, a4
; RV64-NEXT: andi a0, a0, 1
; RV64-NEXT: or a0, a0, a2
; RV64-NEXT: andi a0, a0, 1
; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: xor a3, a3, a4
; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: and a3, a3, a0
; RV64-NEXT: xor a4, a4, a5
@@ -320,11 +320,11 @@ define i32 @test_ctselect_double_nested_mixed_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x,
; RV32-LABEL: test_ctselect_double_nested_mixed_i1:
; RV32: # %bb.0:
; RV32-NEXT: and a0, a0, a1
-; RV32-NEXT: xor a3, a3, a4
; RV32-NEXT: andi a0, a0, 1
; RV32-NEXT: or a0, a0, a2
; RV32-NEXT: andi a0, a0, 1
; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: xor a3, a3, a4
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a3, a3, a0
; RV32-NEXT: xor a4, a4, a5
@@ -345,12 +345,12 @@ define i32 @test_ctselect_double_nested_mixed_i1(i1 %c0, i1 %c1, i1 %c2, i32 %x,
define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
; RV64-LABEL: test_ctselect_nested:
; RV64: # %bb.0:
-; RV64-NEXT: xor a2, a2, a3
; RV64-NEXT: slli a1, a1, 63
-; RV64-NEXT: xor a3, a3, a4
-; RV64-NEXT: slli a0, a0, 63
+; RV64-NEXT: xor a2, a2, a3
; RV64-NEXT: srai a1, a1, 63
; RV64-NEXT: and a1, a2, a1
+; RV64-NEXT: xor a3, a3, a4
+; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: xor a1, a3, a1
; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: and a0, a1, a0
@@ -359,12 +359,12 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
;
; RV32-LABEL: test_ctselect_nested:
; RV32: # %bb.0:
-; RV32-NEXT: xor a2, a2, a3
; RV32-NEXT: slli a1, a1, 31
-; RV32-NEXT: xor a3, a3, a4
-; RV32-NEXT: slli a0, a0, 31
+; RV32-NEXT: xor a2, a2, a3
; RV32-NEXT: srai a1, a1, 31
; RV32-NEXT: and a1, a2, a1
+; RV32-NEXT: xor a3, a3, a4
+; RV32-NEXT: slli a0, a0, 31
; RV32-NEXT: xor a1, a3, a1
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a0, a1, a0
@@ -379,8 +379,8 @@ define i32 @test_ctselect_nested(i1 %cond1, i1 %cond2, i32 %a, i32 %b, i32 %c) {
define float @test_ctselect_f32_nan_inf(i1 %cond) {
; RV64-LABEL: test_ctselect_f32_nan_inf:
; RV64: # %bb.0:
-; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: lui a1, 1024
+; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: and a0, a0, a1
; RV64-NEXT: lui a1, 522240
@@ -389,8 +389,8 @@ define float @test_ctselect_f32_nan_inf(i1 %cond) {
;
; RV32-LABEL: test_ctselect_f32_nan_inf:
; RV32: # %bb.0:
-; RV32-NEXT: slli a0, a0, 31
; RV32-NEXT: lui a1, 1024
+; RV32-NEXT: slli a0, a0, 31
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a0, a0, a1
; RV32-NEXT: lui a1, 522240
@@ -403,20 +403,20 @@ define float @test_ctselect_f32_nan_inf(i1 %cond) {
define double @test_ctselect_f64_nan_inf(i1 %cond) {
; RV64-LABEL: test_ctselect_f64_nan_inf:
; RV64: # %bb.0:
-; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: li a1, 1
+; RV64-NEXT: slli a0, a0, 63
; RV64-NEXT: srai a0, a0, 63
; RV64-NEXT: slli a1, a1, 51
+; RV64-NEXT: li a2, 2047
; RV64-NEXT: and a0, a0, a1
-; RV64-NEXT: li a1, 2047
-; RV64-NEXT: slli a1, a1, 52
-; RV64-NEXT: xor a0, a0, a1
+; RV64-NEXT: slli a2, a2, 52
+; RV64-NEXT: xor a0, a0, a2
; RV64-NEXT: ret
;
; RV32-LABEL: test_ctselect_f64_nan_inf:
; RV32: # %bb.0:
-; RV32-NEXT: slli a0, a0, 31
; RV32-NEXT: lui a1, 128
+; RV32-NEXT: slli a0, a0, 31
; RV32-NEXT: srai a0, a0, 31
; RV32-NEXT: and a0, a0, a1
; RV32-NEXT: lui a1, 524032
@@ -462,10 +462,10 @@ define double @test_ctselect_f64(i1 %cond, double %a, double %b) {
;
; RV32-LABEL: test_ctselect_f64:
; RV32: # %bb.0:
-; RV32-NEXT: xor a1, a1, a3
; RV32-NEXT: slli a0, a0, 31
-; RV32-NEXT: xor a2, a2, a4
+; RV32-NEXT: xor a1, a1, a3
; RV32-NEXT: srai a0, a0, 31
+; RV32-NEXT: xor a2, a2, a4
; RV32-NEXT: and a1, a1, a0
; RV32-NEXT: and a2, a2, a0
; RV32-NEXT: xor a0, a3, a1
@@ -480,29 +480,29 @@ define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) {
; RV64-LABEL: test_ctselect_v4i32:
; RV64: # %bb.0:
; RV64-NEXT: lw a4, 0(a3)
-; RV64-NEXT: lw a5, 8(a3)
-; RV64-NEXT: lw a6, 16(a3)
+; RV64-NEXT: lw a5, 0(a2)
+; RV64-NEXT: lw a6, 8(a2)
+; RV64-NEXT: lw a7, 8(a3)
+; RV64-NEXT: lw t0, 16(a3)
; RV64-NEXT: lw a3, 24(a3)
-; RV64-NEXT: lw a7, 0(a2)
-; RV64-NEXT: lw t0, 8(a2)
; RV64-NEXT: lw t1, 16(a2)
; RV64-NEXT: lw a2, 24(a2)
+; RV64-NEXT: xor a5, a5, a4
; RV64-NEXT: slli a1, a1, 63
; RV64-NEXT: srai a1, a1, 63
-; RV64-NEXT: xor a7, a7, a4
-; RV64-NEXT: xor t0, t0, a5
-; RV64-NEXT: xor t1, t1, a6
+; RV64-NEXT: xor a6, a6, a7
+; RV64-NEXT: and a5, a5, a1
+; RV64-NEXT: and a6, a6, a1
+; RV64-NEXT: xor t1, t1, t0
; RV64-NEXT: xor a2, a2, a3
-; RV64-NEXT: and a7, a7, a1
-; RV64-NEXT: and t0, t0, a1
; RV64-NEXT: and t1, t1, a1
; RV64-NEXT: and a1, a2, a1
-; RV64-NEXT: xor a2, a4, a7
-; RV64-NEXT: xor a4, a5, t0
-; RV64-NEXT: xor a5, a6, t1
+; RV64-NEXT: xor a4, a4, a5
+; RV64-NEXT: xor a2, a7, a6
+; RV64-NEXT: xor a5, t0, t1
; RV64-NEXT: xor a1, a3, a1
-; RV64-NEXT: sw a2, 0(a0)
-; RV64-NEXT: sw a4, 4(a0)
+; RV64-NEXT: sw a4, 0(a0)
+; RV64-NEXT: sw a2, 4(a0)
; RV64-NEXT: sw a5, 8(a0)
; RV64-NEXT: sw a1, 12(a0)
; RV64-NEXT: ret
@@ -510,29 +510,29 @@ define <4 x i32> @test_ctselect_v4i32(i1 %cond, <4 x i32> %a, <4 x i32> %b) {
; RV32-LABEL: test_ctselect_v4i32:
; RV32: # %bb.0:
; RV32-NEXT: lw a4, 0(a3)
-; RV32-NEXT: lw a5, 4(a3)
-; RV32-NEXT: lw a6, 8(a3)
+; RV32-NEXT: lw a5, 0(a2)
+; RV32-NEXT: lw a6, 4(a2)
+; RV32-NEXT: lw a7, 4(a3)
+; RV32-NEXT: lw t0, 8(a3)
; RV32-NEXT: lw a3, 12(a3)
-; RV32-NEXT: lw a7, 0(a2)
-; RV32-NEXT: lw t0, 4(a2)
; RV32-NEXT: lw t1, 8(a2)
; RV32-NEXT: lw a2, 12(a2)
+; RV32-NEXT: xor a5, a5, a4
; RV32-NEXT: slli a1, a1, 31
; RV32-NEXT: srai a1, a1, 31
-; RV32-NEXT: xor a7, a7, a4
-; RV32-NEXT: xor t0, t0, a5
-; RV32-NEXT: xor t1, t1, a6
+; RV32-NEXT: xor a6, a6, a7
+; RV32-NEXT: and a5, a5, a1
+; RV32-NEXT: and a6, a6, a1
+; RV32-NEXT: xor t1, t1, t0
; RV32-NEXT: xor a2, a2, a3
-; RV32-NEXT: and a7, a7, a1
-; RV32-NEXT: and t0, t0, a1
; RV32-NEXT: and t1, t1, a1
; RV32-NEXT: and a1, a2, a1
-; RV32-NEXT: xor a2, a4, a7
-; RV32-NEXT: xor a4, a5, t0
-; RV32-NEXT: xor a5, a6, t1
+; RV32-NEXT: xor a4, a4, a5
+; RV32-NEXT: xor a2, a7, a6
+; RV32-NEXT: xor a5, t0, t1
; RV32-NEXT: xor a1, a3, a1
-; RV32-NEXT: sw a2, 0(a0)
-; RV32-NEXT: sw a4, 4(a0)
+; RV32-NEXT: sw a4, 0(a0)
+; RV32-NEXT: sw a2, 4(a0)
; RV32-NEXT: sw a5, 8(a0)
; RV32-NEXT: sw a1, 12(a0)
; RV32-NEXT: ret
@@ -543,29 +543,29 @@ define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b
; RV64-LABEL: test_ctselect_v4f32:
; RV64: # %bb.0:
; RV64-NEXT: lw a4, 0(a3)
-; RV64-NEXT: lw a5, 8(a3)
-; RV64-NEXT: lw a6, 16(a3)
+; RV64-NEXT: lw a5, 0(a2)
+; RV64-NEXT: lw a6, 8(a2)
+; RV64-NEXT: lw a7, 8(a3)
+; RV64-NEXT: lw t0, 16(a3)
; RV64-NEXT: lw a3, 24(a3)
-; RV64-NEXT: lw a7, 0(a2)
-; RV64-NEXT: lw t0, 8(a2)
; RV64-NEXT: lw t1, 16(a2)
; RV64-NEXT: lw a2, 24(a2)
+; RV64-NEXT: xor a5, a5, a4
; RV64-NEXT: slli a1, a1, 63
; RV64-NEXT: srai a1, a1, 63
-; RV64-NEXT: xor a7, a7, a4
-; RV64-NEXT: xor t0, t0, a5
-; RV64-NEXT: xor t1, t1, a6
+; RV64-NEXT: xor a6, a6, a7
+; RV64-NEXT: and a5, a5, a1
+; RV64-NEXT: and a6, a6, a1
+; RV64-NEXT: xor t1, t1, t0
; RV64-NEXT: xor a2, a2, a3
-; RV64-NEXT: and a7, a7, a1
-; RV64-NEXT: and t0, t0, a1
; RV64-NEXT: and t1, t1, a1
; RV64-NEXT: and a1, a2, a1
-; RV64-NEXT: xor a2, a4, a7
-; RV64-NEXT: xor a4, a5, t0
-; RV64-NEXT: xor a5, a6, t1
+; RV64-NEXT: xor a4, a4, a5
+; RV64-NEXT: xor a2, a7, a6
+; RV64-NEXT: xor a5, t0, t1
; RV64-NEXT: xor a1, a3, a1
-; RV64-NEXT: sw a2, 0(a0)
-; RV64-NEXT: sw a4, 4(a0)
+; RV64-NEXT: sw a4, 0(a0)
+; RV64-NEXT: sw a2, 4(a0)
; RV64-NEXT: sw a5, 8(a0)
; RV64-NEXT: sw a1, 12(a0)
; RV64-NEXT: ret
@@ -573,29 +573,29 @@ define <4 x float> @test_ctselect_v4f32(i1 %cond, <4 x float> %a, <4 x float> %b
; RV32-LABEL: test_ctselect_v4f32:
; RV32: # %bb.0:
; RV32-NEXT: lw a4, 0(a3)
-; RV32-NEXT: lw a5, 4(a3)
-; RV32-NEXT: lw a6, 8(a3)
+; RV32-NEXT: lw a5, 0(a2)
+; RV32-NEXT: lw a6, 4(a2)
+; RV32-NEXT: lw a7, 4(a3)
+; RV32-NEXT: lw t0, 8(a3)
; RV32-NEXT: lw a3, 12(a3)
-; RV32-NEXT: lw a7, 0(a2)
-; RV32-NEXT: lw t0, 4(a2)
; RV32-NEXT: lw t1, 8(a2)
; RV32-NEXT: lw a2, 12(a2)
+; RV32-NEXT: xor a5, a5, a4
; RV32-NEXT: slli a1, a1, 31
; RV32-NEXT: srai a1, a1, 31
-; RV32-NEXT: xor a7, a7, a4
-; RV32-NEXT: xor t0, t0, a5
-; RV32-NEXT: xor t1, t1, a6
+; RV32-NEXT: xor a6, a6, a7
+; RV32-NEXT: and a5, a5, a1
+; RV32-NEXT: and a6, a6, a1
+; RV32-NEXT: xor t1, t1, t0
; RV32-NEXT: xor a2, a2, a3
-; RV32-NEXT: and a7, a7, a1
-; RV32-NEXT: and t0, t0, a1
; RV32-NEXT: and t1, t1, a1
; RV32-NEXT: and a1, a2, a1
-; RV32-NEXT: xor a2, a4, a7
-; RV32-NEXT: xor a4, a5, t0
-; RV32-NEXT: xor a5, a6, t1
+; RV32-NEXT: xor a4, a4, a5
+; RV32-NEXT: xor a2, a7, a6
+; RV32-NEXT: xor a5, t0, t1
; RV32-NEXT: xor a1, a3, a1
-; RV32-NEXT: sw a2, 0(a0)
-; RV32-NEXT: sw a4, 4(a0)
+; RV32-NEXT: sw a4, 0(a0)
+; RV32-NEXT: sw a2, 4(a0)
; RV32-NEXT: sw a5, 8(a0)
; RV32-NEXT: sw a1, 12(a0)
; RV32-NEXT: ret
@@ -622,34 +622,34 @@ define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
; RV64-NEXT: lw t2, 48(a2)
; RV64-NEXT: lw t3, 56(a2)
; RV64-NEXT: lw t4, 0(a3)
-; RV64-NEXT: lw t5, 8(a3)
-; RV64-NEXT: lw t6, 16(a3)
+; RV64-NEXT: lw t5, 0(a2)
+; RV64-NEXT: lw t6, 8(a2)
+; RV64-NEXT: lw s0, 8(a3)
+; RV64-NEXT: lw s1, 16(a3)
; RV64-NEXT: lw a3, 24(a3)
-; RV64-NEXT: lw s0, 0(a2)
-; RV64-NEXT: lw s1, 8(a2)
; RV64-NEXT: lw s2, 16(a2)
; RV64-NEXT: lw a2, 24(a2)
+; RV64-NEXT: xor t5, t5, t4
; RV64-NEXT: slli a1, a1, 63
; RV64-NEXT: srai a1, a1, 63
-; RV64-NEXT: xor s0, s0, t4
-; RV64-NEXT: xor s1, s1, t5
-; RV64-NEXT: xor s2, s2, t6
+; RV64-NEXT: xor t6, t6, s0
+; RV64-NEXT: and t5, t5, a1
+; RV64-NEXT: and t6, t6, a1
+; RV64-NEXT: xor s2, s2, s1
; RV64-NEXT: xor a2, a2, a3
-; RV64-NEXT: xor t0, t0, a4
-; RV64-NEXT: xor t1, t1, a5
-; RV64-NEXT: xor t2, t2, a6
-; RV64-NEXT: xor t3, t3, a7
-; RV64-NEXT: and s0, s0, a1
-; RV64-NEXT: and s1, s1, a1
; RV64-NEXT: and s2, s2, a1
; RV64-NEXT: and a2, a2, a1
+; RV64-NEXT: xor t0, t0, a4
+; RV64-NEXT: xor t1, t1, a5
; RV64-NEXT: and t0, t0, a1
; RV64-NEXT: and t1, t1, a1
+; RV64-NEXT: xor t2, t2, a6
+; RV64-NEXT: xor t3, t3, a7
; RV64-NEXT: and t2, t2, a1
; RV64-NEXT: and a1, t3, a1
-; RV64-NEXT: xor t3, t4, s0
-; RV64-NEXT: xor t4, t5, s1
-; RV64-NEXT: xor t5, t6, s2
+; RV64-NEXT: xor t3, t4, t5
+; RV64-NEXT: xor t4, s0, t6
+; RV64-NEXT: xor t5, s1, s2
; RV64-NEXT: xor a2, a3, a2
; RV64-NEXT: xor a3, a4, t0
; RV64-NEXT: xor a4, a5, t1
@@ -692,34 +692,34 @@ define <8 x i32> @test_ctselect_v8i32(i1 %cond, <8 x i32> %a, <8 x i32> %b) {
; RV32-NEXT: lw t2, 24(a2)
; RV32-NEXT: lw t3, 28(a2)
; RV32-NEXT: lw t4, 0(a3)
-; RV32-NEXT: lw t5, 4(a3)
-; RV32-NEXT: lw t6, 8(a3)
+; RV32-NEXT: lw t5, 0(a2)
+; RV32-NEXT: lw t6, 4(a2)
+; RV32-NEXT: lw s0, 4(a3)
+; RV32-NEXT: lw s1, 8(a3)
; RV32-NEXT: lw a3, 12(a3)
-; RV32-NEXT: lw s0, 0(a2)
-; RV32-NEXT: lw s1, 4(a2)
; RV32-NEXT: lw s2, 8(a2)
; RV32-NEXT: lw a2, 12(a2)
+; RV32-NEXT: xor t5, t5, t4
; RV32-NEXT: slli a1, a1, 31
; RV32-NEXT: srai a1, a1, 31
-; RV32-NEXT: xor s0, s0, t4
-; RV32-NEXT: xor s1, s1, t5
-; RV32-NEXT: xor s2, s2, t6
+; RV32-NEXT: xor t6, t6, s0
+; RV32-NEXT: and t5, t5, a1
+; RV32-NEXT: and t6, t6, a1
+; RV32-NEXT: xor s2, s2, s1
; RV32-NEXT: xor a2, a2, a3
-; RV32-NEXT: xor t0, t0, a4
-; RV32-NEXT: xor t1, t1, a5
-; RV32-NEXT: xor t2, t2, a6
-; RV32-NEXT: xor t3, t3, a7
-; RV32-NEXT: and s0, s0, a1
-; RV32-NEXT: and s1, s1, a1
; RV32-NEXT: and s2, s2, a1
; RV32-NEXT: and a2, a2, a1
+; RV32-NEXT: xor t0, t0, a4
+; RV32-NEXT: xor t1, t1, a5
; RV32-NEXT: and t0, t0, a1
; RV32-NEXT: and t1, t1, a1
+; RV32-NEXT: xor t2, t2, a6
+; RV32-NEXT: xor t3, t3, a7
; RV32-NEXT: and t2, t2, a1
; RV32-NEXT: and a1, t3, a1
-; RV32-NEXT: xor t3, t4, s0
-; RV32-NEXT: xor t4, t5, s1
-; RV32-NEXT: xor t5, t6, s2
+; RV32-NEXT: xor t3, t4, t5
+; RV32-NEXT: xor t4, s0, t6
+; RV32-NEXT: xor t5, s1, s2
; RV32-NEXT: xor a2, a3, a2
; RV32-NEXT: xor a3, a4, t0
; RV32-NEXT: xor a4, a5, t1
diff --git a/llvm/test/CodeGen/X86/ctselect.ll b/llvm/test/CodeGen/X86/ctselect.ll
index 9d54c29db9683..5b23642b1c5d0 100644
--- a/llvm/test/CodeGen/X86/ctselect.ll
+++ b/llvm/test/CodeGen/X86/ctselect.ll
@@ -500,42 +500,25 @@ define fp128 @test_ctselect_f128(i1 %cond, fp128 %a, fp128 %b) #0 {
define x86_fp80 @test_ctselect_f80(i1 %cond, x86_fp80 %a, x86_fp80 %b) #0 {
; X64-LABEL: test_ctselect_f80:
; X64: # %bb.0:
+; X64-NEXT: # kill: def $edi killed $edi def $rdi
; X64-NEXT: fldt {{[0-9]+}}(%rsp)
; X64-NEXT: fldt {{[0-9]+}}(%rsp)
; X64-NEXT: fstpt -{{[0-9]+}}(%rsp)
; X64-NEXT: fstpt -{{[0-9]+}}(%rsp)
-; X64-NEXT: movl -{{[0-9]+}}(%rsp), %ecx
-; X64-NEXT: movl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %edx
-; X64-NEXT: xorw %cx, %dx
; X64-NEXT: andl $1, %edi
-; X64-NEXT: negl %edi
-; X64-NEXT: andl %edi, %edx
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: movw %dx, -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %ecx
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %edx
-; X64-NEXT: xorw %cx, %dx
-; X64-NEXT: andl %edi, %edx
-; X64-NEXT: xorl %ecx, %edx
-; X64-NEXT: movw %dx, -{{[0-9]+}}(%rsp)
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %ecx
-; X64-NEXT: xorw %ax, %cx
-; X64-NEXT: andl %edi, %ecx
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: movw %cx, -{{[0-9]+}}(%rsp)
+; X64-NEXT: negq %rdi
+; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rax
+; X64-NEXT: movq -{{[0-9]+}}(%rsp), %rcx
+; X64-NEXT: xorq %rax, %rcx
+; X64-NEXT: andq %rdi, %rcx
+; X64-NEXT: xorq %rax, %rcx
+; X64-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %eax
; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %ecx
-; X64-NEXT: xorw %ax, %cx
-; X64-NEXT: andl %edi, %ecx
; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: movw %cx, -{{[0-9]+}}(%rsp)
-; X64-NEXT: movl -{{[0-9]+}}(%rsp), %eax
-; X64-NEXT: movzwl -{{[0-9]+}}(%rsp), %ecx
-; X64-NEXT: xorw %ax, %cx
-; X64-NEXT: andl %edi, %ecx
-; X64-NEXT: xorl %eax, %ecx
-; X64-NEXT: movw %cx, -{{[0-9]+}}(%rsp)
+; X64-NEXT: andl %ecx, %edi
+; X64-NEXT: xorl %eax, %edi
+; X64-NEXT: movw %di, -{{[0-9]+}}(%rsp)
; X64-NEXT: fldt -{{[0-9]+}}(%rsp)
; X64-NEXT: retq
;
@@ -549,35 +532,23 @@ define x86_fp80 @test_ctselect_f80(i1 %cond, x86_fp80 %a, x86_fp80 %b) #0 {
; X32-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NEXT: fstpt {{[0-9]+}}(%esp)
; X32-NEXT: fstpt {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorw %dx, %si
; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
-; X32-NEXT: andl %eax, %esi
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: movw %si, (%esp)
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %esi
-; X32-NEXT: xorw %dx, %si
; X32-NEXT: andl %eax, %esi
; X32-NEXT: xorl %edx, %esi
-; X32-NEXT: movw %si, {{[0-9]+}}(%esp)
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorw %cx, %dx
+; X32-NEXT: movl %esi, (%esp)
+; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NEXT: xorl %ecx, %edx
; X32-NEXT: andl %eax, %edx
; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movw %dx, {{[0-9]+}}(%esp)
+; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorw %cx, %dx
-; X32-NEXT: andl %eax, %edx
; X32-NEXT: xorl %ecx, %edx
-; X32-NEXT: movw %dx, {{[0-9]+}}(%esp)
-; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NEXT: xorw %cx, %dx
; X32-NEXT: andl %eax, %edx
; X32-NEXT: xorl %ecx, %edx
; X32-NEXT: movw %dx, {{[0-9]+}}(%esp)
@@ -596,35 +567,23 @@ define x86_fp80 @test_ctselect_f80(i1 %cond, x86_fp80 %a, x86_fp80 %b) #0 {
; X32-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorw %dx, %si
; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
-; X32-NOCMOV-NEXT: andl %eax, %esi
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: movw %si, (%esp)
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %esi
-; X32-NOCMOV-NEXT: xorw %dx, %si
; X32-NOCMOV-NEXT: andl %eax, %esi
; X32-NOCMOV-NEXT: xorl %edx, %esi
-; X32-NOCMOV-NEXT: movw %si, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorw %cx, %dx
+; X32-NOCMOV-NEXT: movl %esi, (%esp)
+; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
+; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: andl %eax, %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movw %dx, {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %ecx
; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorw %cx, %dx
-; X32-NOCMOV-NEXT: andl %eax, %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
-; X32-NOCMOV-NEXT: movw %dx, {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movzwl {{[0-9]+}}(%esp), %edx
-; X32-NOCMOV-NEXT: xorw %cx, %dx
; X32-NOCMOV-NEXT: andl %eax, %edx
; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: movw %dx, {{[0-9]+}}(%esp)
>From f4383a332170afc874a6fd6113a42d83a46889d9 Mon Sep 17 00:00:00 2001
From: AkshayK <iit.akshay at gmail.com>
Date: Thu, 16 Jul 2026 21:38:38 -0400
Subject: [PATCH 10/10] [ConstantTime] Apply review fixes for llvm.ct.select
core
- Verifier: reject aggregate types instead of crashing ISel; add test.
- Fix scalable-vector expansion crash when the element type is not a
legal scalar; add RVV test.
- LangRef: document poison/undef, noundef, permitted folds, and FMF
behavior; rewrite Syntax/Overview.
- Deduplicate CT_SELECT type-legalizer handlers into the SELECT paths.
- Misc: FP int-twin assert, dead null check, in-place memory blend,
flag-free PromoteNode, GlobalISel fallback test, comment cleanups.
---
llvm/docs/LangRef.md | 90 +++++++++-----
llvm/include/llvm/CodeGen/SelectionDAG.h | 2 +-
llvm/include/llvm/IR/Intrinsics.td | 7 +-
llvm/lib/Analysis/InstructionSimplify.cpp | 5 +-
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 15 +--
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 115 ++++++++++-------
.../SelectionDAG/LegalizeFloatTypes.cpp | 34 ++---
.../SelectionDAG/LegalizeIntegerTypes.cpp | 26 ++--
llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 5 -
.../SelectionDAG/LegalizeTypesGeneric.cpp | 6 -
.../SelectionDAG/LegalizeVectorTypes.cpp | 20 +--
.../SelectionDAG/SelectionDAGBuilder.cpp | 2 +
llvm/lib/IR/Verifier.cpp | 11 ++
.../CodeGen/RISCV/ctselect-fallback-gisel.ll | 35 ++++++
.../RISCV/ctselect-fallback-scalable.ll | 117 ++++++++++++++++++
llvm/test/CodeGen/X86/ctselect.ll | 48 +++----
llvm/test/Verifier/ct-select.ll | 20 +++
17 files changed, 369 insertions(+), 189 deletions(-)
create mode 100644 llvm/test/CodeGen/RISCV/ctselect-fallback-gisel.ll
create mode 100644 llvm/test/CodeGen/RISCV/ctselect-fallback-scalable.ll
create mode 100644 llvm/test/Verifier/ct-select.ll
diff --git a/llvm/docs/LangRef.md b/llvm/docs/LangRef.md
index c2196cc74e0fb..46ee2ea97371c 100644
--- a/llvm/docs/LangRef.md
+++ b/llvm/docs/LangRef.md
@@ -15759,8 +15759,8 @@ catch:
### Constant-Time Intrinsics
These intrinsics are provided to support constant-time operations for
-security-sensitive code. Constant-time operations execute in time independent
-of secret data values, preventing timing side-channel leaks.
+security-sensitive code. Constant-time operations are designed to execute in
+time independent of secret data values, preventing timing side-channel leaks.
(int_ct_select)=
@@ -15768,36 +15768,33 @@ of secret data values, preventing timing side-channel leaks.
##### Syntax:
-This is an overloaded intrinsic. You can use `llvm.ct.select` on any
-integer or floating-point type, pointer types, or vectors types.
+This is an overloaded intrinsic. You can use `llvm.ct.select` on any integer,
+floating-point, or pointer type, or on any vector of those types, including
+scalable vectors. The declarations below are a representative sample:
```
+declare i8 @llvm.ct.select.i8(i1 <cond>, i8 <val1>, i8 <val2>)
declare i32 @llvm.ct.select.i32(i1 <cond>, i32 <val1>, i32 <val2>)
declare i64 @llvm.ct.select.i64(i1 <cond>, i64 <val1>, i64 <val2>)
+declare half @llvm.ct.select.f16(i1 <cond>, half <val1>, half <val2>)
declare float @llvm.ct.select.f32(i1 <cond>, float <val1>, float <val2>)
declare double @llvm.ct.select.f64(i1 <cond>, double <val1>, double <val2>)
+declare fp128 @llvm.ct.select.f128(i1 <cond>, fp128 <val1>, fp128 <val2>)
declare ptr @llvm.ct.select.p0(i1 <cond>, ptr <val1>, ptr <val2>)
-
-; 128-bit vectors
declare <4 x i32> @llvm.ct.select.v4i32(i1 <cond>, <4 x i32> <val1>, <4 x i32> <val2>)
-declare <2 x i64> @llvm.ct.select.v2i64(i1 <cond>, <2 x i64> <val1>, <2 x i64> <val2>)
-declare <4 x float> @llvm.ct.select.v4f32(i1 <cond>, <4 x float> <val1>, <4 x float> <val2>)
declare <2 x double> @llvm.ct.select.v2f64(i1 <cond>, <2 x double> <val1>, <2 x double> <val2>)
-
-; 256-bit vectors
-declare <8 x i32> @llvm.ct.select.v8i32(i1 <cond>, <8 x i32> <val1>, <8 x i32> <val2>)
-declare <8 x float> @llvm.ct.select.v8f32(i1 <cond>, <8 x float> <val1>, <8 x float> <val2>)
-declare <4 x double> @llvm.ct.select.v4f64(i1 <cond>, <4 x double> <val1>, <4 x double> <val2>)
+declare <2 x ptr> @llvm.ct.select.v2p0(i1 <cond>, <2 x ptr> <val1>, <2 x ptr> <val2>)
+declare <vscale x 4 x i32> @llvm.ct.select.nxv4i32(i1 <cond>, <vscale x 4 x i32> <val1>, <vscale x 4 x i32> <val2>)
```
##### Overview:
The '`llvm.ct.select`' family of intrinsic functions selects one of two
-values based on a condition, with the guarantee that the operation executes
-in constant time. Unlike the standard {ref}`select <i_select>` instruction,
-`llvm.ct.select` ensures that the execution time and observable behavior
-do not depend on the condition value, preventing timing-based side-channel
-leaks.
+values based on a condition, like the standard {ref}`select <i_select>`
+instruction, but is lowered to branchless code whose execution time does not
+depend on the condition value. This keeps the condition from leaking through
+timing side channels; see the Semantics section for the exact guarantee and
+its platform requirements.
##### Arguments:
@@ -15807,8 +15804,10 @@ The '`llvm.ct.select`' intrinsic requires three arguments:
{ref}`select <i_select>` which accepts both scalar 'i1' and vector
'`<N x i1>`' conditions, `llvm.ct.select` only accepts a scalar 'i1'
condition. Vector conditions are not supported.
-2. The first value argument of any {ref}`first class <t_firstclass>` type.
- This can be a scalar or vector type.
+2. The first value argument, which must be an integer, floating-point, or
+ pointer type, or a vector of those types. Other
+ {ref}`first class <t_firstclass>` types, such as aggregates, are not
+ supported.
3. The second value argument, which must have the same type as the first
value argument.
@@ -15817,12 +15816,25 @@ The '`llvm.ct.select`' intrinsic requires three arguments:
If the condition evaluates to true, the intrinsic returns the first value
argument; otherwise, it returns the second value argument.
+If any argument is poison, the result is poison. Unlike `select`, this
+includes the unselected value argument, since the lowering blends both
+arguments with bitwise operations. If any argument is undef, the result is
+undef; an undef condition can blend bits from both arguments, so the result
+is not necessarily one of them. `llvm.ct.select` therefore cannot be used as
+a poison barrier.
+
+The return value carries the `noundef` attribute. A call where the condition
+or either value argument is undef or poison would return undef or poison, and
+is therefore undefined behavior. Frontends that cannot rule this out must
+{ref}`freeze <i_freeze>` the operands first.
+
The key semantic difference from {ref}`select <i_select>` is the constant-time
code generation guarantee: the intrinsic must be lowered to machine code that:
- Does not introduce data-dependent control flow based on the condition value
- Executes the same sequence of instructions regardless of the condition value
-- Computes both value arguments before performing the selection
+- Uses both value arguments unconditionally; the not-selected argument is
+ never skipped or branched around
**Platform Requirements:** The constant-time guarantee is conditional on
hardware support for data-independent execution timing. This may be a
@@ -15842,17 +15854,26 @@ mask = sext(cond) ; sign-extend condition to all 1s or all 0s
result = val2 ^ ((val1 ^ val2) & mask)
```
-Targets with native constant-time select support use target-specific
-instructions to generate optimized bitwise operations with stronger guarantees.
-Targets without native support lower the intrinsic to a sequence of generic
-bitwise operations as shown above, structured to resist pattern recognition
-and preserve the constant-time property through optimization passes.
+Targets with native constant-time select support lower the intrinsic to
+suitable target instructions instead. Targets without native support use the
+generic bitwise expansion shown above. In either case the selection is kept
+as a single opaque operation until late in code generation, so optimization
+passes never see a pattern they could rewrite into a select or a conditional
+branch.
Optimizations must preserve the constant-time code generation semantics.
Transforms that would introduce data-dependent control flow are not permitted.
This includes converting to conditional branches, using predicated instructions
-with data-dependent timing, or optimizing away either value argument before the
-selection completes (both paths must be computed).
+with data-dependent timing, or, except as described below, optimizing away
+either value argument before the selection completes.
+
+The call may be folded to one of its value arguments only when the condition
+is a literal `i1` constant or both value arguments are the same SSA value;
+the unused argument then becomes ordinary dead code. Proving the condition
+constant by other means (known-bits, range, or dominating conditions) does
+not permit the fold. Rewrites that keep the `llvm.ct.select`, such as
+swapping the arguments to remove a negated condition, are allowed. Fast-math
+flags on a call to `llvm.ct.select` are ignored.
##### Examples:
@@ -15861,19 +15882,20 @@ selection completes (both paths must be computed).
%x = call i32 @llvm.ct.select.i32(i1 %cond, i32 42, i32 17)
%key = call i64 @llvm.ct.select.i64(i1 %cond, i64 %k_a, i64 %k_b)
-; Constant-time 128-bit integer vector selection (scalar condition broadcast to all lanes)
+; Constant-time integer vector selection (scalar condition broadcast to all lanes)
%v4 = call <4 x i32> @llvm.ct.select.v4i32(i1 %cond,
<4 x i32> <i32 1, i32 2, i32 3, i32 4>,
<4 x i32> <i32 5, i32 6, i32 7, i32 8>)
-; Constant-time 256-bit integer vector selection
-%v8 = call <8 x i32> @llvm.ct.select.v8i32(i1 %cond,
- <8 x i32> %vec_a, <8 x i32> %vec_b)
-
-; Constant-time 256-bit float vector selection
+; Constant-time float vector selection
%v8f = call <8 x float> @llvm.ct.select.v8f32(i1 %cond,
<8 x float> %fvec_a, <8 x float> %fvec_b)
+; Constant-time scalable vector selection
+%sv = call <vscale x 4 x i32> @llvm.ct.select.nxv4i32(i1 %cond,
+ <vscale x 4 x i32> %sv_a,
+ <vscale x 4 x i32> %sv_b)
+
; Constant-time float selection
%f = call float @llvm.ct.select.f32(i1 %cond, float 1.0, float 0.0)
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h
index b3bd33d7edc38..d22e769987e51 100644
--- a/llvm/include/llvm/CodeGen/SelectionDAG.h
+++ b/llvm/include/llvm/CodeGen/SelectionDAG.h
@@ -1421,7 +1421,7 @@ class SelectionDAG {
SDValue getCTSelect(const SDLoc &DL, EVT VT, SDValue Cond, SDValue LHS,
SDValue RHS) {
assert(LHS.getValueType() == VT && RHS.getValueType() == VT &&
- "Cannot use select on differing types");
+ "Cannot use ct.select on differing types");
assert(!Cond.getValueType().isVector() &&
"ct.select condition must be a scalar");
return getNode(ISD::CT_SELECT, DL, VT, Cond, LHS, RHS);
diff --git a/llvm/include/llvm/IR/Intrinsics.td b/llvm/include/llvm/IR/Intrinsics.td
index 24761ec020605..d8096ecfeba85 100644
--- a/llvm/include/llvm/IR/Intrinsics.td
+++ b/llvm/include/llvm/IR/Intrinsics.td
@@ -2051,14 +2051,15 @@ def int_coro_subfn_addr : DefaultAttrsIntrinsic<
// This prevents passes from deleting it when unused, speculating it, or
// rewriting it into an ordinary select, while avoiding claims about writes to
// accessible program memory. The constant-time lowering contract is described
-// in LangRef.
+// in LangRef. llvm_any_ty alone would also admit aggregates, which codegen
+// does not support; the Verifier restricts the value type to integer,
+// floating-point, or pointer types, or vectors of them.
def int_ct_select
: DefaultAttrsIntrinsic<[llvm_any_ty],
[llvm_i1_ty, LLVMMatchType<0>, LLVMMatchType<0>],
[IntrInaccessibleMemOnly, NoUndef<RetIndex>]>;
-
-////===-------------------------- Other Intrinsics --------------------------===//
+///===------------------------- Other Intrinsics --------------------------===//
// TODO: We should introduce a new memory kind fo traps (and other side effects
// we only model to keep things alive).
def int_trap : Intrinsic<[], [],
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 6275592155b99..bd1d31ee24f32 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -7553,8 +7553,9 @@ static Value *simplifyIntrinsic(CallBase *Call, ArrayRef<Value *> Args,
return nullptr;
}
case Intrinsic::ct_select: {
- // Only fold on a literal IR-constant condition or identical arms. Folding
- // through ValueTracking-derived known bits would defeat the constant-time
+ // Only fold on a literal IR-constant condition or identical arms; these
+ // are the only two folds LangRef permits for ct.select. Folding through
+ // ValueTracking-derived known bits would defeat the constant-time
// contract on conditions the user wants kept opaque.
Value *Cond = Args[0], *TrueVal = Args[1], *FalseVal = Args[2];
if (auto *CI = dyn_cast<ConstantInt>(Cond)) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index 1caa54458cdb3..db18c0adba997 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -13455,17 +13455,10 @@ SDValue DAGCombiner::visitCT_SELECT(SDNode *N) {
return DAG.getCTSelect(DL, VT, F, N2, N1);
if (VT0 == MVT::i1) {
- // Nested CT_SELECT merging optimizations for i1 conditions.
- // These are CT-safe because:
- // 1. AND/OR are bitwise operations that execute in constant time
- // 2. The optimization combines two sequential CT_SELECTs into one,
- // reducing the total number of constant-time operations without
- // changing semantics
- // 3. No data-dependent branches or memory accesses are introduced
- //
+ // Merge nested i1 ct_selects. The AND/OR of the two conditions is itself
+ // constant-time and the result remains a CT_SELECT.
+
// ct_select C0, (ct_select C1, X, Y), Y -> ct_select (C0 & C1), X, Y
- // Semantic equivalence: If C0 is true, evaluate inner select (C1 ? X :
- // Y). If C0 is false, choose Y. This is equivalent to (C0 && C1) ? X : Y.
if (N1.getOpcode() == ISD::CT_SELECT && N1.hasOneUse()) {
SDValue N10 = N1.getOperand(0);
SDValue N11 = N1.getOperand(1);
@@ -13477,8 +13470,6 @@ SDValue DAGCombiner::visitCT_SELECT(SDNode *N) {
}
// ct_select C0, X, (ct_select C1, X, Y) -> ct_select (C0 | C1), X, Y
- // Semantic equivalence: If C0 is true, choose X. If C0 is false, evaluate
- // inner select (C1 ? X : Y). This is equivalent to (C0 || C1) ? X : Y.
if (N2.getOpcode() == ISD::CT_SELECT && N2.hasOneUse()) {
SDValue N20 = N2.getOperand(0);
SDValue N21 = N2.getOperand(1);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 073288a879daa..3838b09f77f9e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -4365,8 +4365,9 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
// and reconstructs a SELECT. The chain edge is defense-in-depth against
// a hypothetical future fold of that form: the dependency partitions the
// bitwise sequence into a region the combiner can't see through. Cost is
- // at most a coalesce-able MOV per call. Swap for ARITH_FENCE when its
- // int/vector extension lands.
+ // at most a coalesce-able MOV per call. ISD::ARITH_FENCE would serve the
+ // same purpose (ISel already selects it for any legal type); switching to
+ // it is left to a follow-up.
Tmp1 = Node->getOperand(0); // cond
Tmp2 = Node->getOperand(1); // T
Tmp3 = Node->getOperand(2); // F
@@ -4375,8 +4376,9 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
// Memory-blend for FP scalars whose same-size integer isn't legal (f64 on
// i386 no-SSE, x86_fp80, fp128). The bitcast-to-int expansion below can't
// run since LegalizeDAG is the last legalization stage. Spill T/F, blend
- // chunk-by-chunk at a legal int width, reload as the FP type. Fixed and
- // scalable FP vectors fall through to the unified path below.
+ // chunk-by-chunk at a legal int width in place over the T slot, reload it
+ // as the FP type. Fixed and scalable FP vectors fall through to the
+ // unified path below.
if (VT.isFloatingPoint() && !VT.isVector() &&
!TLI.isTypeLegal(VT.changeTypeToInteger())) {
const DataLayout &DL = DAG.getDataLayout();
@@ -4399,13 +4401,10 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
MachineFunction &MF = DAG.getMachineFunction();
SDValue StackT = DAG.CreateStackTemporary(VT);
SDValue StackF = DAG.CreateStackTemporary(VT);
- SDValue StackR = DAG.CreateStackTemporary(VT);
int FIT = cast<FrameIndexSDNode>(StackT.getNode())->getIndex();
int FIF = cast<FrameIndexSDNode>(StackF.getNode())->getIndex();
- int FIR = cast<FrameIndexSDNode>(StackR.getNode())->getIndex();
MachinePointerInfo PIT = MachinePointerInfo::getFixedStack(MF, FIT);
MachinePointerInfo PIF = MachinePointerInfo::getFixedStack(MF, FIF);
- MachinePointerInfo PIR = MachinePointerInfo::getFixedStack(MF, FIR);
SDValue Chain = DAG.getEntryNode();
Chain = DAG.getStore(Chain, dl, Tmp2, StackT, PIT);
@@ -4421,7 +4420,6 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
TypeSize Off = TypeSize::getFixed(Offset);
SDValue TPtr = DAG.getMemBasePlusOffset(StackT, Off, dl);
SDValue FPtr = DAG.getMemBasePlusOffset(StackF, Off, dl);
- SDValue RPtr = DAG.getMemBasePlusOffset(StackR, Off, dl);
SDValue Ti, Fi;
if (MemVT == BlendVT) {
@@ -4437,19 +4435,21 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
}
Chain = Fi.getValue(1);
- // Blend this chunk via CT_SELECT on the legal integer type. The
- // recursive node will be Expand'd by the scalar-int branch below.
+ // Blend this chunk via CT_SELECT on the legal integer type (the
+ // recursive node is Expand'd by the scalar-int branch below), then
+ // store it back over the now-dead chunk of the T slot. The serial
+ // chain keeps each chunk's loads ahead of the store.
SDValue Ri = DAG.getCTSelect(dl, BlendVT, Tmp1, Ti, Fi);
if (MemVT == BlendVT)
- Chain = DAG.getStore(Chain, dl, Ri, RPtr, PIR.getWithOffset(Offset));
+ Chain = DAG.getStore(Chain, dl, Ri, TPtr, PIT.getWithOffset(Offset));
else
- Chain = DAG.getTruncStore(Chain, dl, Ri, RPtr,
- PIR.getWithOffset(Offset), MemVT);
+ Chain = DAG.getTruncStore(Chain, dl, Ri, TPtr,
+ PIT.getWithOffset(Offset), MemVT);
Offset += ChunkBytes;
}
- Tmp1 = DAG.getLoad(VT, dl, Chain, StackR, PIR);
+ Tmp1 = DAG.getLoad(VT, dl, Chain, StackT, PIT);
Results.push_back(Tmp1);
break;
}
@@ -4461,30 +4461,53 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
bool IsFP = VT.isVector() ? VT.getVectorElementType().isFloatingPoint()
: VT.isFloatingPoint();
if (IsFP) {
- if (VT.isVector())
- WorkingVT = EVT::getVectorVT(
- *DAG.getContext(),
- EVT::getIntegerVT(*DAG.getContext(), VT.getScalarSizeInBits()),
- VT.getVectorElementCount());
- else
- WorkingVT = VT.changeTypeToInteger();
+ WorkingVT = VT.changeTypeToInteger();
+ // Scalars with an illegal integer twin took the memory path above; FP
+ // vector types are expected to have a legal integer counterpart.
+ assert(TLI.isTypeLegal(WorkingVT) &&
+ "no legal same-width integer type for CT_SELECT expansion");
WorkingT = DAG.getBitcast(WorkingVT, Tmp2);
WorkingF = DAG.getBitcast(WorkingVT, Tmp3);
}
// Compute the all-ones/all-zeros mask as a scalar, then splat for vectors.
+ // The element type of a legal vector is not always a legal scalar type
+ // (i32 on riscv64, i64 on riscv32), so build the scalar mask at a legal
+ // width: BUILD_VECTOR and SPLAT_VECTOR implicitly truncate a wider
+ // scalar, and if every legal scalar is narrower than the element, splat
+ // at that width and sign-extend the mask vector (0 and -1 survive both).
EVT MaskEltVT =
WorkingVT.isVector() ? WorkingVT.getVectorElementType() : WorkingVT;
+ EVT MaskSclVT = MaskEltVT;
+ if (!TLI.isTypeLegal(MaskSclVT)) {
+ assert(WorkingVT.isVector() && "scalar CT_SELECT type must be legal");
+ for (MVT MV : {MVT::i64, MVT::i32, MVT::i16, MVT::i8})
+ if (TLI.isTypeLegal(MV)) {
+ MaskSclVT = MV;
+ break;
+ }
+ assert(TLI.isTypeLegal(MaskSclVT) && "no legal scalar integer type");
+ }
SDValue ScalarCond = Tmp1;
- if (ScalarCond.getValueType() != MaskEltVT)
- ScalarCond = DAG.getAnyExtOrTrunc(ScalarCond, dl, MaskEltVT);
+ if (ScalarCond.getValueType() != MaskSclVT)
+ ScalarCond = DAG.getAnyExtOrTrunc(ScalarCond, dl, MaskSclVT);
SDValue ScalarMask =
- DAG.getNode(ISD::SUB, dl, MaskEltVT, DAG.getConstant(0, dl, MaskEltVT),
- DAG.getNode(ISD::AND, dl, MaskEltVT, ScalarCond,
- DAG.getConstant(1, dl, MaskEltVT)));
- SDValue Mask = WorkingVT.isVector()
- ? DAG.getSplat(WorkingVT, dl, ScalarMask)
- : ScalarMask;
+ DAG.getNode(ISD::SUB, dl, MaskSclVT, DAG.getConstant(0, dl, MaskSclVT),
+ DAG.getNode(ISD::AND, dl, MaskSclVT, ScalarCond,
+ DAG.getConstant(1, dl, MaskSclVT)));
+ SDValue Mask;
+ if (!WorkingVT.isVector()) {
+ Mask = ScalarMask;
+ } else if (MaskSclVT.bitsGE(MaskEltVT)) {
+ Mask = DAG.getSplat(WorkingVT, dl, ScalarMask);
+ } else {
+ EVT NarrowVT =
+ WorkingVT.changeVectorElementType(*DAG.getContext(), MaskSclVT);
+ assert(TLI.isTypeLegal(NarrowVT) &&
+ "no legal vector type for the CT_SELECT mask splat");
+ Mask = DAG.getNode(ISD::SIGN_EXTEND, dl, WorkingVT,
+ DAG.getSplat(NarrowVT, dl, ScalarMask));
+ }
// F ^ ((T ^ F) & Mask)
SDValue XorTF = DAG.getNode(ISD::XOR, dl, WorkingVT, WorkingT, WorkingF);
@@ -4493,19 +4516,17 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
// Forward-looking DAGCombine barrier (see header): route the masked-diff
// through a vreg so the chain edge partitions it from the surrounding
// bitwise ops, guarding against a future combiner that might fold
- // XOR/AND/XOR-with-sext-mask back to SELECT. Skipped where no register
- // class is available (scalable vectors, non-simple or illegal types, or
- // a legal type the target has no register class for).
- if (WorkingVT.isSimple() && !WorkingVT.isScalableVector() &&
- TLI.isTypeLegal(WorkingVT.getSimpleVT())) {
- if (const TargetRegisterClass *RC =
- TLI.getRegClassFor(WorkingVT.getSimpleVT())) {
- MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
- Register TMReg = MRI.createVirtualRegister(RC);
- SDValue Chain = DAG.getEntryNode();
- Chain = DAG.getCopyToReg(Chain, dl, TMReg, TM);
- TM = DAG.getCopyFromReg(Chain, dl, TMReg, WorkingVT);
- }
+ // XOR/AND/XOR-with-sext-mask back to SELECT. WorkingVT is legal here, so
+ // it always has a register class; only scalable vectors are skipped,
+ // conservatively.
+ if (!WorkingVT.isScalableVector()) {
+ const TargetRegisterClass *RC =
+ TLI.getRegClassFor(WorkingVT.getSimpleVT());
+ MachineRegisterInfo &MRI = DAG.getMachineFunction().getRegInfo();
+ Register TMReg = MRI.createVirtualRegister(RC);
+ SDValue Chain = DAG.getEntryNode();
+ Chain = DAG.getCopyToReg(Chain, dl, TMReg, TM);
+ TM = DAG.getCopyFromReg(Chain, dl, TMReg, WorkingVT);
}
Tmp1 = DAG.getNode(ISD::XOR, dl, WorkingVT, WorkingF, TM);
@@ -5910,9 +5931,13 @@ void SelectionDAGLegalize::PromoteNode(SDNode *Node) {
// Promote each of the values to the new type.
Tmp2 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(1));
Tmp3 = DAG.getNode(ExtOp, dl, NVT, Node->getOperand(2));
- // Perform the larger operation, then round down.
- Tmp1 = DAG.getNode(Node->getOpcode(), dl, NVT, Tmp1, Tmp2, Tmp3,
- Node->getFlags());
+ // Perform the larger operation, then round down. CT_SELECT carries no
+ // SDNodeFlags (see ISDOpcodes.h); rebuild it through the helper.
+ if (Node->getOpcode() == ISD::CT_SELECT)
+ Tmp1 = DAG.getCTSelect(dl, NVT, Tmp1, Tmp2, Tmp3);
+ else
+ Tmp1 =
+ DAG.getNode(ISD::SELECT, dl, NVT, Tmp1, Tmp2, Tmp3, Node->getFlags());
if (TruncOp != ISD::FP_ROUND)
Tmp1 = DAG.getNode(TruncOp, dl, Node->getValueType(0), Tmp1);
else
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
index 63d92907b6582..48405f6c759e2 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp
@@ -160,8 +160,8 @@ void DAGTypeLegalizer::SoftenFloatResult(SDNode *N, unsigned ResNo) {
case ISD::LOAD: R = SoftenFloatRes_LOAD(N); break;
case ISD::ATOMIC_LOAD: R = SoftenFloatRes_ATOMIC_LOAD(N); break;
case ISD::ATOMIC_SWAP: R = BitcastToInt_ATOMIC_SWAP(N); break;
- case ISD::SELECT: R = SoftenFloatRes_SELECT(N); break;
- case ISD::CT_SELECT: R = SoftenFloatRes_CT_SELECT(N); break;
+ case ISD::SELECT:
+ case ISD::CT_SELECT: R = SoftenFloatRes_SELECT(N); break;
case ISD::SELECT_CC: R = SoftenFloatRes_SELECT_CC(N); break;
case ISD::FREEZE: R = SoftenFloatRes_FREEZE(N); break;
case ISD::STRICT_SINT_TO_FP:
@@ -1062,15 +1062,8 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_ATOMIC_LOAD(SDNode *N) {
SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT(SDNode *N) {
SDValue LHS = GetSoftenedFloat(N->getOperand(1));
SDValue RHS = GetSoftenedFloat(N->getOperand(2));
- return DAG.getSelect(SDLoc(N),
- LHS.getValueType(), N->getOperand(0), LHS, RHS);
-}
-
-SDValue DAGTypeLegalizer::SoftenFloatRes_CT_SELECT(SDNode *N) {
- SDValue LHS = GetSoftenedFloat(N->getOperand(1));
- SDValue RHS = GetSoftenedFloat(N->getOperand(2));
- return DAG.getCTSelect(SDLoc(N), LHS.getValueType(), N->getOperand(0), LHS,
- RHS);
+ return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(),
+ N->getOperand(0), LHS, RHS);
}
SDValue DAGTypeLegalizer::SoftenFloatRes_SELECT_CC(SDNode *N) {
@@ -1592,8 +1585,8 @@ void DAGTypeLegalizer::ExpandFloatResult(SDNode *N, unsigned ResNo) {
// clang-format off
case ISD::POISON:
case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
- case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
- case ISD::CT_SELECT: SplitRes_CT_SELECT(N, Lo, Hi); break;
+ case ISD::SELECT:
+ case ISD::CT_SELECT: SplitRes_Select(N, Lo, Hi); break;
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
case ISD::MERGE_VALUES: ExpandRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
@@ -2771,9 +2764,9 @@ void DAGTypeLegalizer::SoftPromoteHalfResult(SDNode *N, unsigned ResNo) {
case ISD::ATOMIC_LOAD:
R = SoftPromoteHalfRes_ATOMIC_LOAD(N);
break;
- case ISD::SELECT: R = SoftPromoteHalfRes_SELECT(N); break;
+ case ISD::SELECT:
case ISD::CT_SELECT:
- R = SoftPromoteHalfRes_CT_SELECT(N);
+ R = SoftPromoteHalfRes_SELECT(N);
break;
case ISD::SELECT_CC: R = SoftPromoteHalfRes_SELECT_CC(N); break;
case ISD::STRICT_SINT_TO_FP:
@@ -3038,15 +3031,8 @@ SDValue DAGTypeLegalizer::SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N) {
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT(SDNode *N) {
SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
- return DAG.getSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
- Op2);
-}
-
-SDValue DAGTypeLegalizer::SoftPromoteHalfRes_CT_SELECT(SDNode *N) {
- SDValue Op1 = GetSoftPromotedHalf(N->getOperand(1));
- SDValue Op2 = GetSoftPromotedHalf(N->getOperand(2));
- return DAG.getCTSelect(SDLoc(N), Op1.getValueType(), N->getOperand(0), Op1,
- Op2);
+ return DAG.getNode(N->getOpcode(), SDLoc(N), Op1.getValueType(),
+ N->getOperand(0), Op1, Op2);
}
SDValue DAGTypeLegalizer::SoftPromoteHalfRes_SELECT_CC(SDNode *N) {
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index ede25ab017b29..d2d590bc53b42 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -2173,9 +2173,9 @@ bool DAGTypeLegalizer::PromoteIntegerOperand(SDNode *N, unsigned OpNo) {
Res = PromoteIntOp_ScalarOp(N);
break;
case ISD::VSELECT:
- case ISD::SELECT: Res = PromoteIntOp_SELECT(N, OpNo); break;
+ case ISD::SELECT:
case ISD::CT_SELECT:
- Res = PromoteIntOp_CT_SELECT(N, OpNo);
+ Res = PromoteIntOp_SELECT(N, OpNo);
break;
case ISD::SELECT_CC: Res = PromoteIntOp_SELECT_CC(N, OpNo); break;
case ISD::VP_SETCC:
@@ -2582,27 +2582,15 @@ SDValue DAGTypeLegalizer::PromoteIntOp_SELECT(SDNode *N, unsigned OpNo) {
return DAG.getNode(N->getOpcode(), SDLoc(N), N->getValueType(0),
Res, N->getOperand(1), N->getOperand(2));
- // Promote all the way up to the canonical SetCC type.
- EVT OpVT = N->getOpcode() == ISD::SELECT ? OpTy.getScalarType() : OpTy;
+ // Promote all the way up to the canonical SetCC type. SELECT and CT_SELECT
+ // have a scalar condition; VSELECT's mask type matches the vector operands.
+ EVT OpVT = N->getOpcode() == ISD::VSELECT ? OpTy : OpTy.getScalarType();
Cond = PromoteTargetBoolean(Cond, OpVT);
return SDValue(DAG.UpdateNodeOperands(N, Cond, N->getOperand(1),
N->getOperand(2)), 0);
}
-SDValue DAGTypeLegalizer::PromoteIntOp_CT_SELECT(SDNode *N, unsigned OpNo) {
- assert(OpNo == 0 && "Only know how to promote the condition!");
- SDValue Cond = N->getOperand(0);
- EVT OpTy = N->getOperand(1).getValueType();
-
- // Promote all the way up to the canonical SetCC type. The condition is
- // always scalar, so derive the boolean type from the operands' scalar type.
- Cond = PromoteTargetBoolean(Cond, OpTy.getScalarType());
-
- return SDValue(
- DAG.UpdateNodeOperands(N, Cond, N->getOperand(1), N->getOperand(2)), 0);
-}
-
SDValue DAGTypeLegalizer::PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo) {
assert(OpNo == 0 && "Don't know how to promote this operand!");
@@ -3248,9 +3236,9 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
case ISD::ARITH_FENCE: SplitRes_ARITH_FENCE(N, Lo, Hi); break;
case ISD::MERGE_VALUES: SplitRes_MERGE_VALUES(N, ResNo, Lo, Hi); break;
- case ISD::SELECT: SplitRes_Select(N, Lo, Hi); break;
+ case ISD::SELECT:
case ISD::CT_SELECT:
- SplitRes_CT_SELECT(N, Lo, Hi);
+ SplitRes_Select(N, Lo, Hi);
break;
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
case ISD::POISON:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 6a6966603f355..b0b1ddcf3f409 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -389,7 +389,6 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue PromoteIntOp_CONCAT_VECTORS(SDNode *N);
SDValue PromoteIntOp_ScalarOp(SDNode *N);
SDValue PromoteIntOp_SELECT(SDNode *N, unsigned OpNo);
- SDValue PromoteIntOp_CT_SELECT(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_SELECT_CC(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_SETCC(SDNode *N, unsigned OpNo);
SDValue PromoteIntOp_Shift(SDNode *N);
@@ -630,7 +629,6 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftenFloatRes_LOAD(SDNode *N);
SDValue SoftenFloatRes_ATOMIC_LOAD(SDNode *N);
SDValue SoftenFloatRes_SELECT(SDNode *N);
- SDValue SoftenFloatRes_CT_SELECT(SDNode *N);
SDValue SoftenFloatRes_SELECT_CC(SDNode *N);
SDValue SoftenFloatRes_UNDEF(SDNode *N);
SDValue SoftenFloatRes_VAARG(SDNode *N);
@@ -793,7 +791,6 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue SoftPromoteHalfRes_LOAD(SDNode *N);
SDValue SoftPromoteHalfRes_ATOMIC_LOAD(SDNode *N);
SDValue SoftPromoteHalfRes_SELECT(SDNode *N);
- SDValue SoftPromoteHalfRes_CT_SELECT(SDNode *N);
SDValue SoftPromoteHalfRes_SELECT_CC(SDNode *N);
SDValue SoftPromoteHalfRes_UnaryOp(SDNode *N);
SDValue SoftPromoteHalfRes_FABS(SDNode *N);
@@ -864,7 +861,6 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
SDValue ScalarizeVecRes_SCALAR_TO_VECTOR(SDNode *N);
SDValue ScalarizeVecRes_VSELECT(SDNode *N);
SDValue ScalarizeVecRes_SELECT(SDNode *N);
- SDValue ScalarizeVecRes_CT_SELECT(SDNode *N);
SDValue ScalarizeVecRes_SELECT_CC(SDNode *N);
SDValue ScalarizeVecRes_SETCC(SDNode *N);
SDValue ScalarizeVecRes_UNDEF(SDNode *N);
@@ -1208,7 +1204,6 @@ class LLVM_LIBRARY_VISIBILITY DAGTypeLegalizer {
void SplitVecRes_AssertSext(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_ARITH_FENCE (SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_Select(SDNode *N, SDValue &Lo, SDValue &Hi);
- void SplitRes_CT_SELECT(SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_SELECT_CC (SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_UNDEF (SDNode *N, SDValue &Lo, SDValue &Hi);
void SplitRes_FREEZE (SDNode *N, SDValue &Lo, SDValue &Hi);
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
index 6e0fde28ae51b..0cff883f83186 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypesGeneric.cpp
@@ -569,12 +569,6 @@ void DAGTypeLegalizer::SplitRes_Select(SDNode *N, SDValue &Lo, SDValue &Hi) {
Hi = DAG.getNode(Opcode, dl, LH.getValueType(), CH, LH, RH, EVLHi);
}
-void DAGTypeLegalizer::SplitRes_CT_SELECT(SDNode *N, SDValue &Lo, SDValue &Hi) {
- // Reuse generic select splitting to support scalar and vector conditions.
- // SplitRes_Select rebuilds with N->getOpcode(), so CT_SELECT is preserved.
- SplitRes_Select(N, Lo, Hi);
-}
-
void DAGTypeLegalizer::SplitRes_SELECT_CC(SDNode *N, SDValue &Lo,
SDValue &Hi) {
SDValue LL, LH, RL, RH;
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 49b38c1c7cf4a..e9c332691e43a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -86,9 +86,9 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
case ISD::SCALAR_TO_VECTOR: R = ScalarizeVecRes_SCALAR_TO_VECTOR(N); break;
case ISD::SIGN_EXTEND_INREG: R = ScalarizeVecRes_InregOp(N); break;
case ISD::VSELECT: R = ScalarizeVecRes_VSELECT(N); break;
- case ISD::SELECT: R = ScalarizeVecRes_SELECT(N); break;
+ case ISD::SELECT:
case ISD::CT_SELECT:
- R = ScalarizeVecRes_CT_SELECT(N);
+ R = ScalarizeVecRes_SELECT(N);
break;
case ISD::SELECT_CC: R = ScalarizeVecRes_SELECT_CC(N); break;
case ISD::SETCC: R = ScalarizeVecRes_SETCC(N); break;
@@ -762,15 +762,9 @@ SDValue DAGTypeLegalizer::ScalarizeVecRes_VSELECT(SDNode *N) {
SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT(SDNode *N) {
SDValue LHS = GetScalarizedVector(N->getOperand(1));
- return DAG.getSelect(SDLoc(N),
- LHS.getValueType(), N->getOperand(0), LHS,
- GetScalarizedVector(N->getOperand(2)));
-}
-
-SDValue DAGTypeLegalizer::ScalarizeVecRes_CT_SELECT(SDNode *N) {
- SDValue LHS = GetScalarizedVector(N->getOperand(1));
- return DAG.getCTSelect(SDLoc(N), LHS.getValueType(), N->getOperand(0), LHS,
- GetScalarizedVector(N->getOperand(2)));
+ return DAG.getNode(N->getOpcode(), SDLoc(N), LHS.getValueType(),
+ N->getOperand(0), LHS,
+ GetScalarizedVector(N->getOperand(2)));
}
SDValue DAGTypeLegalizer::ScalarizeVecRes_SELECT_CC(SDNode *N) {
@@ -1388,11 +1382,9 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
case ISD::AssertSext: SplitVecRes_AssertSext(N, Lo, Hi); break;
case ISD::VSELECT:
case ISD::SELECT:
+ case ISD::CT_SELECT:
case ISD::VP_MERGE:
case ISD::VP_SELECT: SplitRes_Select(N, Lo, Hi); break;
- case ISD::CT_SELECT:
- SplitRes_CT_SELECT(N, Lo, Hi);
- break;
case ISD::SELECT_CC: SplitRes_SELECT_CC(N, Lo, Hi); break;
case ISD::POISON:
case ISD::UNDEF: SplitRes_UNDEF(N, Lo, Hi); break;
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index a6f9652889a31..f418828445158 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -6894,6 +6894,8 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I,
return;
}
case Intrinsic::ct_select: {
+ // Fast-math flags on the call are intentionally dropped: CT_SELECT
+ // carries no SDNodeFlags, so no FMF-driven combine can apply to it.
SDValue Cond = getValue(I.getArgOperand(0));
SDValue A = getValue(I.getArgOperand(1));
SDValue B = getValue(I.getArgOperand(2));
diff --git a/llvm/lib/IR/Verifier.cpp b/llvm/lib/IR/Verifier.cpp
index 28459a39fedae..55da53a7a4ad8 100644
--- a/llvm/lib/IR/Verifier.cpp
+++ b/llvm/lib/IR/Verifier.cpp
@@ -6073,6 +6073,17 @@ void Verifier::visitIntrinsicCall(Intrinsic::ID ID, CallBase &Call) {
}
break;
}
+ case Intrinsic::ct_select: {
+ // The value type is only constrained to first-class by tablegen's
+ // llvm_any_ty; SelectionDAG lowering only supports single-value types.
+ Type *ValTy = Call.getType();
+ Check(ValTy->isIntOrIntVectorTy() || ValTy->isFPOrFPVectorTy() ||
+ ValTy->isPtrOrPtrVectorTy(),
+ "llvm.ct.select only supports integer, floating-point, or pointer "
+ "types, or vectors of them",
+ Call);
+ break;
+ }
case Intrinsic::coro_begin:
case Intrinsic::coro_begin_custom_abi:
Check(isa<AnyCoroIdInst>(Call.getArgOperand(0)),
diff --git a/llvm/test/CodeGen/RISCV/ctselect-fallback-gisel.ll b/llvm/test/CodeGen/RISCV/ctselect-fallback-gisel.ll
new file mode 100644
index 0000000000000..756d1154b5ef8
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/ctselect-fallback-gisel.ll
@@ -0,0 +1,35 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=riscv64 -global-isel -global-isel-abort=0 < %s | FileCheck %s
+
+; llvm.ct.select has no GlobalISel lowering yet. This pins the per-function
+; SelectionDAG fallback: the function must still compile to the branchless
+; expansion rather than failing in the GlobalISel legalizer.
+
+declare i32 @llvm.ct.select.i32(i1, i32, i32)
+declare i64 @llvm.ct.select.i64(i1, i64, i64)
+
+define i32 @ctsel_fallback_i32(i1 %c, i32 %x, i32 %y) {
+; CHECK-LABEL: ctsel_fallback_i32:
+; CHECK: # %bb.0:
+; CHECK-NEXT: xor a1, a1, a2
+; CHECK-NEXT: slli a0, a0, 63
+; CHECK-NEXT: srai a0, a0, 63
+; CHECK-NEXT: and a0, a1, a0
+; CHECK-NEXT: xor a0, a2, a0
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.ct.select.i32(i1 %c, i32 %x, i32 %y)
+ ret i32 %r
+}
+
+define i64 @ctsel_fallback_i64(i1 %c, i64 %x, i64 %y) {
+; CHECK-LABEL: ctsel_fallback_i64:
+; CHECK: # %bb.0:
+; CHECK-NEXT: xor a1, a1, a2
+; CHECK-NEXT: slli a0, a0, 63
+; CHECK-NEXT: srai a0, a0, 63
+; CHECK-NEXT: and a0, a1, a0
+; CHECK-NEXT: xor a0, a2, a0
+; CHECK-NEXT: ret
+ %r = call i64 @llvm.ct.select.i64(i1 %c, i64 %x, i64 %y)
+ ret i64 %r
+}
diff --git a/llvm/test/CodeGen/RISCV/ctselect-fallback-scalable.ll b/llvm/test/CodeGen/RISCV/ctselect-fallback-scalable.ll
new file mode 100644
index 0000000000000..d721ab1d76a40
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/ctselect-fallback-scalable.ll
@@ -0,0 +1,117 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=riscv64 -mattr=+v < %s | FileCheck %s --check-prefix=RV64V
+; RUN: llc -mtriple=riscv32 -mattr=+v < %s | FileCheck %s --check-prefix=RV32V
+
+; Generic CT_SELECT expansion for scalable vectors. The mask is built at a
+; legal scalar width: elements narrower than it splat with implicit
+; truncation (nxv4i32 on rv64), and elements wider than every legal scalar
+; splat narrow and sign-extend (nxv2i64 on rv32).
+
+declare <vscale x 4 x i32> @llvm.ct.select.nxv4i32(i1, <vscale x 4 x i32>, <vscale x 4 x i32>)
+declare <vscale x 2 x i64> @llvm.ct.select.nxv2i64(i1, <vscale x 2 x i64>, <vscale x 2 x i64>)
+declare <vscale x 4 x float> @llvm.ct.select.nxv4f32(i1, <vscale x 4 x float>, <vscale x 4 x float>)
+declare <vscale x 2 x double> @llvm.ct.select.nxv2f64(i1, <vscale x 2 x double>, <vscale x 2 x double>)
+
+define <vscale x 4 x i32> @ctsel_nxv4i32(i1 %c, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y) {
+; RV64V-LABEL: ctsel_nxv4i32:
+; RV64V: # %bb.0:
+; RV64V-NEXT: vsetvli a1, zero, e32, m2, ta, ma
+; RV64V-NEXT: vxor.vv v8, v8, v10
+; RV64V-NEXT: slli a0, a0, 63
+; RV64V-NEXT: srai a0, a0, 63
+; RV64V-NEXT: vand.vx v8, v8, a0
+; RV64V-NEXT: vxor.vv v8, v10, v8
+; RV64V-NEXT: ret
+;
+; RV32V-LABEL: ctsel_nxv4i32:
+; RV32V: # %bb.0:
+; RV32V-NEXT: vsetvli a1, zero, e32, m2, ta, ma
+; RV32V-NEXT: vxor.vv v8, v8, v10
+; RV32V-NEXT: slli a0, a0, 31
+; RV32V-NEXT: srai a0, a0, 31
+; RV32V-NEXT: vand.vx v8, v8, a0
+; RV32V-NEXT: vxor.vv v8, v10, v8
+; RV32V-NEXT: ret
+ %r = call <vscale x 4 x i32> @llvm.ct.select.nxv4i32(i1 %c, <vscale x 4 x i32> %x, <vscale x 4 x i32> %y)
+ ret <vscale x 4 x i32> %r
+}
+
+define <vscale x 2 x i64> @ctsel_nxv2i64(i1 %c, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y) {
+; RV64V-LABEL: ctsel_nxv2i64:
+; RV64V: # %bb.0:
+; RV64V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; RV64V-NEXT: vxor.vv v8, v8, v10
+; RV64V-NEXT: slli a0, a0, 63
+; RV64V-NEXT: srai a0, a0, 63
+; RV64V-NEXT: vand.vx v8, v8, a0
+; RV64V-NEXT: vxor.vv v8, v10, v8
+; RV64V-NEXT: ret
+;
+; RV32V-LABEL: ctsel_nxv2i64:
+; RV32V: # %bb.0:
+; RV32V-NEXT: slli a0, a0, 31
+; RV32V-NEXT: srai a0, a0, 31
+; RV32V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; RV32V-NEXT: vxor.vv v8, v8, v10
+; RV32V-NEXT: vsetvli zero, zero, e32, m1, ta, ma
+; RV32V-NEXT: vmv.v.x v14, a0
+; RV32V-NEXT: vsetvli zero, zero, e64, m2, ta, ma
+; RV32V-NEXT: vsext.vf2 v12, v14
+; RV32V-NEXT: vand.vv v8, v8, v12
+; RV32V-NEXT: vxor.vv v8, v10, v8
+; RV32V-NEXT: ret
+ %r = call <vscale x 2 x i64> @llvm.ct.select.nxv2i64(i1 %c, <vscale x 2 x i64> %x, <vscale x 2 x i64> %y)
+ ret <vscale x 2 x i64> %r
+}
+
+define <vscale x 4 x float> @ctsel_nxv4f32(i1 %c, <vscale x 4 x float> %x, <vscale x 4 x float> %y) {
+; RV64V-LABEL: ctsel_nxv4f32:
+; RV64V: # %bb.0:
+; RV64V-NEXT: vsetvli a1, zero, e32, m2, ta, ma
+; RV64V-NEXT: vxor.vv v8, v8, v10
+; RV64V-NEXT: slli a0, a0, 63
+; RV64V-NEXT: srai a0, a0, 63
+; RV64V-NEXT: vand.vx v8, v8, a0
+; RV64V-NEXT: vxor.vv v8, v10, v8
+; RV64V-NEXT: ret
+;
+; RV32V-LABEL: ctsel_nxv4f32:
+; RV32V: # %bb.0:
+; RV32V-NEXT: vsetvli a1, zero, e32, m2, ta, ma
+; RV32V-NEXT: vxor.vv v8, v8, v10
+; RV32V-NEXT: slli a0, a0, 31
+; RV32V-NEXT: srai a0, a0, 31
+; RV32V-NEXT: vand.vx v8, v8, a0
+; RV32V-NEXT: vxor.vv v8, v10, v8
+; RV32V-NEXT: ret
+ %r = call <vscale x 4 x float> @llvm.ct.select.nxv4f32(i1 %c, <vscale x 4 x float> %x, <vscale x 4 x float> %y)
+ ret <vscale x 4 x float> %r
+}
+
+define <vscale x 2 x double> @ctsel_nxv2f64(i1 %c, <vscale x 2 x double> %x, <vscale x 2 x double> %y) {
+; RV64V-LABEL: ctsel_nxv2f64:
+; RV64V: # %bb.0:
+; RV64V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; RV64V-NEXT: vxor.vv v8, v8, v10
+; RV64V-NEXT: slli a0, a0, 63
+; RV64V-NEXT: srai a0, a0, 63
+; RV64V-NEXT: vand.vx v8, v8, a0
+; RV64V-NEXT: vxor.vv v8, v10, v8
+; RV64V-NEXT: ret
+;
+; RV32V-LABEL: ctsel_nxv2f64:
+; RV32V: # %bb.0:
+; RV32V-NEXT: slli a0, a0, 31
+; RV32V-NEXT: srai a0, a0, 31
+; RV32V-NEXT: vsetvli a1, zero, e64, m2, ta, ma
+; RV32V-NEXT: vxor.vv v8, v8, v10
+; RV32V-NEXT: vsetvli zero, zero, e32, m1, ta, ma
+; RV32V-NEXT: vmv.v.x v14, a0
+; RV32V-NEXT: vsetvli zero, zero, e64, m2, ta, ma
+; RV32V-NEXT: vsext.vf2 v12, v14
+; RV32V-NEXT: vand.vv v8, v8, v12
+; RV32V-NEXT: vxor.vv v8, v10, v8
+; RV32V-NEXT: ret
+ %r = call <vscale x 2 x double> @llvm.ct.select.nxv2f64(i1 %c, <vscale x 2 x double> %x, <vscale x 2 x double> %y)
+ ret <vscale x 2 x double> %r
+}
diff --git a/llvm/test/CodeGen/X86/ctselect.ll b/llvm/test/CodeGen/X86/ctselect.ll
index 5b23642b1c5d0..2246530619150 100644
--- a/llvm/test/CodeGen/X86/ctselect.ll
+++ b/llvm/test/CodeGen/X86/ctselect.ll
@@ -217,18 +217,18 @@ define double @test_ctselect_f64(i1 %cond, double %a, double %b) #0 {
; X32-LABEL: test_ctselect_f64:
; X32: # %bb.0:
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $24, %esp
+; X32-NEXT: subl $16, %esp
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NEXT: andb $1, %al
; X32-NEXT: fldl {{[0-9]+}}(%esp)
; X32-NEXT: fldl {{[0-9]+}}(%esp)
; X32-NEXT: fstpl {{[0-9]+}}(%esp)
-; X32-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NEXT: fstpl (%esp)
; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl (%esp), %esi
; X32-NEXT: xorl %edx, %esi
; X32-NEXT: andl %eax, %esi
; X32-NEXT: xorl %edx, %esi
@@ -239,25 +239,25 @@ define double @test_ctselect_f64(i1 %cond, double %a, double %b) #0 {
; X32-NEXT: xorl %ecx, %edx
; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NEXT: fldl (%esp)
-; X32-NEXT: addl $24, %esp
+; X32-NEXT: addl $16, %esp
; X32-NEXT: popl %esi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f64:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $24, %esp
+; X32-NOCMOV-NEXT: subl $16, %esp
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: fldl {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fldl {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstpl (%esp)
; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl (%esp), %esi
; X32-NOCMOV-NEXT: xorl %edx, %esi
; X32-NOCMOV-NEXT: andl %eax, %esi
; X32-NOCMOV-NEXT: xorl %edx, %esi
@@ -268,7 +268,7 @@ define double @test_ctselect_f64(i1 %cond, double %a, double %b) #0 {
; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fldl (%esp)
-; X32-NOCMOV-NEXT: addl $24, %esp
+; X32-NOCMOV-NEXT: addl $16, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: retl
%result = call double @llvm.ct.select.f64(i1 %cond, double %a, double %b)
@@ -525,18 +525,18 @@ define x86_fp80 @test_ctselect_f80(i1 %cond, x86_fp80 %a, x86_fp80 %b) #0 {
; X32-LABEL: test_ctselect_f80:
; X32: # %bb.0:
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $36, %esp
+; X32-NEXT: subl $24, %esp
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NEXT: andb $1, %al
; X32-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NEXT: fstpt {{[0-9]+}}(%esp)
-; X32-NEXT: fstpt {{[0-9]+}}(%esp)
+; X32-NEXT: fstpt (%esp)
; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl (%esp), %esi
; X32-NEXT: xorl %edx, %esi
; X32-NEXT: andl %eax, %esi
; X32-NEXT: xorl %edx, %esi
@@ -553,25 +553,25 @@ define x86_fp80 @test_ctselect_f80(i1 %cond, x86_fp80 %a, x86_fp80 %b) #0 {
; X32-NEXT: xorl %ecx, %edx
; X32-NEXT: movw %dx, {{[0-9]+}}(%esp)
; X32-NEXT: fldt (%esp)
-; X32-NEXT: addl $36, %esp
+; X32-NEXT: addl $24, %esp
; X32-NEXT: popl %esi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f80:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $36, %esp
+; X32-NOCMOV-NEXT: subl $24, %esp
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fldt {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
-; X32-NOCMOV-NEXT: fstpt {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstpt (%esp)
; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl (%esp), %esi
; X32-NOCMOV-NEXT: xorl %edx, %esi
; X32-NOCMOV-NEXT: andl %eax, %esi
; X32-NOCMOV-NEXT: xorl %edx, %esi
@@ -588,7 +588,7 @@ define x86_fp80 @test_ctselect_f80(i1 %cond, x86_fp80 %a, x86_fp80 %b) #0 {
; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: movw %dx, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fldt (%esp)
-; X32-NOCMOV-NEXT: addl $36, %esp
+; X32-NOCMOV-NEXT: addl $24, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: retl
%result = call x86_fp80 @llvm.ct.select.f80(i1 %cond, x86_fp80 %a, x86_fp80 %b)
@@ -2374,18 +2374,18 @@ define double @test_ctselect_f64_nan_inf(i1 %cond) #0 {
; X32-LABEL: test_ctselect_f64_nan_inf:
; X32: # %bb.0:
; X32-NEXT: pushl %esi
-; X32-NEXT: subl $24, %esp
+; X32-NEXT: subl $16, %esp
; X32-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NEXT: andb $1, %al
; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
; X32-NEXT: fstpl {{[0-9]+}}(%esp)
; X32-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
-; X32-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NEXT: fstpl (%esp)
; X32-NEXT: movzbl %al, %eax
; X32-NEXT: negl %eax
; X32-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NEXT: movl (%esp), %esi
; X32-NEXT: xorl %edx, %esi
; X32-NEXT: andl %eax, %esi
; X32-NEXT: xorl %edx, %esi
@@ -2396,25 +2396,25 @@ define double @test_ctselect_f64_nan_inf(i1 %cond) #0 {
; X32-NEXT: xorl %ecx, %edx
; X32-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NEXT: fldl (%esp)
-; X32-NEXT: addl $24, %esp
+; X32-NEXT: addl $16, %esp
; X32-NEXT: popl %esi
; X32-NEXT: retl
;
; X32-NOCMOV-LABEL: test_ctselect_f64_nan_inf:
; X32-NOCMOV: # %bb.0:
; X32-NOCMOV-NEXT: pushl %esi
-; X32-NOCMOV-NEXT: subl $24, %esp
+; X32-NOCMOV-NEXT: subl $16, %esp
; X32-NOCMOV-NEXT: movzbl {{[0-9]+}}(%esp), %eax
; X32-NOCMOV-NEXT: andb $1, %al
; X32-NOCMOV-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: flds {{\.?LCPI[0-9]+_[0-9]+}}
-; X32-NOCMOV-NEXT: fstpl {{[0-9]+}}(%esp)
+; X32-NOCMOV-NEXT: fstpl (%esp)
; X32-NOCMOV-NEXT: movzbl %al, %eax
; X32-NOCMOV-NEXT: negl %eax
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %edx
; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %ecx
-; X32-NOCMOV-NEXT: movl {{[0-9]+}}(%esp), %esi
+; X32-NOCMOV-NEXT: movl (%esp), %esi
; X32-NOCMOV-NEXT: xorl %edx, %esi
; X32-NOCMOV-NEXT: andl %eax, %esi
; X32-NOCMOV-NEXT: xorl %edx, %esi
@@ -2425,7 +2425,7 @@ define double @test_ctselect_f64_nan_inf(i1 %cond) #0 {
; X32-NOCMOV-NEXT: xorl %ecx, %edx
; X32-NOCMOV-NEXT: movl %edx, {{[0-9]+}}(%esp)
; X32-NOCMOV-NEXT: fldl (%esp)
-; X32-NOCMOV-NEXT: addl $24, %esp
+; X32-NOCMOV-NEXT: addl $16, %esp
; X32-NOCMOV-NEXT: popl %esi
; X32-NOCMOV-NEXT: retl
%result = call double @llvm.ct.select.f64(i1 %cond, double 0x7FF8000000000000, double 0x7FF0000000000000)
diff --git a/llvm/test/Verifier/ct-select.ll b/llvm/test/Verifier/ct-select.ll
new file mode 100644
index 0000000000000..56f897a8a53c3
--- /dev/null
+++ b/llvm/test/Verifier/ct-select.ll
@@ -0,0 +1,20 @@
+; RUN: not llvm-as -disable-output < %s 2>&1 | FileCheck %s
+
+; ct.select's value type is restricted to single-value types: aggregates are
+; accepted by the intrinsic signature (llvm_any_ty) but rejected here because
+; codegen does not support them.
+
+declare {i32, i32} @llvm.ct.select.sl_i32i32s(i1, {i32, i32}, {i32, i32})
+declare [2 x i64] @llvm.ct.select.a2i64(i1, [2 x i64], [2 x i64])
+
+; CHECK: llvm.ct.select only supports integer, floating-point, or pointer types, or vectors of them
+define {i32, i32} @ct_select_struct(i1 %c, {i32, i32} %x, {i32, i32} %y) {
+ %r = call {i32, i32} @llvm.ct.select.sl_i32i32s(i1 %c, {i32, i32} %x, {i32, i32} %y)
+ ret {i32, i32} %r
+}
+
+; CHECK: llvm.ct.select only supports integer, floating-point, or pointer types, or vectors of them
+define [2 x i64] @ct_select_array(i1 %c, [2 x i64] %x, [2 x i64] %y) {
+ %r = call [2 x i64] @llvm.ct.select.a2i64(i1 %c, [2 x i64] %x, [2 x i64] %y)
+ ret [2 x i64] %r
+}
More information about the llvm-commits
mailing list