[llvm] [AArch64] optimize lowering for icmp on i128 when RHS is an immediate (PR #181822)
Cheng Lingfei via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 6 03:58:50 PDT 2026
https://github.com/clingfei updated https://github.com/llvm/llvm-project/pull/181822
>From 3caeb6658972cbd04ebd67842908df102bc3aebc Mon Sep 17 00:00:00 2001
From: clingfei <1599101385 at qq.com>
Date: Tue, 17 Feb 2026 21:22:02 +0800
Subject: [PATCH 1/9] [AArch64] optimize lowering for icmp on i128 when RHS is
an immediate
---
.../Target/AArch64/AArch64ISelLowering.cpp | 118 ++++++++++++++++++
.../CodeGen/AArch64/i128-imm-compare-ccmp.ll | 61 +++++++++
llvm/test/CodeGen/AArch64/isinf.ll | 10 +-
3 files changed, 184 insertions(+), 5 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 66c22db0491d1..0880e5ce210f5 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27145,6 +27145,121 @@ performVecReduceBitwiseCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
return SDValue();
}
+static bool splitI128ValueToI64Halves(SDValue V, SelectionDAG &DAG,
+ const SDLoc &DL, SDValue &Lo,
+ SDValue &Hi) {
+ EVT VT = V.getValueType();
+ if (!VT.isInteger() || VT.getFixedSizeInBits() != 128)
+ return false;
+ Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, V);
+ Hi = DAG.getNode(
+ ISD::TRUNCATE, DL, MVT::i64,
+ DAG.getNode(ISD::SRL, DL, VT, V, DAG.getShiftAmountConstant(64, VT, DL)));
+ return true;
+}
+
+static SDValue
+performExpandedI128CmpCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
+ SelectionDAG &DAG, ISD::CondCode CCCode) {
+ SDValue NewLHS = N->getOperand(0);
+ SDValue NewRHS = N->getOperand(1);
+ if (N->hasOneUse()) {
+ auto User = N->user_begin();
+ if ((User)->getOpcode() == ISD::BRCOND)
+ return SDValue();
+ }
+ // Keep the dedicated shift+cmp-zero combine opportunities for wide integers.
+ // Canonicalizing these into split compares here tends to introduce an extra
+ // EXTR on AArch64 (see icmp-shift-opt.ll).
+ if ((CCCode == ISD::SETEQ || CCCode == ISD::SETNE) &&
+ isNullConstant(NewRHS) &&
+ (NewLHS.getOpcode() == ISD::SRL || NewLHS.getOpcode() == ISD::SHL))
+ return SDValue();
+
+ if (NewLHS.getNumOperands() != 2)
+ return SDValue();
+ if (NewLHS.getValueType().isScalableVT() ||
+ NewRHS.getValueType().isScalableVT() || NewLHS.getNumOperands() != 2 ||
+ !(NewLHS.getValueType().isInteger() &&
+ NewLHS.getValueSizeInBits() == 128) ||
+ !(NewRHS.getValueType().isInteger() &&
+ NewRHS.getValueSizeInBits() == 128))
+ return SDValue();
+ ConstantSDNode *ConstLHS = dyn_cast<ConstantSDNode>(NewLHS.getNode());
+ ConstantSDNode *ConstRHS = dyn_cast<ConstantSDNode>(NewRHS.getNode());
+ if (ConstLHS || !ConstRHS)
+ return SDValue();
+
+ SDValue LHSLo = NewLHS.getOperand(0);
+ SDValue LHSHi = NewLHS.getOperand(1);
+ SDValue RHSLo =
+ DAG.getConstant(ConstRHS->getAPIntValue().trunc(64), SDLoc(N), MVT::i64);
+ SDValue RHSHi = DAG.getConstant(ConstRHS->getAPIntValue().lshr(64).trunc(64),
+ SDLoc(N), MVT::i64);
+ if (!splitI128ValueToI64Halves(NewLHS, DAG, SDLoc(N), LHSLo, LHSHi)) {
+ return SDValue();
+ }
+
+ if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
+ SDValue LoCmpF =
+ DAG.FoldSetCC(N->getValueType(0), LHSLo, RHSLo, CCCode, SDLoc(N));
+ SDValue HiCmpF =
+ DAG.FoldSetCC(N->getValueType(0), LHSHi, RHSHi, CCCode, SDLoc(N));
+ auto IsConstBool = [](SDValue V) {
+ return isa_and_nonnull<ConstantSDNode>(V.getNode());
+ };
+ if (IsConstBool(LoCmpF) || IsConstBool(HiCmpF))
+ return SDValue();
+
+ SDValue LoCmp =
+ DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSLo, RHSLo, CCCode);
+ SDValue HiCmp =
+ DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSHi, RHSHi, CCCode);
+ unsigned Opcode = (CCCode == ISD::SETEQ) ? ISD::AND : ISD::OR;
+ return DAG.getNode(Opcode, SDLoc(N), LoCmp.getValueType(), LoCmp, HiCmp);
+ }
+
+ if (CCCode == ISD::SETUGT || CCCode == ISD::SETUGE) {
+ // x > K <=> (xhi > khi) || (xhi==khi && xlo > klo)
+ // x >= K <=> (xhi > khi) || (xhi==khi && xlo >= klo)
+ SDValue ZeroHi = DAG.getConstant(0, SDLoc(N), LHSHi.getValueType());
+
+ ISD::CondCode LoCC = (CCCode == ISD::SETUGT) ? ISD::SETULE : ISD::SETULT;
+
+ SDValue HiEq =
+ DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSHi, ZeroHi, ISD::SETEQ);
+ SDValue LoCmp =
+ DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSLo, RHSLo, LoCC);
+ SDValue Tree =
+ DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), HiEq, LoCmp);
+
+ SDValue Zero = DAG.getConstant(0, SDLoc(N), N->getValueType(0));
+ return DAG.getSetCC(SDLoc(N), N->getValueType(0), Tree, Zero, ISD::SETEQ);
+ }
+
+ if (CCCode == ISD::SETULT || CCCode == ISD::SETULE) {
+ // x < K <=> (xhi < khi) || (xhi==khi && xlo < klo)
+ // x <= K <=> (xhi < khi) || (xhi==khi && xlo <= klo)
+ ISD::CondCode Opcode = ISD::SETULT;
+
+ SDValue HiCmp =
+ DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSHi, RHSHi, Opcode);
+ SDValue HiEq =
+ DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSHi, RHSHi, ISD::SETEQ);
+ SDValue LoCmp =
+ DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSLo, RHSLo, CCCode);
+
+ SDValue LoAnd =
+ DAG.getNode(ISD::AND, SDLoc(N), HiEq.getValueType(), HiEq, LoCmp);
+ SDValue Tree =
+ DAG.getNode(ISD::OR, SDLoc(N), N->getValueType(0), HiCmp, LoAnd);
+
+ SDValue Zero = DAG.getConstant(0, SDLoc(N), N->getValueType(0));
+ return DAG.getSetCC(SDLoc(N), N->getValueType(0), Tree, Zero, ISD::SETNE);
+ }
+ return SDValue();
+}
+
static SDValue performSETCCCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG) {
@@ -27225,6 +27340,9 @@ static SDValue performSETCCCombine(SDNode *N,
SplatLHSVal.isOne())
return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, CmpVT), RHS, ISD::SETGE);
+ if (SDValue V = performExpandedI128CmpCombine(N, DCI, DAG, Cond))
+ return V;
+
return SDValue();
}
diff --git a/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll b/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
new file mode 100644
index 0000000000000..7422cbbfd19a0
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
@@ -0,0 +1,61 @@
+; RUN: llc -mtriple=aarch64-linux-gnu -O3 -o - < %s | FileCheck %s
+
+define i1 @eq_imm(i128 %x) {
+; CHECK-LABEL: eq_imm:
+; CHECK: cmp x0, #5
+; CHECK-NEXT: ccmp x1, #0, #0, eq
+; CHECK-NEXT: cset w0, eq
+; CHECK-NEXT: ret
+ %cmp = icmp eq i128 %x, 5
+ ret i1 %cmp
+}
+
+define i1 @ne_imm(i128 %x) {
+; CHECK-LABEL: ne_imm:
+; CHECK: cmp x0, #5
+; CHECK-NEXT: ccmp x1, #0, #0, eq
+; CHECK-NEXT: cset w0, ne
+; CHECK-NEXT: ret
+ %cmp = icmp ne i128 %x, 5
+ ret i1 %cmp
+}
+
+define i1 @ult_imm(i128 %x) {
+; CHECK-LABEL: ult_imm:
+; CHECK: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #5, #2, eq
+; CHECK-NEXT: cset w0, {{cc|lo}}
+; CHECK-NEXT: ret
+ %cmp = icmp ult i128 %x, 5
+ ret i1 %cmp
+}
+
+define i1 @ule_imm(i128 %x) {
+; CHECK-LABEL: ule_imm:
+; CHECK: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #6, #2, eq
+; CHECK-NEXT: cset w0, {{cc|lo}}
+; CHECK-NEXT: ret
+ %cmp = icmp ule i128 %x, 5
+ ret i1 %cmp
+}
+
+define i1 @ugt_imm(i128 %x) {
+; CHECK-LABEL: ugt_imm:
+; CHECK: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #5, #2, eq
+; CHECK-NEXT: cset w0, hi
+; CHECK-NEXT: ret
+ %cmp = icmp ugt i128 %x, 5
+ ret i1 %cmp
+}
+
+define i1 @uge_imm(i128 %x) {
+; CHECK-LABEL: uge_imm:
+; CHECK: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #4, #2, eq
+; CHECK-NEXT: cset w0, hi
+; CHECK-NEXT: ret
+ %cmp = icmp uge i128 %x, 5
+ ret i1 %cmp
+}
\ No newline at end of file
diff --git a/llvm/test/CodeGen/AArch64/isinf.ll b/llvm/test/CodeGen/AArch64/isinf.ll
index e8bbaf96395f0..59e1ab44f16b4 100644
--- a/llvm/test/CodeGen/AArch64/isinf.ll
+++ b/llvm/test/CodeGen/AArch64/isinf.ll
@@ -58,11 +58,11 @@ define i32 @replace_isinf_call_f128(fp128 %x) {
; CHECK: // %bb.0:
; CHECK-NEXT: str q0, [sp, #-16]!
; CHECK-NEXT: .cfi_def_cfa_offset 16
-; CHECK-NEXT: ldp x9, x8, [sp], #16
-; CHECK-NEXT: and x8, x8, #0x7fffffffffffffff
-; CHECK-NEXT: eor x8, x8, #0x7fff000000000000
-; CHECK-NEXT: orr x8, x9, x8
-; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: mov x8, #-562949953421312
+; CHECK-NEXT: ldp x10, x9, [sp], #16
+; CHECK-NEXT: lsl x9, x9, #1
+; CHECK-NEXT: cmp x10, #0
+; CHECK-NEXT: ccmp x8, x9, #0, eq
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
%abs = tail call fp128 @llvm.fabs.f128(fp128 %x)
>From bf60f9f64421d695b97c73be30d5c71bfbf0d5e4 Mon Sep 17 00:00:00 2001
From: clingfei <1599101385 at qq.com>
Date: Fri, 10 Apr 2026 23:30:38 +0800
Subject: [PATCH 2/9] fix
---
llvm/lib/Target/AArch64/AArch64ISelLowering.cpp | 8 +++++++-
llvm/test/CodeGen/AArch64/isinf.ll | 14 +++++++-------
2 files changed, 14 insertions(+), 8 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c9f6e0d651251..305d22859d51d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -27769,8 +27769,14 @@ performExpandedI128CmpCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
if (CCCode == ISD::SETUGT || CCCode == ISD::SETUGE) {
// x > K <=> (xhi > khi) || (xhi==khi && xlo > klo)
// x >= K <=> (xhi > khi) || (xhi==khi && xlo >= klo)
- SDValue ZeroHi = DAG.getConstant(0, SDLoc(N), LHSHi.getValueType());
+ // This specialized form relies on the immediate living entirely in the low
+ // 64 bits. When the high half is non-zero, reducing the compare to a
+ // high==0 test is invalid and can miscompile widened expressions such as
+ // (shl i128 X, 1) > C.
+ if (!isNullConstant(RHSHi))
+ return SDValue();
+ SDValue ZeroHi = DAG.getConstant(0, SDLoc(N), LHSHi.getValueType());
ISD::CondCode LoCC = (CCCode == ISD::SETUGT) ? ISD::SETULE : ISD::SETULT;
SDValue HiEq =
diff --git a/llvm/test/CodeGen/AArch64/isinf.ll b/llvm/test/CodeGen/AArch64/isinf.ll
index f3283d2cf7ec2..92d53d7086e46 100644
--- a/llvm/test/CodeGen/AArch64/isinf.ll
+++ b/llvm/test/CodeGen/AArch64/isinf.ll
@@ -80,14 +80,14 @@ define i32 @replace_isinf_call_f64(double %x) {
define i32 @replace_isinf_call_f128(fp128 %x) {
; CHECK-SD-LABEL: replace_isinf_call_f128:
; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: str q0, [sp, #-16]!
+; CHECK-SD-NEXT: str q0, [sp, #-16]!
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT: ldp x9, x8, [sp], #16
-; CHECK-SD-NEXT: and x8, x8, #0x7fffffffffffffff
-; CHECK-SD-NEXT: eor x8, x8, #0x7fff000000000000
-; CHECK-SD-NEXT: orr x8, x9, x8
-; CHECK-SD-NEXT: cmp x8, #0
-; CHECK-SD-NEXT: cset w0, eq
+; CHECK-SD-NEXT: mov x8, #-562949953421312 // =0xfffe000000000000
+; CHECK-SD-NEXT: ldp x10, x9, [sp], #16
+; CHECK-SD-NEXT: lsl x9, x9, #1
+; CHECK-SD-NEXT: cmp x10, #0
+; CHECK-SD-NEXT: ccmp x8, x9, #0, eq
+; CHECK-SD-NEXT: cset w0, eq
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: replace_isinf_call_f128:
>From 25d8e9f018ae8558682bbbc1ad9bbe588c5a1a1e Mon Sep 17 00:00:00 2001
From: clingfei <1599101385 at qq.com>
Date: Thu, 7 May 2026 23:45:29 +0800
Subject: [PATCH 3/9] Optimize at later stage
---
.../Target/AArch64/AArch64ISelLowering.cpp | 266 +++++++++---------
.../CodeGen/AArch64/i128-imm-compare-ccmp.ll | 116 +++++++-
2 files changed, 242 insertions(+), 140 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index fb95b6e2cb67c..071149ef46585 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1155,6 +1155,7 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
// Try and combine setcc with csel
setTargetDAGCombine(ISD::SETCC);
+ setTargetDAGCombine(ISD::SETCCCARRY);
setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
@@ -11863,9 +11864,62 @@ SDValue AArch64TargetLowering::LowerBitreverse(SDValue Op,
}
// Check whether the continuous comparison sequence.
+static bool hasLegalCmpImmediateOperand(SDValue V) {
+ ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
+ return !C || isLegalCmpImmed(C->getAPIntValue());
+}
+
+static bool hasLegalCmpImmediate(SDValue LHS, SDValue RHS) {
+ return hasLegalCmpImmediateOperand(LHS) && hasLegalCmpImmediateOperand(RHS);
+}
+
+static bool isLegalCondCmpImmediate(const APInt &Imm) {
+ if (Imm.isNegative())
+ return Imm.sgt(-32);
+ return Imm.getLimitedValue(32) <= 31;
+}
+
+static bool hasLegalCondCmpImmediateOperand(SDValue V) {
+ ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
+ return !C || isLegalCondCmpImmediate(C->getAPIntValue());
+}
+
+static bool hasLegalCondCmpImmediate(SDValue LHS, SDValue RHS) {
+ return hasLegalCondCmpImmediateOperand(LHS) &&
+ hasLegalCondCmpImmediateOperand(RHS);
+}
+
+static bool isLegalXorImmediate(const APInt &Imm) {
+ unsigned BitWidth = Imm.getBitWidth() <= 32 ? 32 : 64;
+ if (Imm.getBitWidth() > BitWidth)
+ return false;
+ return AArch64_AM::isLogicalImmediate(Imm.getZExtValue(), BitWidth);
+}
+
+static bool hasCheapXorImmediateThatNeedsCondCmpRegOperand(SDValue V) {
+ ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
+ if (!C)
+ return false;
+
+ const APInt &Imm = C->getAPIntValue();
+ return !isLegalCondCmpImmediate(Imm) && isLegalXorImmediate(Imm);
+}
+
+static bool hasCheapXorImmediateThatNeedsCondCmpReg(
+ const std::pair<SDValue, SDValue> &Pair) {
+ return hasCheapXorImmediateThatNeedsCondCmpRegOperand(Pair.first) ||
+ hasCheapXorImmediateThatNeedsCondCmpRegOperand(Pair.second);
+}
+
+static bool preferAsFirstCmp(const std::pair<SDValue, SDValue> &Pair) {
+ return hasLegalCmpImmediate(Pair.first, Pair.second) &&
+ !hasLegalCondCmpImmediate(Pair.first, Pair.second);
+}
+
static bool
-isOrXorChain(SDValue N, unsigned &Num,
- SmallVector<std::pair<SDValue, SDValue>, 16> &WorkList) {
+isOrXorChain(SDValue N, SelectionDAG &DAG, unsigned &Num, bool &SawXor,
+ bool RequireLegalCmpImmediates,
+ SmallVectorImpl<std::pair<SDValue, SDValue>> &WorkList) {
if (Num == MaxXors)
return false;
@@ -11873,21 +11927,36 @@ isOrXorChain(SDValue N, unsigned &Num,
if (N->getOpcode() == ISD::ZERO_EXTEND && N->hasOneUse())
N = N->getOperand(0);
- // The leaf node must be XOR
if (N->getOpcode() == ISD::XOR) {
+ if (RequireLegalCmpImmediates &&
+ !hasLegalCmpImmediate(N->getOperand(0), N->getOperand(1)))
+ return false;
WorkList.push_back(std::make_pair(N->getOperand(0), N->getOperand(1)));
Num++;
+ SawXor = true;
return true;
}
// All the non-leaf nodes must be OR.
- if (N->getOpcode() != ISD::OR || !N->hasOneUse())
+ if (N->getOpcode() == ISD::OR && N->hasOneUse())
+ return isOrXorChain(N->getOperand(0), DAG, Num, SawXor,
+ RequireLegalCmpImmediates, WorkList) &&
+ isOrXorChain(N->getOperand(1), DAG, Num, SawXor,
+ RequireLegalCmpImmediates, WorkList);
+ if (N->getOpcode() == ISD::OR)
return false;
- if (isOrXorChain(N->getOperand(0), Num, WorkList) &&
- isOrXorChain(N->getOperand(1), Num, WorkList))
- return true;
- return false;
+ EVT VT = N.getValueType();
+ if (!VT.isScalarInteger())
+ return false;
+
+ // A xor with zero may have been folded away before this combine sees it.
+ // Treat such leaves as comparisons with zero so the original OR/XOR form and
+ // type-legalized wide integer equality compares converge to the same SETCC
+ // tree.
+ WorkList.push_back(std::make_pair(N, DAG.getConstant(0, SDLoc(N), VT)));
+ Num++;
+ return true;
}
// Transform chains of ORs and XORs, which usually outlined by memcmp/bmp.
@@ -11906,9 +11975,26 @@ static SDValue performOrXorChainCombine(SDNode *N, SelectionDAG &DAG) {
// Try to express conjunction "cmp 0 (or (xor A0 A1) (xor B0 B1))" as:
// sub A0, A1; ccmp B0, B1, 0, eq; cmp inv(Cond) flag
unsigned NumXors = 0;
+ bool SawXor = false;
+ bool RequireLegalCmpImmediates =
+ any_of(N->users(), [](SDNode *User) {
+ return User->getOpcode() == ISD::BRCOND ||
+ User->getOpcode() == AArch64ISD::BRCOND;
+ });
if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && isNullConstant(RHS) &&
LHS->getOpcode() == ISD::OR && LHS->hasOneUse() &&
- isOrXorChain(LHS, NumXors, WorkList)) {
+ isOrXorChain(LHS, DAG, NumXors, SawXor, RequireLegalCmpImmediates,
+ WorkList) &&
+ SawXor) {
+ if (WorkList.size() > 2 &&
+ any_of(WorkList, hasCheapXorImmediateThatNeedsCondCmpReg))
+ return SDValue();
+
+ SmallVector<std::pair<SDValue, SDValue>, 16>::iterator First =
+ find_if(WorkList, preferAsFirstCmp);
+ if (First != WorkList.end())
+ std::iter_swap(WorkList.begin(), First);
+
SDValue XOR0, XOR1;
std::tie(XOR0, XOR1) = WorkList[0];
unsigned LogicOp = (Cond == ISD::SETEQ) ? ISD::AND : ISD::OR;
@@ -27384,127 +27470,6 @@ performVecReduceBitwiseCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
return SDValue();
}
-static bool splitI128ValueToI64Halves(SDValue V, SelectionDAG &DAG,
- const SDLoc &DL, SDValue &Lo,
- SDValue &Hi) {
- EVT VT = V.getValueType();
- if (!VT.isInteger() || VT.getFixedSizeInBits() != 128)
- return false;
- Lo = DAG.getNode(ISD::TRUNCATE, DL, MVT::i64, V);
- Hi = DAG.getNode(
- ISD::TRUNCATE, DL, MVT::i64,
- DAG.getNode(ISD::SRL, DL, VT, V, DAG.getShiftAmountConstant(64, VT, DL)));
- return true;
-}
-
-static SDValue
-performExpandedI128CmpCombine(SDNode *N, TargetLowering::DAGCombinerInfo &DCI,
- SelectionDAG &DAG, ISD::CondCode CCCode) {
- SDValue NewLHS = N->getOperand(0);
- SDValue NewRHS = N->getOperand(1);
- if (N->hasOneUse()) {
- auto User = N->user_begin();
- if ((User)->getOpcode() == ISD::BRCOND)
- return SDValue();
- }
- // Keep the dedicated shift+cmp-zero combine opportunities for wide integers.
- // Canonicalizing these into split compares here tends to introduce an extra
- // EXTR on AArch64 (see icmp-shift-opt.ll).
- if ((CCCode == ISD::SETEQ || CCCode == ISD::SETNE) &&
- isNullConstant(NewRHS) &&
- (NewLHS.getOpcode() == ISD::SRL || NewLHS.getOpcode() == ISD::SHL))
- return SDValue();
-
- if (NewLHS.getNumOperands() != 2)
- return SDValue();
- if (NewLHS.getValueType().isScalableVT() ||
- NewRHS.getValueType().isScalableVT() || NewLHS.getNumOperands() != 2 ||
- !(NewLHS.getValueType().isInteger() &&
- NewLHS.getValueSizeInBits() == 128) ||
- !(NewRHS.getValueType().isInteger() &&
- NewRHS.getValueSizeInBits() == 128))
- return SDValue();
- ConstantSDNode *ConstLHS = dyn_cast<ConstantSDNode>(NewLHS.getNode());
- ConstantSDNode *ConstRHS = dyn_cast<ConstantSDNode>(NewRHS.getNode());
- if (ConstLHS || !ConstRHS)
- return SDValue();
-
- SDValue LHSLo = NewLHS.getOperand(0);
- SDValue LHSHi = NewLHS.getOperand(1);
- SDValue RHSLo =
- DAG.getConstant(ConstRHS->getAPIntValue().trunc(64), SDLoc(N), MVT::i64);
- SDValue RHSHi = DAG.getConstant(ConstRHS->getAPIntValue().lshr(64).trunc(64),
- SDLoc(N), MVT::i64);
- if (!splitI128ValueToI64Halves(NewLHS, DAG, SDLoc(N), LHSLo, LHSHi)) {
- return SDValue();
- }
-
- if (CCCode == ISD::SETEQ || CCCode == ISD::SETNE) {
- SDValue LoCmpF =
- DAG.FoldSetCC(N->getValueType(0), LHSLo, RHSLo, CCCode, SDLoc(N));
- SDValue HiCmpF =
- DAG.FoldSetCC(N->getValueType(0), LHSHi, RHSHi, CCCode, SDLoc(N));
- auto IsConstBool = [](SDValue V) {
- return isa_and_nonnull<ConstantSDNode>(V.getNode());
- };
- if (IsConstBool(LoCmpF) || IsConstBool(HiCmpF))
- return SDValue();
-
- SDValue LoCmp =
- DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSLo, RHSLo, CCCode);
- SDValue HiCmp =
- DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSHi, RHSHi, CCCode);
- unsigned Opcode = (CCCode == ISD::SETEQ) ? ISD::AND : ISD::OR;
- return DAG.getNode(Opcode, SDLoc(N), LoCmp.getValueType(), LoCmp, HiCmp);
- }
-
- if (CCCode == ISD::SETUGT || CCCode == ISD::SETUGE) {
- // x > K <=> (xhi > khi) || (xhi==khi && xlo > klo)
- // x >= K <=> (xhi > khi) || (xhi==khi && xlo >= klo)
- // This specialized form relies on the immediate living entirely in the low
- // 64 bits. When the high half is non-zero, reducing the compare to a
- // high==0 test is invalid and can miscompile widened expressions such as
- // (shl i128 X, 1) > C.
- if (!isNullConstant(RHSHi))
- return SDValue();
-
- SDValue ZeroHi = DAG.getConstant(0, SDLoc(N), LHSHi.getValueType());
- ISD::CondCode LoCC = (CCCode == ISD::SETUGT) ? ISD::SETULE : ISD::SETULT;
-
- SDValue HiEq =
- DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSHi, ZeroHi, ISD::SETEQ);
- SDValue LoCmp =
- DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSLo, RHSLo, LoCC);
- SDValue Tree =
- DAG.getNode(ISD::AND, SDLoc(N), N->getValueType(0), HiEq, LoCmp);
-
- SDValue Zero = DAG.getConstant(0, SDLoc(N), N->getValueType(0));
- return DAG.getSetCC(SDLoc(N), N->getValueType(0), Tree, Zero, ISD::SETEQ);
- }
-
- if (CCCode == ISD::SETULT || CCCode == ISD::SETULE) {
- // x < K <=> (xhi < khi) || (xhi==khi && xlo < klo)
- // x <= K <=> (xhi < khi) || (xhi==khi && xlo <= klo)
- ISD::CondCode Opcode = ISD::SETULT;
-
- SDValue HiCmp =
- DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSHi, RHSHi, Opcode);
- SDValue HiEq =
- DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSHi, RHSHi, ISD::SETEQ);
- SDValue LoCmp =
- DAG.getSetCC(SDLoc(N), N->getValueType(0), LHSLo, RHSLo, CCCode);
-
- SDValue LoAnd =
- DAG.getNode(ISD::AND, SDLoc(N), HiEq.getValueType(), HiEq, LoCmp);
- SDValue Tree =
- DAG.getNode(ISD::OR, SDLoc(N), N->getValueType(0), HiCmp, LoAnd);
-
- SDValue Zero = DAG.getConstant(0, SDLoc(N), N->getValueType(0));
- return DAG.getSetCC(SDLoc(N), N->getValueType(0), Tree, Zero, ISD::SETNE);
- }
- return SDValue();
-}
-
static SDValue performSETCCCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG) {
@@ -27585,12 +27550,45 @@ static SDValue performSETCCCombine(SDNode *N,
SplatLHSVal.isOne())
return DAG.getSetCC(DL, VT, DAG.getConstant(0, DL, CmpVT), RHS, ISD::SETGE);
- if (SDValue V = performExpandedI128CmpCombine(N, DCI, DAG, Cond))
- return V;
-
return SDValue();
}
+static SDValue performSETCCCARRYCombine(SDNode *N, SelectionDAG &DAG) {
+ assert(N->getOpcode() == ISD::SETCCCARRY && "Unexpected opcode!");
+
+ // Rebuild narrow high/low compares from type-legalized wide unsigned compares
+ // so the existing CCMP conjunction/disjunction lowering can handle them.
+ SDValue HiLHS = N->getOperand(0);
+ SDValue HiRHS = N->getOperand(1);
+ SDValue Carry = N->getOperand(2);
+ ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(3))->get();
+ if (Cond != ISD::SETULT || Carry.getOpcode() != ISD::USUBO ||
+ Carry.getResNo() != 1)
+ return SDValue();
+
+ if (!isNullConstant(HiLHS) && !isNullConstant(HiRHS))
+ return SDValue();
+
+ SDValue LoLHS = Carry.getOperand(0);
+ SDValue LoRHS = Carry.getOperand(1);
+ if (!isa<ConstantSDNode>(LoLHS) && !isa<ConstantSDNode>(LoRHS))
+ return SDValue();
+
+ EVT VT = N->getValueType(0);
+ SDLoc DL(N);
+
+ SDValue LoCmp = DAG.getSetCC(DL, VT, LoLHS, LoRHS, ISD::SETULT);
+ if (isNullConstant(HiRHS)) {
+ SDValue HiEq = DAG.getSetCC(DL, VT, HiLHS, HiRHS, ISD::SETEQ);
+ return DAG.getNode(ISD::AND, DL, VT, HiEq, LoCmp);
+ }
+
+ SDValue HiNe = DAG.getSetCC(DL, VT, HiRHS,
+ DAG.getConstant(0, DL, HiRHS.getValueType()),
+ ISD::SETNE);
+ return DAG.getNode(ISD::OR, DL, VT, HiNe, LoCmp);
+}
+
// Replace a flag-setting operator (eg ANDS) with the generic version
// (eg AND) if the flag is unused.
static SDValue performFlagSettingCombine(SDNode *N,
@@ -29421,6 +29419,8 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performVSelectCombine(N, DCI, Subtarget);
case ISD::SETCC:
return performSETCCCombine(N, DCI, DAG);
+ case ISD::SETCCCARRY:
+ return performSETCCCARRYCombine(N, DAG);
case ISD::LOAD:
return performLOADCombine(N, DCI, DAG, Subtarget);
case ISD::STORE:
diff --git a/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll b/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
index 7422cbbfd19a0..eb414dbad3858 100644
--- a/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
+++ b/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
@@ -1,5 +1,7 @@
; RUN: llc -mtriple=aarch64-linux-gnu -O3 -o - < %s | FileCheck %s
+declare void @foo()
+
define i1 @eq_imm(i128 %x) {
; CHECK-LABEL: eq_imm:
; CHECK: cmp x0, #5
@@ -20,11 +22,111 @@ define i1 @ne_imm(i128 %x) {
ret i1 %cmp
}
+define i1 @eq_or_xor_zero_hi(i64 %lo, i64 %hi) {
+; CHECK-LABEL: eq_or_xor_zero_hi:
+; CHECK: cmp x0, #5
+; CHECK-NEXT: ccmp x1, #0, #0, eq
+; CHECK-NEXT: cset w0, eq
+; CHECK-NEXT: ret
+ %xorlo = xor i64 %lo, 5
+ %or = or i64 %xorlo, %hi
+ %cmp = icmp eq i64 %or, 0
+ ret i1 %cmp
+}
+
+define i1 @ne_or_xor_zero_hi(i64 %lo, i64 %hi) {
+; CHECK-LABEL: ne_or_xor_zero_hi:
+; CHECK: cmp x0, #5
+; CHECK-NEXT: ccmp x1, #0, #0, eq
+; CHECK-NEXT: cset w0, ne
+; CHECK-NEXT: ret
+ %xorlo = xor i64 %lo, 5
+ %or = or i64 %xorlo, %hi
+ %cmp = icmp ne i64 %or, 0
+ ret i1 %cmp
+}
+
+define void @eq_or_xor_large_cmp_imm_branch(i64 %lo, i64 %hi) {
+; CHECK-LABEL: eq_or_xor_large_cmp_imm_branch:
+; CHECK: eor x8, x1, #0x8000000000000000
+; CHECK-NEXT: orr x8, x0, x8
+; CHECK-NEXT: cbz x8, .LBB4_2
+ %xor = xor i64 %hi, -9223372036854775808
+ %or = or i64 %lo, %xor
+ %cmp = icmp eq i64 %or, 0
+ br i1 %cmp, label %true, label %false
+
+true:
+ call void @foo()
+ ret void
+
+false:
+ ret void
+}
+
+define void @eq_or_xor_large_condcmp_imm_branch(i64 %a, i64 %b, i64 %c, i64 %d) {
+; CHECK-LABEL: eq_or_xor_large_condcmp_imm_branch:
+; CHECK: eor x8, x3, #0xff
+; CHECK-NEXT: orr x9, x0, x1
+; CHECK-NEXT: orr x8, x2, x8
+; CHECK-NEXT: orr x8, x9, x8
+; CHECK-NEXT: cbz x8, .LBB5_2
+ %xor = xor i64 %d, 255
+ %or0 = or i64 %a, %b
+ %or1 = or i64 %c, %xor
+ %or = or i64 %or0, %or1
+ %cmp = icmp eq i64 %or, 0
+ br i1 %cmp, label %true, label %false
+
+true:
+ call void @foo()
+ ret void
+
+false:
+ ret void
+}
+
+define i1 @eq_or_xor_two_condcmp_imm_reordered(i64 %zero, i64 %value) {
+; CHECK-LABEL: eq_or_xor_two_condcmp_imm_reordered:
+; CHECK: cmp x1, #255
+; CHECK-NEXT: ccmp x0, #0, #0, eq
+; CHECK-NEXT: cset w0, eq
+; CHECK-NEXT: ret
+ %xor = xor i64 %value, 255
+ %or = or i64 %zero, %xor
+ %cmp = icmp eq i64 %or, 0
+ ret i1 %cmp
+}
+
+define void @eq_or_xor_multiple_large_condcmp_imm_branch(i64 %a, i64 %b, i64 %c, i64 %d) {
+; CHECK-LABEL: eq_or_xor_multiple_large_condcmp_imm_branch:
+; CHECK: eor x8, x1, #0xff
+; CHECK-NEXT: eor x9, x3, #0xffff
+; CHECK-NEXT: orr x8, x0, x8
+; CHECK-NEXT: orr x9, x2, x9
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: cbz x8, .LBB7_2
+ %xorb = xor i64 %b, 255
+ %xord = xor i64 %d, 65535
+ %or0 = or i64 %a, %xorb
+ %or1 = or i64 %c, %xord
+ %or = or i64 %or0, %or1
+ %cmp = icmp eq i64 %or, 0
+ br i1 %cmp, label %true, label %false
+
+true:
+ call void @foo()
+ ret void
+
+false:
+ ret void
+}
+
define i1 @ult_imm(i128 %x) {
; CHECK-LABEL: ult_imm:
; CHECK: cmp x1, #0
; CHECK-NEXT: ccmp x0, #5, #2, eq
-; CHECK-NEXT: cset w0, {{cc|lo}}
+; CHECK-NEXT: cset w0, lo
; CHECK-NEXT: ret
%cmp = icmp ult i128 %x, 5
ret i1 %cmp
@@ -34,7 +136,7 @@ define i1 @ule_imm(i128 %x) {
; CHECK-LABEL: ule_imm:
; CHECK: cmp x1, #0
; CHECK-NEXT: ccmp x0, #6, #2, eq
-; CHECK-NEXT: cset w0, {{cc|lo}}
+; CHECK-NEXT: cset w0, lo
; CHECK-NEXT: ret
%cmp = icmp ule i128 %x, 5
ret i1 %cmp
@@ -44,7 +146,7 @@ define i1 @ugt_imm(i128 %x) {
; CHECK-LABEL: ugt_imm:
; CHECK: cmp x1, #0
; CHECK-NEXT: ccmp x0, #5, #2, eq
-; CHECK-NEXT: cset w0, hi
+; CHECK-NEXT: cset w0, hi
; CHECK-NEXT: ret
%cmp = icmp ugt i128 %x, 5
ret i1 %cmp
@@ -52,10 +154,10 @@ define i1 @ugt_imm(i128 %x) {
define i1 @uge_imm(i128 %x) {
; CHECK-LABEL: uge_imm:
-; CHECK: cmp x1, #0
-; CHECK-NEXT: ccmp x0, #4, #2, eq
-; CHECK-NEXT: cset w0, hi
+; CHECK: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #4, #2, eq
+; CHECK-NEXT: cset w0, hi
; CHECK-NEXT: ret
%cmp = icmp uge i128 %x, 5
ret i1 %cmp
-}
\ No newline at end of file
+}
>From c20a1861cf0220453828b6b4450465e90cd6960e Mon Sep 17 00:00:00 2001
From: clingfei <1599101385 at qq.com>
Date: Fri, 8 May 2026 09:52:28 +0800
Subject: [PATCH 4/9] Refine eq/ne OR/XOR-chain CCMP immediate handling
---
.../Target/AArch64/AArch64ISelLowering.cpp | 124 ++++++---------
.../CodeGen/AArch64/i128-imm-compare-ccmp.ll | 142 ++++++++++++------
llvm/test/CodeGen/AArch64/isinf.ll | 14 +-
3 files changed, 145 insertions(+), 135 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ce9612f6b14c9..bdfdfc2193dc9 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -11871,59 +11871,66 @@ SDValue AArch64TargetLowering::LowerBitreverse(SDValue Op,
DAG.getNode(ISD::BITREVERSE, DL, VST, REVB));
}
-// Check whether the continuous comparison sequence.
-static bool hasLegalCmpImmediateOperand(SDValue V) {
- ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
- return !C || isLegalCmpImmed(C->getAPIntValue());
-}
-
-static bool hasLegalCmpImmediate(SDValue LHS, SDValue RHS) {
- return hasLegalCmpImmediateOperand(LHS) && hasLegalCmpImmediateOperand(RHS);
-}
-
+// A CCMP folds in only a 5-bit unsigned immediate (or its negation, via CCMN);
+// any other constant must be materialized into a register first.
static bool isLegalCondCmpImmediate(const APInt &Imm) {
if (Imm.isNegative())
return Imm.sgt(-32);
return Imm.getLimitedValue(32) <= 31;
}
-static bool hasLegalCondCmpImmediateOperand(SDValue V) {
- ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
- return !C || isLegalCondCmpImmediate(C->getAPIntValue());
+// True unless one of the operands is a constant that cannot be encoded as a
+// CMP/CMN immediate. A non-constant operand always lives in a register, so it
+// is fine.
+static bool hasLegalCmpImmediate(SDValue LHS, SDValue RHS) {
+ ConstantSDNode *C = dyn_cast<ConstantSDNode>(LHS);
+ if (C && !isLegalCmpImmed(C->getAPIntValue()))
+ return false;
+ C = dyn_cast<ConstantSDNode>(RHS);
+ if (C && !isLegalCmpImmed(C->getAPIntValue()))
+ return false;
+ return true;
}
+// As hasLegalCmpImmediate, but for the tighter CCMP immediate form.
static bool hasLegalCondCmpImmediate(SDValue LHS, SDValue RHS) {
- return hasLegalCondCmpImmediateOperand(LHS) &&
- hasLegalCondCmpImmediateOperand(RHS);
-}
-
-static bool isLegalXorImmediate(const APInt &Imm) {
- unsigned BitWidth = Imm.getBitWidth() <= 32 ? 32 : 64;
- if (Imm.getBitWidth() > BitWidth)
+ ConstantSDNode *C = dyn_cast<ConstantSDNode>(LHS);
+ if (C && !isLegalCondCmpImmediate(C->getAPIntValue()))
return false;
- return AArch64_AM::isLogicalImmediate(Imm.getZExtValue(), BitWidth);
-}
-
-static bool hasCheapXorImmediateThatNeedsCondCmpRegOperand(SDValue V) {
- ConstantSDNode *C = dyn_cast<ConstantSDNode>(V);
- if (!C)
+ C = dyn_cast<ConstantSDNode>(RHS);
+ if (C && !isLegalCondCmpImmediate(C->getAPIntValue()))
return false;
-
- const APInt &Imm = C->getAPIntValue();
- return !isLegalCondCmpImmediate(Imm) && isLegalXorImmediate(Imm);
-}
-
-static bool hasCheapXorImmediateThatNeedsCondCmpReg(
- const std::pair<SDValue, SDValue> &Pair) {
- return hasCheapXorImmediateThatNeedsCondCmpRegOperand(Pair.first) ||
- hasCheapXorImmediateThatNeedsCondCmpRegOperand(Pair.second);
+ return true;
}
+// Only the leading compare of the chain uses the wider CMP immediate; the rest
+// become CCMPs. So a compare whose immediate is a legal CMP operand but not a
+// legal CCMP operand is cheapest at the front.
static bool preferAsFirstCmp(const std::pair<SDValue, SDValue> &Pair) {
return hasLegalCmpImmediate(Pair.first, Pair.second) &&
!hasLegalCondCmpImmediate(Pair.first, Pair.second);
}
+// True if either operand is a constant that does not fit a CCMP immediate but
+// is a legal logical immediate: folding it into the xor is a single cheap
+// instruction, yet the result still needs a scratch register for the CCMP.
+// Chaining more than two such compares costs more than the CCMP form saves.
+static bool hasCheapXorImmediateThatNeedsCondCmpReg(
+ const std::pair<SDValue, SDValue> &Pair) {
+ for (SDValue V : {Pair.first, Pair.second}) {
+ auto *C = dyn_cast<ConstantSDNode>(V);
+ if (!C || isLegalCondCmpImmediate(C->getAPIntValue()))
+ continue;
+ const APInt &Imm = C->getAPIntValue();
+ unsigned BitWidth = Imm.getBitWidth() <= 32 ? 32 : 64;
+ if (Imm.getBitWidth() <= BitWidth &&
+ AArch64_AM::isLogicalImmediate(Imm.getZExtValue(), BitWidth))
+ return true;
+ }
+ return false;
+}
+
+// Check whether the continuous comparison sequence.
static bool
isOrXorChain(SDValue N, SelectionDAG &DAG, unsigned &Num, bool &SawXor,
bool RequireLegalCmpImmediates,
@@ -11984,11 +11991,10 @@ static SDValue performOrXorChainCombine(SDNode *N, SelectionDAG &DAG) {
// sub A0, A1; ccmp B0, B1, 0, eq; cmp inv(Cond) flag
unsigned NumXors = 0;
bool SawXor = false;
- bool RequireLegalCmpImmediates =
- any_of(N->users(), [](SDNode *User) {
- return User->getOpcode() == ISD::BRCOND ||
- User->getOpcode() == AArch64ISD::BRCOND;
- });
+ bool RequireLegalCmpImmediates = any_of(N->users(), [](SDNode *User) {
+ return User->getOpcode() == ISD::BRCOND ||
+ User->getOpcode() == AArch64ISD::BRCOND;
+ });
if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && isNullConstant(RHS) &&
LHS->getOpcode() == ISD::OR && LHS->hasOneUse() &&
isOrXorChain(LHS, DAG, NumXors, SawXor, RequireLegalCmpImmediates,
@@ -27654,42 +27660,6 @@ static SDValue performSETCCCombine(SDNode *N,
return SDValue();
}
-static SDValue performSETCCCARRYCombine(SDNode *N, SelectionDAG &DAG) {
- assert(N->getOpcode() == ISD::SETCCCARRY && "Unexpected opcode!");
-
- // Rebuild narrow high/low compares from type-legalized wide unsigned compares
- // so the existing CCMP conjunction/disjunction lowering can handle them.
- SDValue HiLHS = N->getOperand(0);
- SDValue HiRHS = N->getOperand(1);
- SDValue Carry = N->getOperand(2);
- ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(3))->get();
- if (Cond != ISD::SETULT || Carry.getOpcode() != ISD::USUBO ||
- Carry.getResNo() != 1)
- return SDValue();
-
- if (!isNullConstant(HiLHS) && !isNullConstant(HiRHS))
- return SDValue();
-
- SDValue LoLHS = Carry.getOperand(0);
- SDValue LoRHS = Carry.getOperand(1);
- if (!isa<ConstantSDNode>(LoLHS) && !isa<ConstantSDNode>(LoRHS))
- return SDValue();
-
- EVT VT = N->getValueType(0);
- SDLoc DL(N);
-
- SDValue LoCmp = DAG.getSetCC(DL, VT, LoLHS, LoRHS, ISD::SETULT);
- if (isNullConstant(HiRHS)) {
- SDValue HiEq = DAG.getSetCC(DL, VT, HiLHS, HiRHS, ISD::SETEQ);
- return DAG.getNode(ISD::AND, DL, VT, HiEq, LoCmp);
- }
-
- SDValue HiNe = DAG.getSetCC(DL, VT, HiRHS,
- DAG.getConstant(0, DL, HiRHS.getValueType()),
- ISD::SETNE);
- return DAG.getNode(ISD::OR, DL, VT, HiNe, LoCmp);
-}
-
static SDValue performSELECT_CCCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
SelectionDAG &DAG) {
@@ -29584,8 +29554,6 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performVSelectCombine(N, DCI, Subtarget);
case ISD::SETCC:
return performSETCCCombine(N, DCI, DAG);
- case ISD::SETCCCARRY:
- return performSETCCCARRYCombine(N, DAG);
case ISD::LOAD:
return performLOADCombine(N, DCI, DAG, Subtarget);
case ISD::STORE:
diff --git a/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll b/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
index eb414dbad3858..824d7871cd3d7 100644
--- a/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
+++ b/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
@@ -1,33 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=aarch64-linux-gnu -O3 -o - < %s | FileCheck %s
declare void @foo()
define i1 @eq_imm(i128 %x) {
; CHECK-LABEL: eq_imm:
-; CHECK: cmp x0, #5
-; CHECK-NEXT: ccmp x1, #0, #0, eq
-; CHECK-NEXT: cset w0, eq
-; CHECK-NEXT: ret
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, #5
+; CHECK-NEXT: ccmp x1, #0, #0, eq
+; CHECK-NEXT: cset w0, eq
+; CHECK-NEXT: ret
%cmp = icmp eq i128 %x, 5
ret i1 %cmp
}
define i1 @ne_imm(i128 %x) {
; CHECK-LABEL: ne_imm:
-; CHECK: cmp x0, #5
-; CHECK-NEXT: ccmp x1, #0, #0, eq
-; CHECK-NEXT: cset w0, ne
-; CHECK-NEXT: ret
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, #5
+; CHECK-NEXT: ccmp x1, #0, #0, eq
+; CHECK-NEXT: cset w0, ne
+; CHECK-NEXT: ret
%cmp = icmp ne i128 %x, 5
ret i1 %cmp
}
define i1 @eq_or_xor_zero_hi(i64 %lo, i64 %hi) {
; CHECK-LABEL: eq_or_xor_zero_hi:
-; CHECK: cmp x0, #5
-; CHECK-NEXT: ccmp x1, #0, #0, eq
-; CHECK-NEXT: cset w0, eq
-; CHECK-NEXT: ret
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, #5
+; CHECK-NEXT: ccmp x1, #0, #0, eq
+; CHECK-NEXT: cset w0, eq
+; CHECK-NEXT: ret
%xorlo = xor i64 %lo, 5
%or = or i64 %xorlo, %hi
%cmp = icmp eq i64 %or, 0
@@ -36,10 +40,11 @@ define i1 @eq_or_xor_zero_hi(i64 %lo, i64 %hi) {
define i1 @ne_or_xor_zero_hi(i64 %lo, i64 %hi) {
; CHECK-LABEL: ne_or_xor_zero_hi:
-; CHECK: cmp x0, #5
-; CHECK-NEXT: ccmp x1, #0, #0, eq
-; CHECK-NEXT: cset w0, ne
-; CHECK-NEXT: ret
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, #5
+; CHECK-NEXT: ccmp x1, #0, #0, eq
+; CHECK-NEXT: cset w0, ne
+; CHECK-NEXT: ret
%xorlo = xor i64 %lo, 5
%or = or i64 %xorlo, %hi
%cmp = icmp ne i64 %or, 0
@@ -48,9 +53,19 @@ define i1 @ne_or_xor_zero_hi(i64 %lo, i64 %hi) {
define void @eq_or_xor_large_cmp_imm_branch(i64 %lo, i64 %hi) {
; CHECK-LABEL: eq_or_xor_large_cmp_imm_branch:
-; CHECK: eor x8, x1, #0x8000000000000000
-; CHECK-NEXT: orr x8, x0, x8
-; CHECK-NEXT: cbz x8, .LBB4_2
+; CHECK: // %bb.0:
+; CHECK-NEXT: eor x8, x1, #0x8000000000000000
+; CHECK-NEXT: orr x8, x0, x8
+; CHECK-NEXT: cbz x8, .LBB4_2
+; CHECK-NEXT: // %bb.1: // %common.ret
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB4_2: // %true
+; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset w30, -16
+; CHECK-NEXT: bl foo
+; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
%xor = xor i64 %hi, -9223372036854775808
%or = or i64 %lo, %xor
%cmp = icmp eq i64 %or, 0
@@ -66,11 +81,21 @@ false:
define void @eq_or_xor_large_condcmp_imm_branch(i64 %a, i64 %b, i64 %c, i64 %d) {
; CHECK-LABEL: eq_or_xor_large_condcmp_imm_branch:
-; CHECK: eor x8, x3, #0xff
-; CHECK-NEXT: orr x9, x0, x1
-; CHECK-NEXT: orr x8, x2, x8
-; CHECK-NEXT: orr x8, x9, x8
-; CHECK-NEXT: cbz x8, .LBB5_2
+; CHECK: // %bb.0:
+; CHECK-NEXT: eor x8, x3, #0xff
+; CHECK-NEXT: orr x9, x0, x1
+; CHECK-NEXT: orr x8, x2, x8
+; CHECK-NEXT: orr x8, x9, x8
+; CHECK-NEXT: cbz x8, .LBB5_2
+; CHECK-NEXT: // %bb.1: // %common.ret
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB5_2: // %true
+; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset w30, -16
+; CHECK-NEXT: bl foo
+; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
%xor = xor i64 %d, 255
%or0 = or i64 %a, %b
%or1 = or i64 %c, %xor
@@ -88,10 +113,11 @@ false:
define i1 @eq_or_xor_two_condcmp_imm_reordered(i64 %zero, i64 %value) {
; CHECK-LABEL: eq_or_xor_two_condcmp_imm_reordered:
-; CHECK: cmp x1, #255
-; CHECK-NEXT: ccmp x0, #0, #0, eq
-; CHECK-NEXT: cset w0, eq
-; CHECK-NEXT: ret
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x1, #255
+; CHECK-NEXT: ccmp x0, #0, #0, eq
+; CHECK-NEXT: cset w0, eq
+; CHECK-NEXT: ret
%xor = xor i64 %value, 255
%or = or i64 %zero, %xor
%cmp = icmp eq i64 %or, 0
@@ -100,12 +126,22 @@ define i1 @eq_or_xor_two_condcmp_imm_reordered(i64 %zero, i64 %value) {
define void @eq_or_xor_multiple_large_condcmp_imm_branch(i64 %a, i64 %b, i64 %c, i64 %d) {
; CHECK-LABEL: eq_or_xor_multiple_large_condcmp_imm_branch:
-; CHECK: eor x8, x1, #0xff
-; CHECK-NEXT: eor x9, x3, #0xffff
-; CHECK-NEXT: orr x8, x0, x8
-; CHECK-NEXT: orr x9, x2, x9
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: cbz x8, .LBB7_2
+; CHECK: // %bb.0:
+; CHECK-NEXT: eor x8, x1, #0xff
+; CHECK-NEXT: eor x9, x3, #0xffff
+; CHECK-NEXT: orr x8, x0, x8
+; CHECK-NEXT: orr x9, x2, x9
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: cbz x8, .LBB7_2
+; CHECK-NEXT: // %bb.1: // %common.ret
+; CHECK-NEXT: ret
+; CHECK-NEXT: .LBB7_2: // %true
+; CHECK-NEXT: str x30, [sp, #-16]! // 8-byte Folded Spill
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: .cfi_offset w30, -16
+; CHECK-NEXT: bl foo
+; CHECK-NEXT: ldr x30, [sp], #16 // 8-byte Folded Reload
+; CHECK-NEXT: ret
%xorb = xor i64 %b, 255
%xord = xor i64 %d, 65535
%or0 = or i64 %a, %xorb
@@ -124,40 +160,46 @@ false:
define i1 @ult_imm(i128 %x) {
; CHECK-LABEL: ult_imm:
-; CHECK: cmp x1, #0
-; CHECK-NEXT: ccmp x0, #5, #2, eq
-; CHECK-NEXT: cset w0, lo
-; CHECK-NEXT: ret
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, #5
+; CHECK-NEXT: sbcs xzr, x1, xzr
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
%cmp = icmp ult i128 %x, 5
ret i1 %cmp
}
define i1 @ule_imm(i128 %x) {
; CHECK-LABEL: ule_imm:
-; CHECK: cmp x1, #0
-; CHECK-NEXT: ccmp x0, #6, #2, eq
-; CHECK-NEXT: cset w0, lo
-; CHECK-NEXT: ret
+; CHECK: // %bb.0:
+; CHECK-NEXT: cmp x0, #6
+; CHECK-NEXT: sbcs xzr, x1, xzr
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
%cmp = icmp ule i128 %x, 5
ret i1 %cmp
}
define i1 @ugt_imm(i128 %x) {
; CHECK-LABEL: ugt_imm:
-; CHECK: cmp x1, #0
-; CHECK-NEXT: ccmp x0, #5, #2, eq
-; CHECK-NEXT: cset w0, hi
-; CHECK-NEXT: ret
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov w8, #5 // =0x5
+; CHECK-NEXT: cmp x8, x0
+; CHECK-NEXT: ngcs xzr, x1
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
%cmp = icmp ugt i128 %x, 5
ret i1 %cmp
}
define i1 @uge_imm(i128 %x) {
; CHECK-LABEL: uge_imm:
-; CHECK: cmp x1, #0
-; CHECK-NEXT: ccmp x0, #4, #2, eq
-; CHECK-NEXT: cset w0, hi
-; CHECK-NEXT: ret
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov w8, #4 // =0x4
+; CHECK-NEXT: cmp x8, x0
+; CHECK-NEXT: ngcs xzr, x1
+; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: ret
%cmp = icmp uge i128 %x, 5
ret i1 %cmp
}
diff --git a/llvm/test/CodeGen/AArch64/isinf.ll b/llvm/test/CodeGen/AArch64/isinf.ll
index 92d53d7086e46..249a339f2de0a 100644
--- a/llvm/test/CodeGen/AArch64/isinf.ll
+++ b/llvm/test/CodeGen/AArch64/isinf.ll
@@ -80,14 +80,14 @@ define i32 @replace_isinf_call_f64(double %x) {
define i32 @replace_isinf_call_f128(fp128 %x) {
; CHECK-SD-LABEL: replace_isinf_call_f128:
; CHECK-SD: // %bb.0:
-; CHECK-SD-NEXT: str q0, [sp, #-16]!
+; CHECK-SD-NEXT: str q0, [sp, #-16]!
; CHECK-SD-NEXT: .cfi_def_cfa_offset 16
-; CHECK-SD-NEXT: mov x8, #-562949953421312 // =0xfffe000000000000
-; CHECK-SD-NEXT: ldp x10, x9, [sp], #16
-; CHECK-SD-NEXT: lsl x9, x9, #1
-; CHECK-SD-NEXT: cmp x10, #0
-; CHECK-SD-NEXT: ccmp x8, x9, #0, eq
-; CHECK-SD-NEXT: cset w0, eq
+; CHECK-SD-NEXT: mov x8, #-562949953421312 // =0xfffe000000000000
+; CHECK-SD-NEXT: ldp x10, x9, [sp], #16
+; CHECK-SD-NEXT: lsl x9, x9, #1
+; CHECK-SD-NEXT: cmp x10, #0
+; CHECK-SD-NEXT: ccmp x8, x9, #0, eq
+; CHECK-SD-NEXT: cset w0, eq
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: replace_isinf_call_f128:
>From 5020a2a43accdd4b6f56e4415ff0321decbe6bcb Mon Sep 17 00:00:00 2001
From: clingfei <chenglingfei at foxmail.com>
Date: Thu, 18 Jun 2026 10:29:33 +0800
Subject: [PATCH 5/9] update
---
.../Target/AArch64/AArch64ISelLowering.cpp | 22 +++++++++----------
1 file changed, 10 insertions(+), 12 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index ec4fa336b0b34..40af7f1ccfa46 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1157,7 +1157,6 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
// Try and combine setcc/select_cc with csel and bool-vector bitcasts.
setTargetDAGCombine(ISD::SETCC);
setTargetDAGCombine(ISD::SELECT_CC);
- setTargetDAGCombine(ISD::SETCCCARRY);
setTargetDAGCombine(ISD::INTRINSIC_WO_CHAIN);
@@ -12084,10 +12083,10 @@ static bool isLegalCondCmpImmediate(const APInt &Imm) {
// is fine.
static bool hasLegalCmpImmediate(SDValue LHS, SDValue RHS) {
ConstantSDNode *C = dyn_cast<ConstantSDNode>(LHS);
- if (C && !isLegalCmpImmed(C->getAPIntValue()))
+ if (C && !llvm::AArch64_AM::isLegalCmpImmed(C->getAPIntValue()))
return false;
C = dyn_cast<ConstantSDNode>(RHS);
- if (C && !isLegalCmpImmed(C->getAPIntValue()))
+ if (C && !llvm::AArch64_AM::isLegalCmpImmed(C->getAPIntValue()))
return false;
return true;
}
@@ -12103,14 +12102,6 @@ static bool hasLegalCondCmpImmediate(SDValue LHS, SDValue RHS) {
return true;
}
-// Only the leading compare of the chain uses the wider CMP immediate; the rest
-// become CCMPs. So a compare whose immediate is a legal CMP operand but not a
-// legal CCMP operand is cheapest at the front.
-static bool preferAsFirstCmp(const std::pair<SDValue, SDValue> &Pair) {
- return hasLegalCmpImmediate(Pair.first, Pair.second) &&
- !hasLegalCondCmpImmediate(Pair.first, Pair.second);
-}
-
// True if either operand is a constant that does not fit a CCMP immediate but
// is a legal logical immediate: folding it into the xor is a single cheap
// instruction, yet the result still needs a scratch register for the CCMP.
@@ -12204,8 +12195,15 @@ static SDValue performOrXorChainCombine(SDNode *N, SelectionDAG &DAG) {
any_of(WorkList, hasCheapXorImmediateThatNeedsCondCmpReg))
return SDValue();
+ // Only the leading compare of the chain uses the wider CMP immediate; the
+ // rest become CCMPs. So a compare whose immediate is a legal CMP operand
+ // but not a legal CCMP operand is cheapest at the front.
+ auto PreferAsFirstCmp = [](const std::pair<SDValue, SDValue> &Pair) {
+ return hasLegalCmpImmediate(Pair.first, Pair.second) &&
+ !hasLegalCondCmpImmediate(Pair.first, Pair.second);
+ };
SmallVector<std::pair<SDValue, SDValue>, 16>::iterator First =
- find_if(WorkList, preferAsFirstCmp);
+ find_if(WorkList, PreferAsFirstCmp);
if (First != WorkList.end())
std::iter_swap(WorkList.begin(), First);
>From 7470a9063041dce784a5c7020cb162a39b2a2d0d Mon Sep 17 00:00:00 2001
From: clingfei <chenglingfei at foxmail.com>
Date: Wed, 24 Jun 2026 22:03:23 +0800
Subject: [PATCH 6/9] Simplify isLegalCondCmpImmediate and restrain the length
of ccmp chain
---
.../Target/AArch64/AArch64ISelLowering.cpp | 10 ++-
llvm/test/CodeGen/AArch64/bcmp.ll | 80 +++++++++++--------
2 files changed, 52 insertions(+), 38 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 40af7f1ccfa46..f5791a5c93efc 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -12073,9 +12073,7 @@ SDValue AArch64TargetLowering::LowerBitreverse(SDValue Op,
// A CCMP folds in only a 5-bit unsigned immediate (or its negation, via CCMN);
// any other constant must be materialized into a register first.
static bool isLegalCondCmpImmediate(const APInt &Imm) {
- if (Imm.isNegative())
- return Imm.sgt(-32);
- return Imm.getLimitedValue(32) <= 31;
+ return Imm.sgt(-32) && Imm.slt(32);
}
// True unless one of the operands is a constant that cannot be encoded as a
@@ -12191,6 +12189,12 @@ static SDValue performOrXorChainCombine(SDNode *N, SelectionDAG &DAG) {
isOrXorChain(LHS, DAG, NumXors, SawXor, RequireLegalCmpImmediates,
WorkList) &&
SawXor) {
+ // A CCMP sequence serializes the comparisons through NZCV. Keep the
+ // transform to short chains where the instruction-count reduction outweighs
+ // the longer dependency chain.
+ if (WorkList.size() > 5)
+ return SDValue();
+
if (WorkList.size() > 2 &&
any_of(WorkList, hasCheapXorImmediateThatNeedsCondCmpReg))
return SDValue();
diff --git a/llvm/test/CodeGen/AArch64/bcmp.ll b/llvm/test/CodeGen/AArch64/bcmp.ll
index 436a3c2375b52..3379790797f11 100644
--- a/llvm/test/CodeGen/AArch64/bcmp.ll
+++ b/llvm/test/CodeGen/AArch64/bcmp.ll
@@ -359,20 +359,26 @@ define i1 @bcmp38(ptr %a, ptr %b) {
define i1 @bcmp45(ptr %a, ptr %b) {
; CHECK-LABEL: bcmp45:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldp x8, x11, [x1]
-; CHECK-NEXT: ldp x9, x10, [x0]
-; CHECK-NEXT: ldp x12, x13, [x1, #16]
-; CHECK-NEXT: cmp x9, x8
-; CHECK-NEXT: ldp x8, x9, [x0, #16]
-; CHECK-NEXT: ccmp x10, x11, #0, eq
+; CHECK-NEXT: ldp x8, x9, [x0]
+; CHECK-NEXT: ldr x16, [x1, #32]
+; CHECK-NEXT: ldp x10, x11, [x1]
+; CHECK-NEXT: ldur x17, [x1, #37]
+; CHECK-NEXT: ldp x12, x13, [x0, #16]
+; CHECK-NEXT: ldp x14, x15, [x1, #16]
+; CHECK-NEXT: eor x8, x8, x10
+; CHECK-NEXT: eor x9, x9, x11
; CHECK-NEXT: ldr x10, [x0, #32]
-; CHECK-NEXT: ldr x11, [x1, #32]
-; CHECK-NEXT: ccmp x8, x12, #0, eq
-; CHECK-NEXT: ldur x8, [x0, #37]
-; CHECK-NEXT: ldur x12, [x1, #37]
-; CHECK-NEXT: ccmp x9, x13, #0, eq
-; CHECK-NEXT: ccmp x10, x11, #0, eq
-; CHECK-NEXT: ccmp x8, x12, #0, eq
+; CHECK-NEXT: ldur x11, [x0, #37]
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: eor x12, x12, x14
+; CHECK-NEXT: eor x13, x13, x15
+; CHECK-NEXT: eor x10, x10, x16
+; CHECK-NEXT: eor x11, x11, x17
+; CHECK-NEXT: orr x9, x12, x13
+; CHECK-NEXT: orr x10, x10, x11
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: orr x8, x8, x10
+; CHECK-NEXT: cmp x8, #0
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
%cr = call i32 @bcmp(ptr %a, ptr %b, i64 45)
@@ -380,31 +386,35 @@ define i1 @bcmp45(ptr %a, ptr %b) {
ret i1 %r
}
-; Although the large cmp chain may be not profitable on high end CPU, we
-; believe it is better on most cpus, so perform the transform now.
-; 8 xor + 7 or + 1 cmp only need 6 cycles on a 4 width ALU port machine
-; 2 cycle for xor
-; 3 cycle for or
-; 1 cycle for cmp
+; Avoid turning large xor/or reductions into long CCMP chains. The CCMP form
+; saves instructions but serializes the whole chain through NZCV.
define i1 @bcmp64(ptr %a, ptr %b) {
; CHECK-LABEL: bcmp64:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldp x8, x11, [x1]
-; CHECK-NEXT: ldp x9, x10, [x0]
-; CHECK-NEXT: ldp x12, x13, [x1, #16]
-; CHECK-NEXT: cmp x9, x8
-; CHECK-NEXT: ldp x8, x9, [x0, #16]
-; CHECK-NEXT: ccmp x10, x11, #0, eq
-; CHECK-NEXT: ccmp x8, x12, #0, eq
-; CHECK-NEXT: ldp x8, x11, [x0, #32]
-; CHECK-NEXT: ldp x10, x12, [x1, #32]
-; CHECK-NEXT: ccmp x9, x13, #0, eq
-; CHECK-NEXT: ldp x9, x13, [x1, #48]
-; CHECK-NEXT: ccmp x8, x10, #0, eq
-; CHECK-NEXT: ldp x8, x10, [x0, #48]
-; CHECK-NEXT: ccmp x11, x12, #0, eq
-; CHECK-NEXT: ccmp x8, x9, #0, eq
-; CHECK-NEXT: ccmp x10, x13, #0, eq
+; CHECK-NEXT: ldp x8, x9, [x0]
+; CHECK-NEXT: ldp x10, x11, [x1]
+; CHECK-NEXT: ldp x12, x13, [x0, #16]
+; CHECK-NEXT: ldp x14, x15, [x1, #16]
+; CHECK-NEXT: eor x8, x8, x10
+; CHECK-NEXT: eor x9, x9, x11
+; CHECK-NEXT: ldp x17, x18, [x0, #48]
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: eor x10, x12, x14
+; CHECK-NEXT: ldp x11, x12, [x0, #32]
+; CHECK-NEXT: ldp x14, x16, [x1, #32]
+; CHECK-NEXT: eor x13, x13, x15
+; CHECK-NEXT: ldp x0, x1, [x1, #48]
+; CHECK-NEXT: orr x9, x10, x13
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: eor x11, x11, x14
+; CHECK-NEXT: eor x12, x12, x16
+; CHECK-NEXT: eor x14, x17, x0
+; CHECK-NEXT: eor x15, x18, x1
+; CHECK-NEXT: orr x10, x11, x12
+; CHECK-NEXT: orr x11, x14, x15
+; CHECK-NEXT: orr x9, x10, x11
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: cmp x8, #0
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
%cr = call i32 @bcmp(ptr %a, ptr %b, i64 64)
>From 080ee599920ceb4409c35f86fc74b359afe82937 Mon Sep 17 00:00:00 2001
From: clingfei <chenglingfei at foxmail.com>
Date: Thu, 25 Jun 2026 13:41:53 +0800
Subject: [PATCH 7/9] Update test cases
---
.../CodeGen/AArch64/i128-imm-compare-ccmp.ll | 22 +-
llvm/test/CodeGen/AArch64/memcmp.ll | 326 ++++++++++--------
.../umulo-128-legalisation-lowering.ll | 6 +-
3 files changed, 201 insertions(+), 153 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll b/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
index 824d7871cd3d7..77e697781c006 100644
--- a/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
+++ b/llvm/test/CodeGen/AArch64/i128-imm-compare-ccmp.ll
@@ -161,8 +161,8 @@ false:
define i1 @ult_imm(i128 %x) {
; CHECK-LABEL: ult_imm:
; CHECK: // %bb.0:
-; CHECK-NEXT: cmp x0, #5
-; CHECK-NEXT: sbcs xzr, x1, xzr
+; CHECK-NEXT: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #5, #2, eq
; CHECK-NEXT: cset w0, lo
; CHECK-NEXT: ret
%cmp = icmp ult i128 %x, 5
@@ -172,8 +172,8 @@ define i1 @ult_imm(i128 %x) {
define i1 @ule_imm(i128 %x) {
; CHECK-LABEL: ule_imm:
; CHECK: // %bb.0:
-; CHECK-NEXT: cmp x0, #6
-; CHECK-NEXT: sbcs xzr, x1, xzr
+; CHECK-NEXT: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #6, #2, eq
; CHECK-NEXT: cset w0, lo
; CHECK-NEXT: ret
%cmp = icmp ule i128 %x, 5
@@ -183,10 +183,9 @@ define i1 @ule_imm(i128 %x) {
define i1 @ugt_imm(i128 %x) {
; CHECK-LABEL: ugt_imm:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #5 // =0x5
-; CHECK-NEXT: cmp x8, x0
-; CHECK-NEXT: ngcs xzr, x1
-; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #5, #2, eq
+; CHECK-NEXT: cset w0, hi
; CHECK-NEXT: ret
%cmp = icmp ugt i128 %x, 5
ret i1 %cmp
@@ -195,10 +194,9 @@ define i1 @ugt_imm(i128 %x) {
define i1 @uge_imm(i128 %x) {
; CHECK-LABEL: uge_imm:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #4 // =0x4
-; CHECK-NEXT: cmp x8, x0
-; CHECK-NEXT: ngcs xzr, x1
-; CHECK-NEXT: cset w0, lo
+; CHECK-NEXT: cmp x1, #0
+; CHECK-NEXT: ccmp x0, #4, #2, eq
+; CHECK-NEXT: cset w0, hi
; CHECK-NEXT: ret
%cmp = icmp uge i128 %x, 5
ret i1 %cmp
diff --git a/llvm/test/CodeGen/AArch64/memcmp.ll b/llvm/test/CodeGen/AArch64/memcmp.ll
index 6b702c54b183e..92ff1c024a24f 100644
--- a/llvm/test/CodeGen/AArch64/memcmp.ll
+++ b/llvm/test/CodeGen/AArch64/memcmp.ll
@@ -1599,18 +1599,24 @@ define i32 @length48(ptr %X, ptr %Y) nounwind {
define i1 @length48_eq(ptr %x, ptr %y) nounwind {
; CHECK-LABEL: length48_eq:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldp x8, x11, [x1]
-; CHECK-NEXT: ldp x9, x10, [x0]
-; CHECK-NEXT: ldp x12, x13, [x1, #16]
-; CHECK-NEXT: cmp x9, x8
-; CHECK-NEXT: ldp x8, x9, [x0, #16]
-; CHECK-NEXT: ccmp x10, x11, #0, eq
-; CHECK-NEXT: ccmp x8, x12, #0, eq
-; CHECK-NEXT: ldp x8, x11, [x0, #32]
-; CHECK-NEXT: ldp x10, x12, [x1, #32]
-; CHECK-NEXT: ccmp x9, x13, #0, eq
-; CHECK-NEXT: ccmp x8, x10, #0, eq
-; CHECK-NEXT: ccmp x11, x12, #0, eq
+; CHECK-NEXT: ldp x8, x9, [x0]
+; CHECK-NEXT: ldp x10, x11, [x1]
+; CHECK-NEXT: ldp x12, x13, [x0, #16]
+; CHECK-NEXT: ldp x14, x15, [x1, #16]
+; CHECK-NEXT: eor x8, x8, x10
+; CHECK-NEXT: ldp x16, x17, [x1, #32]
+; CHECK-NEXT: eor x9, x9, x11
+; CHECK-NEXT: ldp x10, x11, [x0, #32]
+; CHECK-NEXT: eor x12, x12, x14
+; CHECK-NEXT: eor x13, x13, x15
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: orr x9, x12, x13
+; CHECK-NEXT: eor x10, x10, x16
+; CHECK-NEXT: eor x11, x11, x17
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: orr x10, x10, x11
+; CHECK-NEXT: orr x8, x8, x10
+; CHECK-NEXT: cmp x8, #0
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
%call = tail call i32 @memcmp(ptr %x, ptr %y, i64 48) nounwind
@@ -1739,18 +1745,24 @@ define i1 @length48_gt(ptr %x, ptr %y) nounwind {
define i1 @length48_eq_prefer128(ptr %x, ptr %y) nounwind "prefer-vector-width"="128" {
; CHECK-LABEL: length48_eq_prefer128:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldp x8, x11, [x1]
-; CHECK-NEXT: ldp x9, x10, [x0]
-; CHECK-NEXT: ldp x12, x13, [x1, #16]
-; CHECK-NEXT: cmp x9, x8
-; CHECK-NEXT: ldp x8, x9, [x0, #16]
-; CHECK-NEXT: ccmp x10, x11, #0, eq
-; CHECK-NEXT: ccmp x8, x12, #0, eq
-; CHECK-NEXT: ldp x8, x11, [x0, #32]
-; CHECK-NEXT: ldp x10, x12, [x1, #32]
-; CHECK-NEXT: ccmp x9, x13, #0, eq
-; CHECK-NEXT: ccmp x8, x10, #0, eq
-; CHECK-NEXT: ccmp x11, x12, #0, eq
+; CHECK-NEXT: ldp x8, x9, [x0]
+; CHECK-NEXT: ldp x10, x11, [x1]
+; CHECK-NEXT: ldp x12, x13, [x0, #16]
+; CHECK-NEXT: ldp x14, x15, [x1, #16]
+; CHECK-NEXT: eor x8, x8, x10
+; CHECK-NEXT: ldp x16, x17, [x1, #32]
+; CHECK-NEXT: eor x9, x9, x11
+; CHECK-NEXT: ldp x10, x11, [x0, #32]
+; CHECK-NEXT: eor x12, x12, x14
+; CHECK-NEXT: eor x13, x13, x15
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: orr x9, x12, x13
+; CHECK-NEXT: eor x10, x10, x16
+; CHECK-NEXT: eor x11, x11, x17
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: orr x10, x10, x11
+; CHECK-NEXT: orr x8, x8, x10
+; CHECK-NEXT: cmp x8, #0
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
%call = tail call i32 @memcmp(ptr %x, ptr %y, i64 48) nounwind
@@ -1761,35 +1773,41 @@ define i1 @length48_eq_prefer128(ptr %x, ptr %y) nounwind "prefer-vector-width"=
define i1 @length48_eq_const(ptr %X) nounwind {
; CHECK-LABEL: length48_eq_const:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov x8, #12592 // =0x3130
-; CHECK-NEXT: ldp x9, x10, [x0]
-; CHECK-NEXT: movk x8, #13106, lsl #16
-; CHECK-NEXT: ldp x11, x12, [x0, #16]
-; CHECK-NEXT: movk x8, #13620, lsl #32
-; CHECK-NEXT: movk x8, #14134, lsl #48
-; CHECK-NEXT: cmp x9, x8
+; CHECK-NEXT: mov x12, #14134 // =0x3736
; CHECK-NEXT: mov x9, #14648 // =0x3938
+; CHECK-NEXT: mov x8, #12592 // =0x3130
+; CHECK-NEXT: movk x12, #14648, lsl #16
+; CHECK-NEXT: ldp x14, x15, [x0, #16]
; CHECK-NEXT: movk x9, #12592, lsl #16
+; CHECK-NEXT: ldp x10, x11, [x0]
+; CHECK-NEXT: movk x12, #12592, lsl #32
; CHECK-NEXT: movk x9, #13106, lsl #32
+; CHECK-NEXT: mov x13, #13620 // =0x3534
+; CHECK-NEXT: movk x12, #13106, lsl #48
+; CHECK-NEXT: movk x8, #13106, lsl #16
; CHECK-NEXT: movk x9, #13620, lsl #48
-; CHECK-NEXT: ccmp x10, x9, #0, eq
-; CHECK-NEXT: mov x9, #14134 // =0x3736
-; CHECK-NEXT: movk x9, #14648, lsl #16
-; CHECK-NEXT: movk x9, #12592, lsl #32
-; CHECK-NEXT: movk x9, #13106, lsl #48
-; CHECK-NEXT: ccmp x11, x9, #0, eq
-; CHECK-NEXT: mov x9, #13620 // =0x3534
-; CHECK-NEXT: movk x9, #14134, lsl #16
-; CHECK-NEXT: ldp x10, x11, [x0, #32]
-; CHECK-NEXT: movk x9, #14648, lsl #32
-; CHECK-NEXT: movk x9, #12592, lsl #48
-; CHECK-NEXT: ccmp x12, x9, #0, eq
-; CHECK-NEXT: mov x9, #13106 // =0x3332
-; CHECK-NEXT: movk x9, #13620, lsl #16
-; CHECK-NEXT: movk x9, #14134, lsl #32
-; CHECK-NEXT: movk x9, #14648, lsl #48
-; CHECK-NEXT: ccmp x10, x9, #0, eq
-; CHECK-NEXT: ccmp x11, x8, #0, eq
+; CHECK-NEXT: movk x13, #14134, lsl #16
+; CHECK-NEXT: eor x12, x14, x12
+; CHECK-NEXT: mov x14, #13106 // =0x3332
+; CHECK-NEXT: movk x8, #13620, lsl #32
+; CHECK-NEXT: movk x13, #14648, lsl #32
+; CHECK-NEXT: eor x9, x11, x9
+; CHECK-NEXT: ldp x11, x16, [x0, #32]
+; CHECK-NEXT: movk x14, #13620, lsl #16
+; CHECK-NEXT: movk x8, #14134, lsl #48
+; CHECK-NEXT: movk x13, #12592, lsl #48
+; CHECK-NEXT: movk x14, #14134, lsl #32
+; CHECK-NEXT: movk x14, #14648, lsl #48
+; CHECK-NEXT: eor x13, x15, x13
+; CHECK-NEXT: eor x10, x10, x8
+; CHECK-NEXT: eor x11, x11, x14
+; CHECK-NEXT: eor x8, x16, x8
+; CHECK-NEXT: orr x9, x10, x9
+; CHECK-NEXT: orr x10, x12, x13
+; CHECK-NEXT: orr x8, x11, x8
+; CHECK-NEXT: orr x9, x9, x10
+; CHECK-NEXT: orr x8, x9, x8
+; CHECK-NEXT: cmp x8, #0
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%m = tail call i32 @memcmp(ptr %X, ptr @.str, i64 48) nounwind
@@ -1870,24 +1888,32 @@ define i32 @length63(ptr %X, ptr %Y) nounwind {
define i1 @length63_eq(ptr %x, ptr %y) nounwind {
; CHECK-LABEL: length63_eq:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldp x8, x11, [x1]
-; CHECK-NEXT: ldp x9, x10, [x0]
-; CHECK-NEXT: ldp x12, x13, [x1, #16]
-; CHECK-NEXT: cmp x9, x8
-; CHECK-NEXT: ldp x8, x9, [x0, #16]
-; CHECK-NEXT: ccmp x10, x11, #0, eq
-; CHECK-NEXT: ccmp x8, x12, #0, eq
-; CHECK-NEXT: ldp x8, x11, [x0, #32]
-; CHECK-NEXT: ldp x10, x12, [x1, #32]
-; CHECK-NEXT: ccmp x9, x13, #0, eq
-; CHECK-NEXT: ldr x9, [x0, #48]
-; CHECK-NEXT: ldr x13, [x1, #48]
-; CHECK-NEXT: ccmp x8, x10, #0, eq
-; CHECK-NEXT: ldur x8, [x0, #55]
-; CHECK-NEXT: ldur x10, [x1, #55]
-; CHECK-NEXT: ccmp x11, x12, #0, eq
-; CHECK-NEXT: ccmp x9, x13, #0, eq
-; CHECK-NEXT: ccmp x8, x10, #0, eq
+; CHECK-NEXT: ldp x8, x9, [x0]
+; CHECK-NEXT: ldr x17, [x0, #48]
+; CHECK-NEXT: ldp x10, x11, [x1]
+; CHECK-NEXT: ldr x18, [x1, #48]
+; CHECK-NEXT: ldp x12, x13, [x0, #16]
+; CHECK-NEXT: ldp x14, x15, [x1, #16]
+; CHECK-NEXT: eor x8, x8, x10
+; CHECK-NEXT: eor x9, x9, x11
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: eor x10, x12, x14
+; CHECK-NEXT: ldp x11, x12, [x0, #32]
+; CHECK-NEXT: ldp x14, x16, [x1, #32]
+; CHECK-NEXT: ldur x0, [x0, #55]
+; CHECK-NEXT: ldur x1, [x1, #55]
+; CHECK-NEXT: eor x13, x13, x15
+; CHECK-NEXT: orr x9, x10, x13
+; CHECK-NEXT: eor x11, x11, x14
+; CHECK-NEXT: eor x12, x12, x16
+; CHECK-NEXT: eor x14, x17, x18
+; CHECK-NEXT: eor x15, x0, x1
+; CHECK-NEXT: orr x10, x11, x12
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: orr x11, x14, x15
+; CHECK-NEXT: orr x9, x10, x11
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: cmp x8, #0
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%call = tail call i32 @memcmp(ptr %x, ptr %y, i64 63) nounwind
@@ -2044,43 +2070,51 @@ define i1 @length63_gt(ptr %x, ptr %y) nounwind {
define i1 @length63_eq_const(ptr %X) nounwind {
; CHECK-LABEL: length63_eq_const:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov x10, #14134 // =0x3736
+; CHECK-NEXT: mov x13, #13620 // =0x3534
; CHECK-NEXT: mov x8, #12592 // =0x3130
-; CHECK-NEXT: ldp x9, x10, [x0]
+; CHECK-NEXT: movk x10, #14648, lsl #16
+; CHECK-NEXT: ldp x12, x15, [x0, #16]
+; CHECK-NEXT: movk x13, #14134, lsl #16
+; CHECK-NEXT: movk x10, #12592, lsl #32
; CHECK-NEXT: movk x8, #13106, lsl #16
-; CHECK-NEXT: ldp x11, x12, [x0, #16]
+; CHECK-NEXT: movk x13, #14648, lsl #32
+; CHECK-NEXT: movk x10, #13106, lsl #48
; CHECK-NEXT: movk x8, #13620, lsl #32
+; CHECK-NEXT: movk x13, #12592, lsl #48
+; CHECK-NEXT: ldp x11, x14, [x0]
+; CHECK-NEXT: eor x10, x12, x10
+; CHECK-NEXT: eor x12, x15, x13
+; CHECK-NEXT: mov x13, #13106 // =0x3332
+; CHECK-NEXT: ldp x16, x15, [x0, #32]
; CHECK-NEXT: movk x8, #14134, lsl #48
-; CHECK-NEXT: cmp x9, x8
+; CHECK-NEXT: movk x13, #13620, lsl #16
; CHECK-NEXT: mov x9, #14648 // =0x3938
+; CHECK-NEXT: eor x11, x11, x8
+; CHECK-NEXT: movk x13, #14134, lsl #32
; CHECK-NEXT: movk x9, #12592, lsl #16
+; CHECK-NEXT: ldur x17, [x0, #55]
+; CHECK-NEXT: eor x8, x15, x8
+; CHECK-NEXT: mov x15, #13877 // =0x3635
+; CHECK-NEXT: movk x13, #14648, lsl #48
+; CHECK-NEXT: movk x15, #14391, lsl #16
; CHECK-NEXT: movk x9, #13106, lsl #32
+; CHECK-NEXT: eor x13, x16, x13
+; CHECK-NEXT: ldr x16, [x0, #48]
+; CHECK-NEXT: movk x15, #12345, lsl #32
; CHECK-NEXT: movk x9, #13620, lsl #48
-; CHECK-NEXT: ccmp x10, x9, #0, eq
-; CHECK-NEXT: mov x10, #14134 // =0x3736
-; CHECK-NEXT: movk x10, #14648, lsl #16
-; CHECK-NEXT: movk x10, #12592, lsl #32
-; CHECK-NEXT: movk x10, #13106, lsl #48
-; CHECK-NEXT: ccmp x11, x10, #0, eq
-; CHECK-NEXT: mov x10, #13620 // =0x3534
-; CHECK-NEXT: movk x10, #14134, lsl #16
-; CHECK-NEXT: ldp x11, x13, [x0, #32]
-; CHECK-NEXT: movk x10, #14648, lsl #32
-; CHECK-NEXT: movk x10, #12592, lsl #48
-; CHECK-NEXT: ccmp x12, x10, #0, eq
-; CHECK-NEXT: mov x10, #13106 // =0x3332
-; CHECK-NEXT: ldr x12, [x0, #48]
-; CHECK-NEXT: movk x10, #13620, lsl #16
-; CHECK-NEXT: movk x10, #14134, lsl #32
-; CHECK-NEXT: movk x10, #14648, lsl #48
-; CHECK-NEXT: ccmp x11, x10, #0, eq
-; CHECK-NEXT: ldur x10, [x0, #55]
-; CHECK-NEXT: ccmp x13, x8, #0, eq
-; CHECK-NEXT: mov x8, #13877 // =0x3635
-; CHECK-NEXT: movk x8, #14391, lsl #16
-; CHECK-NEXT: ccmp x12, x9, #0, eq
-; CHECK-NEXT: movk x8, #12345, lsl #32
-; CHECK-NEXT: movk x8, #12849, lsl #48
-; CHECK-NEXT: ccmp x10, x8, #0, eq
+; CHECK-NEXT: movk x15, #12849, lsl #48
+; CHECK-NEXT: eor x14, x14, x9
+; CHECK-NEXT: orr x10, x10, x12
+; CHECK-NEXT: eor x9, x16, x9
+; CHECK-NEXT: eor x15, x17, x15
+; CHECK-NEXT: orr x11, x11, x14
+; CHECK-NEXT: orr x8, x13, x8
+; CHECK-NEXT: orr x9, x9, x15
+; CHECK-NEXT: orr x10, x11, x10
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: orr x8, x10, x8
+; CHECK-NEXT: cmp x8, #0
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
%m = tail call i32 @memcmp(ptr %X, ptr @.str, i64 63) nounwind
@@ -2161,22 +2195,30 @@ define i32 @length64(ptr %X, ptr %Y) nounwind {
define i1 @length64_eq(ptr %x, ptr %y) nounwind {
; CHECK-LABEL: length64_eq:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldp x8, x11, [x1]
-; CHECK-NEXT: ldp x9, x10, [x0]
-; CHECK-NEXT: ldp x12, x13, [x1, #16]
-; CHECK-NEXT: cmp x9, x8
-; CHECK-NEXT: ldp x8, x9, [x0, #16]
-; CHECK-NEXT: ccmp x10, x11, #0, eq
-; CHECK-NEXT: ccmp x8, x12, #0, eq
-; CHECK-NEXT: ldp x8, x11, [x0, #32]
-; CHECK-NEXT: ldp x10, x12, [x1, #32]
-; CHECK-NEXT: ccmp x9, x13, #0, eq
-; CHECK-NEXT: ldp x9, x13, [x1, #48]
-; CHECK-NEXT: ccmp x8, x10, #0, eq
-; CHECK-NEXT: ldp x8, x10, [x0, #48]
-; CHECK-NEXT: ccmp x11, x12, #0, eq
-; CHECK-NEXT: ccmp x8, x9, #0, eq
-; CHECK-NEXT: ccmp x10, x13, #0, eq
+; CHECK-NEXT: ldp x8, x9, [x0]
+; CHECK-NEXT: ldp x10, x11, [x1]
+; CHECK-NEXT: ldp x12, x13, [x0, #16]
+; CHECK-NEXT: ldp x14, x15, [x1, #16]
+; CHECK-NEXT: eor x8, x8, x10
+; CHECK-NEXT: eor x9, x9, x11
+; CHECK-NEXT: ldp x17, x18, [x0, #48]
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: eor x10, x12, x14
+; CHECK-NEXT: ldp x11, x12, [x0, #32]
+; CHECK-NEXT: ldp x14, x16, [x1, #32]
+; CHECK-NEXT: eor x13, x13, x15
+; CHECK-NEXT: ldp x0, x1, [x1, #48]
+; CHECK-NEXT: orr x9, x10, x13
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: eor x11, x11, x14
+; CHECK-NEXT: eor x12, x12, x16
+; CHECK-NEXT: eor x14, x17, x0
+; CHECK-NEXT: eor x15, x18, x1
+; CHECK-NEXT: orr x10, x11, x12
+; CHECK-NEXT: orr x11, x14, x15
+; CHECK-NEXT: orr x9, x10, x11
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: cmp x8, #0
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%call = tail call i32 @memcmp(ptr %x, ptr %y, i64 64) nounwind
@@ -2333,38 +2375,46 @@ define i1 @length64_gt(ptr %x, ptr %y) nounwind {
define i1 @length64_eq_const(ptr %X) nounwind {
; CHECK-LABEL: length64_eq_const:
; CHECK: // %bb.0:
+; CHECK-NEXT: mov x12, #13620 // =0x3534
+; CHECK-NEXT: ldp x13, x14, [x0, #16]
+; CHECK-NEXT: movk x12, #14134, lsl #16
+; CHECK-NEXT: ldp x16, x17, [x0, #32]
+; CHECK-NEXT: movk x12, #14648, lsl #32
; CHECK-NEXT: mov x8, #12592 // =0x3130
-; CHECK-NEXT: ldp x9, x10, [x0]
-; CHECK-NEXT: movk x8, #13106, lsl #16
-; CHECK-NEXT: ldp x11, x12, [x0, #16]
-; CHECK-NEXT: movk x8, #13620, lsl #32
-; CHECK-NEXT: ldp x13, x14, [x0, #32]
-; CHECK-NEXT: movk x8, #14134, lsl #48
-; CHECK-NEXT: cmp x9, x8
; CHECK-NEXT: mov x9, #14648 // =0x3938
-; CHECK-NEXT: movk x9, #12592, lsl #16
-; CHECK-NEXT: movk x9, #13106, lsl #32
-; CHECK-NEXT: movk x9, #13620, lsl #48
-; CHECK-NEXT: ccmp x10, x9, #0, eq
+; CHECK-NEXT: movk x12, #12592, lsl #48
; CHECK-NEXT: mov x10, #14134 // =0x3736
+; CHECK-NEXT: movk x8, #13106, lsl #16
+; CHECK-NEXT: eor x12, x14, x12
+; CHECK-NEXT: mov x14, #13106 // =0x3332
+; CHECK-NEXT: movk x9, #12592, lsl #16
+; CHECK-NEXT: movk x14, #13620, lsl #16
; CHECK-NEXT: movk x10, #14648, lsl #16
+; CHECK-NEXT: movk x8, #13620, lsl #32
+; CHECK-NEXT: movk x14, #14134, lsl #32
+; CHECK-NEXT: ldp x11, x15, [x0]
+; CHECK-NEXT: movk x14, #14648, lsl #48
+; CHECK-NEXT: movk x9, #13106, lsl #32
; CHECK-NEXT: movk x10, #12592, lsl #32
+; CHECK-NEXT: eor x14, x16, x14
+; CHECK-NEXT: ldp x16, x18, [x0, #48]
+; CHECK-NEXT: movk x8, #14134, lsl #48
+; CHECK-NEXT: movk x9, #13620, lsl #48
; CHECK-NEXT: movk x10, #13106, lsl #48
-; CHECK-NEXT: ccmp x11, x10, #0, eq
-; CHECK-NEXT: mov x11, #13620 // =0x3534
-; CHECK-NEXT: movk x11, #14134, lsl #16
-; CHECK-NEXT: movk x11, #14648, lsl #32
-; CHECK-NEXT: movk x11, #12592, lsl #48
-; CHECK-NEXT: ccmp x12, x11, #0, eq
-; CHECK-NEXT: mov x11, #13106 // =0x3332
-; CHECK-NEXT: movk x11, #13620, lsl #16
-; CHECK-NEXT: movk x11, #14134, lsl #32
-; CHECK-NEXT: movk x11, #14648, lsl #48
-; CHECK-NEXT: ccmp x13, x11, #0, eq
-; CHECK-NEXT: ldp x11, x12, [x0, #48]
-; CHECK-NEXT: ccmp x14, x8, #0, eq
-; CHECK-NEXT: ccmp x11, x9, #0, eq
-; CHECK-NEXT: ccmp x12, x10, #0, eq
+; CHECK-NEXT: eor x11, x11, x8
+; CHECK-NEXT: eor x8, x17, x8
+; CHECK-NEXT: eor x15, x15, x9
+; CHECK-NEXT: eor x13, x13, x10
+; CHECK-NEXT: eor x9, x16, x9
+; CHECK-NEXT: eor x10, x18, x10
+; CHECK-NEXT: orr x11, x11, x15
+; CHECK-NEXT: orr x12, x13, x12
+; CHECK-NEXT: orr x8, x14, x8
+; CHECK-NEXT: orr x9, x9, x10
+; CHECK-NEXT: orr x10, x11, x12
+; CHECK-NEXT: orr x8, x8, x9
+; CHECK-NEXT: orr x8, x10, x8
+; CHECK-NEXT: cmp x8, #0
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
%m = tail call i32 @memcmp(ptr %X, ptr @.str, i64 64) nounwind
diff --git a/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll b/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll
index d7348ea154b24..f9993e53c318a 100644
--- a/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll
+++ b/llvm/test/CodeGen/AArch64/umulo-128-legalisation-lowering.ll
@@ -81,9 +81,9 @@ define i128 @__muloti4(i128 %0, i128 %1, ptr nocapture nonnull writeonly align 4
; AARCH-NEXT: mov w9, wzr
; AARCH-NEXT: mul x0, x0, x2
; AARCH-NEXT: .LBB1_3: // %overflow.res
-; AARCH-NEXT: eor x10, x3, #0x8000000000000000
-; AARCH-NEXT: orr x10, x2, x10
-; AARCH-NEXT: cmp x10, #0
+; AARCH-NEXT: mov x10, #-9223372036854775808 // =0x8000000000000000
+; AARCH-NEXT: cmp x3, x10
+; AARCH-NEXT: ccmp x2, #0, #0, eq
; AARCH-NEXT: ccmp x1, #0, #0, eq
; AARCH-NEXT: b.mi .LBB1_5
; AARCH-NEXT: // %bb.4: // %Else2
>From 13e92b8f022f1a8bb44d6ab1a46b8769b63830fc Mon Sep 17 00:00:00 2001
From: clingfei <chenglingfei at foxmail.com>
Date: Mon, 6 Jul 2026 17:44:36 +0800
Subject: [PATCH 8/9] Relax optsize
---
.../Target/AArch64/AArch64ISelLowering.cpp | 33 +++--
llvm/test/CodeGen/AArch64/bcmp.ll | 32 ++---
llvm/test/CodeGen/AArch64/memcmp.ll | 116 ++++++++----------
3 files changed, 83 insertions(+), 98 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index c8837ee5c9e5f..2fa392f0761be 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -12219,10 +12219,10 @@ static bool hasCheapXorImmediateThatNeedsCondCmpReg(
// Check whether the continuous comparison sequence.
static bool
-isOrXorChain(SDValue N, SelectionDAG &DAG, unsigned &Num, bool &SawXor,
- bool RequireLegalCmpImmediates,
+isOrXorChain(SDValue N, SelectionDAG &DAG, unsigned &NumLeaves,
+ unsigned &NumXors, bool &SawXor, bool RequireLegalCmpImmediates,
SmallVectorImpl<std::pair<SDValue, SDValue>> &WorkList) {
- if (Num == MaxXors)
+ if (NumLeaves == MaxXors)
return false;
// Skip the one-use zext
@@ -12234,16 +12234,17 @@ isOrXorChain(SDValue N, SelectionDAG &DAG, unsigned &Num, bool &SawXor,
!hasLegalCmpImmediate(N->getOperand(0), N->getOperand(1)))
return false;
WorkList.push_back(std::make_pair(N->getOperand(0), N->getOperand(1)));
- Num++;
+ NumLeaves++;
+ NumXors++;
SawXor = true;
return true;
}
// All the non-leaf nodes must be OR.
if (N->getOpcode() == ISD::OR && N->hasOneUse())
- return isOrXorChain(N->getOperand(0), DAG, Num, SawXor,
+ return isOrXorChain(N->getOperand(0), DAG, NumLeaves, NumXors, SawXor,
RequireLegalCmpImmediates, WorkList) &&
- isOrXorChain(N->getOperand(1), DAG, Num, SawXor,
+ isOrXorChain(N->getOperand(1), DAG, NumLeaves, NumXors, SawXor,
RequireLegalCmpImmediates, WorkList);
if (N->getOpcode() == ISD::OR)
return false;
@@ -12257,7 +12258,7 @@ isOrXorChain(SDValue N, SelectionDAG &DAG, unsigned &Num, bool &SawXor,
// type-legalized wide integer equality compares converge to the same SETCC
// tree.
WorkList.push_back(std::make_pair(N, DAG.getConstant(0, SDLoc(N), VT)));
- Num++;
+ NumLeaves++;
return true;
}
@@ -12276,6 +12277,7 @@ static SDValue performOrXorChainCombine(SDNode *N, SelectionDAG &DAG) {
ISD::CondCode Cond = cast<CondCodeSDNode>(N->getOperand(2))->get();
// Try to express conjunction "cmp 0 (or (xor A0 A1) (xor B0 B1))" as:
// sub A0, A1; ccmp B0, B1, 0, eq; cmp inv(Cond) flag
+ unsigned NumLeaves = 0;
unsigned NumXors = 0;
bool SawXor = false;
bool RequireLegalCmpImmediates = any_of(N->users(), [](SDNode *User) {
@@ -12284,13 +12286,20 @@ static SDValue performOrXorChainCombine(SDNode *N, SelectionDAG &DAG) {
});
if ((Cond == ISD::SETEQ || Cond == ISD::SETNE) && isNullConstant(RHS) &&
LHS->getOpcode() == ISD::OR && LHS->hasOneUse() &&
- isOrXorChain(LHS, DAG, NumXors, SawXor, RequireLegalCmpImmediates,
- WorkList) &&
+ isOrXorChain(LHS, DAG, NumLeaves, NumXors, SawXor,
+ RequireLegalCmpImmediates, WorkList) &&
SawXor) {
// A CCMP sequence serializes the comparisons through NZCV. Keep the
- // transform to short chains where the instruction-count reduction outweighs
- // the longer dependency chain.
- if (WorkList.size() > 5)
+ // default transform to short chains, but account for real XOR leaves: each
+ // one removed is a code-size and front-end win. Under size optimization,
+ // prefer the smaller CCMP form unless another guard rejects it.
+ const Function &F = DAG.getMachineFunction().getFunction();
+ unsigned Limit = 5;
+ if (NumXors >= 6)
+ Limit = 6;
+ if (F.hasOptSize() || F.hasMinSize())
+ Limit = MaxXors;
+ if (WorkList.size() > Limit)
return SDValue();
if (WorkList.size() > 2 &&
diff --git a/llvm/test/CodeGen/AArch64/bcmp.ll b/llvm/test/CodeGen/AArch64/bcmp.ll
index 3379790797f11..c6ea5a5b189b2 100644
--- a/llvm/test/CodeGen/AArch64/bcmp.ll
+++ b/llvm/test/CodeGen/AArch64/bcmp.ll
@@ -359,26 +359,20 @@ define i1 @bcmp38(ptr %a, ptr %b) {
define i1 @bcmp45(ptr %a, ptr %b) {
; CHECK-LABEL: bcmp45:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldp x8, x9, [x0]
-; CHECK-NEXT: ldr x16, [x1, #32]
-; CHECK-NEXT: ldp x10, x11, [x1]
-; CHECK-NEXT: ldur x17, [x1, #37]
-; CHECK-NEXT: ldp x12, x13, [x0, #16]
-; CHECK-NEXT: ldp x14, x15, [x1, #16]
-; CHECK-NEXT: eor x8, x8, x10
-; CHECK-NEXT: eor x9, x9, x11
+; CHECK-NEXT: ldp x8, x11, [x1]
+; CHECK-NEXT: ldp x9, x10, [x0]
+; CHECK-NEXT: ldp x12, x13, [x1, #16]
+; CHECK-NEXT: cmp x9, x8
+; CHECK-NEXT: ldp x8, x9, [x0, #16]
+; CHECK-NEXT: ccmp x10, x11, #0, eq
; CHECK-NEXT: ldr x10, [x0, #32]
-; CHECK-NEXT: ldur x11, [x0, #37]
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: eor x12, x12, x14
-; CHECK-NEXT: eor x13, x13, x15
-; CHECK-NEXT: eor x10, x10, x16
-; CHECK-NEXT: eor x11, x11, x17
-; CHECK-NEXT: orr x9, x12, x13
-; CHECK-NEXT: orr x10, x10, x11
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: orr x8, x8, x10
-; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: ldr x11, [x1, #32]
+; CHECK-NEXT: ccmp x8, x12, #0, eq
+; CHECK-NEXT: ldur x8, [x0, #37]
+; CHECK-NEXT: ldur x12, [x1, #37]
+; CHECK-NEXT: ccmp x9, x13, #0, eq
+; CHECK-NEXT: ccmp x10, x11, #0, eq
+; CHECK-NEXT: ccmp x8, x12, #0, eq
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
%cr = call i32 @bcmp(ptr %a, ptr %b, i64 45)
diff --git a/llvm/test/CodeGen/AArch64/memcmp.ll b/llvm/test/CodeGen/AArch64/memcmp.ll
index 92ff1c024a24f..2ab944f63e045 100644
--- a/llvm/test/CodeGen/AArch64/memcmp.ll
+++ b/llvm/test/CodeGen/AArch64/memcmp.ll
@@ -1599,24 +1599,18 @@ define i32 @length48(ptr %X, ptr %Y) nounwind {
define i1 @length48_eq(ptr %x, ptr %y) nounwind {
; CHECK-LABEL: length48_eq:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldp x8, x9, [x0]
-; CHECK-NEXT: ldp x10, x11, [x1]
-; CHECK-NEXT: ldp x12, x13, [x0, #16]
-; CHECK-NEXT: ldp x14, x15, [x1, #16]
-; CHECK-NEXT: eor x8, x8, x10
-; CHECK-NEXT: ldp x16, x17, [x1, #32]
-; CHECK-NEXT: eor x9, x9, x11
-; CHECK-NEXT: ldp x10, x11, [x0, #32]
-; CHECK-NEXT: eor x12, x12, x14
-; CHECK-NEXT: eor x13, x13, x15
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: orr x9, x12, x13
-; CHECK-NEXT: eor x10, x10, x16
-; CHECK-NEXT: eor x11, x11, x17
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: orr x10, x10, x11
-; CHECK-NEXT: orr x8, x8, x10
-; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: ldp x8, x11, [x1]
+; CHECK-NEXT: ldp x9, x10, [x0]
+; CHECK-NEXT: ldp x12, x13, [x1, #16]
+; CHECK-NEXT: cmp x9, x8
+; CHECK-NEXT: ldp x8, x9, [x0, #16]
+; CHECK-NEXT: ccmp x10, x11, #0, eq
+; CHECK-NEXT: ccmp x8, x12, #0, eq
+; CHECK-NEXT: ldp x8, x11, [x0, #32]
+; CHECK-NEXT: ldp x10, x12, [x1, #32]
+; CHECK-NEXT: ccmp x9, x13, #0, eq
+; CHECK-NEXT: ccmp x8, x10, #0, eq
+; CHECK-NEXT: ccmp x11, x12, #0, eq
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
%call = tail call i32 @memcmp(ptr %x, ptr %y, i64 48) nounwind
@@ -1745,24 +1739,18 @@ define i1 @length48_gt(ptr %x, ptr %y) nounwind {
define i1 @length48_eq_prefer128(ptr %x, ptr %y) nounwind "prefer-vector-width"="128" {
; CHECK-LABEL: length48_eq_prefer128:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldp x8, x9, [x0]
-; CHECK-NEXT: ldp x10, x11, [x1]
-; CHECK-NEXT: ldp x12, x13, [x0, #16]
-; CHECK-NEXT: ldp x14, x15, [x1, #16]
-; CHECK-NEXT: eor x8, x8, x10
-; CHECK-NEXT: ldp x16, x17, [x1, #32]
-; CHECK-NEXT: eor x9, x9, x11
-; CHECK-NEXT: ldp x10, x11, [x0, #32]
-; CHECK-NEXT: eor x12, x12, x14
-; CHECK-NEXT: eor x13, x13, x15
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: orr x9, x12, x13
-; CHECK-NEXT: eor x10, x10, x16
-; CHECK-NEXT: eor x11, x11, x17
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: orr x10, x10, x11
-; CHECK-NEXT: orr x8, x8, x10
-; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: ldp x8, x11, [x1]
+; CHECK-NEXT: ldp x9, x10, [x0]
+; CHECK-NEXT: ldp x12, x13, [x1, #16]
+; CHECK-NEXT: cmp x9, x8
+; CHECK-NEXT: ldp x8, x9, [x0, #16]
+; CHECK-NEXT: ccmp x10, x11, #0, eq
+; CHECK-NEXT: ccmp x8, x12, #0, eq
+; CHECK-NEXT: ldp x8, x11, [x0, #32]
+; CHECK-NEXT: ldp x10, x12, [x1, #32]
+; CHECK-NEXT: ccmp x9, x13, #0, eq
+; CHECK-NEXT: ccmp x8, x10, #0, eq
+; CHECK-NEXT: ccmp x11, x12, #0, eq
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
%call = tail call i32 @memcmp(ptr %x, ptr %y, i64 48) nounwind
@@ -1773,41 +1761,35 @@ define i1 @length48_eq_prefer128(ptr %x, ptr %y) nounwind "prefer-vector-width"=
define i1 @length48_eq_const(ptr %X) nounwind {
; CHECK-LABEL: length48_eq_const:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov x12, #14134 // =0x3736
-; CHECK-NEXT: mov x9, #14648 // =0x3938
; CHECK-NEXT: mov x8, #12592 // =0x3130
-; CHECK-NEXT: movk x12, #14648, lsl #16
-; CHECK-NEXT: ldp x14, x15, [x0, #16]
-; CHECK-NEXT: movk x9, #12592, lsl #16
-; CHECK-NEXT: ldp x10, x11, [x0]
-; CHECK-NEXT: movk x12, #12592, lsl #32
-; CHECK-NEXT: movk x9, #13106, lsl #32
-; CHECK-NEXT: mov x13, #13620 // =0x3534
-; CHECK-NEXT: movk x12, #13106, lsl #48
+; CHECK-NEXT: ldp x9, x10, [x0]
; CHECK-NEXT: movk x8, #13106, lsl #16
-; CHECK-NEXT: movk x9, #13620, lsl #48
-; CHECK-NEXT: movk x13, #14134, lsl #16
-; CHECK-NEXT: eor x12, x14, x12
-; CHECK-NEXT: mov x14, #13106 // =0x3332
+; CHECK-NEXT: ldp x11, x12, [x0, #16]
; CHECK-NEXT: movk x8, #13620, lsl #32
-; CHECK-NEXT: movk x13, #14648, lsl #32
-; CHECK-NEXT: eor x9, x11, x9
-; CHECK-NEXT: ldp x11, x16, [x0, #32]
-; CHECK-NEXT: movk x14, #13620, lsl #16
; CHECK-NEXT: movk x8, #14134, lsl #48
-; CHECK-NEXT: movk x13, #12592, lsl #48
-; CHECK-NEXT: movk x14, #14134, lsl #32
-; CHECK-NEXT: movk x14, #14648, lsl #48
-; CHECK-NEXT: eor x13, x15, x13
-; CHECK-NEXT: eor x10, x10, x8
-; CHECK-NEXT: eor x11, x11, x14
-; CHECK-NEXT: eor x8, x16, x8
-; CHECK-NEXT: orr x9, x10, x9
-; CHECK-NEXT: orr x10, x12, x13
-; CHECK-NEXT: orr x8, x11, x8
-; CHECK-NEXT: orr x9, x9, x10
-; CHECK-NEXT: orr x8, x9, x8
-; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: cmp x9, x8
+; CHECK-NEXT: mov x9, #14648 // =0x3938
+; CHECK-NEXT: movk x9, #12592, lsl #16
+; CHECK-NEXT: movk x9, #13106, lsl #32
+; CHECK-NEXT: movk x9, #13620, lsl #48
+; CHECK-NEXT: ccmp x10, x9, #0, eq
+; CHECK-NEXT: mov x9, #14134 // =0x3736
+; CHECK-NEXT: movk x9, #14648, lsl #16
+; CHECK-NEXT: movk x9, #12592, lsl #32
+; CHECK-NEXT: movk x9, #13106, lsl #48
+; CHECK-NEXT: ccmp x11, x9, #0, eq
+; CHECK-NEXT: mov x9, #13620 // =0x3534
+; CHECK-NEXT: movk x9, #14134, lsl #16
+; CHECK-NEXT: ldp x10, x11, [x0, #32]
+; CHECK-NEXT: movk x9, #14648, lsl #32
+; CHECK-NEXT: movk x9, #12592, lsl #48
+; CHECK-NEXT: ccmp x12, x9, #0, eq
+; CHECK-NEXT: mov x9, #13106 // =0x3332
+; CHECK-NEXT: movk x9, #13620, lsl #16
+; CHECK-NEXT: movk x9, #14134, lsl #32
+; CHECK-NEXT: movk x9, #14648, lsl #48
+; CHECK-NEXT: ccmp x10, x9, #0, eq
+; CHECK-NEXT: ccmp x11, x8, #0, eq
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%m = tail call i32 @memcmp(ptr %X, ptr @.str, i64 48) nounwind
>From d99cb09bbda4e648ab95aba6961936d114f0623e Mon Sep 17 00:00:00 2001
From: clingfei <chenglingfei at foxmail.com>
Date: Mon, 6 Jul 2026 18:56:02 +0800
Subject: [PATCH 9/9] Relax optsize
---
.../Target/AArch64/AArch64ISelLowering.cpp | 5 +-
llvm/test/CodeGen/AArch64/bcmp.ll | 48 ++--
llvm/test/CodeGen/AArch64/memcmp.ll | 210 ++++++++----------
3 files changed, 115 insertions(+), 148 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 2fa392f0761be..a9ece3188dd94 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -12295,8 +12295,11 @@ static SDValue performOrXorChainCombine(SDNode *N, SelectionDAG &DAG) {
// prefer the smaller CCMP form unless another guard rejects it.
const Function &F = DAG.getMachineFunction().getFunction();
unsigned Limit = 5;
- if (NumXors >= 6)
+ if (NumXors >= 6) {
Limit = 6;
+ if (NumXors == NumLeaves)
+ Limit = std::min<unsigned>(8, NumXors);
+ }
if (F.hasOptSize() || F.hasMinSize())
Limit = MaxXors;
if (WorkList.size() > Limit)
diff --git a/llvm/test/CodeGen/AArch64/bcmp.ll b/llvm/test/CodeGen/AArch64/bcmp.ll
index c6ea5a5b189b2..436a3c2375b52 100644
--- a/llvm/test/CodeGen/AArch64/bcmp.ll
+++ b/llvm/test/CodeGen/AArch64/bcmp.ll
@@ -380,35 +380,31 @@ define i1 @bcmp45(ptr %a, ptr %b) {
ret i1 %r
}
-; Avoid turning large xor/or reductions into long CCMP chains. The CCMP form
-; saves instructions but serializes the whole chain through NZCV.
+; Although the large cmp chain may be not profitable on high end CPU, we
+; believe it is better on most cpus, so perform the transform now.
+; 8 xor + 7 or + 1 cmp only need 6 cycles on a 4 width ALU port machine
+; 2 cycle for xor
+; 3 cycle for or
+; 1 cycle for cmp
define i1 @bcmp64(ptr %a, ptr %b) {
; CHECK-LABEL: bcmp64:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldp x8, x9, [x0]
-; CHECK-NEXT: ldp x10, x11, [x1]
-; CHECK-NEXT: ldp x12, x13, [x0, #16]
-; CHECK-NEXT: ldp x14, x15, [x1, #16]
-; CHECK-NEXT: eor x8, x8, x10
-; CHECK-NEXT: eor x9, x9, x11
-; CHECK-NEXT: ldp x17, x18, [x0, #48]
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: eor x10, x12, x14
-; CHECK-NEXT: ldp x11, x12, [x0, #32]
-; CHECK-NEXT: ldp x14, x16, [x1, #32]
-; CHECK-NEXT: eor x13, x13, x15
-; CHECK-NEXT: ldp x0, x1, [x1, #48]
-; CHECK-NEXT: orr x9, x10, x13
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: eor x11, x11, x14
-; CHECK-NEXT: eor x12, x12, x16
-; CHECK-NEXT: eor x14, x17, x0
-; CHECK-NEXT: eor x15, x18, x1
-; CHECK-NEXT: orr x10, x11, x12
-; CHECK-NEXT: orr x11, x14, x15
-; CHECK-NEXT: orr x9, x10, x11
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: ldp x8, x11, [x1]
+; CHECK-NEXT: ldp x9, x10, [x0]
+; CHECK-NEXT: ldp x12, x13, [x1, #16]
+; CHECK-NEXT: cmp x9, x8
+; CHECK-NEXT: ldp x8, x9, [x0, #16]
+; CHECK-NEXT: ccmp x10, x11, #0, eq
+; CHECK-NEXT: ccmp x8, x12, #0, eq
+; CHECK-NEXT: ldp x8, x11, [x0, #32]
+; CHECK-NEXT: ldp x10, x12, [x1, #32]
+; CHECK-NEXT: ccmp x9, x13, #0, eq
+; CHECK-NEXT: ldp x9, x13, [x1, #48]
+; CHECK-NEXT: ccmp x8, x10, #0, eq
+; CHECK-NEXT: ldp x8, x10, [x0, #48]
+; CHECK-NEXT: ccmp x11, x12, #0, eq
+; CHECK-NEXT: ccmp x8, x9, #0, eq
+; CHECK-NEXT: ccmp x10, x13, #0, eq
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
%cr = call i32 @bcmp(ptr %a, ptr %b, i64 64)
diff --git a/llvm/test/CodeGen/AArch64/memcmp.ll b/llvm/test/CodeGen/AArch64/memcmp.ll
index 2ab944f63e045..6b702c54b183e 100644
--- a/llvm/test/CodeGen/AArch64/memcmp.ll
+++ b/llvm/test/CodeGen/AArch64/memcmp.ll
@@ -1870,32 +1870,24 @@ define i32 @length63(ptr %X, ptr %Y) nounwind {
define i1 @length63_eq(ptr %x, ptr %y) nounwind {
; CHECK-LABEL: length63_eq:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldp x8, x9, [x0]
-; CHECK-NEXT: ldr x17, [x0, #48]
-; CHECK-NEXT: ldp x10, x11, [x1]
-; CHECK-NEXT: ldr x18, [x1, #48]
-; CHECK-NEXT: ldp x12, x13, [x0, #16]
-; CHECK-NEXT: ldp x14, x15, [x1, #16]
-; CHECK-NEXT: eor x8, x8, x10
-; CHECK-NEXT: eor x9, x9, x11
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: eor x10, x12, x14
-; CHECK-NEXT: ldp x11, x12, [x0, #32]
-; CHECK-NEXT: ldp x14, x16, [x1, #32]
-; CHECK-NEXT: ldur x0, [x0, #55]
-; CHECK-NEXT: ldur x1, [x1, #55]
-; CHECK-NEXT: eor x13, x13, x15
-; CHECK-NEXT: orr x9, x10, x13
-; CHECK-NEXT: eor x11, x11, x14
-; CHECK-NEXT: eor x12, x12, x16
-; CHECK-NEXT: eor x14, x17, x18
-; CHECK-NEXT: eor x15, x0, x1
-; CHECK-NEXT: orr x10, x11, x12
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: orr x11, x14, x15
-; CHECK-NEXT: orr x9, x10, x11
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: ldp x8, x11, [x1]
+; CHECK-NEXT: ldp x9, x10, [x0]
+; CHECK-NEXT: ldp x12, x13, [x1, #16]
+; CHECK-NEXT: cmp x9, x8
+; CHECK-NEXT: ldp x8, x9, [x0, #16]
+; CHECK-NEXT: ccmp x10, x11, #0, eq
+; CHECK-NEXT: ccmp x8, x12, #0, eq
+; CHECK-NEXT: ldp x8, x11, [x0, #32]
+; CHECK-NEXT: ldp x10, x12, [x1, #32]
+; CHECK-NEXT: ccmp x9, x13, #0, eq
+; CHECK-NEXT: ldr x9, [x0, #48]
+; CHECK-NEXT: ldr x13, [x1, #48]
+; CHECK-NEXT: ccmp x8, x10, #0, eq
+; CHECK-NEXT: ldur x8, [x0, #55]
+; CHECK-NEXT: ldur x10, [x1, #55]
+; CHECK-NEXT: ccmp x11, x12, #0, eq
+; CHECK-NEXT: ccmp x9, x13, #0, eq
+; CHECK-NEXT: ccmp x8, x10, #0, eq
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%call = tail call i32 @memcmp(ptr %x, ptr %y, i64 63) nounwind
@@ -2052,51 +2044,43 @@ define i1 @length63_gt(ptr %x, ptr %y) nounwind {
define i1 @length63_eq_const(ptr %X) nounwind {
; CHECK-LABEL: length63_eq_const:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov x10, #14134 // =0x3736
-; CHECK-NEXT: mov x13, #13620 // =0x3534
; CHECK-NEXT: mov x8, #12592 // =0x3130
-; CHECK-NEXT: movk x10, #14648, lsl #16
-; CHECK-NEXT: ldp x12, x15, [x0, #16]
-; CHECK-NEXT: movk x13, #14134, lsl #16
-; CHECK-NEXT: movk x10, #12592, lsl #32
+; CHECK-NEXT: ldp x9, x10, [x0]
; CHECK-NEXT: movk x8, #13106, lsl #16
-; CHECK-NEXT: movk x13, #14648, lsl #32
-; CHECK-NEXT: movk x10, #13106, lsl #48
+; CHECK-NEXT: ldp x11, x12, [x0, #16]
; CHECK-NEXT: movk x8, #13620, lsl #32
-; CHECK-NEXT: movk x13, #12592, lsl #48
-; CHECK-NEXT: ldp x11, x14, [x0]
-; CHECK-NEXT: eor x10, x12, x10
-; CHECK-NEXT: eor x12, x15, x13
-; CHECK-NEXT: mov x13, #13106 // =0x3332
-; CHECK-NEXT: ldp x16, x15, [x0, #32]
; CHECK-NEXT: movk x8, #14134, lsl #48
-; CHECK-NEXT: movk x13, #13620, lsl #16
+; CHECK-NEXT: cmp x9, x8
; CHECK-NEXT: mov x9, #14648 // =0x3938
-; CHECK-NEXT: eor x11, x11, x8
-; CHECK-NEXT: movk x13, #14134, lsl #32
; CHECK-NEXT: movk x9, #12592, lsl #16
-; CHECK-NEXT: ldur x17, [x0, #55]
-; CHECK-NEXT: eor x8, x15, x8
-; CHECK-NEXT: mov x15, #13877 // =0x3635
-; CHECK-NEXT: movk x13, #14648, lsl #48
-; CHECK-NEXT: movk x15, #14391, lsl #16
; CHECK-NEXT: movk x9, #13106, lsl #32
-; CHECK-NEXT: eor x13, x16, x13
-; CHECK-NEXT: ldr x16, [x0, #48]
-; CHECK-NEXT: movk x15, #12345, lsl #32
; CHECK-NEXT: movk x9, #13620, lsl #48
-; CHECK-NEXT: movk x15, #12849, lsl #48
-; CHECK-NEXT: eor x14, x14, x9
-; CHECK-NEXT: orr x10, x10, x12
-; CHECK-NEXT: eor x9, x16, x9
-; CHECK-NEXT: eor x15, x17, x15
-; CHECK-NEXT: orr x11, x11, x14
-; CHECK-NEXT: orr x8, x13, x8
-; CHECK-NEXT: orr x9, x9, x15
-; CHECK-NEXT: orr x10, x11, x10
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: orr x8, x10, x8
-; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: ccmp x10, x9, #0, eq
+; CHECK-NEXT: mov x10, #14134 // =0x3736
+; CHECK-NEXT: movk x10, #14648, lsl #16
+; CHECK-NEXT: movk x10, #12592, lsl #32
+; CHECK-NEXT: movk x10, #13106, lsl #48
+; CHECK-NEXT: ccmp x11, x10, #0, eq
+; CHECK-NEXT: mov x10, #13620 // =0x3534
+; CHECK-NEXT: movk x10, #14134, lsl #16
+; CHECK-NEXT: ldp x11, x13, [x0, #32]
+; CHECK-NEXT: movk x10, #14648, lsl #32
+; CHECK-NEXT: movk x10, #12592, lsl #48
+; CHECK-NEXT: ccmp x12, x10, #0, eq
+; CHECK-NEXT: mov x10, #13106 // =0x3332
+; CHECK-NEXT: ldr x12, [x0, #48]
+; CHECK-NEXT: movk x10, #13620, lsl #16
+; CHECK-NEXT: movk x10, #14134, lsl #32
+; CHECK-NEXT: movk x10, #14648, lsl #48
+; CHECK-NEXT: ccmp x11, x10, #0, eq
+; CHECK-NEXT: ldur x10, [x0, #55]
+; CHECK-NEXT: ccmp x13, x8, #0, eq
+; CHECK-NEXT: mov x8, #13877 // =0x3635
+; CHECK-NEXT: movk x8, #14391, lsl #16
+; CHECK-NEXT: ccmp x12, x9, #0, eq
+; CHECK-NEXT: movk x8, #12345, lsl #32
+; CHECK-NEXT: movk x8, #12849, lsl #48
+; CHECK-NEXT: ccmp x10, x8, #0, eq
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
%m = tail call i32 @memcmp(ptr %X, ptr @.str, i64 63) nounwind
@@ -2177,30 +2161,22 @@ define i32 @length64(ptr %X, ptr %Y) nounwind {
define i1 @length64_eq(ptr %x, ptr %y) nounwind {
; CHECK-LABEL: length64_eq:
; CHECK: // %bb.0:
-; CHECK-NEXT: ldp x8, x9, [x0]
-; CHECK-NEXT: ldp x10, x11, [x1]
-; CHECK-NEXT: ldp x12, x13, [x0, #16]
-; CHECK-NEXT: ldp x14, x15, [x1, #16]
-; CHECK-NEXT: eor x8, x8, x10
-; CHECK-NEXT: eor x9, x9, x11
-; CHECK-NEXT: ldp x17, x18, [x0, #48]
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: eor x10, x12, x14
-; CHECK-NEXT: ldp x11, x12, [x0, #32]
-; CHECK-NEXT: ldp x14, x16, [x1, #32]
-; CHECK-NEXT: eor x13, x13, x15
-; CHECK-NEXT: ldp x0, x1, [x1, #48]
-; CHECK-NEXT: orr x9, x10, x13
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: eor x11, x11, x14
-; CHECK-NEXT: eor x12, x12, x16
-; CHECK-NEXT: eor x14, x17, x0
-; CHECK-NEXT: eor x15, x18, x1
-; CHECK-NEXT: orr x10, x11, x12
-; CHECK-NEXT: orr x11, x14, x15
-; CHECK-NEXT: orr x9, x10, x11
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: ldp x8, x11, [x1]
+; CHECK-NEXT: ldp x9, x10, [x0]
+; CHECK-NEXT: ldp x12, x13, [x1, #16]
+; CHECK-NEXT: cmp x9, x8
+; CHECK-NEXT: ldp x8, x9, [x0, #16]
+; CHECK-NEXT: ccmp x10, x11, #0, eq
+; CHECK-NEXT: ccmp x8, x12, #0, eq
+; CHECK-NEXT: ldp x8, x11, [x0, #32]
+; CHECK-NEXT: ldp x10, x12, [x1, #32]
+; CHECK-NEXT: ccmp x9, x13, #0, eq
+; CHECK-NEXT: ldp x9, x13, [x1, #48]
+; CHECK-NEXT: ccmp x8, x10, #0, eq
+; CHECK-NEXT: ldp x8, x10, [x0, #48]
+; CHECK-NEXT: ccmp x11, x12, #0, eq
+; CHECK-NEXT: ccmp x8, x9, #0, eq
+; CHECK-NEXT: ccmp x10, x13, #0, eq
; CHECK-NEXT: cset w0, ne
; CHECK-NEXT: ret
%call = tail call i32 @memcmp(ptr %x, ptr %y, i64 64) nounwind
@@ -2357,46 +2333,38 @@ define i1 @length64_gt(ptr %x, ptr %y) nounwind {
define i1 @length64_eq_const(ptr %X) nounwind {
; CHECK-LABEL: length64_eq_const:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov x12, #13620 // =0x3534
-; CHECK-NEXT: ldp x13, x14, [x0, #16]
-; CHECK-NEXT: movk x12, #14134, lsl #16
-; CHECK-NEXT: ldp x16, x17, [x0, #32]
-; CHECK-NEXT: movk x12, #14648, lsl #32
; CHECK-NEXT: mov x8, #12592 // =0x3130
-; CHECK-NEXT: mov x9, #14648 // =0x3938
-; CHECK-NEXT: movk x12, #12592, lsl #48
-; CHECK-NEXT: mov x10, #14134 // =0x3736
+; CHECK-NEXT: ldp x9, x10, [x0]
; CHECK-NEXT: movk x8, #13106, lsl #16
-; CHECK-NEXT: eor x12, x14, x12
-; CHECK-NEXT: mov x14, #13106 // =0x3332
-; CHECK-NEXT: movk x9, #12592, lsl #16
-; CHECK-NEXT: movk x14, #13620, lsl #16
-; CHECK-NEXT: movk x10, #14648, lsl #16
+; CHECK-NEXT: ldp x11, x12, [x0, #16]
; CHECK-NEXT: movk x8, #13620, lsl #32
-; CHECK-NEXT: movk x14, #14134, lsl #32
-; CHECK-NEXT: ldp x11, x15, [x0]
-; CHECK-NEXT: movk x14, #14648, lsl #48
-; CHECK-NEXT: movk x9, #13106, lsl #32
-; CHECK-NEXT: movk x10, #12592, lsl #32
-; CHECK-NEXT: eor x14, x16, x14
-; CHECK-NEXT: ldp x16, x18, [x0, #48]
+; CHECK-NEXT: ldp x13, x14, [x0, #32]
; CHECK-NEXT: movk x8, #14134, lsl #48
+; CHECK-NEXT: cmp x9, x8
+; CHECK-NEXT: mov x9, #14648 // =0x3938
+; CHECK-NEXT: movk x9, #12592, lsl #16
+; CHECK-NEXT: movk x9, #13106, lsl #32
; CHECK-NEXT: movk x9, #13620, lsl #48
+; CHECK-NEXT: ccmp x10, x9, #0, eq
+; CHECK-NEXT: mov x10, #14134 // =0x3736
+; CHECK-NEXT: movk x10, #14648, lsl #16
+; CHECK-NEXT: movk x10, #12592, lsl #32
; CHECK-NEXT: movk x10, #13106, lsl #48
-; CHECK-NEXT: eor x11, x11, x8
-; CHECK-NEXT: eor x8, x17, x8
-; CHECK-NEXT: eor x15, x15, x9
-; CHECK-NEXT: eor x13, x13, x10
-; CHECK-NEXT: eor x9, x16, x9
-; CHECK-NEXT: eor x10, x18, x10
-; CHECK-NEXT: orr x11, x11, x15
-; CHECK-NEXT: orr x12, x13, x12
-; CHECK-NEXT: orr x8, x14, x8
-; CHECK-NEXT: orr x9, x9, x10
-; CHECK-NEXT: orr x10, x11, x12
-; CHECK-NEXT: orr x8, x8, x9
-; CHECK-NEXT: orr x8, x10, x8
-; CHECK-NEXT: cmp x8, #0
+; CHECK-NEXT: ccmp x11, x10, #0, eq
+; CHECK-NEXT: mov x11, #13620 // =0x3534
+; CHECK-NEXT: movk x11, #14134, lsl #16
+; CHECK-NEXT: movk x11, #14648, lsl #32
+; CHECK-NEXT: movk x11, #12592, lsl #48
+; CHECK-NEXT: ccmp x12, x11, #0, eq
+; CHECK-NEXT: mov x11, #13106 // =0x3332
+; CHECK-NEXT: movk x11, #13620, lsl #16
+; CHECK-NEXT: movk x11, #14134, lsl #32
+; CHECK-NEXT: movk x11, #14648, lsl #48
+; CHECK-NEXT: ccmp x13, x11, #0, eq
+; CHECK-NEXT: ldp x11, x12, [x0, #48]
+; CHECK-NEXT: ccmp x14, x8, #0, eq
+; CHECK-NEXT: ccmp x11, x9, #0, eq
+; CHECK-NEXT: ccmp x12, x10, #0, eq
; CHECK-NEXT: cset w0, eq
; CHECK-NEXT: ret
%m = tail call i32 @memcmp(ptr %X, ptr @.str, i64 64) nounwind
More information about the llvm-commits
mailing list