[llvm] [X86] Optimize 1-bit RCL/RCR and ADC(X, X, CF) patterns (PR #218285)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Aug 23 13:18:09 PDT 2026
https://github.com/AZero13 updated https://github.com/llvm/llvm-project/pull/218285
>From e949a2a8b36430aa0c23c4c12fb7b24deaad0493 Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Sun, 23 Aug 2026 16:10:24 -0400
Subject: [PATCH 1/2] [X86] Precommit tests for RCL/RCR funnel shift
optimizations
---
llvm/test/CodeGen/X86/funnel-shift-rcl-rcr.ll | 102 ++++++++++++++++++
1 file changed, 102 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/funnel-shift-rcl-rcr.ll
diff --git a/llvm/test/CodeGen/X86/funnel-shift-rcl-rcr.ll b/llvm/test/CodeGen/X86/funnel-shift-rcl-rcr.ll
new file mode 100644
index 0000000000000..29a42b9703a81
--- /dev/null
+++ b/llvm/test/CodeGen/X86/funnel-shift-rcl-rcr.ll
@@ -0,0 +1,102 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=-fast-shld-rotate,-ndd | FileCheck %s --check-prefix=AMD
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=-fast-shld-rotate,+ndd | FileCheck %s --check-prefix=NDD
+; RUN: llc < %s -mtriple=x86_64-unknown-linux-gnu -mattr=+fast-shld-rotate,-ndd | FileCheck %s --check-prefix=SANDYBRIDGE
+
+define void @rcl_mem_testA(ptr %ptr, i32 %b) {
+; AMD-LABEL: rcl_mem_testA:
+; AMD: # %bb.0: # %entry
+; AMD-NEXT: movl (%rdi), %eax
+; AMD-NEXT: cmpl $70, %esi
+; AMD-NEXT: adcl %eax, %eax
+; AMD-NEXT: movl %eax, (%rdi)
+; AMD-NEXT: retq
+;
+; NDD-LABEL: rcl_mem_testA:
+; NDD: # %bb.0: # %entry
+; NDD-NEXT: movl (%rdi), %eax
+; NDD-NEXT: cmpl $70, %esi
+; NDD-NEXT: adcl %eax, %eax
+; NDD-NEXT: movl %eax, (%rdi)
+; NDD-NEXT: retq
+;
+; SANDYBRIDGE-LABEL: rcl_mem_testA:
+; SANDYBRIDGE: # %bb.0: # %entry
+; SANDYBRIDGE-NEXT: movl (%rdi), %eax
+; SANDYBRIDGE-NEXT: cmpl $70, %esi
+; SANDYBRIDGE-NEXT: adcl %eax, %eax
+; SANDYBRIDGE-NEXT: movl %eax, (%rdi)
+; SANDYBRIDGE-NEXT: retq
+entry:
+ %dst = load i32, ptr %ptr
+ %cmp = icmp ult i32 %b, 70
+ %conv = zext i1 %cmp to i32
+ %add1 = add i32 %dst, %dst
+ %add2 = add i32 %add1, %conv
+ store i32 %add2, ptr %ptr
+ ret void
+}
+
+define i32 @rcl_mem_testB(ptr %ptr, i32 %b) {
+; AMD-LABEL: rcl_mem_testB:
+; AMD: # %bb.0: # %entry
+; AMD-NEXT: movl (%rdi), %eax
+; AMD-NEXT: cmpl $70, %esi
+; AMD-NEXT: adcl %eax, %eax
+; AMD-NEXT: retq
+;
+; NDD-LABEL: rcl_mem_testB:
+; NDD: # %bb.0: # %entry
+; NDD-NEXT: movl (%rdi), %eax
+; NDD-NEXT: cmpl $70, %esi
+; NDD-NEXT: adcl %eax, %eax
+; NDD-NEXT: retq
+;
+; SANDYBRIDGE-LABEL: rcl_mem_testB:
+; SANDYBRIDGE: # %bb.0: # %entry
+; SANDYBRIDGE-NEXT: movl (%rdi), %eax
+; SANDYBRIDGE-NEXT: cmpl $70, %esi
+; SANDYBRIDGE-NEXT: adcl %eax, %eax
+; SANDYBRIDGE-NEXT: retq
+entry:
+ %dst = load i32, ptr %ptr
+ %cmp = icmp ult i32 %b, 70
+ %conv = zext i1 %cmp to i32
+ %add1 = add i32 %dst, %dst
+ %add2 = add i32 %add1, %conv
+ ret i32 %add2
+}
+
+define i32 @rcr_test(i32 %a, i32 %b) {
+; AMD-LABEL: rcr_test:
+; AMD: # %bb.0: # %entry
+; AMD-NEXT: xorl %eax, %eax
+; AMD-NEXT: cmpl $70, %esi
+; AMD-NEXT: setb %al
+; AMD-NEXT: shldl $31, %edi, %eax
+; AMD-NEXT: retq
+;
+; NDD-LABEL: rcr_test:
+; NDD: # %bb.0: # %entry
+; NDD-NEXT: xorl %eax, %eax
+; NDD-NEXT: cmpl $70, %esi
+; NDD-NEXT: setb %al
+; NDD-NEXT: shldl $31, %edi, %eax
+; NDD-NEXT: retq
+;
+; SANDYBRIDGE-LABEL: rcr_test:
+; SANDYBRIDGE: # %bb.0: # %entry
+; SANDYBRIDGE-NEXT: xorl %eax, %eax
+; SANDYBRIDGE-NEXT: cmpl $70, %esi
+; SANDYBRIDGE-NEXT: setb %al
+; SANDYBRIDGE-NEXT: shldl $31, %edi, %eax
+; SANDYBRIDGE-NEXT: retq
+entry:
+ %cmp = icmp ult i32 %b, 70
+ %conv = zext i1 %cmp to i32
+ %0 = tail call i32 @llvm.fshr.i32(i32 %conv, i32 %a, i32 1)
+ ret i32 %0
+}
+
+declare i32 @llvm.fshl.i32(i32, i32, i32)
+declare i32 @llvm.fshr.i32(i32, i32, i32)
>From 05616c5bdbe1bc590427a161971e8c7930f9ed86 Mon Sep 17 00:00:00 2001
From: AZero13 <gfunni234 at gmail.com>
Date: Sun, 23 Aug 2026 16:11:00 -0400
Subject: [PATCH 2/2] [X86] Optimize RCL/RCR funnel shifts and ADC matching
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 69 +++++++++++++++++++
llvm/lib/Target/X86/X86InstrFragments.td | 8 +++
llvm/lib/Target/X86/X86InstrPredicates.td | 1 +
llvm/lib/Target/X86/X86InstrShiftRotate.td | 39 +++++++++++
llvm/test/CodeGen/X86/funnel-shift-rcl-rcr.ll | 63 +++++++++++++----
5 files changed, 167 insertions(+), 13 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index bf435b0c2f849..475272eef890c 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -59272,6 +59272,66 @@ static SDValue combineRotate(SDNode *N, SelectionDAG &DAG,
}
// Combiner: turn uniform-constant splat funnel shifts into VSHLD/VSHRD
+static SDValue combineScalarFunnelShift(SDNode *N, SelectionDAG &DAG,
+ TargetLowering::DAGCombinerInfo &DCI,
+ const X86Subtarget &Subtarget) {
+
+ EVT VT = N->getValueType(0);
+ if (VT.isVector())
+ return SDValue();
+
+ SDLoc DL(N);
+ unsigned Opc = N->getOpcode();
+ SDValue Op0 = N->getOperand(0);
+ SDValue Op1 = N->getOperand(1);
+ SDValue Amt = N->getOperand(2);
+
+ auto *AmtC = dyn_cast<ConstantSDNode>(Amt);
+ if (!AmtC)
+ return SDValue();
+
+ unsigned ShiftAmt = AmtC->getZExtValue();
+ unsigned BitWidth = VT.getSizeInBits();
+
+ // We want to match:
+ // fshr(Carry, Dst, 1) -> RCR Dst, 1
+ // fshl(Dst, Carry, 1) -> RCL Dst, 1
+ bool IsRCR = (Opc == ISD::FSHR && ShiftAmt == 1) ||
+ (Opc == ISD::FSHL && ShiftAmt == BitWidth - 1);
+ bool IsRCL = (Opc == ISD::FSHL && ShiftAmt == 1) ||
+ (Opc == ISD::FSHR && ShiftAmt == BitWidth - 1);
+
+ if (!IsRCR && !IsRCL)
+ return SDValue();
+
+ // For RCR, we only emit it if the target doesn't have fast SHLD rotate
+ // (e.g. AMD, pre-Sandy Bridge) or if it has APX NDD.
+ if (IsRCR && Subtarget.hasFastSHLDRotate() && !Subtarget.hasNDD())
+ return SDValue();
+
+ SDValue Carry = IsRCR ? Op0 : Op1;
+ SDValue Dst = IsRCR ? Op1 : Op0;
+
+ if (IsRCL) {
+ SDValue Shift = DAG.getConstant(BitWidth - 1, DL, VT);
+ Carry = DAG.getNode(ISD::SRL, DL, VT, Carry, Shift);
+ }
+
+ // EFLAGS = SUB(0, AND(Carry, 1)). This sets CF=1 if lowest bit is 1.
+ SDVTList VTs = DAG.getVTList(VT, MVT::i32);
+ SDValue And =
+ DAG.getNode(ISD::AND, DL, VT, Carry, DAG.getConstant(1, DL, VT));
+ SDValue Sub =
+ DAG.getNode(X86ISD::SUB, DL, VTs, DAG.getConstant(0, DL, VT), And);
+ SDValue EFLAGS = SDValue(Sub.getNode(), 1);
+
+ // RCR/RCL
+ unsigned TargetOpc = IsRCR ? X86ISD::RCR : X86ISD::RCL;
+ SDValue Rot = DAG.getNode(TargetOpc, DL, VTs, Dst, EFLAGS);
+
+ return SDValue(Rot.getNode(), 0);
+}
+
static SDValue combineFunnelShift(SDNode *N, SelectionDAG &DAG,
TargetLowering::DAGCombinerInfo &DCI,
const X86Subtarget &Subtarget) {
@@ -59281,6 +59341,9 @@ static SDValue combineFunnelShift(SDNode *N, SelectionDAG &DAG,
SDValue Amt = N->getOperand(2);
EVT VT = Op0.getValueType();
+ if (SDValue ScalarRes = combineScalarFunnelShift(N, DAG, DCI, Subtarget))
+ return ScalarRes;
+
if (!VT.isVector())
return SDValue();
@@ -59673,6 +59736,12 @@ static SDValue combineADC(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(X86ISD::ADC, SDLoc(N), N->getVTList(), RHS, LHS,
CarryIn);
+ // ADC(X, X, Carry) -> RCL(X, Carry)
+ // RCL only defines CF and OF. ADC defines all flags.
+ // We can only do this fold if the EFLAGS result of ADC is dead.
+ if (LHS == RHS && !N->hasAnyUseOfValue(1))
+ return DAG.getNode(X86ISD::RCL, SDLoc(N), N->getVTList(), LHS, CarryIn);
+
// If the LHS and RHS of the ADC node are zero, then it can't overflow and
// the result is either zero or one (depending on the input carry bit).
// Strength reduce this down to a "set on carry" aka SETCC_CARRY&1.
diff --git a/llvm/lib/Target/X86/X86InstrFragments.td b/llvm/lib/Target/X86/X86InstrFragments.td
index c183849d4f575..44a184a72eca2 100644
--- a/llvm/lib/Target/X86/X86InstrFragments.td
+++ b/llvm/lib/Target/X86/X86InstrFragments.td
@@ -38,6 +38,14 @@ def SDTBinaryArithWithFlags : SDTypeProfile<2, 2,
SDTCisSameAs<0, 3>,
SDTCisInt<0>, SDTCisVT<1, i32>]>;
+def SDTUnaryArithWithFlagsInOut : SDTypeProfile<2, 2,
+ [SDTCisSameAs<0, 2>,
+ SDTCisInt<0>, SDTCisVT<1, i32>,
+ SDTCisVT<3, i32>]>;
+
+def X86rcl_flag : SDNode<"X86ISD::RCL", SDTUnaryArithWithFlagsInOut>;
+def X86rcr_flag : SDNode<"X86ISD::RCR", SDTUnaryArithWithFlagsInOut>;
+
// SDTBinaryArithWithFlagsInOut - RES1, EFLAGS = op LHS, RHS, EFLAGS
def SDTBinaryArithWithFlagsInOut : SDTypeProfile<2, 3,
[SDTCisSameAs<0, 2>,
diff --git a/llvm/lib/Target/X86/X86InstrPredicates.td b/llvm/lib/Target/X86/X86InstrPredicates.td
index fd03fa24a18c8..8858b1435c4a2 100644
--- a/llvm/lib/Target/X86/X86InstrPredicates.td
+++ b/llvm/lib/Target/X86/X86InstrPredicates.td
@@ -255,6 +255,7 @@ def FavorMemIndirectCall : Predicate<"!Subtarget->slowTwoMemOps() && "
def HasFastMem32 : Predicate<"!Subtarget->isUnalignedMem32Slow()">;
def HasFastLZCNT : Predicate<"Subtarget->hasFastLZCNT()">;
def HasFastSHLDRotate : Predicate<"Subtarget->hasFastSHLDRotate()">;
+def NotFastSHLDRotate : Predicate<"!Subtarget->hasFastSHLDRotate()">;
def HasERMSB : Predicate<"Subtarget->hasERMSB()">;
def HasFSRM : Predicate<"Subtarget->hasFSRM()">;
def HasMFence : Predicate<"Subtarget->hasMFence()">;
diff --git a/llvm/lib/Target/X86/X86InstrShiftRotate.td b/llvm/lib/Target/X86/X86InstrShiftRotate.td
index 7e7c2f97c5793..4567c5039452b 100644
--- a/llvm/lib/Target/X86/X86InstrShiftRotate.td
+++ b/llvm/lib/Target/X86/X86InstrShiftRotate.td
@@ -249,6 +249,45 @@ let Predicates = [NoNDD] in {
def : Pat<(rotr GR16:$src1, (i8 15)), (ROL16r1 GR16:$src1)>;
def : Pat<(rotr GR32:$src1, (i8 31)), (ROL32r1 GR32:$src1)>;
def : Pat<(rotr GR64:$src1, (i8 63)), (ROL64r1 GR64:$src1)>;
+
+ def : Pat<(X86rcl_flag GR8:$src1, EFLAGS), (ADC8rr GR8:$src1, GR8:$src1)>;
+ def : Pat<(X86rcl_flag GR16:$src1, EFLAGS), (ADC16rr GR16:$src1, GR16:$src1)>;
+ def : Pat<(X86rcl_flag GR32:$src1, EFLAGS), (ADC32rr GR32:$src1, GR32:$src1)>;
+ def : Pat<(X86rcl_flag GR64:$src1, EFLAGS), (ADC64rr GR64:$src1, GR64:$src1)>;
+
+ let Predicates = [NotFastSHLDRotate] in {
+ def : Pat<(store (X86rcl_flag (loadi8 addr:$dst), EFLAGS), addr:$dst), (RCL8m1 addr:$dst)>;
+ def : Pat<(store (X86rcl_flag (loadi16 addr:$dst), EFLAGS), addr:$dst), (RCL16m1 addr:$dst)>;
+ def : Pat<(store (X86rcl_flag (loadi32 addr:$dst), EFLAGS), addr:$dst), (RCL32m1 addr:$dst)>;
+ def : Pat<(store (X86rcl_flag (loadi64 addr:$dst), EFLAGS), addr:$dst), (RCL64m1 addr:$dst)>;
+ }
+
+ let Predicates = [HasNDD] in {
+ def : Pat<(X86rcl_flag (loadi8 addr:$src), EFLAGS), (RCL8m1_ND addr:$src)>;
+ def : Pat<(X86rcl_flag (loadi16 addr:$src), EFLAGS), (RCL16m1_ND addr:$src)>;
+ def : Pat<(X86rcl_flag (loadi32 addr:$src), EFLAGS), (RCL32m1_ND addr:$src)>;
+ def : Pat<(X86rcl_flag (loadi64 addr:$src), EFLAGS), (RCL64m1_ND addr:$src)>;
+
+ def : Pat<(X86rcr_flag GR8:$src, EFLAGS), (RCR8r1_ND GR8:$src)>;
+ def : Pat<(X86rcr_flag GR16:$src, EFLAGS), (RCR16r1_ND GR16:$src)>;
+ def : Pat<(X86rcr_flag GR32:$src, EFLAGS), (RCR32r1_ND GR32:$src)>;
+ def : Pat<(X86rcr_flag GR64:$src, EFLAGS), (RCR64r1_ND GR64:$src)>;
+
+ def : Pat<(X86rcr_flag (loadi8 addr:$src), EFLAGS), (RCR8m1_ND addr:$src)>;
+ def : Pat<(X86rcr_flag (loadi16 addr:$src), EFLAGS), (RCR16m1_ND addr:$src)>;
+ def : Pat<(X86rcr_flag (loadi32 addr:$src), EFLAGS), (RCR32m1_ND addr:$src)>;
+ def : Pat<(X86rcr_flag (loadi64 addr:$src), EFLAGS), (RCR64m1_ND addr:$src)>;
+ }
+
+ def : Pat<(X86rcr_flag GR8:$src1, EFLAGS), (RCR8r1 GR8:$src1)>;
+ def : Pat<(X86rcr_flag GR16:$src1, EFLAGS), (RCR16r1 GR16:$src1)>;
+ def : Pat<(X86rcr_flag GR32:$src1, EFLAGS), (RCR32r1 GR32:$src1)>;
+ def : Pat<(X86rcr_flag GR64:$src1, EFLAGS), (RCR64r1 GR64:$src1)>;
+
+ def : Pat<(store (X86rcr_flag (loadi8 addr:$dst), EFLAGS), addr:$dst), (RCR8m1 addr:$dst)>;
+ def : Pat<(store (X86rcr_flag (loadi16 addr:$dst), EFLAGS), addr:$dst), (RCR16m1 addr:$dst)>;
+ def : Pat<(store (X86rcr_flag (loadi32 addr:$dst), EFLAGS), addr:$dst), (RCR32m1 addr:$dst)>;
+ def : Pat<(store (X86rcr_flag (loadi64 addr:$dst), EFLAGS), addr:$dst), (RCR64m1 addr:$dst)>;
}
let Predicates = [HasNDD] in {
def : Pat<(rotl GR8:$src1, (i8 7)), (ROR8r1_ND GR8:$src1)>;
diff --git a/llvm/test/CodeGen/X86/funnel-shift-rcl-rcr.ll b/llvm/test/CodeGen/X86/funnel-shift-rcl-rcr.ll
index 29a42b9703a81..fa2b23637117a 100644
--- a/llvm/test/CodeGen/X86/funnel-shift-rcl-rcr.ll
+++ b/llvm/test/CodeGen/X86/funnel-shift-rcl-rcr.ll
@@ -6,18 +6,14 @@
define void @rcl_mem_testA(ptr %ptr, i32 %b) {
; AMD-LABEL: rcl_mem_testA:
; AMD: # %bb.0: # %entry
-; AMD-NEXT: movl (%rdi), %eax
; AMD-NEXT: cmpl $70, %esi
-; AMD-NEXT: adcl %eax, %eax
-; AMD-NEXT: movl %eax, (%rdi)
+; AMD-NEXT: rcll (%rdi)
; AMD-NEXT: retq
;
; NDD-LABEL: rcl_mem_testA:
; NDD: # %bb.0: # %entry
-; NDD-NEXT: movl (%rdi), %eax
; NDD-NEXT: cmpl $70, %esi
-; NDD-NEXT: adcl %eax, %eax
-; NDD-NEXT: movl %eax, (%rdi)
+; NDD-NEXT: rcll (%rdi)
; NDD-NEXT: retq
;
; SANDYBRIDGE-LABEL: rcl_mem_testA:
@@ -47,9 +43,8 @@ define i32 @rcl_mem_testB(ptr %ptr, i32 %b) {
;
; NDD-LABEL: rcl_mem_testB:
; NDD: # %bb.0: # %entry
-; NDD-NEXT: movl (%rdi), %eax
; NDD-NEXT: cmpl $70, %esi
-; NDD-NEXT: adcl %eax, %eax
+; NDD-NEXT: rcll (%rdi), %eax
; NDD-NEXT: retq
;
; SANDYBRIDGE-LABEL: rcl_mem_testB:
@@ -70,18 +65,19 @@ entry:
define i32 @rcr_test(i32 %a, i32 %b) {
; AMD-LABEL: rcr_test:
; AMD: # %bb.0: # %entry
-; AMD-NEXT: xorl %eax, %eax
+; AMD-NEXT: movl %edi, %eax
+; AMD-NEXT: xorl %ecx, %ecx
; AMD-NEXT: cmpl $70, %esi
-; AMD-NEXT: setb %al
-; AMD-NEXT: shldl $31, %edi, %eax
+; AMD-NEXT: sbbl %ecx, %ecx
+; AMD-NEXT: rcrl %eax
; AMD-NEXT: retq
;
; NDD-LABEL: rcr_test:
; NDD: # %bb.0: # %entry
; NDD-NEXT: xorl %eax, %eax
; NDD-NEXT: cmpl $70, %esi
-; NDD-NEXT: setb %al
-; NDD-NEXT: shldl $31, %edi, %eax
+; NDD-NEXT: sbbl %eax, %eax
+; NDD-NEXT: rcrl %edi, %eax
; NDD-NEXT: retq
;
; SANDYBRIDGE-LABEL: rcr_test:
@@ -100,3 +96,44 @@ entry:
declare i32 @llvm.fshl.i32(i32, i32, i32)
declare i32 @llvm.fshr.i32(i32, i32, i32)
+
+define i32 @adc_flags_used(ptr %ptr, i32 %b) {
+; AMD-LABEL: adc_flags_used:
+; AMD: # %bb.0: # %entry
+; AMD-NEXT: movl (%rdi), %eax
+; AMD-NEXT: cmpl $70, %esi
+; AMD-NEXT: adcl %eax, %eax
+; AMD-NEXT: movl $10, %ecx
+; AMD-NEXT: movl $20, %eax
+; AMD-NEXT: cmovel %ecx, %eax
+; AMD-NEXT: retq
+;
+; NDD-LABEL: adc_flags_used:
+; NDD: # %bb.0: # %entry
+; NDD-NEXT: movl (%rdi), %eax
+; NDD-NEXT: cmpl $70, %esi
+; NDD-NEXT: adcl %eax, %eax
+; NDD-NEXT: movl $10, %eax
+; NDD-NEXT: movl $20, %ecx
+; NDD-NEXT: cmovnel %ecx, %eax
+; NDD-NEXT: retq
+;
+; SANDYBRIDGE-LABEL: adc_flags_used:
+; SANDYBRIDGE: # %bb.0: # %entry
+; SANDYBRIDGE-NEXT: movl (%rdi), %eax
+; SANDYBRIDGE-NEXT: cmpl $70, %esi
+; SANDYBRIDGE-NEXT: adcl %eax, %eax
+; SANDYBRIDGE-NEXT: movl $10, %ecx
+; SANDYBRIDGE-NEXT: movl $20, %eax
+; SANDYBRIDGE-NEXT: cmovel %ecx, %eax
+; SANDYBRIDGE-NEXT: retq
+entry:
+ %dst = load i32, ptr %ptr
+ %cmp = icmp ult i32 %b, 70
+ %conv = zext i1 %cmp to i32
+ %add1 = add i32 %dst, %dst
+ %add2 = add i32 %add1, %conv
+ %cmp2 = icmp eq i32 %add2, 0
+ %sel = select i1 %cmp2, i32 10, i32 20
+ ret i32 %sel
+}
More information about the llvm-commits
mailing list