[llvm] [RISCV] Optimize CLMULH of i32 loads on RV64 (PR #228669)
via llvm-commits
llvm-commits at lists.llvm.org
Sat Oct 3 21:02:01 PDT 2026
https://github.com/tinfengyu updated https://github.com/llvm/llvm-project/pull/228669
>From 8588244c9f0a75ecab9f5f2b746e9780659fb722 Mon Sep 17 00:00:00 2001
From: tinfeng <1768309929 at qq.com>
Date: Fri, 2 Oct 2026 20:55:24 +0800
Subject: [PATCH 1/2] [RISCV] Optimize CLMULH of i32 loads on RV64
RV64 i32 CLMULH lowering shifts both promoted operands left before using
CLMULH. For two simple, nonindexed, single-use i32 loads, zero extension
can instead fold into LWU.
Use i64 CLMUL and extract the high 32 bits of the 32-by-32 carry-less
product, avoiding the two left shifts. Preserve the existing lowering
for other operands and CLMULR.
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 26 ++
llvm/test/CodeGen/RISCV/clmulh-load-rv64.ll | 277 ++++++++++++++++++++
2 files changed, 303 insertions(+)
create mode 100644 llvm/test/CodeGen/RISCV/clmulh-load-rv64.ll
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 13669fa56766d..0e277d0eb88bd 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -18184,6 +18184,32 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
return;
+ // A simple single-use sign-extending i32 load can use lwu directly.
+ // Widening these operands and using clmul avoids the shifts required by
+ // the general register-input lowering below.
+ auto IsSingleUseLoad = [](SDValue Op) {
+ if (Op.getOpcode() != ISD::TRUNCATE || !Op.hasOneUse())
+ return false;
+ auto *Load = dyn_cast<LoadSDNode>(Op.getOperand(0));
+ return Load && Load->isSimple() && !Load->isIndexed() &&
+ Load->getExtensionType() == ISD::SEXTLOAD &&
+ Load->getMemoryVT() == MVT::i32 && Op.getOperand(0).hasOneUse();
+ };
+ if (IntNo == Intrinsic::riscv_clmulh &&
+ (Subtarget.hasStdExtZbc() || Subtarget.hasStdExtZbkc()) &&
+ IsSingleUseLoad(N->getOperand(1)) &&
+ IsSingleUseLoad(N->getOperand(2))) {
+ SDValue NewOp0 =
+ DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
+ SDValue NewOp1 =
+ DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(2));
+ SDValue Res = DAG.getNode(ISD::CLMUL, DL, MVT::i64, NewOp0, NewOp1);
+ Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res,
+ DAG.getConstant(32, DL, MVT::i64));
+ Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
+ return;
+ }
+
// Extend inputs to XLen, and shift by 32. This will add 64 trailing zeros
// to the full 128-bit clmul result of multiplying two xlen values.
// Perform clmulr or clmulh on the shifted values. Finally, extract the
diff --git a/llvm/test/CodeGen/RISCV/clmulh-load-rv64.ll b/llvm/test/CodeGen/RISCV/clmulh-load-rv64.ll
new file mode 100644
index 0000000000000..238f3ba1853cb
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/clmulh-load-rv64.ll
@@ -0,0 +1,277 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=riscv64 -mattr=+zbc -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZBC,NOA
+; RUN: llc -mtriple=riscv64 -mattr=+zbkc -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZBKC,NOA
+; RUN: llc -mtriple=riscv64 -mattr=+zbc,+a -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZBC,ATOMIC
+; RUN: llc -mtriple=riscv64 -mattr=+zbkc,+a -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZBKC,ATOMIC
+
+declare i32 @llvm.riscv.clmulh.i32(i32, i32)
+declare i32 @llvm.riscv.clmulr.i32(i32, i32)
+
+; Zero extensions of simple single-use i32 loads can fold into lwu.
+define i32 @clmulh_load(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lwu a0, 0(a0)
+; CHECK-NEXT: lwu a1, 0(a1)
+; CHECK-NEXT: clmul a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ %b = load i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; The result still follows the signed i32 return convention when requested.
+define signext i32 @clmulh_load_signext(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_load_signext:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lwu a0, 0(a0)
+; CHECK-NEXT: lwu a1, 0(a1)
+; CHECK-NEXT: clmul a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ %b = load i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; Unaligned loads may need to be expanded before the intrinsic is legalized.
+define i32 @clmulh_unaligned_load(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_unaligned_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lbu a2, 1(a0)
+; CHECK-NEXT: lbu a3, 0(a0)
+; CHECK-NEXT: lbu a4, 2(a0)
+; CHECK-NEXT: lbu a0, 3(a0)
+; CHECK-NEXT: slli a2, a2, 8
+; CHECK-NEXT: lbu a5, 1(a1)
+; CHECK-NEXT: lbu a6, 2(a1)
+; CHECK-NEXT: lbu a7, 3(a1)
+; CHECK-NEXT: lbu a1, 0(a1)
+; CHECK-NEXT: or a2, a2, a3
+; CHECK-NEXT: slli a4, a4, 16
+; CHECK-NEXT: slli a0, a0, 24
+; CHECK-NEXT: or a0, a0, a4
+; CHECK-NEXT: slli a5, a5, 8
+; CHECK-NEXT: slli a6, a6, 16
+; CHECK-NEXT: slli a7, a7, 24
+; CHECK-NEXT: or a1, a5, a1
+; CHECK-NEXT: or a3, a7, a6
+; CHECK-NEXT: or a0, a0, a2
+; CHECK-NEXT: or a1, a3, a1
+; CHECK-NEXT: clmul a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 1
+ %b = load i32, ptr %q, align 1
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+define i32 @clmulh_register(i32 %a, i32 %b) {
+; CHECK-LABEL: clmulh_register:
+; CHECK: # %bb.0:
+; CHECK-NEXT: slli a1, a1, 32
+; CHECK-NEXT: slli a0, a0, 32
+; CHECK-NEXT: clmulh a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; Both operands must be freely zero extendable loads.
+define i32 @clmulh_load_register(ptr %p, i32 %b) {
+; CHECK-LABEL: clmulh_load_register:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lw a0, 0(a0)
+; CHECK-NEXT: slli a1, a1, 32
+; CHECK-NEXT: slli a0, a0, 32
+; CHECK-NEXT: clmulh a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+define i32 @clmulh_register_load(i32 %a, ptr %q) {
+; CHECK-LABEL: clmulh_register_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lw a1, 0(a1)
+; CHECK-NEXT: slli a0, a0, 32
+; CHECK-NEXT: slli a1, a1, 32
+; CHECK-NEXT: clmulh a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %b = load i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; An additional value use prevents changing the load's extension.
+define i32 @clmulh_multiuse_load(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_multiuse_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lw a0, 0(a0)
+; CHECK-NEXT: lw a1, 0(a1)
+; CHECK-NEXT: slli a1, a1, 32
+; CHECK-NEXT: slli a2, a0, 32
+; CHECK-NEXT: clmulh a1, a2, a1
+; CHECK-NEXT: srli a1, a1, 32
+; CHECK-NEXT: addw a0, a1, a0
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ %b = load i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ %sum = add i32 %r, %a
+ ret i32 %sum
+}
+
+; Volatile and atomic loads keep their original lowering.
+define i32 @clmulh_volatile_load(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_volatile_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lw a0, 0(a0)
+; CHECK-NEXT: lw a1, 0(a1)
+; CHECK-NEXT: slli a0, a0, 32
+; CHECK-NEXT: slli a1, a1, 32
+; CHECK-NEXT: clmulh a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load volatile i32, ptr %p, align 4
+ %b = load volatile i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+define i32 @clmulh_atomic_load(ptr %p, ptr %q) {
+; NOA-LABEL: clmulh_atomic_load:
+; NOA: # %bb.0:
+; NOA-NEXT: addi sp, sp, -32
+; NOA-NEXT: .cfi_def_cfa_offset 32
+; NOA-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
+; NOA-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; NOA-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
+; NOA-NEXT: .cfi_offset ra, -8
+; NOA-NEXT: .cfi_offset s0, -16
+; NOA-NEXT: .cfi_offset s1, -24
+; NOA-NEXT: mv s0, a1
+; NOA-NEXT: li a1, 2
+; NOA-NEXT: call __atomic_load_4
+; NOA-NEXT: mv s1, a0
+; NOA-NEXT: li a1, 2
+; NOA-NEXT: mv a0, s0
+; NOA-NEXT: call __atomic_load_4
+; NOA-NEXT: slli s1, s1, 32
+; NOA-NEXT: slli a0, a0, 32
+; NOA-NEXT: clmulh a0, s1, a0
+; NOA-NEXT: srli a0, a0, 32
+; NOA-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
+; NOA-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; NOA-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
+; NOA-NEXT: .cfi_restore ra
+; NOA-NEXT: .cfi_restore s0
+; NOA-NEXT: .cfi_restore s1
+; NOA-NEXT: addi sp, sp, 32
+; NOA-NEXT: .cfi_def_cfa_offset 0
+; NOA-NEXT: ret
+;
+; ATOMIC-LABEL: clmulh_atomic_load:
+; ATOMIC: # %bb.0:
+; ATOMIC-NEXT: lw a0, 0(a0)
+; ATOMIC-NEXT: fence r, rw
+; ATOMIC-NEXT: lw a1, 0(a1)
+; ATOMIC-NEXT: slli a0, a0, 32
+; ATOMIC-NEXT: slli a1, a1, 32
+; ATOMIC-NEXT: clmulh a0, a0, a1
+; ATOMIC-NEXT: srli a0, a0, 32
+; ATOMIC-NEXT: fence r, rw
+; ATOMIC-NEXT: ret
+ %a = load atomic i32, ptr %p acquire, align 4
+ %b = load atomic i32, ptr %q acquire, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; Using one load for both operands is not a single value use.
+define i32 @clmulh_same_load(ptr %p) {
+; CHECK-LABEL: clmulh_same_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lw a0, 0(a0)
+; CHECK-NEXT: slli a0, a0, 32
+; CHECK-NEXT: clmulh a0, a0, a0
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %a)
+ ret i32 %r
+}
+
+; A chain use does not prevent changing a single-use load's extension. The
+; aliasing store must remain between the first load and the second load.
+define i32 @clmulh_alias_chain(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_alias_chain:
+; CHECK: # %bb.0:
+; CHECK-NEXT: li a2, 7
+; CHECK-NEXT: lwu a3, 0(a0)
+; CHECK-NEXT: sw a2, 0(a0)
+; CHECK-NEXT: lwu a0, 0(a1)
+; CHECK-NEXT: clmul a0, a3, a0
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ store i32 7, ptr %p, align 4
+ %b = load i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; A load of a narrower memory type is outside the i32-load matcher.
+define i32 @clmulh_narrow_load(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_narrow_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lh a0, 0(a0)
+; CHECK-NEXT: lh a1, 0(a1)
+; CHECK-NEXT: slli a1, a1, 32
+; CHECK-NEXT: slli a0, a0, 32
+; CHECK-NEXT: clmulh a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a16 = load i16, ptr %p, align 2
+ %b16 = load i16, ptr %q, align 2
+ %a = sext i16 %a16 to i32
+ %b = sext i16 %b16 to i32
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; CLMULR retains its existing lowering for i32 loads.
+define i32 @clmulr_load(ptr %p, ptr %q) {
+; ZBC-LABEL: clmulr_load:
+; ZBC: # %bb.0:
+; ZBC-NEXT: lw a0, 0(a0)
+; ZBC-NEXT: lw a1, 0(a1)
+; ZBC-NEXT: slli a1, a1, 32
+; ZBC-NEXT: slli a0, a0, 32
+; ZBC-NEXT: clmulr a0, a0, a1
+; ZBC-NEXT: srli a0, a0, 32
+; ZBC-NEXT: ret
+;
+; ZBKC-LABEL: clmulr_load:
+; ZBKC: # %bb.0:
+; ZBKC-NEXT: lw a0, 0(a0)
+; ZBKC-NEXT: lw a1, 0(a1)
+; ZBKC-NEXT: slli a1, a1, 32
+; ZBKC-NEXT: slli a0, a0, 32
+; ZBKC-NEXT: clmulh a0, a0, a1
+; ZBKC-NEXT: slli a0, a0, 1
+; ZBKC-NEXT: srli a0, a0, 32
+; ZBKC-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ %b = load i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulr.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
>From 3ccd69c6908274ba70d63b33e249a0d2ed70619a Mon Sep 17 00:00:00 2001
From: tinfeng <1768309929 at qq.com>
Date: Sun, 4 Oct 2026 12:01:14 +0800
Subject: [PATCH 2/2] [RISCV] Remove redundant Zbc feature check
---
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 0e277d0eb88bd..ccdf420d539fd 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -18195,8 +18195,7 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
Load->getExtensionType() == ISD::SEXTLOAD &&
Load->getMemoryVT() == MVT::i32 && Op.getOperand(0).hasOneUse();
};
- if (IntNo == Intrinsic::riscv_clmulh &&
- (Subtarget.hasStdExtZbc() || Subtarget.hasStdExtZbkc()) &&
+ if (IntNo == Intrinsic::riscv_clmulh && Subtarget.hasStdExtZbkc() &&
IsSingleUseLoad(N->getOperand(1)) &&
IsSingleUseLoad(N->getOperand(2))) {
SDValue NewOp0 =
More information about the llvm-commits
mailing list