[llvm] [RISCV] Optimize CLMULH of i32 loads on RV64 (PR #228669)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 2 23:44:32 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-risc-v
Author: tinfengyu
<details>
<summary>Changes</summary>
On RV64, lowering an i32 `llvm.riscv.clmulh` currently extends both operands, shifts them left by 32, performs `CLMULH`, and shifts the result right by 32.
When both operands come from simple, non-indexed, single-use i32 loads, their zero extensions can instead fold into `LWU`. The 32-by-32 carry-less product fits in the low 64 bits, allowing the two left shifts and `CLMULH` to be replaced by `LWU` loads and `CLMUL`.
In short:
lw + lw + 2×slli + clmulh + srli
↓
lwu + lwu + clmul + srli
The existing lowering is retained when the operands do not meet these conditions, including multi-use, volatile, and atomic loads. `CLMULR` is unchanged.
Tests cover Zbc and Zbkc, the optimized load cases, and fallback cases including register operands, multi-use loads, volatile/atomic loads, narrower loads, and `CLMULR`.
AI tools were used to assist with investigating the optimization opportunity, reviewing the implementation, and preparing tests and documentation. The final changes and their correctness were reviewed and validated by the author.
---
Full diff: https://github.com/llvm/llvm-project/pull/228669.diff
2 Files Affected:
- (modified) llvm/lib/Target/RISCV/RISCVISelLowering.cpp (+26)
- (added) llvm/test/CodeGen/RISCV/clmulh-load-rv64.ll (+277)
``````````diff
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 13669fa56766d..0e277d0eb88bd 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -18184,6 +18184,32 @@ void RISCVTargetLowering::ReplaceNodeResults(SDNode *N,
if (!Subtarget.is64Bit() || N->getValueType(0) != MVT::i32)
return;
+ // A simple single-use sign-extending i32 load can use lwu directly.
+ // Widening these operands and using clmul avoids the shifts required by
+ // the general register-input lowering below.
+ auto IsSingleUseLoad = [](SDValue Op) {
+ if (Op.getOpcode() != ISD::TRUNCATE || !Op.hasOneUse())
+ return false;
+ auto *Load = dyn_cast<LoadSDNode>(Op.getOperand(0));
+ return Load && Load->isSimple() && !Load->isIndexed() &&
+ Load->getExtensionType() == ISD::SEXTLOAD &&
+ Load->getMemoryVT() == MVT::i32 && Op.getOperand(0).hasOneUse();
+ };
+ if (IntNo == Intrinsic::riscv_clmulh &&
+ (Subtarget.hasStdExtZbc() || Subtarget.hasStdExtZbkc()) &&
+ IsSingleUseLoad(N->getOperand(1)) &&
+ IsSingleUseLoad(N->getOperand(2))) {
+ SDValue NewOp0 =
+ DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(1));
+ SDValue NewOp1 =
+ DAG.getNode(ISD::ZERO_EXTEND, DL, MVT::i64, N->getOperand(2));
+ SDValue Res = DAG.getNode(ISD::CLMUL, DL, MVT::i64, NewOp0, NewOp1);
+ Res = DAG.getNode(ISD::SRL, DL, MVT::i64, Res,
+ DAG.getConstant(32, DL, MVT::i64));
+ Results.push_back(DAG.getNode(ISD::TRUNCATE, DL, MVT::i32, Res));
+ return;
+ }
+
// Extend inputs to XLen, and shift by 32. This will add 64 trailing zeros
// to the full 128-bit clmul result of multiplying two xlen values.
// Perform clmulr or clmulh on the shifted values. Finally, extract the
diff --git a/llvm/test/CodeGen/RISCV/clmulh-load-rv64.ll b/llvm/test/CodeGen/RISCV/clmulh-load-rv64.ll
new file mode 100644
index 0000000000000..238f3ba1853cb
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/clmulh-load-rv64.ll
@@ -0,0 +1,277 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=riscv64 -mattr=+zbc -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZBC,NOA
+; RUN: llc -mtriple=riscv64 -mattr=+zbkc -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZBKC,NOA
+; RUN: llc -mtriple=riscv64 -mattr=+zbc,+a -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZBC,ATOMIC
+; RUN: llc -mtriple=riscv64 -mattr=+zbkc,+a -verify-machineinstrs < %s | FileCheck %s --check-prefixes=CHECK,ZBKC,ATOMIC
+
+declare i32 @llvm.riscv.clmulh.i32(i32, i32)
+declare i32 @llvm.riscv.clmulr.i32(i32, i32)
+
+; Zero extensions of simple single-use i32 loads can fold into lwu.
+define i32 @clmulh_load(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lwu a0, 0(a0)
+; CHECK-NEXT: lwu a1, 0(a1)
+; CHECK-NEXT: clmul a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ %b = load i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; The result still follows the signed i32 return convention when requested.
+define signext i32 @clmulh_load_signext(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_load_signext:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lwu a0, 0(a0)
+; CHECK-NEXT: lwu a1, 0(a1)
+; CHECK-NEXT: clmul a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ %b = load i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; Unaligned loads may need to be expanded before the intrinsic is legalized.
+define i32 @clmulh_unaligned_load(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_unaligned_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lbu a2, 1(a0)
+; CHECK-NEXT: lbu a3, 0(a0)
+; CHECK-NEXT: lbu a4, 2(a0)
+; CHECK-NEXT: lbu a0, 3(a0)
+; CHECK-NEXT: slli a2, a2, 8
+; CHECK-NEXT: lbu a5, 1(a1)
+; CHECK-NEXT: lbu a6, 2(a1)
+; CHECK-NEXT: lbu a7, 3(a1)
+; CHECK-NEXT: lbu a1, 0(a1)
+; CHECK-NEXT: or a2, a2, a3
+; CHECK-NEXT: slli a4, a4, 16
+; CHECK-NEXT: slli a0, a0, 24
+; CHECK-NEXT: or a0, a0, a4
+; CHECK-NEXT: slli a5, a5, 8
+; CHECK-NEXT: slli a6, a6, 16
+; CHECK-NEXT: slli a7, a7, 24
+; CHECK-NEXT: or a1, a5, a1
+; CHECK-NEXT: or a3, a7, a6
+; CHECK-NEXT: or a0, a0, a2
+; CHECK-NEXT: or a1, a3, a1
+; CHECK-NEXT: clmul a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 1
+ %b = load i32, ptr %q, align 1
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+define i32 @clmulh_register(i32 %a, i32 %b) {
+; CHECK-LABEL: clmulh_register:
+; CHECK: # %bb.0:
+; CHECK-NEXT: slli a1, a1, 32
+; CHECK-NEXT: slli a0, a0, 32
+; CHECK-NEXT: clmulh a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; Both operands must be freely zero extendable loads.
+define i32 @clmulh_load_register(ptr %p, i32 %b) {
+; CHECK-LABEL: clmulh_load_register:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lw a0, 0(a0)
+; CHECK-NEXT: slli a1, a1, 32
+; CHECK-NEXT: slli a0, a0, 32
+; CHECK-NEXT: clmulh a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+define i32 @clmulh_register_load(i32 %a, ptr %q) {
+; CHECK-LABEL: clmulh_register_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lw a1, 0(a1)
+; CHECK-NEXT: slli a0, a0, 32
+; CHECK-NEXT: slli a1, a1, 32
+; CHECK-NEXT: clmulh a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %b = load i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; An additional value use prevents changing the load's extension.
+define i32 @clmulh_multiuse_load(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_multiuse_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lw a0, 0(a0)
+; CHECK-NEXT: lw a1, 0(a1)
+; CHECK-NEXT: slli a1, a1, 32
+; CHECK-NEXT: slli a2, a0, 32
+; CHECK-NEXT: clmulh a1, a2, a1
+; CHECK-NEXT: srli a1, a1, 32
+; CHECK-NEXT: addw a0, a1, a0
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ %b = load i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ %sum = add i32 %r, %a
+ ret i32 %sum
+}
+
+; Volatile and atomic loads keep their original lowering.
+define i32 @clmulh_volatile_load(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_volatile_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lw a0, 0(a0)
+; CHECK-NEXT: lw a1, 0(a1)
+; CHECK-NEXT: slli a0, a0, 32
+; CHECK-NEXT: slli a1, a1, 32
+; CHECK-NEXT: clmulh a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load volatile i32, ptr %p, align 4
+ %b = load volatile i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+define i32 @clmulh_atomic_load(ptr %p, ptr %q) {
+; NOA-LABEL: clmulh_atomic_load:
+; NOA: # %bb.0:
+; NOA-NEXT: addi sp, sp, -32
+; NOA-NEXT: .cfi_def_cfa_offset 32
+; NOA-NEXT: sd ra, 24(sp) # 8-byte Folded Spill
+; NOA-NEXT: sd s0, 16(sp) # 8-byte Folded Spill
+; NOA-NEXT: sd s1, 8(sp) # 8-byte Folded Spill
+; NOA-NEXT: .cfi_offset ra, -8
+; NOA-NEXT: .cfi_offset s0, -16
+; NOA-NEXT: .cfi_offset s1, -24
+; NOA-NEXT: mv s0, a1
+; NOA-NEXT: li a1, 2
+; NOA-NEXT: call __atomic_load_4
+; NOA-NEXT: mv s1, a0
+; NOA-NEXT: li a1, 2
+; NOA-NEXT: mv a0, s0
+; NOA-NEXT: call __atomic_load_4
+; NOA-NEXT: slli s1, s1, 32
+; NOA-NEXT: slli a0, a0, 32
+; NOA-NEXT: clmulh a0, s1, a0
+; NOA-NEXT: srli a0, a0, 32
+; NOA-NEXT: ld ra, 24(sp) # 8-byte Folded Reload
+; NOA-NEXT: ld s0, 16(sp) # 8-byte Folded Reload
+; NOA-NEXT: ld s1, 8(sp) # 8-byte Folded Reload
+; NOA-NEXT: .cfi_restore ra
+; NOA-NEXT: .cfi_restore s0
+; NOA-NEXT: .cfi_restore s1
+; NOA-NEXT: addi sp, sp, 32
+; NOA-NEXT: .cfi_def_cfa_offset 0
+; NOA-NEXT: ret
+;
+; ATOMIC-LABEL: clmulh_atomic_load:
+; ATOMIC: # %bb.0:
+; ATOMIC-NEXT: lw a0, 0(a0)
+; ATOMIC-NEXT: fence r, rw
+; ATOMIC-NEXT: lw a1, 0(a1)
+; ATOMIC-NEXT: slli a0, a0, 32
+; ATOMIC-NEXT: slli a1, a1, 32
+; ATOMIC-NEXT: clmulh a0, a0, a1
+; ATOMIC-NEXT: srli a0, a0, 32
+; ATOMIC-NEXT: fence r, rw
+; ATOMIC-NEXT: ret
+ %a = load atomic i32, ptr %p acquire, align 4
+ %b = load atomic i32, ptr %q acquire, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; Using one load for both operands is not a single value use.
+define i32 @clmulh_same_load(ptr %p) {
+; CHECK-LABEL: clmulh_same_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lw a0, 0(a0)
+; CHECK-NEXT: slli a0, a0, 32
+; CHECK-NEXT: clmulh a0, a0, a0
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %a)
+ ret i32 %r
+}
+
+; A chain use does not prevent changing a single-use load's extension. The
+; aliasing store must remain between the first load and the second load.
+define i32 @clmulh_alias_chain(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_alias_chain:
+; CHECK: # %bb.0:
+; CHECK-NEXT: li a2, 7
+; CHECK-NEXT: lwu a3, 0(a0)
+; CHECK-NEXT: sw a2, 0(a0)
+; CHECK-NEXT: lwu a0, 0(a1)
+; CHECK-NEXT: clmul a0, a3, a0
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ store i32 7, ptr %p, align 4
+ %b = load i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; A load of a narrower memory type is outside the i32-load matcher.
+define i32 @clmulh_narrow_load(ptr %p, ptr %q) {
+; CHECK-LABEL: clmulh_narrow_load:
+; CHECK: # %bb.0:
+; CHECK-NEXT: lh a0, 0(a0)
+; CHECK-NEXT: lh a1, 0(a1)
+; CHECK-NEXT: slli a1, a1, 32
+; CHECK-NEXT: slli a0, a0, 32
+; CHECK-NEXT: clmulh a0, a0, a1
+; CHECK-NEXT: srli a0, a0, 32
+; CHECK-NEXT: ret
+ %a16 = load i16, ptr %p, align 2
+ %b16 = load i16, ptr %q, align 2
+ %a = sext i16 %a16 to i32
+ %b = sext i16 %b16 to i32
+ %r = call i32 @llvm.riscv.clmulh.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
+
+; CLMULR retains its existing lowering for i32 loads.
+define i32 @clmulr_load(ptr %p, ptr %q) {
+; ZBC-LABEL: clmulr_load:
+; ZBC: # %bb.0:
+; ZBC-NEXT: lw a0, 0(a0)
+; ZBC-NEXT: lw a1, 0(a1)
+; ZBC-NEXT: slli a1, a1, 32
+; ZBC-NEXT: slli a0, a0, 32
+; ZBC-NEXT: clmulr a0, a0, a1
+; ZBC-NEXT: srli a0, a0, 32
+; ZBC-NEXT: ret
+;
+; ZBKC-LABEL: clmulr_load:
+; ZBKC: # %bb.0:
+; ZBKC-NEXT: lw a0, 0(a0)
+; ZBKC-NEXT: lw a1, 0(a1)
+; ZBKC-NEXT: slli a1, a1, 32
+; ZBKC-NEXT: slli a0, a0, 32
+; ZBKC-NEXT: clmulh a0, a0, a1
+; ZBKC-NEXT: slli a0, a0, 1
+; ZBKC-NEXT: srli a0, a0, 32
+; ZBKC-NEXT: ret
+ %a = load i32, ptr %p, align 4
+ %b = load i32, ptr %q, align 4
+ %r = call i32 @llvm.riscv.clmulr.i32(i32 %a, i32 %b)
+ ret i32 %r
+}
``````````
</details>
https://github.com/llvm/llvm-project/pull/228669
More information about the llvm-commits
mailing list