[llvm-branch-commits] [clang] [llvm] [AArch64] Use 64-bit registers for wide "r" inline asm operands (PR #239127)
Bill Wendling via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Thu Nov 19 22:09:30 PST 2026
https://github.com/isanbard updated https://github.com/llvm/llvm-project/pull/239127
>From 01297f9d3490c48a79ecb047c6618ee7bc3f9499 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Wed, 4 Nov 2026 23:31:04 -0800
Subject: [PATCH 1/4] [AArch64] Use 64-bit registers for wide "r" inline asm
operands
An integer wider than 64 bits, such as an i128, in an "r" operand was
given a 32-bit register for each 64 bits of it, so half of its bits were
lost. Give it 64-bit registers instead.
Fixes #238926.
Assisted-by: Claude Opus 5.5
---
.../Target/AArch64/AArch64ISelLowering.cpp | 8 +++-
llvm/test/CodeGen/AArch64/build-pair-isel.ll | 7 +---
.../CodeGen/AArch64/inline-asm-wide-gpr.ll | 41 +++++++++++++++++++
3 files changed, 50 insertions(+), 6 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index a51e6bb18cdd8e..741cad2f162a09 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -14484,8 +14484,14 @@ AArch64TargetLowering::getRegForInlineAsmConstraint(
return std::make_pair(0U, nullptr);
if (Subtarget->hasLS64() && VT.getSizeInBits() == 512)
return std::make_pair(0U, &AArch64::GPR64x8ClassRegClass);
- if (VT.getFixedSizeInBits() == 64)
+
+ // An integer wider than 64 bits, such as an i128, takes a 64-bit
+ // register for each 64 bits of it. 32-bit registers would hold only half
+ // of it.
+ if (VT.getFixedSizeInBits() == 64 ||
+ (VT.isScalarInteger() && VT.getFixedSizeInBits() > 64))
return std::make_pair(0U, &AArch64::GPR64commonRegClass);
+
return std::make_pair(0U, &AArch64::GPR32commonRegClass);
case 'w': {
if (!Subtarget->hasFPARMv8())
diff --git a/llvm/test/CodeGen/AArch64/build-pair-isel.ll b/llvm/test/CodeGen/AArch64/build-pair-isel.ll
index cd68bde0f8da73..ee5fc8b7df13cf 100644
--- a/llvm/test/CodeGen/AArch64/build-pair-isel.ll
+++ b/llvm/test/CodeGen/AArch64/build-pair-isel.ll
@@ -10,11 +10,8 @@ define void @compare_and_swap128() {
; CHECK-NEXT: //APP
; CHECK-NEXT: nop
; CHECK-NEXT: //NO_APP
-; CHECK-NEXT: // implicit-def: $x9
-; CHECK-NEXT: mov w9, w10
-; CHECK-NEXT: mov w8, w8
-; CHECK-NEXT: // kill: def $x8 killed $w8
-; CHECK-NEXT: orr x8, x8, x9, lsl #32
+; CHECK-NEXT: // implicit-def: $x10
+; CHECK-NEXT: str x9, [x10]
; CHECK-NEXT: // implicit-def: $x9
; CHECK-NEXT: str x8, [x9]
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
new file mode 100644
index 00000000000000..05144bcdd69714
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s
+
+; An integer wider than 64 bits in an "r" operand goes in 64-bit registers,
+; one for each 64 bits of it. 32-bit registers would hold only half of it.
+
+define void @test_i128_input(i128 %x) {
+; CHECK-LABEL: test_i128_input:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: //APP
+; CHECK-NEXT: // x0
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: ret
+entry:
+ call void asm sideeffect "// $0", "r"(i128 %x)
+ ret void
+}
+
+define i128 @test_i128_output() {
+; CHECK-LABEL: test_i128_output:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: //APP
+; CHECK-NEXT: // x0
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: ret
+entry:
+ %r = call i128 asm sideeffect "// $0", "=r"()
+ ret i128 %r
+}
+
+; The asm doesn't change its operand, so the function returns its argument.
+define i128 @test_i128_tied(i128 %x) {
+; CHECK-LABEL: test_i128_tied:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: //APP
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: ret
+entry:
+ %r = call i128 asm sideeffect "", "=&r,0"(i128 %x)
+ ret i128 %r
+}
>From a3644124427fd6094aff3e16b552642c22d29fcd Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Wed, 18 Nov 2026 11:13:28 -0800
Subject: [PATCH 2/4] [AArch64] Add the H modifier for the second register of a
128-bit operand
A 128-bit integer in an "r" operand takes two X registers, but an asm
could name only the first, so it couldn't use the other half. GCC has
the H modifier for this, so support it too.
Assisted-by: Claude Opus 5.5
---
llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 18 ++++++
.../AArch64/inline-asm-wide-gpr-error.ll | 18 ++++++
.../CodeGen/AArch64/inline-asm-wide-gpr.ll | 55 ++++++++++++++++++-
3 files changed, 90 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AArch64/inline-asm-wide-gpr-error.ll
diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 0df6c2cae06303..83bc5e3f1f70f0 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -48,6 +48,7 @@
#include "llvm/IR/Analysis.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/InlineAsm.h"
#include "llvm/IR/Mangler.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/PassManager.h"
@@ -1290,6 +1291,23 @@ bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
}
printOperand(MI, OpNum, O);
return false;
+ case 'H': { // Print the second X register of a 128-bit operand.
+ // A 128-bit integer in an "r" operand takes two X registers, and $N
+ // prints the first, which holds the doubleword at the lower address.
+ // 'H' prints the other, as in GCC, where the two are an even/odd pair.
+ // Here they needn't be. The operand before the first register is the
+ // flag that says how many registers there are.
+ const InlineAsm::Flag F(MI->getOperand(OpNum - 1).getImm());
+ if (F.getNumOperandRegisters() != 2)
+ return true;
+
+ const MachineOperand &SecondMO = MI->getOperand(OpNum + 1);
+ if (!SecondMO.isReg() ||
+ !AArch64::GPR64allRegClass.contains(SecondMO.getReg()))
+ return true;
+
+ return printAsmMRegister(SecondMO, 'x', O);
+ }
case 'b': // Print B register.
case 'h': // Print H register.
case 's': // Print S register.
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr-error.ll b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr-error.ll
new file mode 100644
index 00000000000000..c16a613d52ed99
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr-error.ll
@@ -0,0 +1,18 @@
+; RUN: not llc -mtriple=aarch64-linux-gnu < %s 2>&1 | FileCheck %s
+
+; %H names the second register of an operand that takes two, such as an i128,
+; so it's an error on an operand that takes one, or more than two.
+
+; CHECK: error: invalid operand in inline asm: '// ${0:H}'
+define void @test_one_register(i64 %x) {
+entry:
+ call void asm sideeffect "// ${0:H}", "r"(i64 %x)
+ ret void
+}
+
+; CHECK: error: invalid operand in inline asm: '// ${0:H} four'
+define void @test_four_registers(i256 %x) {
+entry:
+ call void asm sideeffect "// ${0:H} four", "r"(i256 %x)
+ ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
index 05144bcdd69714..78d948acf108fa 100644
--- a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
+++ b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
@@ -1,8 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,LE
+; RUN: llc -mtriple=aarch64_be-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,BE
; An integer wider than 64 bits in an "r" operand goes in 64-bit registers,
; one for each 64 bits of it. 32-bit registers would hold only half of it.
+; Big-endian puts the halves of an i128 in the other order (see
+; @test_i128_high_half).
define void @test_i128_input(i128 %x) {
; CHECK-LABEL: test_i128_input:
@@ -39,3 +42,53 @@ entry:
%r = call i128 asm sideeffect "", "=&r,0"(i128 %x)
ret i128 %r
}
+
+; %H names the second register of an i128, so an asm can use both halves.
+define void @test_i128_store(ptr %p, i128 %x) {
+; CHECK-LABEL: test_i128_store:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: //APP
+; CHECK-NEXT: stp x2, x3, [x0]
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: ret
+entry:
+ call void asm sideeffect "stp $1, ${1:H}, [$0]", "r,r,~{memory}"(ptr %p, i128 %x)
+ ret void
+}
+
+define i128 @test_i128_load(ptr %p) {
+; CHECK-LABEL: test_i128_load:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: //APP
+; CHECK-NEXT: ldp x8, x1, [x0]
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: mov x0, x8
+; CHECK-NEXT: ret
+entry:
+ %r = call i128 asm sideeffect "ldp $0, ${0:H}, [$1]", "=&r,r,~{memory}"(ptr %p)
+ ret i128 %r
+}
+
+; As in GCC, the first register holds the doubleword at the lower address. So
+; the high half is in the second register on little-endian, and in the first
+; on big-endian.
+define i64 @test_i128_high_half() {
+; LE-LABEL: test_i128_high_half:
+; LE: // %bb.0: // %entry
+; LE-NEXT: //APP
+; LE-NEXT: // x8 x0
+; LE-NEXT: //NO_APP
+; LE-NEXT: ret
+;
+; BE-LABEL: test_i128_high_half:
+; BE: // %bb.0: // %entry
+; BE-NEXT: //APP
+; BE-NEXT: // x0 x8
+; BE-NEXT: //NO_APP
+; BE-NEXT: ret
+entry:
+ %r = call i128 asm sideeffect "// $0 ${0:H}", "=r"()
+ %hi = lshr i128 %r, 64
+ %t = trunc i128 %hi to i64
+ ret i64 %t
+}
>From 8c5060157d05b2ae061a582b7868e8040907a52c Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Wed, 18 Nov 2026 11:13:28 -0800
Subject: [PATCH 3/4] [Clang][AArch64] Don't warn about a 128-bit "r" operand
Clang warned that a 128-bit value in an "r" operand didn't fit its
register and suggested %w. But the value takes two X registers, and %H
names the second, so don't warn about it.
Assisted-by: Claude Opus 5.5
---
clang/lib/Basic/Targets/AArch64.cpp | 4 ++++
clang/test/Sema/arm64-inline-asm.c | 5 +++++
2 files changed, 9 insertions(+)
diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 7ddf9d87ad163e..7474b2a7649965 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -1766,6 +1766,10 @@ bool AArch64TargetInfo::validateConstraintModifier(
if (Size == 64)
return true;
+ // A 128-bit value takes two 'x' registers, and %H names the second.
+ if (Size == 128)
+ return true;
+
if (Size == 512)
return HasLS64;
diff --git a/clang/test/Sema/arm64-inline-asm.c b/clang/test/Sema/arm64-inline-asm.c
index c804478ea68d55..22757faa3bc770 100644
--- a/clang/test/Sema/arm64-inline-asm.c
+++ b/clang/test/Sema/arm64-inline-asm.c
@@ -13,3 +13,8 @@ void test_clobber_conflict(void) {
asm volatile("nop" :: "r"(x) : "%x1"); // expected-error {{conflicts with asm clobber list}}
asm volatile("nop" : "=r"(x) :: "%x1"); // expected-error {{conflicts with asm clobber list}}
}
+
+// A 128-bit value takes two 64-bit registers, and %H names the second.
+void test_int128(unsigned __int128 *p, unsigned __int128 x) {
+ asm volatile("stp %1, %H1, [%0]" :: "r"(p), "r"(x) : "memory");
+}
>From 17bd37dfeac9b55b54c2ab96a30aa055252d5fa7 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Thu, 19 Nov 2026 21:05:14 -0800
Subject: [PATCH 4/4] [AArch64] Use 64-bit registers for wide FP and vector "r"
operands
An fp128 or a 128-bit vector, such as a <2 x double>, in an "r" operand
was still given 32-bit registers, so half of its bits were lost, as with
an i128 before. Give it 64-bit registers too.
Assisted-by: Claude Opus 5.5
---
.../Target/AArch64/AArch64ISelLowering.cpp | 9 ++-
llvm/test/CodeGen/AArch64/fmlal-loreg.ll | 7 +--
.../CodeGen/AArch64/inline-asm-wide-gpr.ll | 63 ++++++++++++++++++-
3 files changed, 68 insertions(+), 11 deletions(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index fc01dcfb402619..9f9b4afb31af6a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -14488,11 +14488,10 @@ AArch64TargetLowering::getRegForInlineAsmConstraint(
if (Subtarget->hasLS64() && VT.getSizeInBits() == 512)
return std::make_pair(0U, &AArch64::GPR64x8ClassRegClass);
- // An integer wider than 64 bits, such as an i128, takes a 64-bit
- // register for each 64 bits of it. 32-bit registers would hold only half
- // of it.
- if (VT.getFixedSizeInBits() == 64 ||
- (VT.isScalarInteger() && VT.getFixedSizeInBits() > 64))
+ // A value wider than 64 bits, such as an i128 or an fp128, takes a
+ // 64-bit register for each 64 bits of it. 32-bit registers would hold
+ // only half of it.
+ if (VT.getFixedSizeInBits() >= 64)
return std::make_pair(0U, &AArch64::GPR64commonRegClass);
return std::make_pair(0U, &AArch64::GPR32commonRegClass);
diff --git a/llvm/test/CodeGen/AArch64/fmlal-loreg.ll b/llvm/test/CodeGen/AArch64/fmlal-loreg.ll
index 20737a73183944..18988969684484 100644
--- a/llvm/test/CodeGen/AArch64/fmlal-loreg.ll
+++ b/llvm/test/CodeGen/AArch64/fmlal-loreg.ll
@@ -10,16 +10,15 @@ define <4 x float> @test(ptr %lhs_panel, ptr %rhs_panel, <4 x float> %a) {
; CHECK-NEXT: str d8, [sp, #-16]! // 8-byte Folded Spill
; CHECK-NEXT: .cfi_def_cfa_offset 16
; CHECK-NEXT: .cfi_offset b8, -16
-; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: mov x8, v0.d[1]
+; CHECK-NEXT: fmov x9, d0
; CHECK-NEXT: ldr q8, [x0]
; CHECK-NEXT: ldr q16, [x1]
-; CHECK-NEXT: lsr x9, x8, #32
; CHECK-NEXT: //APP
; CHECK-NEXT: nop
; CHECK-NEXT: //NO_APP
-; CHECK-NEXT: mov w8, w8
-; CHECK-NEXT: orr x8, x8, x9, lsl #32
; CHECK-NEXT: fmov d0, x8
+; CHECK-NEXT: mov v0.d[1], x9
; CHECK-NEXT: fmlal v0.4s, v16.4h, v8.h[0]
; CHECK-NEXT: mov v1.16b, v0.16b
; CHECK-NEXT: fmlal2 v1.4s, v16.4h, v8.h[0]
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
index 78d948acf108fa..659e40854903dc 100644
--- a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
+++ b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
@@ -2,8 +2,9 @@
; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,LE
; RUN: llc -mtriple=aarch64_be-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,BE
-; An integer wider than 64 bits in an "r" operand goes in 64-bit registers,
-; one for each 64 bits of it. 32-bit registers would hold only half of it.
+; A value wider than 64 bits in an "r" operand, such as an i128 or an fp128,
+; goes in 64-bit registers, one for each 64 bits of it. 32-bit registers would
+; hold only half of it.
; Big-endian puts the halves of an i128 in the other order (see
; @test_i128_high_half).
@@ -92,3 +93,61 @@ entry:
%t = trunc i128 %hi to i64
ret i64 %t
}
+
+; A floating-point value or a vector goes in X registers the same way, so
+; "stp $1, ${1:H}" stores the same bytes that a plain store of it would.
+define void @test_fp128_store(ptr %p, fp128 %x) {
+; CHECK-LABEL: test_fp128_store:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: str q0, [sp, #-16]!
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: ldp x8, x9, [sp]
+; CHECK-NEXT: //APP
+; CHECK-NEXT: stp x8, x9, [x0]
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: add sp, sp, #16
+; CHECK-NEXT: ret
+entry:
+ call void asm sideeffect "stp $1, ${1:H}, [$0]", "r,r,~{memory}"(ptr %p, fp128 %x)
+ ret void
+}
+
+define fp128 @test_fp128_load(ptr %p) {
+; CHECK-LABEL: test_fp128_load:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: //APP
+; CHECK-NEXT: ldp x8, x9, [x0]
+; CHECK-NEXT: //NO_APP
+; CHECK-NEXT: stp x8, x9, [sp, #-16]!
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: ldr q0, [sp], #16
+; CHECK-NEXT: ret
+entry:
+ %r = call fp128 asm sideeffect "ldp $0, ${0:H}, [$1]", "=&r,r,~{memory}"(ptr %p)
+ ret fp128 %r
+}
+
+define void @test_v2f64_store(ptr %p, <2 x double> %x) {
+; LE-LABEL: test_v2f64_store:
+; LE: // %bb.0: // %entry
+; LE-NEXT: mov x8, v0.d[1]
+; LE-NEXT: fmov x9, d0
+; LE-NEXT: //APP
+; LE-NEXT: stp x9, x8, [x0]
+; LE-NEXT: //NO_APP
+; LE-NEXT: ret
+;
+; BE-LABEL: test_v2f64_store:
+; BE: // %bb.0: // %entry
+; BE-NEXT: str q0, [sp, #-16]!
+; BE-NEXT: .cfi_def_cfa_offset 16
+; BE-NEXT: ldp x8, x9, [sp]
+; BE-NEXT: //APP
+; BE-NEXT: stp x8, x9, [x0]
+; BE-NEXT: //NO_APP
+; BE-NEXT: add sp, sp, #16
+; BE-NEXT: ret
+entry:
+ call void asm sideeffect "stp $1, ${1:H}, [$0]", "r,r,~{memory}"(ptr %p, <2 x double> %x)
+ ret void
+}
More information about the llvm-branch-commits
mailing list