[llvm-branch-commits] [clang] [llvm] [AArch64] Use 64-bit registers for wide "r" inline asm operands (PR #239127)

Bill Wendling via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Fri Nov 20 02:42:54 PST 2026


https://github.com/isanbard updated https://github.com/llvm/llvm-project/pull/239127

>From 01297f9d3490c48a79ecb047c6618ee7bc3f9499 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Wed, 4 Nov 2026 23:31:04 -0800
Subject: [PATCH 1/4] [AArch64] Use 64-bit registers for wide "r" inline asm
 operands

An integer wider than 64 bits, such as an i128, in an "r" operand was
given a 32-bit register for each 64 bits of it, so half of its bits were
lost. Give it 64-bit registers instead.

Fixes #238926.

Assisted-by: Claude Opus 5.5
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  8 +++-
 llvm/test/CodeGen/AArch64/build-pair-isel.ll  |  7 +---
 .../CodeGen/AArch64/inline-asm-wide-gpr.ll    | 41 +++++++++++++++++++
 3 files changed, 50 insertions(+), 6 deletions(-)
 create mode 100644 llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index a51e6bb18cdd8e..741cad2f162a09 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -14484,8 +14484,14 @@ AArch64TargetLowering::getRegForInlineAsmConstraint(
         return std::make_pair(0U, nullptr);
       if (Subtarget->hasLS64() && VT.getSizeInBits() == 512)
         return std::make_pair(0U, &AArch64::GPR64x8ClassRegClass);
-      if (VT.getFixedSizeInBits() == 64)
+
+      // An integer wider than 64 bits, such as an i128, takes a 64-bit
+      // register for each 64 bits of it. 32-bit registers would hold only half
+      // of it.
+      if (VT.getFixedSizeInBits() == 64 ||
+          (VT.isScalarInteger() && VT.getFixedSizeInBits() > 64))
         return std::make_pair(0U, &AArch64::GPR64commonRegClass);
+
       return std::make_pair(0U, &AArch64::GPR32commonRegClass);
     case 'w': {
       if (!Subtarget->hasFPARMv8())
diff --git a/llvm/test/CodeGen/AArch64/build-pair-isel.ll b/llvm/test/CodeGen/AArch64/build-pair-isel.ll
index cd68bde0f8da73..ee5fc8b7df13cf 100644
--- a/llvm/test/CodeGen/AArch64/build-pair-isel.ll
+++ b/llvm/test/CodeGen/AArch64/build-pair-isel.ll
@@ -10,11 +10,8 @@ define void @compare_and_swap128() {
 ; CHECK-NEXT:    //APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    //NO_APP
-; CHECK-NEXT:    // implicit-def: $x9
-; CHECK-NEXT:    mov w9, w10
-; CHECK-NEXT:    mov w8, w8
-; CHECK-NEXT:    // kill: def $x8 killed $w8
-; CHECK-NEXT:    orr x8, x8, x9, lsl #32
+; CHECK-NEXT:    // implicit-def: $x10
+; CHECK-NEXT:    str x9, [x10]
 ; CHECK-NEXT:    // implicit-def: $x9
 ; CHECK-NEXT:    str x8, [x9]
 ; CHECK-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
new file mode 100644
index 00000000000000..05144bcdd69714
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
@@ -0,0 +1,41 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s
+
+; An integer wider than 64 bits in an "r" operand goes in 64-bit registers,
+; one for each 64 bits of it. 32-bit registers would hold only half of it.
+
+define void @test_i128_input(i128 %x) {
+; CHECK-LABEL: test_i128_input:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    // x0
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    ret
+entry:
+  call void asm sideeffect "// $0", "r"(i128 %x)
+  ret void
+}
+
+define i128 @test_i128_output() {
+; CHECK-LABEL: test_i128_output:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    // x0
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    ret
+entry:
+  %r = call i128 asm sideeffect "// $0", "=r"()
+  ret i128 %r
+}
+
+; The asm doesn't change its operand, so the function returns its argument.
+define i128 @test_i128_tied(i128 %x) {
+; CHECK-LABEL: test_i128_tied:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    ret
+entry:
+  %r = call i128 asm sideeffect "", "=&r,0"(i128 %x)
+  ret i128 %r
+}

>From a3644124427fd6094aff3e16b552642c22d29fcd Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Wed, 18 Nov 2026 11:13:28 -0800
Subject: [PATCH 2/4] [AArch64] Add the H modifier for the second register of a
 128-bit operand

A 128-bit integer in an "r" operand takes two X registers, but an asm
could name only the first, so it couldn't use the other half. GCC has
the H modifier for this, so support it too.

Assisted-by: Claude Opus 5.5
---
 llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp | 18 ++++++
 .../AArch64/inline-asm-wide-gpr-error.ll      | 18 ++++++
 .../CodeGen/AArch64/inline-asm-wide-gpr.ll    | 55 ++++++++++++++++++-
 3 files changed, 90 insertions(+), 1 deletion(-)
 create mode 100644 llvm/test/CodeGen/AArch64/inline-asm-wide-gpr-error.ll

diff --git a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
index 0df6c2cae06303..83bc5e3f1f70f0 100644
--- a/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
+++ b/llvm/lib/Target/AArch64/AArch64AsmPrinter.cpp
@@ -48,6 +48,7 @@
 #include "llvm/IR/Analysis.h"
 #include "llvm/IR/DataLayout.h"
 #include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/InlineAsm.h"
 #include "llvm/IR/Mangler.h"
 #include "llvm/IR/Module.h"
 #include "llvm/IR/PassManager.h"
@@ -1290,6 +1291,23 @@ bool AArch64AsmPrinter::PrintAsmOperand(const MachineInstr *MI, unsigned OpNum,
       }
       printOperand(MI, OpNum, O);
       return false;
+    case 'H': { // Print the second X register of a 128-bit operand.
+      // A 128-bit integer in an "r" operand takes two X registers, and $N
+      // prints the first, which holds the doubleword at the lower address.
+      // 'H' prints the other, as in GCC, where the two are an even/odd pair.
+      // Here they needn't be. The operand before the first register is the
+      // flag that says how many registers there are.
+      const InlineAsm::Flag F(MI->getOperand(OpNum - 1).getImm());
+      if (F.getNumOperandRegisters() != 2)
+        return true;
+
+      const MachineOperand &SecondMO = MI->getOperand(OpNum + 1);
+      if (!SecondMO.isReg() ||
+          !AArch64::GPR64allRegClass.contains(SecondMO.getReg()))
+        return true;
+
+      return printAsmMRegister(SecondMO, 'x', O);
+    }
     case 'b': // Print B register.
     case 'h': // Print H register.
     case 's': // Print S register.
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr-error.ll b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr-error.ll
new file mode 100644
index 00000000000000..c16a613d52ed99
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr-error.ll
@@ -0,0 +1,18 @@
+; RUN: not llc -mtriple=aarch64-linux-gnu < %s 2>&1 | FileCheck %s
+
+; %H names the second register of an operand that takes two, such as an i128,
+; so it's an error on an operand that takes one, or more than two.
+
+; CHECK: error: invalid operand in inline asm: '// ${0:H}'
+define void @test_one_register(i64 %x) {
+entry:
+  call void asm sideeffect "// ${0:H}", "r"(i64 %x)
+  ret void
+}
+
+; CHECK: error: invalid operand in inline asm: '// ${0:H} four'
+define void @test_four_registers(i256 %x) {
+entry:
+  call void asm sideeffect "// ${0:H} four", "r"(i256 %x)
+  ret void
+}
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
index 05144bcdd69714..78d948acf108fa 100644
--- a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
+++ b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
@@ -1,8 +1,11 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s
+; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,LE
+; RUN: llc -mtriple=aarch64_be-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,BE
 
 ; An integer wider than 64 bits in an "r" operand goes in 64-bit registers,
 ; one for each 64 bits of it. 32-bit registers would hold only half of it.
+; Big-endian puts the halves of an i128 in the other order (see
+; @test_i128_high_half).
 
 define void @test_i128_input(i128 %x) {
 ; CHECK-LABEL: test_i128_input:
@@ -39,3 +42,53 @@ entry:
   %r = call i128 asm sideeffect "", "=&r,0"(i128 %x)
   ret i128 %r
 }
+
+; %H names the second register of an i128, so an asm can use both halves.
+define void @test_i128_store(ptr %p, i128 %x) {
+; CHECK-LABEL: test_i128_store:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    stp x2, x3, [x0]
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    ret
+entry:
+  call void asm sideeffect "stp $1, ${1:H}, [$0]", "r,r,~{memory}"(ptr %p, i128 %x)
+  ret void
+}
+
+define i128 @test_i128_load(ptr %p) {
+; CHECK-LABEL: test_i128_load:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    ldp x8, x1, [x0]
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    mov x0, x8
+; CHECK-NEXT:    ret
+entry:
+  %r = call i128 asm sideeffect "ldp $0, ${0:H}, [$1]", "=&r,r,~{memory}"(ptr %p)
+  ret i128 %r
+}
+
+; As in GCC, the first register holds the doubleword at the lower address. So
+; the high half is in the second register on little-endian, and in the first
+; on big-endian.
+define i64 @test_i128_high_half() {
+; LE-LABEL: test_i128_high_half:
+; LE:       // %bb.0: // %entry
+; LE-NEXT:    //APP
+; LE-NEXT:    // x8 x0
+; LE-NEXT:    //NO_APP
+; LE-NEXT:    ret
+;
+; BE-LABEL: test_i128_high_half:
+; BE:       // %bb.0: // %entry
+; BE-NEXT:    //APP
+; BE-NEXT:    // x0 x8
+; BE-NEXT:    //NO_APP
+; BE-NEXT:    ret
+entry:
+  %r = call i128 asm sideeffect "// $0 ${0:H}", "=r"()
+  %hi = lshr i128 %r, 64
+  %t = trunc i128 %hi to i64
+  ret i64 %t
+}

>From 8c5060157d05b2ae061a582b7868e8040907a52c Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Wed, 18 Nov 2026 11:13:28 -0800
Subject: [PATCH 3/4] [Clang][AArch64] Don't warn about a 128-bit "r" operand

Clang warned that a 128-bit value in an "r" operand didn't fit its
register and suggested %w. But the value takes two X registers, and %H
names the second, so don't warn about it.

Assisted-by: Claude Opus 5.5
---
 clang/lib/Basic/Targets/AArch64.cpp | 4 ++++
 clang/test/Sema/arm64-inline-asm.c  | 5 +++++
 2 files changed, 9 insertions(+)

diff --git a/clang/lib/Basic/Targets/AArch64.cpp b/clang/lib/Basic/Targets/AArch64.cpp
index 7ddf9d87ad163e..7474b2a7649965 100644
--- a/clang/lib/Basic/Targets/AArch64.cpp
+++ b/clang/lib/Basic/Targets/AArch64.cpp
@@ -1766,6 +1766,10 @@ bool AArch64TargetInfo::validateConstraintModifier(
       if (Size == 64)
         return true;
 
+      // A 128-bit value takes two 'x' registers, and %H names the second.
+      if (Size == 128)
+        return true;
+
       if (Size == 512)
         return HasLS64;
 
diff --git a/clang/test/Sema/arm64-inline-asm.c b/clang/test/Sema/arm64-inline-asm.c
index c804478ea68d55..22757faa3bc770 100644
--- a/clang/test/Sema/arm64-inline-asm.c
+++ b/clang/test/Sema/arm64-inline-asm.c
@@ -13,3 +13,8 @@ void test_clobber_conflict(void) {
   asm volatile("nop" :: "r"(x) : "%x1"); // expected-error {{conflicts with asm clobber list}}
   asm volatile("nop" : "=r"(x) :: "%x1"); // expected-error {{conflicts with asm clobber list}}
 }
+
+// A 128-bit value takes two 64-bit registers, and %H names the second.
+void test_int128(unsigned __int128 *p, unsigned __int128 x) {
+  asm volatile("stp %1, %H1, [%0]" :: "r"(p), "r"(x) : "memory");
+}

>From 17bd37dfeac9b55b54c2ab96a30aa055252d5fa7 Mon Sep 17 00:00:00 2001
From: Bill Wendling <isanbard at gmail.com>
Date: Thu, 19 Nov 2026 21:05:14 -0800
Subject: [PATCH 4/4] [AArch64] Use 64-bit registers for wide FP and vector "r"
 operands

An fp128 or a 128-bit vector, such as a <2 x double>, in an "r" operand
was still given 32-bit registers, so half of its bits were lost, as with
an i128 before. Give it 64-bit registers too.

Assisted-by: Claude Opus 5.5
---
 .../Target/AArch64/AArch64ISelLowering.cpp    |  9 ++-
 llvm/test/CodeGen/AArch64/fmlal-loreg.ll      |  7 +--
 .../CodeGen/AArch64/inline-asm-wide-gpr.ll    | 63 ++++++++++++++++++-
 3 files changed, 68 insertions(+), 11 deletions(-)

diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index fc01dcfb402619..9f9b4afb31af6a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -14488,11 +14488,10 @@ AArch64TargetLowering::getRegForInlineAsmConstraint(
       if (Subtarget->hasLS64() && VT.getSizeInBits() == 512)
         return std::make_pair(0U, &AArch64::GPR64x8ClassRegClass);
 
-      // An integer wider than 64 bits, such as an i128, takes a 64-bit
-      // register for each 64 bits of it. 32-bit registers would hold only half
-      // of it.
-      if (VT.getFixedSizeInBits() == 64 ||
-          (VT.isScalarInteger() && VT.getFixedSizeInBits() > 64))
+      // A value wider than 64 bits, such as an i128 or an fp128, takes a
+      // 64-bit register for each 64 bits of it. 32-bit registers would hold
+      // only half of it.
+      if (VT.getFixedSizeInBits() >= 64)
         return std::make_pair(0U, &AArch64::GPR64commonRegClass);
 
       return std::make_pair(0U, &AArch64::GPR32commonRegClass);
diff --git a/llvm/test/CodeGen/AArch64/fmlal-loreg.ll b/llvm/test/CodeGen/AArch64/fmlal-loreg.ll
index 20737a73183944..18988969684484 100644
--- a/llvm/test/CodeGen/AArch64/fmlal-loreg.ll
+++ b/llvm/test/CodeGen/AArch64/fmlal-loreg.ll
@@ -10,16 +10,15 @@ define <4 x float> @test(ptr %lhs_panel, ptr %rhs_panel, <4 x float> %a) {
 ; CHECK-NEXT:    str d8, [sp, #-16]! // 8-byte Folded Spill
 ; CHECK-NEXT:    .cfi_def_cfa_offset 16
 ; CHECK-NEXT:    .cfi_offset b8, -16
-; CHECK-NEXT:    fmov x8, d0
+; CHECK-NEXT:    mov x8, v0.d[1]
+; CHECK-NEXT:    fmov x9, d0
 ; CHECK-NEXT:    ldr q8, [x0]
 ; CHECK-NEXT:    ldr q16, [x1]
-; CHECK-NEXT:    lsr x9, x8, #32
 ; CHECK-NEXT:    //APP
 ; CHECK-NEXT:    nop
 ; CHECK-NEXT:    //NO_APP
-; CHECK-NEXT:    mov w8, w8
-; CHECK-NEXT:    orr x8, x8, x9, lsl #32
 ; CHECK-NEXT:    fmov d0, x8
+; CHECK-NEXT:    mov v0.d[1], x9
 ; CHECK-NEXT:    fmlal v0.4s, v16.4h, v8.h[0]
 ; CHECK-NEXT:    mov v1.16b, v0.16b
 ; CHECK-NEXT:    fmlal2 v1.4s, v16.4h, v8.h[0]
diff --git a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
index 78d948acf108fa..659e40854903dc 100644
--- a/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
+++ b/llvm/test/CodeGen/AArch64/inline-asm-wide-gpr.ll
@@ -2,8 +2,9 @@
 ; RUN: llc -mtriple=aarch64-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,LE
 ; RUN: llc -mtriple=aarch64_be-linux-gnu < %s | FileCheck %s --check-prefixes=CHECK,BE
 
-; An integer wider than 64 bits in an "r" operand goes in 64-bit registers,
-; one for each 64 bits of it. 32-bit registers would hold only half of it.
+; A value wider than 64 bits in an "r" operand, such as an i128 or an fp128,
+; goes in 64-bit registers, one for each 64 bits of it. 32-bit registers would
+; hold only half of it.
 ; Big-endian puts the halves of an i128 in the other order (see
 ; @test_i128_high_half).
 
@@ -92,3 +93,61 @@ entry:
   %t = trunc i128 %hi to i64
   ret i64 %t
 }
+
+; A floating-point value or a vector goes in X registers the same way, so
+; "stp $1, ${1:H}" stores the same bytes that a plain store of it would.
+define void @test_fp128_store(ptr %p, fp128 %x) {
+; CHECK-LABEL: test_fp128_store:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    str q0, [sp, #-16]!
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    ldp x8, x9, [sp]
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    stp x8, x9, [x0]
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    ret
+entry:
+  call void asm sideeffect "stp $1, ${1:H}, [$0]", "r,r,~{memory}"(ptr %p, fp128 %x)
+  ret void
+}
+
+define fp128 @test_fp128_load(ptr %p) {
+; CHECK-LABEL: test_fp128_load:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    //APP
+; CHECK-NEXT:    ldp x8, x9, [x0]
+; CHECK-NEXT:    //NO_APP
+; CHECK-NEXT:    stp x8, x9, [sp, #-16]!
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    ldr q0, [sp], #16
+; CHECK-NEXT:    ret
+entry:
+  %r = call fp128 asm sideeffect "ldp $0, ${0:H}, [$1]", "=&r,r,~{memory}"(ptr %p)
+  ret fp128 %r
+}
+
+define void @test_v2f64_store(ptr %p, <2 x double> %x) {
+; LE-LABEL: test_v2f64_store:
+; LE:       // %bb.0: // %entry
+; LE-NEXT:    mov x8, v0.d[1]
+; LE-NEXT:    fmov x9, d0
+; LE-NEXT:    //APP
+; LE-NEXT:    stp x9, x8, [x0]
+; LE-NEXT:    //NO_APP
+; LE-NEXT:    ret
+;
+; BE-LABEL: test_v2f64_store:
+; BE:       // %bb.0: // %entry
+; BE-NEXT:    str q0, [sp, #-16]!
+; BE-NEXT:    .cfi_def_cfa_offset 16
+; BE-NEXT:    ldp x8, x9, [sp]
+; BE-NEXT:    //APP
+; BE-NEXT:    stp x8, x9, [x0]
+; BE-NEXT:    //NO_APP
+; BE-NEXT:    add sp, sp, #16
+; BE-NEXT:    ret
+entry:
+  call void asm sideeffect "stp $1, ${1:H}, [$0]", "r,r,~{memory}"(ptr %p, <2 x double> %x)
+  ret void
+}



More information about the llvm-branch-commits mailing list