[llvm] [CodeGen][AArch64] Use vector parts for internal non-power-of-two vectors (PR #213982)
Hari Limaye via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 7 08:32:35 PDT 2026
https://github.com/hazzlim updated https://github.com/llvm/llvm-project/pull/213982
>From 5537b4c897c493d2ebb64ef523cd23363f0c2370 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Mon, 3 Aug 2026 10:28:50 +0000
Subject: [PATCH 1/6] Add test
---
.../CodeGen/AArch64/non-pow2-fixed-vectors.ll | 672 ++++++++++++++++++
1 file changed, 672 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
diff --git a/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
new file mode 100644
index 0000000000000..047a377710ecf
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
@@ -0,0 +1,672 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64-linux-gnu -O3 | FileCheck %s
+
+define void @v5i32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v5i32_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp w12, w13, [x0]
+; CHECK-NEXT: mov w1, wzr
+; CHECK-NEXT: mov w11, wzr
+; CHECK-NEXT: mov w9, wzr
+; CHECK-NEXT: mov w8, wzr
+; CHECK-NEXT: mov w10, wzr
+; CHECK-NEXT: fmov s0, w12
+; CHECK-NEXT: mov v0.s[1], w13
+; CHECK-NEXT: ldp w13, w12, [x0, #8]
+; CHECK-NEXT: mov v0.s[2], w13
+; CHECK-NEXT: mov v0.s[3], w12
+; CHECK-NEXT: ldr w12, [x0, #16]
+; CHECK-NEXT: fmov s1, w12
+; CHECK-NEXT: .LBB0_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: fmov s2, w1
+; CHECK-NEXT: fmov s3, w10
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: mov v2.s[1], w11
+; CHECK-NEXT: add v3.4s, v3.4s, v1.4s
+; CHECK-NEXT: fmov w10, s3
+; CHECK-NEXT: mov v2.s[2], w9
+; CHECK-NEXT: mov v2.s[3], w8
+; CHECK-NEXT: add v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: mov w8, v2.s[3]
+; CHECK-NEXT: mov w9, v2.s[2]
+; CHECK-NEXT: mov w11, v2.s[1]
+; CHECK-NEXT: fmov w1, s2
+; CHECK-NEXT: b.ne .LBB0_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v2.s[1], w11
+; CHECK-NEXT: str w10, [x2, #16]
+; CHECK-NEXT: mov v2.s[2], w9
+; CHECK-NEXT: mov v2.s[3], w8
+; CHECK-NEXT: str q2, [x2]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <5 x i32>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <5 x i32> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = add <5 x i32> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <5 x i32> %acc.next, ptr %out, align 8
+ ret void
+}
+
+define void @v6i32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6i32_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp w11, w12, [x0]
+; CHECK-NEXT: mov w4, wzr
+; CHECK-NEXT: mov w10, wzr
+; CHECK-NEXT: mov w9, wzr
+; CHECK-NEXT: mov w8, wzr
+; CHECK-NEXT: mov w1, wzr
+; CHECK-NEXT: fmov s0, w11
+; CHECK-NEXT: ldp w11, w13, [x0, #8]
+; CHECK-NEXT: mov v0.s[1], w12
+; CHECK-NEXT: mov v0.s[2], w11
+; CHECK-NEXT: ldp w11, w12, [x0, #16]
+; CHECK-NEXT: fmov s1, w11
+; CHECK-NEXT: mov v0.s[3], w13
+; CHECK-NEXT: mov w11, wzr
+; CHECK-NEXT: mov v1.s[1], w12
+; CHECK-NEXT: .LBB1_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: fmov s2, w4
+; CHECK-NEXT: fmov s4, w1
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: mov v2.s[1], w10
+; CHECK-NEXT: mov v4.s[1], w11
+; CHECK-NEXT: mov v2.s[2], w9
+; CHECK-NEXT: mov v2.s[3], w8
+; CHECK-NEXT: add v3.4s, v2.4s, v0.4s
+; CHECK-NEXT: add v2.4s, v4.4s, v1.4s
+; CHECK-NEXT: mov w8, v3.s[3]
+; CHECK-NEXT: mov w9, v3.s[2]
+; CHECK-NEXT: mov w10, v3.s[1]
+; CHECK-NEXT: mov w11, v2.s[1]
+; CHECK-NEXT: fmov w4, s3
+; CHECK-NEXT: fmov w1, s2
+; CHECK-NEXT: b.ne .LBB1_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v3.s[1], w10
+; CHECK-NEXT: mov v2.s[1], w11
+; CHECK-NEXT: mov v3.s[2], w9
+; CHECK-NEXT: str d2, [x2, #16]
+; CHECK-NEXT: mov v3.s[3], w8
+; CHECK-NEXT: str q3, [x2]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <6 x i32>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <6 x i32> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = add <6 x i32> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <6 x i32> %acc.next, ptr %out, align 8
+ ret void
+}
+
+define <6 x i32> @v6i32_cc(<6 x i32> %c.val, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6i32_cc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov w10, w0
+; CHECK-NEXT: mov w11, w1
+; CHECK-NEXT: mov w12, w2
+; CHECK-NEXT: fmov s0, w10
+; CHECK-NEXT: mov w8, w4
+; CHECK-NEXT: mov w9, w3
+; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: ldr x8, [sp]
+; CHECK-NEXT: mov w0, wzr
+; CHECK-NEXT: mov w1, wzr
+; CHECK-NEXT: mov w2, wzr
+; CHECK-NEXT: mov w3, wzr
+; CHECK-NEXT: mov v0.s[1], w11
+; CHECK-NEXT: mov w4, wzr
+; CHECK-NEXT: mov v1.s[1], w5
+; CHECK-NEXT: mov w5, wzr
+; CHECK-NEXT: mov v0.s[2], w12
+; CHECK-NEXT: mov v0.s[3], w9
+; CHECK-NEXT: .LBB2_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: fmov s2, w0
+; CHECK-NEXT: fmov s3, w4
+; CHECK-NEXT: subs x8, x8, #1
+; CHECK-NEXT: mov v2.s[1], w1
+; CHECK-NEXT: mov v3.s[1], w5
+; CHECK-NEXT: mov v2.s[2], w2
+; CHECK-NEXT: add v3.4s, v3.4s, v1.4s
+; CHECK-NEXT: mov w5, v3.s[1]
+; CHECK-NEXT: fmov w4, s3
+; CHECK-NEXT: mov v2.s[3], w3
+; CHECK-NEXT: add v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: mov w3, v2.s[3]
+; CHECK-NEXT: mov w2, v2.s[2]
+; CHECK-NEXT: mov w1, v2.s[1]
+; CHECK-NEXT: fmov w0, s2
+; CHECK-NEXT: b.ne .LBB2_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %loop
+
+loop:
+ %acc = phi <6 x i32> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = add <6 x i32> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ ret <6 x i32> %acc.next
+}
+
+define void @v5i64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v5i64_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: ldp q2, q1, [x0]
+; CHECK-NEXT: ldr d7, [x0, #32]
+; CHECK-NEXT: mov d3, v1.d[1]
+; CHECK-NEXT: mov d6, v2.d[1]
+; CHECK-NEXT: fmov d17, d0
+; CHECK-NEXT: fmov d4, d0
+; CHECK-NEXT: fmov d16, d0
+; CHECK-NEXT: fmov d5, d0
+; CHECK-NEXT: .LBB3_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: add d0, d0, d2
+; CHECK-NEXT: add d17, d17, d6
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: add d4, d4, d1
+; CHECK-NEXT: add d16, d16, d3
+; CHECK-NEXT: add d5, d5, d7
+; CHECK-NEXT: b.ne .LBB3_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v0.d[1], v17.d[0]
+; CHECK-NEXT: mov v4.d[1], v16.d[0]
+; CHECK-NEXT: str d5, [x2, #32]
+; CHECK-NEXT: stp q0, q4, [x2]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <5 x i64>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <5 x i64> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = add <5 x i64> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <5 x i64> %acc.next, ptr %out, align 8
+ ret void
+}
+
+define void @v6i64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6i64_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: ldp q4, q1, [x0, #16]
+; CHECK-NEXT: ldr q6, [x0]
+; CHECK-NEXT: mov d7, v1.d[1]
+; CHECK-NEXT: mov d16, v4.d[1]
+; CHECK-NEXT: mov d18, v6.d[1]
+; CHECK-NEXT: fmov d19, d0
+; CHECK-NEXT: fmov d2, d0
+; CHECK-NEXT: fmov d17, d0
+; CHECK-NEXT: fmov d3, d0
+; CHECK-NEXT: fmov d5, d0
+; CHECK-NEXT: .LBB4_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: add d0, d0, d6
+; CHECK-NEXT: add d19, d19, d18
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: add d2, d2, d4
+; CHECK-NEXT: add d17, d17, d16
+; CHECK-NEXT: add d3, d3, d1
+; CHECK-NEXT: add d5, d5, d7
+; CHECK-NEXT: b.ne .LBB4_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v0.d[1], v19.d[0]
+; CHECK-NEXT: mov v2.d[1], v17.d[0]
+; CHECK-NEXT: mov v3.d[1], v5.d[0]
+; CHECK-NEXT: stp q0, q2, [x2]
+; CHECK-NEXT: str q3, [x2, #32]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <6 x i64>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <6 x i64> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = add <6 x i64> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <6 x i64> %acc.next, ptr %out, align 8
+ ret void
+}
+
+define <6 x i64> @v6i64_cc(<6 x i64> %c.val, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6i64_cc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d6, #0000000000000000
+; CHECK-NEXT: fmov d7, d6
+; CHECK-NEXT: fmov d16, d6
+; CHECK-NEXT: fmov d17, d6
+; CHECK-NEXT: fmov d18, d6
+; CHECK-NEXT: fmov d19, d6
+; CHECK-NEXT: .LBB5_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: add d6, d6, d0
+; CHECK-NEXT: add d7, d7, d1
+; CHECK-NEXT: subs x2, x2, #1
+; CHECK-NEXT: add d16, d16, d2
+; CHECK-NEXT: add d17, d17, d3
+; CHECK-NEXT: add d18, d18, d4
+; CHECK-NEXT: add d19, d19, d5
+; CHECK-NEXT: b.ne .LBB5_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: fmov d0, d6
+; CHECK-NEXT: fmov d1, d7
+; CHECK-NEXT: fmov d2, d16
+; CHECK-NEXT: fmov d3, d17
+; CHECK-NEXT: fmov d4, d18
+; CHECK-NEXT: fmov d5, d19
+; CHECK-NEXT: ret
+entry:
+ br label %loop
+
+loop:
+ %acc = phi <6 x i64> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = add <6 x i64> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ ret <6 x i64> %acc.next
+}
+
+define void @v5f32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v5f32_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldr q0, [x0]
+; CHECK-NEXT: movi d5, #0000000000000000
+; CHECK-NEXT: movi d4, #0000000000000000
+; CHECK-NEXT: ldr s6, [x0, #16]
+; CHECK-NEXT: mov s1, v0.s[1]
+; CHECK-NEXT: mov s2, v0.s[3]
+; CHECK-NEXT: mov s3, v0.s[2]
+; CHECK-NEXT: mov v0.s[1], v1.s[0]
+; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: mov v0.s[2], v3.s[0]
+; CHECK-NEXT: movi d3, #0000000000000000
+; CHECK-NEXT: mov v0.s[3], v2.s[0]
+; CHECK-NEXT: movi d2, #0000000000000000
+; CHECK-NEXT: .LBB6_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov v1.s[1], v5.s[0]
+; CHECK-NEXT: fadd v2.4s, v2.4s, v6.4s
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: mov v1.s[2], v4.s[0]
+; CHECK-NEXT: mov v1.s[3], v3.s[0]
+; CHECK-NEXT: fadd v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: mov s3, v1.s[3]
+; CHECK-NEXT: mov s4, v1.s[2]
+; CHECK-NEXT: mov s5, v1.s[1]
+; CHECK-NEXT: b.ne .LBB6_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v1.s[1], v5.s[0]
+; CHECK-NEXT: str s2, [x2, #16]
+; CHECK-NEXT: mov v1.s[2], v4.s[0]
+; CHECK-NEXT: mov v1.s[3], v3.s[0]
+; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <5 x float>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <5 x float> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = fadd <5 x float> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <5 x float> %acc.next, ptr %out, align 8
+ ret void
+}
+
+define void @v6f32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6f32_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q0, q2, [x0]
+; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: movi d6, #0000000000000000
+; CHECK-NEXT: mov s1, v0.s[1]
+; CHECK-NEXT: mov s3, v0.s[3]
+; CHECK-NEXT: mov s4, v0.s[2]
+; CHECK-NEXT: mov s5, v2.s[1]
+; CHECK-NEXT: mov v0.s[1], v1.s[0]
+; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: mov v2.s[1], v5.s[0]
+; CHECK-NEXT: movi d5, #0000000000000000
+; CHECK-NEXT: mov v0.s[2], v4.s[0]
+; CHECK-NEXT: movi d4, #0000000000000000
+; CHECK-NEXT: mov v0.s[3], v3.s[0]
+; CHECK-NEXT: movi d3, #0000000000000000
+; CHECK-NEXT: .LBB7_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov v1.s[1], v7.s[0]
+; CHECK-NEXT: mov v3.s[1], v4.s[0]
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: mov v1.s[2], v6.s[0]
+; CHECK-NEXT: fadd v3.4s, v3.4s, v2.4s
+; CHECK-NEXT: mov v1.s[3], v5.s[0]
+; CHECK-NEXT: mov s4, v3.s[1]
+; CHECK-NEXT: fadd v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: mov s5, v1.s[3]
+; CHECK-NEXT: mov s6, v1.s[2]
+; CHECK-NEXT: mov s7, v1.s[1]
+; CHECK-NEXT: b.ne .LBB7_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v1.s[1], v7.s[0]
+; CHECK-NEXT: mov v3.s[1], v4.s[0]
+; CHECK-NEXT: mov v1.s[2], v6.s[0]
+; CHECK-NEXT: str d3, [x2, #16]
+; CHECK-NEXT: mov v1.s[3], v5.s[0]
+; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <6 x float>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <6 x float> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = fadd <6 x float> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <6 x float> %acc.next, ptr %out, align 8
+ ret void
+}
+
+define <6 x float> @v6f32_cc(<6 x float> %c.val, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6f32_cc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1
+; CHECK-NEXT: // kill: def $s2 killed $s2 def $q2
+; CHECK-NEXT: // kill: def $s5 killed $s5 def $q5
+; CHECK-NEXT: // kill: def $s4 killed $s4 def $q4
+; CHECK-NEXT: // kill: def $s3 killed $s3 def $q3
+; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: movi d6, #0000000000000000
+; CHECK-NEXT: mov v0.s[1], v1.s[0]
+; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: mov v4.s[1], v5.s[0]
+; CHECK-NEXT: movi d5, #0000000000000000
+; CHECK-NEXT: mov v0.s[2], v2.s[0]
+; CHECK-NEXT: movi d2, #0000000000000000
+; CHECK-NEXT: mov v0.s[3], v3.s[0]
+; CHECK-NEXT: movi d3, #0000000000000000
+; CHECK-NEXT: .LBB8_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov v7.s[1], v1.s[0]
+; CHECK-NEXT: mov v6.s[1], v5.s[0]
+; CHECK-NEXT: subs x2, x2, #1
+; CHECK-NEXT: mov v7.s[2], v2.s[0]
+; CHECK-NEXT: fadd v6.4s, v6.4s, v4.4s
+; CHECK-NEXT: mov v7.s[3], v3.s[0]
+; CHECK-NEXT: mov s5, v6.s[1]
+; CHECK-NEXT: fadd v7.4s, v7.4s, v0.4s
+; CHECK-NEXT: mov s3, v7.s[3]
+; CHECK-NEXT: mov s2, v7.s[2]
+; CHECK-NEXT: mov s1, v7.s[1]
+; CHECK-NEXT: b.ne .LBB8_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: fmov s0, s7
+; CHECK-NEXT: fmov s4, s6
+; CHECK-NEXT: // kill: def $s1 killed $s1 killed $q1
+; CHECK-NEXT: // kill: def $s2 killed $s2 killed $q2
+; CHECK-NEXT: // kill: def $s3 killed $s3 killed $q3
+; CHECK-NEXT: // kill: def $s5 killed $s5 killed $q5
+; CHECK-NEXT: ret
+entry:
+ br label %loop
+
+loop:
+ %acc = phi <6 x float> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = fadd <6 x float> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ ret <6 x float> %acc.next
+}
+
+define void @v6f64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6f64_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q2, q1, [x0, #16]
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: ldr q3, [x0]
+; CHECK-NEXT: movi d16, #0000000000000000
+; CHECK-NEXT: movi d4, #0000000000000000
+; CHECK-NEXT: mov d5, v1.d[1]
+; CHECK-NEXT: mov d6, v2.d[1]
+; CHECK-NEXT: mov d7, v3.d[1]
+; CHECK-NEXT: mov v3.d[1], v7.d[0]
+; CHECK-NEXT: mov v2.d[1], v6.d[0]
+; CHECK-NEXT: mov v1.d[1], v5.d[0]
+; CHECK-NEXT: movi d6, #0000000000000000
+; CHECK-NEXT: movi d5, #0000000000000000
+; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: .LBB9_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov v0.d[1], v16.d[0]
+; CHECK-NEXT: mov v4.d[1], v6.d[0]
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: mov v5.d[1], v7.d[0]
+; CHECK-NEXT: fadd v0.2d, v0.2d, v3.2d
+; CHECK-NEXT: fadd v4.2d, v4.2d, v2.2d
+; CHECK-NEXT: fadd v5.2d, v5.2d, v1.2d
+; CHECK-NEXT: mov d16, v0.d[1]
+; CHECK-NEXT: mov d6, v4.d[1]
+; CHECK-NEXT: mov d7, v5.d[1]
+; CHECK-NEXT: b.ne .LBB9_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v0.d[1], v16.d[0]
+; CHECK-NEXT: mov v4.d[1], v7.d[0]
+; CHECK-NEXT: mov v6.d[1], v5.d[0]
+; CHECK-NEXT: stp q0, q4, [x2]
+; CHECK-NEXT: str q6, [x2, #32]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <6 x double>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <6 x double> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = fadd fast <6 x double> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ %lanes01 = shufflevector <6 x double> %acc.next, <6 x double> poison, <2 x i32> <i32 0, i32 1>
+ %lanes25 = shufflevector <6 x double> %acc.next, <6 x double> poison, <2 x i32> <i32 2, i32 5>
+ %lane3 = extractelement <6 x double> %acc.next, i64 3
+ %lane4 = extractelement <6 x double> %acc.next, i64 4
+ store <2 x double> %lanes01, ptr %out, align 8
+ %p2 = getelementptr double, ptr %out, i64 2
+ store <2 x double> %lanes25, ptr %p2, align 8
+ %p4 = getelementptr double, ptr %out, i64 4
+ store double %lane3, ptr %p4, align 8
+ %p5 = getelementptr double, ptr %out, i64 5
+ store double %lane4, ptr %p5, align 8
+ ret void
+}
+
+define <6 x double> @v6f64_cc(<6 x double> %c.val, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6f64_cc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fmov d16, d0
+; CHECK-NEXT: // kill: def $d5 killed $d5 def $q5
+; CHECK-NEXT: // kill: def $d4 killed $d4 def $q4
+; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-NEXT: mov v4.d[1], v5.d[0]
+; CHECK-NEXT: movi d3, #0000000000000000
+; CHECK-NEXT: movi d6, #0000000000000000
+; CHECK-NEXT: movi d5, #0000000000000000
+; CHECK-NEXT: mov v16.d[1], v1.d[0]
+; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: .LBB10_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NEXT: mov v7.d[1], v3.d[0]
+; CHECK-NEXT: subs x2, x2, #1
+; CHECK-NEXT: mov v6.d[1], v5.d[0]
+; CHECK-NEXT: fadd v0.2d, v0.2d, v16.2d
+; CHECK-NEXT: fadd v7.2d, v7.2d, v2.2d
+; CHECK-NEXT: fadd v6.2d, v6.2d, v4.2d
+; CHECK-NEXT: mov d1, v0.d[1]
+; CHECK-NEXT: mov d3, v7.d[1]
+; CHECK-NEXT: mov d5, v6.d[1]
+; CHECK-NEXT: b.ne .LBB10_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: fmov d2, d7
+; CHECK-NEXT: fmov d4, d6
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT: // kill: def $d3 killed $d3 killed $q3
+; CHECK-NEXT: // kill: def $d5 killed $d5 killed $q5
+; CHECK-NEXT: ret
+entry:
+ br label %loop
+
+loop:
+ %acc = phi <6 x double> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = fadd fast <6 x double> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ ret <6 x double> %acc.next
+}
+
+define void @v12f64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v12f64_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q2, q1, [x0]
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: ldp q4, q3, [x0, #32]
+; CHECK-NEXT: movi d25, #0000000000000000
+; CHECK-NEXT: ldp q5, q6, [x0, #64]
+; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: mov d16, v1.d[1]
+; CHECK-NEXT: mov d17, v2.d[1]
+; CHECK-NEXT: movi d24, #0000000000000000
+; CHECK-NEXT: mov d19, v3.d[1]
+; CHECK-NEXT: mov d20, v4.d[1]
+; CHECK-NEXT: movi d23, #0000000000000000
+; CHECK-NEXT: mov d18, v5.d[1]
+; CHECK-NEXT: mov d21, v6.d[1]
+; CHECK-NEXT: movi d22, #0000000000000000
+; CHECK-NEXT: mov v2.d[1], v17.d[0]
+; CHECK-NEXT: mov v1.d[1], v16.d[0]
+; CHECK-NEXT: movi d16, #0000000000000000
+; CHECK-NEXT: mov v4.d[1], v20.d[0]
+; CHECK-NEXT: mov v3.d[1], v19.d[0]
+; CHECK-NEXT: movi d17, #0000000000000000
+; CHECK-NEXT: mov v5.d[1], v18.d[0]
+; CHECK-NEXT: mov v6.d[1], v21.d[0]
+; CHECK-NEXT: movi d18, #0000000000000000
+; CHECK-NEXT: movi d21, #0000000000000000
+; CHECK-NEXT: movi d19, #0000000000000000
+; CHECK-NEXT: movi d20, #0000000000000000
+; CHECK-NEXT: .LBB11_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov v0.d[1], v25.d[0]
+; CHECK-NEXT: mov v7.d[1], v24.d[0]
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: mov v16.d[1], v23.d[0]
+; CHECK-NEXT: mov v17.d[1], v22.d[0]
+; CHECK-NEXT: mov v18.d[1], v21.d[0]
+; CHECK-NEXT: mov v19.d[1], v20.d[0]
+; CHECK-NEXT: fadd v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: fadd v7.2d, v7.2d, v1.2d
+; CHECK-NEXT: fadd v16.2d, v16.2d, v4.2d
+; CHECK-NEXT: fadd v17.2d, v17.2d, v3.2d
+; CHECK-NEXT: fadd v18.2d, v18.2d, v5.2d
+; CHECK-NEXT: fadd v19.2d, v19.2d, v6.2d
+; CHECK-NEXT: mov d25, v0.d[1]
+; CHECK-NEXT: mov d24, v7.d[1]
+; CHECK-NEXT: mov d23, v16.d[1]
+; CHECK-NEXT: mov d22, v17.d[1]
+; CHECK-NEXT: mov d21, v18.d[1]
+; CHECK-NEXT: mov d20, v19.d[1]
+; CHECK-NEXT: b.ne .LBB11_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v0.d[1], v25.d[0]
+; CHECK-NEXT: mov v7.d[1], v24.d[0]
+; CHECK-NEXT: mov v16.d[1], v23.d[0]
+; CHECK-NEXT: mov v17.d[1], v22.d[0]
+; CHECK-NEXT: mov v18.d[1], v21.d[0]
+; CHECK-NEXT: mov v19.d[1], v20.d[0]
+; CHECK-NEXT: stp q0, q7, [x2]
+; CHECK-NEXT: stp q16, q17, [x2, #32]
+; CHECK-NEXT: stp q18, q19, [x2, #64]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <12 x double>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <12 x double> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = fadd fast <12 x double> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <12 x double> %acc.next, ptr %out
+ ret void
+}
>From 79fdbda77eee0036faa87088cadfc4341d70bbfe Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Tue, 4 Aug 2026 14:22:50 +0000
Subject: [PATCH 2/6] [CodeGen][AArch64] Use vector parts for internal
non-power-of-two vectors
Decompose fixed-length, non-power-of-two vectors into the largest legal
vector type that exactly divides their element count. Apply this only to
internal register values.
Continue using the existing scalar breakdown for arguments and returns,
preserving the existing calling convention behaviour.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 47 +-
llvm/lib/CodeGen/TargetLoweringBase.cpp | 61 ++-
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 5 +
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 9 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 3 +-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 9 +-
llvm/lib/Target/RISCV/RISCVISelLowering.h | 6 +-
llvm/lib/Target/SPIRV/SPIRVISelLowering.h | 6 +-
llvm/lib/Target/SystemZ/SystemZISelLowering.h | 9 +-
.../CodeGen/AArch64/non-pow2-fixed-vectors.ll | 401 ++++++------------
10 files changed, 246 insertions(+), 310 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index c663bb8ea65b7..e8caf0c2471e8 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1226,8 +1226,14 @@ class LLVM_ABI TargetLoweringBase {
/// before they are promoted/expanded.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT,
EVT &IntermediateVT,
- unsigned &NumIntermediates,
- MVT &RegisterVT) const;
+ unsigned &NumIntermediates, MVT &RegisterVT,
+ bool ForCallingConv = false) const;
+
+ /// Return true if fixed-length, non-power-of-two vectors should be broken
+ /// down into legal vector parts instead of scalars for internal values.
+ virtual bool preferVectorizedNonPowerOfTwoTypeBreakdown() const {
+ return false;
+ }
/// Certain targets such as MIPS require that some types such as vectors are
/// always broken down into scalars in some contexts. This occurs even if the
@@ -1236,7 +1242,7 @@ class LLVM_ABI TargetLoweringBase {
LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
unsigned &NumIntermediates, MVT &RegisterVT) const {
return getVectorTypeBreakdown(Context, VT, IntermediateVT, NumIntermediates,
- RegisterVT);
+ RegisterVT, /*ForCallingConv=*/true);
}
struct IntrinsicInfo {
@@ -1851,19 +1857,23 @@ class LLVM_ABI TargetLoweringBase {
}
/// Return the type of registers that this ValueType will eventually require.
- MVT getRegisterType(LLVMContext &Context, EVT VT) const {
- if (VT.isSimple())
+ MVT getRegisterType(LLVMContext &Context, EVT VT,
+ bool ForCallingConv = false) const {
+ if (VT.isSimple() && (!ForCallingConv || !VT.isFixedLengthVector() ||
+ isPowerOf2_32(VT.getVectorNumElements()) ||
+ !preferVectorizedNonPowerOfTwoTypeBreakdown()))
return getRegisterType(VT.getSimpleVT());
if (VT.isVector()) {
EVT VT1;
MVT RegisterVT;
unsigned NumIntermediates;
- (void)getVectorTypeBreakdown(Context, VT, VT1,
- NumIntermediates, RegisterVT);
+ (void)getVectorTypeBreakdown(Context, VT, VT1, NumIntermediates,
+ RegisterVT, ForCallingConv);
return RegisterVT;
}
if (VT.isInteger()) {
- return getRegisterType(Context, getTypeToTransformTo(Context, VT));
+ return getRegisterType(Context, getTypeToTransformTo(Context, VT),
+ ForCallingConv);
}
llvm_unreachable("Unsupported extended type!");
}
@@ -1879,10 +1889,12 @@ class LLVM_ABI TargetLoweringBase {
///
/// RegisterVT may be passed as a way to override the default settings, for
/// instance with i128 inline assembly operands on SystemZ.
- virtual unsigned
- getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt) const {
- if (VT.isSimple()) {
+ virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt,
+ bool ForCallingConv = false) const {
+ if (VT.isSimple() && (!ForCallingConv || !VT.isFixedLengthVector() ||
+ isPowerOf2_32(VT.getVectorNumElements()) ||
+ !preferVectorizedNonPowerOfTwoTypeBreakdown())) {
assert((unsigned)VT.getSimpleVT().SimpleTy <
std::size(NumRegistersForVT));
return NumRegistersForVT[VT.getSimpleVT().SimpleTy];
@@ -1891,11 +1903,13 @@ class LLVM_ABI TargetLoweringBase {
EVT VT1;
MVT VT2;
unsigned NumIntermediates;
- return getVectorTypeBreakdown(Context, VT, VT1, NumIntermediates, VT2);
+ return getVectorTypeBreakdown(Context, VT, VT1, NumIntermediates, VT2,
+ ForCallingConv);
}
if (VT.isInteger()) {
unsigned BitWidth = VT.getSizeInBits();
- unsigned RegWidth = getRegisterType(Context, VT).getSizeInBits();
+ unsigned RegWidth =
+ getRegisterType(Context, VT, ForCallingConv).getSizeInBits();
return (BitWidth + RegWidth - 1) / RegWidth;
}
llvm_unreachable("Unsupported extended type!");
@@ -1906,7 +1920,7 @@ class LLVM_ABI TargetLoweringBase {
/// For MIPS all vector types must be passed through the integer register set.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context,
CallingConv::ID CC, EVT VT) const {
- return getRegisterType(Context, VT);
+ return getRegisterType(Context, VT, /*ForCallingConv=*/true);
}
/// Certain targets require unusual breakdowns of certain types. For MIPS,
@@ -1915,7 +1929,8 @@ class LLVM_ABI TargetLoweringBase {
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context,
CallingConv::ID CC,
EVT VT) const {
- return getNumRegisters(Context, VT);
+ return getNumRegisters(Context, VT, /*RegisterVT=*/std::nullopt,
+ /*ForCallingConv=*/true);
}
/// Certain targets have context sensitive alignment requirements, where one
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index e5f2f3fc0e80e..9a9c87c1bced6 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1545,11 +1545,33 @@ static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
llvm_unreachable(
"Splitting or widening of non-power-of-2 MVTs is not implemented.");
- // FIXME: We don't support non-power-of-2-sized vectors for now.
+ // FIXME: We don't generically support non-power-of-2-sized vectors for now.
// Ideally we could break down into LHS/RHS like LegalizeDAG does.
if (!isPowerOf2_32(EC.getKnownMinValue())) {
- // Split EC to unit size (scalable property is preserved).
- NumVectorRegs = EC.getKnownMinValue();
+ assert(VT.isFixedLengthVector() && "Expected a fixed-length vector VT");
+ unsigned NumElts = EC.getKnownMinValue();
+
+ // Find the largest legal vector type that exactly divides a
+ // non-power-of-two vector.
+ if (TLI->preferVectorizedNonPowerOfTwoTypeBreakdown()) {
+ for (unsigned PartElts = llvm::bit_floor(NumElts); PartElts > 1;
+ PartElts >>= 1) {
+ if (NumElts % PartElts != 0)
+ continue;
+
+ MVT PartVT = MVT::getVectorVT(EltTy, ElementCount::getFixed(PartElts));
+ if (PartVT == MVT() || !TLI->isTypeLegal(PartVT))
+ continue;
+
+ IntermediateVT = PartVT;
+ NumIntermediates = NumElts / PartElts;
+ RegisterVT = PartVT;
+ return NumIntermediates;
+ }
+ }
+
+ // Fall back to scalars if there is no exact legal vector decomposition.
+ NumVectorRegs = NumElts;
EC = ElementCount::getFixed(1);
}
@@ -1984,7 +2006,8 @@ EVT TargetLoweringBase::getSetCCResultType(const DataLayout &DL, LLVMContext &,
unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,
EVT VT, EVT &IntermediateVT,
unsigned &NumIntermediates,
- MVT &RegisterVT) const {
+ MVT &RegisterVT,
+ bool ForCallingConv) const {
ElementCount EltCnt = VT.getVectorElementCount();
// If there is a wider vector type with the same element type as this one,
@@ -2033,10 +2056,34 @@ unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,
return NumIntermediates;
}
- // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally
- // we could break down into LHS/RHS like LegalizeDAG does.
+ // FIXME: We don't generically support non-power-of-2-sized vectors for now.
+ // Ideally we could break down into LHS/RHS like LegalizeDAG does.
if (!isPowerOf2_32(EltCnt.getKnownMinValue())) {
- NumVectorRegs = EltCnt.getKnownMinValue();
+ assert(VT.isFixedLengthVector() && "Expected a fixed-length vector VT");
+ unsigned NumElts = EltCnt.getKnownMinValue();
+
+ // Find the largest legal vector type that exactly divides a
+ // non-power-of-two vector.
+ if (!ForCallingConv && preferVectorizedNonPowerOfTwoTypeBreakdown()) {
+ for (unsigned PartElts = llvm::bit_floor(NumElts); PartElts > 1;
+ PartElts >>= 1) {
+ if (NumElts % PartElts != 0)
+ continue;
+
+ EVT PartVT =
+ EVT::getVectorVT(Context, EltTy, ElementCount::getFixed(PartElts));
+ if (!isTypeLegal(PartVT))
+ continue;
+
+ IntermediateVT = PartVT;
+ NumIntermediates = NumElts / PartElts;
+ RegisterVT = PartVT.getSimpleVT();
+ return NumIntermediates;
+ }
+ }
+
+ // Fall back to scalars if there is no exact legal vector decomposition.
+ NumVectorRegs = NumElts;
EltCnt = ElementCount::getFixed(1);
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 35f7ef0e2151e..c5a47414ddf18 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -20,6 +20,7 @@
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Instruction.h"
+#include "llvm/Support/MathExtras.h"
namespace llvm {
@@ -602,6 +603,10 @@ class AArch64TargetLowering : public TargetLowering {
unsigned &NumIntermediates,
MVT &RegisterVT) const override;
+ bool preferVectorizedNonPowerOfTwoTypeBreakdown() const override {
+ return true;
+ }
+
/// True if stack clash protection is enabled for this functions.
bool hasInlineStackProbe(const MachineFunction &MF) const override;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 29921ff86b352..dc777bacd47bf 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -4028,12 +4028,13 @@ SDValue NVPTXTargetLowering::LowerCopyToReg_128(SDValue Op,
return DAG.getNode(ISD::CopyToReg, DL, ResultsType, NewOps);
}
-unsigned NVPTXTargetLowering::getNumRegisters(
- LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt) const {
+unsigned NVPTXTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT,
+ bool ForCallingConv) const {
if (VT == MVT::i128 && RegisterVT == MVT::i128)
return 1;
- return TargetLoweringBase::getNumRegisters(Context, VT, RegisterVT);
+ return TargetLoweringBase::getNumRegisters(Context, VT, RegisterVT,
+ ForCallingConv);
}
bool NVPTXTargetLowering::splitValueIntoRegisterParts(
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 42741ba98b438..c7238444ae61c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -224,7 +224,8 @@ class NVPTXTargetLowering : public TargetLowering {
SDValue LowerCopyToReg_128(SDValue Op, SelectionDAG &DAG) const;
unsigned getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT) const override;
+ std::optional<MVT> RegisterVT,
+ bool ForCallingConv = false) const override;
bool
splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
SDValue *Parts, unsigned NumParts, MVT PartVT,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index e23427482c1e4..b9d3871ef2374 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -2926,15 +2926,16 @@ MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
}
-unsigned
-RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT) const {
+unsigned RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT,
+ bool ForCallingConv) const {
// Pair inline assembly operand
if (VT == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
*RegisterVT == MVT::Untyped)
return 1;
- return TargetLowering::getNumRegisters(Context, VT, RegisterVT);
+ return TargetLowering::getNumRegisters(Context, VT, RegisterVT,
+ ForCallingConv);
}
unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 2ca7c392639f7..d2f0e1f62c845 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -82,9 +82,9 @@ class RISCVTargetLowering : public TargetLowering {
EVT VT) const override;
/// Return the number of registers for a given MVT, for inline assembly
- unsigned
- getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt) const override;
+ unsigned getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt,
+ bool ForCallingConv = false) const override;
/// Return the number of registers for a given MVT, ensuring vectors are
/// treated as a series of gpr sized integers.
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
index ff843649fd6d3..a4e91b3f06d64 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
@@ -53,9 +53,9 @@ class SPIRVTargetLowering : public TargetLowering {
std::pair<unsigned, const TargetRegisterClass *>
getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
StringRef Constraint, MVT VT) const override;
- unsigned
- getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt) const override {
+ unsigned getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt,
+ bool ForCallingConv = false) const override {
return 1;
}
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index e6da481353497..dc2da7aea77bf 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -75,13 +75,14 @@ class SystemZTargetLowering : public TargetLowering {
return TypeWidenVector;
return TargetLoweringBase::getPreferredVectorAction(VT);
}
- unsigned
- getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT) const override {
+ unsigned getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT,
+ bool ForCallingConv = false) const override {
// i128 inline assembly operand.
if (VT == MVT::i128 && RegisterVT && *RegisterVT == MVT::Untyped)
return 1;
- return TargetLowering::getNumRegisters(Context, VT);
+ return TargetLowering::getNumRegisters(
+ Context, VT, /*RegisterVT=*/std::nullopt, ForCallingConv);
}
unsigned
getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC,
diff --git a/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
index 047a377710ecf..ea945cdbcec43 100644
--- a/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
+++ b/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
@@ -60,46 +60,23 @@ exit:
define void @v6i32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6i32_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp w11, w12, [x0]
-; CHECK-NEXT: mov w4, wzr
-; CHECK-NEXT: mov w10, wzr
-; CHECK-NEXT: mov w9, wzr
-; CHECK-NEXT: mov w8, wzr
-; CHECK-NEXT: mov w1, wzr
-; CHECK-NEXT: fmov s0, w11
-; CHECK-NEXT: ldp w11, w13, [x0, #8]
-; CHECK-NEXT: mov v0.s[1], w12
-; CHECK-NEXT: mov v0.s[2], w11
-; CHECK-NEXT: ldp w11, w12, [x0, #16]
-; CHECK-NEXT: fmov s1, w11
-; CHECK-NEXT: mov v0.s[3], w13
-; CHECK-NEXT: mov w11, wzr
-; CHECK-NEXT: mov v1.s[1], w12
+; CHECK-NEXT: ldr q1, [x0]
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: ldr d5, [x0, #16]
+; CHECK-NEXT: mov d3, v1.d[1]
; CHECK-NEXT: .LBB1_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: fmov s2, w4
-; CHECK-NEXT: fmov s4, w1
+; CHECK-NEXT: add v0.2s, v0.2s, v1.2s
+; CHECK-NEXT: add v4.2s, v4.2s, v3.2s
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: mov v2.s[1], w10
-; CHECK-NEXT: mov v4.s[1], w11
-; CHECK-NEXT: mov v2.s[2], w9
-; CHECK-NEXT: mov v2.s[3], w8
-; CHECK-NEXT: add v3.4s, v2.4s, v0.4s
-; CHECK-NEXT: add v2.4s, v4.4s, v1.4s
-; CHECK-NEXT: mov w8, v3.s[3]
-; CHECK-NEXT: mov w9, v3.s[2]
-; CHECK-NEXT: mov w10, v3.s[1]
-; CHECK-NEXT: mov w11, v2.s[1]
-; CHECK-NEXT: fmov w4, s3
-; CHECK-NEXT: fmov w1, s2
+; CHECK-NEXT: add v2.2s, v2.2s, v5.2s
; CHECK-NEXT: b.ne .LBB1_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v3.s[1], w10
-; CHECK-NEXT: mov v2.s[1], w11
-; CHECK-NEXT: mov v3.s[2], w9
+; CHECK-NEXT: mov v0.d[1], v4.d[0]
; CHECK-NEXT: str d2, [x2, #16]
-; CHECK-NEXT: mov v3.s[3], w8
-; CHECK-NEXT: str q3, [x2]
+; CHECK-NEXT: str q0, [x2]
; CHECK-NEXT: ret
entry:
%c.val = load <6 x i32>, ptr %c
@@ -121,43 +98,30 @@ exit:
define <6 x i32> @v6i32_cc(<6 x i32> %c.val, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6i32_cc:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov w10, w0
-; CHECK-NEXT: mov w11, w1
-; CHECK-NEXT: mov w12, w2
-; CHECK-NEXT: fmov s0, w10
-; CHECK-NEXT: mov w8, w4
-; CHECK-NEXT: mov w9, w3
-; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: fmov s0, w4
+; CHECK-NEXT: fmov s1, w2
; CHECK-NEXT: ldr x8, [sp]
-; CHECK-NEXT: mov w0, wzr
-; CHECK-NEXT: mov w1, wzr
-; CHECK-NEXT: mov w2, wzr
-; CHECK-NEXT: mov w3, wzr
-; CHECK-NEXT: mov v0.s[1], w11
-; CHECK-NEXT: mov w4, wzr
-; CHECK-NEXT: mov v1.s[1], w5
-; CHECK-NEXT: mov w5, wzr
-; CHECK-NEXT: mov v0.s[2], w12
-; CHECK-NEXT: mov v0.s[3], w9
+; CHECK-NEXT: fmov s3, w0
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: mov v0.s[1], w5
+; CHECK-NEXT: mov v1.s[1], w3
+; CHECK-NEXT: mov v3.s[1], w1
; CHECK-NEXT: .LBB2_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: fmov s2, w0
-; CHECK-NEXT: fmov s3, w4
+; CHECK-NEXT: add v2.2s, v2.2s, v3.2s
+; CHECK-NEXT: add v4.2s, v4.2s, v1.2s
; CHECK-NEXT: subs x8, x8, #1
-; CHECK-NEXT: mov v2.s[1], w1
-; CHECK-NEXT: mov v3.s[1], w5
-; CHECK-NEXT: mov v2.s[2], w2
-; CHECK-NEXT: add v3.4s, v3.4s, v1.4s
-; CHECK-NEXT: mov w5, v3.s[1]
-; CHECK-NEXT: fmov w4, s3
-; CHECK-NEXT: mov v2.s[3], w3
-; CHECK-NEXT: add v2.4s, v2.4s, v0.4s
-; CHECK-NEXT: mov w3, v2.s[3]
-; CHECK-NEXT: mov w2, v2.s[2]
-; CHECK-NEXT: mov w1, v2.s[1]
-; CHECK-NEXT: fmov w0, s2
+; CHECK-NEXT: add v5.2s, v5.2s, v0.2s
; CHECK-NEXT: b.ne .LBB2_1
; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov w1, v2.s[1]
+; CHECK-NEXT: mov w3, v4.s[1]
+; CHECK-NEXT: mov w5, v5.s[1]
+; CHECK-NEXT: fmov w0, s2
+; CHECK-NEXT: fmov w2, s4
+; CHECK-NEXT: fmov w4, s5
; CHECK-NEXT: ret
entry:
br label %loop
@@ -221,33 +185,21 @@ exit:
define void @v6i64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6i64_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: ldp q4, q1, [x0, #16]
-; CHECK-NEXT: ldr q6, [x0]
-; CHECK-NEXT: mov d7, v1.d[1]
-; CHECK-NEXT: mov d16, v4.d[1]
-; CHECK-NEXT: mov d18, v6.d[1]
-; CHECK-NEXT: fmov d19, d0
-; CHECK-NEXT: fmov d2, d0
-; CHECK-NEXT: fmov d17, d0
-; CHECK-NEXT: fmov d3, d0
-; CHECK-NEXT: fmov d5, d0
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldr q5, [x0]
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: ldp q4, q3, [x0, #16]
; CHECK-NEXT: .LBB4_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add d0, d0, d6
-; CHECK-NEXT: add d19, d19, d18
+; CHECK-NEXT: add v1.2d, v1.2d, v5.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v4.2d
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: add d2, d2, d4
-; CHECK-NEXT: add d17, d17, d16
-; CHECK-NEXT: add d3, d3, d1
-; CHECK-NEXT: add d5, d5, d7
+; CHECK-NEXT: add v2.2d, v2.2d, v3.2d
; CHECK-NEXT: b.ne .LBB4_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v0.d[1], v19.d[0]
-; CHECK-NEXT: mov v2.d[1], v17.d[0]
-; CHECK-NEXT: mov v3.d[1], v5.d[0]
-; CHECK-NEXT: stp q0, q2, [x2]
-; CHECK-NEXT: str q3, [x2, #32]
+; CHECK-NEXT: stp q1, q0, [x2]
+; CHECK-NEXT: str q2, [x2, #32]
; CHECK-NEXT: ret
entry:
%c.val = load <6 x i64>, ptr %c
@@ -269,29 +221,32 @@ exit:
define <6 x i64> @v6i64_cc(<6 x i64> %c.val, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6i64_cc:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi d6, #0000000000000000
-; CHECK-NEXT: fmov d7, d6
-; CHECK-NEXT: fmov d16, d6
-; CHECK-NEXT: fmov d17, d6
-; CHECK-NEXT: fmov d18, d6
-; CHECK-NEXT: fmov d19, d6
+; CHECK-NEXT: // kill: def $d4 killed $d4 def $q4
+; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: // kill: def $d5 killed $d5 def $q5
+; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: movi v6.2d, #0000000000000000
+; CHECK-NEXT: movi v7.2d, #0000000000000000
+; CHECK-NEXT: movi v16.2d, #0000000000000000
+; CHECK-NEXT: mov v4.d[1], v5.d[0]
+; CHECK-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-NEXT: mov v0.d[1], v1.d[0]
; CHECK-NEXT: .LBB5_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add d6, d6, d0
-; CHECK-NEXT: add d7, d7, d1
+; CHECK-NEXT: add v6.2d, v6.2d, v0.2d
+; CHECK-NEXT: add v7.2d, v7.2d, v2.2d
; CHECK-NEXT: subs x2, x2, #1
-; CHECK-NEXT: add d16, d16, d2
-; CHECK-NEXT: add d17, d17, d3
-; CHECK-NEXT: add d18, d18, d4
-; CHECK-NEXT: add d19, d19, d5
+; CHECK-NEXT: add v16.2d, v16.2d, v4.2d
; CHECK-NEXT: b.ne .LBB5_1
; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov d1, v6.d[1]
+; CHECK-NEXT: mov d3, v7.d[1]
+; CHECK-NEXT: mov d5, v16.d[1]
; CHECK-NEXT: fmov d0, d6
-; CHECK-NEXT: fmov d1, d7
-; CHECK-NEXT: fmov d2, d16
-; CHECK-NEXT: fmov d3, d17
-; CHECK-NEXT: fmov d4, d18
-; CHECK-NEXT: fmov d5, d19
+; CHECK-NEXT: fmov d2, d7
+; CHECK-NEXT: fmov d4, d16
; CHECK-NEXT: ret
entry:
br label %loop
@@ -363,42 +318,23 @@ exit:
define void @v6f32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6f32_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp q0, q2, [x0]
-; CHECK-NEXT: movi d7, #0000000000000000
-; CHECK-NEXT: movi d6, #0000000000000000
-; CHECK-NEXT: mov s1, v0.s[1]
-; CHECK-NEXT: mov s3, v0.s[3]
-; CHECK-NEXT: mov s4, v0.s[2]
-; CHECK-NEXT: mov s5, v2.s[1]
-; CHECK-NEXT: mov v0.s[1], v1.s[0]
-; CHECK-NEXT: movi d1, #0000000000000000
-; CHECK-NEXT: mov v2.s[1], v5.s[0]
-; CHECK-NEXT: movi d5, #0000000000000000
-; CHECK-NEXT: mov v0.s[2], v4.s[0]
+; CHECK-NEXT: ldr q1, [x0]
+; CHECK-NEXT: movi d0, #0000000000000000
; CHECK-NEXT: movi d4, #0000000000000000
-; CHECK-NEXT: mov v0.s[3], v3.s[0]
-; CHECK-NEXT: movi d3, #0000000000000000
+; CHECK-NEXT: movi d2, #0000000000000000
+; CHECK-NEXT: ldr d5, [x0, #16]
+; CHECK-NEXT: mov d3, v1.d[1]
; CHECK-NEXT: .LBB7_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: mov v1.s[1], v7.s[0]
-; CHECK-NEXT: mov v3.s[1], v4.s[0]
+; CHECK-NEXT: fadd v0.2s, v0.2s, v1.2s
+; CHECK-NEXT: fadd v4.2s, v4.2s, v3.2s
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: mov v1.s[2], v6.s[0]
-; CHECK-NEXT: fadd v3.4s, v3.4s, v2.4s
-; CHECK-NEXT: mov v1.s[3], v5.s[0]
-; CHECK-NEXT: mov s4, v3.s[1]
-; CHECK-NEXT: fadd v1.4s, v1.4s, v0.4s
-; CHECK-NEXT: mov s5, v1.s[3]
-; CHECK-NEXT: mov s6, v1.s[2]
-; CHECK-NEXT: mov s7, v1.s[1]
+; CHECK-NEXT: fadd v2.2s, v2.2s, v5.2s
; CHECK-NEXT: b.ne .LBB7_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v1.s[1], v7.s[0]
-; CHECK-NEXT: mov v3.s[1], v4.s[0]
-; CHECK-NEXT: mov v1.s[2], v6.s[0]
-; CHECK-NEXT: str d3, [x2, #16]
-; CHECK-NEXT: mov v1.s[3], v5.s[0]
-; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: mov v0.d[1], v4.d[0]
+; CHECK-NEXT: str d2, [x2, #16]
+; CHECK-NEXT: str q0, [x2]
; CHECK-NEXT: ret
entry:
%c.val = load <6 x float>, ptr %c
@@ -420,43 +356,32 @@ exit:
define <6 x float> @v6f32_cc(<6 x float> %c.val, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6f32_cc:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
-; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1
+; CHECK-NEXT: // kill: def $s4 killed $s4 def $q4
; CHECK-NEXT: // kill: def $s2 killed $s2 def $q2
+; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
; CHECK-NEXT: // kill: def $s5 killed $s5 def $q5
-; CHECK-NEXT: // kill: def $s4 killed $s4 def $q4
; CHECK-NEXT: // kill: def $s3 killed $s3 def $q3
-; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1
; CHECK-NEXT: movi d6, #0000000000000000
-; CHECK-NEXT: mov v0.s[1], v1.s[0]
-; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: movi d16, #0000000000000000
; CHECK-NEXT: mov v4.s[1], v5.s[0]
-; CHECK-NEXT: movi d5, #0000000000000000
-; CHECK-NEXT: mov v0.s[2], v2.s[0]
-; CHECK-NEXT: movi d2, #0000000000000000
-; CHECK-NEXT: mov v0.s[3], v3.s[0]
-; CHECK-NEXT: movi d3, #0000000000000000
+; CHECK-NEXT: mov v2.s[1], v3.s[0]
+; CHECK-NEXT: mov v0.s[1], v1.s[0]
; CHECK-NEXT: .LBB8_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: mov v7.s[1], v1.s[0]
-; CHECK-NEXT: mov v6.s[1], v5.s[0]
+; CHECK-NEXT: fadd v6.2s, v6.2s, v0.2s
+; CHECK-NEXT: fadd v7.2s, v7.2s, v2.2s
; CHECK-NEXT: subs x2, x2, #1
-; CHECK-NEXT: mov v7.s[2], v2.s[0]
-; CHECK-NEXT: fadd v6.4s, v6.4s, v4.4s
-; CHECK-NEXT: mov v7.s[3], v3.s[0]
-; CHECK-NEXT: mov s5, v6.s[1]
-; CHECK-NEXT: fadd v7.4s, v7.4s, v0.4s
-; CHECK-NEXT: mov s3, v7.s[3]
-; CHECK-NEXT: mov s2, v7.s[2]
-; CHECK-NEXT: mov s1, v7.s[1]
+; CHECK-NEXT: fadd v16.2s, v16.2s, v4.2s
; CHECK-NEXT: b.ne .LBB8_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: fmov s0, s7
-; CHECK-NEXT: fmov s4, s6
-; CHECK-NEXT: // kill: def $s1 killed $s1 killed $q1
-; CHECK-NEXT: // kill: def $s2 killed $s2 killed $q2
-; CHECK-NEXT: // kill: def $s3 killed $s3 killed $q3
-; CHECK-NEXT: // kill: def $s5 killed $s5 killed $q5
+; CHECK-NEXT: mov s1, v6.s[1]
+; CHECK-NEXT: mov s3, v7.s[1]
+; CHECK-NEXT: mov s5, v16.s[1]
+; CHECK-NEXT: fmov s0, s6
+; CHECK-NEXT: fmov s2, s7
+; CHECK-NEXT: fmov s4, s16
; CHECK-NEXT: ret
entry:
br label %loop
@@ -476,39 +401,24 @@ exit:
define void @v6f64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6f64_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp q2, q1, [x0, #16]
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: ldr q3, [x0]
-; CHECK-NEXT: movi d16, #0000000000000000
-; CHECK-NEXT: movi d4, #0000000000000000
-; CHECK-NEXT: mov d5, v1.d[1]
-; CHECK-NEXT: mov d6, v2.d[1]
-; CHECK-NEXT: mov d7, v3.d[1]
-; CHECK-NEXT: mov v3.d[1], v7.d[0]
-; CHECK-NEXT: mov v2.d[1], v6.d[0]
-; CHECK-NEXT: mov v1.d[1], v5.d[0]
-; CHECK-NEXT: movi d6, #0000000000000000
-; CHECK-NEXT: movi d5, #0000000000000000
-; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldr q5, [x0]
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: ldp q4, q3, [x0, #16]
; CHECK-NEXT: .LBB9_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: mov v0.d[1], v16.d[0]
-; CHECK-NEXT: mov v4.d[1], v6.d[0]
+; CHECK-NEXT: fadd v1.2d, v1.2d, v5.2d
+; CHECK-NEXT: fadd v0.2d, v0.2d, v4.2d
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: mov v5.d[1], v7.d[0]
-; CHECK-NEXT: fadd v0.2d, v0.2d, v3.2d
-; CHECK-NEXT: fadd v4.2d, v4.2d, v2.2d
-; CHECK-NEXT: fadd v5.2d, v5.2d, v1.2d
-; CHECK-NEXT: mov d16, v0.d[1]
-; CHECK-NEXT: mov d6, v4.d[1]
-; CHECK-NEXT: mov d7, v5.d[1]
+; CHECK-NEXT: fadd v2.2d, v2.2d, v3.2d
; CHECK-NEXT: b.ne .LBB9_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v0.d[1], v16.d[0]
-; CHECK-NEXT: mov v4.d[1], v7.d[0]
-; CHECK-NEXT: mov v6.d[1], v5.d[0]
-; CHECK-NEXT: stp q0, q4, [x2]
-; CHECK-NEXT: str q6, [x2, #32]
+; CHECK-NEXT: mov v3.16b, v0.16b
+; CHECK-NEXT: ext v0.16b, v0.16b, v2.16b, #8
+; CHECK-NEXT: mov v3.d[1], v2.d[1]
+; CHECK-NEXT: str q0, [x2, #32]
+; CHECK-NEXT: stp q1, q3, [x2]
; CHECK-NEXT: ret
entry:
%c.val = load <6 x double>, ptr %c
@@ -540,41 +450,32 @@ exit:
define <6 x double> @v6f64_cc(<6 x double> %c.val, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6f64_cc:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: fmov d16, d0
-; CHECK-NEXT: // kill: def $d5 killed $d5 def $q5
; CHECK-NEXT: // kill: def $d4 killed $d4 def $q4
-; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3
; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: // kill: def $d5 killed $d5 def $q5
+; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3
; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: movi d7, #0000000000000000
-; CHECK-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-NEXT: movi v6.2d, #0000000000000000
+; CHECK-NEXT: movi v7.2d, #0000000000000000
+; CHECK-NEXT: movi v16.2d, #0000000000000000
; CHECK-NEXT: mov v4.d[1], v5.d[0]
-; CHECK-NEXT: movi d3, #0000000000000000
-; CHECK-NEXT: movi d6, #0000000000000000
-; CHECK-NEXT: movi d5, #0000000000000000
-; CHECK-NEXT: mov v16.d[1], v1.d[0]
-; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-NEXT: mov v0.d[1], v1.d[0]
; CHECK-NEXT: .LBB10_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-NEXT: mov v7.d[1], v3.d[0]
-; CHECK-NEXT: subs x2, x2, #1
-; CHECK-NEXT: mov v6.d[1], v5.d[0]
-; CHECK-NEXT: fadd v0.2d, v0.2d, v16.2d
+; CHECK-NEXT: fadd v6.2d, v6.2d, v0.2d
; CHECK-NEXT: fadd v7.2d, v7.2d, v2.2d
-; CHECK-NEXT: fadd v6.2d, v6.2d, v4.2d
-; CHECK-NEXT: mov d1, v0.d[1]
-; CHECK-NEXT: mov d3, v7.d[1]
-; CHECK-NEXT: mov d5, v6.d[1]
+; CHECK-NEXT: subs x2, x2, #1
+; CHECK-NEXT: fadd v16.2d, v16.2d, v4.2d
; CHECK-NEXT: b.ne .LBB10_1
; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov d1, v6.d[1]
+; CHECK-NEXT: mov d3, v7.d[1]
+; CHECK-NEXT: mov d5, v16.d[1]
+; CHECK-NEXT: fmov d0, d6
; CHECK-NEXT: fmov d2, d7
-; CHECK-NEXT: fmov d4, d6
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT: // kill: def $d1 killed $d1 killed $q1
-; CHECK-NEXT: // kill: def $d3 killed $d3 killed $q3
-; CHECK-NEXT: // kill: def $d5 killed $d5 killed $q5
+; CHECK-NEXT: fmov d4, d16
; CHECK-NEXT: ret
entry:
br label %loop
@@ -594,65 +495,29 @@ exit:
define void @v12f64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v12f64_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp q2, q1, [x0]
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: ldp q4, q3, [x0, #32]
-; CHECK-NEXT: movi d25, #0000000000000000
-; CHECK-NEXT: ldp q5, q6, [x0, #64]
-; CHECK-NEXT: movi d7, #0000000000000000
-; CHECK-NEXT: mov d16, v1.d[1]
-; CHECK-NEXT: mov d17, v2.d[1]
-; CHECK-NEXT: movi d24, #0000000000000000
-; CHECK-NEXT: mov d19, v3.d[1]
-; CHECK-NEXT: mov d20, v4.d[1]
-; CHECK-NEXT: movi d23, #0000000000000000
-; CHECK-NEXT: mov d18, v5.d[1]
-; CHECK-NEXT: mov d21, v6.d[1]
-; CHECK-NEXT: movi d22, #0000000000000000
-; CHECK-NEXT: mov v2.d[1], v17.d[0]
-; CHECK-NEXT: mov v1.d[1], v16.d[0]
-; CHECK-NEXT: movi d16, #0000000000000000
-; CHECK-NEXT: mov v4.d[1], v20.d[0]
-; CHECK-NEXT: mov v3.d[1], v19.d[0]
-; CHECK-NEXT: movi d17, #0000000000000000
-; CHECK-NEXT: mov v5.d[1], v18.d[0]
-; CHECK-NEXT: mov v6.d[1], v21.d[0]
-; CHECK-NEXT: movi d18, #0000000000000000
-; CHECK-NEXT: movi d21, #0000000000000000
-; CHECK-NEXT: movi d19, #0000000000000000
-; CHECK-NEXT: movi d20, #0000000000000000
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: ldp q7, q6, [x0, #64]
+; CHECK-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: ldp q17, q16, [x0, #32]
+; CHECK-NEXT: ldp q19, q18, [x0]
; CHECK-NEXT: .LBB11_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: mov v0.d[1], v25.d[0]
-; CHECK-NEXT: mov v7.d[1], v24.d[0]
+; CHECK-NEXT: fadd v1.2d, v1.2d, v19.2d
+; CHECK-NEXT: fadd v0.2d, v0.2d, v18.2d
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: mov v16.d[1], v23.d[0]
-; CHECK-NEXT: mov v17.d[1], v22.d[0]
-; CHECK-NEXT: mov v18.d[1], v21.d[0]
-; CHECK-NEXT: mov v19.d[1], v20.d[0]
-; CHECK-NEXT: fadd v0.2d, v0.2d, v2.2d
-; CHECK-NEXT: fadd v7.2d, v7.2d, v1.2d
-; CHECK-NEXT: fadd v16.2d, v16.2d, v4.2d
-; CHECK-NEXT: fadd v17.2d, v17.2d, v3.2d
-; CHECK-NEXT: fadd v18.2d, v18.2d, v5.2d
-; CHECK-NEXT: fadd v19.2d, v19.2d, v6.2d
-; CHECK-NEXT: mov d25, v0.d[1]
-; CHECK-NEXT: mov d24, v7.d[1]
-; CHECK-NEXT: mov d23, v16.d[1]
-; CHECK-NEXT: mov d22, v17.d[1]
-; CHECK-NEXT: mov d21, v18.d[1]
-; CHECK-NEXT: mov d20, v19.d[1]
+; CHECK-NEXT: fadd v2.2d, v2.2d, v17.2d
+; CHECK-NEXT: fadd v3.2d, v3.2d, v16.2d
+; CHECK-NEXT: fadd v4.2d, v4.2d, v7.2d
+; CHECK-NEXT: fadd v5.2d, v5.2d, v6.2d
; CHECK-NEXT: b.ne .LBB11_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v0.d[1], v25.d[0]
-; CHECK-NEXT: mov v7.d[1], v24.d[0]
-; CHECK-NEXT: mov v16.d[1], v23.d[0]
-; CHECK-NEXT: mov v17.d[1], v22.d[0]
-; CHECK-NEXT: mov v18.d[1], v21.d[0]
-; CHECK-NEXT: mov v19.d[1], v20.d[0]
-; CHECK-NEXT: stp q0, q7, [x2]
-; CHECK-NEXT: stp q16, q17, [x2, #32]
-; CHECK-NEXT: stp q18, q19, [x2, #64]
+; CHECK-NEXT: stp q1, q0, [x2]
+; CHECK-NEXT: stp q2, q3, [x2, #32]
+; CHECK-NEXT: stp q4, q5, [x2, #64]
; CHECK-NEXT: ret
entry:
%c.val = load <12 x double>, ptr %c
>From 0fd65f96d3015554f2526b29105ad85472122133 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Tue, 4 Aug 2026 20:49:18 +0000
Subject: [PATCH 3/6] Refactor to avoid changing MVT Tables
---
llvm/include/llvm/CodeGen/TargetLowering.h | 16 ++++++++-----
llvm/lib/CodeGen/TargetLoweringBase.cpp | 28 +++-------------------
2 files changed, 13 insertions(+), 31 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index e8caf0c2471e8..30c11b28ee380 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1235,6 +1235,12 @@ class LLVM_ABI TargetLoweringBase {
return false;
}
+ bool shouldUseDynamicVectorTypeBreakdown(EVT VT, bool ForCallingConv) const {
+ return preferVectorizedNonPowerOfTwoTypeBreakdown() && !ForCallingConv &&
+ VT.isFixedLengthVector() &&
+ !isPowerOf2_32(VT.getVectorNumElements());
+ }
+
/// Certain targets such as MIPS require that some types such as vectors are
/// always broken down into scalars in some contexts. This occurs even if the
/// vector type is legal.
@@ -1859,9 +1865,8 @@ class LLVM_ABI TargetLoweringBase {
/// Return the type of registers that this ValueType will eventually require.
MVT getRegisterType(LLVMContext &Context, EVT VT,
bool ForCallingConv = false) const {
- if (VT.isSimple() && (!ForCallingConv || !VT.isFixedLengthVector() ||
- isPowerOf2_32(VT.getVectorNumElements()) ||
- !preferVectorizedNonPowerOfTwoTypeBreakdown()))
+ if (VT.isSimple() &&
+ !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv))
return getRegisterType(VT.getSimpleVT());
if (VT.isVector()) {
EVT VT1;
@@ -1892,9 +1897,8 @@ class LLVM_ABI TargetLoweringBase {
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT,
std::optional<MVT> RegisterVT = std::nullopt,
bool ForCallingConv = false) const {
- if (VT.isSimple() && (!ForCallingConv || !VT.isFixedLengthVector() ||
- isPowerOf2_32(VT.getVectorNumElements()) ||
- !preferVectorizedNonPowerOfTwoTypeBreakdown())) {
+ if (VT.isSimple() &&
+ !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv)) {
assert((unsigned)VT.getSimpleVT().SimpleTy <
std::size(NumRegistersForVT));
return NumRegistersForVT[VT.getSimpleVT().SimpleTy];
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 9a9c87c1bced6..1acc2a0b5d7b2 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1545,33 +1545,11 @@ static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
llvm_unreachable(
"Splitting or widening of non-power-of-2 MVTs is not implemented.");
- // FIXME: We don't generically support non-power-of-2-sized vectors for now.
+ // FIXME: We don't support non-power-of-2-sized vectors for now.
// Ideally we could break down into LHS/RHS like LegalizeDAG does.
if (!isPowerOf2_32(EC.getKnownMinValue())) {
- assert(VT.isFixedLengthVector() && "Expected a fixed-length vector VT");
- unsigned NumElts = EC.getKnownMinValue();
-
- // Find the largest legal vector type that exactly divides a
- // non-power-of-two vector.
- if (TLI->preferVectorizedNonPowerOfTwoTypeBreakdown()) {
- for (unsigned PartElts = llvm::bit_floor(NumElts); PartElts > 1;
- PartElts >>= 1) {
- if (NumElts % PartElts != 0)
- continue;
-
- MVT PartVT = MVT::getVectorVT(EltTy, ElementCount::getFixed(PartElts));
- if (PartVT == MVT() || !TLI->isTypeLegal(PartVT))
- continue;
-
- IntermediateVT = PartVT;
- NumIntermediates = NumElts / PartElts;
- RegisterVT = PartVT;
- return NumIntermediates;
- }
- }
-
- // Fall back to scalars if there is no exact legal vector decomposition.
- NumVectorRegs = NumElts;
+ // Split EC to unit size (scalable property is preserved).
+ NumVectorRegs = EC.getKnownMinValue();
EC = ElementCount::getFixed(1);
}
>From a04a51c4d7ad99f8306963d5aaa7040233cc2993 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Wed, 5 Aug 2026 00:54:29 +0000
Subject: [PATCH 4/6] Reuse existing getTypeConversion logic from isScalable
block
---
llvm/lib/CodeGen/TargetLoweringBase.cpp | 44 ++--
.../CodeGen/AArch64/non-pow2-fixed-vectors.ll | 197 ++++++------------
llvm/test/CodeGen/AArch64/phi.ll | 76 +++++--
3 files changed, 139 insertions(+), 178 deletions(-)
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 1acc2a0b5d7b2..4c0541b37afe5 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -2010,9 +2010,7 @@ unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,
unsigned NumVectorRegs = 1;
- // Scalable vectors cannot be scalarized, so handle the legalisation of the
- // types like done elsewhere in SelectionDAG.
- if (EltCnt.isScalable()) {
+ auto GetLegalVectorBreakdown = [&]() -> std::optional<unsigned> {
LegalizeKind LK;
EVT PartVT = VT;
do {
@@ -2021,17 +2019,25 @@ unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,
PartVT = LK.second;
} while (LK.first != TypeLegal);
- if (!PartVT.isVector()) {
- report_fatal_error(
- "Don't know how to legalize this scalable vector type");
- }
+ if (!PartVT.isVector())
+ return std::nullopt;
+ assert(PartVT.isScalableVector() == VT.isScalableVector() &&
+ "Vector legalization changed scalability");
NumIntermediates =
divideCeil(VT.getVectorElementCount().getKnownMinValue(),
PartVT.getVectorElementCount().getKnownMinValue());
IntermediateVT = PartVT;
RegisterVT = getRegisterType(Context, IntermediateVT);
return NumIntermediates;
+ };
+
+ // Scalable vectors cannot be scalarized, so handle the legalisation of the
+ // types like done elsewhere in SelectionDAG.
+ if (EltCnt.isScalable()) {
+ if (std::optional<unsigned> NumRegs = GetLegalVectorBreakdown())
+ return *NumRegs;
+ report_fatal_error("Don't know how to legalize this scalable vector type");
}
// FIXME: We don't generically support non-power-of-2-sized vectors for now.
@@ -2040,27 +2046,11 @@ unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,
assert(VT.isFixedLengthVector() && "Expected a fixed-length vector VT");
unsigned NumElts = EltCnt.getKnownMinValue();
- // Find the largest legal vector type that exactly divides a
- // non-power-of-two vector.
- if (!ForCallingConv && preferVectorizedNonPowerOfTwoTypeBreakdown()) {
- for (unsigned PartElts = llvm::bit_floor(NumElts); PartElts > 1;
- PartElts >>= 1) {
- if (NumElts % PartElts != 0)
- continue;
-
- EVT PartVT =
- EVT::getVectorVT(Context, EltTy, ElementCount::getFixed(PartElts));
- if (!isTypeLegal(PartVT))
- continue;
-
- IntermediateVT = PartVT;
- NumIntermediates = NumElts / PartElts;
- RegisterVT = PartVT.getSimpleVT();
- return NumIntermediates;
- }
- }
+ if (!ForCallingConv && preferVectorizedNonPowerOfTwoTypeBreakdown())
+ if (std::optional<unsigned> NumRegs = GetLegalVectorBreakdown())
+ return *NumRegs;
- // Fall back to scalars if there is no exact legal vector decomposition.
+ // Fall back to scalars if there is no legal vector decomposition.
NumVectorRegs = NumElts;
EltCnt = ElementCount::getFixed(1);
}
diff --git a/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
index ea945cdbcec43..f5357c2a67181 100644
--- a/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
+++ b/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
@@ -4,41 +4,19 @@
define void @v5i32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v5i32_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp w12, w13, [x0]
-; CHECK-NEXT: mov w1, wzr
-; CHECK-NEXT: mov w11, wzr
-; CHECK-NEXT: mov w9, wzr
-; CHECK-NEXT: mov w8, wzr
-; CHECK-NEXT: mov w10, wzr
-; CHECK-NEXT: fmov s0, w12
-; CHECK-NEXT: mov v0.s[1], w13
-; CHECK-NEXT: ldp w13, w12, [x0, #8]
-; CHECK-NEXT: mov v0.s[2], w13
-; CHECK-NEXT: mov v0.s[3], w12
-; CHECK-NEXT: ldr w12, [x0, #16]
-; CHECK-NEXT: fmov s1, w12
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldr q2, [x0]
+; CHECK-NEXT: ldr s3, [x0, #16]
; CHECK-NEXT: .LBB0_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: fmov s2, w1
-; CHECK-NEXT: fmov s3, w10
+; CHECK-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: add v0.4s, v0.4s, v3.4s
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: mov v2.s[1], w11
-; CHECK-NEXT: add v3.4s, v3.4s, v1.4s
-; CHECK-NEXT: fmov w10, s3
-; CHECK-NEXT: mov v2.s[2], w9
-; CHECK-NEXT: mov v2.s[3], w8
-; CHECK-NEXT: add v2.4s, v2.4s, v0.4s
-; CHECK-NEXT: mov w8, v2.s[3]
-; CHECK-NEXT: mov w9, v2.s[2]
-; CHECK-NEXT: mov w11, v2.s[1]
-; CHECK-NEXT: fmov w1, s2
; CHECK-NEXT: b.ne .LBB0_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v2.s[1], w11
-; CHECK-NEXT: str w10, [x2, #16]
-; CHECK-NEXT: mov v2.s[2], w9
-; CHECK-NEXT: mov v2.s[3], w8
-; CHECK-NEXT: str q2, [x2]
+; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: str s0, [x2, #16]
; CHECK-NEXT: ret
entry:
%c.val = load <5 x i32>, ptr %c
@@ -60,23 +38,18 @@ exit:
define void @v6i32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6i32_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr q1, [x0]
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: movi v4.2d, #0000000000000000
-; CHECK-NEXT: movi v2.2d, #0000000000000000
-; CHECK-NEXT: ldr d5, [x0, #16]
-; CHECK-NEXT: mov d3, v1.d[1]
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldp q3, q2, [x0]
; CHECK-NEXT: .LBB1_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add v0.2s, v0.2s, v1.2s
-; CHECK-NEXT: add v4.2s, v4.2s, v3.2s
+; CHECK-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: add v2.2s, v2.2s, v5.2s
; CHECK-NEXT: b.ne .LBB1_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v0.d[1], v4.d[0]
-; CHECK-NEXT: str d2, [x2, #16]
-; CHECK-NEXT: str q0, [x2]
+; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: str d0, [x2, #16]
; CHECK-NEXT: ret
entry:
%c.val = load <6 x i32>, ptr %c
@@ -98,30 +71,28 @@ exit:
define <6 x i32> @v6i32_cc(<6 x i32> %c.val, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6i32_cc:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: fmov s0, w4
-; CHECK-NEXT: fmov s1, w2
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: fmov s2, w4
; CHECK-NEXT: ldr x8, [sp]
-; CHECK-NEXT: fmov s3, w0
-; CHECK-NEXT: movi v2.2d, #0000000000000000
-; CHECK-NEXT: movi v4.2d, #0000000000000000
-; CHECK-NEXT: movi v5.2d, #0000000000000000
-; CHECK-NEXT: mov v0.s[1], w5
-; CHECK-NEXT: mov v1.s[1], w3
-; CHECK-NEXT: mov v3.s[1], w1
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: mov v0.s[1], w1
+; CHECK-NEXT: mov v2.s[1], w5
+; CHECK-NEXT: mov v0.s[2], w2
+; CHECK-NEXT: mov v0.s[3], w3
; CHECK-NEXT: .LBB2_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add v2.2s, v2.2s, v3.2s
-; CHECK-NEXT: add v4.2s, v4.2s, v1.2s
+; CHECK-NEXT: add v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: add v3.4s, v3.4s, v2.4s
; CHECK-NEXT: subs x8, x8, #1
-; CHECK-NEXT: add v5.2s, v5.2s, v0.2s
; CHECK-NEXT: b.ne .LBB2_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov w1, v2.s[1]
-; CHECK-NEXT: mov w3, v4.s[1]
-; CHECK-NEXT: mov w5, v5.s[1]
-; CHECK-NEXT: fmov w0, s2
-; CHECK-NEXT: fmov w2, s4
-; CHECK-NEXT: fmov w4, s5
+; CHECK-NEXT: mov w1, v1.s[1]
+; CHECK-NEXT: mov w2, v1.s[2]
+; CHECK-NEXT: mov w3, v1.s[3]
+; CHECK-NEXT: mov w5, v3.s[1]
+; CHECK-NEXT: fmov w0, s1
+; CHECK-NEXT: fmov w4, s3
; CHECK-NEXT: ret
entry:
br label %loop
@@ -141,29 +112,21 @@ exit:
define void @v5i64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v5i64_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: ldp q2, q1, [x0]
-; CHECK-NEXT: ldr d7, [x0, #32]
-; CHECK-NEXT: mov d3, v1.d[1]
-; CHECK-NEXT: mov d6, v2.d[1]
-; CHECK-NEXT: fmov d17, d0
-; CHECK-NEXT: fmov d4, d0
-; CHECK-NEXT: fmov d16, d0
-; CHECK-NEXT: fmov d5, d0
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldr d5, [x0, #32]
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: ldp q4, q3, [x0]
; CHECK-NEXT: .LBB3_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add d0, d0, d2
-; CHECK-NEXT: add d17, d17, d6
+; CHECK-NEXT: add v1.2d, v1.2d, v4.2d
+; CHECK-NEXT: add v2.2d, v2.2d, v3.2d
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: add d4, d4, d1
-; CHECK-NEXT: add d16, d16, d3
-; CHECK-NEXT: add d5, d5, d7
+; CHECK-NEXT: add v0.2d, v0.2d, v5.2d
; CHECK-NEXT: b.ne .LBB3_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v0.d[1], v17.d[0]
-; CHECK-NEXT: mov v4.d[1], v16.d[0]
-; CHECK-NEXT: str d5, [x2, #32]
-; CHECK-NEXT: stp q0, q4, [x2]
+; CHECK-NEXT: stp q1, q2, [x2]
+; CHECK-NEXT: str d0, [x2, #32]
; CHECK-NEXT: ret
entry:
%c.val = load <5 x i64>, ptr %c
@@ -266,37 +229,19 @@ exit:
define void @v5f32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v5f32_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr q0, [x0]
-; CHECK-NEXT: movi d5, #0000000000000000
-; CHECK-NEXT: movi d4, #0000000000000000
-; CHECK-NEXT: ldr s6, [x0, #16]
-; CHECK-NEXT: mov s1, v0.s[1]
-; CHECK-NEXT: mov s2, v0.s[3]
-; CHECK-NEXT: mov s3, v0.s[2]
-; CHECK-NEXT: mov v0.s[1], v1.s[0]
-; CHECK-NEXT: movi d1, #0000000000000000
-; CHECK-NEXT: mov v0.s[2], v3.s[0]
-; CHECK-NEXT: movi d3, #0000000000000000
-; CHECK-NEXT: mov v0.s[3], v2.s[0]
-; CHECK-NEXT: movi d2, #0000000000000000
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldr q2, [x0]
+; CHECK-NEXT: ldr s3, [x0, #16]
; CHECK-NEXT: .LBB6_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: mov v1.s[1], v5.s[0]
-; CHECK-NEXT: fadd v2.4s, v2.4s, v6.4s
+; CHECK-NEXT: fadd v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: fadd v0.4s, v0.4s, v3.4s
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: mov v1.s[2], v4.s[0]
-; CHECK-NEXT: mov v1.s[3], v3.s[0]
-; CHECK-NEXT: fadd v1.4s, v1.4s, v0.4s
-; CHECK-NEXT: mov s3, v1.s[3]
-; CHECK-NEXT: mov s4, v1.s[2]
-; CHECK-NEXT: mov s5, v1.s[1]
; CHECK-NEXT: b.ne .LBB6_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v1.s[1], v5.s[0]
-; CHECK-NEXT: str s2, [x2, #16]
-; CHECK-NEXT: mov v1.s[2], v4.s[0]
-; CHECK-NEXT: mov v1.s[3], v3.s[0]
; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: str s0, [x2, #16]
; CHECK-NEXT: ret
entry:
%c.val = load <5 x float>, ptr %c
@@ -318,23 +263,18 @@ exit:
define void @v6f32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6f32_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr q1, [x0]
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: movi d4, #0000000000000000
-; CHECK-NEXT: movi d2, #0000000000000000
-; CHECK-NEXT: ldr d5, [x0, #16]
-; CHECK-NEXT: mov d3, v1.d[1]
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldp q3, q2, [x0]
; CHECK-NEXT: .LBB7_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: fadd v0.2s, v0.2s, v1.2s
-; CHECK-NEXT: fadd v4.2s, v4.2s, v3.2s
+; CHECK-NEXT: fadd v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: fadd v0.4s, v0.4s, v2.4s
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: fadd v2.2s, v2.2s, v5.2s
; CHECK-NEXT: b.ne .LBB7_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v0.d[1], v4.d[0]
-; CHECK-NEXT: str d2, [x2, #16]
-; CHECK-NEXT: str q0, [x2]
+; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: str d0, [x2, #16]
; CHECK-NEXT: ret
entry:
%c.val = load <6 x float>, ptr %c
@@ -356,32 +296,31 @@ exit:
define <6 x float> @v6f32_cc(<6 x float> %c.val, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6f32_cc:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $s4 killed $s4 def $q4
-; CHECK-NEXT: // kill: def $s2 killed $s2 def $q2
; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1
+; CHECK-NEXT: // kill: def $s2 killed $s2 def $q2
+; CHECK-NEXT: // kill: def $s4 killed $s4 def $q4
; CHECK-NEXT: // kill: def $s5 killed $s5 def $q5
; CHECK-NEXT: // kill: def $s3 killed $s3 def $q3
-; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1
-; CHECK-NEXT: movi d6, #0000000000000000
-; CHECK-NEXT: movi d7, #0000000000000000
-; CHECK-NEXT: movi d16, #0000000000000000
-; CHECK-NEXT: mov v4.s[1], v5.s[0]
-; CHECK-NEXT: mov v2.s[1], v3.s[0]
+; CHECK-NEXT: movi v6.2d, #0000000000000000
+; CHECK-NEXT: movi v7.2d, #0000000000000000
; CHECK-NEXT: mov v0.s[1], v1.s[0]
+; CHECK-NEXT: mov v4.s[1], v5.s[0]
+; CHECK-NEXT: mov v0.s[2], v2.s[0]
+; CHECK-NEXT: mov v0.s[3], v3.s[0]
; CHECK-NEXT: .LBB8_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: fadd v6.2s, v6.2s, v0.2s
-; CHECK-NEXT: fadd v7.2s, v7.2s, v2.2s
+; CHECK-NEXT: fadd v6.4s, v6.4s, v0.4s
+; CHECK-NEXT: fadd v7.4s, v7.4s, v4.4s
; CHECK-NEXT: subs x2, x2, #1
-; CHECK-NEXT: fadd v16.2s, v16.2s, v4.2s
; CHECK-NEXT: b.ne .LBB8_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: mov s1, v6.s[1]
-; CHECK-NEXT: mov s3, v7.s[1]
-; CHECK-NEXT: mov s5, v16.s[1]
+; CHECK-NEXT: mov s2, v6.s[2]
+; CHECK-NEXT: mov s3, v6.s[3]
+; CHECK-NEXT: mov s5, v7.s[1]
; CHECK-NEXT: fmov s0, s6
-; CHECK-NEXT: fmov s2, s7
-; CHECK-NEXT: fmov s4, s16
+; CHECK-NEXT: fmov s4, s7
; CHECK-NEXT: ret
entry:
br label %loop
diff --git a/llvm/test/CodeGen/AArch64/phi.ll b/llvm/test/CodeGen/AArch64/phi.ll
index cad959c077990..399c19dbd7b2c 100644
--- a/llvm/test/CodeGen/AArch64/phi.ll
+++ b/llvm/test/CodeGen/AArch64/phi.ll
@@ -336,14 +336,19 @@ define <3 x i8> @tv3i8(i1 %c, ptr %p, <3 x i8> %a, <3 x i8> %b) {
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: tbz w0, #0, .LBB11_2
; CHECK-SD-NEXT: // %bb.1: // %t
-; CHECK-SD-NEXT: mov w5, w2
-; CHECK-SD-NEXT: mov w6, w3
-; CHECK-SD-NEXT: mov w7, w4
+; CHECK-SD-NEXT: fmov s0, w2
; CHECK-SD-NEXT: str wzr, [x1]
-; CHECK-SD-NEXT: .LBB11_2: // %e
-; CHECK-SD-NEXT: mov w0, w5
-; CHECK-SD-NEXT: mov w1, w6
-; CHECK-SD-NEXT: mov w2, w7
+; CHECK-SD-NEXT: mov v0.h[1], w3
+; CHECK-SD-NEXT: mov v0.h[2], w4
+; CHECK-SD-NEXT: b .LBB11_3
+; CHECK-SD-NEXT: .LBB11_2:
+; CHECK-SD-NEXT: fmov s0, w5
+; CHECK-SD-NEXT: mov v0.h[1], w6
+; CHECK-SD-NEXT: mov v0.h[2], w7
+; CHECK-SD-NEXT: .LBB11_3: // %e
+; CHECK-SD-NEXT: umov w0, v0.h[0]
+; CHECK-SD-NEXT: umov w1, v0.h[1]
+; CHECK-SD-NEXT: umov w2, v0.h[2]
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: tv3i8:
@@ -728,15 +733,24 @@ e:
define <3 x i64> @tv3i64(i1 %c, ptr %p, <3 x i64> %a, <3 x i64> %b) {
; CHECK-SD-LABEL: tv3i64:
; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $d5 killed $d5 def $q5
+; CHECK-SD-NEXT: // kill: def $d4 killed $d4 def $q4
+; CHECK-SD-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT: tbz w0, #0, .LBB26_2
; CHECK-SD-NEXT: // %bb.1: // %t
-; CHECK-SD-NEXT: fmov d3, d0
-; CHECK-SD-NEXT: fmov d4, d1
+; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT: mov v5.16b, v2.16b
; CHECK-SD-NEXT: str wzr, [x1]
-; CHECK-SD-NEXT: fmov d5, d2
-; CHECK-SD-NEXT: .LBB26_2: // %e
+; CHECK-SD-NEXT: mov v3.16b, v0.16b
+; CHECK-SD-NEXT: b .LBB26_3
+; CHECK-SD-NEXT: .LBB26_2:
+; CHECK-SD-NEXT: mov v3.d[1], v4.d[0]
+; CHECK-SD-NEXT: .LBB26_3: // %e
+; CHECK-SD-NEXT: mov d1, v3.d[1]
; CHECK-SD-NEXT: fmov d0, d3
-; CHECK-SD-NEXT: fmov d1, d4
; CHECK-SD-NEXT: fmov d2, d5
; CHECK-SD-NEXT: ret
;
@@ -877,15 +891,24 @@ e:
define <3 x ptr> @tv3p0(i1 %c, ptr %p, <3 x ptr> %a, <3 x ptr> %b) {
; CHECK-SD-LABEL: tv3p0:
; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $d5 killed $d5 def $q5
+; CHECK-SD-NEXT: // kill: def $d4 killed $d4 def $q4
+; CHECK-SD-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT: tbz w0, #0, .LBB30_2
; CHECK-SD-NEXT: // %bb.1: // %t
-; CHECK-SD-NEXT: fmov d3, d0
-; CHECK-SD-NEXT: fmov d4, d1
+; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT: mov v5.16b, v2.16b
; CHECK-SD-NEXT: str wzr, [x1]
-; CHECK-SD-NEXT: fmov d5, d2
-; CHECK-SD-NEXT: .LBB30_2: // %e
+; CHECK-SD-NEXT: mov v3.16b, v0.16b
+; CHECK-SD-NEXT: b .LBB30_3
+; CHECK-SD-NEXT: .LBB30_2:
+; CHECK-SD-NEXT: mov v3.d[1], v4.d[0]
+; CHECK-SD-NEXT: .LBB30_3: // %e
+; CHECK-SD-NEXT: mov d1, v3.d[1]
; CHECK-SD-NEXT: fmov d0, d3
-; CHECK-SD-NEXT: fmov d1, d4
; CHECK-SD-NEXT: fmov d2, d5
; CHECK-SD-NEXT: ret
;
@@ -1191,15 +1214,24 @@ e:
define <3 x double> @tv3f64(i1 %c, ptr %p, <3 x double> %a, <3 x double> %b) {
; CHECK-SD-LABEL: tv3f64:
; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $d5 killed $d5 def $q5
+; CHECK-SD-NEXT: // kill: def $d4 killed $d4 def $q4
+; CHECK-SD-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT: tbz w0, #0, .LBB42_2
; CHECK-SD-NEXT: // %bb.1: // %t
-; CHECK-SD-NEXT: fmov d3, d0
-; CHECK-SD-NEXT: fmov d4, d1
+; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT: mov v5.16b, v2.16b
; CHECK-SD-NEXT: str wzr, [x1]
-; CHECK-SD-NEXT: fmov d5, d2
-; CHECK-SD-NEXT: .LBB42_2: // %e
+; CHECK-SD-NEXT: mov v3.16b, v0.16b
+; CHECK-SD-NEXT: b .LBB42_3
+; CHECK-SD-NEXT: .LBB42_2:
+; CHECK-SD-NEXT: mov v3.d[1], v4.d[0]
+; CHECK-SD-NEXT: .LBB42_3: // %e
+; CHECK-SD-NEXT: mov d1, v3.d[1]
; CHECK-SD-NEXT: fmov d0, d3
-; CHECK-SD-NEXT: fmov d1, d4
; CHECK-SD-NEXT: fmov d2, d5
; CHECK-SD-NEXT: ret
;
>From c9ef3d3d267beeb4004139bdd1ca77e5d6fdd132 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Wed, 5 Aug 2026 10:22:04 +0000
Subject: [PATCH 5/6] Remove spurious include
---
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index c5a47414ddf18..f53534e3edfa7 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -20,7 +20,6 @@
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Instruction.h"
-#include "llvm/Support/MathExtras.h"
namespace llvm {
>From f7379e2d7bea9bd17c4101f84218ebde5cf48ccd Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Fri, 7 Aug 2026 14:29:01 +0000
Subject: [PATCH 6/6] Address review
Refactor to use *Impl methods and remove other Target changes.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 115 ++++++++++--------
llvm/lib/CodeGen/TargetLoweringBase.cpp | 8 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 9 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 3 +-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 9 +-
llvm/lib/Target/RISCV/RISCVISelLowering.h | 6 +-
llvm/lib/Target/SPIRV/SPIRVISelLowering.h | 6 +-
llvm/lib/Target/SystemZ/SystemZISelLowering.h | 9 +-
8 files changed, 89 insertions(+), 76 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 30c11b28ee380..09afc37f90d16 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1226,8 +1226,12 @@ class LLVM_ABI TargetLoweringBase {
/// before they are promoted/expanded.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT,
EVT &IntermediateVT,
- unsigned &NumIntermediates, MVT &RegisterVT,
- bool ForCallingConv = false) const;
+ unsigned &NumIntermediates,
+ MVT &RegisterVT) const {
+ return getVectorTypeBreakdownImpl(Context, VT, IntermediateVT,
+ NumIntermediates, RegisterVT,
+ /*ForCallingConv=*/false);
+ }
/// Return true if fixed-length, non-power-of-two vectors should be broken
/// down into legal vector parts instead of scalars for internal values.
@@ -1247,8 +1251,9 @@ class LLVM_ABI TargetLoweringBase {
virtual unsigned getVectorTypeBreakdownForCallingConv(
LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
unsigned &NumIntermediates, MVT &RegisterVT) const {
- return getVectorTypeBreakdown(Context, VT, IntermediateVT, NumIntermediates,
- RegisterVT, /*ForCallingConv=*/true);
+ return getVectorTypeBreakdownImpl(Context, VT, IntermediateVT,
+ NumIntermediates, RegisterVT,
+ /*ForCallingConv=*/true);
}
struct IntrinsicInfo {
@@ -1863,24 +1868,8 @@ class LLVM_ABI TargetLoweringBase {
}
/// Return the type of registers that this ValueType will eventually require.
- MVT getRegisterType(LLVMContext &Context, EVT VT,
- bool ForCallingConv = false) const {
- if (VT.isSimple() &&
- !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv))
- return getRegisterType(VT.getSimpleVT());
- if (VT.isVector()) {
- EVT VT1;
- MVT RegisterVT;
- unsigned NumIntermediates;
- (void)getVectorTypeBreakdown(Context, VT, VT1, NumIntermediates,
- RegisterVT, ForCallingConv);
- return RegisterVT;
- }
- if (VT.isInteger()) {
- return getRegisterType(Context, getTypeToTransformTo(Context, VT),
- ForCallingConv);
- }
- llvm_unreachable("Unsupported extended type!");
+ MVT getRegisterType(LLVMContext &Context, EVT VT) const {
+ return getRegisterTypeImpl(Context, VT, /*ForCallingConv=*/false);
}
/// Return the number of registers that this ValueType will eventually
@@ -1894,29 +1883,10 @@ class LLVM_ABI TargetLoweringBase {
///
/// RegisterVT may be passed as a way to override the default settings, for
/// instance with i128 inline assembly operands on SystemZ.
- virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt,
- bool ForCallingConv = false) const {
- if (VT.isSimple() &&
- !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv)) {
- assert((unsigned)VT.getSimpleVT().SimpleTy <
- std::size(NumRegistersForVT));
- return NumRegistersForVT[VT.getSimpleVT().SimpleTy];
- }
- if (VT.isVector()) {
- EVT VT1;
- MVT VT2;
- unsigned NumIntermediates;
- return getVectorTypeBreakdown(Context, VT, VT1, NumIntermediates, VT2,
- ForCallingConv);
- }
- if (VT.isInteger()) {
- unsigned BitWidth = VT.getSizeInBits();
- unsigned RegWidth =
- getRegisterType(Context, VT, ForCallingConv).getSizeInBits();
- return (BitWidth + RegWidth - 1) / RegWidth;
- }
- llvm_unreachable("Unsupported extended type!");
+ virtual unsigned
+ getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt) const {
+ return getNumRegistersImpl(Context, VT, /*ForCallingConv=*/false);
}
/// Certain combinations of ABIs, Targets and features require that types
@@ -1924,7 +1894,7 @@ class LLVM_ABI TargetLoweringBase {
/// For MIPS all vector types must be passed through the integer register set.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context,
CallingConv::ID CC, EVT VT) const {
- return getRegisterType(Context, VT, /*ForCallingConv=*/true);
+ return getRegisterTypeImpl(Context, VT, /*ForCallingConv=*/true);
}
/// Certain targets require unusual breakdowns of certain types. For MIPS,
@@ -1933,8 +1903,7 @@ class LLVM_ABI TargetLoweringBase {
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context,
CallingConv::ID CC,
EVT VT) const {
- return getNumRegisters(Context, VT, /*RegisterVT=*/std::nullopt,
- /*ForCallingConv=*/true);
+ return getNumRegistersImpl(Context, VT, /*ForCallingConv=*/true);
}
/// Certain targets have context sensitive alignment requirements, where one
@@ -3991,6 +3960,56 @@ class LLVM_ABI TargetLoweringBase {
return (LegalizeAction)((IndexedModeActions[Ty][IdxMode] >> Shift) & 0xf);
}
+ unsigned getVectorTypeBreakdownImpl(LLVMContext &Context, EVT VT,
+ EVT &IntermediateVT,
+ unsigned &NumIntermediates,
+ MVT &RegisterVT,
+ bool ForCallingConv) const;
+
+ MVT getRegisterTypeImpl(LLVMContext &Context, EVT VT,
+ bool ForCallingConv) const {
+ if (VT.isSimple() &&
+ !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv))
+ return getRegisterType(VT.getSimpleVT());
+ if (VT.isVector()) {
+ EVT VT1;
+ MVT RegisterVT;
+ unsigned NumIntermediates;
+ (void)getVectorTypeBreakdownImpl(Context, VT, VT1, NumIntermediates,
+ RegisterVT, ForCallingConv);
+ return RegisterVT;
+ }
+ if (VT.isInteger()) {
+ return getRegisterTypeImpl(Context, getTypeToTransformTo(Context, VT),
+ ForCallingConv);
+ }
+ llvm_unreachable("Unsupported extended type!");
+ }
+
+ unsigned getNumRegistersImpl(LLVMContext &Context, EVT VT,
+ bool ForCallingConv) const {
+ if (VT.isSimple() &&
+ !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv)) {
+ assert((unsigned)VT.getSimpleVT().SimpleTy <
+ std::size(NumRegistersForVT));
+ return NumRegistersForVT[VT.getSimpleVT().SimpleTy];
+ }
+ if (VT.isVector()) {
+ EVT VT1;
+ MVT VT2;
+ unsigned NumIntermediates;
+ return getVectorTypeBreakdownImpl(Context, VT, VT1, NumIntermediates, VT2,
+ ForCallingConv);
+ }
+ if (VT.isInteger()) {
+ unsigned BitWidth = VT.getSizeInBits();
+ unsigned RegWidth =
+ getRegisterTypeImpl(Context, VT, ForCallingConv).getSizeInBits();
+ return (BitWidth + RegWidth - 1) / RegWidth;
+ }
+ llvm_unreachable("Unsupported extended type!");
+ }
+
protected:
/// Return true if the extension represented by \p I is free.
/// \pre \p I is a sign, zero, or fp extension and
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 4c0541b37afe5..f2c8663fdfa7c 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1981,11 +1981,9 @@ EVT TargetLoweringBase::getSetCCResultType(const DataLayout &DL, LLVMContext &,
/// This method returns the number of registers needed, and the VT for each
/// register. It also returns the VT and quantity of the intermediate values
/// before they are promoted/expanded.
-unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,
- EVT VT, EVT &IntermediateVT,
- unsigned &NumIntermediates,
- MVT &RegisterVT,
- bool ForCallingConv) const {
+unsigned TargetLoweringBase::getVectorTypeBreakdownImpl(
+ LLVMContext &Context, EVT VT, EVT &IntermediateVT,
+ unsigned &NumIntermediates, MVT &RegisterVT, bool ForCallingConv) const {
ElementCount EltCnt = VT.getVectorElementCount();
// If there is a wider vector type with the same element type as this one,
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index dc777bacd47bf..29921ff86b352 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -4028,13 +4028,12 @@ SDValue NVPTXTargetLowering::LowerCopyToReg_128(SDValue Op,
return DAG.getNode(ISD::CopyToReg, DL, ResultsType, NewOps);
}
-unsigned NVPTXTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT,
- bool ForCallingConv) const {
+unsigned NVPTXTargetLowering::getNumRegisters(
+ LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt) const {
if (VT == MVT::i128 && RegisterVT == MVT::i128)
return 1;
- return TargetLoweringBase::getNumRegisters(Context, VT, RegisterVT,
- ForCallingConv);
+ return TargetLoweringBase::getNumRegisters(Context, VT, RegisterVT);
}
bool NVPTXTargetLowering::splitValueIntoRegisterParts(
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index c7238444ae61c..42741ba98b438 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -224,8 +224,7 @@ class NVPTXTargetLowering : public TargetLowering {
SDValue LowerCopyToReg_128(SDValue Op, SelectionDAG &DAG) const;
unsigned getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT,
- bool ForCallingConv = false) const override;
+ std::optional<MVT> RegisterVT) const override;
bool
splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
SDValue *Parts, unsigned NumParts, MVT PartVT,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index b9d3871ef2374..e23427482c1e4 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -2926,16 +2926,15 @@ MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
}
-unsigned RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT,
- bool ForCallingConv) const {
+unsigned
+RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT) const {
// Pair inline assembly operand
if (VT == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
*RegisterVT == MVT::Untyped)
return 1;
- return TargetLowering::getNumRegisters(Context, VT, RegisterVT,
- ForCallingConv);
+ return TargetLowering::getNumRegisters(Context, VT, RegisterVT);
}
unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index d2f0e1f62c845..2ca7c392639f7 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -82,9 +82,9 @@ class RISCVTargetLowering : public TargetLowering {
EVT VT) const override;
/// Return the number of registers for a given MVT, for inline assembly
- unsigned getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt,
- bool ForCallingConv = false) const override;
+ unsigned
+ getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt) const override;
/// Return the number of registers for a given MVT, ensuring vectors are
/// treated as a series of gpr sized integers.
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
index a4e91b3f06d64..ff843649fd6d3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
@@ -53,9 +53,9 @@ class SPIRVTargetLowering : public TargetLowering {
std::pair<unsigned, const TargetRegisterClass *>
getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
StringRef Constraint, MVT VT) const override;
- unsigned getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt,
- bool ForCallingConv = false) const override {
+ unsigned
+ getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt) const override {
return 1;
}
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index dc2da7aea77bf..e6da481353497 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -75,14 +75,13 @@ class SystemZTargetLowering : public TargetLowering {
return TypeWidenVector;
return TargetLoweringBase::getPreferredVectorAction(VT);
}
- unsigned getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT,
- bool ForCallingConv = false) const override {
+ unsigned
+ getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT) const override {
// i128 inline assembly operand.
if (VT == MVT::i128 && RegisterVT && *RegisterVT == MVT::Untyped)
return 1;
- return TargetLowering::getNumRegisters(
- Context, VT, /*RegisterVT=*/std::nullopt, ForCallingConv);
+ return TargetLowering::getNumRegisters(Context, VT);
}
unsigned
getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC,
More information about the llvm-commits
mailing list