[llvm] [CodeGen][AArch64] Use vector parts for internal non-power-of-two vectors (PR #213982)
Hari Limaye via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 19 02:55:57 PDT 2026
https://github.com/hazzlim updated https://github.com/llvm/llvm-project/pull/213982
>From cb4d9f28fc01dc33c27813690f2e40181c0306d9 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Mon, 3 Aug 2026 10:28:50 +0000
Subject: [PATCH 1/7] Add test
---
.../CodeGen/AArch64/non-pow2-fixed-vectors.ll | 672 ++++++++++++++++++
1 file changed, 672 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
diff --git a/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
new file mode 100644
index 0000000000000..047a377710ecf
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
@@ -0,0 +1,672 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -mtriple=aarch64-linux-gnu -O3 | FileCheck %s
+
+define void @v5i32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v5i32_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp w12, w13, [x0]
+; CHECK-NEXT: mov w1, wzr
+; CHECK-NEXT: mov w11, wzr
+; CHECK-NEXT: mov w9, wzr
+; CHECK-NEXT: mov w8, wzr
+; CHECK-NEXT: mov w10, wzr
+; CHECK-NEXT: fmov s0, w12
+; CHECK-NEXT: mov v0.s[1], w13
+; CHECK-NEXT: ldp w13, w12, [x0, #8]
+; CHECK-NEXT: mov v0.s[2], w13
+; CHECK-NEXT: mov v0.s[3], w12
+; CHECK-NEXT: ldr w12, [x0, #16]
+; CHECK-NEXT: fmov s1, w12
+; CHECK-NEXT: .LBB0_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: fmov s2, w1
+; CHECK-NEXT: fmov s3, w10
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: mov v2.s[1], w11
+; CHECK-NEXT: add v3.4s, v3.4s, v1.4s
+; CHECK-NEXT: fmov w10, s3
+; CHECK-NEXT: mov v2.s[2], w9
+; CHECK-NEXT: mov v2.s[3], w8
+; CHECK-NEXT: add v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: mov w8, v2.s[3]
+; CHECK-NEXT: mov w9, v2.s[2]
+; CHECK-NEXT: mov w11, v2.s[1]
+; CHECK-NEXT: fmov w1, s2
+; CHECK-NEXT: b.ne .LBB0_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v2.s[1], w11
+; CHECK-NEXT: str w10, [x2, #16]
+; CHECK-NEXT: mov v2.s[2], w9
+; CHECK-NEXT: mov v2.s[3], w8
+; CHECK-NEXT: str q2, [x2]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <5 x i32>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <5 x i32> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = add <5 x i32> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <5 x i32> %acc.next, ptr %out, align 8
+ ret void
+}
+
+define void @v6i32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6i32_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp w11, w12, [x0]
+; CHECK-NEXT: mov w4, wzr
+; CHECK-NEXT: mov w10, wzr
+; CHECK-NEXT: mov w9, wzr
+; CHECK-NEXT: mov w8, wzr
+; CHECK-NEXT: mov w1, wzr
+; CHECK-NEXT: fmov s0, w11
+; CHECK-NEXT: ldp w11, w13, [x0, #8]
+; CHECK-NEXT: mov v0.s[1], w12
+; CHECK-NEXT: mov v0.s[2], w11
+; CHECK-NEXT: ldp w11, w12, [x0, #16]
+; CHECK-NEXT: fmov s1, w11
+; CHECK-NEXT: mov v0.s[3], w13
+; CHECK-NEXT: mov w11, wzr
+; CHECK-NEXT: mov v1.s[1], w12
+; CHECK-NEXT: .LBB1_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: fmov s2, w4
+; CHECK-NEXT: fmov s4, w1
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: mov v2.s[1], w10
+; CHECK-NEXT: mov v4.s[1], w11
+; CHECK-NEXT: mov v2.s[2], w9
+; CHECK-NEXT: mov v2.s[3], w8
+; CHECK-NEXT: add v3.4s, v2.4s, v0.4s
+; CHECK-NEXT: add v2.4s, v4.4s, v1.4s
+; CHECK-NEXT: mov w8, v3.s[3]
+; CHECK-NEXT: mov w9, v3.s[2]
+; CHECK-NEXT: mov w10, v3.s[1]
+; CHECK-NEXT: mov w11, v2.s[1]
+; CHECK-NEXT: fmov w4, s3
+; CHECK-NEXT: fmov w1, s2
+; CHECK-NEXT: b.ne .LBB1_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v3.s[1], w10
+; CHECK-NEXT: mov v2.s[1], w11
+; CHECK-NEXT: mov v3.s[2], w9
+; CHECK-NEXT: str d2, [x2, #16]
+; CHECK-NEXT: mov v3.s[3], w8
+; CHECK-NEXT: str q3, [x2]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <6 x i32>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <6 x i32> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = add <6 x i32> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <6 x i32> %acc.next, ptr %out, align 8
+ ret void
+}
+
+define <6 x i32> @v6i32_cc(<6 x i32> %c.val, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6i32_cc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov w10, w0
+; CHECK-NEXT: mov w11, w1
+; CHECK-NEXT: mov w12, w2
+; CHECK-NEXT: fmov s0, w10
+; CHECK-NEXT: mov w8, w4
+; CHECK-NEXT: mov w9, w3
+; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: ldr x8, [sp]
+; CHECK-NEXT: mov w0, wzr
+; CHECK-NEXT: mov w1, wzr
+; CHECK-NEXT: mov w2, wzr
+; CHECK-NEXT: mov w3, wzr
+; CHECK-NEXT: mov v0.s[1], w11
+; CHECK-NEXT: mov w4, wzr
+; CHECK-NEXT: mov v1.s[1], w5
+; CHECK-NEXT: mov w5, wzr
+; CHECK-NEXT: mov v0.s[2], w12
+; CHECK-NEXT: mov v0.s[3], w9
+; CHECK-NEXT: .LBB2_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: fmov s2, w0
+; CHECK-NEXT: fmov s3, w4
+; CHECK-NEXT: subs x8, x8, #1
+; CHECK-NEXT: mov v2.s[1], w1
+; CHECK-NEXT: mov v3.s[1], w5
+; CHECK-NEXT: mov v2.s[2], w2
+; CHECK-NEXT: add v3.4s, v3.4s, v1.4s
+; CHECK-NEXT: mov w5, v3.s[1]
+; CHECK-NEXT: fmov w4, s3
+; CHECK-NEXT: mov v2.s[3], w3
+; CHECK-NEXT: add v2.4s, v2.4s, v0.4s
+; CHECK-NEXT: mov w3, v2.s[3]
+; CHECK-NEXT: mov w2, v2.s[2]
+; CHECK-NEXT: mov w1, v2.s[1]
+; CHECK-NEXT: fmov w0, s2
+; CHECK-NEXT: b.ne .LBB2_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: ret
+entry:
+ br label %loop
+
+loop:
+ %acc = phi <6 x i32> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = add <6 x i32> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ ret <6 x i32> %acc.next
+}
+
+define void @v5i64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v5i64_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: ldp q2, q1, [x0]
+; CHECK-NEXT: ldr d7, [x0, #32]
+; CHECK-NEXT: mov d3, v1.d[1]
+; CHECK-NEXT: mov d6, v2.d[1]
+; CHECK-NEXT: fmov d17, d0
+; CHECK-NEXT: fmov d4, d0
+; CHECK-NEXT: fmov d16, d0
+; CHECK-NEXT: fmov d5, d0
+; CHECK-NEXT: .LBB3_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: add d0, d0, d2
+; CHECK-NEXT: add d17, d17, d6
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: add d4, d4, d1
+; CHECK-NEXT: add d16, d16, d3
+; CHECK-NEXT: add d5, d5, d7
+; CHECK-NEXT: b.ne .LBB3_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v0.d[1], v17.d[0]
+; CHECK-NEXT: mov v4.d[1], v16.d[0]
+; CHECK-NEXT: str d5, [x2, #32]
+; CHECK-NEXT: stp q0, q4, [x2]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <5 x i64>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <5 x i64> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = add <5 x i64> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <5 x i64> %acc.next, ptr %out, align 8
+ ret void
+}
+
+define void @v6i64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6i64_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: ldp q4, q1, [x0, #16]
+; CHECK-NEXT: ldr q6, [x0]
+; CHECK-NEXT: mov d7, v1.d[1]
+; CHECK-NEXT: mov d16, v4.d[1]
+; CHECK-NEXT: mov d18, v6.d[1]
+; CHECK-NEXT: fmov d19, d0
+; CHECK-NEXT: fmov d2, d0
+; CHECK-NEXT: fmov d17, d0
+; CHECK-NEXT: fmov d3, d0
+; CHECK-NEXT: fmov d5, d0
+; CHECK-NEXT: .LBB4_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: add d0, d0, d6
+; CHECK-NEXT: add d19, d19, d18
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: add d2, d2, d4
+; CHECK-NEXT: add d17, d17, d16
+; CHECK-NEXT: add d3, d3, d1
+; CHECK-NEXT: add d5, d5, d7
+; CHECK-NEXT: b.ne .LBB4_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v0.d[1], v19.d[0]
+; CHECK-NEXT: mov v2.d[1], v17.d[0]
+; CHECK-NEXT: mov v3.d[1], v5.d[0]
+; CHECK-NEXT: stp q0, q2, [x2]
+; CHECK-NEXT: str q3, [x2, #32]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <6 x i64>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <6 x i64> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = add <6 x i64> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <6 x i64> %acc.next, ptr %out, align 8
+ ret void
+}
+
+define <6 x i64> @v6i64_cc(<6 x i64> %c.val, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6i64_cc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d6, #0000000000000000
+; CHECK-NEXT: fmov d7, d6
+; CHECK-NEXT: fmov d16, d6
+; CHECK-NEXT: fmov d17, d6
+; CHECK-NEXT: fmov d18, d6
+; CHECK-NEXT: fmov d19, d6
+; CHECK-NEXT: .LBB5_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: add d6, d6, d0
+; CHECK-NEXT: add d7, d7, d1
+; CHECK-NEXT: subs x2, x2, #1
+; CHECK-NEXT: add d16, d16, d2
+; CHECK-NEXT: add d17, d17, d3
+; CHECK-NEXT: add d18, d18, d4
+; CHECK-NEXT: add d19, d19, d5
+; CHECK-NEXT: b.ne .LBB5_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: fmov d0, d6
+; CHECK-NEXT: fmov d1, d7
+; CHECK-NEXT: fmov d2, d16
+; CHECK-NEXT: fmov d3, d17
+; CHECK-NEXT: fmov d4, d18
+; CHECK-NEXT: fmov d5, d19
+; CHECK-NEXT: ret
+entry:
+ br label %loop
+
+loop:
+ %acc = phi <6 x i64> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = add <6 x i64> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ ret <6 x i64> %acc.next
+}
+
+define void @v5f32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v5f32_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldr q0, [x0]
+; CHECK-NEXT: movi d5, #0000000000000000
+; CHECK-NEXT: movi d4, #0000000000000000
+; CHECK-NEXT: ldr s6, [x0, #16]
+; CHECK-NEXT: mov s1, v0.s[1]
+; CHECK-NEXT: mov s2, v0.s[3]
+; CHECK-NEXT: mov s3, v0.s[2]
+; CHECK-NEXT: mov v0.s[1], v1.s[0]
+; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: mov v0.s[2], v3.s[0]
+; CHECK-NEXT: movi d3, #0000000000000000
+; CHECK-NEXT: mov v0.s[3], v2.s[0]
+; CHECK-NEXT: movi d2, #0000000000000000
+; CHECK-NEXT: .LBB6_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov v1.s[1], v5.s[0]
+; CHECK-NEXT: fadd v2.4s, v2.4s, v6.4s
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: mov v1.s[2], v4.s[0]
+; CHECK-NEXT: mov v1.s[3], v3.s[0]
+; CHECK-NEXT: fadd v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: mov s3, v1.s[3]
+; CHECK-NEXT: mov s4, v1.s[2]
+; CHECK-NEXT: mov s5, v1.s[1]
+; CHECK-NEXT: b.ne .LBB6_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v1.s[1], v5.s[0]
+; CHECK-NEXT: str s2, [x2, #16]
+; CHECK-NEXT: mov v1.s[2], v4.s[0]
+; CHECK-NEXT: mov v1.s[3], v3.s[0]
+; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <5 x float>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <5 x float> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = fadd <5 x float> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <5 x float> %acc.next, ptr %out, align 8
+ ret void
+}
+
+define void @v6f32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6f32_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q0, q2, [x0]
+; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: movi d6, #0000000000000000
+; CHECK-NEXT: mov s1, v0.s[1]
+; CHECK-NEXT: mov s3, v0.s[3]
+; CHECK-NEXT: mov s4, v0.s[2]
+; CHECK-NEXT: mov s5, v2.s[1]
+; CHECK-NEXT: mov v0.s[1], v1.s[0]
+; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: mov v2.s[1], v5.s[0]
+; CHECK-NEXT: movi d5, #0000000000000000
+; CHECK-NEXT: mov v0.s[2], v4.s[0]
+; CHECK-NEXT: movi d4, #0000000000000000
+; CHECK-NEXT: mov v0.s[3], v3.s[0]
+; CHECK-NEXT: movi d3, #0000000000000000
+; CHECK-NEXT: .LBB7_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov v1.s[1], v7.s[0]
+; CHECK-NEXT: mov v3.s[1], v4.s[0]
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: mov v1.s[2], v6.s[0]
+; CHECK-NEXT: fadd v3.4s, v3.4s, v2.4s
+; CHECK-NEXT: mov v1.s[3], v5.s[0]
+; CHECK-NEXT: mov s4, v3.s[1]
+; CHECK-NEXT: fadd v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: mov s5, v1.s[3]
+; CHECK-NEXT: mov s6, v1.s[2]
+; CHECK-NEXT: mov s7, v1.s[1]
+; CHECK-NEXT: b.ne .LBB7_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v1.s[1], v7.s[0]
+; CHECK-NEXT: mov v3.s[1], v4.s[0]
+; CHECK-NEXT: mov v1.s[2], v6.s[0]
+; CHECK-NEXT: str d3, [x2, #16]
+; CHECK-NEXT: mov v1.s[3], v5.s[0]
+; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <6 x float>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <6 x float> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = fadd <6 x float> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <6 x float> %acc.next, ptr %out, align 8
+ ret void
+}
+
+define <6 x float> @v6f32_cc(<6 x float> %c.val, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6f32_cc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1
+; CHECK-NEXT: // kill: def $s2 killed $s2 def $q2
+; CHECK-NEXT: // kill: def $s5 killed $s5 def $q5
+; CHECK-NEXT: // kill: def $s4 killed $s4 def $q4
+; CHECK-NEXT: // kill: def $s3 killed $s3 def $q3
+; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: movi d6, #0000000000000000
+; CHECK-NEXT: mov v0.s[1], v1.s[0]
+; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: mov v4.s[1], v5.s[0]
+; CHECK-NEXT: movi d5, #0000000000000000
+; CHECK-NEXT: mov v0.s[2], v2.s[0]
+; CHECK-NEXT: movi d2, #0000000000000000
+; CHECK-NEXT: mov v0.s[3], v3.s[0]
+; CHECK-NEXT: movi d3, #0000000000000000
+; CHECK-NEXT: .LBB8_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov v7.s[1], v1.s[0]
+; CHECK-NEXT: mov v6.s[1], v5.s[0]
+; CHECK-NEXT: subs x2, x2, #1
+; CHECK-NEXT: mov v7.s[2], v2.s[0]
+; CHECK-NEXT: fadd v6.4s, v6.4s, v4.4s
+; CHECK-NEXT: mov v7.s[3], v3.s[0]
+; CHECK-NEXT: mov s5, v6.s[1]
+; CHECK-NEXT: fadd v7.4s, v7.4s, v0.4s
+; CHECK-NEXT: mov s3, v7.s[3]
+; CHECK-NEXT: mov s2, v7.s[2]
+; CHECK-NEXT: mov s1, v7.s[1]
+; CHECK-NEXT: b.ne .LBB8_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: fmov s0, s7
+; CHECK-NEXT: fmov s4, s6
+; CHECK-NEXT: // kill: def $s1 killed $s1 killed $q1
+; CHECK-NEXT: // kill: def $s2 killed $s2 killed $q2
+; CHECK-NEXT: // kill: def $s3 killed $s3 killed $q3
+; CHECK-NEXT: // kill: def $s5 killed $s5 killed $q5
+; CHECK-NEXT: ret
+entry:
+ br label %loop
+
+loop:
+ %acc = phi <6 x float> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = fadd <6 x float> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ ret <6 x float> %acc.next
+}
+
+define void @v6f64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6f64_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q2, q1, [x0, #16]
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: ldr q3, [x0]
+; CHECK-NEXT: movi d16, #0000000000000000
+; CHECK-NEXT: movi d4, #0000000000000000
+; CHECK-NEXT: mov d5, v1.d[1]
+; CHECK-NEXT: mov d6, v2.d[1]
+; CHECK-NEXT: mov d7, v3.d[1]
+; CHECK-NEXT: mov v3.d[1], v7.d[0]
+; CHECK-NEXT: mov v2.d[1], v6.d[0]
+; CHECK-NEXT: mov v1.d[1], v5.d[0]
+; CHECK-NEXT: movi d6, #0000000000000000
+; CHECK-NEXT: movi d5, #0000000000000000
+; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: .LBB9_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov v0.d[1], v16.d[0]
+; CHECK-NEXT: mov v4.d[1], v6.d[0]
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: mov v5.d[1], v7.d[0]
+; CHECK-NEXT: fadd v0.2d, v0.2d, v3.2d
+; CHECK-NEXT: fadd v4.2d, v4.2d, v2.2d
+; CHECK-NEXT: fadd v5.2d, v5.2d, v1.2d
+; CHECK-NEXT: mov d16, v0.d[1]
+; CHECK-NEXT: mov d6, v4.d[1]
+; CHECK-NEXT: mov d7, v5.d[1]
+; CHECK-NEXT: b.ne .LBB9_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v0.d[1], v16.d[0]
+; CHECK-NEXT: mov v4.d[1], v7.d[0]
+; CHECK-NEXT: mov v6.d[1], v5.d[0]
+; CHECK-NEXT: stp q0, q4, [x2]
+; CHECK-NEXT: str q6, [x2, #32]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <6 x double>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <6 x double> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = fadd fast <6 x double> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ %lanes01 = shufflevector <6 x double> %acc.next, <6 x double> poison, <2 x i32> <i32 0, i32 1>
+ %lanes25 = shufflevector <6 x double> %acc.next, <6 x double> poison, <2 x i32> <i32 2, i32 5>
+ %lane3 = extractelement <6 x double> %acc.next, i64 3
+ %lane4 = extractelement <6 x double> %acc.next, i64 4
+ store <2 x double> %lanes01, ptr %out, align 8
+ %p2 = getelementptr double, ptr %out, i64 2
+ store <2 x double> %lanes25, ptr %p2, align 8
+ %p4 = getelementptr double, ptr %out, i64 4
+ store double %lane3, ptr %p4, align 8
+ %p5 = getelementptr double, ptr %out, i64 5
+ store double %lane4, ptr %p5, align 8
+ ret void
+}
+
+define <6 x double> @v6f64_cc(<6 x double> %c.val, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v6f64_cc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fmov d16, d0
+; CHECK-NEXT: // kill: def $d5 killed $d5 def $q5
+; CHECK-NEXT: // kill: def $d4 killed $d4 def $q4
+; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-NEXT: mov v4.d[1], v5.d[0]
+; CHECK-NEXT: movi d3, #0000000000000000
+; CHECK-NEXT: movi d6, #0000000000000000
+; CHECK-NEXT: movi d5, #0000000000000000
+; CHECK-NEXT: mov v16.d[1], v1.d[0]
+; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: .LBB10_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-NEXT: mov v7.d[1], v3.d[0]
+; CHECK-NEXT: subs x2, x2, #1
+; CHECK-NEXT: mov v6.d[1], v5.d[0]
+; CHECK-NEXT: fadd v0.2d, v0.2d, v16.2d
+; CHECK-NEXT: fadd v7.2d, v7.2d, v2.2d
+; CHECK-NEXT: fadd v6.2d, v6.2d, v4.2d
+; CHECK-NEXT: mov d1, v0.d[1]
+; CHECK-NEXT: mov d3, v7.d[1]
+; CHECK-NEXT: mov d5, v6.d[1]
+; CHECK-NEXT: b.ne .LBB10_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: fmov d2, d7
+; CHECK-NEXT: fmov d4, d6
+; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT: // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT: // kill: def $d3 killed $d3 killed $q3
+; CHECK-NEXT: // kill: def $d5 killed $d5 killed $q5
+; CHECK-NEXT: ret
+entry:
+ br label %loop
+
+loop:
+ %acc = phi <6 x double> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = fadd fast <6 x double> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ ret <6 x double> %acc.next
+}
+
+define void @v12f64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
+; CHECK-LABEL: v12f64_internal:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ldp q2, q1, [x0]
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: ldp q4, q3, [x0, #32]
+; CHECK-NEXT: movi d25, #0000000000000000
+; CHECK-NEXT: ldp q5, q6, [x0, #64]
+; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: mov d16, v1.d[1]
+; CHECK-NEXT: mov d17, v2.d[1]
+; CHECK-NEXT: movi d24, #0000000000000000
+; CHECK-NEXT: mov d19, v3.d[1]
+; CHECK-NEXT: mov d20, v4.d[1]
+; CHECK-NEXT: movi d23, #0000000000000000
+; CHECK-NEXT: mov d18, v5.d[1]
+; CHECK-NEXT: mov d21, v6.d[1]
+; CHECK-NEXT: movi d22, #0000000000000000
+; CHECK-NEXT: mov v2.d[1], v17.d[0]
+; CHECK-NEXT: mov v1.d[1], v16.d[0]
+; CHECK-NEXT: movi d16, #0000000000000000
+; CHECK-NEXT: mov v4.d[1], v20.d[0]
+; CHECK-NEXT: mov v3.d[1], v19.d[0]
+; CHECK-NEXT: movi d17, #0000000000000000
+; CHECK-NEXT: mov v5.d[1], v18.d[0]
+; CHECK-NEXT: mov v6.d[1], v21.d[0]
+; CHECK-NEXT: movi d18, #0000000000000000
+; CHECK-NEXT: movi d21, #0000000000000000
+; CHECK-NEXT: movi d19, #0000000000000000
+; CHECK-NEXT: movi d20, #0000000000000000
+; CHECK-NEXT: .LBB11_1: // %loop
+; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+; CHECK-NEXT: mov v0.d[1], v25.d[0]
+; CHECK-NEXT: mov v7.d[1], v24.d[0]
+; CHECK-NEXT: subs x3, x3, #1
+; CHECK-NEXT: mov v16.d[1], v23.d[0]
+; CHECK-NEXT: mov v17.d[1], v22.d[0]
+; CHECK-NEXT: mov v18.d[1], v21.d[0]
+; CHECK-NEXT: mov v19.d[1], v20.d[0]
+; CHECK-NEXT: fadd v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: fadd v7.2d, v7.2d, v1.2d
+; CHECK-NEXT: fadd v16.2d, v16.2d, v4.2d
+; CHECK-NEXT: fadd v17.2d, v17.2d, v3.2d
+; CHECK-NEXT: fadd v18.2d, v18.2d, v5.2d
+; CHECK-NEXT: fadd v19.2d, v19.2d, v6.2d
+; CHECK-NEXT: mov d25, v0.d[1]
+; CHECK-NEXT: mov d24, v7.d[1]
+; CHECK-NEXT: mov d23, v16.d[1]
+; CHECK-NEXT: mov d22, v17.d[1]
+; CHECK-NEXT: mov d21, v18.d[1]
+; CHECK-NEXT: mov d20, v19.d[1]
+; CHECK-NEXT: b.ne .LBB11_1
+; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov v0.d[1], v25.d[0]
+; CHECK-NEXT: mov v7.d[1], v24.d[0]
+; CHECK-NEXT: mov v16.d[1], v23.d[0]
+; CHECK-NEXT: mov v17.d[1], v22.d[0]
+; CHECK-NEXT: mov v18.d[1], v21.d[0]
+; CHECK-NEXT: mov v19.d[1], v20.d[0]
+; CHECK-NEXT: stp q0, q7, [x2]
+; CHECK-NEXT: stp q16, q17, [x2, #32]
+; CHECK-NEXT: stp q18, q19, [x2, #64]
+; CHECK-NEXT: ret
+entry:
+ %c.val = load <12 x double>, ptr %c
+ br label %loop
+
+loop:
+ %acc = phi <12 x double> [ zeroinitializer, %entry ], [ %acc.next, %loop ]
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %i.next = add nuw i64 %i, 1
+ %acc.next = fadd fast <12 x double> %acc, %c.val
+ %cond = icmp eq i64 %i.next, %n
+ br i1 %cond, label %exit, label %loop
+
+exit:
+ store <12 x double> %acc.next, ptr %out
+ ret void
+}
>From fd2558ad3aaa757a441d1ee202068715dcc433c6 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Tue, 4 Aug 2026 14:22:50 +0000
Subject: [PATCH 2/7] [CodeGen][AArch64] Use vector parts for internal
non-power-of-two vectors
Decompose fixed-length, non-power-of-two vectors into the largest legal
vector type that exactly divides their element count. Apply this only to
internal register values.
Continue using the existing scalar breakdown for arguments and returns,
preserving the existing calling convention behaviour.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 47 +-
llvm/lib/CodeGen/TargetLoweringBase.cpp | 61 ++-
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 5 +
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 9 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 3 +-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 9 +-
llvm/lib/Target/RISCV/RISCVISelLowering.h | 6 +-
llvm/lib/Target/SPIRV/SPIRVISelLowering.h | 6 +-
llvm/lib/Target/SystemZ/SystemZISelLowering.h | 9 +-
.../CodeGen/AArch64/non-pow2-fixed-vectors.ll | 401 ++++++------------
10 files changed, 246 insertions(+), 310 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 34b1857fd4adc..2b66f46aedfeb 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1226,8 +1226,14 @@ class LLVM_ABI TargetLoweringBase {
/// before they are promoted/expanded.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT,
EVT &IntermediateVT,
- unsigned &NumIntermediates,
- MVT &RegisterVT) const;
+ unsigned &NumIntermediates, MVT &RegisterVT,
+ bool ForCallingConv = false) const;
+
+ /// Return true if fixed-length, non-power-of-two vectors should be broken
+ /// down into legal vector parts instead of scalars for internal values.
+ virtual bool preferVectorizedNonPowerOfTwoTypeBreakdown() const {
+ return false;
+ }
/// Certain targets such as MIPS require that some types such as vectors are
/// always broken down into scalars in some contexts. This occurs even if the
@@ -1236,7 +1242,7 @@ class LLVM_ABI TargetLoweringBase {
LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
unsigned &NumIntermediates, MVT &RegisterVT) const {
return getVectorTypeBreakdown(Context, VT, IntermediateVT, NumIntermediates,
- RegisterVT);
+ RegisterVT, /*ForCallingConv=*/true);
}
struct IntrinsicInfo {
@@ -1851,19 +1857,23 @@ class LLVM_ABI TargetLoweringBase {
}
/// Return the type of registers that this ValueType will eventually require.
- MVT getRegisterType(LLVMContext &Context, EVT VT) const {
- if (VT.isSimple())
+ MVT getRegisterType(LLVMContext &Context, EVT VT,
+ bool ForCallingConv = false) const {
+ if (VT.isSimple() && (!ForCallingConv || !VT.isFixedLengthVector() ||
+ isPowerOf2_32(VT.getVectorNumElements()) ||
+ !preferVectorizedNonPowerOfTwoTypeBreakdown()))
return getRegisterType(VT.getSimpleVT());
if (VT.isVector()) {
EVT VT1;
MVT RegisterVT;
unsigned NumIntermediates;
- (void)getVectorTypeBreakdown(Context, VT, VT1,
- NumIntermediates, RegisterVT);
+ (void)getVectorTypeBreakdown(Context, VT, VT1, NumIntermediates,
+ RegisterVT, ForCallingConv);
return RegisterVT;
}
if (VT.isInteger()) {
- return getRegisterType(Context, getTypeToTransformTo(Context, VT));
+ return getRegisterType(Context, getTypeToTransformTo(Context, VT),
+ ForCallingConv);
}
llvm_unreachable("Unsupported extended type!");
}
@@ -1879,10 +1889,12 @@ class LLVM_ABI TargetLoweringBase {
///
/// RegisterVT may be passed as a way to override the default settings, for
/// instance with i128 inline assembly operands on SystemZ.
- virtual unsigned
- getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt) const {
- if (VT.isSimple()) {
+ virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt,
+ bool ForCallingConv = false) const {
+ if (VT.isSimple() && (!ForCallingConv || !VT.isFixedLengthVector() ||
+ isPowerOf2_32(VT.getVectorNumElements()) ||
+ !preferVectorizedNonPowerOfTwoTypeBreakdown())) {
assert((unsigned)VT.getSimpleVT().SimpleTy <
std::size(NumRegistersForVT));
return NumRegistersForVT[VT.getSimpleVT().SimpleTy];
@@ -1891,11 +1903,13 @@ class LLVM_ABI TargetLoweringBase {
EVT VT1;
MVT VT2;
unsigned NumIntermediates;
- return getVectorTypeBreakdown(Context, VT, VT1, NumIntermediates, VT2);
+ return getVectorTypeBreakdown(Context, VT, VT1, NumIntermediates, VT2,
+ ForCallingConv);
}
if (VT.isInteger()) {
unsigned BitWidth = VT.getSizeInBits();
- unsigned RegWidth = getRegisterType(Context, VT).getSizeInBits();
+ unsigned RegWidth =
+ getRegisterType(Context, VT, ForCallingConv).getSizeInBits();
return (BitWidth + RegWidth - 1) / RegWidth;
}
llvm_unreachable("Unsupported extended type!");
@@ -1906,7 +1920,7 @@ class LLVM_ABI TargetLoweringBase {
/// For MIPS all vector types must be passed through the integer register set.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context,
CallingConv::ID CC, EVT VT) const {
- return getRegisterType(Context, VT);
+ return getRegisterType(Context, VT, /*ForCallingConv=*/true);
}
/// Certain targets require unusual breakdowns of certain types. For MIPS,
@@ -1915,7 +1929,8 @@ class LLVM_ABI TargetLoweringBase {
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context,
CallingConv::ID CC,
EVT VT) const {
- return getNumRegisters(Context, VT);
+ return getNumRegisters(Context, VT, /*RegisterVT=*/std::nullopt,
+ /*ForCallingConv=*/true);
}
/// Certain targets have context sensitive alignment requirements, where one
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index b3c396af129fa..a887156427ebe 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1252,11 +1252,33 @@ static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
llvm_unreachable(
"Splitting or widening of non-power-of-2 MVTs is not implemented.");
- // FIXME: We don't support non-power-of-2-sized vectors for now.
+ // FIXME: We don't generically support non-power-of-2-sized vectors for now.
// Ideally we could break down into LHS/RHS like LegalizeDAG does.
if (!isPowerOf2_32(EC.getKnownMinValue())) {
- // Split EC to unit size (scalable property is preserved).
- NumVectorRegs = EC.getKnownMinValue();
+ assert(VT.isFixedLengthVector() && "Expected a fixed-length vector VT");
+ unsigned NumElts = EC.getKnownMinValue();
+
+ // Find the largest legal vector type that exactly divides a
+ // non-power-of-two vector.
+ if (TLI->preferVectorizedNonPowerOfTwoTypeBreakdown()) {
+ for (unsigned PartElts = llvm::bit_floor(NumElts); PartElts > 1;
+ PartElts >>= 1) {
+ if (NumElts % PartElts != 0)
+ continue;
+
+ MVT PartVT = MVT::getVectorVT(EltTy, ElementCount::getFixed(PartElts));
+ if (PartVT == MVT() || !TLI->isTypeLegal(PartVT))
+ continue;
+
+ IntermediateVT = PartVT;
+ NumIntermediates = NumElts / PartElts;
+ RegisterVT = PartVT;
+ return NumIntermediates;
+ }
+ }
+
+ // Fall back to scalars if there is no exact legal vector decomposition.
+ NumVectorRegs = NumElts;
EC = ElementCount::getFixed(1);
}
@@ -1691,7 +1713,8 @@ EVT TargetLoweringBase::getSetCCResultType(const DataLayout &DL, LLVMContext &,
unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,
EVT VT, EVT &IntermediateVT,
unsigned &NumIntermediates,
- MVT &RegisterVT) const {
+ MVT &RegisterVT,
+ bool ForCallingConv) const {
ElementCount EltCnt = VT.getVectorElementCount();
// If there is a wider vector type with the same element type as this one,
@@ -1740,10 +1763,34 @@ unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,
return NumIntermediates;
}
- // FIXME: We don't support non-power-of-2-sized vectors for now. Ideally
- // we could break down into LHS/RHS like LegalizeDAG does.
+ // FIXME: We don't generically support non-power-of-2-sized vectors for now.
+ // Ideally we could break down into LHS/RHS like LegalizeDAG does.
if (!isPowerOf2_32(EltCnt.getKnownMinValue())) {
- NumVectorRegs = EltCnt.getKnownMinValue();
+ assert(VT.isFixedLengthVector() && "Expected a fixed-length vector VT");
+ unsigned NumElts = EltCnt.getKnownMinValue();
+
+ // Find the largest legal vector type that exactly divides a
+ // non-power-of-two vector.
+ if (!ForCallingConv && preferVectorizedNonPowerOfTwoTypeBreakdown()) {
+ for (unsigned PartElts = llvm::bit_floor(NumElts); PartElts > 1;
+ PartElts >>= 1) {
+ if (NumElts % PartElts != 0)
+ continue;
+
+ EVT PartVT =
+ EVT::getVectorVT(Context, EltTy, ElementCount::getFixed(PartElts));
+ if (!isTypeLegal(PartVT))
+ continue;
+
+ IntermediateVT = PartVT;
+ NumIntermediates = NumElts / PartElts;
+ RegisterVT = PartVT.getSimpleVT();
+ return NumIntermediates;
+ }
+ }
+
+ // Fall back to scalars if there is no exact legal vector decomposition.
+ NumVectorRegs = NumElts;
EltCnt = ElementCount::getFixed(1);
}
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index b641c68770cb9..9740f38893abc 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -20,6 +20,7 @@
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Instruction.h"
+#include "llvm/Support/MathExtras.h"
namespace llvm {
@@ -602,6 +603,10 @@ class AArch64TargetLowering : public TargetLowering {
unsigned &NumIntermediates,
MVT &RegisterVT) const override;
+ bool preferVectorizedNonPowerOfTwoTypeBreakdown() const override {
+ return true;
+ }
+
/// True if stack clash protection is enabled for this functions.
bool hasInlineStackProbe(const MachineFunction &MF) const override;
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index e788b0e44041f..7ba25e63f6854 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -4025,12 +4025,13 @@ SDValue NVPTXTargetLowering::LowerCopyToReg_128(SDValue Op,
return DAG.getNode(ISD::CopyToReg, DL, ResultsType, NewOps);
}
-unsigned NVPTXTargetLowering::getNumRegisters(
- LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt) const {
+unsigned NVPTXTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT,
+ bool ForCallingConv) const {
if (VT == MVT::i128 && RegisterVT == MVT::i128)
return 1;
- return TargetLoweringBase::getNumRegisters(Context, VT, RegisterVT);
+ return TargetLoweringBase::getNumRegisters(Context, VT, RegisterVT,
+ ForCallingConv);
}
bool NVPTXTargetLowering::splitValueIntoRegisterParts(
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 42741ba98b438..c7238444ae61c 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -224,7 +224,8 @@ class NVPTXTargetLowering : public TargetLowering {
SDValue LowerCopyToReg_128(SDValue Op, SelectionDAG &DAG) const;
unsigned getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT) const override;
+ std::optional<MVT> RegisterVT,
+ bool ForCallingConv = false) const override;
bool
splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
SDValue *Parts, unsigned NumParts, MVT PartVT,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index 44359a9e6155f..ff13d62172e87 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -2934,15 +2934,16 @@ MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
}
-unsigned
-RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT) const {
+unsigned RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT,
+ bool ForCallingConv) const {
// Pair inline assembly operand
if (VT == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
*RegisterVT == MVT::Untyped)
return 1;
- return TargetLowering::getNumRegisters(Context, VT, RegisterVT);
+ return TargetLowering::getNumRegisters(Context, VT, RegisterVT,
+ ForCallingConv);
}
unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index b9bea162d6d4c..56d62d83272dc 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -82,9 +82,9 @@ class RISCVTargetLowering : public TargetLowering {
EVT VT) const override;
/// Return the number of registers for a given MVT, for inline assembly
- unsigned
- getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt) const override;
+ unsigned getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt,
+ bool ForCallingConv = false) const override;
/// Return the number of registers for a given MVT, ensuring vectors are
/// treated as a series of gpr sized integers.
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
index ff843649fd6d3..a4e91b3f06d64 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
@@ -53,9 +53,9 @@ class SPIRVTargetLowering : public TargetLowering {
std::pair<unsigned, const TargetRegisterClass *>
getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
StringRef Constraint, MVT VT) const override;
- unsigned
- getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt) const override {
+ unsigned getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt,
+ bool ForCallingConv = false) const override {
return 1;
}
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index f01f7dcc6ab1d..6aa72bf89213d 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -75,13 +75,14 @@ class SystemZTargetLowering : public TargetLowering {
return TypeWidenVector;
return TargetLoweringBase::getPreferredVectorAction(VT);
}
- unsigned
- getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT) const override {
+ unsigned getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT,
+ bool ForCallingConv = false) const override {
// i128 inline assembly operand.
if (VT == MVT::i128 && RegisterVT && *RegisterVT == MVT::Untyped)
return 1;
- return TargetLowering::getNumRegisters(Context, VT);
+ return TargetLowering::getNumRegisters(
+ Context, VT, /*RegisterVT=*/std::nullopt, ForCallingConv);
}
unsigned
getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC,
diff --git a/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
index 047a377710ecf..ea945cdbcec43 100644
--- a/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
+++ b/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
@@ -60,46 +60,23 @@ exit:
define void @v6i32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6i32_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp w11, w12, [x0]
-; CHECK-NEXT: mov w4, wzr
-; CHECK-NEXT: mov w10, wzr
-; CHECK-NEXT: mov w9, wzr
-; CHECK-NEXT: mov w8, wzr
-; CHECK-NEXT: mov w1, wzr
-; CHECK-NEXT: fmov s0, w11
-; CHECK-NEXT: ldp w11, w13, [x0, #8]
-; CHECK-NEXT: mov v0.s[1], w12
-; CHECK-NEXT: mov v0.s[2], w11
-; CHECK-NEXT: ldp w11, w12, [x0, #16]
-; CHECK-NEXT: fmov s1, w11
-; CHECK-NEXT: mov v0.s[3], w13
-; CHECK-NEXT: mov w11, wzr
-; CHECK-NEXT: mov v1.s[1], w12
+; CHECK-NEXT: ldr q1, [x0]
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: ldr d5, [x0, #16]
+; CHECK-NEXT: mov d3, v1.d[1]
; CHECK-NEXT: .LBB1_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: fmov s2, w4
-; CHECK-NEXT: fmov s4, w1
+; CHECK-NEXT: add v0.2s, v0.2s, v1.2s
+; CHECK-NEXT: add v4.2s, v4.2s, v3.2s
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: mov v2.s[1], w10
-; CHECK-NEXT: mov v4.s[1], w11
-; CHECK-NEXT: mov v2.s[2], w9
-; CHECK-NEXT: mov v2.s[3], w8
-; CHECK-NEXT: add v3.4s, v2.4s, v0.4s
-; CHECK-NEXT: add v2.4s, v4.4s, v1.4s
-; CHECK-NEXT: mov w8, v3.s[3]
-; CHECK-NEXT: mov w9, v3.s[2]
-; CHECK-NEXT: mov w10, v3.s[1]
-; CHECK-NEXT: mov w11, v2.s[1]
-; CHECK-NEXT: fmov w4, s3
-; CHECK-NEXT: fmov w1, s2
+; CHECK-NEXT: add v2.2s, v2.2s, v5.2s
; CHECK-NEXT: b.ne .LBB1_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v3.s[1], w10
-; CHECK-NEXT: mov v2.s[1], w11
-; CHECK-NEXT: mov v3.s[2], w9
+; CHECK-NEXT: mov v0.d[1], v4.d[0]
; CHECK-NEXT: str d2, [x2, #16]
-; CHECK-NEXT: mov v3.s[3], w8
-; CHECK-NEXT: str q3, [x2]
+; CHECK-NEXT: str q0, [x2]
; CHECK-NEXT: ret
entry:
%c.val = load <6 x i32>, ptr %c
@@ -121,43 +98,30 @@ exit:
define <6 x i32> @v6i32_cc(<6 x i32> %c.val, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6i32_cc:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov w10, w0
-; CHECK-NEXT: mov w11, w1
-; CHECK-NEXT: mov w12, w2
-; CHECK-NEXT: fmov s0, w10
-; CHECK-NEXT: mov w8, w4
-; CHECK-NEXT: mov w9, w3
-; CHECK-NEXT: fmov s1, w8
+; CHECK-NEXT: fmov s0, w4
+; CHECK-NEXT: fmov s1, w2
; CHECK-NEXT: ldr x8, [sp]
-; CHECK-NEXT: mov w0, wzr
-; CHECK-NEXT: mov w1, wzr
-; CHECK-NEXT: mov w2, wzr
-; CHECK-NEXT: mov w3, wzr
-; CHECK-NEXT: mov v0.s[1], w11
-; CHECK-NEXT: mov w4, wzr
-; CHECK-NEXT: mov v1.s[1], w5
-; CHECK-NEXT: mov w5, wzr
-; CHECK-NEXT: mov v0.s[2], w12
-; CHECK-NEXT: mov v0.s[3], w9
+; CHECK-NEXT: fmov s3, w0
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: mov v0.s[1], w5
+; CHECK-NEXT: mov v1.s[1], w3
+; CHECK-NEXT: mov v3.s[1], w1
; CHECK-NEXT: .LBB2_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: fmov s2, w0
-; CHECK-NEXT: fmov s3, w4
+; CHECK-NEXT: add v2.2s, v2.2s, v3.2s
+; CHECK-NEXT: add v4.2s, v4.2s, v1.2s
; CHECK-NEXT: subs x8, x8, #1
-; CHECK-NEXT: mov v2.s[1], w1
-; CHECK-NEXT: mov v3.s[1], w5
-; CHECK-NEXT: mov v2.s[2], w2
-; CHECK-NEXT: add v3.4s, v3.4s, v1.4s
-; CHECK-NEXT: mov w5, v3.s[1]
-; CHECK-NEXT: fmov w4, s3
-; CHECK-NEXT: mov v2.s[3], w3
-; CHECK-NEXT: add v2.4s, v2.4s, v0.4s
-; CHECK-NEXT: mov w3, v2.s[3]
-; CHECK-NEXT: mov w2, v2.s[2]
-; CHECK-NEXT: mov w1, v2.s[1]
-; CHECK-NEXT: fmov w0, s2
+; CHECK-NEXT: add v5.2s, v5.2s, v0.2s
; CHECK-NEXT: b.ne .LBB2_1
; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov w1, v2.s[1]
+; CHECK-NEXT: mov w3, v4.s[1]
+; CHECK-NEXT: mov w5, v5.s[1]
+; CHECK-NEXT: fmov w0, s2
+; CHECK-NEXT: fmov w2, s4
+; CHECK-NEXT: fmov w4, s5
; CHECK-NEXT: ret
entry:
br label %loop
@@ -221,33 +185,21 @@ exit:
define void @v6i64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6i64_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: ldp q4, q1, [x0, #16]
-; CHECK-NEXT: ldr q6, [x0]
-; CHECK-NEXT: mov d7, v1.d[1]
-; CHECK-NEXT: mov d16, v4.d[1]
-; CHECK-NEXT: mov d18, v6.d[1]
-; CHECK-NEXT: fmov d19, d0
-; CHECK-NEXT: fmov d2, d0
-; CHECK-NEXT: fmov d17, d0
-; CHECK-NEXT: fmov d3, d0
-; CHECK-NEXT: fmov d5, d0
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldr q5, [x0]
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: ldp q4, q3, [x0, #16]
; CHECK-NEXT: .LBB4_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add d0, d0, d6
-; CHECK-NEXT: add d19, d19, d18
+; CHECK-NEXT: add v1.2d, v1.2d, v5.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v4.2d
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: add d2, d2, d4
-; CHECK-NEXT: add d17, d17, d16
-; CHECK-NEXT: add d3, d3, d1
-; CHECK-NEXT: add d5, d5, d7
+; CHECK-NEXT: add v2.2d, v2.2d, v3.2d
; CHECK-NEXT: b.ne .LBB4_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v0.d[1], v19.d[0]
-; CHECK-NEXT: mov v2.d[1], v17.d[0]
-; CHECK-NEXT: mov v3.d[1], v5.d[0]
-; CHECK-NEXT: stp q0, q2, [x2]
-; CHECK-NEXT: str q3, [x2, #32]
+; CHECK-NEXT: stp q1, q0, [x2]
+; CHECK-NEXT: str q2, [x2, #32]
; CHECK-NEXT: ret
entry:
%c.val = load <6 x i64>, ptr %c
@@ -269,29 +221,32 @@ exit:
define <6 x i64> @v6i64_cc(<6 x i64> %c.val, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6i64_cc:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi d6, #0000000000000000
-; CHECK-NEXT: fmov d7, d6
-; CHECK-NEXT: fmov d16, d6
-; CHECK-NEXT: fmov d17, d6
-; CHECK-NEXT: fmov d18, d6
-; CHECK-NEXT: fmov d19, d6
+; CHECK-NEXT: // kill: def $d4 killed $d4 def $q4
+; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: // kill: def $d5 killed $d5 def $q5
+; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT: movi v6.2d, #0000000000000000
+; CHECK-NEXT: movi v7.2d, #0000000000000000
+; CHECK-NEXT: movi v16.2d, #0000000000000000
+; CHECK-NEXT: mov v4.d[1], v5.d[0]
+; CHECK-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-NEXT: mov v0.d[1], v1.d[0]
; CHECK-NEXT: .LBB5_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add d6, d6, d0
-; CHECK-NEXT: add d7, d7, d1
+; CHECK-NEXT: add v6.2d, v6.2d, v0.2d
+; CHECK-NEXT: add v7.2d, v7.2d, v2.2d
; CHECK-NEXT: subs x2, x2, #1
-; CHECK-NEXT: add d16, d16, d2
-; CHECK-NEXT: add d17, d17, d3
-; CHECK-NEXT: add d18, d18, d4
-; CHECK-NEXT: add d19, d19, d5
+; CHECK-NEXT: add v16.2d, v16.2d, v4.2d
; CHECK-NEXT: b.ne .LBB5_1
; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov d1, v6.d[1]
+; CHECK-NEXT: mov d3, v7.d[1]
+; CHECK-NEXT: mov d5, v16.d[1]
; CHECK-NEXT: fmov d0, d6
-; CHECK-NEXT: fmov d1, d7
-; CHECK-NEXT: fmov d2, d16
-; CHECK-NEXT: fmov d3, d17
-; CHECK-NEXT: fmov d4, d18
-; CHECK-NEXT: fmov d5, d19
+; CHECK-NEXT: fmov d2, d7
+; CHECK-NEXT: fmov d4, d16
; CHECK-NEXT: ret
entry:
br label %loop
@@ -363,42 +318,23 @@ exit:
define void @v6f32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6f32_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp q0, q2, [x0]
-; CHECK-NEXT: movi d7, #0000000000000000
-; CHECK-NEXT: movi d6, #0000000000000000
-; CHECK-NEXT: mov s1, v0.s[1]
-; CHECK-NEXT: mov s3, v0.s[3]
-; CHECK-NEXT: mov s4, v0.s[2]
-; CHECK-NEXT: mov s5, v2.s[1]
-; CHECK-NEXT: mov v0.s[1], v1.s[0]
-; CHECK-NEXT: movi d1, #0000000000000000
-; CHECK-NEXT: mov v2.s[1], v5.s[0]
-; CHECK-NEXT: movi d5, #0000000000000000
-; CHECK-NEXT: mov v0.s[2], v4.s[0]
+; CHECK-NEXT: ldr q1, [x0]
+; CHECK-NEXT: movi d0, #0000000000000000
; CHECK-NEXT: movi d4, #0000000000000000
-; CHECK-NEXT: mov v0.s[3], v3.s[0]
-; CHECK-NEXT: movi d3, #0000000000000000
+; CHECK-NEXT: movi d2, #0000000000000000
+; CHECK-NEXT: ldr d5, [x0, #16]
+; CHECK-NEXT: mov d3, v1.d[1]
; CHECK-NEXT: .LBB7_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: mov v1.s[1], v7.s[0]
-; CHECK-NEXT: mov v3.s[1], v4.s[0]
+; CHECK-NEXT: fadd v0.2s, v0.2s, v1.2s
+; CHECK-NEXT: fadd v4.2s, v4.2s, v3.2s
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: mov v1.s[2], v6.s[0]
-; CHECK-NEXT: fadd v3.4s, v3.4s, v2.4s
-; CHECK-NEXT: mov v1.s[3], v5.s[0]
-; CHECK-NEXT: mov s4, v3.s[1]
-; CHECK-NEXT: fadd v1.4s, v1.4s, v0.4s
-; CHECK-NEXT: mov s5, v1.s[3]
-; CHECK-NEXT: mov s6, v1.s[2]
-; CHECK-NEXT: mov s7, v1.s[1]
+; CHECK-NEXT: fadd v2.2s, v2.2s, v5.2s
; CHECK-NEXT: b.ne .LBB7_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v1.s[1], v7.s[0]
-; CHECK-NEXT: mov v3.s[1], v4.s[0]
-; CHECK-NEXT: mov v1.s[2], v6.s[0]
-; CHECK-NEXT: str d3, [x2, #16]
-; CHECK-NEXT: mov v1.s[3], v5.s[0]
-; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: mov v0.d[1], v4.d[0]
+; CHECK-NEXT: str d2, [x2, #16]
+; CHECK-NEXT: str q0, [x2]
; CHECK-NEXT: ret
entry:
%c.val = load <6 x float>, ptr %c
@@ -420,43 +356,32 @@ exit:
define <6 x float> @v6f32_cc(<6 x float> %c.val, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6f32_cc:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
-; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1
+; CHECK-NEXT: // kill: def $s4 killed $s4 def $q4
; CHECK-NEXT: // kill: def $s2 killed $s2 def $q2
+; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
; CHECK-NEXT: // kill: def $s5 killed $s5 def $q5
-; CHECK-NEXT: // kill: def $s4 killed $s4 def $q4
; CHECK-NEXT: // kill: def $s3 killed $s3 def $q3
-; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1
; CHECK-NEXT: movi d6, #0000000000000000
-; CHECK-NEXT: mov v0.s[1], v1.s[0]
-; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: movi d16, #0000000000000000
; CHECK-NEXT: mov v4.s[1], v5.s[0]
-; CHECK-NEXT: movi d5, #0000000000000000
-; CHECK-NEXT: mov v0.s[2], v2.s[0]
-; CHECK-NEXT: movi d2, #0000000000000000
-; CHECK-NEXT: mov v0.s[3], v3.s[0]
-; CHECK-NEXT: movi d3, #0000000000000000
+; CHECK-NEXT: mov v2.s[1], v3.s[0]
+; CHECK-NEXT: mov v0.s[1], v1.s[0]
; CHECK-NEXT: .LBB8_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: mov v7.s[1], v1.s[0]
-; CHECK-NEXT: mov v6.s[1], v5.s[0]
+; CHECK-NEXT: fadd v6.2s, v6.2s, v0.2s
+; CHECK-NEXT: fadd v7.2s, v7.2s, v2.2s
; CHECK-NEXT: subs x2, x2, #1
-; CHECK-NEXT: mov v7.s[2], v2.s[0]
-; CHECK-NEXT: fadd v6.4s, v6.4s, v4.4s
-; CHECK-NEXT: mov v7.s[3], v3.s[0]
-; CHECK-NEXT: mov s5, v6.s[1]
-; CHECK-NEXT: fadd v7.4s, v7.4s, v0.4s
-; CHECK-NEXT: mov s3, v7.s[3]
-; CHECK-NEXT: mov s2, v7.s[2]
-; CHECK-NEXT: mov s1, v7.s[1]
+; CHECK-NEXT: fadd v16.2s, v16.2s, v4.2s
; CHECK-NEXT: b.ne .LBB8_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: fmov s0, s7
-; CHECK-NEXT: fmov s4, s6
-; CHECK-NEXT: // kill: def $s1 killed $s1 killed $q1
-; CHECK-NEXT: // kill: def $s2 killed $s2 killed $q2
-; CHECK-NEXT: // kill: def $s3 killed $s3 killed $q3
-; CHECK-NEXT: // kill: def $s5 killed $s5 killed $q5
+; CHECK-NEXT: mov s1, v6.s[1]
+; CHECK-NEXT: mov s3, v7.s[1]
+; CHECK-NEXT: mov s5, v16.s[1]
+; CHECK-NEXT: fmov s0, s6
+; CHECK-NEXT: fmov s2, s7
+; CHECK-NEXT: fmov s4, s16
; CHECK-NEXT: ret
entry:
br label %loop
@@ -476,39 +401,24 @@ exit:
define void @v6f64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6f64_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp q2, q1, [x0, #16]
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: ldr q3, [x0]
-; CHECK-NEXT: movi d16, #0000000000000000
-; CHECK-NEXT: movi d4, #0000000000000000
-; CHECK-NEXT: mov d5, v1.d[1]
-; CHECK-NEXT: mov d6, v2.d[1]
-; CHECK-NEXT: mov d7, v3.d[1]
-; CHECK-NEXT: mov v3.d[1], v7.d[0]
-; CHECK-NEXT: mov v2.d[1], v6.d[0]
-; CHECK-NEXT: mov v1.d[1], v5.d[0]
-; CHECK-NEXT: movi d6, #0000000000000000
-; CHECK-NEXT: movi d5, #0000000000000000
-; CHECK-NEXT: movi d7, #0000000000000000
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldr q5, [x0]
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: ldp q4, q3, [x0, #16]
; CHECK-NEXT: .LBB9_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: mov v0.d[1], v16.d[0]
-; CHECK-NEXT: mov v4.d[1], v6.d[0]
+; CHECK-NEXT: fadd v1.2d, v1.2d, v5.2d
+; CHECK-NEXT: fadd v0.2d, v0.2d, v4.2d
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: mov v5.d[1], v7.d[0]
-; CHECK-NEXT: fadd v0.2d, v0.2d, v3.2d
-; CHECK-NEXT: fadd v4.2d, v4.2d, v2.2d
-; CHECK-NEXT: fadd v5.2d, v5.2d, v1.2d
-; CHECK-NEXT: mov d16, v0.d[1]
-; CHECK-NEXT: mov d6, v4.d[1]
-; CHECK-NEXT: mov d7, v5.d[1]
+; CHECK-NEXT: fadd v2.2d, v2.2d, v3.2d
; CHECK-NEXT: b.ne .LBB9_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v0.d[1], v16.d[0]
-; CHECK-NEXT: mov v4.d[1], v7.d[0]
-; CHECK-NEXT: mov v6.d[1], v5.d[0]
-; CHECK-NEXT: stp q0, q4, [x2]
-; CHECK-NEXT: str q6, [x2, #32]
+; CHECK-NEXT: mov v3.16b, v0.16b
+; CHECK-NEXT: ext v0.16b, v0.16b, v2.16b, #8
+; CHECK-NEXT: mov v3.d[1], v2.d[1]
+; CHECK-NEXT: str q0, [x2, #32]
+; CHECK-NEXT: stp q1, q3, [x2]
; CHECK-NEXT: ret
entry:
%c.val = load <6 x double>, ptr %c
@@ -540,41 +450,32 @@ exit:
define <6 x double> @v6f64_cc(<6 x double> %c.val, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6f64_cc:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: fmov d16, d0
-; CHECK-NEXT: // kill: def $d5 killed $d5 def $q5
; CHECK-NEXT: // kill: def $d4 killed $d4 def $q4
-; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3
; CHECK-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: // kill: def $d5 killed $d5 def $q5
+; CHECK-NEXT: // kill: def $d3 killed $d3 def $q3
; CHECK-NEXT: // kill: def $d1 killed $d1 def $q1
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: movi d7, #0000000000000000
-; CHECK-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-NEXT: movi v6.2d, #0000000000000000
+; CHECK-NEXT: movi v7.2d, #0000000000000000
+; CHECK-NEXT: movi v16.2d, #0000000000000000
; CHECK-NEXT: mov v4.d[1], v5.d[0]
-; CHECK-NEXT: movi d3, #0000000000000000
-; CHECK-NEXT: movi d6, #0000000000000000
-; CHECK-NEXT: movi d5, #0000000000000000
-; CHECK-NEXT: mov v16.d[1], v1.d[0]
-; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-NEXT: mov v0.d[1], v1.d[0]
; CHECK-NEXT: .LBB10_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: mov v0.d[1], v1.d[0]
-; CHECK-NEXT: mov v7.d[1], v3.d[0]
-; CHECK-NEXT: subs x2, x2, #1
-; CHECK-NEXT: mov v6.d[1], v5.d[0]
-; CHECK-NEXT: fadd v0.2d, v0.2d, v16.2d
+; CHECK-NEXT: fadd v6.2d, v6.2d, v0.2d
; CHECK-NEXT: fadd v7.2d, v7.2d, v2.2d
-; CHECK-NEXT: fadd v6.2d, v6.2d, v4.2d
-; CHECK-NEXT: mov d1, v0.d[1]
-; CHECK-NEXT: mov d3, v7.d[1]
-; CHECK-NEXT: mov d5, v6.d[1]
+; CHECK-NEXT: subs x2, x2, #1
+; CHECK-NEXT: fadd v16.2d, v16.2d, v4.2d
; CHECK-NEXT: b.ne .LBB10_1
; CHECK-NEXT: // %bb.2: // %exit
+; CHECK-NEXT: mov d1, v6.d[1]
+; CHECK-NEXT: mov d3, v7.d[1]
+; CHECK-NEXT: mov d5, v16.d[1]
+; CHECK-NEXT: fmov d0, d6
; CHECK-NEXT: fmov d2, d7
-; CHECK-NEXT: fmov d4, d6
-; CHECK-NEXT: // kill: def $d0 killed $d0 killed $q0
-; CHECK-NEXT: // kill: def $d1 killed $d1 killed $q1
-; CHECK-NEXT: // kill: def $d3 killed $d3 killed $q3
-; CHECK-NEXT: // kill: def $d5 killed $d5 killed $q5
+; CHECK-NEXT: fmov d4, d16
; CHECK-NEXT: ret
entry:
br label %loop
@@ -594,65 +495,29 @@ exit:
define void @v12f64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v12f64_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp q2, q1, [x0]
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: ldp q4, q3, [x0, #32]
-; CHECK-NEXT: movi d25, #0000000000000000
-; CHECK-NEXT: ldp q5, q6, [x0, #64]
-; CHECK-NEXT: movi d7, #0000000000000000
-; CHECK-NEXT: mov d16, v1.d[1]
-; CHECK-NEXT: mov d17, v2.d[1]
-; CHECK-NEXT: movi d24, #0000000000000000
-; CHECK-NEXT: mov d19, v3.d[1]
-; CHECK-NEXT: mov d20, v4.d[1]
-; CHECK-NEXT: movi d23, #0000000000000000
-; CHECK-NEXT: mov d18, v5.d[1]
-; CHECK-NEXT: mov d21, v6.d[1]
-; CHECK-NEXT: movi d22, #0000000000000000
-; CHECK-NEXT: mov v2.d[1], v17.d[0]
-; CHECK-NEXT: mov v1.d[1], v16.d[0]
-; CHECK-NEXT: movi d16, #0000000000000000
-; CHECK-NEXT: mov v4.d[1], v20.d[0]
-; CHECK-NEXT: mov v3.d[1], v19.d[0]
-; CHECK-NEXT: movi d17, #0000000000000000
-; CHECK-NEXT: mov v5.d[1], v18.d[0]
-; CHECK-NEXT: mov v6.d[1], v21.d[0]
-; CHECK-NEXT: movi d18, #0000000000000000
-; CHECK-NEXT: movi d21, #0000000000000000
-; CHECK-NEXT: movi d19, #0000000000000000
-; CHECK-NEXT: movi d20, #0000000000000000
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: ldp q7, q6, [x0, #64]
+; CHECK-NEXT: movi v4.2d, #0000000000000000
+; CHECK-NEXT: movi v5.2d, #0000000000000000
+; CHECK-NEXT: ldp q17, q16, [x0, #32]
+; CHECK-NEXT: ldp q19, q18, [x0]
; CHECK-NEXT: .LBB11_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: mov v0.d[1], v25.d[0]
-; CHECK-NEXT: mov v7.d[1], v24.d[0]
+; CHECK-NEXT: fadd v1.2d, v1.2d, v19.2d
+; CHECK-NEXT: fadd v0.2d, v0.2d, v18.2d
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: mov v16.d[1], v23.d[0]
-; CHECK-NEXT: mov v17.d[1], v22.d[0]
-; CHECK-NEXT: mov v18.d[1], v21.d[0]
-; CHECK-NEXT: mov v19.d[1], v20.d[0]
-; CHECK-NEXT: fadd v0.2d, v0.2d, v2.2d
-; CHECK-NEXT: fadd v7.2d, v7.2d, v1.2d
-; CHECK-NEXT: fadd v16.2d, v16.2d, v4.2d
-; CHECK-NEXT: fadd v17.2d, v17.2d, v3.2d
-; CHECK-NEXT: fadd v18.2d, v18.2d, v5.2d
-; CHECK-NEXT: fadd v19.2d, v19.2d, v6.2d
-; CHECK-NEXT: mov d25, v0.d[1]
-; CHECK-NEXT: mov d24, v7.d[1]
-; CHECK-NEXT: mov d23, v16.d[1]
-; CHECK-NEXT: mov d22, v17.d[1]
-; CHECK-NEXT: mov d21, v18.d[1]
-; CHECK-NEXT: mov d20, v19.d[1]
+; CHECK-NEXT: fadd v2.2d, v2.2d, v17.2d
+; CHECK-NEXT: fadd v3.2d, v3.2d, v16.2d
+; CHECK-NEXT: fadd v4.2d, v4.2d, v7.2d
+; CHECK-NEXT: fadd v5.2d, v5.2d, v6.2d
; CHECK-NEXT: b.ne .LBB11_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v0.d[1], v25.d[0]
-; CHECK-NEXT: mov v7.d[1], v24.d[0]
-; CHECK-NEXT: mov v16.d[1], v23.d[0]
-; CHECK-NEXT: mov v17.d[1], v22.d[0]
-; CHECK-NEXT: mov v18.d[1], v21.d[0]
-; CHECK-NEXT: mov v19.d[1], v20.d[0]
-; CHECK-NEXT: stp q0, q7, [x2]
-; CHECK-NEXT: stp q16, q17, [x2, #32]
-; CHECK-NEXT: stp q18, q19, [x2, #64]
+; CHECK-NEXT: stp q1, q0, [x2]
+; CHECK-NEXT: stp q2, q3, [x2, #32]
+; CHECK-NEXT: stp q4, q5, [x2, #64]
; CHECK-NEXT: ret
entry:
%c.val = load <12 x double>, ptr %c
>From 4b1bd28a70c6398c400932f13abe5b43987f3162 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Tue, 4 Aug 2026 20:49:18 +0000
Subject: [PATCH 3/7] Refactor to avoid changing MVT Tables
---
llvm/include/llvm/CodeGen/TargetLowering.h | 16 ++++++++-----
llvm/lib/CodeGen/TargetLoweringBase.cpp | 28 +++-------------------
2 files changed, 13 insertions(+), 31 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 2b66f46aedfeb..32434707f6bf3 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1235,6 +1235,12 @@ class LLVM_ABI TargetLoweringBase {
return false;
}
+ bool shouldUseDynamicVectorTypeBreakdown(EVT VT, bool ForCallingConv) const {
+ return preferVectorizedNonPowerOfTwoTypeBreakdown() && !ForCallingConv &&
+ VT.isFixedLengthVector() &&
+ !isPowerOf2_32(VT.getVectorNumElements());
+ }
+
/// Certain targets such as MIPS require that some types such as vectors are
/// always broken down into scalars in some contexts. This occurs even if the
/// vector type is legal.
@@ -1859,9 +1865,8 @@ class LLVM_ABI TargetLoweringBase {
/// Return the type of registers that this ValueType will eventually require.
MVT getRegisterType(LLVMContext &Context, EVT VT,
bool ForCallingConv = false) const {
- if (VT.isSimple() && (!ForCallingConv || !VT.isFixedLengthVector() ||
- isPowerOf2_32(VT.getVectorNumElements()) ||
- !preferVectorizedNonPowerOfTwoTypeBreakdown()))
+ if (VT.isSimple() &&
+ !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv))
return getRegisterType(VT.getSimpleVT());
if (VT.isVector()) {
EVT VT1;
@@ -1892,9 +1897,8 @@ class LLVM_ABI TargetLoweringBase {
virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT,
std::optional<MVT> RegisterVT = std::nullopt,
bool ForCallingConv = false) const {
- if (VT.isSimple() && (!ForCallingConv || !VT.isFixedLengthVector() ||
- isPowerOf2_32(VT.getVectorNumElements()) ||
- !preferVectorizedNonPowerOfTwoTypeBreakdown())) {
+ if (VT.isSimple() &&
+ !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv)) {
assert((unsigned)VT.getSimpleVT().SimpleTy <
std::size(NumRegistersForVT));
return NumRegistersForVT[VT.getSimpleVT().SimpleTy];
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index a887156427ebe..6a47a5faeacfd 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1252,33 +1252,11 @@ static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
llvm_unreachable(
"Splitting or widening of non-power-of-2 MVTs is not implemented.");
- // FIXME: We don't generically support non-power-of-2-sized vectors for now.
+ // FIXME: We don't support non-power-of-2-sized vectors for now.
// Ideally we could break down into LHS/RHS like LegalizeDAG does.
if (!isPowerOf2_32(EC.getKnownMinValue())) {
- assert(VT.isFixedLengthVector() && "Expected a fixed-length vector VT");
- unsigned NumElts = EC.getKnownMinValue();
-
- // Find the largest legal vector type that exactly divides a
- // non-power-of-two vector.
- if (TLI->preferVectorizedNonPowerOfTwoTypeBreakdown()) {
- for (unsigned PartElts = llvm::bit_floor(NumElts); PartElts > 1;
- PartElts >>= 1) {
- if (NumElts % PartElts != 0)
- continue;
-
- MVT PartVT = MVT::getVectorVT(EltTy, ElementCount::getFixed(PartElts));
- if (PartVT == MVT() || !TLI->isTypeLegal(PartVT))
- continue;
-
- IntermediateVT = PartVT;
- NumIntermediates = NumElts / PartElts;
- RegisterVT = PartVT;
- return NumIntermediates;
- }
- }
-
- // Fall back to scalars if there is no exact legal vector decomposition.
- NumVectorRegs = NumElts;
+ // Split EC to unit size (scalable property is preserved).
+ NumVectorRegs = EC.getKnownMinValue();
EC = ElementCount::getFixed(1);
}
>From f498ec92c4f20f54ce90f453b5c319b17e92b1b3 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Wed, 5 Aug 2026 00:54:29 +0000
Subject: [PATCH 4/7] Reuse existing getTypeConversion logic from isScalable
block
---
llvm/lib/CodeGen/TargetLoweringBase.cpp | 44 ++--
.../CodeGen/AArch64/non-pow2-fixed-vectors.ll | 197 ++++++------------
llvm/test/CodeGen/AArch64/phi.ll | 76 +++++--
3 files changed, 139 insertions(+), 178 deletions(-)
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 6a47a5faeacfd..876e03715cf1b 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1717,9 +1717,7 @@ unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,
unsigned NumVectorRegs = 1;
- // Scalable vectors cannot be scalarized, so handle the legalisation of the
- // types like done elsewhere in SelectionDAG.
- if (EltCnt.isScalable()) {
+ auto GetLegalVectorBreakdown = [&]() -> std::optional<unsigned> {
LegalizeKind LK;
EVT PartVT = VT;
do {
@@ -1728,17 +1726,25 @@ unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,
PartVT = LK.second;
} while (LK.first != TypeLegal);
- if (!PartVT.isVector()) {
- report_fatal_error(
- "Don't know how to legalize this scalable vector type");
- }
+ if (!PartVT.isVector())
+ return std::nullopt;
+ assert(PartVT.isScalableVector() == VT.isScalableVector() &&
+ "Vector legalization changed scalability");
NumIntermediates =
divideCeil(VT.getVectorElementCount().getKnownMinValue(),
PartVT.getVectorElementCount().getKnownMinValue());
IntermediateVT = PartVT;
RegisterVT = getRegisterType(Context, IntermediateVT);
return NumIntermediates;
+ };
+
+ // Scalable vectors cannot be scalarized, so handle the legalisation of the
+ // types like done elsewhere in SelectionDAG.
+ if (EltCnt.isScalable()) {
+ if (std::optional<unsigned> NumRegs = GetLegalVectorBreakdown())
+ return *NumRegs;
+ report_fatal_error("Don't know how to legalize this scalable vector type");
}
// FIXME: We don't generically support non-power-of-2-sized vectors for now.
@@ -1747,27 +1753,11 @@ unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,
assert(VT.isFixedLengthVector() && "Expected a fixed-length vector VT");
unsigned NumElts = EltCnt.getKnownMinValue();
- // Find the largest legal vector type that exactly divides a
- // non-power-of-two vector.
- if (!ForCallingConv && preferVectorizedNonPowerOfTwoTypeBreakdown()) {
- for (unsigned PartElts = llvm::bit_floor(NumElts); PartElts > 1;
- PartElts >>= 1) {
- if (NumElts % PartElts != 0)
- continue;
-
- EVT PartVT =
- EVT::getVectorVT(Context, EltTy, ElementCount::getFixed(PartElts));
- if (!isTypeLegal(PartVT))
- continue;
-
- IntermediateVT = PartVT;
- NumIntermediates = NumElts / PartElts;
- RegisterVT = PartVT.getSimpleVT();
- return NumIntermediates;
- }
- }
+ if (!ForCallingConv && preferVectorizedNonPowerOfTwoTypeBreakdown())
+ if (std::optional<unsigned> NumRegs = GetLegalVectorBreakdown())
+ return *NumRegs;
- // Fall back to scalars if there is no exact legal vector decomposition.
+ // Fall back to scalars if there is no legal vector decomposition.
NumVectorRegs = NumElts;
EltCnt = ElementCount::getFixed(1);
}
diff --git a/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll b/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
index ea945cdbcec43..f5357c2a67181 100644
--- a/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
+++ b/llvm/test/CodeGen/AArch64/non-pow2-fixed-vectors.ll
@@ -4,41 +4,19 @@
define void @v5i32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v5i32_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldp w12, w13, [x0]
-; CHECK-NEXT: mov w1, wzr
-; CHECK-NEXT: mov w11, wzr
-; CHECK-NEXT: mov w9, wzr
-; CHECK-NEXT: mov w8, wzr
-; CHECK-NEXT: mov w10, wzr
-; CHECK-NEXT: fmov s0, w12
-; CHECK-NEXT: mov v0.s[1], w13
-; CHECK-NEXT: ldp w13, w12, [x0, #8]
-; CHECK-NEXT: mov v0.s[2], w13
-; CHECK-NEXT: mov v0.s[3], w12
-; CHECK-NEXT: ldr w12, [x0, #16]
-; CHECK-NEXT: fmov s1, w12
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldr q2, [x0]
+; CHECK-NEXT: ldr s3, [x0, #16]
; CHECK-NEXT: .LBB0_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: fmov s2, w1
-; CHECK-NEXT: fmov s3, w10
+; CHECK-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: add v0.4s, v0.4s, v3.4s
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: mov v2.s[1], w11
-; CHECK-NEXT: add v3.4s, v3.4s, v1.4s
-; CHECK-NEXT: fmov w10, s3
-; CHECK-NEXT: mov v2.s[2], w9
-; CHECK-NEXT: mov v2.s[3], w8
-; CHECK-NEXT: add v2.4s, v2.4s, v0.4s
-; CHECK-NEXT: mov w8, v2.s[3]
-; CHECK-NEXT: mov w9, v2.s[2]
-; CHECK-NEXT: mov w11, v2.s[1]
-; CHECK-NEXT: fmov w1, s2
; CHECK-NEXT: b.ne .LBB0_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v2.s[1], w11
-; CHECK-NEXT: str w10, [x2, #16]
-; CHECK-NEXT: mov v2.s[2], w9
-; CHECK-NEXT: mov v2.s[3], w8
-; CHECK-NEXT: str q2, [x2]
+; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: str s0, [x2, #16]
; CHECK-NEXT: ret
entry:
%c.val = load <5 x i32>, ptr %c
@@ -60,23 +38,18 @@ exit:
define void @v6i32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6i32_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr q1, [x0]
; CHECK-NEXT: movi v0.2d, #0000000000000000
-; CHECK-NEXT: movi v4.2d, #0000000000000000
-; CHECK-NEXT: movi v2.2d, #0000000000000000
-; CHECK-NEXT: ldr d5, [x0, #16]
-; CHECK-NEXT: mov d3, v1.d[1]
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldp q3, q2, [x0]
; CHECK-NEXT: .LBB1_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add v0.2s, v0.2s, v1.2s
-; CHECK-NEXT: add v4.2s, v4.2s, v3.2s
+; CHECK-NEXT: add v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: add v0.4s, v0.4s, v2.4s
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: add v2.2s, v2.2s, v5.2s
; CHECK-NEXT: b.ne .LBB1_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v0.d[1], v4.d[0]
-; CHECK-NEXT: str d2, [x2, #16]
-; CHECK-NEXT: str q0, [x2]
+; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: str d0, [x2, #16]
; CHECK-NEXT: ret
entry:
%c.val = load <6 x i32>, ptr %c
@@ -98,30 +71,28 @@ exit:
define <6 x i32> @v6i32_cc(<6 x i32> %c.val, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6i32_cc:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: fmov s0, w4
-; CHECK-NEXT: fmov s1, w2
+; CHECK-NEXT: fmov s0, w0
+; CHECK-NEXT: fmov s2, w4
; CHECK-NEXT: ldr x8, [sp]
-; CHECK-NEXT: fmov s3, w0
-; CHECK-NEXT: movi v2.2d, #0000000000000000
-; CHECK-NEXT: movi v4.2d, #0000000000000000
-; CHECK-NEXT: movi v5.2d, #0000000000000000
-; CHECK-NEXT: mov v0.s[1], w5
-; CHECK-NEXT: mov v1.s[1], w3
-; CHECK-NEXT: mov v3.s[1], w1
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: movi v3.2d, #0000000000000000
+; CHECK-NEXT: mov v0.s[1], w1
+; CHECK-NEXT: mov v2.s[1], w5
+; CHECK-NEXT: mov v0.s[2], w2
+; CHECK-NEXT: mov v0.s[3], w3
; CHECK-NEXT: .LBB2_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add v2.2s, v2.2s, v3.2s
-; CHECK-NEXT: add v4.2s, v4.2s, v1.2s
+; CHECK-NEXT: add v1.4s, v1.4s, v0.4s
+; CHECK-NEXT: add v3.4s, v3.4s, v2.4s
; CHECK-NEXT: subs x8, x8, #1
-; CHECK-NEXT: add v5.2s, v5.2s, v0.2s
; CHECK-NEXT: b.ne .LBB2_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov w1, v2.s[1]
-; CHECK-NEXT: mov w3, v4.s[1]
-; CHECK-NEXT: mov w5, v5.s[1]
-; CHECK-NEXT: fmov w0, s2
-; CHECK-NEXT: fmov w2, s4
-; CHECK-NEXT: fmov w4, s5
+; CHECK-NEXT: mov w1, v1.s[1]
+; CHECK-NEXT: mov w2, v1.s[2]
+; CHECK-NEXT: mov w3, v1.s[3]
+; CHECK-NEXT: mov w5, v3.s[1]
+; CHECK-NEXT: fmov w0, s1
+; CHECK-NEXT: fmov w4, s3
; CHECK-NEXT: ret
entry:
br label %loop
@@ -141,29 +112,21 @@ exit:
define void @v5i64_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v5i64_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: ldp q2, q1, [x0]
-; CHECK-NEXT: ldr d7, [x0, #32]
-; CHECK-NEXT: mov d3, v1.d[1]
-; CHECK-NEXT: mov d6, v2.d[1]
-; CHECK-NEXT: fmov d17, d0
-; CHECK-NEXT: fmov d4, d0
-; CHECK-NEXT: fmov d16, d0
-; CHECK-NEXT: fmov d5, d0
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldr d5, [x0, #32]
+; CHECK-NEXT: movi v2.2d, #0000000000000000
+; CHECK-NEXT: ldp q4, q3, [x0]
; CHECK-NEXT: .LBB3_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: add d0, d0, d2
-; CHECK-NEXT: add d17, d17, d6
+; CHECK-NEXT: add v1.2d, v1.2d, v4.2d
+; CHECK-NEXT: add v2.2d, v2.2d, v3.2d
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: add d4, d4, d1
-; CHECK-NEXT: add d16, d16, d3
-; CHECK-NEXT: add d5, d5, d7
+; CHECK-NEXT: add v0.2d, v0.2d, v5.2d
; CHECK-NEXT: b.ne .LBB3_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v0.d[1], v17.d[0]
-; CHECK-NEXT: mov v4.d[1], v16.d[0]
-; CHECK-NEXT: str d5, [x2, #32]
-; CHECK-NEXT: stp q0, q4, [x2]
+; CHECK-NEXT: stp q1, q2, [x2]
+; CHECK-NEXT: str d0, [x2, #32]
; CHECK-NEXT: ret
entry:
%c.val = load <5 x i64>, ptr %c
@@ -266,37 +229,19 @@ exit:
define void @v5f32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v5f32_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr q0, [x0]
-; CHECK-NEXT: movi d5, #0000000000000000
-; CHECK-NEXT: movi d4, #0000000000000000
-; CHECK-NEXT: ldr s6, [x0, #16]
-; CHECK-NEXT: mov s1, v0.s[1]
-; CHECK-NEXT: mov s2, v0.s[3]
-; CHECK-NEXT: mov s3, v0.s[2]
-; CHECK-NEXT: mov v0.s[1], v1.s[0]
-; CHECK-NEXT: movi d1, #0000000000000000
-; CHECK-NEXT: mov v0.s[2], v3.s[0]
-; CHECK-NEXT: movi d3, #0000000000000000
-; CHECK-NEXT: mov v0.s[3], v2.s[0]
-; CHECK-NEXT: movi d2, #0000000000000000
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldr q2, [x0]
+; CHECK-NEXT: ldr s3, [x0, #16]
; CHECK-NEXT: .LBB6_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: mov v1.s[1], v5.s[0]
-; CHECK-NEXT: fadd v2.4s, v2.4s, v6.4s
+; CHECK-NEXT: fadd v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: fadd v0.4s, v0.4s, v3.4s
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: mov v1.s[2], v4.s[0]
-; CHECK-NEXT: mov v1.s[3], v3.s[0]
-; CHECK-NEXT: fadd v1.4s, v1.4s, v0.4s
-; CHECK-NEXT: mov s3, v1.s[3]
-; CHECK-NEXT: mov s4, v1.s[2]
-; CHECK-NEXT: mov s5, v1.s[1]
; CHECK-NEXT: b.ne .LBB6_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v1.s[1], v5.s[0]
-; CHECK-NEXT: str s2, [x2, #16]
-; CHECK-NEXT: mov v1.s[2], v4.s[0]
-; CHECK-NEXT: mov v1.s[3], v3.s[0]
; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: str s0, [x2, #16]
; CHECK-NEXT: ret
entry:
%c.val = load <5 x float>, ptr %c
@@ -318,23 +263,18 @@ exit:
define void @v6f32_internal(ptr %c, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6f32_internal:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr q1, [x0]
-; CHECK-NEXT: movi d0, #0000000000000000
-; CHECK-NEXT: movi d4, #0000000000000000
-; CHECK-NEXT: movi d2, #0000000000000000
-; CHECK-NEXT: ldr d5, [x0, #16]
-; CHECK-NEXT: mov d3, v1.d[1]
+; CHECK-NEXT: movi v0.2d, #0000000000000000
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: ldp q3, q2, [x0]
; CHECK-NEXT: .LBB7_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: fadd v0.2s, v0.2s, v1.2s
-; CHECK-NEXT: fadd v4.2s, v4.2s, v3.2s
+; CHECK-NEXT: fadd v1.4s, v1.4s, v3.4s
+; CHECK-NEXT: fadd v0.4s, v0.4s, v2.4s
; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: fadd v2.2s, v2.2s, v5.2s
; CHECK-NEXT: b.ne .LBB7_1
; CHECK-NEXT: // %bb.2: // %exit
-; CHECK-NEXT: mov v0.d[1], v4.d[0]
-; CHECK-NEXT: str d2, [x2, #16]
-; CHECK-NEXT: str q0, [x2]
+; CHECK-NEXT: str q1, [x2]
+; CHECK-NEXT: str d0, [x2, #16]
; CHECK-NEXT: ret
entry:
%c.val = load <6 x float>, ptr %c
@@ -356,32 +296,31 @@ exit:
define <6 x float> @v6f32_cc(<6 x float> %c.val, ptr %in, ptr %out, i64 %n) {
; CHECK-LABEL: v6f32_cc:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $s4 killed $s4 def $q4
-; CHECK-NEXT: // kill: def $s2 killed $s2 def $q2
; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
+; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1
+; CHECK-NEXT: // kill: def $s2 killed $s2 def $q2
+; CHECK-NEXT: // kill: def $s4 killed $s4 def $q4
; CHECK-NEXT: // kill: def $s5 killed $s5 def $q5
; CHECK-NEXT: // kill: def $s3 killed $s3 def $q3
-; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1
-; CHECK-NEXT: movi d6, #0000000000000000
-; CHECK-NEXT: movi d7, #0000000000000000
-; CHECK-NEXT: movi d16, #0000000000000000
-; CHECK-NEXT: mov v4.s[1], v5.s[0]
-; CHECK-NEXT: mov v2.s[1], v3.s[0]
+; CHECK-NEXT: movi v6.2d, #0000000000000000
+; CHECK-NEXT: movi v7.2d, #0000000000000000
; CHECK-NEXT: mov v0.s[1], v1.s[0]
+; CHECK-NEXT: mov v4.s[1], v5.s[0]
+; CHECK-NEXT: mov v0.s[2], v2.s[0]
+; CHECK-NEXT: mov v0.s[3], v3.s[0]
; CHECK-NEXT: .LBB8_1: // %loop
; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: fadd v6.2s, v6.2s, v0.2s
-; CHECK-NEXT: fadd v7.2s, v7.2s, v2.2s
+; CHECK-NEXT: fadd v6.4s, v6.4s, v0.4s
+; CHECK-NEXT: fadd v7.4s, v7.4s, v4.4s
; CHECK-NEXT: subs x2, x2, #1
-; CHECK-NEXT: fadd v16.2s, v16.2s, v4.2s
; CHECK-NEXT: b.ne .LBB8_1
; CHECK-NEXT: // %bb.2: // %exit
; CHECK-NEXT: mov s1, v6.s[1]
-; CHECK-NEXT: mov s3, v7.s[1]
-; CHECK-NEXT: mov s5, v16.s[1]
+; CHECK-NEXT: mov s2, v6.s[2]
+; CHECK-NEXT: mov s3, v6.s[3]
+; CHECK-NEXT: mov s5, v7.s[1]
; CHECK-NEXT: fmov s0, s6
-; CHECK-NEXT: fmov s2, s7
-; CHECK-NEXT: fmov s4, s16
+; CHECK-NEXT: fmov s4, s7
; CHECK-NEXT: ret
entry:
br label %loop
diff --git a/llvm/test/CodeGen/AArch64/phi.ll b/llvm/test/CodeGen/AArch64/phi.ll
index cad959c077990..399c19dbd7b2c 100644
--- a/llvm/test/CodeGen/AArch64/phi.ll
+++ b/llvm/test/CodeGen/AArch64/phi.ll
@@ -336,14 +336,19 @@ define <3 x i8> @tv3i8(i1 %c, ptr %p, <3 x i8> %a, <3 x i8> %b) {
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: tbz w0, #0, .LBB11_2
; CHECK-SD-NEXT: // %bb.1: // %t
-; CHECK-SD-NEXT: mov w5, w2
-; CHECK-SD-NEXT: mov w6, w3
-; CHECK-SD-NEXT: mov w7, w4
+; CHECK-SD-NEXT: fmov s0, w2
; CHECK-SD-NEXT: str wzr, [x1]
-; CHECK-SD-NEXT: .LBB11_2: // %e
-; CHECK-SD-NEXT: mov w0, w5
-; CHECK-SD-NEXT: mov w1, w6
-; CHECK-SD-NEXT: mov w2, w7
+; CHECK-SD-NEXT: mov v0.h[1], w3
+; CHECK-SD-NEXT: mov v0.h[2], w4
+; CHECK-SD-NEXT: b .LBB11_3
+; CHECK-SD-NEXT: .LBB11_2:
+; CHECK-SD-NEXT: fmov s0, w5
+; CHECK-SD-NEXT: mov v0.h[1], w6
+; CHECK-SD-NEXT: mov v0.h[2], w7
+; CHECK-SD-NEXT: .LBB11_3: // %e
+; CHECK-SD-NEXT: umov w0, v0.h[0]
+; CHECK-SD-NEXT: umov w1, v0.h[1]
+; CHECK-SD-NEXT: umov w2, v0.h[2]
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: tv3i8:
@@ -728,15 +733,24 @@ e:
define <3 x i64> @tv3i64(i1 %c, ptr %p, <3 x i64> %a, <3 x i64> %b) {
; CHECK-SD-LABEL: tv3i64:
; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $d5 killed $d5 def $q5
+; CHECK-SD-NEXT: // kill: def $d4 killed $d4 def $q4
+; CHECK-SD-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT: tbz w0, #0, .LBB26_2
; CHECK-SD-NEXT: // %bb.1: // %t
-; CHECK-SD-NEXT: fmov d3, d0
-; CHECK-SD-NEXT: fmov d4, d1
+; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT: mov v5.16b, v2.16b
; CHECK-SD-NEXT: str wzr, [x1]
-; CHECK-SD-NEXT: fmov d5, d2
-; CHECK-SD-NEXT: .LBB26_2: // %e
+; CHECK-SD-NEXT: mov v3.16b, v0.16b
+; CHECK-SD-NEXT: b .LBB26_3
+; CHECK-SD-NEXT: .LBB26_2:
+; CHECK-SD-NEXT: mov v3.d[1], v4.d[0]
+; CHECK-SD-NEXT: .LBB26_3: // %e
+; CHECK-SD-NEXT: mov d1, v3.d[1]
; CHECK-SD-NEXT: fmov d0, d3
-; CHECK-SD-NEXT: fmov d1, d4
; CHECK-SD-NEXT: fmov d2, d5
; CHECK-SD-NEXT: ret
;
@@ -877,15 +891,24 @@ e:
define <3 x ptr> @tv3p0(i1 %c, ptr %p, <3 x ptr> %a, <3 x ptr> %b) {
; CHECK-SD-LABEL: tv3p0:
; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $d5 killed $d5 def $q5
+; CHECK-SD-NEXT: // kill: def $d4 killed $d4 def $q4
+; CHECK-SD-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT: tbz w0, #0, .LBB30_2
; CHECK-SD-NEXT: // %bb.1: // %t
-; CHECK-SD-NEXT: fmov d3, d0
-; CHECK-SD-NEXT: fmov d4, d1
+; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT: mov v5.16b, v2.16b
; CHECK-SD-NEXT: str wzr, [x1]
-; CHECK-SD-NEXT: fmov d5, d2
-; CHECK-SD-NEXT: .LBB30_2: // %e
+; CHECK-SD-NEXT: mov v3.16b, v0.16b
+; CHECK-SD-NEXT: b .LBB30_3
+; CHECK-SD-NEXT: .LBB30_2:
+; CHECK-SD-NEXT: mov v3.d[1], v4.d[0]
+; CHECK-SD-NEXT: .LBB30_3: // %e
+; CHECK-SD-NEXT: mov d1, v3.d[1]
; CHECK-SD-NEXT: fmov d0, d3
-; CHECK-SD-NEXT: fmov d1, d4
; CHECK-SD-NEXT: fmov d2, d5
; CHECK-SD-NEXT: ret
;
@@ -1191,15 +1214,24 @@ e:
define <3 x double> @tv3f64(i1 %c, ptr %p, <3 x double> %a, <3 x double> %b) {
; CHECK-SD-LABEL: tv3f64:
; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: // kill: def $d5 killed $d5 def $q5
+; CHECK-SD-NEXT: // kill: def $d4 killed $d4 def $q4
+; CHECK-SD-NEXT: // kill: def $d3 killed $d3 def $q3
+; CHECK-SD-NEXT: // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT: // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT: // kill: def $d0 killed $d0 def $q0
; CHECK-SD-NEXT: tbz w0, #0, .LBB42_2
; CHECK-SD-NEXT: // %bb.1: // %t
-; CHECK-SD-NEXT: fmov d3, d0
-; CHECK-SD-NEXT: fmov d4, d1
+; CHECK-SD-NEXT: mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT: mov v5.16b, v2.16b
; CHECK-SD-NEXT: str wzr, [x1]
-; CHECK-SD-NEXT: fmov d5, d2
-; CHECK-SD-NEXT: .LBB42_2: // %e
+; CHECK-SD-NEXT: mov v3.16b, v0.16b
+; CHECK-SD-NEXT: b .LBB42_3
+; CHECK-SD-NEXT: .LBB42_2:
+; CHECK-SD-NEXT: mov v3.d[1], v4.d[0]
+; CHECK-SD-NEXT: .LBB42_3: // %e
+; CHECK-SD-NEXT: mov d1, v3.d[1]
; CHECK-SD-NEXT: fmov d0, d3
-; CHECK-SD-NEXT: fmov d1, d4
; CHECK-SD-NEXT: fmov d2, d5
; CHECK-SD-NEXT: ret
;
>From 0a8dfcc37eb607959b76269730a70c2e09c0ee40 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Wed, 5 Aug 2026 10:22:04 +0000
Subject: [PATCH 5/7] Remove spurious include
---
llvm/lib/Target/AArch64/AArch64ISelLowering.h | 1 -
1 file changed, 1 deletion(-)
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.h b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
index 9740f38893abc..931b98bf8a41d 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.h
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.h
@@ -20,7 +20,6 @@
#include "llvm/CodeGen/TargetLowering.h"
#include "llvm/IR/CallingConv.h"
#include "llvm/IR/Instruction.h"
-#include "llvm/Support/MathExtras.h"
namespace llvm {
>From 015d1829c64d434a8dbd1f46e8bcabd65d0f02bc Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Fri, 7 Aug 2026 14:29:01 +0000
Subject: [PATCH 6/7] Address review
Refactor to use *Impl methods and remove other Target changes.
---
llvm/include/llvm/CodeGen/TargetLowering.h | 115 ++++++++++--------
llvm/lib/CodeGen/TargetLoweringBase.cpp | 8 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 9 +-
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 3 +-
llvm/lib/Target/RISCV/RISCVISelLowering.cpp | 9 +-
llvm/lib/Target/RISCV/RISCVISelLowering.h | 6 +-
llvm/lib/Target/SPIRV/SPIRVISelLowering.h | 6 +-
llvm/lib/Target/SystemZ/SystemZISelLowering.h | 9 +-
8 files changed, 89 insertions(+), 76 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 32434707f6bf3..6eb0f347cbe49 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1226,8 +1226,12 @@ class LLVM_ABI TargetLoweringBase {
/// before they are promoted/expanded.
unsigned getVectorTypeBreakdown(LLVMContext &Context, EVT VT,
EVT &IntermediateVT,
- unsigned &NumIntermediates, MVT &RegisterVT,
- bool ForCallingConv = false) const;
+ unsigned &NumIntermediates,
+ MVT &RegisterVT) const {
+ return getVectorTypeBreakdownImpl(Context, VT, IntermediateVT,
+ NumIntermediates, RegisterVT,
+ /*ForCallingConv=*/false);
+ }
/// Return true if fixed-length, non-power-of-two vectors should be broken
/// down into legal vector parts instead of scalars for internal values.
@@ -1247,8 +1251,9 @@ class LLVM_ABI TargetLoweringBase {
virtual unsigned getVectorTypeBreakdownForCallingConv(
LLVMContext &Context, CallingConv::ID CC, EVT VT, EVT &IntermediateVT,
unsigned &NumIntermediates, MVT &RegisterVT) const {
- return getVectorTypeBreakdown(Context, VT, IntermediateVT, NumIntermediates,
- RegisterVT, /*ForCallingConv=*/true);
+ return getVectorTypeBreakdownImpl(Context, VT, IntermediateVT,
+ NumIntermediates, RegisterVT,
+ /*ForCallingConv=*/true);
}
struct IntrinsicInfo {
@@ -1863,24 +1868,8 @@ class LLVM_ABI TargetLoweringBase {
}
/// Return the type of registers that this ValueType will eventually require.
- MVT getRegisterType(LLVMContext &Context, EVT VT,
- bool ForCallingConv = false) const {
- if (VT.isSimple() &&
- !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv))
- return getRegisterType(VT.getSimpleVT());
- if (VT.isVector()) {
- EVT VT1;
- MVT RegisterVT;
- unsigned NumIntermediates;
- (void)getVectorTypeBreakdown(Context, VT, VT1, NumIntermediates,
- RegisterVT, ForCallingConv);
- return RegisterVT;
- }
- if (VT.isInteger()) {
- return getRegisterType(Context, getTypeToTransformTo(Context, VT),
- ForCallingConv);
- }
- llvm_unreachable("Unsupported extended type!");
+ MVT getRegisterType(LLVMContext &Context, EVT VT) const {
+ return getRegisterTypeImpl(Context, VT, /*ForCallingConv=*/false);
}
/// Return the number of registers that this ValueType will eventually
@@ -1894,29 +1883,10 @@ class LLVM_ABI TargetLoweringBase {
///
/// RegisterVT may be passed as a way to override the default settings, for
/// instance with i128 inline assembly operands on SystemZ.
- virtual unsigned getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt,
- bool ForCallingConv = false) const {
- if (VT.isSimple() &&
- !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv)) {
- assert((unsigned)VT.getSimpleVT().SimpleTy <
- std::size(NumRegistersForVT));
- return NumRegistersForVT[VT.getSimpleVT().SimpleTy];
- }
- if (VT.isVector()) {
- EVT VT1;
- MVT VT2;
- unsigned NumIntermediates;
- return getVectorTypeBreakdown(Context, VT, VT1, NumIntermediates, VT2,
- ForCallingConv);
- }
- if (VT.isInteger()) {
- unsigned BitWidth = VT.getSizeInBits();
- unsigned RegWidth =
- getRegisterType(Context, VT, ForCallingConv).getSizeInBits();
- return (BitWidth + RegWidth - 1) / RegWidth;
- }
- llvm_unreachable("Unsupported extended type!");
+ virtual unsigned
+ getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt) const {
+ return getNumRegistersImpl(Context, VT, /*ForCallingConv=*/false);
}
/// Certain combinations of ABIs, Targets and features require that types
@@ -1924,7 +1894,7 @@ class LLVM_ABI TargetLoweringBase {
/// For MIPS all vector types must be passed through the integer register set.
virtual MVT getRegisterTypeForCallingConv(LLVMContext &Context,
CallingConv::ID CC, EVT VT) const {
- return getRegisterType(Context, VT, /*ForCallingConv=*/true);
+ return getRegisterTypeImpl(Context, VT, /*ForCallingConv=*/true);
}
/// Certain targets require unusual breakdowns of certain types. For MIPS,
@@ -1933,8 +1903,7 @@ class LLVM_ABI TargetLoweringBase {
virtual unsigned getNumRegistersForCallingConv(LLVMContext &Context,
CallingConv::ID CC,
EVT VT) const {
- return getNumRegisters(Context, VT, /*RegisterVT=*/std::nullopt,
- /*ForCallingConv=*/true);
+ return getNumRegistersImpl(Context, VT, /*ForCallingConv=*/true);
}
/// Certain targets have context sensitive alignment requirements, where one
@@ -3997,6 +3966,56 @@ class LLVM_ABI TargetLoweringBase {
return (LegalizeAction)((IndexedModeActions[Ty][IdxMode] >> Shift) & 0xf);
}
+ unsigned getVectorTypeBreakdownImpl(LLVMContext &Context, EVT VT,
+ EVT &IntermediateVT,
+ unsigned &NumIntermediates,
+ MVT &RegisterVT,
+ bool ForCallingConv) const;
+
+ MVT getRegisterTypeImpl(LLVMContext &Context, EVT VT,
+ bool ForCallingConv) const {
+ if (VT.isSimple() &&
+ !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv))
+ return getRegisterType(VT.getSimpleVT());
+ if (VT.isVector()) {
+ EVT VT1;
+ MVT RegisterVT;
+ unsigned NumIntermediates;
+ (void)getVectorTypeBreakdownImpl(Context, VT, VT1, NumIntermediates,
+ RegisterVT, ForCallingConv);
+ return RegisterVT;
+ }
+ if (VT.isInteger()) {
+ return getRegisterTypeImpl(Context, getTypeToTransformTo(Context, VT),
+ ForCallingConv);
+ }
+ llvm_unreachable("Unsupported extended type!");
+ }
+
+ unsigned getNumRegistersImpl(LLVMContext &Context, EVT VT,
+ bool ForCallingConv) const {
+ if (VT.isSimple() &&
+ !shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv)) {
+ assert((unsigned)VT.getSimpleVT().SimpleTy <
+ std::size(NumRegistersForVT));
+ return NumRegistersForVT[VT.getSimpleVT().SimpleTy];
+ }
+ if (VT.isVector()) {
+ EVT VT1;
+ MVT VT2;
+ unsigned NumIntermediates;
+ return getVectorTypeBreakdownImpl(Context, VT, VT1, NumIntermediates, VT2,
+ ForCallingConv);
+ }
+ if (VT.isInteger()) {
+ unsigned BitWidth = VT.getSizeInBits();
+ unsigned RegWidth =
+ getRegisterTypeImpl(Context, VT, ForCallingConv).getSizeInBits();
+ return (BitWidth + RegWidth - 1) / RegWidth;
+ }
+ llvm_unreachable("Unsupported extended type!");
+ }
+
protected:
/// Return true if the extension represented by \p I is free.
/// \pre \p I is a sign, zero, or fp extension and
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 876e03715cf1b..1810cd0bf83a2 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1688,11 +1688,9 @@ EVT TargetLoweringBase::getSetCCResultType(const DataLayout &DL, LLVMContext &,
/// This method returns the number of registers needed, and the VT for each
/// register. It also returns the VT and quantity of the intermediate values
/// before they are promoted/expanded.
-unsigned TargetLoweringBase::getVectorTypeBreakdown(LLVMContext &Context,
- EVT VT, EVT &IntermediateVT,
- unsigned &NumIntermediates,
- MVT &RegisterVT,
- bool ForCallingConv) const {
+unsigned TargetLoweringBase::getVectorTypeBreakdownImpl(
+ LLVMContext &Context, EVT VT, EVT &IntermediateVT,
+ unsigned &NumIntermediates, MVT &RegisterVT, bool ForCallingConv) const {
ElementCount EltCnt = VT.getVectorElementCount();
// If there is a wider vector type with the same element type as this one,
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 7ba25e63f6854..e788b0e44041f 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -4025,13 +4025,12 @@ SDValue NVPTXTargetLowering::LowerCopyToReg_128(SDValue Op,
return DAG.getNode(ISD::CopyToReg, DL, ResultsType, NewOps);
}
-unsigned NVPTXTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT,
- bool ForCallingConv) const {
+unsigned NVPTXTargetLowering::getNumRegisters(
+ LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt) const {
if (VT == MVT::i128 && RegisterVT == MVT::i128)
return 1;
- return TargetLoweringBase::getNumRegisters(Context, VT, RegisterVT,
- ForCallingConv);
+ return TargetLoweringBase::getNumRegisters(Context, VT, RegisterVT);
}
bool NVPTXTargetLowering::splitValueIntoRegisterParts(
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index c7238444ae61c..42741ba98b438 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -224,8 +224,7 @@ class NVPTXTargetLowering : public TargetLowering {
SDValue LowerCopyToReg_128(SDValue Op, SelectionDAG &DAG) const;
unsigned getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT,
- bool ForCallingConv = false) const override;
+ std::optional<MVT> RegisterVT) const override;
bool
splitValueIntoRegisterParts(SelectionDAG &DAG, const SDLoc &DL, SDValue Val,
SDValue *Parts, unsigned NumParts, MVT PartVT,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
index ff13d62172e87..44359a9e6155f 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.cpp
@@ -2934,16 +2934,15 @@ MVT RISCVTargetLowering::getRegisterTypeForCallingConv(LLVMContext &Context,
return TargetLowering::getRegisterTypeForCallingConv(Context, CC, VT);
}
-unsigned RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT,
- bool ForCallingConv) const {
+unsigned
+RISCVTargetLowering::getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT) const {
// Pair inline assembly operand
if (VT == (Subtarget.is64Bit() ? MVT::i128 : MVT::i64) && RegisterVT &&
*RegisterVT == MVT::Untyped)
return 1;
- return TargetLowering::getNumRegisters(Context, VT, RegisterVT,
- ForCallingConv);
+ return TargetLowering::getNumRegisters(Context, VT, RegisterVT);
}
unsigned RISCVTargetLowering::getNumRegistersForCallingConv(LLVMContext &Context,
diff --git a/llvm/lib/Target/RISCV/RISCVISelLowering.h b/llvm/lib/Target/RISCV/RISCVISelLowering.h
index 56d62d83272dc..b9bea162d6d4c 100644
--- a/llvm/lib/Target/RISCV/RISCVISelLowering.h
+++ b/llvm/lib/Target/RISCV/RISCVISelLowering.h
@@ -82,9 +82,9 @@ class RISCVTargetLowering : public TargetLowering {
EVT VT) const override;
/// Return the number of registers for a given MVT, for inline assembly
- unsigned getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt,
- bool ForCallingConv = false) const override;
+ unsigned
+ getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt) const override;
/// Return the number of registers for a given MVT, ensuring vectors are
/// treated as a series of gpr sized integers.
diff --git a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
index a4e91b3f06d64..ff843649fd6d3 100644
--- a/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
+++ b/llvm/lib/Target/SPIRV/SPIRVISelLowering.h
@@ -53,9 +53,9 @@ class SPIRVTargetLowering : public TargetLowering {
std::pair<unsigned, const TargetRegisterClass *>
getRegForInlineAsmConstraint(const TargetRegisterInfo *TRI,
StringRef Constraint, MVT VT) const override;
- unsigned getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT = std::nullopt,
- bool ForCallingConv = false) const override {
+ unsigned
+ getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT = std::nullopt) const override {
return 1;
}
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.h b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
index 6aa72bf89213d..f01f7dcc6ab1d 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.h
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.h
@@ -75,14 +75,13 @@ class SystemZTargetLowering : public TargetLowering {
return TypeWidenVector;
return TargetLoweringBase::getPreferredVectorAction(VT);
}
- unsigned getNumRegisters(LLVMContext &Context, EVT VT,
- std::optional<MVT> RegisterVT,
- bool ForCallingConv = false) const override {
+ unsigned
+ getNumRegisters(LLVMContext &Context, EVT VT,
+ std::optional<MVT> RegisterVT) const override {
// i128 inline assembly operand.
if (VT == MVT::i128 && RegisterVT && *RegisterVT == MVT::Untyped)
return 1;
- return TargetLowering::getNumRegisters(
- Context, VT, /*RegisterVT=*/std::nullopt, ForCallingConv);
+ return TargetLowering::getNumRegisters(Context, VT);
}
unsigned
getVectorTypeBreakdownForCallingConv(LLVMContext &Context, CallingConv::ID CC,
>From c6d0db7ed1c3b7be7f82dfa1304a5bb261eb4572 Mon Sep 17 00:00:00 2001
From: Hari Limaye <hari.limaye at arm.com>
Date: Mon, 17 Aug 2026 13:59:09 +0000
Subject: [PATCH 7/7] Fix uses of getRegisterType(MVT VT)
---
llvm/include/llvm/CodeGen/TargetLowering.h | 20 +++++++++++--------
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 8 +++++---
llvm/lib/CodeGen/TargetLoweringBase.cpp | 16 +++++++--------
llvm/lib/Target/Mips/MipsISelLowering.cpp | 2 +-
4 files changed, 25 insertions(+), 21 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 6eb0f347cbe49..a91b575dc82d2 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -1861,12 +1861,6 @@ class LLVM_ABI TargetLoweringBase {
/// arguments in the caller parameter area.
virtual Align getByValTypeAlignment(Type *Ty, const DataLayout &DL) const;
- /// Return the type of registers that this ValueType will eventually require.
- MVT getRegisterType(MVT VT) const {
- assert((unsigned)VT.SimpleTy < std::size(RegisterTypeForVT));
- return RegisterTypeForVT[VT.SimpleTy];
- }
-
/// Return the type of registers that this ValueType will eventually require.
MVT getRegisterType(LLVMContext &Context, EVT VT) const {
return getRegisterTypeImpl(Context, VT, /*ForCallingConv=*/false);
@@ -3972,11 +3966,21 @@ class LLVM_ABI TargetLoweringBase {
MVT &RegisterVT,
bool ForCallingConv) const;
+ unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
+ unsigned &NumIntermediates,
+ MVT &RegisterVT);
+
+ /// Return the type of registers that this ValueType will eventually require.
+ MVT getCachedRegisterType(MVT VT) const {
+ assert((unsigned)VT.SimpleTy < std::size(RegisterTypeForVT));
+ return RegisterTypeForVT[VT.SimpleTy];
+ }
+
MVT getRegisterTypeImpl(LLVMContext &Context, EVT VT,
bool ForCallingConv) const {
if (VT.isSimple() &&
!shouldUseDynamicVectorTypeBreakdown(VT, ForCallingConv))
- return getRegisterType(VT.getSimpleVT());
+ return getCachedRegisterType(VT.getSimpleVT());
if (VT.isVector()) {
EVT VT1;
MVT RegisterVT;
@@ -5247,7 +5251,7 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
/// necessary information.
virtual EVT getTypeForExtReturn(LLVMContext &Context, EVT VT,
ISD::NodeType /*ExtendKind*/) const {
- EVT MinVT = getRegisterType(MVT::i32);
+ EVT MinVT = getRegisterType(Context, MVT::i32);
return VT.bitsLT(MinVT) ? MinVT : VT;
}
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 93869ee001388..70e094bc0d255 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -894,7 +894,8 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
ISD::EXTLOAD, false)) {
// If the source type is not legal, see if there is a legal extload to
// an intermediate type that we can then extend further.
- EVT LoadVT = TLI.getRegisterType(SrcVT.getSimpleVT());
+ EVT LoadVT =
+ TLI.getRegisterType(*DAG.getContext(), SrcVT.getSimpleVT());
if ((LoadVT.isFloatingPoint() == SrcVT.isFloatingPoint()) &&
(TLI.isTypeLegal(SrcVT) || // Same as SrcVT == LoadVT?
TLI.isLoadLegal(LoadVT, SrcVT, LD->getAlign(),
@@ -921,7 +922,8 @@ void SelectionDAGLegalize::LegalizeLoadOps(SDNode *Node) {
if (SVT == MVT::f16 || SVT == MVT::bf16) {
EVT ISrcVT = SrcVT.changeTypeToInteger();
EVT IDestVT = DestVT.changeTypeToInteger();
- EVT ILoadVT = TLI.getRegisterType(IDestVT.getSimpleVT());
+ EVT ILoadVT =
+ TLI.getRegisterType(*DAG.getContext(), IDestVT.getSimpleVT());
SDValue Result = DAG.getExtLoad(ISD::ZEXTLOAD, dl, ILoadVT, Chain,
Ptr, ISrcVT, LD->getMemOperand());
@@ -1646,7 +1648,7 @@ void SelectionDAGLegalize::getSignAsIntValue(FloatSignAsInt &State,
auto &DataLayout = DAG.getDataLayout();
// Store the float to memory, then load the sign part out as an integer.
- MVT LoadTy = TLI.getRegisterType(MVT::i8);
+ MVT LoadTy = TLI.getRegisterType(*DAG.getContext(), MVT::i8);
// First create a temporary that is aligned for both the load and store.
SDValue StackPtr = DAG.CreateStackTemporary(FloatVT, LoadTy);
int FI = cast<FrameIndexSDNode>(StackPtr.getNode())->getIndex();
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index 1810cd0bf83a2..31e3f79080862 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -1236,10 +1236,8 @@ TargetLoweringBase::getTypeConversion(LLVMContext &Context, EVT VT) const {
return LegalizeKind(TypeSplitVector, NVT);
}
-static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
- unsigned &NumIntermediates,
- MVT &RegisterVT,
- TargetLoweringBase *TLI) {
+unsigned TargetLoweringBase::getVectorTypeBreakdownMVT(
+ MVT VT, MVT &IntermediateVT, unsigned &NumIntermediates, MVT &RegisterVT) {
// Figure out the right, legal destination reg to copy into.
ElementCount EC = VT.getVectorElementCount();
MVT EltTy = VT.getVectorElementType();
@@ -1264,7 +1262,7 @@ static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
// always end up with an EC that represent a scalar or a scalable
// scalar.
while (EC.getKnownMinValue() > 1 &&
- !TLI->isTypeLegal(MVT::getVectorVT(EltTy, EC))) {
+ !isTypeLegal(MVT::getVectorVT(EltTy, EC))) {
EC = EC.divideCoefficientBy(2);
NumVectorRegs <<= 1;
}
@@ -1272,7 +1270,7 @@ static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
NumIntermediates = NumVectorRegs;
MVT NewVT = MVT::getVectorVT(EltTy, EC);
- if (!TLI->isTypeLegal(NewVT))
+ if (!isTypeLegal(NewVT))
NewVT = EltTy;
IntermediateVT = NewVT;
@@ -1281,7 +1279,7 @@ static unsigned getVectorTypeBreakdownMVT(MVT VT, MVT &IntermediateVT,
// Convert sizes such as i33 to i64.
LaneSizeInBits = llvm::bit_ceil(LaneSizeInBits);
- MVT DestVT = TLI->getRegisterType(NewVT);
+ MVT DestVT = getCachedRegisterType(NewVT);
RegisterVT = DestVT;
if (EVT(DestVT).bitsLT(NewVT)) // Value is expanded, e.g. i64 -> i16.
return NumVectorRegs * (LaneSizeInBits / DestVT.getScalarSizeInBits());
@@ -1621,8 +1619,8 @@ void TargetLoweringBase::computeRegisterProperties(
MVT IntermediateVT;
MVT RegisterVT;
unsigned NumIntermediates;
- unsigned NumRegisters = getVectorTypeBreakdownMVT(VT, IntermediateVT,
- NumIntermediates, RegisterVT, this);
+ unsigned NumRegisters = getVectorTypeBreakdownMVT(
+ VT, IntermediateVT, NumIntermediates, RegisterVT);
NumRegistersForVT[i] = NumRegisters;
assert(NumRegistersForVT[i] == NumRegisters &&
"NumRegistersForVT size cannot represent NumRegisters!");
diff --git a/llvm/lib/Target/Mips/MipsISelLowering.cpp b/llvm/lib/Target/Mips/MipsISelLowering.cpp
index 076ca31649c43..31e184df78690 100644
--- a/llvm/lib/Target/Mips/MipsISelLowering.cpp
+++ b/llvm/lib/Target/Mips/MipsISelLowering.cpp
@@ -4189,7 +4189,7 @@ static std::pair<bool, bool> parsePhysicalReg(StringRef C, StringRef &Prefix,
EVT MipsTargetLowering::getTypeForExtReturn(LLVMContext &Context, EVT VT,
ISD::NodeType) const {
bool Cond = !Subtarget.isABI_O32() && VT.getSizeInBits() == 32;
- EVT MinVT = getRegisterType(Cond ? MVT::i64 : MVT::i32);
+ EVT MinVT = getRegisterType(Context, Cond ? MVT::i64 : MVT::i32);
return VT.bitsLT(MinVT) ? MinVT : VT;
}
More information about the llvm-commits
mailing list