[llvm] Port `NVPTXTargetLowering::LowerCONCAT_VECTORS` to SelectionDAG (PR #120030)
Ethan Kaji via llvm-commits
llvm-commits at lists.llvm.org
Wed Mar 26 12:42:43 PDT 2025
https://github.com/Esan5 updated https://github.com/llvm/llvm-project/pull/120030
>From fbba746bb4973b12a71b61c133dac524ad0b99c8 Mon Sep 17 00:00:00 2001
From: Ethan Kaji <ethan.kaji at gmail.com>
Date: Sun, 15 Dec 2024 19:31:53 -0600
Subject: [PATCH 01/16] first try
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 26 ++++++++++++++++---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 24 -----------------
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 1 -
3 files changed, 22 insertions(+), 29 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index e2ae31c86bc48..a1a354823a996 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -191,6 +191,7 @@ class SelectionDAGLegalize {
SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
SDValue ExpandInsertToVectorThroughStack(SDValue Op);
SDValue ExpandVectorBuildThroughStack(SDNode* Node);
+ SDValue ExpandConcatVectors(SDNode* Node);
SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
SDValue ExpandConstant(ConstantSDNode *CP);
@@ -1525,10 +1526,27 @@ SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
BaseVecAlignment);
}
+SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
+ assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
+ SDLoc Dl(Node);
+ SmallVector<SDValue, 0> Ops;
+ unsigned NumOperands = Node->getNumOperands();
+ for (unsigned I = 0; I < NumOperands; ++I) {
+ SDValue SubOp = Node->getOperand(I);
+ EVT VectorValueType =
+ SubOp->getValueType(0);
+ EVT ElementValueType = VectorValueType.getVectorElementType();
+ unsigned NumSubElem = VectorValueType.getVectorNumElements();
+ for (unsigned J = 0; J < NumSubElem; ++J) {
+ Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, Dl, ElementValueType,
+ SubOp, DAG.getIntPtrConstant(J, Dl)));
+ }
+ }
+ return DAG.getBuildVector(Node->getValueType(0), Dl, Ops);
+}
+
SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
- assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
- Node->getOpcode() == ISD::CONCAT_VECTORS) &&
- "Unexpected opcode!");
+ assert(Node->getOpcode() == ISD::BUILD_VECTOR && "Unexpected opcode!");
// We can't handle this case efficiently. Allocate a sufficiently
// aligned object on the stack, store each operand into it, then load
@@ -3383,7 +3401,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
break;
case ISD::CONCAT_VECTORS:
- Results.push_back(ExpandVectorBuildThroughStack(Node));
+ Results.push_back(ExpandConcatVectors(Node));
break;
case ISD::SCALAR_TO_VECTOR:
Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 06e221777b7ea..0384903dd2cac 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -2111,28 +2111,6 @@ SDValue NVPTXTargetLowering::LowerSTACKSAVE(SDValue Op,
return DAG.getMergeValues({ASC, SDValue(SS.getNode(), 1)}, DL);
}
-// By default CONCAT_VECTORS is lowered by ExpandVectorBuildThroughStack()
-// (see LegalizeDAG.cpp). This is slow and uses local memory.
-// We use extract/insert/build vector just as what LegalizeOp() does in llvm 2.5
-SDValue
-NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
- SDNode *Node = Op.getNode();
- SDLoc dl(Node);
- SmallVector<SDValue, 8> Ops;
- unsigned NumOperands = Node->getNumOperands();
- for (unsigned i = 0; i < NumOperands; ++i) {
- SDValue SubOp = Node->getOperand(i);
- EVT VVT = SubOp.getNode()->getValueType(0);
- EVT EltVT = VVT.getVectorElementType();
- unsigned NumSubElem = VVT.getVectorNumElements();
- for (unsigned j = 0; j < NumSubElem; ++j) {
- Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, SubOp,
- DAG.getIntPtrConstant(j, dl)));
- }
- }
- return DAG.getBuildVector(Node->getValueType(0), dl, Ops);
-}
-
SDValue NVPTXTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
// Handle bitcasting from v2i8 without hitting the default promotion
// strategy which goes through stack memory.
@@ -2846,8 +2824,6 @@ NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
return LowerINSERT_VECTOR_ELT(Op, DAG);
case ISD::VECTOR_SHUFFLE:
return LowerVECTOR_SHUFFLE(Op, DAG);
- case ISD::CONCAT_VECTORS:
- return LowerCONCAT_VECTORS(Op, DAG);
case ISD::STORE:
return LowerSTORE(Op, DAG);
case ISD::LOAD:
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 39470be254efa..9202e0773fbd0 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -292,7 +292,6 @@ class NVPTXTargetLowering : public TargetLowering {
SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
- SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
>From 34be6571a09aa0cbe56671264fa098a0b65c03be Mon Sep 17 00:00:00 2001
From: Ethan Kaji <ethan.kaji at gmail.com>
Date: Mon, 16 Dec 2024 19:17:29 -0600
Subject: [PATCH 02/16] fix tests
---
.../sve-streaming-mode-fixed-length-concat.ll | 82 +++++++++++++------
...ve-streaming-mode-fixed-length-int-mulh.ll | 72 +++++++---------
...treaming-mode-fixed-length-trunc-stores.ll | 4 +-
3 files changed, 86 insertions(+), 72 deletions(-)
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
index 6ec2b837eed2a..1b5a37bfb932b 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
@@ -70,9 +70,12 @@ define <16 x i8> @concat_v16i8(<8 x i8> %op1, <8 x i8> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v16i8:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
+; NONEON-NOSVE-NEXT: ldp x8, x9, [sp]
+; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <8 x i8> %op1, <8 x i8> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -181,9 +184,12 @@ define <8 x i16> @concat_v8i16(<4 x i16> %op1, <4 x i16> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v8i16:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
+; NONEON-NOSVE-NEXT: ldp x8, x9, [sp]
+; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <4 x i16> %op1, <4 x i16> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
ret <8 x i16> %res
@@ -279,9 +285,14 @@ define <4 x i32> @concat_v4i32(<2 x i32> %op1, <2 x i32> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v4i32:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
+; NONEON-NOSVE-NEXT: ldp w8, w9, [sp, #8]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #24]
+; NONEON-NOSVE-NEXT: ldp w8, w9, [sp]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <2 x i32> %op1, <2 x i32> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
ret <4 x i32> %res
@@ -441,9 +452,12 @@ define <8 x half> @concat_v8f16(<4 x half> %op1, <4 x half> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v8f16:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
+; NONEON-NOSVE-NEXT: ldp x8, x9, [sp]
+; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <4 x half> %op1, <4 x half> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
ret <8 x half> %res
@@ -539,9 +553,14 @@ define <4 x float> @concat_v4f32(<2 x float> %op1, <2 x float> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v4f32:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
+; NONEON-NOSVE-NEXT: ldp s0, s1, [sp, #8]
+; NONEON-NOSVE-NEXT: stp s0, s1, [sp, #24]
+; NONEON-NOSVE-NEXT: ldp s0, s1, [sp]
+; NONEON-NOSVE-NEXT: stp s0, s1, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <2 x float> %op1, <2 x float> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
ret <4 x float> %res
@@ -754,12 +773,15 @@ define void @concat_v32i8_4op(ptr %a, ptr %b) {
;
; NONEON-NOSVE-LABEL: concat_v32i8_4op:
; NONEON-NOSVE: // %bb.0:
+; NONEON-NOSVE-NEXT: sub sp, sp, #32
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr d0, [x0]
-; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldr q0, [sp]
+; NONEON-NOSVE-NEXT: str d0, [sp, #8]
+; NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; NONEON-NOSVE-NEXT: str x8, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: str q0, [x1]
-; NONEON-NOSVE-NEXT: add sp, sp, #16
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%shuffle = shufflevector <8 x i8> %op1, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -781,12 +803,15 @@ define void @concat_v16i16_4op(ptr %a, ptr %b) {
;
; NONEON-NOSVE-LABEL: concat_v16i16_4op:
; NONEON-NOSVE: // %bb.0:
+; NONEON-NOSVE-NEXT: sub sp, sp, #32
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr d0, [x0]
-; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldr q0, [sp]
+; NONEON-NOSVE-NEXT: str d0, [sp, #8]
+; NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; NONEON-NOSVE-NEXT: str x8, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: str q0, [x1]
-; NONEON-NOSVE-NEXT: add sp, sp, #16
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%op1 = load <4 x i16>, ptr %a
%shuffle = shufflevector <4 x i16> %op1, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -805,12 +830,15 @@ define void @concat_v8i32_4op(ptr %a, ptr %b) {
;
; NONEON-NOSVE-LABEL: concat_v8i32_4op:
; NONEON-NOSVE: // %bb.0:
+; NONEON-NOSVE-NEXT: sub sp, sp, #32
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr d0, [x0]
-; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldr q0, [sp]
+; NONEON-NOSVE-NEXT: str d0, [sp, #8]
+; NONEON-NOSVE-NEXT: ldp w8, w9, [sp, #8]
+; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: str q0, [x1]
-; NONEON-NOSVE-NEXT: add sp, sp, #16
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%op1 = load <2 x i32>, ptr %a
%shuffle = shufflevector <2 x i32> %op1, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll
index 97f2e7a1e66cb..6510fb4818ef8 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll
@@ -1132,17 +1132,15 @@ define <2 x i64> @smulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
;
; NONEON-NOSVE-LABEL: smulh_v2i64:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp q0, q1, [sp, #-64]!
+; NONEON-NOSVE-NEXT: sub sp, sp, #64
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
-; NONEON-NOSVE-NEXT: ldp x9, x8, [sp]
-; NONEON-NOSVE-NEXT: ldp x11, x10, [sp, #16]
+; NONEON-NOSVE-NEXT: stp q0, q1, [sp, #16]
+; NONEON-NOSVE-NEXT: ldp x9, x8, [sp, #16]
+; NONEON-NOSVE-NEXT: ldp x11, x10, [sp, #32]
; NONEON-NOSVE-NEXT: smulh x8, x8, x10
; NONEON-NOSVE-NEXT: smulh x9, x9, x11
-; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #32]
-; NONEON-NOSVE-NEXT: ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #48]
-; NONEON-NOSVE-NEXT: add sp, sp, #64
+; NONEON-NOSVE-NEXT: stp x9, x8, [sp]
+; NONEON-NOSVE-NEXT: ldr q0, [sp], #64
; NONEON-NOSVE-NEXT: ret
%1 = sext <2 x i64> %op1 to <2 x i128>
%2 = sext <2 x i64> %op2 to <2 x i128>
@@ -1179,23 +1177,19 @@ define void @smulh_v4i64(ptr %a, ptr %b) {
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 128
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT: ldp q2, q3, [x1]
-; NONEON-NOSVE-NEXT: stp q1, q2, [sp]
-; NONEON-NOSVE-NEXT: ldp x11, x10, [sp]
-; NONEON-NOSVE-NEXT: stp q0, q3, [sp, #32]
-; NONEON-NOSVE-NEXT: ldp x13, x12, [sp, #16]
-; NONEON-NOSVE-NEXT: ldp x9, x8, [sp, #32]
+; NONEON-NOSVE-NEXT: stp q1, q2, [sp, #32]
+; NONEON-NOSVE-NEXT: ldp x11, x10, [sp, #32]
+; NONEON-NOSVE-NEXT: stp q0, q3, [sp, #64]
+; NONEON-NOSVE-NEXT: ldp x13, x12, [sp, #48]
+; NONEON-NOSVE-NEXT: ldp x9, x8, [sp, #64]
; NONEON-NOSVE-NEXT: smulh x10, x10, x12
-; NONEON-NOSVE-NEXT: ldp x14, x12, [sp, #48]
+; NONEON-NOSVE-NEXT: ldp x14, x12, [sp, #80]
; NONEON-NOSVE-NEXT: smulh x11, x11, x13
; NONEON-NOSVE-NEXT: smulh x8, x8, x12
; NONEON-NOSVE-NEXT: smulh x9, x9, x14
-; NONEON-NOSVE-NEXT: stp x11, x10, [sp, #64]
-; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #80]
-; NONEON-NOSVE-NEXT: ldp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #112]
-; NONEON-NOSVE-NEXT: ldp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #96]
-; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #96]
+; NONEON-NOSVE-NEXT: stp x11, x10, [sp, #16]
+; NONEON-NOSVE-NEXT: stp x9, x8, [sp]
+; NONEON-NOSVE-NEXT: ldp q1, q0, [sp]
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
; NONEON-NOSVE-NEXT: add sp, sp, #128
; NONEON-NOSVE-NEXT: ret
@@ -2331,17 +2325,15 @@ define <2 x i64> @umulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
;
; NONEON-NOSVE-LABEL: umulh_v2i64:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp q0, q1, [sp, #-64]!
+; NONEON-NOSVE-NEXT: sub sp, sp, #64
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
-; NONEON-NOSVE-NEXT: ldp x9, x8, [sp]
-; NONEON-NOSVE-NEXT: ldp x11, x10, [sp, #16]
+; NONEON-NOSVE-NEXT: stp q0, q1, [sp, #16]
+; NONEON-NOSVE-NEXT: ldp x9, x8, [sp, #16]
+; NONEON-NOSVE-NEXT: ldp x11, x10, [sp, #32]
; NONEON-NOSVE-NEXT: umulh x8, x8, x10
; NONEON-NOSVE-NEXT: umulh x9, x9, x11
-; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #32]
-; NONEON-NOSVE-NEXT: ldp d0, d1, [sp, #32]
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #48]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #48]
-; NONEON-NOSVE-NEXT: add sp, sp, #64
+; NONEON-NOSVE-NEXT: stp x9, x8, [sp]
+; NONEON-NOSVE-NEXT: ldr q0, [sp], #64
; NONEON-NOSVE-NEXT: ret
%1 = zext <2 x i64> %op1 to <2 x i128>
%2 = zext <2 x i64> %op2 to <2 x i128>
@@ -2378,23 +2370,19 @@ define void @umulh_v4i64(ptr %a, ptr %b) {
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 128
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT: ldp q2, q3, [x1]
-; NONEON-NOSVE-NEXT: stp q1, q2, [sp]
-; NONEON-NOSVE-NEXT: ldp x11, x10, [sp]
-; NONEON-NOSVE-NEXT: stp q0, q3, [sp, #32]
-; NONEON-NOSVE-NEXT: ldp x13, x12, [sp, #16]
-; NONEON-NOSVE-NEXT: ldp x9, x8, [sp, #32]
+; NONEON-NOSVE-NEXT: stp q1, q2, [sp, #32]
+; NONEON-NOSVE-NEXT: ldp x11, x10, [sp, #32]
+; NONEON-NOSVE-NEXT: stp q0, q3, [sp, #64]
+; NONEON-NOSVE-NEXT: ldp x13, x12, [sp, #48]
+; NONEON-NOSVE-NEXT: ldp x9, x8, [sp, #64]
; NONEON-NOSVE-NEXT: umulh x10, x10, x12
-; NONEON-NOSVE-NEXT: ldp x14, x12, [sp, #48]
+; NONEON-NOSVE-NEXT: ldp x14, x12, [sp, #80]
; NONEON-NOSVE-NEXT: umulh x11, x11, x13
; NONEON-NOSVE-NEXT: umulh x8, x8, x12
; NONEON-NOSVE-NEXT: umulh x9, x9, x14
-; NONEON-NOSVE-NEXT: stp x11, x10, [sp, #64]
-; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #80]
-; NONEON-NOSVE-NEXT: ldp d0, d1, [sp, #80]
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #112]
-; NONEON-NOSVE-NEXT: ldp d0, d1, [sp, #64]
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #96]
-; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #96]
+; NONEON-NOSVE-NEXT: stp x11, x10, [sp, #16]
+; NONEON-NOSVE-NEXT: stp x9, x8, [sp]
+; NONEON-NOSVE-NEXT: ldp q1, q0, [sp]
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
; NONEON-NOSVE-NEXT: add sp, sp, #128
; NONEON-NOSVE-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll
index 13fcd94ea8a26..ae87128b5c3f9 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll
@@ -142,9 +142,7 @@ define void @store_trunc_v2i256i64(ptr %ap, ptr %dest) {
; NONEON-NOSVE-NEXT: ldr x9, [x0]
; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #-32]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT: ldp d0, d1, [sp]
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #16]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp]
; NONEON-NOSVE-NEXT: str q0, [x1]
; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
>From 22ecb84c37a4c8c38049e8c212fd8926b7721834 Mon Sep 17 00:00:00 2001
From: Ethan Kaji <ethan.kaji at gmail.com>
Date: Mon, 16 Dec 2024 22:07:02 -0600
Subject: [PATCH 03/16] comments
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 5 +++--
1 file changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index a1a354823a996..9dd2cd74c171c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1531,15 +1531,16 @@ SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
SDLoc Dl(Node);
SmallVector<SDValue, 0> Ops;
unsigned NumOperands = Node->getNumOperands();
+ MVT VectorIdxType = TLI.getVectorIdxTy(DAG.getDataLayout());
for (unsigned I = 0; I < NumOperands; ++I) {
SDValue SubOp = Node->getOperand(I);
EVT VectorValueType =
SubOp->getValueType(0);
EVT ElementValueType = VectorValueType.getVectorElementType();
unsigned NumSubElem = VectorValueType.getVectorNumElements();
- for (unsigned J = 0; J < NumSubElem; ++J) {
+ for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, Dl, ElementValueType,
- SubOp, DAG.getIntPtrConstant(J, Dl)));
+ SubOp, DAG.getConstant(Idx, Dl, VectorIdxType)));
}
}
return DAG.getBuildVector(Node->getValueType(0), Dl, Ops);
>From 2d1bfd7febf07289aac228947090903557df4d85 Mon Sep 17 00:00:00 2001
From: Ethan Kaji <ethan.kaji at gmail.com>
Date: Mon, 16 Dec 2024 22:49:07 -0600
Subject: [PATCH 04/16] prefer size 16 in small vector
Co-authored-by: Matt Arsenault <arsenm2 at gmail.com>
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 9dd2cd74c171c..bb40491bd4022 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1529,7 +1529,7 @@ SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
SDLoc Dl(Node);
- SmallVector<SDValue, 0> Ops;
+ SmallVector<SDValue, 16> Ops;
unsigned NumOperands = Node->getNumOperands();
MVT VectorIdxType = TLI.getVectorIdxTy(DAG.getDataLayout());
for (unsigned I = 0; I < NumOperands; ++I) {
>From 4e72483f5471d8d53f1280692cecb74892b99677 Mon Sep 17 00:00:00 2001
From: Ethan Kaji <ethan.kaji at gmail.com>
Date: Tue, 17 Dec 2024 15:54:49 -0600
Subject: [PATCH 05/16] trigger builds
>From ade270716add7d27dd7299f25012e59bfdf52301 Mon Sep 17 00:00:00 2001
From: Ethan Kaji <ethan.kaji at gmail.com>
Date: Tue, 17 Dec 2024 20:36:49 -0600
Subject: [PATCH 06/16] comments
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 11 +++++------
1 file changed, 5 insertions(+), 6 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index bb40491bd4022..30a5c06757ee1 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1528,22 +1528,21 @@ SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
- SDLoc Dl(Node);
+ SDLoc DL(Node);
SmallVector<SDValue, 16> Ops;
unsigned NumOperands = Node->getNumOperands();
MVT VectorIdxType = TLI.getVectorIdxTy(DAG.getDataLayout());
for (unsigned I = 0; I < NumOperands; ++I) {
SDValue SubOp = Node->getOperand(I);
- EVT VectorValueType =
- SubOp->getValueType(0);
+ EVT VectorValueType = SubOp.getValueType();
EVT ElementValueType = VectorValueType.getVectorElementType();
unsigned NumSubElem = VectorValueType.getVectorNumElements();
for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
- Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, Dl, ElementValueType,
- SubOp, DAG.getConstant(Idx, Dl, VectorIdxType)));
+ Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElementValueType,
+ SubOp, DAG.getConstant(Idx, DL, VectorIdxType)));
}
}
- return DAG.getBuildVector(Node->getValueType(0), Dl, Ops);
+ return DAG.getBuildVector(Node->getValueType(0), DL, Ops);
}
SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
>From 5682483dd9dae2bfd01d6ac3e22cf0ff546371ae Mon Sep 17 00:00:00 2001
From: Ethan Kaji <ethan.kaji at gmail.com>
Date: Thu, 19 Dec 2024 18:04:23 -0600
Subject: [PATCH 07/16] extract to legal type
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 30a5c06757ee1..483e328a5a09a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -191,7 +191,7 @@ class SelectionDAGLegalize {
SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
SDValue ExpandInsertToVectorThroughStack(SDValue Op);
SDValue ExpandVectorBuildThroughStack(SDNode* Node);
- SDValue ExpandConcatVectors(SDNode* Node);
+ SDValue ExpandConcatVectors(SDNode *Node);
SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
SDValue ExpandConstant(ConstantSDNode *CP);
@@ -1535,11 +1535,13 @@ SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
for (unsigned I = 0; I < NumOperands; ++I) {
SDValue SubOp = Node->getOperand(I);
EVT VectorValueType = SubOp.getValueType();
- EVT ElementValueType = VectorValueType.getVectorElementType();
+ EVT ElementValueType = TLI.getTypeToTransformTo(
+ *DAG.getContext(), VectorValueType.getVectorElementType());
unsigned NumSubElem = VectorValueType.getVectorNumElements();
for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElementValueType,
- SubOp, DAG.getConstant(Idx, DL, VectorIdxType)));
+ SubOp,
+ DAG.getConstant(Idx, DL, VectorIdxType)));
}
}
return DAG.getBuildVector(Node->getValueType(0), DL, Ops);
@@ -1553,8 +1555,7 @@ SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
// the result as a vector.
// Create the stack frame object.
EVT VT = Node->getValueType(0);
- EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
- : Node->getOperand(0).getValueType();
+ EVT MemVT = VT.getVectorElementType();
SDLoc dl(Node);
SDValue FIPtr = DAG.CreateStackTemporary(VT);
int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
@@ -1568,8 +1569,7 @@ SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
// If the destination vector element type of a BUILD_VECTOR is narrower than
// the source element type, only store the bits necessary.
- bool Truncate = isa<BuildVectorSDNode>(Node) &&
- MemVT.bitsLT(Node->getOperand(0).getValueType());
+ bool Truncate = MemVT.bitsLT(Node->getOperand(0).getValueType());
// Store (in the right endianness) the elements to memory.
for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
>From 1ac6bd54360f34e8149c1f4deb1827d7730548ce Mon Sep 17 00:00:00 2001
From: Ethan Kaji <ethan.kaji at gmail.com>
Date: Fri, 20 Dec 2024 14:11:12 -0600
Subject: [PATCH 08/16] use build through stack if type changes
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 19 ++++++---
.../sve-streaming-mode-fixed-length-concat.ll | 40 +++++++------------
2 files changed, 27 insertions(+), 32 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 483e328a5a09a..9e68f92f2633c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1532,11 +1532,14 @@ SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
SmallVector<SDValue, 16> Ops;
unsigned NumOperands = Node->getNumOperands();
MVT VectorIdxType = TLI.getVectorIdxTy(DAG.getDataLayout());
+ EVT VectorValueType = Node->getOperand(0).getValueType();
+ EVT ElementValueType = VectorValueType.getVectorElementType();
+ if (ElementValueType !=
+ TLI.getTypeToTransformTo(*DAG.getContext(), ElementValueType)) {
+ return ExpandVectorBuildThroughStack(Node);
+ }
for (unsigned I = 0; I < NumOperands; ++I) {
SDValue SubOp = Node->getOperand(I);
- EVT VectorValueType = SubOp.getValueType();
- EVT ElementValueType = TLI.getTypeToTransformTo(
- *DAG.getContext(), VectorValueType.getVectorElementType());
unsigned NumSubElem = VectorValueType.getVectorNumElements();
for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElementValueType,
@@ -1548,14 +1551,17 @@ SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
}
SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
- assert(Node->getOpcode() == ISD::BUILD_VECTOR && "Unexpected opcode!");
+ assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
+ Node->getOpcode() == ISD::CONCAT_VECTORS) &&
+ "Unexpected opcode!");
// We can't handle this case efficiently. Allocate a sufficiently
// aligned object on the stack, store each operand into it, then load
// the result as a vector.
// Create the stack frame object.
EVT VT = Node->getValueType(0);
- EVT MemVT = VT.getVectorElementType();
+ EVT MemVT = isa<BuildVectorSDNode>(Node) ? VT.getVectorElementType()
+ : Node->getOperand(0).getValueType();
SDLoc dl(Node);
SDValue FIPtr = DAG.CreateStackTemporary(VT);
int FI = cast<FrameIndexSDNode>(FIPtr.getNode())->getIndex();
@@ -1569,7 +1575,8 @@ SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
// If the destination vector element type of a BUILD_VECTOR is narrower than
// the source element type, only store the bits necessary.
- bool Truncate = MemVT.bitsLT(Node->getOperand(0).getValueType());
+ bool Truncate = isa<BuildVectorSDNode>(Node) &&
+ MemVT.bitsLT(Node->getOperand(0).getValueType());
// Store (in the right endianness) the elements to memory.
for (unsigned i = 0, e = Node->getNumOperands(); i != e; ++i) {
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
index 1b5a37bfb932b..3997bf16c2aaf 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
@@ -70,12 +70,9 @@ define <16 x i8> @concat_v16i8(<8 x i8> %op1, <8 x i8> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v16i8:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT: ldp x8, x9, [sp]
-; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #16]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
-; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <8 x i8> %op1, <8 x i8> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -184,12 +181,9 @@ define <8 x i16> @concat_v8i16(<4 x i16> %op1, <4 x i16> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v8i16:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT: ldp x8, x9, [sp]
-; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #16]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
-; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <4 x i16> %op1, <4 x i16> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
ret <8 x i16> %res
@@ -773,15 +767,12 @@ define void @concat_v32i8_4op(ptr %a, ptr %b) {
;
; NONEON-NOSVE-LABEL: concat_v32i8_4op:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: sub sp, sp, #32
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr d0, [x0]
-; NONEON-NOSVE-NEXT: str d0, [sp, #8]
-; NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
-; NONEON-NOSVE-NEXT: str x8, [sp, #16]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldr q0, [sp]
; NONEON-NOSVE-NEXT: str q0, [x1]
-; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%shuffle = shufflevector <8 x i8> %op1, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -803,15 +794,12 @@ define void @concat_v16i16_4op(ptr %a, ptr %b) {
;
; NONEON-NOSVE-LABEL: concat_v16i16_4op:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: sub sp, sp, #32
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr d0, [x0]
-; NONEON-NOSVE-NEXT: str d0, [sp, #8]
-; NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
-; NONEON-NOSVE-NEXT: str x8, [sp, #16]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldr q0, [sp]
; NONEON-NOSVE-NEXT: str q0, [x1]
-; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
%op1 = load <4 x i16>, ptr %a
%shuffle = shufflevector <4 x i16> %op1, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
>From d184705a05c44340f037bf50beac033629cea1c0 Mon Sep 17 00:00:00 2001
From: Ethan Kaji <ethan.kaji at gmail.com>
Date: Fri, 20 Dec 2024 14:50:17 -0600
Subject: [PATCH 09/16] just checking if the type changed is not sufficient
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 8 +---
.../sve-streaming-mode-fixed-length-concat.ll | 40 ++++++++++++-------
2 files changed, 28 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 9e68f92f2633c..f36a5fda9904e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1533,14 +1533,10 @@ SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
unsigned NumOperands = Node->getNumOperands();
MVT VectorIdxType = TLI.getVectorIdxTy(DAG.getDataLayout());
EVT VectorValueType = Node->getOperand(0).getValueType();
- EVT ElementValueType = VectorValueType.getVectorElementType();
- if (ElementValueType !=
- TLI.getTypeToTransformTo(*DAG.getContext(), ElementValueType)) {
- return ExpandVectorBuildThroughStack(Node);
- }
+ unsigned NumSubElem = VectorValueType.getVectorNumElements();
+ EVT ElementValueType = TLI.getTypeToTransformTo(*DAG.getContext(), VectorValueType.getVectorElementType());
for (unsigned I = 0; I < NumOperands; ++I) {
SDValue SubOp = Node->getOperand(I);
- unsigned NumSubElem = VectorValueType.getVectorNumElements();
for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElementValueType,
SubOp,
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
index 3997bf16c2aaf..1b5a37bfb932b 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
@@ -70,9 +70,12 @@ define <16 x i8> @concat_v16i8(<8 x i8> %op1, <8 x i8> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v16i8:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
+; NONEON-NOSVE-NEXT: ldp x8, x9, [sp]
+; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <8 x i8> %op1, <8 x i8> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -181,9 +184,12 @@ define <8 x i16> @concat_v8i16(<4 x i16> %op1, <4 x i16> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v8i16:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
+; NONEON-NOSVE-NEXT: ldp x8, x9, [sp]
+; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <4 x i16> %op1, <4 x i16> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
ret <8 x i16> %res
@@ -767,12 +773,15 @@ define void @concat_v32i8_4op(ptr %a, ptr %b) {
;
; NONEON-NOSVE-LABEL: concat_v32i8_4op:
; NONEON-NOSVE: // %bb.0:
+; NONEON-NOSVE-NEXT: sub sp, sp, #32
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr d0, [x0]
-; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldr q0, [sp]
+; NONEON-NOSVE-NEXT: str d0, [sp, #8]
+; NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; NONEON-NOSVE-NEXT: str x8, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: str q0, [x1]
-; NONEON-NOSVE-NEXT: add sp, sp, #16
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%shuffle = shufflevector <8 x i8> %op1, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -794,12 +803,15 @@ define void @concat_v16i16_4op(ptr %a, ptr %b) {
;
; NONEON-NOSVE-LABEL: concat_v16i16_4op:
; NONEON-NOSVE: // %bb.0:
+; NONEON-NOSVE-NEXT: sub sp, sp, #32
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr d0, [x0]
-; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
-; NONEON-NOSVE-NEXT: ldr q0, [sp]
+; NONEON-NOSVE-NEXT: str d0, [sp, #8]
+; NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
+; NONEON-NOSVE-NEXT: str x8, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: str q0, [x1]
-; NONEON-NOSVE-NEXT: add sp, sp, #16
+; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
%op1 = load <4 x i16>, ptr %a
%shuffle = shufflevector <4 x i16> %op1, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
>From cef586ae3a57a9496cddf109c42e76e62e77a4d1 Mon Sep 17 00:00:00 2001
From: Ethan Kaji <ethan.kaji at gmail.com>
Date: Sat, 21 Dec 2024 15:50:34 -0600
Subject: [PATCH 10/16] use ExpandVectorBuildThroughStack if extract vector elt
is not is not legal or custom.
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 12 +++++-
.../sve-streaming-mode-fixed-length-concat.ll | 40 +++++++------------
2 files changed, 24 insertions(+), 28 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index f36a5fda9904e..9f91cc94b2af9 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1534,7 +1534,8 @@ SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
MVT VectorIdxType = TLI.getVectorIdxTy(DAG.getDataLayout());
EVT VectorValueType = Node->getOperand(0).getValueType();
unsigned NumSubElem = VectorValueType.getVectorNumElements();
- EVT ElementValueType = TLI.getTypeToTransformTo(*DAG.getContext(), VectorValueType.getVectorElementType());
+ EVT ElementValueType = TLI.getTypeToTransformTo(
+ *DAG.getContext(), VectorValueType.getVectorElementType());
for (unsigned I = 0; I < NumOperands; ++I) {
SDValue SubOp = Node->getOperand(I);
for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
@@ -3404,7 +3405,14 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
break;
case ISD::CONCAT_VECTORS:
- Results.push_back(ExpandConcatVectors(Node));
+ if (EVT ElementValueType =
+ Node->getOperand(0).getValueType().getVectorElementType();
+ TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT,
+ ElementValueType)) {
+ Results.push_back(ExpandConcatVectors(Node));
+ } else {
+ Results.push_back(ExpandVectorBuildThroughStack(Node));
+ }
break;
case ISD::SCALAR_TO_VECTOR:
Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
index 1b5a37bfb932b..3997bf16c2aaf 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
@@ -70,12 +70,9 @@ define <16 x i8> @concat_v16i8(<8 x i8> %op1, <8 x i8> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v16i8:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT: ldp x8, x9, [sp]
-; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #16]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
-; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <8 x i8> %op1, <8 x i8> %op2, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
@@ -184,12 +181,9 @@ define <8 x i16> @concat_v8i16(<4 x i16> %op1, <4 x i16> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v8i16:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT: ldp x8, x9, [sp]
-; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #16]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
-; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <4 x i16> %op1, <4 x i16> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
ret <8 x i16> %res
@@ -773,15 +767,12 @@ define void @concat_v32i8_4op(ptr %a, ptr %b) {
;
; NONEON-NOSVE-LABEL: concat_v32i8_4op:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: sub sp, sp, #32
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr d0, [x0]
-; NONEON-NOSVE-NEXT: str d0, [sp, #8]
-; NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
-; NONEON-NOSVE-NEXT: str x8, [sp, #16]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldr q0, [sp]
; NONEON-NOSVE-NEXT: str q0, [x1]
-; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
%op1 = load <8 x i8>, ptr %a
%shuffle = shufflevector <8 x i8> %op1, <8 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7,
@@ -803,15 +794,12 @@ define void @concat_v16i16_4op(ptr %a, ptr %b) {
;
; NONEON-NOSVE-LABEL: concat_v16i16_4op:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: sub sp, sp, #32
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr d0, [x0]
-; NONEON-NOSVE-NEXT: str d0, [sp, #8]
-; NONEON-NOSVE-NEXT: ldr x8, [sp, #8]
-; NONEON-NOSVE-NEXT: str x8, [sp, #16]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldr q0, [sp]
; NONEON-NOSVE-NEXT: str q0, [x1]
-; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
%op1 = load <4 x i16>, ptr %a
%shuffle = shufflevector <4 x i16> %op1, <4 x i16> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
>From cb1bfeaf759fbd841c7a556f303a0513bdfeb6c8 Mon Sep 17 00:00:00 2001
From: Ethan Kaji <97700380+Esan5 at users.noreply.github.com>
Date: Tue, 21 Jan 2025 15:49:54 -0500
Subject: [PATCH 11/16] try ExpandConcatVectors if extract_vector_elt is not an
Expand operation
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 5 +-
.../sve-streaming-mode-fixed-length-concat.ll | 42 ++++-------
...ve-streaming-mode-fixed-length-int-mulh.ll | 72 +++++++++++--------
...treaming-mode-fixed-length-trunc-stores.ll | 4 +-
4 files changed, 59 insertions(+), 64 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 9f91cc94b2af9..f86494753dfd6 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3405,10 +3405,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
break;
case ISD::CONCAT_VECTORS:
- if (EVT ElementValueType =
- Node->getOperand(0).getValueType().getVectorElementType();
- TLI.isOperationLegalOrCustom(ISD::EXTRACT_VECTOR_ELT,
- ElementValueType)) {
+ if (!TLI.isOperationExpand(ISD::EXTRACT_VECTOR_ELT, Node->getOperand(0).getValueType())) {
Results.push_back(ExpandConcatVectors(Node));
} else {
Results.push_back(ExpandVectorBuildThroughStack(Node));
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
index 3997bf16c2aaf..6ec2b837eed2a 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-concat.ll
@@ -279,14 +279,9 @@ define <4 x i32> @concat_v4i32(<2 x i32> %op1, <2 x i32> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v4i32:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT: ldp w8, w9, [sp, #8]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #24]
-; NONEON-NOSVE-NEXT: ldp w8, w9, [sp]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #16]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
-; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <2 x i32> %op1, <2 x i32> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
ret <4 x i32> %res
@@ -446,12 +441,9 @@ define <8 x half> @concat_v8f16(<4 x half> %op1, <4 x half> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v8f16:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT: ldp x8, x9, [sp]
-; NONEON-NOSVE-NEXT: stp x8, x9, [sp, #16]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
-; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <4 x half> %op1, <4 x half> %op2, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
ret <8 x half> %res
@@ -547,14 +539,9 @@ define <4 x float> @concat_v4f32(<2 x float> %op1, <2 x float> %op2) {
;
; NONEON-NOSVE-LABEL: concat_v4f32:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-32]!
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT: ldp s0, s1, [sp, #8]
-; NONEON-NOSVE-NEXT: stp s0, s1, [sp, #24]
-; NONEON-NOSVE-NEXT: ldp s0, s1, [sp]
-; NONEON-NOSVE-NEXT: stp s0, s1, [sp, #16]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
-; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldr q0, [sp], #16
; NONEON-NOSVE-NEXT: ret
%res = shufflevector <2 x float> %op1, <2 x float> %op2, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
ret <4 x float> %res
@@ -818,15 +805,12 @@ define void @concat_v8i32_4op(ptr %a, ptr %b) {
;
; NONEON-NOSVE-LABEL: concat_v8i32_4op:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: sub sp, sp, #32
-; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
; NONEON-NOSVE-NEXT: ldr d0, [x0]
-; NONEON-NOSVE-NEXT: str d0, [sp, #8]
-; NONEON-NOSVE-NEXT: ldp w8, w9, [sp, #8]
-; NONEON-NOSVE-NEXT: stp w8, w9, [sp, #16]
-; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
+; NONEON-NOSVE-NEXT: str d0, [sp, #-16]!
+; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 16
+; NONEON-NOSVE-NEXT: ldr q0, [sp]
; NONEON-NOSVE-NEXT: str q0, [x1]
-; NONEON-NOSVE-NEXT: add sp, sp, #32
+; NONEON-NOSVE-NEXT: add sp, sp, #16
; NONEON-NOSVE-NEXT: ret
%op1 = load <2 x i32>, ptr %a
%shuffle = shufflevector <2 x i32> %op1, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll
index 6510fb4818ef8..97f2e7a1e66cb 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-int-mulh.ll
@@ -1132,15 +1132,17 @@ define <2 x i64> @smulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
;
; NONEON-NOSVE-LABEL: smulh_v2i64:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: sub sp, sp, #64
+; NONEON-NOSVE-NEXT: stp q0, q1, [sp, #-64]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
-; NONEON-NOSVE-NEXT: stp q0, q1, [sp, #16]
-; NONEON-NOSVE-NEXT: ldp x9, x8, [sp, #16]
-; NONEON-NOSVE-NEXT: ldp x11, x10, [sp, #32]
+; NONEON-NOSVE-NEXT: ldp x9, x8, [sp]
+; NONEON-NOSVE-NEXT: ldp x11, x10, [sp, #16]
; NONEON-NOSVE-NEXT: smulh x8, x8, x10
; NONEON-NOSVE-NEXT: smulh x9, x9, x11
-; NONEON-NOSVE-NEXT: stp x9, x8, [sp]
-; NONEON-NOSVE-NEXT: ldr q0, [sp], #64
+; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #32]
+; NONEON-NOSVE-NEXT: ldp d0, d1, [sp, #32]
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #48]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #48]
+; NONEON-NOSVE-NEXT: add sp, sp, #64
; NONEON-NOSVE-NEXT: ret
%1 = sext <2 x i64> %op1 to <2 x i128>
%2 = sext <2 x i64> %op2 to <2 x i128>
@@ -1177,19 +1179,23 @@ define void @smulh_v4i64(ptr %a, ptr %b) {
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 128
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT: ldp q2, q3, [x1]
-; NONEON-NOSVE-NEXT: stp q1, q2, [sp, #32]
-; NONEON-NOSVE-NEXT: ldp x11, x10, [sp, #32]
-; NONEON-NOSVE-NEXT: stp q0, q3, [sp, #64]
-; NONEON-NOSVE-NEXT: ldp x13, x12, [sp, #48]
-; NONEON-NOSVE-NEXT: ldp x9, x8, [sp, #64]
+; NONEON-NOSVE-NEXT: stp q1, q2, [sp]
+; NONEON-NOSVE-NEXT: ldp x11, x10, [sp]
+; NONEON-NOSVE-NEXT: stp q0, q3, [sp, #32]
+; NONEON-NOSVE-NEXT: ldp x13, x12, [sp, #16]
+; NONEON-NOSVE-NEXT: ldp x9, x8, [sp, #32]
; NONEON-NOSVE-NEXT: smulh x10, x10, x12
-; NONEON-NOSVE-NEXT: ldp x14, x12, [sp, #80]
+; NONEON-NOSVE-NEXT: ldp x14, x12, [sp, #48]
; NONEON-NOSVE-NEXT: smulh x11, x11, x13
; NONEON-NOSVE-NEXT: smulh x8, x8, x12
; NONEON-NOSVE-NEXT: smulh x9, x9, x14
-; NONEON-NOSVE-NEXT: stp x11, x10, [sp, #16]
-; NONEON-NOSVE-NEXT: stp x9, x8, [sp]
-; NONEON-NOSVE-NEXT: ldp q1, q0, [sp]
+; NONEON-NOSVE-NEXT: stp x11, x10, [sp, #64]
+; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #80]
+; NONEON-NOSVE-NEXT: ldp d0, d1, [sp, #80]
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #112]
+; NONEON-NOSVE-NEXT: ldp d0, d1, [sp, #64]
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #96]
+; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #96]
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
; NONEON-NOSVE-NEXT: add sp, sp, #128
; NONEON-NOSVE-NEXT: ret
@@ -2325,15 +2331,17 @@ define <2 x i64> @umulh_v2i64(<2 x i64> %op1, <2 x i64> %op2) {
;
; NONEON-NOSVE-LABEL: umulh_v2i64:
; NONEON-NOSVE: // %bb.0:
-; NONEON-NOSVE-NEXT: sub sp, sp, #64
+; NONEON-NOSVE-NEXT: stp q0, q1, [sp, #-64]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 64
-; NONEON-NOSVE-NEXT: stp q0, q1, [sp, #16]
-; NONEON-NOSVE-NEXT: ldp x9, x8, [sp, #16]
-; NONEON-NOSVE-NEXT: ldp x11, x10, [sp, #32]
+; NONEON-NOSVE-NEXT: ldp x9, x8, [sp]
+; NONEON-NOSVE-NEXT: ldp x11, x10, [sp, #16]
; NONEON-NOSVE-NEXT: umulh x8, x8, x10
; NONEON-NOSVE-NEXT: umulh x9, x9, x11
-; NONEON-NOSVE-NEXT: stp x9, x8, [sp]
-; NONEON-NOSVE-NEXT: ldr q0, [sp], #64
+; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #32]
+; NONEON-NOSVE-NEXT: ldp d0, d1, [sp, #32]
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #48]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #48]
+; NONEON-NOSVE-NEXT: add sp, sp, #64
; NONEON-NOSVE-NEXT: ret
%1 = zext <2 x i64> %op1 to <2 x i128>
%2 = zext <2 x i64> %op2 to <2 x i128>
@@ -2370,19 +2378,23 @@ define void @umulh_v4i64(ptr %a, ptr %b) {
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 128
; NONEON-NOSVE-NEXT: ldp q1, q0, [x0]
; NONEON-NOSVE-NEXT: ldp q2, q3, [x1]
-; NONEON-NOSVE-NEXT: stp q1, q2, [sp, #32]
-; NONEON-NOSVE-NEXT: ldp x11, x10, [sp, #32]
-; NONEON-NOSVE-NEXT: stp q0, q3, [sp, #64]
-; NONEON-NOSVE-NEXT: ldp x13, x12, [sp, #48]
-; NONEON-NOSVE-NEXT: ldp x9, x8, [sp, #64]
+; NONEON-NOSVE-NEXT: stp q1, q2, [sp]
+; NONEON-NOSVE-NEXT: ldp x11, x10, [sp]
+; NONEON-NOSVE-NEXT: stp q0, q3, [sp, #32]
+; NONEON-NOSVE-NEXT: ldp x13, x12, [sp, #16]
+; NONEON-NOSVE-NEXT: ldp x9, x8, [sp, #32]
; NONEON-NOSVE-NEXT: umulh x10, x10, x12
-; NONEON-NOSVE-NEXT: ldp x14, x12, [sp, #80]
+; NONEON-NOSVE-NEXT: ldp x14, x12, [sp, #48]
; NONEON-NOSVE-NEXT: umulh x11, x11, x13
; NONEON-NOSVE-NEXT: umulh x8, x8, x12
; NONEON-NOSVE-NEXT: umulh x9, x9, x14
-; NONEON-NOSVE-NEXT: stp x11, x10, [sp, #16]
-; NONEON-NOSVE-NEXT: stp x9, x8, [sp]
-; NONEON-NOSVE-NEXT: ldp q1, q0, [sp]
+; NONEON-NOSVE-NEXT: stp x11, x10, [sp, #64]
+; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #80]
+; NONEON-NOSVE-NEXT: ldp d0, d1, [sp, #80]
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #112]
+; NONEON-NOSVE-NEXT: ldp d0, d1, [sp, #64]
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #96]
+; NONEON-NOSVE-NEXT: ldp q0, q1, [sp, #96]
; NONEON-NOSVE-NEXT: stp q0, q1, [x0]
; NONEON-NOSVE-NEXT: add sp, sp, #128
; NONEON-NOSVE-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll
index ae87128b5c3f9..13fcd94ea8a26 100644
--- a/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll
+++ b/llvm/test/CodeGen/AArch64/sve-streaming-mode-fixed-length-trunc-stores.ll
@@ -142,7 +142,9 @@ define void @store_trunc_v2i256i64(ptr %ap, ptr %dest) {
; NONEON-NOSVE-NEXT: ldr x9, [x0]
; NONEON-NOSVE-NEXT: stp x9, x8, [sp, #-32]!
; NONEON-NOSVE-NEXT: .cfi_def_cfa_offset 32
-; NONEON-NOSVE-NEXT: ldr q0, [sp]
+; NONEON-NOSVE-NEXT: ldp d0, d1, [sp]
+; NONEON-NOSVE-NEXT: stp d0, d1, [sp, #16]
+; NONEON-NOSVE-NEXT: ldr q0, [sp, #16]
; NONEON-NOSVE-NEXT: str q0, [x1]
; NONEON-NOSVE-NEXT: add sp, sp, #32
; NONEON-NOSVE-NEXT: ret
>From 3c377659beef0100a8a71f1bcb06425963483d38 Mon Sep 17 00:00:00 2001
From: Ethan Kaji <97700380+Esan5 at users.noreply.github.com>
Date: Thu, 23 Jan 2025 11:59:16 -0500
Subject: [PATCH 12/16] remove braces negate and swap order
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index f86494753dfd6..a6f4fae13fe13 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3405,11 +3405,12 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
break;
case ISD::CONCAT_VECTORS:
- if (!TLI.isOperationExpand(ISD::EXTRACT_VECTOR_ELT, Node->getOperand(0).getValueType())) {
- Results.push_back(ExpandConcatVectors(Node));
- } else {
+ if (Node->getValueType().isScalableVector() ||
+ TLI.isOperationExpand(ISD::EXTRACT_VECTOR_ELT,
+ Node->getOperand(0).getValueType()))
Results.push_back(ExpandVectorBuildThroughStack(Node));
- }
+ else
+ Results.push_back(ExpandConcatVectors(Node));
break;
case ISD::SCALAR_TO_VECTOR:
Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
>From f7c9f312a57454b3b56c3a22327c8363c027f9f4 Mon Sep 17 00:00:00 2001
From: Ethan Kaji <97700380+Esan5 at users.noreply.github.com>
Date: Mon, 27 Jan 2025 13:58:52 -0500
Subject: [PATCH 13/16] scalable vector guard with vector value type
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index a6f4fae13fe13..754c29b6ba868 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -3405,9 +3405,9 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
break;
case ISD::CONCAT_VECTORS:
- if (Node->getValueType().isScalableVector() ||
- TLI.isOperationExpand(ISD::EXTRACT_VECTOR_ELT,
- Node->getOperand(0).getValueType()))
+ if (EVT VectorValueType = Node->getOperand(0).getValueType();
+ VectorValueType.isScalableVector() ||
+ TLI.isOperationExpand(ISD::EXTRACT_VECTOR_ELT, VectorValueType))
Results.push_back(ExpandVectorBuildThroughStack(Node));
else
Results.push_back(ExpandConcatVectors(Node));
>From cb51c4ea00a4ab4cdf5ca17572bef2fc04f8baef Mon Sep 17 00:00:00 2001
From: Ethan Kaji <ethan.kaji at gmail.com>
Date: Tue, 4 Mar 2025 18:14:42 -0500
Subject: [PATCH 14/16] remove dead code
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 29 +------------------
1 file changed, 1 insertion(+), 28 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index 754c29b6ba868..e2ae31c86bc48 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -191,7 +191,6 @@ class SelectionDAGLegalize {
SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
SDValue ExpandInsertToVectorThroughStack(SDValue Op);
SDValue ExpandVectorBuildThroughStack(SDNode* Node);
- SDValue ExpandConcatVectors(SDNode *Node);
SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
SDValue ExpandConstant(ConstantSDNode *CP);
@@ -1526,27 +1525,6 @@ SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
BaseVecAlignment);
}
-SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
- assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
- SDLoc DL(Node);
- SmallVector<SDValue, 16> Ops;
- unsigned NumOperands = Node->getNumOperands();
- MVT VectorIdxType = TLI.getVectorIdxTy(DAG.getDataLayout());
- EVT VectorValueType = Node->getOperand(0).getValueType();
- unsigned NumSubElem = VectorValueType.getVectorNumElements();
- EVT ElementValueType = TLI.getTypeToTransformTo(
- *DAG.getContext(), VectorValueType.getVectorElementType());
- for (unsigned I = 0; I < NumOperands; ++I) {
- SDValue SubOp = Node->getOperand(I);
- for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
- Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElementValueType,
- SubOp,
- DAG.getConstant(Idx, DL, VectorIdxType)));
- }
- }
- return DAG.getBuildVector(Node->getValueType(0), DL, Ops);
-}
-
SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
Node->getOpcode() == ISD::CONCAT_VECTORS) &&
@@ -3405,12 +3383,7 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
break;
case ISD::CONCAT_VECTORS:
- if (EVT VectorValueType = Node->getOperand(0).getValueType();
- VectorValueType.isScalableVector() ||
- TLI.isOperationExpand(ISD::EXTRACT_VECTOR_ELT, VectorValueType))
- Results.push_back(ExpandVectorBuildThroughStack(Node));
- else
- Results.push_back(ExpandConcatVectors(Node));
+ Results.push_back(ExpandVectorBuildThroughStack(Node));
break;
case ISD::SCALAR_TO_VECTOR:
Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
>From 1effb919eec6ebc8ae2673fba62298ba0957e465 Mon Sep 17 00:00:00 2001
From: Ethan Kaji <ethan.kaji at gmail.com>
Date: Wed, 5 Mar 2025 11:44:05 -0500
Subject: [PATCH 15/16] Revert "remove dead code"
This reverts commit d1bab4d606aebd486bf4fca759d8ada23da816de.
---
llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp | 29 ++++++++++++++++++-
1 file changed, 28 insertions(+), 1 deletion(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index e2ae31c86bc48..754c29b6ba868 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -191,6 +191,7 @@ class SelectionDAGLegalize {
SDValue ExpandExtractFromVectorThroughStack(SDValue Op);
SDValue ExpandInsertToVectorThroughStack(SDValue Op);
SDValue ExpandVectorBuildThroughStack(SDNode* Node);
+ SDValue ExpandConcatVectors(SDNode *Node);
SDValue ExpandConstantFP(ConstantFPSDNode *CFP, bool UseCP);
SDValue ExpandConstant(ConstantSDNode *CP);
@@ -1525,6 +1526,27 @@ SDValue SelectionDAGLegalize::ExpandInsertToVectorThroughStack(SDValue Op) {
BaseVecAlignment);
}
+SDValue SelectionDAGLegalize::ExpandConcatVectors(SDNode *Node) {
+ assert(Node->getOpcode() == ISD::CONCAT_VECTORS && "Unexpected opcode!");
+ SDLoc DL(Node);
+ SmallVector<SDValue, 16> Ops;
+ unsigned NumOperands = Node->getNumOperands();
+ MVT VectorIdxType = TLI.getVectorIdxTy(DAG.getDataLayout());
+ EVT VectorValueType = Node->getOperand(0).getValueType();
+ unsigned NumSubElem = VectorValueType.getVectorNumElements();
+ EVT ElementValueType = TLI.getTypeToTransformTo(
+ *DAG.getContext(), VectorValueType.getVectorElementType());
+ for (unsigned I = 0; I < NumOperands; ++I) {
+ SDValue SubOp = Node->getOperand(I);
+ for (unsigned Idx = 0; Idx < NumSubElem; ++Idx) {
+ Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ElementValueType,
+ SubOp,
+ DAG.getConstant(Idx, DL, VectorIdxType)));
+ }
+ }
+ return DAG.getBuildVector(Node->getValueType(0), DL, Ops);
+}
+
SDValue SelectionDAGLegalize::ExpandVectorBuildThroughStack(SDNode* Node) {
assert((Node->getOpcode() == ISD::BUILD_VECTOR ||
Node->getOpcode() == ISD::CONCAT_VECTORS) &&
@@ -3383,7 +3405,12 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
Results.push_back(ExpandInsertToVectorThroughStack(SDValue(Node, 0)));
break;
case ISD::CONCAT_VECTORS:
- Results.push_back(ExpandVectorBuildThroughStack(Node));
+ if (EVT VectorValueType = Node->getOperand(0).getValueType();
+ VectorValueType.isScalableVector() ||
+ TLI.isOperationExpand(ISD::EXTRACT_VECTOR_ELT, VectorValueType))
+ Results.push_back(ExpandVectorBuildThroughStack(Node));
+ else
+ Results.push_back(ExpandConcatVectors(Node));
break;
case ISD::SCALAR_TO_VECTOR:
Results.push_back(ExpandSCALAR_TO_VECTOR(Node));
>From c318b9170eca8661676739cb75adba54c98c9615 Mon Sep 17 00:00:00 2001
From: Ethan Kaji <ethan.kaji at gmail.com>
Date: Wed, 26 Mar 2025 15:40:29 -0400
Subject: [PATCH 16/16] handled in a seperate PR
---
llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp | 24 +++++++++++++++++++++
llvm/lib/Target/NVPTX/NVPTXISelLowering.h | 1 +
2 files changed, 25 insertions(+)
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 0384903dd2cac..06e221777b7ea 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -2111,6 +2111,28 @@ SDValue NVPTXTargetLowering::LowerSTACKSAVE(SDValue Op,
return DAG.getMergeValues({ASC, SDValue(SS.getNode(), 1)}, DL);
}
+// By default CONCAT_VECTORS is lowered by ExpandVectorBuildThroughStack()
+// (see LegalizeDAG.cpp). This is slow and uses local memory.
+// We use extract/insert/build vector just as what LegalizeOp() does in llvm 2.5
+SDValue
+NVPTXTargetLowering::LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const {
+ SDNode *Node = Op.getNode();
+ SDLoc dl(Node);
+ SmallVector<SDValue, 8> Ops;
+ unsigned NumOperands = Node->getNumOperands();
+ for (unsigned i = 0; i < NumOperands; ++i) {
+ SDValue SubOp = Node->getOperand(i);
+ EVT VVT = SubOp.getNode()->getValueType(0);
+ EVT EltVT = VVT.getVectorElementType();
+ unsigned NumSubElem = VVT.getVectorNumElements();
+ for (unsigned j = 0; j < NumSubElem; ++j) {
+ Ops.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, dl, EltVT, SubOp,
+ DAG.getIntPtrConstant(j, dl)));
+ }
+ }
+ return DAG.getBuildVector(Node->getValueType(0), dl, Ops);
+}
+
SDValue NVPTXTargetLowering::LowerBITCAST(SDValue Op, SelectionDAG &DAG) const {
// Handle bitcasting from v2i8 without hitting the default promotion
// strategy which goes through stack memory.
@@ -2824,6 +2846,8 @@ NVPTXTargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
return LowerINSERT_VECTOR_ELT(Op, DAG);
case ISD::VECTOR_SHUFFLE:
return LowerVECTOR_SHUFFLE(Op, DAG);
+ case ISD::CONCAT_VECTORS:
+ return LowerCONCAT_VECTORS(Op, DAG);
case ISD::STORE:
return LowerSTORE(Op, DAG);
case ISD::LOAD:
diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
index 9202e0773fbd0..39470be254efa 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.h
@@ -292,6 +292,7 @@ class NVPTXTargetLowering : public TargetLowering {
SDValue LowerBITCAST(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerBUILD_VECTOR(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerCONCAT_VECTORS(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerEXTRACT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerINSERT_VECTOR_ELT(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerVECTOR_SHUFFLE(SDValue Op, SelectionDAG &DAG) const;
More information about the llvm-commits
mailing list