[llvm-branch-commits] [llvm] [AMDGPU] Price under-aligned memory operations (PR #233262)
Dmitry Sidorov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sun Oct 18 09:20:23 PDT 2026
https://github.com/MrSidims created https://github.com/llvm/llvm-project/pull/233262
Without unaligned access for the address space, an access aligned below
4 bytes is split into pieces of its alignment. LDS accesses aligned to 4
use 8-byte pieces, and take a 12-byte piece only when aligned to 16 or
when unaligned DS access is enabled.
---
<sub>Stack created with <a href="https://github.com/github/gh-stack">GitHub Stacks CLI</a> • <a href="https://gh.io/stacks-feedback">Give Feedback 💬</a></sub>
>From 1769ec8ee986e9d77e979bcd894f113b83c82666 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 9 Oct 2026 10:56:06 +0200
Subject: [PATCH] [AMDGPU] Price under-aligned memory operations
Without unaligned access for the address space, an access aligned below
4 bytes is split into pieces of its alignment. LDS accesses aligned to 4
use 8-byte pieces, and take a 12-byte piece only when aligned to 16 or
when unaligned DS access is enabled.
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 11 +-
llvm/test/Analysis/CostModel/AMDGPU/load.ll | 81 ++++----
llvm/test/Analysis/CostModel/AMDGPU/store.ll | 81 ++++----
.../SLPVectorizer/AMDGPU/packed-math.ll | 178 +++++++++++++-----
.../SLPVectorizer/AMDGPU/vectorize-i8.ll | 24 ++-
5 files changed, 259 insertions(+), 116 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index bfb455ec758e6c..05726165ba1e44 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -2157,21 +2157,30 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
unsigned Bytes = DL.getTypeStoreSize(Src);
unsigned MaxBytes = 16;
bool HasDwordx3 = ST->hasDwordx3LoadStores();
+ bool HasUnalignedAccess = ST->hasUnalignedBufferAccessEnabled();
switch (AddressSpace) {
case AMDGPUAS::PRIVATE_ADDRESS:
MaxBytes = ST->getMaxPrivateElementSize();
+ HasUnalignedAccess = ST->hasUnalignedScratchAccessEnabled();
break;
case AMDGPUAS::FLAT_ADDRESS:
if (!ST->hasMultiDwordFlatScratchAddressing())
MaxBytes = ST->getMaxPrivateElementSize();
+ HasUnalignedAccess = ST->hasUnalignedScratchAccessEnabled();
break;
case AMDGPUAS::LOCAL_ADDRESS:
case AMDGPUAS::REGION_ADDRESS:
- HasDwordx3 = ST->hasDS96AndDS128();
+ HasUnalignedAccess = ST->hasUnalignedDSAccessEnabled();
+ if (Alignment >= 4 && Alignment < 8)
+ MaxBytes = 8;
+ HasDwordx3 = ST->hasDS96AndDS128() &&
+ (Alignment >= 16 || (Alignment < 4 && HasUnalignedAccess));
break;
default:
break;
}
+ if (Alignment < 4 && !HasUnalignedAccess)
+ MaxBytes = std::min<unsigned>(MaxBytes, Alignment.value());
if (Opcode == Instruction::Load) {
if (Alignment >= PowerOf2Ceil(Bytes))
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/load.ll b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
index 073e504080c49c..52e5e31eeb93f1 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/load.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
@@ -21,20 +21,35 @@ define void @loads_i1(i32 %arg) {
}
define void @loads_i8(i32 %arg) {
-; CHECK-LABEL: 'loads_i8'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr poison, align 2
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i8, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i8>, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i8>, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i8>, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i8, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i8>, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i8>, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i8>, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX8-LABEL: 'loads_i8'
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr poison, align 2
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i8, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <2 x i8>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %7 = load <3 x i8>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %8 = load <4 x i8>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i8, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i8>, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i8>, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i8>, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90APLUS-LABEL: 'loads_i8'
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr poison, align 1
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr poison, align 2
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr poison, align 4
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr poison, align 4
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i8, ptr poison, align 1
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i8>, ptr poison, align 1
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i8>, ptr poison, align 1
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i8>, ptr poison, align 1
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i8, ptr poison, align 4
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i8>, ptr poison, align 4
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i8>, ptr poison, align 4
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i8>, ptr poison, align 4
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
load i8, ptr poison
load <2 x i8>, ptr poison
@@ -60,10 +75,10 @@ define void @loads_i16(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i16>, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <3 x i16>, ptr poison, align 8
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <4 x i16>, ptr poison, align 8
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i16, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i16>, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i16>, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %8 = load <4 x i16>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load i16, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %6 = load <2 x i16>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %7 = load <3 x i16>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %8 = load <4 x i16>, ptr poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i16, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i16>, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <3 x i16>, ptr poison, align 4
@@ -122,10 +137,10 @@ define void @loads_i32(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <2 x i32>, ptr poison, align 8
; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %3 = load <3 x i32>, ptr poison, align 16
; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %4 = load <4 x i32>, ptr poison, align 16
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i32, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <2 x i32>, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %7 = load <3 x i32>, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %8 = load <4 x i32>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %5 = load i32, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %6 = load <2 x i32>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %7 = load <3 x i32>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %8 = load <4 x i32>, ptr poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i32, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %10 = load <2 x i32>, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %11 = load <3 x i32>, ptr poison, align 4
@@ -532,26 +547,26 @@ define void @loads_addrspace_5(i32 %arg) {
define void @loads_underaligned(i32 %arg) {
; GFX8-LABEL: 'loads_underaligned'
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4
-; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; GFX90A-LABEL: 'loads_underaligned'
; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4
; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1
; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; GFX12-LABEL: 'loads_underaligned'
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1
; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/store.ll b/llvm/test/Analysis/CostModel/AMDGPU/store.ll
index aa9cbd92fad559..a72eddfa70796b 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/store.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/store.ll
@@ -20,20 +20,35 @@ define void @stores_i1(i32 %arg) {
}
define void @stores_i8(i32 %arg) {
-; CHECK-LABEL: 'stores_i8'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 2
-; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX8-LABEL: 'stores_i8'
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 2
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i8> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90APLUS-LABEL: 'stores_i8'
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 2
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 1
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 1
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 1
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 4
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 4
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
store i8 poison, ptr poison
store <2 x i8> poison, ptr poison
@@ -59,10 +74,10 @@ define void @stores_i16(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 8
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> poison, ptr poison, align 8
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i16 poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <2 x i16> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 6 for instruction: store <3 x i16> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <4 x i16> poison, ptr poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 4
@@ -121,10 +136,10 @@ define void @stores_i32(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> poison, ptr poison, align 8
; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr poison, align 16
; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr poison, align 16
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> poison, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store i32 poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <2 x i32> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <3 x i32> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <4 x i32> poison, ptr poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> poison, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr poison, align 4
@@ -534,26 +549,26 @@ define void @stores_addrspace_5(i32 %arg) {
define void @stores_underaligned(i32 %arg) {
; GFX8-LABEL: 'stores_underaligned'
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4
-; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; GFX90A-LABEL: 'stores_underaligned'
; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4
; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1
; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; GFX12-LABEL: 'stores_underaligned'
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1
; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll
index 49d7ee569c8711..32a5ac77bd3472 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll
@@ -72,13 +72,30 @@ define amdgpu_kernel void @test1_as_0_0_3_v2f16(ptr %a, ptr %b, ptr addrspace(3)
}
define amdgpu_kernel void @test1_fma_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d) {
-; GCN-LABEL: @test1_fma_v2f16(
-; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
-; GCN-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[B:%.*]], align 2
-; GCN-NEXT: [[TMP3:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2
-; GCN-NEXT: [[TMP4:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP1]], <2 x half> [[TMP2]], <2 x half> [[TMP3]])
-; GCN-NEXT: store <2 x half> [[TMP4]], ptr addrspace(3) [[D:%.*]], align 2
-; GCN-NEXT: ret void
+; GFX9-LABEL: @test1_fma_v2f16(
+; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
+; GFX9-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[B:%.*]], align 2
+; GFX9-NEXT: [[TMP3:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2
+; GFX9-NEXT: [[TMP4:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP1]], <2 x half> [[TMP2]], <2 x half> [[TMP3]])
+; GFX9-NEXT: store <2 x half> [[TMP4]], ptr addrspace(3) [[D:%.*]], align 2
+; GFX9-NEXT: ret void
+;
+; VI-LABEL: @test1_fma_v2f16(
+; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2
+; VI-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2
+; VI-NEXT: [[I2:%.*]] = load half, ptr addrspace(3) [[C:%.*]], align 2
+; VI-NEXT: [[FMA0:%.*]] = call half @llvm.fma.f16(half [[I0]], half [[I1]], half [[I2]])
+; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1
+; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2
+; VI-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1
+; VI-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2
+; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1
+; VI-NEXT: [[I5:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX5]], align 2
+; VI-NEXT: [[FMA1:%.*]] = call half @llvm.fma.f16(half [[I3]], half [[I4]], half [[I5]])
+; VI-NEXT: store half [[FMA0]], ptr addrspace(3) [[D:%.*]], align 2
+; VI-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[D]], i64 1
+; VI-NEXT: store half [[FMA1]], ptr addrspace(3) [[ARRAYIDX6]], align 2
+; VI-NEXT: ret void
;
%i0 = load half, ptr addrspace(3) %a, align 2
%i1 = load half, ptr addrspace(3) %b, align 2
@@ -98,13 +115,24 @@ define amdgpu_kernel void @test1_fma_v2f16(ptr addrspace(3) %a, ptr addrspace(3)
}
define amdgpu_kernel void @mul_scalar_v2f16(ptr addrspace(3) %a, half %scalar, ptr addrspace(3) %c) {
-; GCN-LABEL: @mul_scalar_v2f16(
-; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
-; GCN-NEXT: [[TMP2:%.*]] = insertelement <2 x half> poison, half [[SCALAR:%.*]], i64 0
-; GCN-NEXT: [[TMP3:%.*]] = shufflevector <2 x half> [[TMP2]], <2 x half> poison, <2 x i32> zeroinitializer
-; GCN-NEXT: [[TMP4:%.*]] = fmul <2 x half> [[TMP1]], [[TMP3]]
-; GCN-NEXT: store <2 x half> [[TMP4]], ptr addrspace(3) [[C:%.*]], align 2
-; GCN-NEXT: ret void
+; GFX9-LABEL: @mul_scalar_v2f16(
+; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
+; GFX9-NEXT: [[TMP2:%.*]] = insertelement <2 x half> poison, half [[SCALAR:%.*]], i64 0
+; GFX9-NEXT: [[TMP3:%.*]] = shufflevector <2 x half> [[TMP2]], <2 x half> poison, <2 x i32> zeroinitializer
+; GFX9-NEXT: [[TMP4:%.*]] = fmul <2 x half> [[TMP1]], [[TMP3]]
+; GFX9-NEXT: store <2 x half> [[TMP4]], ptr addrspace(3) [[C:%.*]], align 2
+; GFX9-NEXT: ret void
+;
+; VI-LABEL: @mul_scalar_v2f16(
+; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2
+; VI-NEXT: [[MUL:%.*]] = fmul half [[I0]], [[SCALAR:%.*]]
+; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1
+; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2
+; VI-NEXT: [[MUL5:%.*]] = fmul half [[I3]], [[SCALAR]]
+; VI-NEXT: store half [[MUL]], ptr addrspace(3) [[C:%.*]], align 2
+; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1
+; VI-NEXT: store half [[MUL5]], ptr addrspace(3) [[ARRAYIDX5]], align 2
+; VI-NEXT: ret void
;
%i0 = load half, ptr addrspace(3) %a, align 2
%mul = fmul half %i0, %scalar
@@ -118,11 +146,22 @@ define amdgpu_kernel void @mul_scalar_v2f16(ptr addrspace(3) %a, half %scalar, p
}
define amdgpu_kernel void @fabs_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %c) {
-; GCN-LABEL: @fabs_v2f16(
-; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
-; GCN-NEXT: [[TMP2:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[TMP1]])
-; GCN-NEXT: store <2 x half> [[TMP2]], ptr addrspace(3) [[C:%.*]], align 2
-; GCN-NEXT: ret void
+; GFX9-LABEL: @fabs_v2f16(
+; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
+; GFX9-NEXT: [[TMP2:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[TMP1]])
+; GFX9-NEXT: store <2 x half> [[TMP2]], ptr addrspace(3) [[C:%.*]], align 2
+; GFX9-NEXT: ret void
+;
+; VI-LABEL: @fabs_v2f16(
+; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2
+; VI-NEXT: [[FABS0:%.*]] = call half @llvm.fabs.f16(half [[I0]])
+; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1
+; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2
+; VI-NEXT: [[FABS1:%.*]] = call half @llvm.fabs.f16(half [[I3]])
+; VI-NEXT: store half [[FABS0]], ptr addrspace(3) [[C:%.*]], align 2
+; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1
+; VI-NEXT: store half [[FABS1]], ptr addrspace(3) [[ARRAYIDX5]], align 2
+; VI-NEXT: ret void
;
%i0 = load half, ptr addrspace(3) %a, align 2
%fabs0 = call half @llvm.fabs.f16(half %i0)
@@ -136,14 +175,33 @@ define amdgpu_kernel void @fabs_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %c)
}
define amdgpu_kernel void @test1_fabs_fma_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d) {
-; GCN-LABEL: @test1_fabs_fma_v2f16(
-; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
-; GCN-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[B:%.*]], align 2
-; GCN-NEXT: [[TMP3:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2
-; GCN-NEXT: [[TMP4:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[TMP1]])
-; GCN-NEXT: [[TMP5:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP4]], <2 x half> [[TMP2]], <2 x half> [[TMP3]])
-; GCN-NEXT: store <2 x half> [[TMP5]], ptr addrspace(3) [[D:%.*]], align 2
-; GCN-NEXT: ret void
+; GFX9-LABEL: @test1_fabs_fma_v2f16(
+; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
+; GFX9-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[B:%.*]], align 2
+; GFX9-NEXT: [[TMP3:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2
+; GFX9-NEXT: [[TMP4:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[TMP1]])
+; GFX9-NEXT: [[TMP5:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP4]], <2 x half> [[TMP2]], <2 x half> [[TMP3]])
+; GFX9-NEXT: store <2 x half> [[TMP5]], ptr addrspace(3) [[D:%.*]], align 2
+; GFX9-NEXT: ret void
+;
+; VI-LABEL: @test1_fabs_fma_v2f16(
+; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2
+; VI-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2
+; VI-NEXT: [[I2:%.*]] = load half, ptr addrspace(3) [[C:%.*]], align 2
+; VI-NEXT: [[I0_FABS:%.*]] = call half @llvm.fabs.f16(half [[I0]])
+; VI-NEXT: [[FMA0:%.*]] = call half @llvm.fma.f16(half [[I0_FABS]], half [[I1]], half [[I2]])
+; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1
+; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2
+; VI-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1
+; VI-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2
+; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1
+; VI-NEXT: [[I5:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX5]], align 2
+; VI-NEXT: [[I3_FABS:%.*]] = call half @llvm.fabs.f16(half [[I3]])
+; VI-NEXT: [[FMA1:%.*]] = call half @llvm.fma.f16(half [[I3_FABS]], half [[I4]], half [[I5]])
+; VI-NEXT: store half [[FMA0]], ptr addrspace(3) [[D:%.*]], align 2
+; VI-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[D]], i64 1
+; VI-NEXT: store half [[FMA1]], ptr addrspace(3) [[ARRAYIDX6]], align 2
+; VI-NEXT: ret void
;
%i0 = load half, ptr addrspace(3) %a, align 2
%i1 = load half, ptr addrspace(3) %b, align 2
@@ -167,18 +225,36 @@ define amdgpu_kernel void @test1_fabs_fma_v2f16(ptr addrspace(3) %a, ptr addrspa
}
define amdgpu_kernel void @test1_fabs_scalar_fma_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d) {
-; GCN-LABEL: @test1_fabs_scalar_fma_v2f16(
-; GCN-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2
-; GCN-NEXT: [[I1_FABS:%.*]] = call half @llvm.fabs.f16(half [[I1]])
-; GCN-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1
-; GCN-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2
-; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
-; GCN-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2
-; GCN-NEXT: [[TMP3:%.*]] = insertelement <2 x half> poison, half [[I1_FABS]], i64 0
-; GCN-NEXT: [[TMP4:%.*]] = insertelement <2 x half> [[TMP3]], half [[I4]], i64 1
-; GCN-NEXT: [[TMP5:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP1]], <2 x half> [[TMP4]], <2 x half> [[TMP2]])
-; GCN-NEXT: store <2 x half> [[TMP5]], ptr addrspace(3) [[D:%.*]], align 2
-; GCN-NEXT: ret void
+; GFX9-LABEL: @test1_fabs_scalar_fma_v2f16(
+; GFX9-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2
+; GFX9-NEXT: [[I1_FABS:%.*]] = call half @llvm.fabs.f16(half [[I1]])
+; GFX9-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1
+; GFX9-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2
+; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
+; GFX9-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2
+; GFX9-NEXT: [[TMP3:%.*]] = insertelement <2 x half> poison, half [[I1_FABS]], i64 0
+; GFX9-NEXT: [[TMP4:%.*]] = insertelement <2 x half> [[TMP3]], half [[I4]], i64 1
+; GFX9-NEXT: [[TMP5:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP1]], <2 x half> [[TMP4]], <2 x half> [[TMP2]])
+; GFX9-NEXT: store <2 x half> [[TMP5]], ptr addrspace(3) [[D:%.*]], align 2
+; GFX9-NEXT: ret void
+;
+; VI-LABEL: @test1_fabs_scalar_fma_v2f16(
+; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2
+; VI-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2
+; VI-NEXT: [[I2:%.*]] = load half, ptr addrspace(3) [[C:%.*]], align 2
+; VI-NEXT: [[I1_FABS:%.*]] = call half @llvm.fabs.f16(half [[I1]])
+; VI-NEXT: [[FMA0:%.*]] = call half @llvm.fma.f16(half [[I0]], half [[I1_FABS]], half [[I2]])
+; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1
+; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2
+; VI-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1
+; VI-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2
+; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1
+; VI-NEXT: [[I5:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX5]], align 2
+; VI-NEXT: [[FMA1:%.*]] = call half @llvm.fma.f16(half [[I3]], half [[I4]], half [[I5]])
+; VI-NEXT: store half [[FMA0]], ptr addrspace(3) [[D:%.*]], align 2
+; VI-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[D]], i64 1
+; VI-NEXT: store half [[FMA1]], ptr addrspace(3) [[ARRAYIDX6]], align 2
+; VI-NEXT: ret void
;
%i0 = load half, ptr addrspace(3) %a, align 2
%i1 = load half, ptr addrspace(3) %b, align 2
@@ -200,11 +276,22 @@ define amdgpu_kernel void @test1_fabs_scalar_fma_v2f16(ptr addrspace(3) %a, ptr
}
define amdgpu_kernel void @canonicalize_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %c) {
-; GCN-LABEL: @canonicalize_v2f16(
-; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
-; GCN-NEXT: [[TMP2:%.*]] = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> [[TMP1]])
-; GCN-NEXT: store <2 x half> [[TMP2]], ptr addrspace(3) [[C:%.*]], align 2
-; GCN-NEXT: ret void
+; GFX9-LABEL: @canonicalize_v2f16(
+; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
+; GFX9-NEXT: [[TMP2:%.*]] = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> [[TMP1]])
+; GFX9-NEXT: store <2 x half> [[TMP2]], ptr addrspace(3) [[C:%.*]], align 2
+; GFX9-NEXT: ret void
+;
+; VI-LABEL: @canonicalize_v2f16(
+; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2
+; VI-NEXT: [[CANONICALIZE0:%.*]] = call half @llvm.canonicalize.f16(half [[I0]])
+; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1
+; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2
+; VI-NEXT: [[CANONICALIZE1:%.*]] = call half @llvm.canonicalize.f16(half [[I3]])
+; VI-NEXT: store half [[CANONICALIZE0]], ptr addrspace(3) [[C:%.*]], align 2
+; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1
+; VI-NEXT: store half [[CANONICALIZE1]], ptr addrspace(3) [[ARRAYIDX5]], align 2
+; VI-NEXT: ret void
;
%i0 = load half, ptr addrspace(3) %a, align 2
%canonicalize0 = call half @llvm.canonicalize.f16(half %i0)
@@ -223,6 +310,3 @@ declare half @llvm.canonicalize.f16(half) #1
attributes #0 = { nounwind }
attributes #1 = { nounwind readnone }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX9: {{.*}}
-; VI: {{.*}}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll
index 414ffe6cbec23e..4d20feaa6b94f5 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll
@@ -787,6 +787,10 @@ define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(<4 x i8> %in
; GFX7-LABEL: define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(
; GFX7-SAME: <4 x i8> [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr addrspace(3) [[OUT1:%.*]], i32 [[FLAG:%.*]]) {
; GFX7-NEXT: [[ENTRY:.*]]:
+; GFX7-NEXT: [[OTHERELE0:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 0
+; GFX7-NEXT: [[OTHERELE1:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 1
+; GFX7-NEXT: [[OTHERELE2:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 2
+; GFX7-NEXT: [[OTHERELE3:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 3
; GFX7-NEXT: br label %[[DO_BODY:.*]]
; GFX7: [[DO_BODY]]:
; GFX7-NEXT: [[TMP0:%.*]] = phi <4 x i8> [ [[INPTR0]], %[[ENTRY]] ], [ [[INPTR0]], %[[DO_BODY]] ]
@@ -796,12 +800,22 @@ define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(<4 x i8> %in
; GFX7-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
; GFX7: [[EXIT]]:
; GFX7-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 0
-; GFX7-NEXT: store <4 x i8> [[INPTR0]], ptr addrspace(3) [[GEP0]], align 1
+; GFX7-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 1
+; GFX7-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 2
+; GFX7-NEXT: [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 3
+; GFX7-NEXT: store i8 [[OTHERELE0]], ptr addrspace(3) [[GEP0]], align 1
+; GFX7-NEXT: store i8 [[OTHERELE1]], ptr addrspace(3) [[GEP1]], align 1
+; GFX7-NEXT: store i8 [[OTHERELE2]], ptr addrspace(3) [[GEP2]], align 1
+; GFX7-NEXT: store i8 [[OTHERELE3]], ptr addrspace(3) [[GEP3]], align 1
; GFX7-NEXT: ret void
;
; GFX8-LABEL: define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(
; GFX8-SAME: <4 x i8> [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr addrspace(3) [[OUT1:%.*]], i32 [[FLAG:%.*]]) {
; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: [[OTHERELE0:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 0
+; GFX8-NEXT: [[OTHERELE1:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 1
+; GFX8-NEXT: [[OTHERELE2:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 2
+; GFX8-NEXT: [[OTHERELE3:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 3
; GFX8-NEXT: br label %[[DO_BODY:.*]]
; GFX8: [[DO_BODY]]:
; GFX8-NEXT: [[TMP0:%.*]] = phi <4 x i8> [ [[INPTR0]], %[[ENTRY]] ], [ [[INPTR0]], %[[DO_BODY]] ]
@@ -811,7 +825,13 @@ define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(<4 x i8> %in
; GFX8-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
; GFX8: [[EXIT]]:
; GFX8-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 0
-; GFX8-NEXT: store <4 x i8> [[INPTR0]], ptr addrspace(3) [[GEP0]], align 1
+; GFX8-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 1
+; GFX8-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 2
+; GFX8-NEXT: [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 3
+; GFX8-NEXT: store i8 [[OTHERELE0]], ptr addrspace(3) [[GEP0]], align 1
+; GFX8-NEXT: store i8 [[OTHERELE1]], ptr addrspace(3) [[GEP1]], align 1
+; GFX8-NEXT: store i8 [[OTHERELE2]], ptr addrspace(3) [[GEP2]], align 1
+; GFX8-NEXT: store i8 [[OTHERELE3]], ptr addrspace(3) [[GEP3]], align 1
; GFX8-NEXT: ret void
;
; GFX9-LABEL: define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(
More information about the llvm-branch-commits
mailing list