[llvm-branch-commits] [llvm] [AMDGPU] Price under-aligned memory operations (PR #233262)
Dmitry Sidorov via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Sat Oct 31 11:52:31 PDT 2026
https://github.com/MrSidims updated https://github.com/llvm/llvm-project/pull/233262
>From c0fd7ee04879cd54b893b6d958a054a2d22df901 Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Fri, 9 Oct 2026 10:56:06 +0200
Subject: [PATCH 1/2] [AMDGPU] Price under-aligned memory operations
Without unaligned access for the address space, an access aligned below
4 bytes is split into pieces of its alignment. LDS accesses aligned to 4
use 8-byte pieces, and take a 12-byte piece only when aligned to 16 or
when unaligned DS access is enabled.
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 13 +
llvm/test/Analysis/CostModel/AMDGPU/load.ll | 403 +++++++++++++-----
llvm/test/Analysis/CostModel/AMDGPU/store.ll | 366 +++++++++++-----
.../SLPVectorizer/AMDGPU/packed-math.ll | 178 ++++++--
.../SLPVectorizer/AMDGPU/vectorize-i8.ll | 24 +-
5 files changed, 715 insertions(+), 269 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 75fe4c5ce1b7b4..3e4e0a2b16fef5 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -2222,6 +2222,19 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
bool IsDS = AddressSpace == AMDGPUAS::LOCAL_ADDRESS ||
AddressSpace == AMDGPUAS::REGION_ADDRESS;
bool HasDwordx3 = ST->hasDwordx3LoadStores();
+ bool HasUnalignedAccess = ST->hasUnalignedBufferAccessEnabled();
+ if (AddressSpace == AMDGPUAS::PRIVATE_ADDRESS ||
+ AddressSpace == AMDGPUAS::FLAT_ADDRESS)
+ HasUnalignedAccess = ST->hasUnalignedScratchAccessEnabled();
+ if (IsDS) {
+ HasUnalignedAccess = ST->hasUnalignedDSAccessEnabled();
+ if (Alignment >= 4 && Alignment < 8)
+ MaxBytes = 8;
+ HasDwordx3 = HasDwordx3 &&
+ (Alignment >= 16 || (Alignment < 4 && HasUnalignedAccess));
+ }
+ if (Alignment < 4 && !HasUnalignedAccess)
+ MaxBytes = std::min<unsigned>(MaxBytes, Alignment.value());
bool MergesDwords = ST->hasFlatInstOffsets() &&
(AMDGPU::isExtendedGlobalAddrSpace(AddressSpace) ||
(AddressSpace == AMDGPUAS::FLAT_ADDRESS &&
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/load.ll b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
index b89ce60082bef7..326b3c9f438c42 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/load.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu8.03-unknown-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX8
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.0a-unknown-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX90APLUS,GFX90A
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.0a-unknown-amdhsa -mattr=-unaligned-access-mode < %s | FileCheck %s --check-prefixes=CHECK,GFX90APLUS,GFX90A
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu12.01-unknown-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX90APLUS,GFX12
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.0a-unknown-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX90A,GFX90A-UNALIGNED
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.0a-unknown-amdhsa -mattr=-unaligned-access-mode < %s | FileCheck %s --check-prefixes=CHECK,GFX90A,GFX90A-NOUNALIGNED
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu12.01-unknown-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX12
define void @loads_i1(i32 %arg) {
@@ -22,20 +22,65 @@ define void @loads_i1(i32 %arg) {
}
define void @loads_i8(i32 %arg) {
-; CHECK-LABEL: 'loads_i8'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr poison, align 2
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i8, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i8>, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i8>, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i8>, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i8, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i8>, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i8>, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i8>, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX8-LABEL: 'loads_i8'
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr poison, align 2
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i8, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <2 x i8>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %7 = load <3 x i8>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %8 = load <4 x i8>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i8, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i8>, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i8>, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i8>, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90A-UNALIGNED-LABEL: 'loads_i8'
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i8, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i8>, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i8>, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i8>, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i8, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i8>, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i8>, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i8>, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90A-NOUNALIGNED-LABEL: 'loads_i8'
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i8, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <2 x i8>, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %7 = load <3 x i8>, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %8 = load <4 x i8>, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i8, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i8>, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i8>, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i8>, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX12-LABEL: 'loads_i8'
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr poison, align 2
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i8, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i8>, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i8>, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i8>, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i8, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i8>, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i8>, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i8>, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
load i8, ptr poison
load <2 x i8>, ptr poison
@@ -61,10 +106,10 @@ define void @loads_i16(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i16>, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <3 x i16>, ptr poison, align 8
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <4 x i16>, ptr poison, align 8
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i16, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i16>, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i16>, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %8 = load <4 x i16>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load i16, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %6 = load <2 x i16>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %7 = load <3 x i16>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %8 = load <4 x i16>, ptr poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i16, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i16>, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <3 x i16>, ptr poison, align 4
@@ -75,24 +120,62 @@ define void @loads_i16(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %16 = load <4 x i16>, ptr poison, align 8
; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; GFX90APLUS-LABEL: 'loads_i16'
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i16, ptr poison, align 2
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i16>, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i16>, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i16>, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i16, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i16>, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i16>, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i16>, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i16, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i16>, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <3 x i16>, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i16>, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load i16, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <2 x i16>, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <3 x i16>, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = load <4 x i16>, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX90A-UNALIGNED-LABEL: 'loads_i16'
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i16, ptr poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i16>, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i16>, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i16>, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i16, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i16>, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i16>, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i16>, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i16, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i16>, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <3 x i16>, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i16>, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load i16, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <2 x i16>, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <3 x i16>, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = load <4 x i16>, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90A-NOUNALIGNED-LABEL: 'loads_i16'
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i16, ptr poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i16>, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i16>, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i16>, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load i16, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %6 = load <2 x i16>, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %7 = load <3 x i16>, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %8 = load <4 x i16>, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i16, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i16>, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <3 x i16>, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i16>, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load i16, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <2 x i16>, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <3 x i16>, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = load <4 x i16>, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX12-LABEL: 'loads_i16'
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i16, ptr poison, align 2
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i16>, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i16>, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i16>, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i16, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i16>, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i16>, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i16>, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i16, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i16>, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <3 x i16>, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i16>, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load i16, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <2 x i16>, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <3 x i16>, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = load <4 x i16>, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
load i16, ptr poison
load <2 x i16>, ptr poison
@@ -123,10 +206,10 @@ define void @loads_i32(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <2 x i32>, ptr poison, align 8
; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %3 = load <3 x i32>, ptr poison, align 16
; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %4 = load <4 x i32>, ptr poison, align 16
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i32, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load <2 x i32>, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %7 = load <3 x i32>, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %8 = load <4 x i32>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %5 = load i32, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %6 = load <2 x i32>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %7 = load <3 x i32>, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %8 = load <4 x i32>, ptr poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i32, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %10 = load <2 x i32>, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %11 = load <3 x i32>, ptr poison, align 4
@@ -141,28 +224,74 @@ define void @loads_i32(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %20 = load <4 x i32>, ptr poison, align 16
; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; GFX90APLUS-LABEL: 'loads_i32'
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i32, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i32>, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr poison, align 16
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i32>, ptr poison, align 16
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i32, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i32>, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <3 x i32>, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i32>, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i32, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i32>, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i32>, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i32>, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load i32, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <2 x i32>, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <3 x i32>, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = load <4 x i32>, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %17 = load i32, ptr poison, align 16
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %18 = load <2 x i32>, ptr poison, align 16
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %19 = load <3 x i32>, ptr poison, align 16
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %20 = load <4 x i32>, ptr poison, align 16
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX90A-UNALIGNED-LABEL: 'loads_i32'
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i32, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i32>, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i32>, ptr poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i32, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i32>, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <3 x i32>, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i32>, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i32, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i32>, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i32>, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i32>, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load i32, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <2 x i32>, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <3 x i32>, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = load <4 x i32>, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %17 = load i32, ptr poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %18 = load <2 x i32>, ptr poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %19 = load <3 x i32>, ptr poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %20 = load <4 x i32>, ptr poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90A-NOUNALIGNED-LABEL: 'loads_i32'
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i32, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i32>, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i32>, ptr poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %5 = load i32, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %6 = load <2 x i32>, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %7 = load <3 x i32>, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %8 = load <4 x i32>, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i32, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i32>, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i32>, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i32>, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load i32, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <2 x i32>, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <3 x i32>, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = load <4 x i32>, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %17 = load i32, ptr poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %18 = load <2 x i32>, ptr poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %19 = load <3 x i32>, ptr poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %20 = load <4 x i32>, ptr poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX12-LABEL: 'loads_i32'
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i32, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i32>, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr poison, align 16
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i32>, ptr poison, align 16
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load i32, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <2 x i32>, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <3 x i32>, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i32>, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load i32, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i32>, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <3 x i32>, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <4 x i32>, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load i32, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <2 x i32>, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <3 x i32>, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %16 = load <4 x i32>, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %17 = load i32, ptr poison, align 16
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %18 = load <2 x i32>, ptr poison, align 16
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %19 = load <3 x i32>, ptr poison, align 16
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %20 = load <4 x i32>, ptr poison, align 16
+; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
load i32, ptr poison
load <2 x i32>, ptr poison
@@ -533,38 +662,52 @@ define void @loads_addrspace_5(i32 %arg) {
define void @loads_underaligned(i32 %arg) {
; GFX8-LABEL: 'loads_underaligned'
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4
-; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load i32, ptr addrspace(1) poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load i32, ptr addrspace(1) poison, align 2
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i32>, ptr addrspace(1) poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(1) poison, align 2
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load i32, ptr poison, align 2
-; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %11 = load <4 x i32>, ptr poison, align 2
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %10 = load i32, ptr poison, align 2
+; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %11 = load <4 x i32>, ptr poison, align 2
; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; GFX90A-LABEL: 'loads_underaligned'
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load i32, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load i32, ptr addrspace(1) poison, align 2
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i32>, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(1) poison, align 2
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load i32, ptr poison, align 2
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <4 x i32>, ptr poison, align 2
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX90A-UNALIGNED-LABEL: 'loads_underaligned'
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load i32, ptr addrspace(1) poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load i32, ptr addrspace(1) poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i32>, ptr addrspace(1) poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <4 x i32>, ptr addrspace(1) poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load i32, ptr poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <4 x i32>, ptr poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90A-NOUNALIGNED-LABEL: 'loads_underaligned'
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 12 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %6 = load i32, ptr addrspace(1) poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load i32, ptr addrspace(1) poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %8 = load <4 x i32>, ptr addrspace(1) poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %9 = load <4 x i32>, ptr addrspace(1) poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %10 = load i32, ptr poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %11 = load <4 x i32>, ptr poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; GFX12-LABEL: 'loads_underaligned'
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <3 x i32>, ptr addrspace(3) poison, align 1
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load <3 x i32>, ptr addrspace(3) poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <3 x i32>, ptr addrspace(3) poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load <4 x i32>, ptr addrspace(3) poison, align 4
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <4 x i32>, ptr addrspace(5) poison, align 1
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load i32, ptr addrspace(1) poison, align 1
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load i32, ptr addrspace(1) poison, align 2
@@ -639,18 +782,31 @@ define void @loads_widened_tail(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = load <7 x i16>, ptr addrspace(5) poison, align 8
; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; GFX90A-LABEL: 'loads_widened_tail'
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = load <5 x i8>, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <6 x i8>, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <7 x i8>, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %4 = load <7 x i8>, ptr addrspace(1) poison, align 2
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load <7 x i16>, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <7 x i16>, ptr addrspace(1) poison, align 8
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <7 x i8>, ptr addrspace(3) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <7 x i8>, ptr poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %9 = load <13 x i8>, ptr addrspace(5) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = load <7 x i16>, ptr addrspace(5) poison, align 8
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX90A-UNALIGNED-LABEL: 'loads_widened_tail'
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = load <5 x i8>, ptr addrspace(1) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <6 x i8>, ptr addrspace(1) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <7 x i8>, ptr addrspace(1) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %4 = load <7 x i8>, ptr addrspace(1) poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load <7 x i16>, ptr addrspace(1) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <7 x i16>, ptr addrspace(1) poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <7 x i8>, ptr addrspace(3) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <7 x i8>, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %9 = load <13 x i8>, ptr addrspace(5) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = load <7 x i16>, ptr addrspace(5) poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90A-NOUNALIGNED-LABEL: 'loads_widened_tail'
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = load <5 x i8>, ptr addrspace(1) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <6 x i8>, ptr addrspace(1) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <7 x i8>, ptr addrspace(1) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %4 = load <7 x i8>, ptr addrspace(1) poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load <7 x i16>, ptr addrspace(1) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <7 x i16>, ptr addrspace(1) poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <7 x i8>, ptr addrspace(3) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <7 x i8>, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %9 = load <13 x i8>, ptr addrspace(5) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %10 = load <7 x i16>, ptr addrspace(5) poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; GFX12-LABEL: 'loads_widened_tail'
; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = load <5 x i8>, ptr addrspace(1) poison, align 4
@@ -690,24 +846,39 @@ define void @loads_odd_widths(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i24>, ptr addrspace(1) poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %9 = load <3 x i24>, ptr addrspace(1) poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i48>, ptr addrspace(1) poison, align 8
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <4 x i24>, ptr addrspace(3) poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <4 x i24>, ptr addrspace(3) poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %12 = load <2 x i96>, ptr addrspace(5) poison, align 16
; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; GFX90A-LABEL: 'loads_odd_widths'
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = load i24, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load i48, ptr addrspace(1) poison, align 2
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load i24, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load i48, ptr addrspace(1) poison, align 8
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load i80, ptr addrspace(1) poison, align 16
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load i112, ptr addrspace(1) poison, align 16
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i12>, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i24>, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %9 = load <3 x i24>, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i48>, ptr addrspace(1) poison, align 8
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <4 x i24>, ptr addrspace(3) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %12 = load <2 x i96>, ptr addrspace(5) poison, align 16
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX90A-UNALIGNED-LABEL: 'loads_odd_widths'
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = load i24, ptr addrspace(1) poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %2 = load i48, ptr addrspace(1) poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load i24, ptr addrspace(1) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load i48, ptr addrspace(1) poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load i80, ptr addrspace(1) poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load i112, ptr addrspace(1) poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i12>, ptr addrspace(1) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i24>, ptr addrspace(1) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %9 = load <3 x i24>, ptr addrspace(1) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i48>, ptr addrspace(1) poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <4 x i24>, ptr addrspace(3) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %12 = load <2 x i96>, ptr addrspace(5) poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90A-NOUNALIGNED-LABEL: 'loads_odd_widths'
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %1 = load i24, ptr addrspace(1) poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %2 = load i48, ptr addrspace(1) poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load i24, ptr addrspace(1) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %4 = load i48, ptr addrspace(1) poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %5 = load i80, ptr addrspace(1) poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %6 = load i112, ptr addrspace(1) poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %7 = load <3 x i12>, ptr addrspace(1) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i24>, ptr addrspace(1) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 9 for instruction: %9 = load <3 x i24>, ptr addrspace(1) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i48>, ptr addrspace(1) poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <4 x i24>, ptr addrspace(3) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %12 = load <2 x i96>, ptr addrspace(5) poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; GFX12-LABEL: 'loads_odd_widths'
; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %1 = load i24, ptr addrspace(1) poison, align 1
@@ -720,7 +891,7 @@ define void @loads_odd_widths(i32 %arg) {
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <4 x i24>, ptr addrspace(1) poison, align 4
; GFX12-NEXT: Cost Model: Found an estimated cost of 6 for instruction: %9 = load <3 x i24>, ptr addrspace(1) poison, align 4
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <2 x i48>, ptr addrspace(1) poison, align 8
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <4 x i24>, ptr addrspace(3) poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %11 = load <4 x i24>, ptr addrspace(3) poison, align 4
; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %12 = load <2 x i96>, ptr addrspace(5) poison, align 16
; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/store.ll b/llvm/test/Analysis/CostModel/AMDGPU/store.ll
index ce618c13f6a419..902fb5476148cd 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/store.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/store.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu8.03-unknown-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX8
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.0a-unknown-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX90APLUS,GFX90A
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.0a-unknown-amdhsa -mattr=-unaligned-access-mode < %s | FileCheck %s --check-prefixes=CHECK,GFX90APLUS,GFX90A
-; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu12.01-unknown-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX90APLUS,GFX12
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.0a-unknown-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX90A,GFX90A-UNALIGNED
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu9.0a-unknown-amdhsa -mattr=-unaligned-access-mode < %s | FileCheck %s --check-prefixes=CHECK,GFX90A,GFX90A-NOUNALIGNED
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgpu12.01-unknown-amdhsa < %s | FileCheck %s --check-prefixes=CHECK,GFX12
define void @stores_i1(i32 %arg) {
; CHECK-LABEL: 'stores_i1'
@@ -21,20 +21,65 @@ define void @stores_i1(i32 %arg) {
}
define void @stores_i8(i32 %arg) {
-; CHECK-LABEL: 'stores_i8'
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 2
-; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 1
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
-; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX8-LABEL: 'stores_i8'
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 2
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i8> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90A-UNALIGNED-LABEL: 'stores_i8'
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90A-NOUNALIGNED-LABEL: 'stores_i8'
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i8> poison, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i8> poison, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i8> poison, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX12-LABEL: 'stores_i8'
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 2
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i8 poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i8> poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i8> poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i8> poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
store i8 poison, ptr poison
store <2 x i8> poison, ptr poison
@@ -60,10 +105,10 @@ define void @stores_i16(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 8
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> poison, ptr poison, align 8
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i16 poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <2 x i16> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 6 for instruction: store <3 x i16> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <4 x i16> poison, ptr poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 4
@@ -74,24 +119,62 @@ define void @stores_i16(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i16> poison, ptr poison, align 8
; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; GFX90APLUS-LABEL: 'stores_i16'
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 2
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX90A-UNALIGNED-LABEL: 'stores_i16'
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90A-NOUNALIGNED-LABEL: 'stores_i16'
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i16 poison, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <2 x i16> poison, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 6 for instruction: store <3 x i16> poison, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <4 x i16> poison, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX12-LABEL: 'stores_i16'
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 2
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i16 poison, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i16> poison, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i16> poison, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i16> poison, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
store i16 poison, ptr poison
store <2 x i16> poison, ptr poison
@@ -122,10 +205,10 @@ define void @stores_i32(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> poison, ptr poison, align 8
; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr poison, align 16
; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr poison, align 16
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> poison, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store i32 poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <2 x i32> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <3 x i32> poison, ptr poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <4 x i32> poison, ptr poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i32> poison, ptr poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store <3 x i32> poison, ptr poison, align 4
@@ -140,28 +223,74 @@ define void @stores_i32(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr poison, align 16
; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; GFX90APLUS-LABEL: 'stores_i32'
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 16
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 16
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 1
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 4
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 8
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 16
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 16
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 16
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 16
-; GFX90APLUS-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX90A-UNALIGNED-LABEL: 'stores_i32'
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90A-NOUNALIGNED-LABEL: 'stores_i32'
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store i32 poison, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <2 x i32> poison, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <3 x i32> poison, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <4 x i32> poison, ptr poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX12-LABEL: 'stores_i32'
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 16
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 16
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 1
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 16
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i32> poison, ptr poison, align 16
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr poison, align 16
+; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 16
+; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
store i32 poison, ptr poison
store <2 x i32> poison, ptr poison
@@ -535,38 +664,52 @@ define void @stores_addrspace_5(i32 %arg) {
define void @stores_underaligned(i32 %arg) {
; GFX8-LABEL: 'stores_underaligned'
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4
-; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(1) poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(1) poison, align 2
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(1) poison, align 1
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(1) poison, align 2
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 2
-; GFX8-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr poison, align 2
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i32 poison, ptr poison, align 2
+; GFX8-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <4 x i32> poison, ptr poison, align 2
; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; GFX90A-LABEL: 'stores_underaligned'
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(1) poison, align 2
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(1) poison, align 2
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 2
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 2
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX90A-UNALIGNED-LABEL: 'stores_underaligned'
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(1) poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(1) poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(1) poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(1) poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90A-NOUNALIGNED-LABEL: 'stores_underaligned'
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 12 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 4 for instruction: store i32 poison, ptr addrspace(1) poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i32 poison, ptr addrspace(1) poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 16 for instruction: store <4 x i32> poison, ptr addrspace(1) poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <4 x i32> poison, ptr addrspace(1) poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i32 poison, ptr poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 8 for instruction: store <4 x i32> poison, ptr poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; GFX12-LABEL: 'stores_underaligned'
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 1
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i32> poison, ptr addrspace(3) poison, align 8
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i32> poison, ptr addrspace(3) poison, align 4
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i32> poison, ptr addrspace(5) poison, align 1
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(1) poison, align 1
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store i32 poison, ptr addrspace(1) poison, align 2
@@ -647,24 +790,39 @@ define void @stores_odd_widths(i32 %arg) {
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i24> poison, ptr addrspace(1) poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 6 for instruction: store <3 x i24> poison, ptr addrspace(1) poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i48> poison, ptr addrspace(1) poison, align 8
-; GFX8-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i24> poison, ptr addrspace(3) poison, align 4
+; GFX8-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i24> poison, ptr addrspace(3) poison, align 4
; GFX8-NEXT: Cost Model: Found an estimated cost of 6 for instruction: store <2 x i96> poison, ptr addrspace(5) poison, align 16
; GFX8-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
-; GFX90A-LABEL: 'stores_odd_widths'
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i24 poison, ptr addrspace(1) poison, align 1
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i48 poison, ptr addrspace(1) poison, align 2
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i24 poison, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i48 poison, ptr addrspace(1) poison, align 8
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i80 poison, ptr addrspace(1) poison, align 16
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i112 poison, ptr addrspace(1) poison, align 16
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i12> poison, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i24> poison, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 6 for instruction: store <3 x i24> poison, ptr addrspace(1) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i48> poison, ptr addrspace(1) poison, align 8
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i24> poison, ptr addrspace(3) poison, align 4
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 6 for instruction: store <2 x i96> poison, ptr addrspace(5) poison, align 16
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+; GFX90A-UNALIGNED-LABEL: 'stores_odd_widths'
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i24 poison, ptr addrspace(1) poison, align 1
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i48 poison, ptr addrspace(1) poison, align 2
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i24 poison, ptr addrspace(1) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i48 poison, ptr addrspace(1) poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i80 poison, ptr addrspace(1) poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i112 poison, ptr addrspace(1) poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i12> poison, ptr addrspace(1) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i24> poison, ptr addrspace(1) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 6 for instruction: store <3 x i24> poison, ptr addrspace(1) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i48> poison, ptr addrspace(1) poison, align 8
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i24> poison, ptr addrspace(3) poison, align 4
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 6 for instruction: store <2 x i96> poison, ptr addrspace(5) poison, align 16
+; GFX90A-UNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+; GFX90A-NOUNALIGNED-LABEL: 'stores_odd_widths'
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store i24 poison, ptr addrspace(1) poison, align 1
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 3 for instruction: store i48 poison, ptr addrspace(1) poison, align 2
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i24 poison, ptr addrspace(1) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i48 poison, ptr addrspace(1) poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i80 poison, ptr addrspace(1) poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i112 poison, ptr addrspace(1) poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <3 x i12> poison, ptr addrspace(1) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i24> poison, ptr addrspace(1) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 9 for instruction: store <3 x i24> poison, ptr addrspace(1) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i48> poison, ptr addrspace(1) poison, align 8
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i24> poison, ptr addrspace(3) poison, align 4
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 6 for instruction: store <2 x i96> poison, ptr addrspace(5) poison, align 16
+; GFX90A-NOUNALIGNED-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
; GFX12-LABEL: 'stores_odd_widths'
; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store i24 poison, ptr addrspace(1) poison, align 1
@@ -677,7 +835,7 @@ define void @stores_odd_widths(i32 %arg) {
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i24> poison, ptr addrspace(1) poison, align 4
; GFX12-NEXT: Cost Model: Found an estimated cost of 6 for instruction: store <3 x i24> poison, ptr addrspace(1) poison, align 4
; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <2 x i48> poison, ptr addrspace(1) poison, align 8
-; GFX12-NEXT: Cost Model: Found an estimated cost of 1 for instruction: store <4 x i24> poison, ptr addrspace(3) poison, align 4
+; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <4 x i24> poison, ptr addrspace(3) poison, align 4
; GFX12-NEXT: Cost Model: Found an estimated cost of 2 for instruction: store <2 x i96> poison, ptr addrspace(5) poison, align 16
; GFX12-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll
index 49d7ee569c8711..32a5ac77bd3472 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/packed-math.ll
@@ -72,13 +72,30 @@ define amdgpu_kernel void @test1_as_0_0_3_v2f16(ptr %a, ptr %b, ptr addrspace(3)
}
define amdgpu_kernel void @test1_fma_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d) {
-; GCN-LABEL: @test1_fma_v2f16(
-; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
-; GCN-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[B:%.*]], align 2
-; GCN-NEXT: [[TMP3:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2
-; GCN-NEXT: [[TMP4:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP1]], <2 x half> [[TMP2]], <2 x half> [[TMP3]])
-; GCN-NEXT: store <2 x half> [[TMP4]], ptr addrspace(3) [[D:%.*]], align 2
-; GCN-NEXT: ret void
+; GFX9-LABEL: @test1_fma_v2f16(
+; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
+; GFX9-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[B:%.*]], align 2
+; GFX9-NEXT: [[TMP3:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2
+; GFX9-NEXT: [[TMP4:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP1]], <2 x half> [[TMP2]], <2 x half> [[TMP3]])
+; GFX9-NEXT: store <2 x half> [[TMP4]], ptr addrspace(3) [[D:%.*]], align 2
+; GFX9-NEXT: ret void
+;
+; VI-LABEL: @test1_fma_v2f16(
+; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2
+; VI-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2
+; VI-NEXT: [[I2:%.*]] = load half, ptr addrspace(3) [[C:%.*]], align 2
+; VI-NEXT: [[FMA0:%.*]] = call half @llvm.fma.f16(half [[I0]], half [[I1]], half [[I2]])
+; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1
+; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2
+; VI-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1
+; VI-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2
+; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1
+; VI-NEXT: [[I5:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX5]], align 2
+; VI-NEXT: [[FMA1:%.*]] = call half @llvm.fma.f16(half [[I3]], half [[I4]], half [[I5]])
+; VI-NEXT: store half [[FMA0]], ptr addrspace(3) [[D:%.*]], align 2
+; VI-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[D]], i64 1
+; VI-NEXT: store half [[FMA1]], ptr addrspace(3) [[ARRAYIDX6]], align 2
+; VI-NEXT: ret void
;
%i0 = load half, ptr addrspace(3) %a, align 2
%i1 = load half, ptr addrspace(3) %b, align 2
@@ -98,13 +115,24 @@ define amdgpu_kernel void @test1_fma_v2f16(ptr addrspace(3) %a, ptr addrspace(3)
}
define amdgpu_kernel void @mul_scalar_v2f16(ptr addrspace(3) %a, half %scalar, ptr addrspace(3) %c) {
-; GCN-LABEL: @mul_scalar_v2f16(
-; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
-; GCN-NEXT: [[TMP2:%.*]] = insertelement <2 x half> poison, half [[SCALAR:%.*]], i64 0
-; GCN-NEXT: [[TMP3:%.*]] = shufflevector <2 x half> [[TMP2]], <2 x half> poison, <2 x i32> zeroinitializer
-; GCN-NEXT: [[TMP4:%.*]] = fmul <2 x half> [[TMP1]], [[TMP3]]
-; GCN-NEXT: store <2 x half> [[TMP4]], ptr addrspace(3) [[C:%.*]], align 2
-; GCN-NEXT: ret void
+; GFX9-LABEL: @mul_scalar_v2f16(
+; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
+; GFX9-NEXT: [[TMP2:%.*]] = insertelement <2 x half> poison, half [[SCALAR:%.*]], i64 0
+; GFX9-NEXT: [[TMP3:%.*]] = shufflevector <2 x half> [[TMP2]], <2 x half> poison, <2 x i32> zeroinitializer
+; GFX9-NEXT: [[TMP4:%.*]] = fmul <2 x half> [[TMP1]], [[TMP3]]
+; GFX9-NEXT: store <2 x half> [[TMP4]], ptr addrspace(3) [[C:%.*]], align 2
+; GFX9-NEXT: ret void
+;
+; VI-LABEL: @mul_scalar_v2f16(
+; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2
+; VI-NEXT: [[MUL:%.*]] = fmul half [[I0]], [[SCALAR:%.*]]
+; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1
+; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2
+; VI-NEXT: [[MUL5:%.*]] = fmul half [[I3]], [[SCALAR]]
+; VI-NEXT: store half [[MUL]], ptr addrspace(3) [[C:%.*]], align 2
+; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1
+; VI-NEXT: store half [[MUL5]], ptr addrspace(3) [[ARRAYIDX5]], align 2
+; VI-NEXT: ret void
;
%i0 = load half, ptr addrspace(3) %a, align 2
%mul = fmul half %i0, %scalar
@@ -118,11 +146,22 @@ define amdgpu_kernel void @mul_scalar_v2f16(ptr addrspace(3) %a, half %scalar, p
}
define amdgpu_kernel void @fabs_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %c) {
-; GCN-LABEL: @fabs_v2f16(
-; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
-; GCN-NEXT: [[TMP2:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[TMP1]])
-; GCN-NEXT: store <2 x half> [[TMP2]], ptr addrspace(3) [[C:%.*]], align 2
-; GCN-NEXT: ret void
+; GFX9-LABEL: @fabs_v2f16(
+; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
+; GFX9-NEXT: [[TMP2:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[TMP1]])
+; GFX9-NEXT: store <2 x half> [[TMP2]], ptr addrspace(3) [[C:%.*]], align 2
+; GFX9-NEXT: ret void
+;
+; VI-LABEL: @fabs_v2f16(
+; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2
+; VI-NEXT: [[FABS0:%.*]] = call half @llvm.fabs.f16(half [[I0]])
+; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1
+; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2
+; VI-NEXT: [[FABS1:%.*]] = call half @llvm.fabs.f16(half [[I3]])
+; VI-NEXT: store half [[FABS0]], ptr addrspace(3) [[C:%.*]], align 2
+; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1
+; VI-NEXT: store half [[FABS1]], ptr addrspace(3) [[ARRAYIDX5]], align 2
+; VI-NEXT: ret void
;
%i0 = load half, ptr addrspace(3) %a, align 2
%fabs0 = call half @llvm.fabs.f16(half %i0)
@@ -136,14 +175,33 @@ define amdgpu_kernel void @fabs_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %c)
}
define amdgpu_kernel void @test1_fabs_fma_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d) {
-; GCN-LABEL: @test1_fabs_fma_v2f16(
-; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
-; GCN-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[B:%.*]], align 2
-; GCN-NEXT: [[TMP3:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2
-; GCN-NEXT: [[TMP4:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[TMP1]])
-; GCN-NEXT: [[TMP5:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP4]], <2 x half> [[TMP2]], <2 x half> [[TMP3]])
-; GCN-NEXT: store <2 x half> [[TMP5]], ptr addrspace(3) [[D:%.*]], align 2
-; GCN-NEXT: ret void
+; GFX9-LABEL: @test1_fabs_fma_v2f16(
+; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
+; GFX9-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[B:%.*]], align 2
+; GFX9-NEXT: [[TMP3:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2
+; GFX9-NEXT: [[TMP4:%.*]] = call <2 x half> @llvm.fabs.v2f16(<2 x half> [[TMP1]])
+; GFX9-NEXT: [[TMP5:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP4]], <2 x half> [[TMP2]], <2 x half> [[TMP3]])
+; GFX9-NEXT: store <2 x half> [[TMP5]], ptr addrspace(3) [[D:%.*]], align 2
+; GFX9-NEXT: ret void
+;
+; VI-LABEL: @test1_fabs_fma_v2f16(
+; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2
+; VI-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2
+; VI-NEXT: [[I2:%.*]] = load half, ptr addrspace(3) [[C:%.*]], align 2
+; VI-NEXT: [[I0_FABS:%.*]] = call half @llvm.fabs.f16(half [[I0]])
+; VI-NEXT: [[FMA0:%.*]] = call half @llvm.fma.f16(half [[I0_FABS]], half [[I1]], half [[I2]])
+; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1
+; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2
+; VI-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1
+; VI-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2
+; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1
+; VI-NEXT: [[I5:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX5]], align 2
+; VI-NEXT: [[I3_FABS:%.*]] = call half @llvm.fabs.f16(half [[I3]])
+; VI-NEXT: [[FMA1:%.*]] = call half @llvm.fma.f16(half [[I3_FABS]], half [[I4]], half [[I5]])
+; VI-NEXT: store half [[FMA0]], ptr addrspace(3) [[D:%.*]], align 2
+; VI-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[D]], i64 1
+; VI-NEXT: store half [[FMA1]], ptr addrspace(3) [[ARRAYIDX6]], align 2
+; VI-NEXT: ret void
;
%i0 = load half, ptr addrspace(3) %a, align 2
%i1 = load half, ptr addrspace(3) %b, align 2
@@ -167,18 +225,36 @@ define amdgpu_kernel void @test1_fabs_fma_v2f16(ptr addrspace(3) %a, ptr addrspa
}
define amdgpu_kernel void @test1_fabs_scalar_fma_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %b, ptr addrspace(3) %c, ptr addrspace(3) %d) {
-; GCN-LABEL: @test1_fabs_scalar_fma_v2f16(
-; GCN-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2
-; GCN-NEXT: [[I1_FABS:%.*]] = call half @llvm.fabs.f16(half [[I1]])
-; GCN-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1
-; GCN-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2
-; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
-; GCN-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2
-; GCN-NEXT: [[TMP3:%.*]] = insertelement <2 x half> poison, half [[I1_FABS]], i64 0
-; GCN-NEXT: [[TMP4:%.*]] = insertelement <2 x half> [[TMP3]], half [[I4]], i64 1
-; GCN-NEXT: [[TMP5:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP1]], <2 x half> [[TMP4]], <2 x half> [[TMP2]])
-; GCN-NEXT: store <2 x half> [[TMP5]], ptr addrspace(3) [[D:%.*]], align 2
-; GCN-NEXT: ret void
+; GFX9-LABEL: @test1_fabs_scalar_fma_v2f16(
+; GFX9-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2
+; GFX9-NEXT: [[I1_FABS:%.*]] = call half @llvm.fabs.f16(half [[I1]])
+; GFX9-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1
+; GFX9-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2
+; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
+; GFX9-NEXT: [[TMP2:%.*]] = load <2 x half>, ptr addrspace(3) [[C:%.*]], align 2
+; GFX9-NEXT: [[TMP3:%.*]] = insertelement <2 x half> poison, half [[I1_FABS]], i64 0
+; GFX9-NEXT: [[TMP4:%.*]] = insertelement <2 x half> [[TMP3]], half [[I4]], i64 1
+; GFX9-NEXT: [[TMP5:%.*]] = call <2 x half> @llvm.fma.v2f16(<2 x half> [[TMP1]], <2 x half> [[TMP4]], <2 x half> [[TMP2]])
+; GFX9-NEXT: store <2 x half> [[TMP5]], ptr addrspace(3) [[D:%.*]], align 2
+; GFX9-NEXT: ret void
+;
+; VI-LABEL: @test1_fabs_scalar_fma_v2f16(
+; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2
+; VI-NEXT: [[I1:%.*]] = load half, ptr addrspace(3) [[B:%.*]], align 2
+; VI-NEXT: [[I2:%.*]] = load half, ptr addrspace(3) [[C:%.*]], align 2
+; VI-NEXT: [[I1_FABS:%.*]] = call half @llvm.fabs.f16(half [[I1]])
+; VI-NEXT: [[FMA0:%.*]] = call half @llvm.fma.f16(half [[I0]], half [[I1_FABS]], half [[I2]])
+; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1
+; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2
+; VI-NEXT: [[ARRAYIDX4:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[B]], i64 1
+; VI-NEXT: [[I4:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX4]], align 2
+; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1
+; VI-NEXT: [[I5:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX5]], align 2
+; VI-NEXT: [[FMA1:%.*]] = call half @llvm.fma.f16(half [[I3]], half [[I4]], half [[I5]])
+; VI-NEXT: store half [[FMA0]], ptr addrspace(3) [[D:%.*]], align 2
+; VI-NEXT: [[ARRAYIDX6:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[D]], i64 1
+; VI-NEXT: store half [[FMA1]], ptr addrspace(3) [[ARRAYIDX6]], align 2
+; VI-NEXT: ret void
;
%i0 = load half, ptr addrspace(3) %a, align 2
%i1 = load half, ptr addrspace(3) %b, align 2
@@ -200,11 +276,22 @@ define amdgpu_kernel void @test1_fabs_scalar_fma_v2f16(ptr addrspace(3) %a, ptr
}
define amdgpu_kernel void @canonicalize_v2f16(ptr addrspace(3) %a, ptr addrspace(3) %c) {
-; GCN-LABEL: @canonicalize_v2f16(
-; GCN-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
-; GCN-NEXT: [[TMP2:%.*]] = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> [[TMP1]])
-; GCN-NEXT: store <2 x half> [[TMP2]], ptr addrspace(3) [[C:%.*]], align 2
-; GCN-NEXT: ret void
+; GFX9-LABEL: @canonicalize_v2f16(
+; GFX9-NEXT: [[TMP1:%.*]] = load <2 x half>, ptr addrspace(3) [[A:%.*]], align 2
+; GFX9-NEXT: [[TMP2:%.*]] = call <2 x half> @llvm.canonicalize.v2f16(<2 x half> [[TMP1]])
+; GFX9-NEXT: store <2 x half> [[TMP2]], ptr addrspace(3) [[C:%.*]], align 2
+; GFX9-NEXT: ret void
+;
+; VI-LABEL: @canonicalize_v2f16(
+; VI-NEXT: [[I0:%.*]] = load half, ptr addrspace(3) [[A:%.*]], align 2
+; VI-NEXT: [[CANONICALIZE0:%.*]] = call half @llvm.canonicalize.f16(half [[I0]])
+; VI-NEXT: [[ARRAYIDX3:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[A]], i64 1
+; VI-NEXT: [[I3:%.*]] = load half, ptr addrspace(3) [[ARRAYIDX3]], align 2
+; VI-NEXT: [[CANONICALIZE1:%.*]] = call half @llvm.canonicalize.f16(half [[I3]])
+; VI-NEXT: store half [[CANONICALIZE0]], ptr addrspace(3) [[C:%.*]], align 2
+; VI-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds half, ptr addrspace(3) [[C]], i64 1
+; VI-NEXT: store half [[CANONICALIZE1]], ptr addrspace(3) [[ARRAYIDX5]], align 2
+; VI-NEXT: ret void
;
%i0 = load half, ptr addrspace(3) %a, align 2
%canonicalize0 = call half @llvm.canonicalize.f16(half %i0)
@@ -223,6 +310,3 @@ declare half @llvm.canonicalize.f16(half) #1
attributes #0 = { nounwind }
attributes #1 = { nounwind readnone }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX9: {{.*}}
-; VI: {{.*}}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll
index 414ffe6cbec23e..4d20feaa6b94f5 100644
--- a/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8.ll
@@ -787,6 +787,10 @@ define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(<4 x i8> %in
; GFX7-LABEL: define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(
; GFX7-SAME: <4 x i8> [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr addrspace(3) [[OUT1:%.*]], i32 [[FLAG:%.*]]) {
; GFX7-NEXT: [[ENTRY:.*]]:
+; GFX7-NEXT: [[OTHERELE0:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 0
+; GFX7-NEXT: [[OTHERELE1:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 1
+; GFX7-NEXT: [[OTHERELE2:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 2
+; GFX7-NEXT: [[OTHERELE3:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 3
; GFX7-NEXT: br label %[[DO_BODY:.*]]
; GFX7: [[DO_BODY]]:
; GFX7-NEXT: [[TMP0:%.*]] = phi <4 x i8> [ [[INPTR0]], %[[ENTRY]] ], [ [[INPTR0]], %[[DO_BODY]] ]
@@ -796,12 +800,22 @@ define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(<4 x i8> %in
; GFX7-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
; GFX7: [[EXIT]]:
; GFX7-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 0
-; GFX7-NEXT: store <4 x i8> [[INPTR0]], ptr addrspace(3) [[GEP0]], align 1
+; GFX7-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 1
+; GFX7-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 2
+; GFX7-NEXT: [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 3
+; GFX7-NEXT: store i8 [[OTHERELE0]], ptr addrspace(3) [[GEP0]], align 1
+; GFX7-NEXT: store i8 [[OTHERELE1]], ptr addrspace(3) [[GEP1]], align 1
+; GFX7-NEXT: store i8 [[OTHERELE2]], ptr addrspace(3) [[GEP2]], align 1
+; GFX7-NEXT: store i8 [[OTHERELE3]], ptr addrspace(3) [[GEP3]], align 1
; GFX7-NEXT: ret void
;
; GFX8-LABEL: define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(
; GFX8-SAME: <4 x i8> [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr addrspace(3) [[OUT1:%.*]], i32 [[FLAG:%.*]]) {
; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: [[OTHERELE0:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 0
+; GFX8-NEXT: [[OTHERELE1:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 1
+; GFX8-NEXT: [[OTHERELE2:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 2
+; GFX8-NEXT: [[OTHERELE3:%.*]] = extractelement <4 x i8> [[INPTR0]], i32 3
; GFX8-NEXT: br label %[[DO_BODY:.*]]
; GFX8: [[DO_BODY]]:
; GFX8-NEXT: [[TMP0:%.*]] = phi <4 x i8> [ [[INPTR0]], %[[ENTRY]] ], [ [[INPTR0]], %[[DO_BODY]] ]
@@ -811,7 +825,13 @@ define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(<4 x i8> %in
; GFX8-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
; GFX8: [[EXIT]]:
; GFX8-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 0
-; GFX8-NEXT: store <4 x i8> [[INPTR0]], ptr addrspace(3) [[GEP0]], align 1
+; GFX8-NEXT: [[GEP1:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 1
+; GFX8-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 2
+; GFX8-NEXT: [[GEP3:%.*]] = getelementptr i8, ptr addrspace(3) [[OUT1]], i32 3
+; GFX8-NEXT: store i8 [[OTHERELE0]], ptr addrspace(3) [[GEP0]], align 1
+; GFX8-NEXT: store i8 [[OTHERELE1]], ptr addrspace(3) [[GEP1]], align 1
+; GFX8-NEXT: store i8 [[OTHERELE2]], ptr addrspace(3) [[GEP2]], align 1
+; GFX8-NEXT: store i8 [[OTHERELE3]], ptr addrspace(3) [[GEP3]], align 1
; GFX8-NEXT: ret void
;
; GFX9-LABEL: define protected amdgpu_kernel void @phi_4_with_stores_outside_loop(
>From 819d2505d4ca1ad8fd4f055340a88c463f653b7d Mon Sep 17 00:00:00 2001
From: Dmitry Sidorov <Dmitry.Sidorov at amd.com>
Date: Sun, 25 Oct 2026 19:59:40 +0100
Subject: [PATCH 2/2] if -> elif
---
llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 3e4e0a2b16fef5..544c8914669f98 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -2226,7 +2226,7 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
if (AddressSpace == AMDGPUAS::PRIVATE_ADDRESS ||
AddressSpace == AMDGPUAS::FLAT_ADDRESS)
HasUnalignedAccess = ST->hasUnalignedScratchAccessEnabled();
- if (IsDS) {
+ else if (IsDS) {
HasUnalignedAccess = ST->hasUnalignedDSAccessEnabled();
if (Alignment >= 4 && Alignment < 8)
MaxBytes = 8;
More information about the llvm-branch-commits
mailing list