[llvm] [AMDGPU] Enable vectorization of i8 values. (PR #134934)
Gheorghe-Teodor Bercea via llvm-commits
llvm-commits at lists.llvm.org
Tue Apr 22 16:15:12 PDT 2025
https://github.com/doru1004 updated https://github.com/llvm/llvm-project/pull/134934
>From ab2a49c23ff1aca4927ec079844da77275aa3b9c Mon Sep 17 00:00:00 2001
From: Doru Bercea <doru.bercea at amd.com>
Date: Mon, 7 Apr 2025 11:03:20 -0400
Subject: [PATCH] Enable vectorization of i8 values.
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 36 +-
.../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 14 +
llvm/test/Analysis/CostModel/AMDGPU/load.ll | 466 ++++++++
.../AMDGPU/vectorize-i8-as-i32.ll | 1012 +++++++++++++++++
4 files changed, 1525 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/Analysis/CostModel/AMDGPU/load.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8-as-i32.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 09f7877b13b3a..38344c42f7f1b 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -344,9 +344,10 @@ unsigned GCNTTIImpl::getMinVectorRegisterBitWidth() const {
unsigned GCNTTIImpl::getMaximumVF(unsigned ElemWidth, unsigned Opcode) const {
if (Opcode == Instruction::Load || Opcode == Instruction::Store)
return 32 * 4 / ElemWidth;
- return (ElemWidth == 16 && ST->has16BitInsts()) ? 2
- : (ElemWidth == 32 && ST->hasPackedFP32Ops()) ? 2
- : 1;
+ return ElemWidth == 8 ? 4
+ : (ElemWidth == 16 && ST->has16BitInsts()) ? 2
+ : (ElemWidth == 32 && ST->hasPackedFP32Ops()) ? 2
+ : 1;
}
unsigned GCNTTIImpl::getLoadVectorFactor(unsigned VF, unsigned LoadSize,
@@ -1423,3 +1424,32 @@ void GCNTTIImpl::collectKernelLaunchBounds(
LB.push_back({"amdgpu-waves-per-eu[0]", WavesPerEU.first});
LB.push_back({"amdgpu-waves-per-eu[1]", WavesPerEU.second});
}
+
+InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
+ Align Alignment,
+ unsigned AddressSpace,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo OpInfo,
+ const Instruction *I) {
+ if (VectorType *VecTy = dyn_cast<VectorType>(Src))
+ if (Opcode == Instruction::Load &&
+ VecTy->getElementType() ==
+ IntegerType::getInt8Ty(VecTy->getContext())) {
+ unsigned ElementCount = VecTy->getElementCount().getFixedValue();
+ return (8 * (ElementCount - 1) /
+ getLoadStoreVecRegBitWidth(AddressSpace)) +
+ 1;
+ }
+ return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind,
+ OpInfo, I);
+}
+
+unsigned GCNTTIImpl::getNumberOfParts(Type *Tp) {
+ if (VectorType *VecTy = dyn_cast<VectorType>(Tp))
+ if (VecTy->getElementType() ==
+ IntegerType::getInt8Ty(VecTy->getContext())) {
+ unsigned ElementCount = VecTy->getElementCount().getFixedValue();
+ return ((ElementCount - 1) / 4) + 1;
+ }
+ return BaseT::getNumberOfParts(Tp);
+}
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index f5062070ac6f4..dcd3c70fbe4b6 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -282,6 +282,20 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
void collectKernelLaunchBounds(
const Function &F,
SmallVectorImpl<std::pair<StringRef, int64_t>> &LB) const;
+
+ /// Account for loads of i8 vector types to have reduced cost. For
+ /// example the cost of load 4 i8s values is one is the cost of loading
+ /// a single i32 value.
+ InstructionCost getMemoryOpCost(
+ unsigned Opcode, Type *Src, Align Alignment, unsigned AddressSpace,
+ TTI::TargetCostKind CostKind,
+ TTI::OperandValueInfo OpInfo = {TTI::OK_AnyValue, TTI::OP_None},
+ const Instruction *I = nullptr);
+
+ /// When counting parts on AMD GPUs, account for i8s being grouped
+ /// together under a single i32 value. Otherwise fall back to base
+ /// implementation.
+ unsigned getNumberOfParts(Type *Tp);
};
} // end namespace llvm
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/load.ll b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
new file mode 100644
index 0000000000000..a847440e25339
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
@@ -0,0 +1,466 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgcn-unknown-amdhsa -mcpu=gfx90a < %s | FileCheck %s --check-prefixes=GFX90A
+
+define i32 @loads_align4(i32 %arg) {
+ ; Scalars
+; GFX90A-LABEL: 'loads_align4'
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <16 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <32 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %4 = load <64 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %5 = load <128 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %6 = load <256 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret i32 undef
+;
+ load i8, ptr undef, align 4
+ load <16 x i8>, ptr undef, align 4
+ load <32 x i8>, ptr undef, align 4
+ load <64 x i8>, ptr undef, align 4
+ load <128 x i8>, ptr undef, align 4
+ load <256 x i8>, ptr undef, align 4
+
+ ret i32 undef
+}
+
+define i32 @loads_partial_align4(i32 %arg) {
+ ; Partial vectors
+; GFX90A-LABEL: 'loads_partial_align4'
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <1 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <5 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <6 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <7 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <8 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <9 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <10 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <11 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <12 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load <13 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <14 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <15 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %16 = load <17 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %17 = load <18 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %18 = load <19 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %19 = load <20 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %20 = load <21 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %21 = load <22 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %22 = load <23 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %23 = load <24 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %24 = load <25 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %25 = load <26 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %26 = load <27 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %27 = load <28 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %28 = load <29 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %29 = load <30 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %30 = load <31 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %31 = load <33 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %32 = load <34 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %33 = load <35 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %34 = load <36 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %35 = load <37 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %36 = load <38 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %37 = load <39 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %38 = load <40 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %39 = load <41 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %40 = load <42 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %41 = load <43 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %42 = load <44 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %43 = load <45 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %44 = load <46 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %45 = load <47 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %46 = load <48 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %47 = load <49 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %48 = load <50 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %49 = load <51 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %50 = load <52 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %51 = load <53 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %52 = load <54 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %53 = load <55 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %54 = load <56 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %55 = load <57 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %56 = load <58 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %57 = load <59 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %58 = load <60 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %59 = load <61 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %60 = load <62 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %61 = load <63 x i8>, ptr undef, align 4
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret i32 undef
+;
+ load <1 x i8>, ptr undef, align 4
+ load <2 x i8>, ptr undef, align 4
+ load <3 x i8>, ptr undef, align 4
+ load <4 x i8>, ptr undef, align 4
+ load <5 x i8>, ptr undef, align 4
+ load <6 x i8>, ptr undef, align 4
+ load <7 x i8>, ptr undef, align 4
+ load <8 x i8>, ptr undef, align 4
+ load <9 x i8>, ptr undef, align 4
+ load <10 x i8>, ptr undef, align 4
+ load <11 x i8>, ptr undef, align 4
+ load <12 x i8>, ptr undef, align 4
+ load <13 x i8>, ptr undef, align 4
+ load <14 x i8>, ptr undef, align 4
+ load <15 x i8>, ptr undef, align 4
+
+ load <17 x i8>, ptr undef, align 4
+ load <18 x i8>, ptr undef, align 4
+ load <19 x i8>, ptr undef, align 4
+ load <20 x i8>, ptr undef, align 4
+ load <21 x i8>, ptr undef, align 4
+ load <22 x i8>, ptr undef, align 4
+ load <23 x i8>, ptr undef, align 4
+ load <24 x i8>, ptr undef, align 4
+ load <25 x i8>, ptr undef, align 4
+ load <26 x i8>, ptr undef, align 4
+ load <27 x i8>, ptr undef, align 4
+ load <28 x i8>, ptr undef, align 4
+ load <29 x i8>, ptr undef, align 4
+ load <30 x i8>, ptr undef, align 4
+ load <31 x i8>, ptr undef, align 4
+
+ load <33 x i8>, ptr undef, align 4
+ load <34 x i8>, ptr undef, align 4
+ load <35 x i8>, ptr undef, align 4
+ load <36 x i8>, ptr undef, align 4
+ load <37 x i8>, ptr undef, align 4
+ load <38 x i8>, ptr undef, align 4
+ load <39 x i8>, ptr undef, align 4
+ load <40 x i8>, ptr undef, align 4
+ load <41 x i8>, ptr undef, align 4
+ load <42 x i8>, ptr undef, align 4
+ load <43 x i8>, ptr undef, align 4
+ load <44 x i8>, ptr undef, align 4
+ load <45 x i8>, ptr undef, align 4
+ load <46 x i8>, ptr undef, align 4
+ load <47 x i8>, ptr undef, align 4
+ load <48 x i8>, ptr undef, align 4
+ load <49 x i8>, ptr undef, align 4
+ load <50 x i8>, ptr undef, align 4
+ load <51 x i8>, ptr undef, align 4
+ load <52 x i8>, ptr undef, align 4
+ load <53 x i8>, ptr undef, align 4
+ load <54 x i8>, ptr undef, align 4
+ load <55 x i8>, ptr undef, align 4
+ load <56 x i8>, ptr undef, align 4
+ load <57 x i8>, ptr undef, align 4
+ load <58 x i8>, ptr undef, align 4
+ load <59 x i8>, ptr undef, align 4
+ load <60 x i8>, ptr undef, align 4
+ load <61 x i8>, ptr undef, align 4
+ load <62 x i8>, ptr undef, align 4
+ load <63 x i8>, ptr undef, align 4
+
+ ret i32 undef
+}
+
+define i32 @loads_align1(i32 %arg) {
+ ; Scalars
+; GFX90A-LABEL: 'loads_align1'
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <16 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <32 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %4 = load <64 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %5 = load <128 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %6 = load <256 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret i32 undef
+;
+ load i8, ptr undef, align 1
+ load <16 x i8>, ptr undef, align 1
+ load <32 x i8>, ptr undef, align 1
+ load <64 x i8>, ptr undef, align 1
+ load <128 x i8>, ptr undef, align 1
+ load <256 x i8>, ptr undef, align 1
+
+ ret i32 undef
+}
+
+define i32 @loads_partial_align1(i32 %arg) {
+ ; Partial vectors
+; GFX90A-LABEL: 'loads_partial_align1'
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <1 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <5 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <6 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <7 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <8 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <9 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <10 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <11 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <12 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load <13 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <14 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <15 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %16 = load <17 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %17 = load <18 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %18 = load <19 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %19 = load <20 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %20 = load <21 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %21 = load <22 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %22 = load <23 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %23 = load <24 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %24 = load <25 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %25 = load <26 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %26 = load <27 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %27 = load <28 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %28 = load <29 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %29 = load <30 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %30 = load <31 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %31 = load <33 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %32 = load <34 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %33 = load <35 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %34 = load <36 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %35 = load <37 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %36 = load <38 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %37 = load <39 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %38 = load <40 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %39 = load <41 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %40 = load <42 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %41 = load <43 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %42 = load <44 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %43 = load <45 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %44 = load <46 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %45 = load <47 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %46 = load <48 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %47 = load <49 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %48 = load <50 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %49 = load <51 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %50 = load <52 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %51 = load <53 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %52 = load <54 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %53 = load <55 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %54 = load <56 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %55 = load <57 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %56 = load <58 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %57 = load <59 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %58 = load <60 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %59 = load <61 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %60 = load <62 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %61 = load <63 x i8>, ptr undef, align 1
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret i32 undef
+;
+
+ load <1 x i8>, ptr undef, align 1
+ load <2 x i8>, ptr undef, align 1
+ load <3 x i8>, ptr undef, align 1
+ load <4 x i8>, ptr undef, align 1
+ load <5 x i8>, ptr undef, align 1
+ load <6 x i8>, ptr undef, align 1
+ load <7 x i8>, ptr undef, align 1
+ load <8 x i8>, ptr undef, align 1
+ load <9 x i8>, ptr undef, align 1
+ load <10 x i8>, ptr undef, align 1
+ load <11 x i8>, ptr undef, align 1
+ load <12 x i8>, ptr undef, align 1
+ load <13 x i8>, ptr undef, align 1
+ load <14 x i8>, ptr undef, align 1
+ load <15 x i8>, ptr undef, align 1
+
+ load <17 x i8>, ptr undef, align 1
+ load <18 x i8>, ptr undef, align 1
+ load <19 x i8>, ptr undef, align 1
+ load <20 x i8>, ptr undef, align 1
+ load <21 x i8>, ptr undef, align 1
+ load <22 x i8>, ptr undef, align 1
+ load <23 x i8>, ptr undef, align 1
+ load <24 x i8>, ptr undef, align 1
+ load <25 x i8>, ptr undef, align 1
+ load <26 x i8>, ptr undef, align 1
+ load <27 x i8>, ptr undef, align 1
+ load <28 x i8>, ptr undef, align 1
+ load <29 x i8>, ptr undef, align 1
+ load <30 x i8>, ptr undef, align 1
+ load <31 x i8>, ptr undef, align 1
+
+ load <33 x i8>, ptr undef, align 1
+ load <34 x i8>, ptr undef, align 1
+ load <35 x i8>, ptr undef, align 1
+ load <36 x i8>, ptr undef, align 1
+ load <37 x i8>, ptr undef, align 1
+ load <38 x i8>, ptr undef, align 1
+ load <39 x i8>, ptr undef, align 1
+ load <40 x i8>, ptr undef, align 1
+ load <41 x i8>, ptr undef, align 1
+ load <42 x i8>, ptr undef, align 1
+ load <43 x i8>, ptr undef, align 1
+ load <44 x i8>, ptr undef, align 1
+ load <45 x i8>, ptr undef, align 1
+ load <46 x i8>, ptr undef, align 1
+ load <47 x i8>, ptr undef, align 1
+ load <48 x i8>, ptr undef, align 1
+ load <49 x i8>, ptr undef, align 1
+ load <50 x i8>, ptr undef, align 1
+ load <51 x i8>, ptr undef, align 1
+ load <52 x i8>, ptr undef, align 1
+ load <53 x i8>, ptr undef, align 1
+ load <54 x i8>, ptr undef, align 1
+ load <55 x i8>, ptr undef, align 1
+ load <56 x i8>, ptr undef, align 1
+ load <57 x i8>, ptr undef, align 1
+ load <58 x i8>, ptr undef, align 1
+ load <59 x i8>, ptr undef, align 1
+ load <60 x i8>, ptr undef, align 1
+ load <61 x i8>, ptr undef, align 1
+ load <62 x i8>, ptr undef, align 1
+ load <63 x i8>, ptr undef, align 1
+
+ ret i32 undef
+}
+
+define i32 @loads_align64(i32 %arg) {
+ ; Scalars
+; GFX90A-LABEL: 'loads_align64'
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load i8, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <16 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %3 = load <32 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %4 = load <64 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %5 = load <128 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %6 = load <256 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret i32 undef
+;
+ load i8, ptr undef, align 64
+ load <16 x i8>, ptr undef, align 64
+ load <32 x i8>, ptr undef, align 64
+ load <64 x i8>, ptr undef, align 64
+ load <128 x i8>, ptr undef, align 64
+ load <256 x i8>, ptr undef, align 64
+
+ ret i32 undef
+}
+
+define i32 @loads_partial_align64(i32 %arg) {
+ ; Partial vectors
+; GFX90A-LABEL: 'loads_partial_align64'
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %1 = load <1 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %2 = load <2 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %3 = load <3 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %4 = load <4 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %5 = load <5 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %6 = load <6 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %7 = load <7 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %8 = load <8 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %9 = load <9 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %10 = load <10 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %11 = load <11 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %12 = load <12 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %13 = load <13 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %14 = load <14 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %15 = load <15 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %16 = load <17 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %17 = load <18 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %18 = load <19 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %19 = load <20 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %20 = load <21 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %21 = load <22 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %22 = load <23 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %23 = load <24 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %24 = load <25 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %25 = load <26 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %26 = load <27 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %27 = load <28 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %28 = load <29 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %29 = load <30 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %30 = load <31 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %31 = load <33 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %32 = load <34 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %33 = load <35 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %34 = load <36 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %35 = load <37 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %36 = load <38 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %37 = load <39 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %38 = load <40 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %39 = load <41 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %40 = load <42 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %41 = load <43 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %42 = load <44 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %43 = load <45 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %44 = load <46 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %45 = load <47 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %46 = load <48 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %47 = load <49 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %48 = load <50 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %49 = load <51 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %50 = load <52 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %51 = load <53 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %52 = load <54 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %53 = load <55 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %54 = load <56 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %55 = load <57 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %56 = load <58 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %57 = load <59 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %58 = load <60 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %59 = load <61 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %60 = load <62 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %61 = load <63 x i8>, ptr undef, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret i32 undef
+;
+
+ load <1 x i8>, ptr undef, align 64
+ load <2 x i8>, ptr undef, align 64
+ load <3 x i8>, ptr undef, align 64
+ load <4 x i8>, ptr undef, align 64
+ load <5 x i8>, ptr undef, align 64
+ load <6 x i8>, ptr undef, align 64
+ load <7 x i8>, ptr undef, align 64
+ load <8 x i8>, ptr undef, align 64
+ load <9 x i8>, ptr undef, align 64
+ load <10 x i8>, ptr undef, align 64
+ load <11 x i8>, ptr undef, align 64
+ load <12 x i8>, ptr undef, align 64
+ load <13 x i8>, ptr undef, align 64
+ load <14 x i8>, ptr undef, align 64
+ load <15 x i8>, ptr undef, align 64
+
+ load <17 x i8>, ptr undef, align 64
+ load <18 x i8>, ptr undef, align 64
+ load <19 x i8>, ptr undef, align 64
+ load <20 x i8>, ptr undef, align 64
+ load <21 x i8>, ptr undef, align 64
+ load <22 x i8>, ptr undef, align 64
+ load <23 x i8>, ptr undef, align 64
+ load <24 x i8>, ptr undef, align 64
+ load <25 x i8>, ptr undef, align 64
+ load <26 x i8>, ptr undef, align 64
+ load <27 x i8>, ptr undef, align 64
+ load <28 x i8>, ptr undef, align 64
+ load <29 x i8>, ptr undef, align 64
+ load <30 x i8>, ptr undef, align 64
+ load <31 x i8>, ptr undef, align 64
+
+ load <33 x i8>, ptr undef, align 64
+ load <34 x i8>, ptr undef, align 64
+ load <35 x i8>, ptr undef, align 64
+ load <36 x i8>, ptr undef, align 64
+ load <37 x i8>, ptr undef, align 64
+ load <38 x i8>, ptr undef, align 64
+ load <39 x i8>, ptr undef, align 64
+ load <40 x i8>, ptr undef, align 64
+ load <41 x i8>, ptr undef, align 64
+ load <42 x i8>, ptr undef, align 64
+ load <43 x i8>, ptr undef, align 64
+ load <44 x i8>, ptr undef, align 64
+ load <45 x i8>, ptr undef, align 64
+ load <46 x i8>, ptr undef, align 64
+ load <47 x i8>, ptr undef, align 64
+ load <48 x i8>, ptr undef, align 64
+ load <49 x i8>, ptr undef, align 64
+ load <50 x i8>, ptr undef, align 64
+ load <51 x i8>, ptr undef, align 64
+ load <52 x i8>, ptr undef, align 64
+ load <53 x i8>, ptr undef, align 64
+ load <54 x i8>, ptr undef, align 64
+ load <55 x i8>, ptr undef, align 64
+ load <56 x i8>, ptr undef, align 64
+ load <57 x i8>, ptr undef, align 64
+ load <58 x i8>, ptr undef, align 64
+ load <59 x i8>, ptr undef, align 64
+ load <60 x i8>, ptr undef, align 64
+ load <61 x i8>, ptr undef, align 64
+ load <62 x i8>, ptr undef, align 64
+ load <63 x i8>, ptr undef, align 64
+
+ ret i32 undef
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8-as-i32.ll b/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8-as-i32.ll
new file mode 100644
index 0000000000000..9c25f604cb64b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AMDGPU/vectorize-i8-as-i32.ll
@@ -0,0 +1,1012 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=hawaii -passes=slp-vectorizer %s | FileCheck -check-prefix=GFX7 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=fiji -passes=slp-vectorizer %s | FileCheck -check-prefix=GFX8 %s
+; RUN: opt -S -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -passes=slp-vectorizer %s | FileCheck -check-prefix=GFX9 %s
+
+define protected amdgpu_kernel void @arith_1(<16 x i8> %invec, ptr %out, i32 %flag) {
+; GFX7-LABEL: define protected amdgpu_kernel void @arith_1(
+; GFX7-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX7-NEXT: [[ENTRY:.*:]]
+; GFX7-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX7-NEXT: [[MUL0:%.*]] = mul i8 [[EL0]], 1
+; GFX7-NEXT: [[ADD0:%.*]] = add i8 [[MUL0]], 1
+; GFX7-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
+; GFX7-NEXT: store <16 x i8> [[VECINS0]], ptr [[OUT]], align 16
+; GFX7-NEXT: ret void
+;
+; GFX8-LABEL: define protected amdgpu_kernel void @arith_1(
+; GFX8-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX8-NEXT: [[ENTRY:.*:]]
+; GFX8-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX8-NEXT: [[MUL0:%.*]] = mul i8 [[EL0]], 1
+; GFX8-NEXT: [[ADD0:%.*]] = add i8 [[MUL0]], 1
+; GFX8-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
+; GFX8-NEXT: store <16 x i8> [[VECINS0]], ptr [[OUT]], align 16
+; GFX8-NEXT: ret void
+;
+; GFX9-LABEL: define protected amdgpu_kernel void @arith_1(
+; GFX9-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0:[0-9]+]] {
+; GFX9-NEXT: [[ENTRY:.*:]]
+; GFX9-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX9-NEXT: [[MUL0:%.*]] = mul i8 [[EL0]], 1
+; GFX9-NEXT: [[ADD0:%.*]] = add i8 [[MUL0]], 1
+; GFX9-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
+; GFX9-NEXT: store <16 x i8> [[VECINS0]], ptr [[OUT]], align 16
+; GFX9-NEXT: ret void
+;
+entry:
+ %el0 = extractelement <16 x i8> %invec, i64 0
+ %mul0 = mul i8 %el0, 1
+ %add0 = add i8 %mul0, 1
+ %vecins0 = insertelement <16 x i8> poison, i8 %add0, i64 0
+ store <16 x i8> %vecins0, ptr %out
+ ret void
+}
+
+define protected amdgpu_kernel void @arith_2(<16 x i8> %invec, ptr %out, i32 %flag) {
+; GFX7-LABEL: define protected amdgpu_kernel void @arith_2(
+; GFX7-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[ENTRY:.*:]]
+; GFX7-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX7-NEXT: [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
+; GFX7-NEXT: [[MUL0:%.*]] = mul i8 [[EL0]], 1
+; GFX7-NEXT: [[MUL1:%.*]] = mul i8 [[EL1]], 1
+; GFX7-NEXT: [[ADD0:%.*]] = add i8 [[MUL0]], 1
+; GFX7-NEXT: [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX7-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
+; GFX7-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX7-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT]], align 16
+; GFX7-NEXT: ret void
+;
+; GFX8-LABEL: define protected amdgpu_kernel void @arith_2(
+; GFX8-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*:]]
+; GFX8-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX8-NEXT: [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
+; GFX8-NEXT: [[MUL0:%.*]] = mul i8 [[EL0]], 1
+; GFX8-NEXT: [[MUL1:%.*]] = mul i8 [[EL1]], 1
+; GFX8-NEXT: [[ADD0:%.*]] = add i8 [[MUL0]], 1
+; GFX8-NEXT: [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX8-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
+; GFX8-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX8-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT]], align 16
+; GFX8-NEXT: ret void
+;
+; GFX9-LABEL: define protected amdgpu_kernel void @arith_2(
+; GFX9-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*:]]
+; GFX9-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX9-NEXT: [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
+; GFX9-NEXT: [[MUL0:%.*]] = mul i8 [[EL0]], 1
+; GFX9-NEXT: [[MUL1:%.*]] = mul i8 [[EL1]], 1
+; GFX9-NEXT: [[ADD0:%.*]] = add i8 [[MUL0]], 1
+; GFX9-NEXT: [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX9-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
+; GFX9-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX9-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT]], align 16
+; GFX9-NEXT: ret void
+;
+entry:
+ %el0 = extractelement <16 x i8> %invec, i64 0
+ %el1 = extractelement <16 x i8> %invec, i64 1
+ %mul0 = mul i8 %el0, 1
+ %mul1 = mul i8 %el1, 1
+ %add0 = add i8 %mul0, 1
+ %add1 = add i8 %mul1, 1
+ %vecins0 = insertelement <16 x i8> poison, i8 %add0, i64 0
+ %vecins1 = insertelement <16 x i8> %vecins0, i8 %add1, i64 1
+ store <16 x i8> %vecins1, ptr %out
+ ret void
+}
+
+define protected amdgpu_kernel void @arith_3(<16 x i8> %invec, ptr %out, i32 %flag) {
+; GFX7-LABEL: define protected amdgpu_kernel void @arith_3(
+; GFX7-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[ENTRY:.*:]]
+; GFX7-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX7-NEXT: [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
+; GFX7-NEXT: [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
+; GFX7-NEXT: [[MUL2:%.*]] = mul i8 [[EL0]], 1
+; GFX7-NEXT: [[MUL1:%.*]] = mul i8 [[EL1]], 1
+; GFX7-NEXT: [[MUL3:%.*]] = mul i8 [[EL2]], 1
+; GFX7-NEXT: [[ADD2:%.*]] = add i8 [[MUL2]], 1
+; GFX7-NEXT: [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX7-NEXT: [[ADD3:%.*]] = add i8 [[MUL3]], 1
+; GFX7-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD2]], i64 0
+; GFX7-NEXT: [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX7-NEXT: [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD3]], i64 2
+; GFX7-NEXT: store <16 x i8> [[VECINS2]], ptr [[OUT]], align 16
+; GFX7-NEXT: ret void
+;
+; GFX8-LABEL: define protected amdgpu_kernel void @arith_3(
+; GFX8-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*:]]
+; GFX8-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX8-NEXT: [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
+; GFX8-NEXT: [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
+; GFX8-NEXT: [[MUL2:%.*]] = mul i8 [[EL0]], 1
+; GFX8-NEXT: [[MUL1:%.*]] = mul i8 [[EL1]], 1
+; GFX8-NEXT: [[MUL3:%.*]] = mul i8 [[EL2]], 1
+; GFX8-NEXT: [[ADD2:%.*]] = add i8 [[MUL2]], 1
+; GFX8-NEXT: [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX8-NEXT: [[ADD3:%.*]] = add i8 [[MUL3]], 1
+; GFX8-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD2]], i64 0
+; GFX8-NEXT: [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX8-NEXT: [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD3]], i64 2
+; GFX8-NEXT: store <16 x i8> [[VECINS2]], ptr [[OUT]], align 16
+; GFX8-NEXT: ret void
+;
+; GFX9-LABEL: define protected amdgpu_kernel void @arith_3(
+; GFX9-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*:]]
+; GFX9-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX9-NEXT: [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
+; GFX9-NEXT: [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
+; GFX9-NEXT: [[MUL2:%.*]] = mul i8 [[EL0]], 1
+; GFX9-NEXT: [[MUL1:%.*]] = mul i8 [[EL1]], 1
+; GFX9-NEXT: [[MUL3:%.*]] = mul i8 [[EL2]], 1
+; GFX9-NEXT: [[ADD2:%.*]] = add i8 [[MUL2]], 1
+; GFX9-NEXT: [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX9-NEXT: [[ADD3:%.*]] = add i8 [[MUL3]], 1
+; GFX9-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD2]], i64 0
+; GFX9-NEXT: [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX9-NEXT: [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD3]], i64 2
+; GFX9-NEXT: store <16 x i8> [[VECINS2]], ptr [[OUT]], align 16
+; GFX9-NEXT: ret void
+;
+entry:
+ %el0 = extractelement <16 x i8> %invec, i64 0
+ %el1 = extractelement <16 x i8> %invec, i64 1
+ %el2 = extractelement <16 x i8> %invec, i64 2
+ %mul0 = mul i8 %el0, 1
+ %mul1 = mul i8 %el1, 1
+ %mul2 = mul i8 %el2, 1
+ %add0 = add i8 %mul0, 1
+ %add1 = add i8 %mul1, 1
+ %add2 = add i8 %mul2, 1
+ %vecins0 = insertelement <16 x i8> poison, i8 %add0, i64 0
+ %vecins1 = insertelement <16 x i8> %vecins0, i8 %add1, i64 1
+ %vecins2 = insertelement <16 x i8> %vecins1, i8 %add2, i64 2
+ store <16 x i8> %vecins2, ptr %out
+ ret void
+}
+
+define protected amdgpu_kernel void @arith_4(<16 x i8> %invec, ptr %out, i32 %flag) {
+; GFX7-LABEL: define protected amdgpu_kernel void @arith_4(
+; GFX7-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[ENTRY:.*:]]
+; GFX7-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX7-NEXT: [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
+; GFX7-NEXT: [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
+; GFX7-NEXT: [[EL3:%.*]] = extractelement <16 x i8> [[INVEC]], i64 3
+; GFX7-NEXT: [[MUL0:%.*]] = mul i8 [[EL0]], 1
+; GFX7-NEXT: [[MUL1:%.*]] = mul i8 [[EL1]], 1
+; GFX7-NEXT: [[MUL2:%.*]] = mul i8 [[EL2]], 1
+; GFX7-NEXT: [[MUL3:%.*]] = mul i8 [[EL3]], 1
+; GFX7-NEXT: [[ADD0:%.*]] = add i8 [[MUL0]], 1
+; GFX7-NEXT: [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX7-NEXT: [[ADD2:%.*]] = add i8 [[MUL2]], 1
+; GFX7-NEXT: [[ADD3:%.*]] = add i8 [[MUL3]], 1
+; GFX7-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
+; GFX7-NEXT: [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX7-NEXT: [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD2]], i64 2
+; GFX7-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> [[VECINS2]], i8 [[ADD3]], i64 3
+; GFX7-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT]], align 16
+; GFX7-NEXT: ret void
+;
+; GFX8-LABEL: define protected amdgpu_kernel void @arith_4(
+; GFX8-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*:]]
+; GFX8-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX8-NEXT: [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
+; GFX8-NEXT: [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
+; GFX8-NEXT: [[EL3:%.*]] = extractelement <16 x i8> [[INVEC]], i64 3
+; GFX8-NEXT: [[MUL0:%.*]] = mul i8 [[EL0]], 1
+; GFX8-NEXT: [[MUL1:%.*]] = mul i8 [[EL1]], 1
+; GFX8-NEXT: [[MUL2:%.*]] = mul i8 [[EL2]], 1
+; GFX8-NEXT: [[MUL3:%.*]] = mul i8 [[EL3]], 1
+; GFX8-NEXT: [[ADD0:%.*]] = add i8 [[MUL0]], 1
+; GFX8-NEXT: [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX8-NEXT: [[ADD2:%.*]] = add i8 [[MUL2]], 1
+; GFX8-NEXT: [[ADD3:%.*]] = add i8 [[MUL3]], 1
+; GFX8-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
+; GFX8-NEXT: [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX8-NEXT: [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD2]], i64 2
+; GFX8-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> [[VECINS2]], i8 [[ADD3]], i64 3
+; GFX8-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT]], align 16
+; GFX8-NEXT: ret void
+;
+; GFX9-LABEL: define protected amdgpu_kernel void @arith_4(
+; GFX9-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*:]]
+; GFX9-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX9-NEXT: [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
+; GFX9-NEXT: [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
+; GFX9-NEXT: [[EL3:%.*]] = extractelement <16 x i8> [[INVEC]], i64 3
+; GFX9-NEXT: [[MUL0:%.*]] = mul i8 [[EL0]], 1
+; GFX9-NEXT: [[MUL1:%.*]] = mul i8 [[EL1]], 1
+; GFX9-NEXT: [[MUL2:%.*]] = mul i8 [[EL2]], 1
+; GFX9-NEXT: [[MUL3:%.*]] = mul i8 [[EL3]], 1
+; GFX9-NEXT: [[ADD0:%.*]] = add i8 [[MUL0]], 1
+; GFX9-NEXT: [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX9-NEXT: [[ADD2:%.*]] = add i8 [[MUL2]], 1
+; GFX9-NEXT: [[ADD3:%.*]] = add i8 [[MUL3]], 1
+; GFX9-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
+; GFX9-NEXT: [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX9-NEXT: [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD2]], i64 2
+; GFX9-NEXT: [[TMP3:%.*]] = insertelement <16 x i8> [[VECINS2]], i8 [[ADD3]], i64 3
+; GFX9-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT]], align 16
+; GFX9-NEXT: ret void
+;
+entry:
+ %el0 = extractelement <16 x i8> %invec, i64 0
+ %el1 = extractelement <16 x i8> %invec, i64 1
+ %el2 = extractelement <16 x i8> %invec, i64 2
+ %el3 = extractelement <16 x i8> %invec, i64 3
+ %mul0 = mul i8 %el0, 1
+ %mul1 = mul i8 %el1, 1
+ %mul2 = mul i8 %el2, 1
+ %mul3 = mul i8 %el3, 1
+ %add0 = add i8 %mul0, 1
+ %add1 = add i8 %mul1, 1
+ %add2 = add i8 %mul2, 1
+ %add3 = add i8 %mul3, 1
+ %vecins0 = insertelement <16 x i8> poison, i8 %add0, i64 0
+ %vecins1 = insertelement <16 x i8> %vecins0, i8 %add1, i64 1
+ %vecins2 = insertelement <16 x i8> %vecins1, i8 %add2, i64 2
+ %vecins3 = insertelement <16 x i8> %vecins2, i8 %add3, i64 3
+ store <16 x i8> %vecins3, ptr %out
+ ret void
+}
+
+define protected amdgpu_kernel void @arith_16(<16 x i8> %invec, ptr %out, i32 %flag) {
+; GFX7-LABEL: define protected amdgpu_kernel void @arith_16(
+; GFX7-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[ENTRY:.*:]]
+; GFX7-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX7-NEXT: [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
+; GFX7-NEXT: [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
+; GFX7-NEXT: [[EL3:%.*]] = extractelement <16 x i8> [[INVEC]], i64 3
+; GFX7-NEXT: [[EL4:%.*]] = extractelement <16 x i8> [[INVEC]], i64 4
+; GFX7-NEXT: [[EL5:%.*]] = extractelement <16 x i8> [[INVEC]], i64 5
+; GFX7-NEXT: [[EL6:%.*]] = extractelement <16 x i8> [[INVEC]], i64 6
+; GFX7-NEXT: [[EL7:%.*]] = extractelement <16 x i8> [[INVEC]], i64 7
+; GFX7-NEXT: [[EL8:%.*]] = extractelement <16 x i8> [[INVEC]], i64 8
+; GFX7-NEXT: [[EL9:%.*]] = extractelement <16 x i8> [[INVEC]], i64 9
+; GFX7-NEXT: [[EL10:%.*]] = extractelement <16 x i8> [[INVEC]], i64 10
+; GFX7-NEXT: [[EL11:%.*]] = extractelement <16 x i8> [[INVEC]], i64 11
+; GFX7-NEXT: [[EL12:%.*]] = extractelement <16 x i8> [[INVEC]], i64 12
+; GFX7-NEXT: [[EL13:%.*]] = extractelement <16 x i8> [[INVEC]], i64 13
+; GFX7-NEXT: [[EL14:%.*]] = extractelement <16 x i8> [[INVEC]], i64 14
+; GFX7-NEXT: [[EL15:%.*]] = extractelement <16 x i8> [[INVEC]], i64 15
+; GFX7-NEXT: [[MUL0:%.*]] = mul i8 [[EL0]], 1
+; GFX7-NEXT: [[MUL1:%.*]] = mul i8 [[EL1]], 1
+; GFX7-NEXT: [[MUL2:%.*]] = mul i8 [[EL2]], 1
+; GFX7-NEXT: [[MUL3:%.*]] = mul i8 [[EL3]], 1
+; GFX7-NEXT: [[MUL4:%.*]] = mul i8 [[EL4]], 1
+; GFX7-NEXT: [[MUL5:%.*]] = mul i8 [[EL5]], 1
+; GFX7-NEXT: [[MUL6:%.*]] = mul i8 [[EL6]], 1
+; GFX7-NEXT: [[MUL7:%.*]] = mul i8 [[EL7]], 1
+; GFX7-NEXT: [[MUL8:%.*]] = mul i8 [[EL8]], 1
+; GFX7-NEXT: [[MUL9:%.*]] = mul i8 [[EL9]], 1
+; GFX7-NEXT: [[MUL10:%.*]] = mul i8 [[EL10]], 1
+; GFX7-NEXT: [[MUL11:%.*]] = mul i8 [[EL11]], 1
+; GFX7-NEXT: [[MUL12:%.*]] = mul i8 [[EL12]], 1
+; GFX7-NEXT: [[MUL13:%.*]] = mul i8 [[EL13]], 1
+; GFX7-NEXT: [[MUL14:%.*]] = mul i8 [[EL14]], 1
+; GFX7-NEXT: [[MUL15:%.*]] = mul i8 [[EL15]], 1
+; GFX7-NEXT: [[ADD0:%.*]] = add i8 [[MUL0]], 1
+; GFX7-NEXT: [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX7-NEXT: [[ADD2:%.*]] = add i8 [[MUL2]], 1
+; GFX7-NEXT: [[ADD3:%.*]] = add i8 [[MUL3]], 1
+; GFX7-NEXT: [[ADD4:%.*]] = add i8 [[MUL4]], 1
+; GFX7-NEXT: [[ADD5:%.*]] = add i8 [[MUL5]], 1
+; GFX7-NEXT: [[ADD6:%.*]] = add i8 [[MUL6]], 1
+; GFX7-NEXT: [[ADD7:%.*]] = add i8 [[MUL7]], 1
+; GFX7-NEXT: [[ADD8:%.*]] = add i8 [[MUL8]], 1
+; GFX7-NEXT: [[ADD9:%.*]] = add i8 [[MUL9]], 1
+; GFX7-NEXT: [[ADD10:%.*]] = add i8 [[MUL10]], 1
+; GFX7-NEXT: [[ADD11:%.*]] = add i8 [[MUL11]], 1
+; GFX7-NEXT: [[ADD12:%.*]] = add i8 [[MUL12]], 1
+; GFX7-NEXT: [[ADD13:%.*]] = add i8 [[MUL13]], 1
+; GFX7-NEXT: [[ADD14:%.*]] = add i8 [[MUL14]], 1
+; GFX7-NEXT: [[ADD15:%.*]] = add i8 [[MUL15]], 1
+; GFX7-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
+; GFX7-NEXT: [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX7-NEXT: [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD2]], i64 2
+; GFX7-NEXT: [[VECINS3:%.*]] = insertelement <16 x i8> [[VECINS2]], i8 [[ADD3]], i64 3
+; GFX7-NEXT: [[VECINS4:%.*]] = insertelement <16 x i8> [[VECINS3]], i8 [[ADD4]], i64 4
+; GFX7-NEXT: [[VECINS5:%.*]] = insertelement <16 x i8> [[VECINS4]], i8 [[ADD5]], i64 5
+; GFX7-NEXT: [[VECINS6:%.*]] = insertelement <16 x i8> [[VECINS5]], i8 [[ADD6]], i64 6
+; GFX7-NEXT: [[VECINS7:%.*]] = insertelement <16 x i8> [[VECINS6]], i8 [[ADD7]], i64 7
+; GFX7-NEXT: [[VECINS8:%.*]] = insertelement <16 x i8> [[VECINS7]], i8 [[ADD8]], i64 8
+; GFX7-NEXT: [[VECINS9:%.*]] = insertelement <16 x i8> [[VECINS8]], i8 [[ADD9]], i64 9
+; GFX7-NEXT: [[VECINS10:%.*]] = insertelement <16 x i8> [[VECINS9]], i8 [[ADD10]], i64 10
+; GFX7-NEXT: [[VECINS11:%.*]] = insertelement <16 x i8> [[VECINS10]], i8 [[ADD11]], i64 11
+; GFX7-NEXT: [[VECINS12:%.*]] = insertelement <16 x i8> [[VECINS11]], i8 [[ADD12]], i64 12
+; GFX7-NEXT: [[VECINS13:%.*]] = insertelement <16 x i8> [[VECINS12]], i8 [[ADD13]], i64 13
+; GFX7-NEXT: [[VECINS14:%.*]] = insertelement <16 x i8> [[VECINS13]], i8 [[ADD14]], i64 14
+; GFX7-NEXT: [[VECINS153:%.*]] = insertelement <16 x i8> [[VECINS14]], i8 [[ADD15]], i64 15
+; GFX7-NEXT: store <16 x i8> [[VECINS153]], ptr [[OUT]], align 16
+; GFX7-NEXT: ret void
+;
+; GFX8-LABEL: define protected amdgpu_kernel void @arith_16(
+; GFX8-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*:]]
+; GFX8-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX8-NEXT: [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
+; GFX8-NEXT: [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
+; GFX8-NEXT: [[EL3:%.*]] = extractelement <16 x i8> [[INVEC]], i64 3
+; GFX8-NEXT: [[EL4:%.*]] = extractelement <16 x i8> [[INVEC]], i64 4
+; GFX8-NEXT: [[EL5:%.*]] = extractelement <16 x i8> [[INVEC]], i64 5
+; GFX8-NEXT: [[EL6:%.*]] = extractelement <16 x i8> [[INVEC]], i64 6
+; GFX8-NEXT: [[EL7:%.*]] = extractelement <16 x i8> [[INVEC]], i64 7
+; GFX8-NEXT: [[EL8:%.*]] = extractelement <16 x i8> [[INVEC]], i64 8
+; GFX8-NEXT: [[EL9:%.*]] = extractelement <16 x i8> [[INVEC]], i64 9
+; GFX8-NEXT: [[EL10:%.*]] = extractelement <16 x i8> [[INVEC]], i64 10
+; GFX8-NEXT: [[EL11:%.*]] = extractelement <16 x i8> [[INVEC]], i64 11
+; GFX8-NEXT: [[EL12:%.*]] = extractelement <16 x i8> [[INVEC]], i64 12
+; GFX8-NEXT: [[EL13:%.*]] = extractelement <16 x i8> [[INVEC]], i64 13
+; GFX8-NEXT: [[EL14:%.*]] = extractelement <16 x i8> [[INVEC]], i64 14
+; GFX8-NEXT: [[EL15:%.*]] = extractelement <16 x i8> [[INVEC]], i64 15
+; GFX8-NEXT: [[MUL0:%.*]] = mul i8 [[EL0]], 1
+; GFX8-NEXT: [[MUL1:%.*]] = mul i8 [[EL1]], 1
+; GFX8-NEXT: [[MUL2:%.*]] = mul i8 [[EL2]], 1
+; GFX8-NEXT: [[MUL3:%.*]] = mul i8 [[EL3]], 1
+; GFX8-NEXT: [[MUL4:%.*]] = mul i8 [[EL4]], 1
+; GFX8-NEXT: [[MUL5:%.*]] = mul i8 [[EL5]], 1
+; GFX8-NEXT: [[MUL6:%.*]] = mul i8 [[EL6]], 1
+; GFX8-NEXT: [[MUL7:%.*]] = mul i8 [[EL7]], 1
+; GFX8-NEXT: [[MUL8:%.*]] = mul i8 [[EL8]], 1
+; GFX8-NEXT: [[MUL9:%.*]] = mul i8 [[EL9]], 1
+; GFX8-NEXT: [[MUL10:%.*]] = mul i8 [[EL10]], 1
+; GFX8-NEXT: [[MUL11:%.*]] = mul i8 [[EL11]], 1
+; GFX8-NEXT: [[MUL12:%.*]] = mul i8 [[EL12]], 1
+; GFX8-NEXT: [[MUL13:%.*]] = mul i8 [[EL13]], 1
+; GFX8-NEXT: [[MUL14:%.*]] = mul i8 [[EL14]], 1
+; GFX8-NEXT: [[MUL15:%.*]] = mul i8 [[EL15]], 1
+; GFX8-NEXT: [[ADD0:%.*]] = add i8 [[MUL0]], 1
+; GFX8-NEXT: [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX8-NEXT: [[ADD2:%.*]] = add i8 [[MUL2]], 1
+; GFX8-NEXT: [[ADD3:%.*]] = add i8 [[MUL3]], 1
+; GFX8-NEXT: [[ADD4:%.*]] = add i8 [[MUL4]], 1
+; GFX8-NEXT: [[ADD5:%.*]] = add i8 [[MUL5]], 1
+; GFX8-NEXT: [[ADD6:%.*]] = add i8 [[MUL6]], 1
+; GFX8-NEXT: [[ADD7:%.*]] = add i8 [[MUL7]], 1
+; GFX8-NEXT: [[ADD8:%.*]] = add i8 [[MUL8]], 1
+; GFX8-NEXT: [[ADD9:%.*]] = add i8 [[MUL9]], 1
+; GFX8-NEXT: [[ADD10:%.*]] = add i8 [[MUL10]], 1
+; GFX8-NEXT: [[ADD11:%.*]] = add i8 [[MUL11]], 1
+; GFX8-NEXT: [[ADD12:%.*]] = add i8 [[MUL12]], 1
+; GFX8-NEXT: [[ADD13:%.*]] = add i8 [[MUL13]], 1
+; GFX8-NEXT: [[ADD14:%.*]] = add i8 [[MUL14]], 1
+; GFX8-NEXT: [[ADD15:%.*]] = add i8 [[MUL15]], 1
+; GFX8-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
+; GFX8-NEXT: [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX8-NEXT: [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD2]], i64 2
+; GFX8-NEXT: [[VECINS3:%.*]] = insertelement <16 x i8> [[VECINS2]], i8 [[ADD3]], i64 3
+; GFX8-NEXT: [[VECINS4:%.*]] = insertelement <16 x i8> [[VECINS3]], i8 [[ADD4]], i64 4
+; GFX8-NEXT: [[VECINS5:%.*]] = insertelement <16 x i8> [[VECINS4]], i8 [[ADD5]], i64 5
+; GFX8-NEXT: [[VECINS6:%.*]] = insertelement <16 x i8> [[VECINS5]], i8 [[ADD6]], i64 6
+; GFX8-NEXT: [[VECINS7:%.*]] = insertelement <16 x i8> [[VECINS6]], i8 [[ADD7]], i64 7
+; GFX8-NEXT: [[VECINS8:%.*]] = insertelement <16 x i8> [[VECINS7]], i8 [[ADD8]], i64 8
+; GFX8-NEXT: [[VECINS9:%.*]] = insertelement <16 x i8> [[VECINS8]], i8 [[ADD9]], i64 9
+; GFX8-NEXT: [[VECINS10:%.*]] = insertelement <16 x i8> [[VECINS9]], i8 [[ADD10]], i64 10
+; GFX8-NEXT: [[VECINS11:%.*]] = insertelement <16 x i8> [[VECINS10]], i8 [[ADD11]], i64 11
+; GFX8-NEXT: [[VECINS12:%.*]] = insertelement <16 x i8> [[VECINS11]], i8 [[ADD12]], i64 12
+; GFX8-NEXT: [[VECINS13:%.*]] = insertelement <16 x i8> [[VECINS12]], i8 [[ADD13]], i64 13
+; GFX8-NEXT: [[VECINS14:%.*]] = insertelement <16 x i8> [[VECINS13]], i8 [[ADD14]], i64 14
+; GFX8-NEXT: [[VECINS153:%.*]] = insertelement <16 x i8> [[VECINS14]], i8 [[ADD15]], i64 15
+; GFX8-NEXT: store <16 x i8> [[VECINS153]], ptr [[OUT]], align 16
+; GFX8-NEXT: ret void
+;
+; GFX9-LABEL: define protected amdgpu_kernel void @arith_16(
+; GFX9-SAME: <16 x i8> [[INVEC:%.*]], ptr [[OUT:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*:]]
+; GFX9-NEXT: [[EL0:%.*]] = extractelement <16 x i8> [[INVEC]], i64 0
+; GFX9-NEXT: [[EL1:%.*]] = extractelement <16 x i8> [[INVEC]], i64 1
+; GFX9-NEXT: [[EL2:%.*]] = extractelement <16 x i8> [[INVEC]], i64 2
+; GFX9-NEXT: [[EL3:%.*]] = extractelement <16 x i8> [[INVEC]], i64 3
+; GFX9-NEXT: [[EL4:%.*]] = extractelement <16 x i8> [[INVEC]], i64 4
+; GFX9-NEXT: [[EL5:%.*]] = extractelement <16 x i8> [[INVEC]], i64 5
+; GFX9-NEXT: [[EL6:%.*]] = extractelement <16 x i8> [[INVEC]], i64 6
+; GFX9-NEXT: [[EL7:%.*]] = extractelement <16 x i8> [[INVEC]], i64 7
+; GFX9-NEXT: [[EL8:%.*]] = extractelement <16 x i8> [[INVEC]], i64 8
+; GFX9-NEXT: [[EL9:%.*]] = extractelement <16 x i8> [[INVEC]], i64 9
+; GFX9-NEXT: [[EL10:%.*]] = extractelement <16 x i8> [[INVEC]], i64 10
+; GFX9-NEXT: [[EL11:%.*]] = extractelement <16 x i8> [[INVEC]], i64 11
+; GFX9-NEXT: [[EL12:%.*]] = extractelement <16 x i8> [[INVEC]], i64 12
+; GFX9-NEXT: [[EL13:%.*]] = extractelement <16 x i8> [[INVEC]], i64 13
+; GFX9-NEXT: [[EL14:%.*]] = extractelement <16 x i8> [[INVEC]], i64 14
+; GFX9-NEXT: [[EL15:%.*]] = extractelement <16 x i8> [[INVEC]], i64 15
+; GFX9-NEXT: [[MUL0:%.*]] = mul i8 [[EL0]], 1
+; GFX9-NEXT: [[MUL1:%.*]] = mul i8 [[EL1]], 1
+; GFX9-NEXT: [[MUL2:%.*]] = mul i8 [[EL2]], 1
+; GFX9-NEXT: [[MUL3:%.*]] = mul i8 [[EL3]], 1
+; GFX9-NEXT: [[MUL4:%.*]] = mul i8 [[EL4]], 1
+; GFX9-NEXT: [[MUL5:%.*]] = mul i8 [[EL5]], 1
+; GFX9-NEXT: [[MUL6:%.*]] = mul i8 [[EL6]], 1
+; GFX9-NEXT: [[MUL7:%.*]] = mul i8 [[EL7]], 1
+; GFX9-NEXT: [[MUL8:%.*]] = mul i8 [[EL8]], 1
+; GFX9-NEXT: [[MUL9:%.*]] = mul i8 [[EL9]], 1
+; GFX9-NEXT: [[MUL10:%.*]] = mul i8 [[EL10]], 1
+; GFX9-NEXT: [[MUL11:%.*]] = mul i8 [[EL11]], 1
+; GFX9-NEXT: [[MUL12:%.*]] = mul i8 [[EL12]], 1
+; GFX9-NEXT: [[MUL13:%.*]] = mul i8 [[EL13]], 1
+; GFX9-NEXT: [[MUL14:%.*]] = mul i8 [[EL14]], 1
+; GFX9-NEXT: [[MUL15:%.*]] = mul i8 [[EL15]], 1
+; GFX9-NEXT: [[ADD0:%.*]] = add i8 [[MUL0]], 1
+; GFX9-NEXT: [[ADD1:%.*]] = add i8 [[MUL1]], 1
+; GFX9-NEXT: [[ADD2:%.*]] = add i8 [[MUL2]], 1
+; GFX9-NEXT: [[ADD3:%.*]] = add i8 [[MUL3]], 1
+; GFX9-NEXT: [[ADD4:%.*]] = add i8 [[MUL4]], 1
+; GFX9-NEXT: [[ADD5:%.*]] = add i8 [[MUL5]], 1
+; GFX9-NEXT: [[ADD6:%.*]] = add i8 [[MUL6]], 1
+; GFX9-NEXT: [[ADD7:%.*]] = add i8 [[MUL7]], 1
+; GFX9-NEXT: [[ADD8:%.*]] = add i8 [[MUL8]], 1
+; GFX9-NEXT: [[ADD9:%.*]] = add i8 [[MUL9]], 1
+; GFX9-NEXT: [[ADD10:%.*]] = add i8 [[MUL10]], 1
+; GFX9-NEXT: [[ADD11:%.*]] = add i8 [[MUL11]], 1
+; GFX9-NEXT: [[ADD12:%.*]] = add i8 [[MUL12]], 1
+; GFX9-NEXT: [[ADD13:%.*]] = add i8 [[MUL13]], 1
+; GFX9-NEXT: [[ADD14:%.*]] = add i8 [[MUL14]], 1
+; GFX9-NEXT: [[ADD15:%.*]] = add i8 [[MUL15]], 1
+; GFX9-NEXT: [[VECINS0:%.*]] = insertelement <16 x i8> poison, i8 [[ADD0]], i64 0
+; GFX9-NEXT: [[VECINS1:%.*]] = insertelement <16 x i8> [[VECINS0]], i8 [[ADD1]], i64 1
+; GFX9-NEXT: [[VECINS2:%.*]] = insertelement <16 x i8> [[VECINS1]], i8 [[ADD2]], i64 2
+; GFX9-NEXT: [[VECINS3:%.*]] = insertelement <16 x i8> [[VECINS2]], i8 [[ADD3]], i64 3
+; GFX9-NEXT: [[VECINS4:%.*]] = insertelement <16 x i8> [[VECINS3]], i8 [[ADD4]], i64 4
+; GFX9-NEXT: [[VECINS5:%.*]] = insertelement <16 x i8> [[VECINS4]], i8 [[ADD5]], i64 5
+; GFX9-NEXT: [[VECINS6:%.*]] = insertelement <16 x i8> [[VECINS5]], i8 [[ADD6]], i64 6
+; GFX9-NEXT: [[VECINS7:%.*]] = insertelement <16 x i8> [[VECINS6]], i8 [[ADD7]], i64 7
+; GFX9-NEXT: [[VECINS8:%.*]] = insertelement <16 x i8> [[VECINS7]], i8 [[ADD8]], i64 8
+; GFX9-NEXT: [[VECINS9:%.*]] = insertelement <16 x i8> [[VECINS8]], i8 [[ADD9]], i64 9
+; GFX9-NEXT: [[VECINS10:%.*]] = insertelement <16 x i8> [[VECINS9]], i8 [[ADD10]], i64 10
+; GFX9-NEXT: [[VECINS11:%.*]] = insertelement <16 x i8> [[VECINS10]], i8 [[ADD11]], i64 11
+; GFX9-NEXT: [[VECINS12:%.*]] = insertelement <16 x i8> [[VECINS11]], i8 [[ADD12]], i64 12
+; GFX9-NEXT: [[VECINS13:%.*]] = insertelement <16 x i8> [[VECINS12]], i8 [[ADD13]], i64 13
+; GFX9-NEXT: [[VECINS14:%.*]] = insertelement <16 x i8> [[VECINS13]], i8 [[ADD14]], i64 14
+; GFX9-NEXT: [[VECINS153:%.*]] = insertelement <16 x i8> [[VECINS14]], i8 [[ADD15]], i64 15
+; GFX9-NEXT: store <16 x i8> [[VECINS153]], ptr [[OUT]], align 16
+; GFX9-NEXT: ret void
+;
+entry:
+ %el0 = extractelement <16 x i8> %invec, i64 0
+ %el1 = extractelement <16 x i8> %invec, i64 1
+ %el2 = extractelement <16 x i8> %invec, i64 2
+ %el3 = extractelement <16 x i8> %invec, i64 3
+ %el4 = extractelement <16 x i8> %invec, i64 4
+ %el5 = extractelement <16 x i8> %invec, i64 5
+ %el6 = extractelement <16 x i8> %invec, i64 6
+ %el7 = extractelement <16 x i8> %invec, i64 7
+ %el8 = extractelement <16 x i8> %invec, i64 8
+ %el9 = extractelement <16 x i8> %invec, i64 9
+ %el10 = extractelement <16 x i8> %invec, i64 10
+ %el11 = extractelement <16 x i8> %invec, i64 11
+ %el12 = extractelement <16 x i8> %invec, i64 12
+ %el13 = extractelement <16 x i8> %invec, i64 13
+ %el14 = extractelement <16 x i8> %invec, i64 14
+ %el15 = extractelement <16 x i8> %invec, i64 15
+ %mul0 = mul i8 %el0, 1
+ %mul1 = mul i8 %el1, 1
+ %mul2 = mul i8 %el2, 1
+ %mul3 = mul i8 %el3, 1
+ %mul4 = mul i8 %el4, 1
+ %mul5 = mul i8 %el5, 1
+ %mul6 = mul i8 %el6, 1
+ %mul7 = mul i8 %el7, 1
+ %mul8 = mul i8 %el8, 1
+ %mul9 = mul i8 %el9, 1
+ %mul10 = mul i8 %el10, 1
+ %mul11 = mul i8 %el11, 1
+ %mul12 = mul i8 %el12, 1
+ %mul13 = mul i8 %el13, 1
+ %mul14 = mul i8 %el14, 1
+ %mul15 = mul i8 %el15, 1
+ %add0 = add i8 %mul0, 1
+ %add1 = add i8 %mul1, 1
+ %add2 = add i8 %mul2, 1
+ %add3 = add i8 %mul3, 1
+ %add4 = add i8 %mul4, 1
+ %add5 = add i8 %mul5, 1
+ %add6 = add i8 %mul6, 1
+ %add7 = add i8 %mul7, 1
+ %add8 = add i8 %mul8, 1
+ %add9 = add i8 %mul9, 1
+ %add10 = add i8 %mul10, 1
+ %add11 = add i8 %mul11, 1
+ %add12 = add i8 %mul12, 1
+ %add13 = add i8 %mul13, 1
+ %add14 = add i8 %mul14, 1
+ %add15 = add i8 %mul15, 1
+ %vecins0 = insertelement <16 x i8> poison, i8 %add0, i64 0
+ %vecins1 = insertelement <16 x i8> %vecins0, i8 %add1, i64 1
+ %vecins2 = insertelement <16 x i8> %vecins1, i8 %add2, i64 2
+ %vecins3 = insertelement <16 x i8> %vecins2, i8 %add3, i64 3
+ %vecins4 = insertelement <16 x i8> %vecins3, i8 %add4, i64 4
+ %vecins5 = insertelement <16 x i8> %vecins4, i8 %add5, i64 5
+ %vecins6 = insertelement <16 x i8> %vecins5, i8 %add6, i64 6
+ %vecins7 = insertelement <16 x i8> %vecins6, i8 %add7, i64 7
+ %vecins8 = insertelement <16 x i8> %vecins7, i8 %add8, i64 8
+ %vecins9 = insertelement <16 x i8> %vecins8, i8 %add9, i64 9
+ %vecins10 = insertelement <16 x i8> %vecins9, i8 %add10, i64 10
+ %vecins11 = insertelement <16 x i8> %vecins10, i8 %add11, i64 11
+ %vecins12 = insertelement <16 x i8> %vecins11, i8 %add12, i64 12
+ %vecins13 = insertelement <16 x i8> %vecins12, i8 %add13, i64 13
+ %vecins14 = insertelement <16 x i8> %vecins13, i8 %add14, i64 14
+ %vecins15 = insertelement <16 x i8> %vecins14, i8 %add15, i64 15
+ store <16 x i8> %vecins15, ptr %out
+ ret void
+}
+
+define protected amdgpu_kernel void @phi_1(ptr addrspace(3) %inptr0, ptr addrspace(3) %inptr1, ptr %out, ptr %out1, i32 %flag) {
+; GFX7-LABEL: define protected amdgpu_kernel void @phi_1(
+; GFX7-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[ENTRY:.*]]:
+; GFX7-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX7-NEXT: [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
+; GFX7-NEXT: br label %[[DO_BODY:.*]]
+; GFX7: [[DO_BODY]]:
+; GFX7-NEXT: [[PHI3:%.*]] = phi i8 [ [[ELE0]], %[[ENTRY]] ], [ [[OTHERELE0:%.*]], %[[DO_BODY]] ]
+; GFX7-NEXT: [[OTHERELE0]] = load i8, ptr addrspace(3) [[GEP0]], align 8
+; GFX7-NEXT: [[VEC00:%.*]] = insertelement <16 x i8> poison, i8 [[OTHERELE0]], i64 8
+; GFX7-NEXT: [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
+; GFX7-NEXT: store <16 x i8> [[VEC10]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX7-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX7-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX7: [[EXIT]]:
+; GFX7-NEXT: store <16 x i8> [[VEC10]], ptr [[OUT]], align 16
+; GFX7-NEXT: store <16 x i8> [[VEC00]], ptr [[OUT1]], align 16
+; GFX7-NEXT: ret void
+;
+; GFX8-LABEL: define protected amdgpu_kernel void @phi_1(
+; GFX8-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX8-NEXT: [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
+; GFX8-NEXT: br label %[[DO_BODY:.*]]
+; GFX8: [[DO_BODY]]:
+; GFX8-NEXT: [[PHI3:%.*]] = phi i8 [ [[ELE0]], %[[ENTRY]] ], [ [[OTHERELE0:%.*]], %[[DO_BODY]] ]
+; GFX8-NEXT: [[OTHERELE0]] = load i8, ptr addrspace(3) [[GEP0]], align 8
+; GFX8-NEXT: [[VEC00:%.*]] = insertelement <16 x i8> poison, i8 [[OTHERELE0]], i64 8
+; GFX8-NEXT: [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
+; GFX8-NEXT: store <16 x i8> [[VEC10]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX8-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX8-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: store <16 x i8> [[VEC10]], ptr [[OUT]], align 16
+; GFX8-NEXT: store <16 x i8> [[VEC00]], ptr [[OUT1]], align 16
+; GFX8-NEXT: ret void
+;
+; GFX9-LABEL: define protected amdgpu_kernel void @phi_1(
+; GFX9-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX9-NEXT: [[ELE0:%.*]] = load i8, ptr addrspace(3) [[GEP0]], align 8
+; GFX9-NEXT: br label %[[DO_BODY:.*]]
+; GFX9: [[DO_BODY]]:
+; GFX9-NEXT: [[PHI3:%.*]] = phi i8 [ [[ELE0]], %[[ENTRY]] ], [ [[OTHERELE0:%.*]], %[[DO_BODY]] ]
+; GFX9-NEXT: [[OTHERELE0]] = load i8, ptr addrspace(3) [[GEP0]], align 8
+; GFX9-NEXT: [[VEC00:%.*]] = insertelement <16 x i8> poison, i8 [[OTHERELE0]], i64 8
+; GFX9-NEXT: [[VEC10:%.*]] = insertelement <16 x i8> poison, i8 [[PHI3]], i64 8
+; GFX9-NEXT: store <16 x i8> [[VEC10]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX9-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX9-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: store <16 x i8> [[VEC10]], ptr [[OUT]], align 16
+; GFX9-NEXT: store <16 x i8> [[VEC00]], ptr [[OUT1]], align 16
+; GFX9-NEXT: ret void
+;
+entry:
+ %gep0 = getelementptr i8, ptr addrspace(3) %inptr0, i32 0
+ %ele0 = load i8, ptr addrspace(3) %gep0, align 8
+ br label %do.body
+
+do.body:
+ %phi3 = phi i8 [ %ele0, %entry ], [ %otherele0, %do.body ]
+ %otherele0 = load i8, ptr addrspace(3) %gep0, align 8
+ %vec00 = insertelement <16 x i8> poison, i8 %otherele0, i64 8
+ %vec10 = insertelement <16 x i8> poison, i8 %phi3, i64 8
+ store <16 x i8> %vec10, ptr addrspace(3) %inptr1, align 2
+ %cmp = icmp eq i32 %flag, 0
+ br i1 %cmp, label %exit, label %do.body
+
+exit:
+ store <16 x i8> %vec10, ptr %out
+ store <16 x i8> %vec00, ptr %out1
+ ret void
+}
+
+define protected amdgpu_kernel void @phi_2(ptr addrspace(3) %inptr0, ptr addrspace(3) %inptr1, ptr %out, ptr %out1, i32 %flag) {
+; GFX7-LABEL: define protected amdgpu_kernel void @phi_2(
+; GFX7-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[ENTRY:.*]]:
+; GFX7-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX7-NEXT: [[TMP0:%.*]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX7-NEXT: br label %[[DO_BODY:.*]]
+; GFX7: [[DO_BODY]]:
+; GFX7-NEXT: [[TMP1:%.*]] = phi <2 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX7-NEXT: [[TMP2]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX7-NEXT: [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX7-NEXT: [[TMP4:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX7-NEXT: [[VEC111:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX7-NEXT: store <16 x i8> [[VEC111]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX7-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX7-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX7: [[EXIT]]:
+; GFX7-NEXT: store <16 x i8> [[VEC111]], ptr [[OUT]], align 16
+; GFX7-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT1]], align 16
+; GFX7-NEXT: ret void
+;
+; GFX8-LABEL: define protected amdgpu_kernel void @phi_2(
+; GFX8-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX8-NEXT: [[TMP0:%.*]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX8-NEXT: br label %[[DO_BODY:.*]]
+; GFX8: [[DO_BODY]]:
+; GFX8-NEXT: [[TMP1:%.*]] = phi <2 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX8-NEXT: [[TMP2]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX8-NEXT: [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT: [[TMP4:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT: [[VEC111:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT: store <16 x i8> [[VEC111]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX8-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX8-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: store <16 x i8> [[VEC111]], ptr [[OUT]], align 16
+; GFX8-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT1]], align 16
+; GFX8-NEXT: ret void
+;
+; GFX9-LABEL: define protected amdgpu_kernel void @phi_2(
+; GFX9-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX9-NEXT: [[TMP0:%.*]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX9-NEXT: br label %[[DO_BODY:.*]]
+; GFX9: [[DO_BODY]]:
+; GFX9-NEXT: [[TMP1:%.*]] = phi <2 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX9-NEXT: [[TMP2]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX9-NEXT: [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT: [[TMP4:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT: [[VEC111:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT: store <16 x i8> [[VEC111]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX9-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX9-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: store <16 x i8> [[VEC111]], ptr [[OUT]], align 16
+; GFX9-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT1]], align 16
+; GFX9-NEXT: ret void
+;
+entry:
+ %gep0 = getelementptr i8, ptr addrspace(3) %inptr0, i32 0
+ %ele0 = load i8, ptr addrspace(3) %gep0, align 8
+ %gep1 = getelementptr i8, ptr addrspace(3) %inptr0, i32 1
+ %ele1 = load i8, ptr addrspace(3) %gep1, align 1
+ br label %do.body
+
+do.body:
+ %phi2 = phi i8 [ %ele1, %entry ], [ %otherele1, %do.body ]
+ %phi3 = phi i8 [ %ele0, %entry ], [ %otherele0, %do.body ]
+ %otherele0 = load i8, ptr addrspace(3) %gep0, align 8
+ %otherele1 = load i8, ptr addrspace(3) %gep1, align 1
+ %vec00 = insertelement <16 x i8> poison, i8 %otherele0, i64 8
+ %vec01 = insertelement <16 x i8> %vec00, i8 %otherele1, i64 9
+ %vec10 = insertelement <16 x i8> poison, i8 %phi3, i64 8
+ %vec11 = insertelement <16 x i8> %vec10, i8 %phi2, i64 9
+ store <16 x i8> %vec11, ptr addrspace(3) %inptr1, align 2
+ %cmp = icmp eq i32 %flag, 0
+ br i1 %cmp, label %exit, label %do.body
+
+exit:
+ store <16 x i8> %vec11, ptr %out
+ store <16 x i8> %vec01, ptr %out1
+ ret void
+}
+
+define protected amdgpu_kernel void @phi_3(ptr addrspace(3) %inptr0, ptr addrspace(3) %inptr1, ptr %out, ptr %out1, i32 %flag) {
+; GFX7-LABEL: define protected amdgpu_kernel void @phi_3(
+; GFX7-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[ENTRY:.*]]:
+; GFX7-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX7-NEXT: [[TMP0:%.*]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX7-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 2
+; GFX7-NEXT: [[ELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
+; GFX7-NEXT: br label %[[DO_BODY:.*]]
+; GFX7: [[DO_BODY]]:
+; GFX7-NEXT: [[PHI1:%.*]] = phi i8 [ [[ELE2]], %[[ENTRY]] ], [ [[OTHERELE2:%.*]], %[[DO_BODY]] ]
+; GFX7-NEXT: [[TMP1:%.*]] = phi <2 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX7-NEXT: [[TMP2]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX7-NEXT: [[OTHERELE2]] = load i8, ptr addrspace(3) [[GEP2]], align 2
+; GFX7-NEXT: [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX7-NEXT: [[VEC02:%.*]] = insertelement <16 x i8> [[TMP3]], i8 [[OTHERELE2]], i64 10
+; GFX7-NEXT: [[TMP4:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX7-NEXT: [[VEC111:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX7-NEXT: [[VEC12:%.*]] = insertelement <16 x i8> [[VEC111]], i8 [[PHI1]], i64 10
+; GFX7-NEXT: store <16 x i8> [[VEC12]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX7-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX7-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX7: [[EXIT]]:
+; GFX7-NEXT: store <16 x i8> [[VEC12]], ptr [[OUT]], align 16
+; GFX7-NEXT: store <16 x i8> [[VEC02]], ptr [[OUT1]], align 16
+; GFX7-NEXT: ret void
+;
+; GFX8-LABEL: define protected amdgpu_kernel void @phi_3(
+; GFX8-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX8-NEXT: [[TMP0:%.*]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX8-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 2
+; GFX8-NEXT: [[ELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
+; GFX8-NEXT: br label %[[DO_BODY:.*]]
+; GFX8: [[DO_BODY]]:
+; GFX8-NEXT: [[PHI1:%.*]] = phi i8 [ [[ELE2]], %[[ENTRY]] ], [ [[OTHERELE2:%.*]], %[[DO_BODY]] ]
+; GFX8-NEXT: [[TMP1:%.*]] = phi <2 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX8-NEXT: [[TMP2]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX8-NEXT: [[OTHERELE2]] = load i8, ptr addrspace(3) [[GEP2]], align 2
+; GFX8-NEXT: [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT: [[VEC02:%.*]] = insertelement <16 x i8> [[TMP3]], i8 [[OTHERELE2]], i64 10
+; GFX8-NEXT: [[TMP4:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT: [[VEC111:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT: [[VEC12:%.*]] = insertelement <16 x i8> [[VEC111]], i8 [[PHI1]], i64 10
+; GFX8-NEXT: store <16 x i8> [[VEC12]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX8-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX8-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: store <16 x i8> [[VEC12]], ptr [[OUT]], align 16
+; GFX8-NEXT: store <16 x i8> [[VEC02]], ptr [[OUT1]], align 16
+; GFX8-NEXT: ret void
+;
+; GFX9-LABEL: define protected amdgpu_kernel void @phi_3(
+; GFX9-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX9-NEXT: [[TMP0:%.*]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX9-NEXT: [[GEP2:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 2
+; GFX9-NEXT: [[ELE2:%.*]] = load i8, ptr addrspace(3) [[GEP2]], align 2
+; GFX9-NEXT: br label %[[DO_BODY:.*]]
+; GFX9: [[DO_BODY]]:
+; GFX9-NEXT: [[PHI1:%.*]] = phi i8 [ [[ELE2]], %[[ENTRY]] ], [ [[OTHERELE2:%.*]], %[[DO_BODY]] ]
+; GFX9-NEXT: [[TMP1:%.*]] = phi <2 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX9-NEXT: [[TMP2]] = load <2 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX9-NEXT: [[OTHERELE2]] = load i8, ptr addrspace(3) [[GEP2]], align 2
+; GFX9-NEXT: [[TMP3:%.*]] = shufflevector <2 x i8> [[TMP2]], <2 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT: [[VEC02:%.*]] = insertelement <16 x i8> [[TMP3]], i8 [[OTHERELE2]], i64 10
+; GFX9-NEXT: [[TMP4:%.*]] = shufflevector <2 x i8> [[TMP1]], <2 x i8> poison, <16 x i32> <i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT: [[VEC111:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT: [[VEC12:%.*]] = insertelement <16 x i8> [[VEC111]], i8 [[PHI1]], i64 10
+; GFX9-NEXT: store <16 x i8> [[VEC12]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX9-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX9-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: store <16 x i8> [[VEC12]], ptr [[OUT]], align 16
+; GFX9-NEXT: store <16 x i8> [[VEC02]], ptr [[OUT1]], align 16
+; GFX9-NEXT: ret void
+;
+entry:
+ %gep0 = getelementptr i8, ptr addrspace(3) %inptr0, i32 0
+ %ele0 = load i8, ptr addrspace(3) %gep0, align 8
+ %gep1 = getelementptr i8, ptr addrspace(3) %inptr0, i32 1
+ %ele1 = load i8, ptr addrspace(3) %gep1, align 1
+ %gep2 = getelementptr i8, ptr addrspace(3) %inptr0, i32 2
+ %ele2 = load i8, ptr addrspace(3) %gep2, align 2
+ br label %do.body
+
+do.body:
+ %phi1 = phi i8 [ %ele2, %entry ], [ %otherele2, %do.body ]
+ %phi2 = phi i8 [ %ele1, %entry ], [ %otherele1, %do.body ]
+ %phi3 = phi i8 [ %ele0, %entry ], [ %otherele0, %do.body ]
+ %otherele0 = load i8, ptr addrspace(3) %gep0, align 8
+ %otherele1 = load i8, ptr addrspace(3) %gep1, align 1
+ %otherele2 = load i8, ptr addrspace(3) %gep2, align 2
+ %vec00 = insertelement <16 x i8> poison, i8 %otherele0, i64 8
+ %vec01 = insertelement <16 x i8> %vec00, i8 %otherele1, i64 9
+ %vec02 = insertelement <16 x i8> %vec01, i8 %otherele2, i64 10
+ %vec10 = insertelement <16 x i8> poison, i8 %phi3, i64 8
+ %vec11 = insertelement <16 x i8> %vec10, i8 %phi2, i64 9
+ %vec12 = insertelement <16 x i8> %vec11, i8 %phi1, i64 10
+ store <16 x i8> %vec12, ptr addrspace(3) %inptr1, align 2
+ %cmp = icmp eq i32 %flag, 0
+ br i1 %cmp, label %exit, label %do.body
+
+exit:
+ store <16 x i8> %vec12, ptr %out
+ store <16 x i8> %vec02, ptr %out1
+ ret void
+}
+
+define protected amdgpu_kernel void @phi_4(ptr addrspace(3) %inptr0, ptr addrspace(3) %inptr1, ptr %out, ptr %out1, i32 %flag) {
+; GFX7-LABEL: define protected amdgpu_kernel void @phi_4(
+; GFX7-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[ENTRY:.*]]:
+; GFX7-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX7-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX7-NEXT: br label %[[DO_BODY:.*]]
+; GFX7: [[DO_BODY]]:
+; GFX7-NEXT: [[TMP1:%.*]] = phi <4 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX7-NEXT: [[TMP2]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX7-NEXT: [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX7-NEXT: [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX7-NEXT: [[VEC131:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX7-NEXT: store <16 x i8> [[VEC131]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX7-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX7-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX7: [[EXIT]]:
+; GFX7-NEXT: store <16 x i8> [[VEC131]], ptr [[OUT]], align 16
+; GFX7-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT1]], align 16
+; GFX7-NEXT: ret void
+;
+; GFX8-LABEL: define protected amdgpu_kernel void @phi_4(
+; GFX8-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX8-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX8-NEXT: br label %[[DO_BODY:.*]]
+; GFX8: [[DO_BODY]]:
+; GFX8-NEXT: [[TMP1:%.*]] = phi <4 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX8-NEXT: [[TMP2]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX8-NEXT: [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT: [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT: [[VEC131:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT: store <16 x i8> [[VEC131]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX8-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX8-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: store <16 x i8> [[VEC131]], ptr [[OUT]], align 16
+; GFX8-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT1]], align 16
+; GFX8-NEXT: ret void
+;
+; GFX9-LABEL: define protected amdgpu_kernel void @phi_4(
+; GFX9-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX9-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX9-NEXT: br label %[[DO_BODY:.*]]
+; GFX9: [[DO_BODY]]:
+; GFX9-NEXT: [[TMP1:%.*]] = phi <4 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX9-NEXT: [[TMP2]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX9-NEXT: [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP2]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT: [[TMP4:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT: [[VEC131:%.*]] = shufflevector <16 x i8> [[TMP4]], <16 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT: store <16 x i8> [[VEC131]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX9-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX9-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: store <16 x i8> [[VEC131]], ptr [[OUT]], align 16
+; GFX9-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT1]], align 16
+; GFX9-NEXT: ret void
+;
+entry:
+ %gep0 = getelementptr i8, ptr addrspace(3) %inptr0, i32 0
+ %ele0 = load i8, ptr addrspace(3) %gep0, align 8
+ %gep1 = getelementptr i8, ptr addrspace(3) %inptr0, i32 1
+ %ele1 = load i8, ptr addrspace(3) %gep1, align 1
+ %gep2 = getelementptr i8, ptr addrspace(3) %inptr0, i32 2
+ %ele2 = load i8, ptr addrspace(3) %gep2, align 2
+ %gep3 = getelementptr i8, ptr addrspace(3) %inptr0, i32 3
+ %ele3 = load i8, ptr addrspace(3) %gep3, align 1
+ br label %do.body
+
+do.body:
+ %phi0 = phi i8 [ %ele3, %entry ], [ %otherele3, %do.body ]
+ %phi1 = phi i8 [ %ele2, %entry ], [ %otherele2, %do.body ]
+ %phi2 = phi i8 [ %ele1, %entry ], [ %otherele1, %do.body ]
+ %phi3 = phi i8 [ %ele0, %entry ], [ %otherele0, %do.body ]
+ %otherele0 = load i8, ptr addrspace(3) %gep0, align 8
+ %otherele1 = load i8, ptr addrspace(3) %gep1, align 1
+ %otherele2 = load i8, ptr addrspace(3) %gep2, align 2
+ %otherele3 = load i8, ptr addrspace(3) %gep3, align 1
+ %vec00 = insertelement <16 x i8> poison, i8 %otherele0, i64 8
+ %vec01 = insertelement <16 x i8> %vec00, i8 %otherele1, i64 9
+ %vec02 = insertelement <16 x i8> %vec01, i8 %otherele2, i64 10
+ %vec03 = insertelement <16 x i8> %vec02, i8 %otherele3, i64 11
+ %vec10 = insertelement <16 x i8> poison, i8 %phi3, i64 8
+ %vec11 = insertelement <16 x i8> %vec10, i8 %phi2, i64 9
+ %vec12 = insertelement <16 x i8> %vec11, i8 %phi1, i64 10
+ %vec13 = insertelement <16 x i8> %vec12, i8 %phi0, i64 11
+ store <16 x i8> %vec13, ptr addrspace(3) %inptr1, align 2
+ %cmp = icmp eq i32 %flag, 0
+ br i1 %cmp, label %exit, label %do.body
+
+exit:
+ store <16 x i8> %vec13, ptr %out
+ store <16 x i8> %vec03, ptr %out1
+ ret void
+}
+
+define protected amdgpu_kernel void @phi_4_with_stores(ptr addrspace(3) %inptr0, ptr addrspace(3) %inptr1, ptr %out, ptr %out1, i32 %flag) {
+; GFX7-LABEL: define protected amdgpu_kernel void @phi_4_with_stores(
+; GFX7-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX7-NEXT: [[ENTRY:.*]]:
+; GFX7-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX7-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX7-NEXT: br label %[[DO_BODY:.*]]
+; GFX7: [[DO_BODY]]:
+; GFX7-NEXT: [[TMP1:%.*]] = phi <4 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX7-NEXT: [[TMP2]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX7-NEXT: store <4 x i8> [[TMP1]], ptr addrspace(3) [[GEP0]], align 2
+; GFX7-NEXT: [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX7-NEXT: store <16 x i8> [[TMP3]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX7-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX7-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX7: [[EXIT]]:
+; GFX7-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT]], align 16
+; GFX7-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT1]], align 16
+; GFX7-NEXT: ret void
+;
+; GFX8-LABEL: define protected amdgpu_kernel void @phi_4_with_stores(
+; GFX8-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX8-NEXT: [[ENTRY:.*]]:
+; GFX8-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX8-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX8-NEXT: br label %[[DO_BODY:.*]]
+; GFX8: [[DO_BODY]]:
+; GFX8-NEXT: [[TMP1:%.*]] = phi <4 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX8-NEXT: [[TMP2]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX8-NEXT: store <4 x i8> [[TMP1]], ptr addrspace(3) [[GEP0]], align 2
+; GFX8-NEXT: [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX8-NEXT: store <16 x i8> [[TMP3]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX8-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX8-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX8: [[EXIT]]:
+; GFX8-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT]], align 16
+; GFX8-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT1]], align 16
+; GFX8-NEXT: ret void
+;
+; GFX9-LABEL: define protected amdgpu_kernel void @phi_4_with_stores(
+; GFX9-SAME: ptr addrspace(3) [[INPTR0:%.*]], ptr addrspace(3) [[INPTR1:%.*]], ptr [[OUT:%.*]], ptr [[OUT1:%.*]], i32 [[FLAG:%.*]]) #[[ATTR0]] {
+; GFX9-NEXT: [[ENTRY:.*]]:
+; GFX9-NEXT: [[GEP0:%.*]] = getelementptr i8, ptr addrspace(3) [[INPTR0]], i32 0
+; GFX9-NEXT: [[TMP0:%.*]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX9-NEXT: br label %[[DO_BODY:.*]]
+; GFX9: [[DO_BODY]]:
+; GFX9-NEXT: [[TMP1:%.*]] = phi <4 x i8> [ [[TMP0]], %[[ENTRY]] ], [ [[TMP2:%.*]], %[[DO_BODY]] ]
+; GFX9-NEXT: [[TMP2]] = load <4 x i8>, ptr addrspace(3) [[GEP0]], align 8
+; GFX9-NEXT: store <4 x i8> [[TMP1]], ptr addrspace(3) [[GEP0]], align 2
+; GFX9-NEXT: [[TMP3:%.*]] = shufflevector <4 x i8> [[TMP1]], <4 x i8> poison, <16 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 0, i32 1, i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison>
+; GFX9-NEXT: store <16 x i8> [[TMP3]], ptr addrspace(3) [[INPTR1]], align 2
+; GFX9-NEXT: [[CMP:%.*]] = icmp eq i32 [[FLAG]], 0
+; GFX9-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[DO_BODY]]
+; GFX9: [[EXIT]]:
+; GFX9-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT]], align 16
+; GFX9-NEXT: store <16 x i8> [[TMP3]], ptr [[OUT1]], align 16
+; GFX9-NEXT: ret void
+;
+entry:
+ %gep0 = getelementptr i8, ptr addrspace(3) %inptr0, i32 0
+ %ele0 = load i8, ptr addrspace(3) %gep0, align 8
+ %gep1 = getelementptr i8, ptr addrspace(3) %inptr0, i32 1
+ %ele1 = load i8, ptr addrspace(3) %gep1, align 1
+ %gep2 = getelementptr i8, ptr addrspace(3) %inptr0, i32 2
+ %ele2 = load i8, ptr addrspace(3) %gep2, align 2
+ %gep3 = getelementptr i8, ptr addrspace(3) %inptr0, i32 3
+ %ele3 = load i8, ptr addrspace(3) %gep3, align 1
+ br label %do.body
+
+do.body:
+ %phi0 = phi i8 [ %ele3, %entry ], [ %otherele3, %do.body ]
+ %phi1 = phi i8 [ %ele2, %entry ], [ %otherele2, %do.body ]
+ %phi2 = phi i8 [ %ele1, %entry ], [ %otherele1, %do.body ]
+ %phi3 = phi i8 [ %ele0, %entry ], [ %otherele0, %do.body ]
+ %otherele0 = load i8, ptr addrspace(3) %gep0, align 8
+ %otherele1 = load i8, ptr addrspace(3) %gep1, align 1
+ %otherele2 = load i8, ptr addrspace(3) %gep2, align 2
+ %otherele3 = load i8, ptr addrspace(3) %gep3, align 1
+ store i8 %phi3, ptr addrspace(3) %gep0, align 2
+ store i8 %phi2, ptr addrspace(3) %gep1, align 2
+ store i8 %phi1, ptr addrspace(3) %gep2, align 2
+ store i8 %phi0, ptr addrspace(3) %gep3, align 2
+ %vec10 = insertelement <16 x i8> poison, i8 %phi3, i64 8
+ %vec11 = insertelement <16 x i8> %vec10, i8 %phi2, i64 9
+ %vec12 = insertelement <16 x i8> %vec11, i8 %phi1, i64 10
+ %vec13 = insertelement <16 x i8> %vec12, i8 %phi0, i64 11
+ store <16 x i8> %vec13, ptr addrspace(3) %inptr1, align 2
+ %cmp = icmp eq i32 %flag, 0
+ br i1 %cmp, label %exit, label %do.body
+
+exit:
+ store <16 x i8> %vec13, ptr %out
+ store <16 x i8> %vec13, ptr %out1
+ ret void
+}
+
More information about the llvm-commits
mailing list