[llvm] [VectorCombine][AMDGPU] Shrink demanded vector loads (PR #202501)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 11 02:03:42 PDT 2026
https://github.com/tianhbai updated https://github.com/llvm/llvm-project/pull/202501
>From 2137b1259c09fbc6a1832df299f8becb0e4a8342 Mon Sep 17 00:00:00 2001
From: tianhbai <tianhbai at gmail.com>
Date: Thu, 11 Jun 2026 11:14:17 +0800
Subject: [PATCH] [VectorCombine][AMDGPU] Shrink vector loads for demanded
extracts
Teach VectorCombine to narrow fixed vector loads when all live users are constant-index extractelement instructions and only a subset of elements is demanded. The transform preserves the element type and only changes the element count.
Update AMDGPU load cost modeling so profitable 32/64/96/128-bit vector load widths are visible to the generic transform.
Co-authored-by: Codex <noreply at openai.com>
---
.../AMDGPU/AMDGPUTargetTransformInfo.cpp | 27 +
.../Transforms/Vectorize/VectorCombine.cpp | 170 ++++++
.../Analysis/CostModel/AMDGPU/load-width.ll | 19 +
llvm/test/Analysis/CostModel/AMDGPU/load.ll | 54 +-
.../AMDGPU/shrink-demanded-load.ll | 514 ++++++++++++++++++
5 files changed, 757 insertions(+), 27 deletions(-)
create mode 100644 llvm/test/Analysis/CostModel/AMDGPU/load-width.ll
create mode 100644 llvm/test/Transforms/VectorCombine/AMDGPU/shrink-demanded-load.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
index 7631bb2dc6828..edaf82d92e142 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
@@ -1805,7 +1805,34 @@ InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
return divideCeil(DL.getTypeSizeInBits(VecTy) - 1,
getLoadStoreVecRegBitWidth(AddressSpace));
}
+
+ InstructionCost BaseCost = BaseT::getMemoryOpCost(
+ Opcode, Src, Alignment, AddressSpace, CostKind, OpInfo, I);
+ if (Opcode != Instruction::Load || CostKind != TTI::TCK_RecipThroughput)
+ return BaseCost;
+
+ switch (AddressSpace) {
+ case AMDGPUAS::GLOBAL_ADDRESS:
+ case AMDGPUAS::CONSTANT_ADDRESS:
+ case AMDGPUAS::CONSTANT_ADDRESS_32BIT:
+ case AMDGPUAS::FLAT_ADDRESS:
+ case AMDGPUAS::LOCAL_ADDRESS:
+ case AMDGPUAS::PRIVATE_ADDRESS:
+ break;
+ default:
+ return BaseCost;
+ }
+
+ TypeSize StoreSize = DL.getTypeStoreSizeInBits(VecTy);
+ if (StoreSize.isScalable())
+ return BaseCost;
+
+ constexpr uint64_t MaxSingleMemoryOpBits = 128;
+ InstructionCost WidthCost =
+ divideCeil(StoreSize.getFixedValue(), MaxSingleMemoryOpBits);
+ return std::max(BaseCost, WidthCost);
}
+
return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace, CostKind,
OpInfo, I);
}
diff --git a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
index 99e45bdc8ee21..08a3946895ec5 100644
--- a/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
+++ b/llvm/lib/Transforms/Vectorize/VectorCombine.cpp
@@ -133,6 +133,7 @@ class VectorCombine {
bool foldBinopOfReductions(Instruction &I);
bool foldSingleElementStore(Instruction &I);
bool scalarizeLoad(Instruction &I);
+ bool shrinkLoadForExtracts(Instruction &I);
bool scalarizeLoadExtract(LoadInst *LI, VectorType *VecTy, Value *Ptr);
bool scalarizeLoadBitcast(LoadInst *LI, VectorType *VecTy, Value *Ptr);
bool scalarizeExtExtract(Instruction &I);
@@ -2157,6 +2158,173 @@ bool VectorCombine::scalarizeLoadExtract(LoadInst *LI, VectorType *VecTy,
return true;
}
+/// Try to shrink vector loads feeding extractelement instructions when some
+/// of the loaded lanes are not demanded.
+bool VectorCombine::shrinkLoadForExtracts(Instruction &I) {
+ auto *OldLoad = dyn_cast<LoadInst>(&I);
+ if (!OldLoad || !OldLoad->isSimple())
+ return false;
+
+ auto *OldLoadTy = dyn_cast<FixedVectorType>(OldLoad->getType());
+ if (!OldLoadTy)
+ return false;
+
+ Type *EltTy = OldLoadTy->getElementType();
+ if (!EltTy->isSized())
+ return false;
+
+ TypeSize EltSizeInBits = DL->getTypeSizeInBits(EltTy);
+ if (EltSizeInBits.isScalable())
+ return false;
+ uint64_t EltBits = EltSizeInBits.getFixedValue();
+ if (EltBits < 8 || EltBits % 8 != 0)
+ return false;
+ uint64_t EltBytes = EltBits / 8;
+
+ TypeSize OldLoadSize = DL->getTypeStoreSize(OldLoadTy);
+ if (OldLoadSize.isScalable())
+ return false;
+ InstructionCost OldLoadCost =
+ TTI.getMemoryOpCost(Instruction::Load, OldLoadTy, OldLoad->getAlign(),
+ OldLoad->getPointerAddressSpace(), CostKind);
+ if (!OldLoadCost.isValid())
+ return false;
+
+ struct ChunkRun {
+ unsigned Start;
+ unsigned Length;
+ };
+
+ unsigned NumElts = OldLoadTy->getNumElements();
+ SmallVector<ExtractElementInst *, 8> Extracts;
+ APInt DemandedElts = APInt::getZero(NumElts);
+
+ for (User *U : OldLoad->users()) {
+ auto *Ext = dyn_cast<ExtractElementInst>(U);
+ if (!Ext)
+ return false;
+ if (Ext->use_empty())
+ continue;
+
+ auto *Idx = dyn_cast<ConstantInt>(Ext->getIndexOperand());
+ if (!Idx || Idx->getValue().uge(NumElts))
+ return false;
+
+ Extracts.push_back(Ext);
+ DemandedElts.setBit(Idx->getZExtValue());
+ }
+
+ if (Extracts.empty() || DemandedElts.isAllOnes())
+ return false;
+
+ SmallVector<ChunkRun, 4> ElementRuns;
+ SmallVector<Type *, 4> NewLoadTys;
+ SmallVector<uint64_t, 4> NewLoadOffsets;
+
+ for (unsigned I = 0; I != NumElts;) {
+ if (!DemandedElts[I]) {
+ ++I;
+ continue;
+ }
+
+ unsigned Start = I;
+ do {
+ ++I;
+ } while (I != NumElts && DemandedElts[I]);
+ unsigned Length = I - Start;
+
+ // Single element runs are better handled by existing scalarization. Keep
+ // this fold focused on forming narrower vector loads.
+ if (Length == 1)
+ return false;
+
+ ElementRuns.push_back({Start, Length});
+ }
+
+ TypeSize NewLoadSize = TypeSize::getFixed(0);
+ InstructionCost NewLoadCost = 0;
+ for (const ChunkRun &Run : ElementRuns) {
+ auto *NewLoadTy = FixedVectorType::get(EltTy, Run.Length);
+ uint64_t NewLoadOffset = Run.Start * EltBytes;
+ NewLoadTys.push_back(NewLoadTy);
+ NewLoadOffsets.push_back(NewLoadOffset);
+ NewLoadSize += DL->getTypeStoreSize(NewLoadTy);
+ NewLoadCost +=
+ TTI.getMemoryOpCost(Instruction::Load, NewLoadTy,
+ commonAlignment(OldLoad->getAlign(), NewLoadOffset),
+ OldLoad->getPointerAddressSpace(), CostKind);
+ }
+
+ if (NewLoadSize >= OldLoadSize)
+ return false;
+
+ if (!NewLoadCost.isValid() || NewLoadCost > OldLoadCost)
+ return false;
+
+ LLVM_DEBUG(dbgs() << "Found a vector load with undemanded elements: " << I
+ << "\n");
+
+ Builder.SetInsertPoint(OldLoad);
+ Builder.SetCurrentDebugLocation(OldLoad->getDebugLoc());
+
+ SmallVector<LoadInst *, 4> NewLoads;
+ SmallVector<unsigned, 4> RunStarts;
+ Type *OffsetTy = DL->getIndexType(OldLoad->getPointerOperandType());
+ auto CreateNarrowLoad = [&](Type *NewLoadTy,
+ uint64_t NewLoadOffset) -> LoadInst * {
+ Value *NewPtr = OldLoad->getPointerOperand();
+ if (NewLoadOffset != 0)
+ NewPtr = Builder.CreateGEP(Builder.getInt8Ty(), NewPtr,
+ ConstantInt::get(OffsetTy, NewLoadOffset));
+
+ auto *NewLoad = cast<LoadInst>(Builder.CreateAlignedLoad(
+ NewLoadTy, NewPtr,
+ commonAlignment(OldLoad->getAlign(), NewLoadOffset)));
+ copyMetadataForLoad(*NewLoad, *OldLoad);
+ if (auto *RangeMD = OldLoad->getMetadata(LLVMContext::MD_range))
+ NewLoad->setMetadata(LLVMContext::MD_range, RangeMD);
+
+ AAMDNodes OldAAMD = OldLoad->getAAMetadata();
+ NewLoad->setAAMetadata(
+ OldAAMD.adjustForAccess(NewLoadOffset, NewLoadTy, *DL));
+ return NewLoad;
+ };
+
+ for (auto [Idx, Run] : enumerate(ElementRuns)) {
+ NewLoads.push_back(CreateNarrowLoad(NewLoadTys[Idx], NewLoadOffsets[Idx]));
+ RunStarts.push_back(Run.Start);
+ }
+
+ auto ExtractSubElt = [&](unsigned SubElt, Instruction *InsertPt) -> Value * {
+ for (auto [I, NewLoad] : enumerate(NewLoads)) {
+ unsigned Start = RunStarts[I];
+ unsigned Length =
+ cast<FixedVectorType>(NewLoad->getType())->getNumElements();
+ if (SubElt < Start || SubElt >= Start + Length)
+ continue;
+
+ Builder.SetInsertPoint(InsertPt);
+ return Builder.CreateExtractElement(NewLoad,
+ Builder.getInt32(SubElt - Start));
+ }
+ llvm_unreachable("Extracted element is not covered by a new load");
+ };
+
+ for (ExtractElementInst *Ext : Extracts) {
+ auto *Idx = cast<ConstantInt>(Ext->getIndexOperand());
+ unsigned EltIdx = Idx->getZExtValue();
+
+ Builder.SetInsertPoint(Ext);
+ Builder.SetCurrentDebugLocation(Ext->getDebugLoc());
+
+ Value *NewValue = ExtractSubElt(EltIdx, Ext);
+ replaceValue(*Ext, *NewValue, false);
+ }
+
+ Worklist.push(OldLoad);
+ return true;
+}
+
/// Try to scalarize vector loads feeding bitcast instructions.
bool VectorCombine::scalarizeLoadBitcast(LoadInst *LI, VectorType *VecTy,
Value *Ptr) {
@@ -6277,6 +6445,8 @@ bool VectorCombine::run() {
return true;
break;
case Instruction::Load:
+ if (shrinkLoadForExtracts(I))
+ return true;
if (shrinkLoadForShuffles(I))
return true;
break;
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/load-width.ll b/llvm/test/Analysis/CostModel/AMDGPU/load-width.ll
new file mode 100644
index 0000000000000..7a7dfec2a36d7
--- /dev/null
+++ b/llvm/test/Analysis/CostModel/AMDGPU/load-width.ll
@@ -0,0 +1,19 @@
+; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes="print<cost-model>" 2>&1 -disable-output -mtriple=amdgcn-amd-amdpal -mcpu=gfx1201 < %s | FileCheck %s
+
+define void @load_width_costs(ptr addrspace(4) %p) {
+; CHECK-LABEL: 'load_width_costs'
+; CHECK-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %wide = load <6 x i64>, ptr addrspace(4) %p, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %b96 = load <3 x i32>, ptr addrspace(4) %p, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %b64 = load i64, ptr addrspace(4) %p, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %b128 = load <4 x i32>, ptr addrspace(4) %p, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %two_b128 = load <8 x i32>, ptr addrspace(4) %p, align 8
+; CHECK-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
+;
+ %wide = load <6 x i64>, ptr addrspace(4) %p, align 8
+ %b96 = load <3 x i32>, ptr addrspace(4) %p, align 8
+ %b64 = load i64, ptr addrspace(4) %p, align 8
+ %b128 = load <4 x i32>, ptr addrspace(4) %p, align 8
+ %two_b128 = load <8 x i32>, ptr addrspace(4) %p, align 8
+ ret void
+}
diff --git a/llvm/test/Analysis/CostModel/AMDGPU/load.ll b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
index 6ec84bd88cd4d..7afc912c5ca1c 100644
--- a/llvm/test/Analysis/CostModel/AMDGPU/load.ll
+++ b/llvm/test/Analysis/CostModel/AMDGPU/load.ll
@@ -168,23 +168,23 @@ define void @loads_addrspace_1(i32 %arg) {
; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %17 = load <8 x i1>, ptr addrspace(1) poison, align 1
; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %18 = load <8 x i8>, ptr addrspace(1) poison, align 8
; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %19 = load <8 x i16>, ptr addrspace(1) poison, align 16
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %20 = load <8 x i32>, ptr addrspace(1) poison, align 32
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %20 = load <8 x i32>, ptr addrspace(1) poison, align 32
; GFX90A-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %21 = load <16 x i1>, ptr addrspace(1) poison, align 2
; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %22 = load <16 x i8>, ptr addrspace(1) poison, align 16
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %23 = load <16 x i16>, ptr addrspace(1) poison, align 32
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %24 = load <16 x i32>, ptr addrspace(1) poison, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %23 = load <16 x i16>, ptr addrspace(1) poison, align 32
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %24 = load <16 x i32>, ptr addrspace(1) poison, align 64
; GFX90A-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %25 = load <32 x i1>, ptr addrspace(1) poison, align 4
; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %26 = load <32 x i8>, ptr addrspace(1) poison, align 32
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %27 = load <32 x i16>, ptr addrspace(1) poison, align 64
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %28 = load <32 x i32>, ptr addrspace(1) poison, align 128
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %27 = load <32 x i16>, ptr addrspace(1) poison, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %28 = load <32 x i32>, ptr addrspace(1) poison, align 128
; GFX90A-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %29 = load <64 x i1>, ptr addrspace(1) poison, align 8
; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %30 = load <64 x i8>, ptr addrspace(1) poison, align 64
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %31 = load <64 x i16>, ptr addrspace(1) poison, align 128
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %32 = load <64 x i32>, ptr addrspace(1) poison, align 256
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %31 = load <64 x i16>, ptr addrspace(1) poison, align 128
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %32 = load <64 x i32>, ptr addrspace(1) poison, align 256
; GFX90A-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %33 = load <128 x i1>, ptr addrspace(1) poison, align 16
; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %34 = load <128 x i8>, ptr addrspace(1) poison, align 128
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %35 = load <128 x i16>, ptr addrspace(1) poison, align 256
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %36 = load <128 x i32>, ptr addrspace(1) poison, align 512
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %35 = load <128 x i16>, ptr addrspace(1) poison, align 256
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %36 = load <128 x i32>, ptr addrspace(1) poison, align 512
; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
load <1 x i1>, ptr addrspace(1) poison
@@ -255,23 +255,23 @@ define void @loads_addrspace_3(i32 %arg) {
; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %17 = load <8 x i1>, ptr addrspace(3) poison, align 1
; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %18 = load <8 x i8>, ptr addrspace(3) poison, align 8
; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %19 = load <8 x i16>, ptr addrspace(3) poison, align 16
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %20 = load <8 x i32>, ptr addrspace(3) poison, align 32
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %20 = load <8 x i32>, ptr addrspace(3) poison, align 32
; GFX90A-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %21 = load <16 x i1>, ptr addrspace(3) poison, align 2
; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %22 = load <16 x i8>, ptr addrspace(3) poison, align 16
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %23 = load <16 x i16>, ptr addrspace(3) poison, align 32
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %24 = load <16 x i32>, ptr addrspace(3) poison, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %23 = load <16 x i16>, ptr addrspace(3) poison, align 32
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %24 = load <16 x i32>, ptr addrspace(3) poison, align 64
; GFX90A-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %25 = load <32 x i1>, ptr addrspace(3) poison, align 4
; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %26 = load <32 x i8>, ptr addrspace(3) poison, align 32
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %27 = load <32 x i16>, ptr addrspace(3) poison, align 64
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %28 = load <32 x i32>, ptr addrspace(3) poison, align 128
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %27 = load <32 x i16>, ptr addrspace(3) poison, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %28 = load <32 x i32>, ptr addrspace(3) poison, align 128
; GFX90A-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %29 = load <64 x i1>, ptr addrspace(3) poison, align 8
; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %30 = load <64 x i8>, ptr addrspace(3) poison, align 64
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %31 = load <64 x i16>, ptr addrspace(3) poison, align 128
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %32 = load <64 x i32>, ptr addrspace(3) poison, align 256
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %31 = load <64 x i16>, ptr addrspace(3) poison, align 128
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %32 = load <64 x i32>, ptr addrspace(3) poison, align 256
; GFX90A-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %33 = load <128 x i1>, ptr addrspace(3) poison, align 16
; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %34 = load <128 x i8>, ptr addrspace(3) poison, align 128
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %35 = load <128 x i16>, ptr addrspace(3) poison, align 256
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %36 = load <128 x i32>, ptr addrspace(3) poison, align 512
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %35 = load <128 x i16>, ptr addrspace(3) poison, align 256
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %36 = load <128 x i32>, ptr addrspace(3) poison, align 512
; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
load <1 x i1>, ptr addrspace(3) poison
@@ -342,23 +342,23 @@ define void @loads_addrspace_5(i32 %arg) {
; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %17 = load <8 x i1>, ptr addrspace(5) poison, align 1
; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %18 = load <8 x i8>, ptr addrspace(5) poison, align 8
; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %19 = load <8 x i16>, ptr addrspace(5) poison, align 16
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %20 = load <8 x i32>, ptr addrspace(5) poison, align 32
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %20 = load <8 x i32>, ptr addrspace(5) poison, align 32
; GFX90A-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %21 = load <16 x i1>, ptr addrspace(5) poison, align 2
; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %22 = load <16 x i8>, ptr addrspace(5) poison, align 16
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %23 = load <16 x i16>, ptr addrspace(5) poison, align 32
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %24 = load <16 x i32>, ptr addrspace(5) poison, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %23 = load <16 x i16>, ptr addrspace(5) poison, align 32
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %24 = load <16 x i32>, ptr addrspace(5) poison, align 64
; GFX90A-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %25 = load <32 x i1>, ptr addrspace(5) poison, align 4
; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %26 = load <32 x i8>, ptr addrspace(5) poison, align 32
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %27 = load <32 x i16>, ptr addrspace(5) poison, align 64
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %28 = load <32 x i32>, ptr addrspace(5) poison, align 128
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %27 = load <32 x i16>, ptr addrspace(5) poison, align 64
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %28 = load <32 x i32>, ptr addrspace(5) poison, align 128
; GFX90A-NEXT: Cost Model: Found an estimated cost of 64 for instruction: %29 = load <64 x i1>, ptr addrspace(5) poison, align 8
; GFX90A-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %30 = load <64 x i8>, ptr addrspace(5) poison, align 64
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %31 = load <64 x i16>, ptr addrspace(5) poison, align 128
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 2 for instruction: %32 = load <64 x i32>, ptr addrspace(5) poison, align 256
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 8 for instruction: %31 = load <64 x i16>, ptr addrspace(5) poison, align 128
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %32 = load <64 x i32>, ptr addrspace(5) poison, align 256
; GFX90A-NEXT: Cost Model: Found an estimated cost of 128 for instruction: %33 = load <128 x i1>, ptr addrspace(5) poison, align 16
; GFX90A-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %34 = load <128 x i8>, ptr addrspace(5) poison, align 128
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %35 = load <128 x i16>, ptr addrspace(5) poison, align 256
-; GFX90A-NEXT: Cost Model: Found an estimated cost of 4 for instruction: %36 = load <128 x i32>, ptr addrspace(5) poison, align 512
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 16 for instruction: %35 = load <128 x i16>, ptr addrspace(5) poison, align 256
+; GFX90A-NEXT: Cost Model: Found an estimated cost of 32 for instruction: %36 = load <128 x i32>, ptr addrspace(5) poison, align 512
; GFX90A-NEXT: Cost Model: Found an estimated cost of 10 for instruction: ret void
;
load <1 x i1>, ptr addrspace(5) poison
diff --git a/llvm/test/Transforms/VectorCombine/AMDGPU/shrink-demanded-load.ll b/llvm/test/Transforms/VectorCombine/AMDGPU/shrink-demanded-load.ll
new file mode 100644
index 0000000000000..7776a319d1480
--- /dev/null
+++ b/llvm/test/Transforms/VectorCombine/AMDGPU/shrink-demanded-load.ll
@@ -0,0 +1,514 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes=vector-combine -S -mtriple=amdgcn-amd-amdpal -mcpu=gfx1201 | FileCheck %s
+
+define void @shrink_i32_load_prefix(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @shrink_i32_load_prefix(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <3 x i32>, ptr addrspace(4) [[P]], align 8
+; CHECK-NEXT: [[E0:%.*]] = extractelement <3 x i32> [[TMP0]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <3 x i32> [[TMP0]], i32 1
+; CHECK-NEXT: [[E2:%.*]] = extractelement <3 x i32> [[TMP0]], i32 2
+; CHECK-NEXT: store i32 [[E0]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: [[OUT1:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i32 [[E1]], ptr addrspace(1) [[OUT1]], align 4
+; CHECK-NEXT: [[OUT2:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 2
+; CHECK-NEXT: store i32 [[E2]], ptr addrspace(1) [[OUT2]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <4 x i32>, ptr addrspace(4) %p, align 8
+ %e0 = extractelement <4 x i32> %v, i32 0
+ %e1 = extractelement <4 x i32> %v, i32 1
+ %e2 = extractelement <4 x i32> %v, i32 2
+ store i32 %e0, ptr addrspace(1) %out, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out, i32 1
+ store i32 %e1, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr i32, ptr addrspace(1) %out, i32 2
+ store i32 %e2, ptr addrspace(1) %out2, align 4
+ ret void
+}
+
+define void @shrink_i16_load_prefix(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @shrink_i16_load_prefix(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x i16>, ptr addrspace(4) [[P]], align 4
+; CHECK-NEXT: [[E0:%.*]] = extractelement <2 x i16> [[TMP0]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <2 x i16> [[TMP0]], i32 1
+; CHECK-NEXT: store i16 [[E0]], ptr addrspace(1) [[OUT]], align 2
+; CHECK-NEXT: [[OUT1:%.*]] = getelementptr i16, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i16 [[E1]], ptr addrspace(1) [[OUT1]], align 2
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <4 x i16>, ptr addrspace(4) %p, align 4
+ %e0 = extractelement <4 x i16> %v, i32 0
+ %e1 = extractelement <4 x i16> %v, i32 1
+ store i16 %e0, ptr addrspace(1) %out, align 2
+ %out1 = getelementptr i16, ptr addrspace(1) %out, i32 1
+ store i16 %e1, ptr addrspace(1) %out1, align 2
+ ret void
+}
+
+define void @shrink_i32_load_multiple_runs(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @shrink_i32_load_multiple_runs(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <3 x i32>, ptr addrspace(4) [[P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = getelementptr i8, ptr addrspace(4) [[P]], i64 16
+; CHECK-NEXT: [[TMP2:%.*]] = load <3 x i32>, ptr addrspace(4) [[TMP1]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = getelementptr i8, ptr addrspace(4) [[P]], i64 32
+; CHECK-NEXT: [[TMP4:%.*]] = load <3 x i32>, ptr addrspace(4) [[TMP3]], align 8
+; CHECK-NEXT: [[D0:%.*]] = extractelement <3 x i32> [[TMP0]], i32 0
+; CHECK-NEXT: [[D1:%.*]] = extractelement <3 x i32> [[TMP0]], i32 1
+; CHECK-NEXT: [[D2:%.*]] = extractelement <3 x i32> [[TMP0]], i32 2
+; CHECK-NEXT: [[D4:%.*]] = extractelement <3 x i32> [[TMP2]], i32 0
+; CHECK-NEXT: [[D5:%.*]] = extractelement <3 x i32> [[TMP2]], i32 1
+; CHECK-NEXT: [[D6:%.*]] = extractelement <3 x i32> [[TMP2]], i32 2
+; CHECK-NEXT: [[D8:%.*]] = extractelement <3 x i32> [[TMP4]], i32 0
+; CHECK-NEXT: [[D9:%.*]] = extractelement <3 x i32> [[TMP4]], i32 1
+; CHECK-NEXT: [[D10:%.*]] = extractelement <3 x i32> [[TMP4]], i32 2
+; CHECK-NEXT: [[OUT0:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 0
+; CHECK-NEXT: store i32 [[D0]], ptr addrspace(1) [[OUT0]], align 4
+; CHECK-NEXT: [[OUT1:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i32 [[D1]], ptr addrspace(1) [[OUT1]], align 4
+; CHECK-NEXT: [[OUT2:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 2
+; CHECK-NEXT: store i32 [[D2]], ptr addrspace(1) [[OUT2]], align 4
+; CHECK-NEXT: [[OUT4:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 4
+; CHECK-NEXT: store i32 [[D4]], ptr addrspace(1) [[OUT4]], align 4
+; CHECK-NEXT: [[OUT5:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 5
+; CHECK-NEXT: store i32 [[D5]], ptr addrspace(1) [[OUT5]], align 4
+; CHECK-NEXT: [[OUT6:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 6
+; CHECK-NEXT: store i32 [[D6]], ptr addrspace(1) [[OUT6]], align 4
+; CHECK-NEXT: [[OUT8:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 8
+; CHECK-NEXT: store i32 [[D8]], ptr addrspace(1) [[OUT8]], align 4
+; CHECK-NEXT: [[OUT9:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 9
+; CHECK-NEXT: store i32 [[D9]], ptr addrspace(1) [[OUT9]], align 4
+; CHECK-NEXT: [[OUT10:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 10
+; CHECK-NEXT: store i32 [[D10]], ptr addrspace(1) [[OUT10]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <12 x i32>, ptr addrspace(4) %p, align 8
+ %d0 = extractelement <12 x i32> %v, i32 0
+ %d1 = extractelement <12 x i32> %v, i32 1
+ %d2 = extractelement <12 x i32> %v, i32 2
+ %d4 = extractelement <12 x i32> %v, i32 4
+ %d5 = extractelement <12 x i32> %v, i32 5
+ %d6 = extractelement <12 x i32> %v, i32 6
+ %d8 = extractelement <12 x i32> %v, i32 8
+ %d9 = extractelement <12 x i32> %v, i32 9
+ %d10 = extractelement <12 x i32> %v, i32 10
+
+ %out0 = getelementptr i32, ptr addrspace(1) %out, i32 0
+ store i32 %d0, ptr addrspace(1) %out0, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out, i32 1
+ store i32 %d1, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr i32, ptr addrspace(1) %out, i32 2
+ store i32 %d2, ptr addrspace(1) %out2, align 4
+ %out4 = getelementptr i32, ptr addrspace(1) %out, i32 4
+ store i32 %d4, ptr addrspace(1) %out4, align 4
+ %out5 = getelementptr i32, ptr addrspace(1) %out, i32 5
+ store i32 %d5, ptr addrspace(1) %out5, align 4
+ %out6 = getelementptr i32, ptr addrspace(1) %out, i32 6
+ store i32 %d6, ptr addrspace(1) %out6, align 4
+ %out8 = getelementptr i32, ptr addrspace(1) %out, i32 8
+ store i32 %d8, ptr addrspace(1) %out8, align 4
+ %out9 = getelementptr i32, ptr addrspace(1) %out, i32 9
+ store i32 %d9, ptr addrspace(1) %out9, align 4
+ %out10 = getelementptr i32, ptr addrspace(1) %out, i32 10
+ store i32 %d10, ptr addrspace(1) %out10, align 4
+ ret void
+}
+
+define void @shrink_float_load_prefix(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @shrink_float_load_prefix(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <3 x float>, ptr addrspace(4) [[P]], align 8
+; CHECK-NEXT: [[E0:%.*]] = extractelement <3 x float> [[TMP0]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <3 x float> [[TMP0]], i32 1
+; CHECK-NEXT: [[E2:%.*]] = extractelement <3 x float> [[TMP0]], i32 2
+; CHECK-NEXT: store float [[E0]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: [[OUT1:%.*]] = getelementptr float, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store float [[E1]], ptr addrspace(1) [[OUT1]], align 4
+; CHECK-NEXT: [[OUT2:%.*]] = getelementptr float, ptr addrspace(1) [[OUT]], i32 2
+; CHECK-NEXT: store float [[E2]], ptr addrspace(1) [[OUT2]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <4 x float>, ptr addrspace(4) %p, align 8
+ %e0 = extractelement <4 x float> %v, i32 0
+ %e1 = extractelement <4 x float> %v, i32 1
+ %e2 = extractelement <4 x float> %v, i32 2
+ store float %e0, ptr addrspace(1) %out, align 4
+ %out1 = getelementptr float, ptr addrspace(1) %out, i32 1
+ store float %e1, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr float, ptr addrspace(1) %out, i32 2
+ store float %e2, ptr addrspace(1) %out2, align 4
+ ret void
+}
+
+define void @do_not_shrink_when_all_elements_are_demanded(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @do_not_shrink_when_all_elements_are_demanded(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr addrspace(4) [[P]], align 8
+; CHECK-NEXT: [[E0:%.*]] = extractelement <4 x i32> [[V]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <4 x i32> [[V]], i32 1
+; CHECK-NEXT: [[E2:%.*]] = extractelement <4 x i32> [[V]], i32 2
+; CHECK-NEXT: [[E3:%.*]] = extractelement <4 x i32> [[V]], i32 3
+; CHECK-NEXT: store i32 [[E0]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: [[OUT1:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i32 [[E1]], ptr addrspace(1) [[OUT1]], align 4
+; CHECK-NEXT: [[OUT2:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 2
+; CHECK-NEXT: store i32 [[E2]], ptr addrspace(1) [[OUT2]], align 4
+; CHECK-NEXT: [[OUT3:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 3
+; CHECK-NEXT: store i32 [[E3]], ptr addrspace(1) [[OUT3]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <4 x i32>, ptr addrspace(4) %p, align 8
+ %e0 = extractelement <4 x i32> %v, i32 0
+ %e1 = extractelement <4 x i32> %v, i32 1
+ %e2 = extractelement <4 x i32> %v, i32 2
+ %e3 = extractelement <4 x i32> %v, i32 3
+ store i32 %e0, ptr addrspace(1) %out, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out, i32 1
+ store i32 %e1, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr i32, ptr addrspace(1) %out, i32 2
+ store i32 %e2, ptr addrspace(1) %out2, align 4
+ %out3 = getelementptr i32, ptr addrspace(1) %out, i32 3
+ store i32 %e3, ptr addrspace(1) %out3, align 4
+ ret void
+}
+
+define void @do_not_shrink_non_contiguous_singletons(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @do_not_shrink_non_contiguous_singletons(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr addrspace(4) [[P]], align 8
+; CHECK-NEXT: [[E0:%.*]] = extractelement <4 x i32> [[V]], i32 0
+; CHECK-NEXT: [[E2:%.*]] = extractelement <4 x i32> [[V]], i32 2
+; CHECK-NEXT: store i32 [[E0]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: [[OUT1:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i32 [[E2]], ptr addrspace(1) [[OUT1]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <4 x i32>, ptr addrspace(4) %p, align 8
+ %e0 = extractelement <4 x i32> %v, i32 0
+ %e2 = extractelement <4 x i32> %v, i32 2
+ store i32 %e0, ptr addrspace(1) %out, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out, i32 1
+ store i32 %e2, ptr addrspace(1) %out1, align 4
+ ret void
+}
+
+define void @do_not_shrink_sub_byte_elements(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @do_not_shrink_sub_byte_elements(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[V:%.*]] = load <16 x i4>, ptr addrspace(4) [[P]], align 8
+; CHECK-NEXT: [[E0:%.*]] = extractelement <16 x i4> [[V]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <16 x i4> [[V]], i32 1
+; CHECK-NEXT: [[E2:%.*]] = extractelement <16 x i4> [[V]], i32 2
+; CHECK-NEXT: store i4 [[E0]], ptr addrspace(1) [[OUT]], align 1
+; CHECK-NEXT: [[OUT1:%.*]] = getelementptr i4, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i4 [[E1]], ptr addrspace(1) [[OUT1]], align 1
+; CHECK-NEXT: [[OUT2:%.*]] = getelementptr i4, ptr addrspace(1) [[OUT]], i32 2
+; CHECK-NEXT: store i4 [[E2]], ptr addrspace(1) [[OUT2]], align 1
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <16 x i4>, ptr addrspace(4) %p, align 8
+ %e0 = extractelement <16 x i4> %v, i32 0
+ %e1 = extractelement <16 x i4> %v, i32 1
+ %e2 = extractelement <16 x i4> %v, i32 2
+ store i4 %e0, ptr addrspace(1) %out, align 1
+ %out1 = getelementptr i4, ptr addrspace(1) %out, i32 1
+ store i4 %e1, ptr addrspace(1) %out1, align 1
+ %out2 = getelementptr i4, ptr addrspace(1) %out, i32 2
+ store i4 %e2, ptr addrspace(1) %out2, align 1
+ ret void
+}
+
+define void @do_not_change_element_type_for_sub_element_uses(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @do_not_change_element_type_for_sub_element_uses(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[V:%.*]] = load <6 x i64>, ptr addrspace(4) [[P]], align 8
+; CHECK-NEXT: [[E0:%.*]] = extractelement <6 x i64> [[V]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <6 x i64> [[V]], i32 1
+; CHECK-NEXT: [[E2:%.*]] = extractelement <6 x i64> [[V]], i32 2
+; CHECK-NEXT: [[E3:%.*]] = extractelement <6 x i64> [[V]], i32 3
+; CHECK-NEXT: [[E4:%.*]] = extractelement <6 x i64> [[V]], i32 4
+; CHECK-NEXT: [[E5:%.*]] = extractelement <6 x i64> [[V]], i32 5
+; CHECK-NEXT: [[D0:%.*]] = trunc i64 [[E0]] to i32
+; CHECK-NEXT: [[E0HI:%.*]] = lshr i64 [[E0]], 32
+; CHECK-NEXT: [[D1:%.*]] = trunc i64 [[E0HI]] to i32
+; CHECK-NEXT: [[D2:%.*]] = trunc i64 [[E1]] to i32
+; CHECK-NEXT: [[D4:%.*]] = trunc i64 [[E2]] to i32
+; CHECK-NEXT: [[E2HI:%.*]] = lshr i64 [[E2]], 32
+; CHECK-NEXT: [[D5:%.*]] = trunc i64 [[E2HI]] to i32
+; CHECK-NEXT: [[D6:%.*]] = trunc i64 [[E3]] to i32
+; CHECK-NEXT: [[D8:%.*]] = trunc i64 [[E4]] to i32
+; CHECK-NEXT: [[E4HI:%.*]] = lshr i64 [[E4]], 32
+; CHECK-NEXT: [[D9:%.*]] = trunc i64 [[E4HI]] to i32
+; CHECK-NEXT: [[D10:%.*]] = trunc i64 [[E5]] to i32
+; CHECK-NEXT: [[OUT0:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 0
+; CHECK-NEXT: store i32 [[D0]], ptr addrspace(1) [[OUT0]], align 4
+; CHECK-NEXT: [[OUT1:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i32 [[D1]], ptr addrspace(1) [[OUT1]], align 4
+; CHECK-NEXT: [[OUT2:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 2
+; CHECK-NEXT: store i32 [[D2]], ptr addrspace(1) [[OUT2]], align 4
+; CHECK-NEXT: [[OUT4:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 4
+; CHECK-NEXT: store i32 [[D4]], ptr addrspace(1) [[OUT4]], align 4
+; CHECK-NEXT: [[OUT5:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 5
+; CHECK-NEXT: store i32 [[D5]], ptr addrspace(1) [[OUT5]], align 4
+; CHECK-NEXT: [[OUT6:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 6
+; CHECK-NEXT: store i32 [[D6]], ptr addrspace(1) [[OUT6]], align 4
+; CHECK-NEXT: [[OUT8:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 8
+; CHECK-NEXT: store i32 [[D8]], ptr addrspace(1) [[OUT8]], align 4
+; CHECK-NEXT: [[OUT9:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 9
+; CHECK-NEXT: store i32 [[D9]], ptr addrspace(1) [[OUT9]], align 4
+; CHECK-NEXT: [[OUT10:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 10
+; CHECK-NEXT: store i32 [[D10]], ptr addrspace(1) [[OUT10]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <6 x i64>, ptr addrspace(4) %p, align 8
+ %e0 = extractelement <6 x i64> %v, i32 0
+ %e1 = extractelement <6 x i64> %v, i32 1
+ %e2 = extractelement <6 x i64> %v, i32 2
+ %e3 = extractelement <6 x i64> %v, i32 3
+ %e4 = extractelement <6 x i64> %v, i32 4
+ %e5 = extractelement <6 x i64> %v, i32 5
+
+ %d0 = trunc i64 %e0 to i32
+ %e0hi = lshr i64 %e0, 32
+ %d1 = trunc i64 %e0hi to i32
+ %d2 = trunc i64 %e1 to i32
+ %d4 = trunc i64 %e2 to i32
+ %e2hi = lshr i64 %e2, 32
+ %d5 = trunc i64 %e2hi to i32
+ %d6 = trunc i64 %e3 to i32
+ %d8 = trunc i64 %e4 to i32
+ %e4hi = lshr i64 %e4, 32
+ %d9 = trunc i64 %e4hi to i32
+ %d10 = trunc i64 %e5 to i32
+
+ %out0 = getelementptr i32, ptr addrspace(1) %out, i32 0
+ store i32 %d0, ptr addrspace(1) %out0, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out, i32 1
+ store i32 %d1, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr i32, ptr addrspace(1) %out, i32 2
+ store i32 %d2, ptr addrspace(1) %out2, align 4
+ %out4 = getelementptr i32, ptr addrspace(1) %out, i32 4
+ store i32 %d4, ptr addrspace(1) %out4, align 4
+ %out5 = getelementptr i32, ptr addrspace(1) %out, i32 5
+ store i32 %d5, ptr addrspace(1) %out5, align 4
+ %out6 = getelementptr i32, ptr addrspace(1) %out, i32 6
+ store i32 %d6, ptr addrspace(1) %out6, align 4
+ %out8 = getelementptr i32, ptr addrspace(1) %out, i32 8
+ store i32 %d8, ptr addrspace(1) %out8, align 4
+ %out9 = getelementptr i32, ptr addrspace(1) %out, i32 9
+ store i32 %d9, ptr addrspace(1) %out9, align 4
+ %out10 = getelementptr i32, ptr addrspace(1) %out, i32 10
+ store i32 %d10, ptr addrspace(1) %out10, align 4
+ ret void
+}
+
+define void @do_not_shrink_volatile_load(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @do_not_shrink_volatile_load(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[V:%.*]] = load volatile <4 x i32>, ptr addrspace(4) [[P]], align 8
+; CHECK-NEXT: [[E0:%.*]] = extractelement <4 x i32> [[V]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <4 x i32> [[V]], i32 1
+; CHECK-NEXT: store i32 [[E0]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: [[OUT1:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i32 [[E1]], ptr addrspace(1) [[OUT1]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load volatile <4 x i32>, ptr addrspace(4) %p, align 8
+ %e0 = extractelement <4 x i32> %v, i32 0
+ %e1 = extractelement <4 x i32> %v, i32 1
+ store i32 %e0, ptr addrspace(1) %out, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out, i32 1
+ store i32 %e1, ptr addrspace(1) %out1, align 4
+ ret void
+}
+
+define void @do_not_shrink_atomic_load(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @do_not_shrink_atomic_load(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[V:%.*]] = load atomic <4 x i32>, ptr addrspace(4) [[P]] unordered, align 16
+; CHECK-NEXT: [[E0:%.*]] = extractelement <4 x i32> [[V]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <4 x i32> [[V]], i32 1
+; CHECK-NEXT: store i32 [[E0]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: [[OUT1:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i32 [[E1]], ptr addrspace(1) [[OUT1]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load atomic <4 x i32>, ptr addrspace(4) %p unordered, align 16
+ %e0 = extractelement <4 x i32> %v, i32 0
+ %e1 = extractelement <4 x i32> %v, i32 1
+ store i32 %e0, ptr addrspace(1) %out, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out, i32 1
+ store i32 %e1, ptr addrspace(1) %out1, align 4
+ ret void
+}
+
+define void @shrink_preserves_metadata_for_shorter_vector(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @shrink_preserves_metadata_for_shorter_vector(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <3 x i64>, ptr addrspace(4) [[P]], align 8, !range [[RNG0:![0-9]+]], !noundef [[META1:![0-9]+]]
+; CHECK-NEXT: [[E0:%.*]] = extractelement <3 x i64> [[TMP0]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <3 x i64> [[TMP0]], i32 1
+; CHECK-NEXT: [[E2:%.*]] = extractelement <3 x i64> [[TMP0]], i32 2
+; CHECK-NEXT: store i64 [[E0]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: [[OUT1:%.*]] = getelementptr i64, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i64 [[E1]], ptr addrspace(1) [[OUT1]], align 8
+; CHECK-NEXT: [[OUT2:%.*]] = getelementptr i64, ptr addrspace(1) [[OUT]], i32 2
+; CHECK-NEXT: store i64 [[E2]], ptr addrspace(1) [[OUT2]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <4 x i64>, ptr addrspace(4) %p, align 8, !range !0, !noundef !1
+ %e0 = extractelement <4 x i64> %v, i32 0
+ %e1 = extractelement <4 x i64> %v, i32 1
+ %e2 = extractelement <4 x i64> %v, i32 2
+ store i64 %e0, ptr addrspace(1) %out, align 8
+ %out1 = getelementptr i64, ptr addrspace(1) %out, i32 1
+ store i64 %e1, ptr addrspace(1) %out1, align 8
+ %out2 = getelementptr i64, ptr addrspace(1) %out, i32 2
+ store i64 %e2, ptr addrspace(1) %out2, align 8
+ ret void
+}
+
+define void @shrink_ptr_load_prefix(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @shrink_ptr_load_prefix(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <3 x ptr>, ptr addrspace(4) [[P]], align 8, !noundef [[META1]]
+; CHECK-NEXT: [[E0:%.*]] = extractelement <3 x ptr> [[TMP0]], i32 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <3 x ptr> [[TMP0]], i32 1
+; CHECK-NEXT: [[E2:%.*]] = extractelement <3 x ptr> [[TMP0]], i32 2
+; CHECK-NEXT: store ptr [[E0]], ptr addrspace(1) [[OUT]], align 8
+; CHECK-NEXT: [[OUT1:%.*]] = getelementptr ptr, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store ptr [[E1]], ptr addrspace(1) [[OUT1]], align 8
+; CHECK-NEXT: [[OUT2:%.*]] = getelementptr ptr, ptr addrspace(1) [[OUT]], i32 2
+; CHECK-NEXT: store ptr [[E2]], ptr addrspace(1) [[OUT2]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <4 x ptr>, ptr addrspace(4) %p, align 8, !noundef !1
+ %e0 = extractelement <4 x ptr> %v, i32 0
+ %e1 = extractelement <4 x ptr> %v, i32 1
+ %e2 = extractelement <4 x ptr> %v, i32 2
+ store ptr %e0, ptr addrspace(1) %out, align 8
+ %out1 = getelementptr ptr, ptr addrspace(1) %out, i32 1
+ store ptr %e1, ptr addrspace(1) %out1, align 8
+ %out2 = getelementptr ptr, ptr addrspace(1) %out, i32 2
+ store ptr %e2, ptr addrspace(1) %out2, align 8
+ ret void
+}
+
+define void @shrink_i32_load_suffix_with_tbaa(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @shrink_i32_load_suffix_with_tbaa(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = getelementptr i8, ptr addrspace(4) [[P]], i64 4
+; CHECK-NEXT: [[TMP1:%.*]] = load <3 x i32>, ptr addrspace(4) [[TMP0]], align 4, !tbaa [[TBAA2:![0-9]+]]
+; CHECK-NEXT: [[E1:%.*]] = extractelement <3 x i32> [[TMP1]], i32 0
+; CHECK-NEXT: [[E2:%.*]] = extractelement <3 x i32> [[TMP1]], i32 1
+; CHECK-NEXT: [[E3:%.*]] = extractelement <3 x i32> [[TMP1]], i32 2
+; CHECK-NEXT: store i32 [[E1]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: [[OUT1:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 1
+; CHECK-NEXT: store i32 [[E2]], ptr addrspace(1) [[OUT1]], align 4
+; CHECK-NEXT: [[OUT2:%.*]] = getelementptr i32, ptr addrspace(1) [[OUT]], i32 2
+; CHECK-NEXT: store i32 [[E3]], ptr addrspace(1) [[OUT2]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <4 x i32>, ptr addrspace(4) %p, align 8, !tbaa.struct !6
+ %e1 = extractelement <4 x i32> %v, i32 1
+ %e2 = extractelement <4 x i32> %v, i32 2
+ %e3 = extractelement <4 x i32> %v, i32 3
+ store i32 %e1, ptr addrspace(1) %out, align 4
+ %out1 = getelementptr i32, ptr addrspace(1) %out, i32 1
+ store i32 %e2, ptr addrspace(1) %out1, align 4
+ %out2 = getelementptr i32, ptr addrspace(1) %out, i32 2
+ store i32 %e3, ptr addrspace(1) %out2, align 4
+ ret void
+}
+
+define void @do_not_shrink_variable_index(ptr addrspace(1) %out, ptr addrspace(4) %p, i32 %idx) {
+; CHECK-LABEL: define void @do_not_shrink_variable_index(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]], i32 [[IDX:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr addrspace(4) [[P]], align 8
+; CHECK-NEXT: [[E:%.*]] = extractelement <4 x i32> [[V]], i32 [[IDX]]
+; CHECK-NEXT: store i32 [[E]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <4 x i32>, ptr addrspace(4) %p, align 8
+ %e = extractelement <4 x i32> %v, i32 %idx
+ store i32 %e, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @do_not_shrink_oob_index(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @do_not_shrink_oob_index(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[V:%.*]] = load <4 x i32>, ptr addrspace(4) [[P]], align 8
+; CHECK-NEXT: [[E:%.*]] = extractelement <4 x i32> [[V]], i32 4
+; CHECK-NEXT: store i32 [[E]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <4 x i32>, ptr addrspace(4) %p, align 8
+ %e = extractelement <4 x i32> %v, i32 4
+ store i32 %e, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+define void @do_not_shrink_one_element_vector(ptr addrspace(1) %out, ptr addrspace(4) %p) {
+; CHECK-LABEL: define void @do_not_shrink_one_element_vector(
+; CHECK-SAME: ptr addrspace(1) [[OUT:%.*]], ptr addrspace(4) [[P:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[V:%.*]] = load <1 x i32>, ptr addrspace(4) [[P]], align 4
+; CHECK-NEXT: [[E:%.*]] = extractelement <1 x i32> [[V]], i32 0
+; CHECK-NEXT: store i32 [[E]], ptr addrspace(1) [[OUT]], align 4
+; CHECK-NEXT: ret void
+;
+entry:
+ %v = load <1 x i32>, ptr addrspace(4) %p, align 4
+ %e = extractelement <1 x i32> %v, i32 0
+ store i32 %e, ptr addrspace(1) %out, align 4
+ ret void
+}
+
+!0 = !{i64 0, i64 4294967296}
+!1 = !{}
+!2 = !{!3, !3, i64 0, i64 4}
+!3 = !{!4, i64 4, !"int"}
+!4 = !{!5, i64 1, !"omnipotent char"}
+!5 = !{!"Simple C/C++ TBAA"}
+!6 = !{i64 0, i64 4, !2, i64 4, i64 12, !2}
+;.
+; CHECK: [[RNG0]] = !{i64 0, i64 4294967296}
+; CHECK: [[META1]] = !{}
+; CHECK: [[TBAA2]] = !{[[META3:![0-9]+]], [[META3]], i64 0, i64 4}
+; CHECK: [[META3]] = !{[[META4:![0-9]+]], i64 4, !"int"}
+; CHECK: [[META4]] = !{[[META5:![0-9]+]], i64 1, !"omnipotent char"}
+; CHECK: [[META5]] = !{!"Simple C/C++ TBAA"}
+;.
More information about the llvm-commits
mailing list