[llvm-branch-commits] [llvm] [AMDGPU] Price memory operations by the number of memory instructions (PR #233261)

[Matt Arsenault via llvm-branch-commits]

================
@@ -2115,22 +2115,79 @@ GCNTTIImpl::fpenvIEEEMode(const Instruction &I) const {
                                                : KnownIEEEMode::On;
 }
 
+static unsigned getNumMemoryInsts(unsigned Bytes, unsigned MaxBytes,
+                                  bool HasDwordx3) {
+  unsigned NumInsts = Bytes / MaxBytes;
+  unsigned Rem = Bytes % MaxBytes;
+  if (HasDwordx3 && Rem >= 12) {
+    ++NumInsts;
+    Rem -= 12;
+  }
+  return NumInsts + llvm::popcount(Rem);
+}
+
 InstructionCost GCNTTIImpl::getMemoryOpCost(unsigned Opcode, Type *Src,
                                             Align Alignment,
                                             unsigned AddressSpace,
                                             TTI::TargetCostKind CostKind,
                                             TTI::OperandValueInfo OpInfo,
                                             const Instruction *I) const {
-  if (VectorType *VecTy = dyn_cast<VectorType>(Src)) {
-    if ((Opcode == Instruction::Load || Opcode == Instruction::Store) &&
-        CostKind != TTI::TCK_Latency &&
-        VecTy->getElementType()->isIntegerTy(8)) {
-      return divideCeil(DL.getTypeSizeInBits(VecTy) - 1,
+  Type *EltTy = Src->getScalarType();
+  if ((Opcode != Instruction::Load && Opcode != Instruction::Store) ||
+      CostKind == TTI::TCK_Latency || isa<ScalableVectorType>(Src) ||
+      (!EltTy->isIntOrPtrTy() && !EltTy->isFloatingPointTy()))
+    return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
+                                  CostKind, OpInfo, I);
+
+  unsigned EltBits = DL.getTypeSizeInBits(EltTy);
+  if (EltBits < 8 || !isPowerOf2_32(EltBits))
+    return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
+                                  CostKind, OpInfo, I);
+
+  if (AddressSpace == AMDGPUAS::CONSTANT_ADDRESS ||
+      AddressSpace == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
+      AddressSpace == AMDGPUAS::BUFFER_FAT_POINTER) {
+    if (Src->isVectorTy() && EltTy->isIntegerTy(8))
+      return divideCeil(DL.getTypeSizeInBits(Src) - 1,
                         getLoadStoreVecRegBitWidth(AddressSpace));
+    return BaseT::getMemoryOpCost(Opcode, Src, Alignment, AddressSpace,
+                                  CostKind, OpInfo, I);
+  }
+
+  unsigned Bytes = DL.getTypeStoreSize(Src);
+  unsigned MaxBytes =
+      std::min(16u, getLoadStoreVecRegBitWidth(AddressSpace) / 8);
+  if (AddressSpace == AMDGPUAS::FLAT_ADDRESS &&
+      !ST->hasMultiDwordFlatScratchAddressing())
+    MaxBytes = getLoadStoreVecRegBitWidth(AMDGPUAS::PRIVATE_ADDRESS) / 8;
+  bool IsDS = AddressSpace == AMDGPUAS::LOCAL_ADDRESS ||
+              AddressSpace == AMDGPUAS::REGION_ADDRESS;
+  bool HasDwordx3 = IsDS ? ST->hasDS96AndDS128() : ST->hasDwordx3LoadStores();
----------------
arsenm wrote:

This doesn't seem right? All the b96 loads and stores were introduced at the same time?

https://github.com/llvm/llvm-project/pull/233261