[llvm] [TTI] NFC: Port TLI.shouldSinkOperands to TTI (PR #110564)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Sep 30 13:26:02 PDT 2024
llvmbot wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-webassembly
Author: Jeffrey Byrnes (jrbyrnes)
<details>
<summary>Changes</summary>
Porting to TTI provides direct access to the instruction cost model, which can enable instruction cost based sinking without introducing code duplication.
Added some folks from various backends this touches for any backend specific concerns.
---
Patch is 75.51 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/110564.diff
29 Files Affected:
- (modified) llvm/include/llvm/Analysis/TargetTransformInfo.h (+11)
- (modified) llvm/include/llvm/Analysis/TargetTransformInfoImpl.h (+4)
- (modified) llvm/include/llvm/CodeGen/TargetLowering.h (-10)
- (modified) llvm/lib/Analysis/TargetTransformInfo.cpp (+5)
- (modified) llvm/lib/CodeGen/CodeGenPrepare.cpp (+1-1)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.cpp (-416)
- (modified) llvm/lib/Target/AArch64/AArch64ISelLowering.h (-3)
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp (+417)
- (modified) llvm/lib/Target/AArch64/AArch64TargetTransformInfo.h (+3-1)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp (-19)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUISelLowering.h (-3)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp (+19)
- (modified) llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h (+2)
- (modified) llvm/lib/Target/ARM/ARMISelLowering.cpp (-143)
- (modified) llvm/lib/Target/ARM/ARMISelLowering.h (-2)
- (modified) llvm/lib/Target/ARM/ARMTargetTransformInfo.cpp (+146)
- (modified) llvm/lib/Target/ARM/ARMTargetTransformInfo.h (+1)
- (modified) llvm/lib/Target/RISCV/RISCVISelLowering.cpp (-139)
- (modified) llvm/lib/Target/RISCV/RISCVISelLowering.h (-8)
- (modified) llvm/lib/Target/RISCV/RISCVTargetTransformInfo.cpp (+140-2)
- (modified) llvm/lib/Target/RISCV/RISCVTargetTransformInfo.h (+8)
- (modified) llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.cpp (-24)
- (modified) llvm/lib/Target/WebAssembly/WebAssemblyISelLowering.h (-2)
- (modified) llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.cpp (+24)
- (modified) llvm/lib/Target/WebAssembly/WebAssemblyTargetTransformInfo.h (+4-1)
- (modified) llvm/lib/Target/X86/X86ISelLowering.cpp (-57)
- (modified) llvm/lib/Target/X86/X86ISelLowering.h (-2)
- (modified) llvm/lib/Target/X86/X86TargetTransformInfo.cpp (+79)
- (modified) llvm/lib/Target/X86/X86TargetTransformInfo.h (+2)
``````````diff
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfo.h b/llvm/include/llvm/Analysis/TargetTransformInfo.h
index 89a85bc8a90864..62b78ba81d32dc 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfo.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfo.h
@@ -1744,6 +1744,10 @@ class TargetTransformInfo {
bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
Align Alignment) const;
+ /// \return True if the operands of \p I should be sunk to \p I . The sinkable
+ /// operands are cached in \p OpsToSink
+ bool shouldSinkOperands(Instruction *I, SmallVectorImpl<Use *> &Ops) const;
+
struct VPLegalization {
enum VPTransform {
// keep the predicating parameter
@@ -2182,6 +2186,8 @@ class TargetTransformInfo::Concept {
virtual bool supportsScalableVectors() const = 0;
virtual bool hasActiveVectorLength(unsigned Opcode, Type *DataType,
Align Alignment) const = 0;
+ virtual bool shouldSinkOperands(Instruction *I,
+ SmallVectorImpl<Use *> &OpsToSink) const = 0;
virtual VPLegalization
getVPLegalizationStrategy(const VPIntrinsic &PI) const = 0;
virtual bool hasArmWideBranch(bool Thumb) const = 0;
@@ -2952,6 +2958,11 @@ class TargetTransformInfo::Model final : public TargetTransformInfo::Concept {
return Impl.hasActiveVectorLength(Opcode, DataType, Alignment);
}
+ bool shouldSinkOperands(Instruction *I,
+ SmallVectorImpl<Use *> &Ops) const override {
+ return Impl.shouldSinkOperands(I, Ops);
+ };
+
VPLegalization
getVPLegalizationStrategy(const VPIntrinsic &PI) const override {
return Impl.getVPLegalizationStrategy(PI);
diff --git a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
index eca8818cc25e62..b55953fc9509bf 100644
--- a/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
+++ b/llvm/include/llvm/Analysis/TargetTransformInfoImpl.h
@@ -968,6 +968,10 @@ class TargetTransformInfoImplBase {
return false;
}
+ bool shouldSinkOperands(Instruction *I, SmallVectorImpl<Use *> &Ops) const {
+ return false;
+ }
+
TargetTransformInfo::VPLegalization
getVPLegalizationStrategy(const VPIntrinsic &PI) const {
return TargetTransformInfo::VPLegalization(
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 3842af56e6b3d7..29a6a1a599be71 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -3085,16 +3085,6 @@ class TargetLoweringBase {
/// a larger type.
virtual bool signExtendConstant(const ConstantInt *C) const { return false; }
- /// Return true if sinking I's operands to the same basic block as I is
- /// profitable, e.g. because the operands can be folded into a target
- /// instruction during instruction selection. After calling the function
- /// \p Ops contains the Uses to sink ordered by dominance (dominating users
- /// come first).
- virtual bool shouldSinkOperands(Instruction *I,
- SmallVectorImpl<Use *> &Ops) const {
- return false;
- }
-
/// Try to optimize extending or truncating conversion instructions (like
/// zext, trunc, fptoui, uitofp) for the target.
virtual bool
diff --git a/llvm/lib/Analysis/TargetTransformInfo.cpp b/llvm/lib/Analysis/TargetTransformInfo.cpp
index b5195f764cbd1c..9d5c086b0c4485 100644
--- a/llvm/lib/Analysis/TargetTransformInfo.cpp
+++ b/llvm/lib/Analysis/TargetTransformInfo.cpp
@@ -1354,6 +1354,11 @@ bool TargetTransformInfo::hasActiveVectorLength(unsigned Opcode, Type *DataType,
return TTIImpl->hasActiveVectorLength(Opcode, DataType, Alignment);
}
+bool TargetTransformInfo::shouldSinkOperands(
+ Instruction *I, SmallVectorImpl<Use *> &OpsToSink) const {
+ return TTIImpl->shouldSinkOperands(I, OpsToSink);
+}
+
TargetTransformInfo::Concept::~Concept() = default;
TargetIRAnalysis::TargetIRAnalysis() : TTICallback(&getDefaultTTI) {}
diff --git a/llvm/lib/CodeGen/CodeGenPrepare.cpp b/llvm/lib/CodeGen/CodeGenPrepare.cpp
index 631cc26d6022fe..45ac5c69125da6 100644
--- a/llvm/lib/CodeGen/CodeGenPrepare.cpp
+++ b/llvm/lib/CodeGen/CodeGenPrepare.cpp
@@ -7566,7 +7566,7 @@ bool CodeGenPrepare::tryToSinkFreeOperands(Instruction *I) {
// If the operands of I can be folded into a target instruction together with
// I, duplicate and sink them.
SmallVector<Use *, 4> OpsToSink;
- if (!TLI->shouldSinkOperands(I, OpsToSink))
+ if (!TTI->shouldSinkOperands(I, OpsToSink))
return false;
// OpsToSink can contain multiple uses in a use chain (e.g.
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 4166d9bd22bc01..a46848c37375ee 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -16246,422 +16246,6 @@ bool AArch64TargetLowering::isExtFreeImpl(const Instruction *Ext) const {
return true;
}
-static bool isSplatShuffle(Value *V) {
- if (auto *Shuf = dyn_cast<ShuffleVectorInst>(V))
- return all_equal(Shuf->getShuffleMask());
- return false;
-}
-
-/// Check if both Op1 and Op2 are shufflevector extracts of either the lower
-/// or upper half of the vector elements.
-static bool areExtractShuffleVectors(Value *Op1, Value *Op2,
- bool AllowSplat = false) {
- // Scalable types can't be extract shuffle vectors.
- if (Op1->getType()->isScalableTy() || Op2->getType()->isScalableTy())
- return false;
-
- auto areTypesHalfed = [](Value *FullV, Value *HalfV) {
- auto *FullTy = FullV->getType();
- auto *HalfTy = HalfV->getType();
- return FullTy->getPrimitiveSizeInBits().getFixedValue() ==
- 2 * HalfTy->getPrimitiveSizeInBits().getFixedValue();
- };
-
- auto extractHalf = [](Value *FullV, Value *HalfV) {
- auto *FullVT = cast<FixedVectorType>(FullV->getType());
- auto *HalfVT = cast<FixedVectorType>(HalfV->getType());
- return FullVT->getNumElements() == 2 * HalfVT->getNumElements();
- };
-
- ArrayRef<int> M1, M2;
- Value *S1Op1 = nullptr, *S2Op1 = nullptr;
- if (!match(Op1, m_Shuffle(m_Value(S1Op1), m_Undef(), m_Mask(M1))) ||
- !match(Op2, m_Shuffle(m_Value(S2Op1), m_Undef(), m_Mask(M2))))
- return false;
-
- // If we allow splats, set S1Op1/S2Op1 to nullptr for the relavant arg so that
- // it is not checked as an extract below.
- if (AllowSplat && isSplatShuffle(Op1))
- S1Op1 = nullptr;
- if (AllowSplat && isSplatShuffle(Op2))
- S2Op1 = nullptr;
-
- // Check that the operands are half as wide as the result and we extract
- // half of the elements of the input vectors.
- if ((S1Op1 && (!areTypesHalfed(S1Op1, Op1) || !extractHalf(S1Op1, Op1))) ||
- (S2Op1 && (!areTypesHalfed(S2Op1, Op2) || !extractHalf(S2Op1, Op2))))
- return false;
-
- // Check the mask extracts either the lower or upper half of vector
- // elements.
- int M1Start = 0;
- int M2Start = 0;
- int NumElements = cast<FixedVectorType>(Op1->getType())->getNumElements() * 2;
- if ((S1Op1 &&
- !ShuffleVectorInst::isExtractSubvectorMask(M1, NumElements, M1Start)) ||
- (S2Op1 &&
- !ShuffleVectorInst::isExtractSubvectorMask(M2, NumElements, M2Start)))
- return false;
-
- if ((M1Start != 0 && M1Start != (NumElements / 2)) ||
- (M2Start != 0 && M2Start != (NumElements / 2)))
- return false;
- if (S1Op1 && S2Op1 && M1Start != M2Start)
- return false;
-
- return true;
-}
-
-/// Check if Ext1 and Ext2 are extends of the same type, doubling the bitwidth
-/// of the vector elements.
-static bool areExtractExts(Value *Ext1, Value *Ext2) {
- auto areExtDoubled = [](Instruction *Ext) {
- return Ext->getType()->getScalarSizeInBits() ==
- 2 * Ext->getOperand(0)->getType()->getScalarSizeInBits();
- };
-
- if (!match(Ext1, m_ZExtOrSExt(m_Value())) ||
- !match(Ext2, m_ZExtOrSExt(m_Value())) ||
- !areExtDoubled(cast<Instruction>(Ext1)) ||
- !areExtDoubled(cast<Instruction>(Ext2)))
- return false;
-
- return true;
-}
-
-/// Check if Op could be used with vmull_high_p64 intrinsic.
-static bool isOperandOfVmullHighP64(Value *Op) {
- Value *VectorOperand = nullptr;
- ConstantInt *ElementIndex = nullptr;
- return match(Op, m_ExtractElt(m_Value(VectorOperand),
- m_ConstantInt(ElementIndex))) &&
- ElementIndex->getValue() == 1 &&
- isa<FixedVectorType>(VectorOperand->getType()) &&
- cast<FixedVectorType>(VectorOperand->getType())->getNumElements() == 2;
-}
-
-/// Check if Op1 and Op2 could be used with vmull_high_p64 intrinsic.
-static bool areOperandsOfVmullHighP64(Value *Op1, Value *Op2) {
- return isOperandOfVmullHighP64(Op1) && isOperandOfVmullHighP64(Op2);
-}
-
-static bool shouldSinkVectorOfPtrs(Value *Ptrs, SmallVectorImpl<Use *> &Ops) {
- // Restrict ourselves to the form CodeGenPrepare typically constructs.
- auto *GEP = dyn_cast<GetElementPtrInst>(Ptrs);
- if (!GEP || GEP->getNumOperands() != 2)
- return false;
-
- Value *Base = GEP->getOperand(0);
- Value *Offsets = GEP->getOperand(1);
-
- // We only care about scalar_base+vector_offsets.
- if (Base->getType()->isVectorTy() || !Offsets->getType()->isVectorTy())
- return false;
-
- // Sink extends that would allow us to use 32-bit offset vectors.
- if (isa<SExtInst>(Offsets) || isa<ZExtInst>(Offsets)) {
- auto *OffsetsInst = cast<Instruction>(Offsets);
- if (OffsetsInst->getType()->getScalarSizeInBits() > 32 &&
- OffsetsInst->getOperand(0)->getType()->getScalarSizeInBits() <= 32)
- Ops.push_back(&GEP->getOperandUse(1));
- }
-
- // Sink the GEP.
- return true;
-}
-
-/// We want to sink following cases:
-/// (add|sub|gep) A, ((mul|shl) vscale, imm); (add|sub|gep) A, vscale;
-/// (add|sub|gep) A, ((mul|shl) zext(vscale), imm);
-static bool shouldSinkVScale(Value *Op, SmallVectorImpl<Use *> &Ops) {
- if (match(Op, m_VScale()))
- return true;
- if (match(Op, m_Shl(m_VScale(), m_ConstantInt())) ||
- match(Op, m_Mul(m_VScale(), m_ConstantInt()))) {
- Ops.push_back(&cast<Instruction>(Op)->getOperandUse(0));
- return true;
- }
- if (match(Op, m_Shl(m_ZExt(m_VScale()), m_ConstantInt())) ||
- match(Op, m_Mul(m_ZExt(m_VScale()), m_ConstantInt()))) {
- Value *ZExtOp = cast<Instruction>(Op)->getOperand(0);
- Ops.push_back(&cast<Instruction>(ZExtOp)->getOperandUse(0));
- Ops.push_back(&cast<Instruction>(Op)->getOperandUse(0));
- return true;
- }
- return false;
-}
-
-/// Check if sinking \p I's operands to I's basic block is profitable, because
-/// the operands can be folded into a target instruction, e.g.
-/// shufflevectors extracts and/or sext/zext can be folded into (u,s)subl(2).
-bool AArch64TargetLowering::shouldSinkOperands(
- Instruction *I, SmallVectorImpl<Use *> &Ops) const {
- if (IntrinsicInst *II = dyn_cast<IntrinsicInst>(I)) {
- switch (II->getIntrinsicID()) {
- case Intrinsic::aarch64_neon_smull:
- case Intrinsic::aarch64_neon_umull:
- if (areExtractShuffleVectors(II->getOperand(0), II->getOperand(1),
- /*AllowSplat=*/true)) {
- Ops.push_back(&II->getOperandUse(0));
- Ops.push_back(&II->getOperandUse(1));
- return true;
- }
- [[fallthrough]];
-
- case Intrinsic::fma:
- case Intrinsic::fmuladd:
- if (isa<VectorType>(I->getType()) &&
- cast<VectorType>(I->getType())->getElementType()->isHalfTy() &&
- !Subtarget->hasFullFP16())
- return false;
- [[fallthrough]];
- case Intrinsic::aarch64_neon_sqdmull:
- case Intrinsic::aarch64_neon_sqdmulh:
- case Intrinsic::aarch64_neon_sqrdmulh:
- // Sink splats for index lane variants
- if (isSplatShuffle(II->getOperand(0)))
- Ops.push_back(&II->getOperandUse(0));
- if (isSplatShuffle(II->getOperand(1)))
- Ops.push_back(&II->getOperandUse(1));
- return !Ops.empty();
- case Intrinsic::aarch64_neon_fmlal:
- case Intrinsic::aarch64_neon_fmlal2:
- case Intrinsic::aarch64_neon_fmlsl:
- case Intrinsic::aarch64_neon_fmlsl2:
- // Sink splats for index lane variants
- if (isSplatShuffle(II->getOperand(1)))
- Ops.push_back(&II->getOperandUse(1));
- if (isSplatShuffle(II->getOperand(2)))
- Ops.push_back(&II->getOperandUse(2));
- return !Ops.empty();
- case Intrinsic::aarch64_sve_ptest_first:
- case Intrinsic::aarch64_sve_ptest_last:
- if (auto *IIOp = dyn_cast<IntrinsicInst>(II->getOperand(0)))
- if (IIOp->getIntrinsicID() == Intrinsic::aarch64_sve_ptrue)
- Ops.push_back(&II->getOperandUse(0));
- return !Ops.empty();
- case Intrinsic::aarch64_sme_write_horiz:
- case Intrinsic::aarch64_sme_write_vert:
- case Intrinsic::aarch64_sme_writeq_horiz:
- case Intrinsic::aarch64_sme_writeq_vert: {
- auto *Idx = dyn_cast<Instruction>(II->getOperand(1));
- if (!Idx || Idx->getOpcode() != Instruction::Add)
- return false;
- Ops.push_back(&II->getOperandUse(1));
- return true;
- }
- case Intrinsic::aarch64_sme_read_horiz:
- case Intrinsic::aarch64_sme_read_vert:
- case Intrinsic::aarch64_sme_readq_horiz:
- case Intrinsic::aarch64_sme_readq_vert:
- case Intrinsic::aarch64_sme_ld1b_vert:
- case Intrinsic::aarch64_sme_ld1h_vert:
- case Intrinsic::aarch64_sme_ld1w_vert:
- case Intrinsic::aarch64_sme_ld1d_vert:
- case Intrinsic::aarch64_sme_ld1q_vert:
- case Intrinsic::aarch64_sme_st1b_vert:
- case Intrinsic::aarch64_sme_st1h_vert:
- case Intrinsic::aarch64_sme_st1w_vert:
- case Intrinsic::aarch64_sme_st1d_vert:
- case Intrinsic::aarch64_sme_st1q_vert:
- case Intrinsic::aarch64_sme_ld1b_horiz:
- case Intrinsic::aarch64_sme_ld1h_horiz:
- case Intrinsic::aarch64_sme_ld1w_horiz:
- case Intrinsic::aarch64_sme_ld1d_horiz:
- case Intrinsic::aarch64_sme_ld1q_horiz:
- case Intrinsic::aarch64_sme_st1b_horiz:
- case Intrinsic::aarch64_sme_st1h_horiz:
- case Intrinsic::aarch64_sme_st1w_horiz:
- case Intrinsic::aarch64_sme_st1d_horiz:
- case Intrinsic::aarch64_sme_st1q_horiz: {
- auto *Idx = dyn_cast<Instruction>(II->getOperand(3));
- if (!Idx || Idx->getOpcode() != Instruction::Add)
- return false;
- Ops.push_back(&II->getOperandUse(3));
- return true;
- }
- case Intrinsic::aarch64_neon_pmull:
- if (!areExtractShuffleVectors(II->getOperand(0), II->getOperand(1)))
- return false;
- Ops.push_back(&II->getOperandUse(0));
- Ops.push_back(&II->getOperandUse(1));
- return true;
- case Intrinsic::aarch64_neon_pmull64:
- if (!areOperandsOfVmullHighP64(II->getArgOperand(0),
- II->getArgOperand(1)))
- return false;
- Ops.push_back(&II->getArgOperandUse(0));
- Ops.push_back(&II->getArgOperandUse(1));
- return true;
- case Intrinsic::masked_gather:
- if (!shouldSinkVectorOfPtrs(II->getArgOperand(0), Ops))
- return false;
- Ops.push_back(&II->getArgOperandUse(0));
- return true;
- case Intrinsic::masked_scatter:
- if (!shouldSinkVectorOfPtrs(II->getArgOperand(1), Ops))
- return false;
- Ops.push_back(&II->getArgOperandUse(1));
- return true;
- default:
- return false;
- }
- }
-
- // Sink vscales closer to uses for better isel
- switch (I->getOpcode()) {
- case Instruction::GetElementPtr:
- case Instruction::Add:
- case Instruction::Sub:
- for (unsigned Op = 0; Op < I->getNumOperands(); ++Op) {
- if (shouldSinkVScale(I->getOperand(Op), Ops)) {
- Ops.push_back(&I->getOperandUse(Op));
- return true;
- }
- }
- break;
- default:
- break;
- }
-
- if (!I->getType()->isVectorTy())
- return false;
-
- switch (I->getOpcode()) {
- case Instruction::Sub:
- case Instruction::Add: {
- if (!areExtractExts(I->getOperand(0), I->getOperand(1)))
- return false;
-
- // If the exts' operands extract either the lower or upper elements, we
- // can sink them too.
- auto Ext1 = cast<Instruction>(I->getOperand(0));
- auto Ext2 = cast<Instruction>(I->getOperand(1));
- if (areExtractShuffleVectors(Ext1->getOperand(0), Ext2->getOperand(0))) {
- Ops.push_back(&Ext1->getOperandUse(0));
- Ops.push_back(&Ext2->getOperandUse(0));
- }
-
- Ops.push_back(&I->getOperandUse(0));
- Ops.push_back(&I->getOperandUse(1));
-
- return true;
- }
- case Instruction::Or: {
- // Pattern: Or(And(MaskValue, A), And(Not(MaskValue), B)) ->
- // bitselect(MaskValue, A, B) where Not(MaskValue) = Xor(MaskValue, -1)
- if (Subtarget->hasNEON()) {
- Instruction *OtherAnd, *IA, *IB;
- Value *MaskValue;
- // MainAnd refers to And instruction that has 'Not' as one of its operands
- if (match(I, m_c_Or(m_OneUse(m_Instruction(OtherAnd)),
- m_OneUse(m_c_And(m_OneUse(m_Not(m_Value(MaskValue))),
- m_Instruction(IA)))))) {
- if (match(OtherAnd,
- m_c_And(m_Specific(MaskValue), m_Instruction(IB)))) {
- Instruction *MainAnd = I->getOperand(0) == OtherAnd
- ? cast<Instruction>(I->getOperand(1))
- : cast<Instruction>(I->getOperand(0));
-
- // Both Ands should be in same basic block as Or
- if (I->getParent() != MainAnd->getParent() ||
- I->getParent() != OtherAnd->getParent())
- return false;
-
- // Non-mask operands of both Ands should also be in same basic block
- if (I->getParent() != IA->getParent() ||
- I->getParent() != IB->getParent())
- return false;
-
- Ops.push_back(&MainAnd->getOperandUse(MainAnd->getOperand(0) == IA ? 1 : 0));
- Ops.push_back(&I->getOperandUse(0));
- Ops.push_back(&I->getOperandUse(1));
-
- return true;
- }
- }
- }
-
- return false;
- }
- case Instruction::Mul: {
- int NumZExts = 0, NumSExts = 0;
- for (auto &Op : I->operands()) {
- // Make sure we are not already sinking this operand
- if (any_of(Ops, [&](Use *U) { return U->get() == Op; }))
- continue;
-
- if (match(&Op, m_SExt(m_Value()))) {
- NumSExts++;
- continue;
- } else if (match(&Op, m_ZExt(m_Value()))) {
- NumZExts++;
- continue;
- }
-
- ShuffleVectorInst *Shuffle = dyn_cast<ShuffleVectorInst>(Op);
-
- // If the Shuffle is a splat and the operand is a zext/sext, sinking the
- // operand and the s/zext can help create indexed s/umull. This is
- // especially useful to prevent i64 mul being scalarized.
- if (Shuffle && isSplatShuffle(Shuffle) &&
- match(Shuffle->getOperand(0), m_ZExtOrSExt(m_Value()))) {
- Ops.push_back(&Shuffle->getOperandUse(0));
- Ops.push_back(&Op);
- if (match(Shuffle->getOperand(0), m_SExt(m_Value())))
- NumSExts++;
- else
- NumZExts++;
- continue;
- }
-
- if (!Shuffle)
- continue;
-
- Value *ShuffleOperand = Shuffle->getOperand(0);
- InsertElementInst *Insert = dyn_cast<InsertElementInst>(ShuffleOperand);
- if (!Insert)
- continue;
-
- Instruction *OperandInstr = dyn_cast<Instruction>(Insert->getOperand(1));
- if (!OperandInstr)
- continue;
-
- ConstantInt *ElementConstant =
- dyn_cast<ConstantInt>(Insert->getOperand(2));
- // Check that the insertelement is inserting into element 0
- if (!ElementConstant || !ElementConstant->isZero())
- continue;
-
- unsigned Opcode = OperandInstr->getOpcode();
- if (Opcode == Instruction::SExt)
- NumSExts++;
- else if (Opcode == Instruction::ZExt)
- NumZExts++;
- else {
- // If we find that the top bits are known 0, then we can sink and allow
- // the backend to generate a umull.
- unsigned Bitwidth = I->getType()->getScalarSizeInBits();
- APInt UpperMask = APInt::getHighBitsSet(Bitwidth, Bitwidth / 2);
- const DataLayout &DL = I->getDataLayout();
- if (!MaskedValueIsZero(OperandInstr, UpperMask, DL))
- continue;
- NumZExts++;...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/110564
More information about the llvm-commits
mailing list