[llvm] [VPlan] Introduce recipes for VP loads and stores. (PR #87816)
Florian Hahn via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 17 08:12:05 PDT 2024
================
@@ -2372,15 +2391,144 @@ class VPWidenMemoryInstructionRecipe : public VPRecipeBase {
bool onlyFirstLaneUsed(const VPValue *Op) const override {
assert(is_contained(operands(), Op) &&
"Op must be an operand of the recipe");
-
// Widened, consecutive memory operations only demand the first lane of
// their address, unless the same operand is also stored. That latter can
// happen with opaque pointers.
- return Op == getAddr() && isConsecutive() &&
- (!isStore() || Op != getStoredValue());
+ return Op == getAddr() && isConsecutive();
}
+};
- Instruction &getIngredient() const { return Ingredient; }
+/// A recipe for widening load operations with vector-predication intrinsics,
+/// using the address to load from, the explicit vector length and an optional
+/// mask.
+struct VPWidenVPLoadRecipe final : public VPWidenMemoryRecipe, public VPValue {
+ VPWidenVPLoadRecipe(LoadInst &Load, VPValue *Addr, VPValue *EVL,
+ VPValue *Mask, bool IsConsecutive, DebugLoc DL)
+ : VPWidenMemoryRecipe(VPDef::VPWidenVPLoadSC, Load, {Addr, EVL},
+ IsConsecutive, false, DL),
+ VPValue(this, &Load) {
+ setMask(Mask);
+ }
+
+ VPRecipeBase *clone() override {
+ return new VPWidenVPLoadRecipe(cast<LoadInst>(Ingredient), getAddr(),
+ getEVL(), getMask(), isConsecutive(),
+ getDebugLoc());
+ }
+
+ VP_CLASSOF_IMPL(VPDef::VPWidenVPLoadSC)
+
+ /// Return the EVL operand.
+ VPValue *getEVL() const { return getOperand(1); }
+
+ /// Generate the wide load/store.
+ void execute(VPTransformState &State) override;
+
+#if !defined(NDEBUG) || defined(LLVM_ENABLE_DUMP)
+ /// Print the recipe.
+ void print(raw_ostream &O, const Twine &Indent,
+ VPSlotTracker &SlotTracker) const override;
+#endif
+
+ /// Returns true if the recipe only uses the first lane of operand \p Op.
+ bool onlyFirstLaneUsed(const VPValue *Op) const override {
+ assert(is_contained(operands(), Op) &&
+ "Op must be an operand of the recipe");
+ if (Op == getEVL())
+ return true;
+ // Widened, consecutive loads operations only demand the first lane of
+ // their address.
+ return Op == getAddr() && isConsecutive();
----------------
fhahn wrote:
merged, thanks!
https://github.com/llvm/llvm-project/pull/87816
More information about the llvm-commits
mailing list