[llvm] [GVN] Teach GVN simple masked load/store forwarding (PR #157689)

[David Sherwood via llvm-commits]

================
@@ -2287,6 +2288,46 @@ bool GVNPass::processLoad(LoadInst *L) {
   return true;
 }
 
+// Attempt to process masked loads which have loaded from
+// masked stores with the same mask
+bool GVNPass::processMaskedLoad(IntrinsicInst *I) {
+  Value *Mask = I->getOperand(2);
+  Value *Passthrough = I->getOperand(3);
+
+  MemDepResult Dep = MD->getDependency(I);
+  Instruction *DepInst = Dep.getInst();
+  if (!DepInst || !Dep.isLocal())
+    return false;
+
+  Value *StoreVal;
+  if (!match(DepInst,
+             m_Intrinsic<Intrinsic::masked_store>(m_Value(StoreVal), m_Value(),
+                                                  m_Value(), m_Specific(Mask))))
+    return false;
+
+  Value *OpToForward = nullptr;
+  if (match(StoreVal, m_MaskedLoad(m_Value(), m_Value(), m_Specific(Mask),
+                                   m_Specific(Passthrough))))
+    // For MaskedLoad->MaskedStore->MaskedLoad, the mask must be the same for
+    // all three instructions. The Passthrough on the two loads must also be the
+    // same.
+    OpToForward = AvailableValue::get(StoreVal).getSimpleValue();
+  else if (match(StoreVal, m_Intrinsic<Intrinsic::masked_load>()))
+    return false;
+  else {
+    // MaskedStore(Op, ptr, mask)->MaskedLoad(ptr, mask, passthrough) can be
+    // replaced with MaskedStore(Op, ptr, mask)->select(mask, Op, passthrough)
+    IRBuilder<> Builder(I);
+    OpToForward = Builder.CreateSelect(Mask, StoreVal, Passthrough);
+  }
+
+  I->replaceAllUsesWith(OpToForward);
+  ICF->removeUsersOf(I);
----------------
david-arm wrote:

I don't if ordering matters, but I noticed that in processLoads we do this the other way around:

```
  ICF->removeUsersOf(L);
  L->replaceAllUsesWith(AvailableValue);
```

https://github.com/llvm/llvm-project/pull/157689