[llvm-branch-commits] [llvm] [WindowScheduler] Preserve loop-carried memory dependences (#231364) (PR #243336)

Ikhlas Ajbar via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Mon Nov 16 21:09:22 PST 2026


https://github.com/iajbar created https://github.com/llvm/llvm-project/pull/243336

The WindowScheduler clones the loop body into TripleMBB and advances the base registers of memory accesses to later iterations, but the clones keep the original iteration's memory operands. The per-window scheduler could therefore treat a store and a load from different copies as non-aliasing and reorder them, even though they touch the same address in consecutive iterations, producing wrong code.
Detect loop-carried memory dependences on the original loop and enforce them with a DAG mutation while list scheduling each window. 
Backport of llvm/llvm-project at b1db34427810b2d14af8293a8c5493119c5ccb45.
Fixes #209946

>From 276421c870322c5341e8c2f9a9642eac4e2d6ec4 Mon Sep 17 00:00:00 2001
From: Ikhlas Ajbar <iajbar at quicinc.com>
Date: Sat, 24 Oct 2026 16:39:09 -0500
Subject: [PATCH] [WindowScheduler] Preserve loop-carried memory dependences
 (#231364)

The WindowScheduler clones the loop body into TripleMBB and advances the
base registers of memory accesses to later iterations, but the clones
keep the original iteration's memory operands. The per-window scheduler
could therefore treat a store and a load from different copies as
non-aliasing and reorder them, even though they touch the same address
in consecutive iterations, producing wrong code.
Detect loop-carried memory dependences on the original loop and enforce
them with a DAG mutation while list scheduling each window.
Fixes #209946
---
 llvm/include/llvm/CodeGen/MachinePipeliner.h  |  21 +-
 llvm/include/llvm/CodeGen/WindowScheduler.h   |  28 +-
 llvm/lib/CodeGen/MachinePipeliner.cpp         |  74 ++--
 llvm/lib/CodeGen/WindowScheduler.cpp          | 198 ++++++++++-
 .../Hexagon/swp-ws-loop-carried-dep.mir       | 325 ++++++++++++++++++
 5 files changed, 599 insertions(+), 47 deletions(-)
 create mode 100644 llvm/test/CodeGen/Hexagon/swp-ws-loop-carried-dep.mir

diff --git a/llvm/include/llvm/CodeGen/MachinePipeliner.h b/llvm/include/llvm/CodeGen/MachinePipeliner.h
index 6c06ea4b2927ea..70364248f4ea07 100644
--- a/llvm/include/llvm/CodeGen/MachinePipeliner.h
+++ b/llvm/include/llvm/CodeGen/MachinePipeliner.h
@@ -63,6 +63,23 @@ class SMSchedule;
 extern LLVM_ABI cl::opt<bool> SwpEnableCopyToPhi;
 extern LLVM_ABI cl::opt<int> SwpForceIssueWidth;
 
+/// Return true if BaseMI and OtherMI may access overlapping memory in a later
+/// iteration of the loop LoopBB. Returns true conservatively whenever overlap
+/// cannot be ruled out.
+LLVM_ABI bool mayOverlapInLaterIter(const MachineInstr *BaseMI,
+                                    const MachineInstr *OtherMI,
+                                    const MachineBasicBlock *LoopBB,
+                                    const MachineRegisterInfo &MRI,
+                                    const TargetInstrInfo &TII);
+
+/// Software pipelining policy for a loop, which a target can customize by
+/// implementing TargetSubtargetInfo::overridePipelinerPolicy.
+struct MachinePipelinerPolicy {
+  /// Limit the register pressure of the scheduled loop, retrying at a higher
+  /// II when a schedule needs too many registers.
+  bool ShouldLimitRegPressure = false;
+};
+
 /// The main class in the implementation of the target independent
 /// software pipeliner pass.
 class LLVM_ABI MachinePipeliner : public MachineFunctionPass {
@@ -449,9 +466,6 @@ class LLVM_ABI SwingSchedulerDAG : public ScheduleDAGInstrs {
 
   const SwingSchedulerDDG *getDDG() const { return DDG.get(); }
 
-  bool mayOverlapInLaterIter(const MachineInstr *BaseMI,
-                             const MachineInstr *OtherMI) const;
-
 private:
   LoopCarriedEdges addLoopCarriedDependences();
   void updatePhiDependences();
@@ -471,7 +485,6 @@ class LLVM_ABI SwingSchedulerDAG : public ScheduleDAGInstrs {
   void computeNodeOrder(NodeSetType &NodeSets);
   void checkValidNodeOrder(const NodeSetType &Circuits) const;
   bool schedulePipeline(SMSchedule &Schedule);
-  bool computeDelta(const MachineInstr &MI, int &Delta) const;
   MachineInstr *findDefInLoop(Register Reg);
   bool canUseLastOffsetValue(MachineInstr *MI, unsigned &BasePos,
                              unsigned &OffsetPos, Register &NewBase,
diff --git a/llvm/include/llvm/CodeGen/WindowScheduler.h b/llvm/include/llvm/CodeGen/WindowScheduler.h
index 12b3474dfa784a..2a89b8aa0ccb8a 100644
--- a/llvm/include/llvm/CodeGen/WindowScheduler.h
+++ b/llvm/include/llvm/CodeGen/WindowScheduler.h
@@ -42,6 +42,7 @@
 #ifndef LLVM_CODEGEN_WINDOWSCHEDULER_H
 #define LLVM_CODEGEN_WINDOWSCHEDULER_H
 
+#include "llvm/ADT/DenseSet.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachineRegisterInfo.h"
 #include "llvm/CodeGen/MachineScheduler.h"
@@ -56,6 +57,13 @@ enum WindowSchedulingFlag {
   WS_Force /// Use window algorithm instead of SMS algorithm.
 };
 
+/// TriCloneInfo describes an MI clone in TripleMBB: the original MI it was
+/// cloned from and the index of the copy (0, 1, or 2) it belongs to.
+struct TriCloneInfo {
+  MachineInstr *Ori = nullptr;
+  unsigned Copy = 0;
+};
+
 /// The main class in the implementation of the target independent window
 /// scheduler.
 class LLVM_ABI WindowScheduler {
@@ -80,11 +88,20 @@ class LLVM_ABI WindowScheduler {
   SmallVector<MachineInstr *> OriMIs;
   /// TriMIs keeps the MIs of TripleMBB, which is used to restore TripleMBB.
   SmallVector<MachineInstr *> TriMIs;
-  /// TriToOri keeps the mappings between the MI clones in TripleMBB and their
-  /// original MI.
-  DenseMap<MachineInstr *, MachineInstr *> TriToOri;
+  /// TriToOri keeps the mappings between the MI clones in TripleMBB and the
+  /// information about their original MI and copy index.
+  DenseMap<MachineInstr *, TriCloneInfo> TriToOri;
   /// OriToCycle keeps the mappings between the original MI and its issue cycle.
   DenseMap<MachineInstr *, int> OriToCycle;
+  /// LoopCarriedMemDeps keeps ordered pairs (Pred, Succ) of original memory MIs
+  /// such that the iteration-N access Pred must be ordered before the
+  /// iteration-(N+1) access Succ.
+  DenseSet<std::pair<MachineInstr *, MachineInstr *>> LoopCarriedMemDeps;
+  /// MemDepEdgeFailed is set when enforcing a loop-carried memory dependence as
+  /// an artificial DAG edge would have created a cycle, so the edge could not
+  /// be added. In that case the current window must be abandoned, as its
+  /// schedule would not honor the dependence.
+  bool MemDepEdgeFailed = false;
   /// SchedResult keeps the result of each list scheduling, and the format of
   /// the tuple is <MI pointer, Cycle, Stage, Order ID>.
   SmallVector<std::tuple<MachineInstr *, int, int, int>, 256> SchedResult;
@@ -118,6 +135,11 @@ class LLVM_ABI WindowScheduler {
   virtual bool initialize();
   /// Add some related processing before running window scheduling.
   virtual void preProcess();
+  /// Detect loop-carried memory dependences in the original MBB that must be
+  /// preserved across loop iterations. This must run on the live MBB before
+  /// backupMBB(), as it analyzes the induction recurrence of memory accesses.
+  /// The hazardous pairs of original MIs are recorded in LoopCarriedMemDeps.
+  void collectLoopCarriedMemDeps();
   /// Add some related processing after running window scheduling.
   virtual void postProcess();
   /// Back up the MIs in the original MBB and remove them from MBB.
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index bfd4fa0c589afd..43348c944aa806 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -325,12 +325,10 @@ class LoopCarriedOrderDepsTracker {
   std::vector<TaggedSUnit> TaggedSUnits;
 
   const TargetInstrInfo *TII = nullptr;
-  const TargetRegisterInfo *TRI = nullptr;
 
 public:
   LoopCarriedOrderDepsTracker(SwingSchedulerDAG *SSD, BatchAAResults *BAA,
-                              const TargetInstrInfo *TII,
-                              const TargetRegisterInfo *TRI);
+                              const TargetInstrInfo *TII);
 
   /// The main function to compute loop-carried order-dependencies.
   void computeDependencies();
@@ -961,7 +959,7 @@ void SwingSchedulerDAG::finishBlock() {
 
 /// Return the register values for  the operands of a Phi instruction.
 /// This function assume the instruction is a Phi.
-static void getPhiRegs(MachineInstr &Phi, MachineBasicBlock *Loop,
+static void getPhiRegs(const MachineInstr &Phi, const MachineBasicBlock *Loop,
                        Register &InitVal, Register &LoopVal) {
   assert(Phi.isPHI() && "Expecting a Phi.");
 
@@ -1055,9 +1053,7 @@ bool SUnitWithMemInfo::getUnderlyingObjects() {
 static bool hasLoopCarriedMemDep(const SUnitWithMemInfo &Src,
                                  const SUnitWithMemInfo &Dst,
                                  BatchAAResults &BAA,
-                                 const TargetInstrInfo *TII,
-                                 const TargetRegisterInfo *TRI,
-                                 const SwingSchedulerDAG *SSD) {
+                                 const TargetInstrInfo *TII) {
   if (Src.isTriviallyDisjoint(Dst))
     return false;
   if (isSuccOrder(Src.SU, Dst.SU))
@@ -1066,7 +1062,9 @@ static bool hasLoopCarriedMemDep(const SUnitWithMemInfo &Src,
   MachineInstr &SrcMI = *Src.SU->getInstr();
   MachineInstr &DstMI = *Dst.SU->getInstr();
 
-  if (!SSD->mayOverlapInLaterIter(&SrcMI, &DstMI))
+  const MachineBasicBlock *LoopBB = SrcMI.getParent();
+  const MachineRegisterInfo &MRI = LoopBB->getParent()->getRegInfo();
+  if (!mayOverlapInLaterIter(&SrcMI, &DstMI, LoopBB, MRI, *TII))
     return false;
 
   // Second, the more expensive check that uses alias analysis on the
@@ -1107,10 +1105,9 @@ void LoopCarriedOrderDepsTracker::NoBarrierInstsChunk::append(SUnit *SU) {
 }
 
 LoopCarriedOrderDepsTracker::LoopCarriedOrderDepsTracker(
-    SwingSchedulerDAG *SSD, BatchAAResults *BAA, const TargetInstrInfo *TII,
-    const TargetRegisterInfo *TRI)
+    SwingSchedulerDAG *SSD, BatchAAResults *BAA, const TargetInstrInfo *TII)
     : DAG(SSD), BAA(BAA), SUnits(DAG->SUnits), N(SUnits.size()),
-      LoopCarried(N, BitVector(N)), TII(TII), TRI(TRI) {}
+      LoopCarried(N, BitVector(N)), TII(TII) {}
 
 void LoopCarriedOrderDepsTracker::computeDependencies() {
   // Traverse all instructions and extract only what we are targetting.
@@ -1148,7 +1145,7 @@ void LoopCarriedOrderDepsTracker::addDependenciesBetweenSUs(
   if (Src.SU == Dst.SU)
     return;
 
-  if (hasLoopCarriedMemDep(Src, Dst, *BAA, TII, TRI, DAG))
+  if (hasLoopCarriedMemDep(Src, Dst, *BAA, TII))
     setLoopCarriedDep(Src.SU, Dst.SU);
 }
 
@@ -1257,7 +1254,7 @@ LoopCarriedEdges SwingSchedulerDAG::addLoopCarriedDependences() {
   LoopCarriedEdges LCE;
 
   // Add loop-carried order-dependencies
-  LoopCarriedOrderDepsTracker LCODTracker(this, &BAA, TII, TRI);
+  LoopCarriedOrderDepsTracker LCODTracker(this, &BAA, TII);
   LCODTracker.computeDependencies();
   for (unsigned I = 0; I != SUnits.size(); I++)
     for (const int Succ : LCODTracker.getLoopCarried(I).set_bits())
@@ -2953,7 +2950,7 @@ static bool findLoopIncrementValue(const MachineOperand &Op, int &Value) {
     if (!CurReg.isValid() || !CurReg.isVirtual())
       return false;
     MachineInstr *Def = MRI.getVRegDef(CurReg);
-    if (Def->getParent() != LoopBB)
+    if (!Def || Def->getParent() != LoopBB)
       return false;
 
     if (Def->isCopy()) {
@@ -3003,12 +3000,13 @@ static bool findLoopIncrementValue(const MachineOperand &Op, int &Value) {
 
 /// Return true if we can compute the amount the instruction changes
 /// during each iteration. Set Delta to the amount of the change.
-bool SwingSchedulerDAG::computeDelta(const MachineInstr &MI, int &Delta) const {
-  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
+static bool computeDelta(const MachineInstr &MI, const TargetInstrInfo &TII,
+                         int &Delta) {
+  const TargetRegisterInfo *TRI = MI.getMF()->getSubtarget().getRegisterInfo();
   const MachineOperand *BaseOp;
   int64_t Offset;
   bool OffsetIsScalable;
-  if (!TII->getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable, TRI))
+  if (!TII.getMemOperandWithOffset(MI, BaseOp, Offset, OffsetIsScalable, TRI))
     return false;
 
   // FIXME: This algorithm assumes instructions have fixed-size offsets.
@@ -3138,22 +3136,26 @@ MachineInstr *SwingSchedulerDAG::findDefInLoop(Register Reg) {
 
 /// Return false if there is no overlap between the region accessed by BaseMI in
 /// an iteration and the region accessed by OtherMI in subsequent iterations.
-bool SwingSchedulerDAG::mayOverlapInLaterIter(
-    const MachineInstr *BaseMI, const MachineInstr *OtherMI) const {
+bool llvm::mayOverlapInLaterIter(const MachineInstr *BaseMI,
+                                 const MachineInstr *OtherMI,
+                                 const MachineBasicBlock *LoopBB,
+                                 const MachineRegisterInfo &MRI,
+                                 const TargetInstrInfo &TII) {
   int DeltaB, DeltaO, Delta;
-  if (!computeDelta(*BaseMI, DeltaB) || !computeDelta(*OtherMI, DeltaO) ||
-      DeltaB != DeltaO)
+  if (!computeDelta(*BaseMI, TII, DeltaB) ||
+      !computeDelta(*OtherMI, TII, DeltaO) || DeltaB != DeltaO)
     return true;
   Delta = DeltaB;
 
   const MachineOperand *BaseOpB, *BaseOpO;
   int64_t OffsetB, OffsetO;
   bool OffsetBIsScalable, OffsetOIsScalable;
-  const TargetRegisterInfo *TRI = MF.getSubtarget().getRegisterInfo();
-  if (!TII->getMemOperandWithOffset(*BaseMI, BaseOpB, OffsetB,
-                                    OffsetBIsScalable, TRI) ||
-      !TII->getMemOperandWithOffset(*OtherMI, BaseOpO, OffsetO,
-                                    OffsetOIsScalable, TRI))
+  const TargetRegisterInfo *TRI =
+      BaseMI->getMF()->getSubtarget().getRegisterInfo();
+  if (!TII.getMemOperandWithOffset(*BaseMI, BaseOpB, OffsetB, OffsetBIsScalable,
+                                   TRI) ||
+      !TII.getMemOperandWithOffset(*OtherMI, BaseOpO, OffsetO,
+                                   OffsetOIsScalable, TRI))
     return true;
 
   if (OffsetBIsScalable || OffsetOIsScalable)
@@ -3169,24 +3171,24 @@ bool SwingSchedulerDAG::mayOverlapInLaterIter(
     if (!RegB.isVirtual() || !RegO.isVirtual())
       return true;
 
-    MachineInstr *DefB = MRI.getVRegDef(BaseOpB->getReg());
-    MachineInstr *DefO = MRI.getVRegDef(BaseOpO->getReg());
+    MachineInstr *DefB = MRI.getVRegDef(RegB);
+    MachineInstr *DefO = MRI.getVRegDef(RegO);
     if (!DefB || !DefO || !DefB->isPHI() || !DefO->isPHI())
       return true;
 
-    Register InitValB;
-    Register LoopValB;
-    Register InitValO;
-    Register LoopValO;
-    getPhiRegs(*DefB, BB, InitValB, LoopValB);
-    getPhiRegs(*DefO, BB, InitValO, LoopValO);
+    Register InitValB, LoopValB, InitValO, LoopValO;
+    getPhiRegs(*DefB, LoopBB, InitValB, LoopValB);
+    getPhiRegs(*DefO, LoopBB, InitValO, LoopValO);
+    if (!InitValB.isValid() || !InitValO.isValid())
+      return true;
     MachineInstr *InitDefB = MRI.getVRegDef(InitValB);
     MachineInstr *InitDefO = MRI.getVRegDef(InitValO);
-
-    if (!InitDefB->isIdenticalTo(*InitDefO))
+    if (!InitDefB || !InitDefO || !InitDefB->isIdenticalTo(*InitDefO))
       return true;
   }
 
+  if (BaseMI->memoperands_empty() || OtherMI->memoperands_empty())
+    return true;
   LocationSize AccessSizeB = (*BaseMI->memoperands_begin())->getSize();
   LocationSize AccessSizeO = (*OtherMI->memoperands_begin())->getSize();
 
diff --git a/llvm/lib/CodeGen/WindowScheduler.cpp b/llvm/lib/CodeGen/WindowScheduler.cpp
index 3849a2c2caddf1..cb4f4dfa9295b4 100644
--- a/llvm/lib/CodeGen/WindowScheduler.cpp
+++ b/llvm/lib/CodeGen/WindowScheduler.cpp
@@ -37,10 +37,14 @@
 //===----------------------------------------------------------------------===//
 #include "llvm/CodeGen/WindowScheduler.h"
 #include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AliasAnalysis.h"
+#include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/ValueTracking.h"
 #include "llvm/CodeGen/LiveIntervals.h"
 #include "llvm/CodeGen/MachineLoopInfo.h"
 #include "llvm/CodeGen/MachinePipeliner.h"
 #include "llvm/CodeGen/ModuloSchedule.h"
+#include "llvm/CodeGen/ScheduleDAGMutation.h"
 #include "llvm/CodeGen/TargetPassConfig.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Debug.h"
@@ -101,6 +105,136 @@ static cl::opt<unsigned>
                   cl::desc("The upper limit of II in the window algorithm."),
                   cl::Hidden, cl::init(1000));
 
+/// Collect the underlying objects and alias metadata for the single memory
+/// operand of MI. Returns false when MI does not have exactly one memory
+/// operand with a known IR value.
+static bool getMemInfo(const MachineInstr *MI, const Value *&MemOpValue,
+                       int64_t &MemOpOffset,
+                       SmallVectorImpl<const Value *> &UnderlyingObjs,
+                       AAMDNodes &AATags, bool &IsAllIdentified) {
+  if (!MI->hasOneMemOperand())
+    return false;
+  MachineMemOperand *MM = *MI->memoperands_begin();
+  if (!MM->getValue())
+    return false;
+  MemOpValue = MM->getValue();
+  MemOpOffset = MM->getOffset();
+  getUnderlyingObjects(MemOpValue, UnderlyingObjs);
+  AATags = MM->getAAInfo();
+  IsAllIdentified = llvm::all_of(UnderlyingObjs, isIdentifiedObject);
+  return true;
+}
+
+/// Return true if there is a loop-carried order dependency from memory access
+/// SrcMI to DstMI in loop LoopBB.
+static bool hasLoopCarriedMemDep(const MachineInstr *SrcMI,
+                                 const MachineInstr *DstMI, BatchAAResults &BAA,
+                                 const MachineBasicBlock *LoopBB,
+                                 const MachineRegisterInfo &MRI,
+                                 const TargetInstrInfo *TII) {
+  const Value *SrcVal = nullptr, *DstVal = nullptr;
+  int64_t SrcOffset = 0, DstOffset = 0;
+  SmallVector<const Value *, 4> SrcObjs, DstObjs;
+  AAMDNodes SrcAATags, DstAATags;
+  bool SrcAllIdentified = false, DstAllIdentified = false;
+  bool SrcKnown = getMemInfo(SrcMI, SrcVal, SrcOffset, SrcObjs, SrcAATags,
+                             SrcAllIdentified);
+  bool DstKnown = getMemInfo(DstMI, DstVal, DstOffset, DstObjs, DstAATags,
+                             DstAllIdentified);
+
+  // If all underlying objects are identified and disjoint, the accesses cannot
+  // alias.
+  if (SrcKnown && DstKnown && SrcAllIdentified && DstAllIdentified &&
+      llvm::none_of(SrcObjs, [&](const Value *Obj) {
+        return llvm::is_contained(DstObjs, Obj);
+      }))
+    return false;
+
+  // The overlap helper inspects only the first memory operand. Multi-MMO
+  // instructions could carry an overlap through a later operand, so be
+  // conservative and keep the dependency for them.
+  if (!SrcMI->hasOneMemOperand() || !DstMI->hasOneMemOperand())
+    return true;
+
+  if (!mayOverlapInLaterIter(SrcMI, DstMI, LoopBB, MRI, *TII))
+    return false;
+
+  // The more expensive check that uses alias analysis on the base values.
+  if (!SrcKnown || !DstKnown)
+    return true;
+  if (SrcVal == DstVal && SrcOffset <= DstOffset)
+    return true;
+
+  if (BAA.isNoAlias(MemoryLocation::getBeforeOrAfter(SrcVal, SrcAATags),
+                    MemoryLocation::getBeforeOrAfter(DstVal, DstAATags)))
+    return false;
+
+  // AliasAnalysis sometimes gives up on following the underlying object. In
+  // such a case, separate checks for underlying objects may prove that there
+  // are no aliases between two accesses.
+  for (const Value *SrcObj : SrcObjs)
+    for (const Value *DstObj : DstObjs)
+      if (!BAA.isNoAlias(MemoryLocation::getBeforeOrAfter(SrcObj, SrcAATags),
+                         MemoryLocation::getBeforeOrAfter(DstObj, DstAATags)))
+        return true;
+
+  return false;
+}
+
+namespace {
+/// A DAG mutation that enforces the loop-carried memory dependences discovered
+/// on the original loop while list scheduling each window of TripleMBB.
+///
+/// For each recorded hazard pair (Pred, Succ) we add an artificial order edge
+/// between the clone of Pred and the clone of Succ belonging to a later copy,
+/// so that the iteration-N Pred is kept before the iteration-(N+1) Succ.
+class WindowMemDepMutation : public ScheduleDAGMutation {
+  const DenseSet<std::pair<MachineInstr *, MachineInstr *>> &LoopCarriedMemDeps;
+  const DenseMap<MachineInstr *, TriCloneInfo> &TriToOri;
+  bool &EdgeFailed;
+
+public:
+  WindowMemDepMutation(
+      const DenseSet<std::pair<MachineInstr *, MachineInstr *>> &Deps,
+      const DenseMap<MachineInstr *, TriCloneInfo> &TriToOri, bool &EdgeFailed)
+      : LoopCarriedMemDeps(Deps), TriToOri(TriToOri), EdgeFailed(EdgeFailed) {}
+
+  void apply(ScheduleDAGInstrs *DAG) override {
+    if (LoopCarriedMemDeps.empty())
+      return;
+    for (SUnit &PredSU : DAG->SUnits) {
+      MachineInstr *PredMI = PredSU.getInstr();
+      if (!PredMI || !PredMI->mayLoadOrStore())
+        continue;
+      auto PredIt = TriToOri.find(PredMI);
+      if (PredIt == TriToOri.end())
+        continue;
+      for (SUnit &SuccSU : DAG->SUnits) {
+        MachineInstr *SuccMI = SuccSU.getInstr();
+        if (&PredSU == &SuccSU || !SuccMI || !SuccMI->mayLoadOrStore())
+          continue;
+        auto SuccIt = TriToOri.find(SuccMI);
+        if (SuccIt == TriToOri.end())
+          continue;
+        // Only order a Pred from an earlier copy before a Succ from a later
+        // copy, which models the iteration-N to iteration-(N+1) dependence.
+        if (PredIt->second.Copy >= SuccIt->second.Copy)
+          continue;
+        if (!LoopCarriedMemDeps.contains(
+                {PredIt->second.Ori, SuccIt->second.Ori}))
+          continue;
+        // If the edge would create a cycle, addEdge() returns false and does
+        // not insert it. The required dependence then cannot be enforced in
+        // this window, so mark it to make the caller abandon this window's
+        // schedule.
+        if (!DAG->addEdge(&SuccSU, SDep(&PredSU, SDep::Barrier)))
+          EdgeFailed = true;
+      }
+    }
+  }
+};
+} // end anonymous namespace
+
 WindowScheduler::WindowScheduler(MachineSchedContext *C, MachineLoop &ML)
     : Context(C), MF(C->MF), MBB(ML.getHeader()), Loop(ML),
       Subtarget(&MF->getSubtarget()), TII(Subtarget->getInstrInfo()),
@@ -122,6 +256,20 @@ bool WindowScheduler::run() {
   preProcess();
   // The main window scheduling begins.
   std::unique_ptr<ScheduleDAGInstrs> SchedDAG(createMachineScheduler());
+  // A target may decline to provide a scheduler, in which case fall back to the
+  // generic live scheduler so that window scheduling still has one to drive.
+  if (!SchedDAG)
+    SchedDAG.reset(createSchedLive(Context));
+  // Attach the mutation that enforces the loop-carried memory dependences.
+  // This requires the scheduler to support DAG mutations, i.e. to be a
+  // ScheduleDAGMI. Every in-tree target's pre-RA machine scheduler is a
+  // ScheduleDAGMI (or ScheduleDAGMILive), and the generic fallback
+  // createSchedLive() above returns one as well, so the cast below is safe.
+  // The assert documents and guards this invariant.
+  assert(SchedDAG && "Expected a scheduler for window scheduling");
+  auto *SchedDAGMI = static_cast<ScheduleDAGMI *>(SchedDAG.get());
+  SchedDAGMI->addMutation(std::make_unique<WindowMemDepMutation>(
+      LoopCarriedMemDeps, TriToOri, MemDepEdgeFailed));
   auto SearchIndexes = getSearchIndexes(WindowSearchNum, WindowSearchRatio);
   for (unsigned Idx : SearchIndexes) {
     OriToCycle.clear();
@@ -130,10 +278,17 @@ bool WindowScheduler::run() {
     // be added to Idx.
     unsigned Offset = Idx + SchedPhiNum;
     auto Range = getScheduleRange(Offset, SchedInstrNum);
+    MemDepEdgeFailed = false;
     SchedDAG->startBlock(MBB);
     SchedDAG->enterRegion(MBB, Range.begin(), Range.end(), SchedInstrNum);
     SchedDAG->schedule();
     LLVM_DEBUG(SchedDAG->dump());
+    if (MemDepEdgeFailed) {
+      restoreTripleMBB();
+      LLVM_DEBUG(dbgs() << "Can't enforce a loop-carried memory dependence. "
+                           "Keep searching...\n");
+      continue;
+    }
     unsigned II = analyseII(*SchedDAG, Offset);
     if (II == WindowIILimit) {
       restoreTripleMBB();
@@ -249,6 +404,9 @@ bool WindowScheduler::initialize() {
 void WindowScheduler::preProcess() {
   // Prior to window scheduling, it's necessary to backup the original MBB,
   // generate a new TripleMBB, and build a TripleDAG based on the TripleMBB.
+  // Collect loop-carried memory dependences first, while the original MBB is
+  // still live (see collectLoopCarriedMemDeps() and run()).
+  collectLoopCarriedMemDeps();
   backupMBB();
   generateTripleMBB();
   TripleDAG->startBlock(MBB);
@@ -258,6 +416,38 @@ void WindowScheduler::preProcess() {
   TripleDAG->buildSchedGraph(Context->AA);
 }
 
+void WindowScheduler::collectLoopCarriedMemDeps() {
+  LoopCarriedMemDeps.clear();
+  MachineBasicBlock *LoopBB = MBB;
+
+  // Collect the memory instructions of the original loop in program order.
+  SmallVector<MachineInstr *> MemMIs;
+  for (MachineInstr &MI : *LoopBB)
+    if (MI.mayLoadOrStore() && !MI.isDebugInstr())
+      MemMIs.push_back(&MI);
+
+  // Nothing to do unless there is at least one store to order against.
+  if (llvm::none_of(MemMIs,
+                    [](const MachineInstr *MI) { return MI->mayStore(); }))
+    return;
+
+  BatchAAResults BAA(*Context->AA);
+  BAA.enableCrossIterationMode();
+
+  for (MachineInstr *SrcMI : MemMIs) {
+    for (MachineInstr *DstMI : MemMIs) {
+      if (SrcMI == DstMI)
+        continue;
+      // A loop-carried order edge is only required when at least one of the
+      // accesses stores.
+      if (!SrcMI->mayStore() && !DstMI->mayStore())
+        continue;
+      if (hasLoopCarriedMemDep(SrcMI, DstMI, BAA, LoopBB, *MRI, TII))
+        LoopCarriedMemDeps.insert({SrcMI, DstMI});
+    }
+  }
+}
+
 void WindowScheduler::postProcess() {
   // After window scheduling, it's necessary to clear the TripleDAG and restore
   // to the original MBB.
@@ -305,7 +495,7 @@ void WindowScheduler::generateTripleMBB() {
     auto *NewMI = MF->CloneMachineInstr(MI);
     MBB->push_back(NewMI);
     TriMIs.push_back(NewMI);
-    TriToOri[NewMI] = MI;
+    TriToOri[NewMI] = {MI, 0};
   }
   // Step 2: Performing the remaining two copies of MBB instructions excluding
   // phis, and the last one contains terminators. At the same time, registers
@@ -365,7 +555,7 @@ void WindowScheduler::generateTripleMBB() {
         DefPairs[NewDef.first] = NewDef.second;
       MBB->push_back(NewMI);
       TriMIs.push_back(NewMI);
-      TriToOri[NewMI] = MI;
+      TriToOri[NewMI] = {MI, static_cast<unsigned>(Cnt)};
     }
   }
   // Step 3: The registers used by phis are updated, and they are generated in
@@ -668,14 +858,14 @@ WindowScheduler::getScheduleRange(unsigned Offset, unsigned Num) {
 
 int WindowScheduler::getOriCycle(MachineInstr *NewMI) {
   assert(TriToOri.count(NewMI) && "Cannot find original MI!");
-  auto *OriMI = TriToOri[NewMI];
+  auto *OriMI = TriToOri[NewMI].Ori;
   assert(OriToCycle.count(OriMI) && "Cannot find schedule cycle!");
   return OriToCycle[OriMI];
 }
 
 MachineInstr *WindowScheduler::getOriMI(MachineInstr *NewMI) {
   assert(TriToOri.count(NewMI) && "Cannot find original MI!");
-  return TriToOri[NewMI];
+  return TriToOri[NewMI].Ori;
 }
 
 unsigned WindowScheduler::getOriStage(MachineInstr *OriMI, unsigned Offset) {
diff --git a/llvm/test/CodeGen/Hexagon/swp-ws-loop-carried-dep.mir b/llvm/test/CodeGen/Hexagon/swp-ws-loop-carried-dep.mir
new file mode 100644
index 00000000000000..1d389465d31a5e
--- /dev/null
+++ b/llvm/test/CodeGen/Hexagon/swp-ws-loop-carried-dep.mir
@@ -0,0 +1,325 @@
+# REQUIRES: asserts
+# RUN: llc --mtriple=hexagon -mcpu=hexagonv68 %s -run-pass=pipeliner \
+# RUN: -debug-only=pipeliner -window-sched=force -filetype=null \
+# RUN: -verify-machineinstrs 2>&1 | FileCheck %s
+
+# This is the SHA-512 message-schedule loop from llvm.org/PR209946. Each
+# iteration advances the base pointer %1 by 32 bytes and stores to offsets
+# +48/+56/+64/+72. The store to +72 of iteration N writes the same address
+# that a load from +40 of iteration N+1 reads (72 == 32 + 40), a loop-carried
+# store->load dependence. If the WindowScheduler does not honor it, the
+# aliasing load is scheduled ahead of the store and the loop computes wrong
+# results. With the dependence modeled, window scheduling finds no profitable
+# reordering here, so it declines (checked below).
+
+# CHECK: Window scheduling is not needed!
+
+--- |
+  target triple = "hexagon"
+
+  ; Function Attrs: nofree noinline norecurse nosync nounwind memory(argmem: readwrite)
+  define dso_local void @schedule(ptr noundef captures(none) %W) local_unnamed_addr #0 {
+  entry:
+    %cgep = getelementptr i8, ptr %W, i32 80
+    br label %for.body
+
+  for.cond.cleanup:                                 ; preds = %for.body
+    ret void
+
+  for.body:                                         ; preds = %for.body, %entry
+    %lsr.iv1 = phi ptr [ %cgep21, %for.body ], [ %cgep, %entry ]
+    %lsr.iv = phi i32 [ %lsr.iv.next, %for.body ], [ 64, %entry ]
+    %cgep20 = getelementptr i8, ptr %lsr.iv1, i32 -72
+    %0 = load i64, ptr %cgep20, align 8, !tbaa !9
+    %or = tail call i64 @llvm.fshl.i64(i64 %0, i64 %0, i64 63)
+    %or9 = tail call i64 @llvm.fshl.i64(i64 %0, i64 %0, i64 56)
+    %xor = xor i64 %or, %or9
+    %shr12 = lshr i64 %0, 7
+    %xor13 = xor i64 %xor, %shr12
+    %cgep21 = getelementptr i8, ptr %lsr.iv1, i32 32
+    %1 = load i64, ptr %cgep21, align 8, !tbaa !9
+    %or20 = tail call i64 @llvm.fshl.i64(i64 %1, i64 %1, i64 45)
+    %or27 = tail call i64 @llvm.fshl.i64(i64 %1, i64 %1, i64 3)
+    %xor28 = xor i64 %or20, %or27
+    %shr31 = lshr i64 %1, 6
+    %xor32 = xor i64 %xor28, %shr31
+    %cgep22 = getelementptr i8, ptr %lsr.iv1, i32 -80
+    %2 = load i64, ptr %cgep22, align 8, !tbaa !9
+    %add = add i64 %xor13, %2
+    %cgep23 = getelementptr i8, ptr %lsr.iv1, i32 -8
+    %3 = load i64, ptr %cgep23, align 8, !tbaa !9
+    %add37 = add i64 %add, %3
+    %add38 = add i64 %add37, %xor32
+    %cgep19 = getelementptr i8, ptr %lsr.iv1, i32 48
+    store i64 %add38, ptr %cgep19, align 8, !tbaa !9
+    %cgep25 = getelementptr i8, ptr %lsr.iv1, i32 -64
+    %4 = load i64, ptr %cgep25, align 8, !tbaa !9
+    %or.1 = tail call i64 @llvm.fshl.i64(i64 %4, i64 %4, i64 63)
+    %or9.1 = tail call i64 @llvm.fshl.i64(i64 %4, i64 %4, i64 56)
+    %xor.1 = xor i64 %or.1, %or9.1
+    %shr12.1 = lshr i64 %4, 7
+    %xor13.1 = xor i64 %xor.1, %shr12.1
+    %cgep26 = getelementptr i8, ptr %lsr.iv1, i32 40
+    %5 = load i64, ptr %cgep26, align 8, !tbaa !9
+    %or20.1 = tail call i64 @llvm.fshl.i64(i64 %5, i64 %5, i64 45)
+    %or27.1 = tail call i64 @llvm.fshl.i64(i64 %5, i64 %5, i64 3)
+    %xor28.1 = xor i64 %or20.1, %or27.1
+    %shr31.1 = lshr i64 %5, 6
+    %xor32.1 = xor i64 %xor28.1, %shr31.1
+    %add.1 = add i64 %xor13.1, %0
+    %6 = load i64, ptr %lsr.iv1, align 8, !tbaa !9
+    %add37.1 = add i64 %add.1, %6
+    %add38.1 = add i64 %add37.1, %xor32.1
+    %cgep24 = getelementptr i8, ptr %lsr.iv1, i32 56
+    store i64 %add38.1, ptr %cgep24, align 8, !tbaa !9
+    %cgep28 = getelementptr i8, ptr %lsr.iv1, i32 -56
+    %7 = load i64, ptr %cgep28, align 8, !tbaa !9
+    %or.2 = tail call i64 @llvm.fshl.i64(i64 %7, i64 %7, i64 63)
+    %or9.2 = tail call i64 @llvm.fshl.i64(i64 %7, i64 %7, i64 56)
+    %xor.2 = xor i64 %or.2, %or9.2
+    %shr12.2 = lshr i64 %7, 7
+    %xor13.2 = xor i64 %xor.2, %shr12.2
+    %8 = load i64, ptr %cgep19, align 8, !tbaa !9
+    %or20.2 = tail call i64 @llvm.fshl.i64(i64 %8, i64 %8, i64 45)
+    %or27.2 = tail call i64 @llvm.fshl.i64(i64 %8, i64 %8, i64 3)
+    %xor28.2 = xor i64 %or20.2, %or27.2
+    %shr31.2 = lshr i64 %8, 6
+    %xor32.2 = xor i64 %xor28.2, %shr31.2
+    %add.2 = add i64 %xor13.2, %4
+    %cgep29 = getelementptr i8, ptr %lsr.iv1, i32 8
+    %9 = load i64, ptr %cgep29, align 8, !tbaa !9
+    %add37.2 = add i64 %add.2, %9
+    %add38.2 = add i64 %add37.2, %xor32.2
+    %cgep27 = getelementptr i8, ptr %lsr.iv1, i32 64
+    store i64 %add38.2, ptr %cgep27, align 8, !tbaa !9
+    %cgep31 = getelementptr i8, ptr %lsr.iv1, i32 -48
+    %10 = load i64, ptr %cgep31, align 8, !tbaa !9
+    %or.3 = tail call i64 @llvm.fshl.i64(i64 %10, i64 %10, i64 63)
+    %or9.3 = tail call i64 @llvm.fshl.i64(i64 %10, i64 %10, i64 56)
+    %xor.3 = xor i64 %or.3, %or9.3
+    %shr12.3 = lshr i64 %10, 7
+    %xor13.3 = xor i64 %xor.3, %shr12.3
+    %11 = load i64, ptr %cgep24, align 8, !tbaa !9
+    %or20.3 = tail call i64 @llvm.fshl.i64(i64 %11, i64 %11, i64 45)
+    %or27.3 = tail call i64 @llvm.fshl.i64(i64 %11, i64 %11, i64 3)
+    %xor28.3 = xor i64 %or20.3, %or27.3
+    %shr31.3 = lshr i64 %11, 6
+    %xor32.3 = xor i64 %xor28.3, %shr31.3
+    %add.3 = add i64 %xor13.3, %7
+    %cgep32 = getelementptr i8, ptr %lsr.iv1, i32 16
+    %12 = load i64, ptr %cgep32, align 8, !tbaa !9
+    %add37.3 = add i64 %add.3, %12
+    %add38.3 = add i64 %add37.3, %xor32.3
+    %cgep30 = getelementptr i8, ptr %lsr.iv1, i32 72
+    store i64 %add38.3, ptr %cgep30, align 8, !tbaa !9
+    %lsr.iv.next = add nsw i32 %lsr.iv, -4
+    %exitcond.not.3 = icmp eq i32 %lsr.iv.next, 0
+    br i1 %exitcond.not.3, label %for.cond.cleanup, label %for.body, !llvm.loop !11
+  }
+
+  ; Function Attrs: nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none)
+  declare i64 @llvm.fshl.i64(i64, i64, i64) #1
+
+  attributes #0 = { nofree noinline norecurse nosync nounwind memory(argmem: readwrite) "frame-pointer"="all" "no-trapping-math"="true" "stack-protector-buffer-size"="8" "target-cpu"="hexagonv68" "target-features"="+v68,-long-calls" }
+  attributes #1 = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) "target-cpu"="hexagonv68" }
+
+  !llvm.module.flags = !{!0, !1, !2, !3}
+  !llvm.ident = !{!4}
+  !llvm.errno.tbaa = !{!5}
+
+  !0 = !{i32 1, !"wchar_size", i32 4}
+  !1 = !{i32 8, !"PIC Level", i32 2}
+  !2 = !{i32 7, !"PIE Level", i32 2}
+  !3 = !{i32 7, !"frame-pointer", i32 2}
+  !4 = !{!"clang version 22.1.8"}
+  !5 = !{!6, !6, i64 0}
+  !6 = !{!"int", !7, i64 0}
+  !7 = !{!"omnipotent char", !8, i64 0}
+  !8 = !{!"Simple C/C++ TBAA"}
+  !9 = !{!10, !10, i64 0}
+  !10 = !{!"long long", !7, i64 0}
+  !11 = distinct !{!11, !12}
+  !12 = !{!"llvm.loop.mustprogress"}
+...
+---
+name:            schedule
+alignment:       4
+exposesReturnsTwice: false
+legalized:       false
+regBankSelected: false
+selected:        false
+failedISel:      false
+tracksRegLiveness: true
+hasWinCFI:       false
+noPhis:          false
+isSSA:           true
+noVRegs:         false
+hasFakeUses:     false
+callsEHReturn:   false
+callsUnwindInit: false
+hasEHContTarget: false
+hasEHScopes:     false
+hasEHFunclets:   false
+isOutlined:      false
+debugInstrRef:   false
+failsVerification: false
+tracksDebugUserValues: false
+registers:
+  - { id: 0, class: intregs, preferred-register: '', flags: [  ] }
+  - { id: 1, class: intregs, preferred-register: '', flags: [  ] }
+  - { id: 2, class: intregs, preferred-register: '', flags: [  ] }
+  - { id: 3, class: intregs, preferred-register: '', flags: [  ] }
+  - { id: 4, class: intregs, preferred-register: '', flags: [  ] }
+  - { id: 5, class: intregs, preferred-register: '', flags: [  ] }
+  - { id: 6, class: intregs, preferred-register: '', flags: [  ] }
+  - { id: 7, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 8, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 9, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 10, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 11, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 12, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 13, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 14, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 15, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 16, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 17, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 18, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 19, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 20, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 21, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 22, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 23, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 24, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 25, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 26, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 27, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 28, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 29, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 30, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 31, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 32, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 33, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 34, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 35, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 36, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 37, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 38, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 39, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 40, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 41, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 42, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 43, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 44, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 45, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 46, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 47, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 48, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 49, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 50, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 51, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 52, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 53, class: doubleregs, preferred-register: '', flags: [  ] }
+  - { id: 54, class: predregs, preferred-register: '', flags: [  ] }
+liveins:
+  - { reg: '$r0', virtual-reg: '%5' }
+frameInfo:
+  isFrameAddressTaken: false
+  isReturnAddressTaken: false
+  hasStackMap:     false
+  hasPatchPoint:   false
+  stackSize:       0
+  offsetAdjustment: 0
+  maxAlignment:    1
+  adjustsStack:    false
+  hasCalls:        false
+  framePointerPolicy: all
+  stackProtector:  ''
+  functionContext: ''
+  maxCallFrameSize: 4294967295
+  cvBytesOfCalleeSavedRegisters: 0
+  hasOpaqueSPAdjustment: false
+  hasVAStart:      false
+  hasMustTailInVarArgFunc: false
+  hasTailCall:     false
+  isCalleeSavedInfoValid: false
+  localFrameSize:  0
+fixedStack:      []
+stack:           []
+entry_values:    []
+callSites:       []
+debugValueSubstitutions: []
+constants:       []
+machineFunctionInfo:
+  stackAlignBaseReg: ''
+body:             |
+  bb.0.entry:
+    successors: %bb.2(0x80000000)
+    liveins: $r0
+
+    %5:intregs = COPY $r0
+    %0:intregs = A2_addi %5, 80
+    J2_loop0i %bb.2, 16, implicit-def $lc0, implicit-def $sa0, implicit-def $usr
+    J2_jump %bb.2, implicit-def dead $pc
+
+  bb.1.for.cond.cleanup:
+    PS_jmpret $r31, implicit-def dead $pc
+
+  bb.2.for.body (machine-block-address-taken):
+    successors: %bb.1(0x04000000), %bb.2(0x7c000000)
+
+    %1:intregs = PHI %0, %bb.0, %3, %bb.2
+    %7:doubleregs = L2_loadrd_io %1, -72 :: (load (s64) from %ir.cgep20, !tbaa !9)
+    %8:doubleregs = S6_rol_i_p %7, 63
+    %9:doubleregs = S6_rol_i_p_xacc %8, %7, 56
+    %10:doubleregs = S2_lsr_i_p_xacc %9, %7, 7
+    %3:intregs = A2_addi %1, 32
+    %11:doubleregs = L2_loadrd_io %1, 32 :: (load (s64) from %ir.cgep21, !tbaa !9)
+    %12:doubleregs = S6_rol_i_p %11, 45
+    %13:doubleregs = S6_rol_i_p_xacc %12, %11, 3
+    %14:doubleregs = S2_lsr_i_p_xacc %13, %11, 6
+    %15:doubleregs = L2_loadrd_io %1, -80 :: (load (s64) from %ir.cgep22, !tbaa !9)
+    %16:doubleregs = A2_addp killed %10, killed %15
+    %17:doubleregs = L2_loadrd_io %1, -8 :: (load (s64) from %ir.cgep23, !tbaa !9)
+    %18:doubleregs = A2_addp killed %16, killed %17
+    %19:doubleregs = A2_addp killed %18, killed %14
+    S2_storerd_io %1, 48, %19 :: (store (s64) into %ir.cgep19, !tbaa !9)
+    %20:doubleregs = L2_loadrd_io %1, -64 :: (load (s64) from %ir.cgep25, !tbaa !9)
+    %21:doubleregs = S6_rol_i_p %20, 63
+    %22:doubleregs = S6_rol_i_p_xacc %21, %20, 56
+    %23:doubleregs = S2_lsr_i_p_xacc %22, %20, 7
+    %24:doubleregs = L2_loadrd_io %1, 40 :: (load (s64) from %ir.cgep26, !tbaa !9)
+    %25:doubleregs = S6_rol_i_p %24, 45
+    %26:doubleregs = S6_rol_i_p_xacc %25, %24, 3
+    %27:doubleregs = S2_lsr_i_p_xacc %26, %24, 6
+    %28:doubleregs = A2_addp killed %23, %7
+    %29:doubleregs = L2_loadrd_io %1, 0 :: (load (s64) from %ir.lsr.iv1, !tbaa !9)
+    %30:doubleregs = A2_addp killed %28, killed %29
+    %31:doubleregs = A2_addp killed %30, killed %27
+    S2_storerd_io %1, 56, %31 :: (store (s64) into %ir.cgep24, !tbaa !9)
+    %32:doubleregs = L2_loadrd_io %1, -56 :: (load (s64) from %ir.cgep28, !tbaa !9)
+    %33:doubleregs = S6_rol_i_p %32, 63
+    %34:doubleregs = S6_rol_i_p_xacc %33, %32, 56
+    %35:doubleregs = S2_lsr_i_p_xacc %34, %32, 7
+    %36:doubleregs = S6_rol_i_p %19, 45
+    %37:doubleregs = S6_rol_i_p_xacc %36, %19, 3
+    %38:doubleregs = S2_lsr_i_p_xacc %37, %19, 6
+    %39:doubleregs = A2_addp killed %35, %20
+    %40:doubleregs = L2_loadrd_io %1, 8 :: (load (s64) from %ir.cgep29, !tbaa !9)
+    %41:doubleregs = A2_addp killed %39, killed %40
+    %42:doubleregs = A2_addp killed %41, killed %38
+    S2_storerd_io %1, 64, killed %42 :: (store (s64) into %ir.cgep27, !tbaa !9)
+    %43:doubleregs = L2_loadrd_io %1, -48 :: (load (s64) from %ir.cgep31, !tbaa !9)
+    %44:doubleregs = S6_rol_i_p %43, 63
+    %45:doubleregs = S6_rol_i_p_xacc %44, %43, 56
+    %46:doubleregs = S2_lsr_i_p_xacc %45, %43, 7
+    %47:doubleregs = S6_rol_i_p %31, 45
+    %48:doubleregs = S6_rol_i_p_xacc %47, %31, 3
+    %49:doubleregs = S2_lsr_i_p_xacc %48, %31, 6
+    %50:doubleregs = A2_addp killed %46, %32
+    %51:doubleregs = L2_loadrd_io %1, 16 :: (load (s64) from %ir.cgep32, !tbaa !9)
+    %52:doubleregs = A2_addp killed %50, killed %51
+    %53:doubleregs = A2_addp killed %52, killed %49
+    S2_storerd_io %1, 72, killed %53 :: (store (s64) into %ir.cgep30, !tbaa !9)
+    ENDLOOP0 %bb.2, implicit-def $pc, implicit-def $lc0, implicit $sa0, implicit $lc0
+    J2_jump %bb.1, implicit-def $pc
+...



More information about the llvm-branch-commits mailing list