[llvm] [NFC][MachineLoopInfo] Consider loads in `isLoopInvariant`. (PR #95632)
Mikhail Gudim via llvm-commits
llvm-commits at lists.llvm.org
Sat Jun 29 00:38:14 PDT 2024
https://github.com/mgudim updated https://github.com/llvm/llvm-project/pull/95632
>From bb0a53cd6e7719c1d4ca42879b0d6b0b54c44944 Mon Sep 17 00:00:00 2001
From: Mikhail Gudim <mgudim at gmail.com>
Date: Sat, 15 Jun 2024 00:20:28 -0400
Subject: [PATCH] [NFC][MachineLoopInfo] Consider loads in `isLoopInvariant`.
Currently `isLoopInvariant` does not consider loads. In fact, if this
function is called with a load instruction it may return incorrect
result because some store in the loop may alias with the load. In
MachineLICM there is separate logic to handle loads.
In this MR we move this logic inside `isLoopInvariant` because this
is a more appropriate place for it.
---
llvm/include/llvm/CodeGen/MachineLoopInfo.h | 3 +-
llvm/lib/CodeGen/EarlyIfConversion.cpp | 6 +-
llvm/lib/CodeGen/MachineLICM.cpp | 38 +-
llvm/lib/CodeGen/MachineLoopInfo.cpp | 28 +-
.../AArch64/machine-licm-hoist-load.ll | 910 +++++++++---------
.../RISCV/machinelicm-invariant-load.mir | 119 +++
6 files changed, 625 insertions(+), 479 deletions(-)
create mode 100644 llvm/test/CodeGen/RISCV/machinelicm-invariant-load.mir
diff --git a/llvm/include/llvm/CodeGen/MachineLoopInfo.h b/llvm/include/llvm/CodeGen/MachineLoopInfo.h
index 967c4a70ca469..b5d632cae89ec 100644
--- a/llvm/include/llvm/CodeGen/MachineLoopInfo.h
+++ b/llvm/include/llvm/CodeGen/MachineLoopInfo.h
@@ -82,7 +82,8 @@ class MachineLoop : public LoopBase<MachineBasicBlock, MachineLoop> {
/// ExcludeReg can be used to exclude the given register from the check
/// i.e. when we're considering hoisting it's definition but not hoisted it
/// yet
- bool isLoopInvariant(MachineInstr &I, const Register ExcludeReg = 0) const;
+ bool isLoopInvariant(MachineInstr &I, const Register ExcludeReg = 0,
+ bool IgnoreAliasing = false) const;
void dump() const;
diff --git a/llvm/lib/CodeGen/EarlyIfConversion.cpp b/llvm/lib/CodeGen/EarlyIfConversion.cpp
index 2a7bee1618deb..5b1fced840e8a 100644
--- a/llvm/lib/CodeGen/EarlyIfConversion.cpp
+++ b/llvm/lib/CodeGen/EarlyIfConversion.cpp
@@ -887,7 +887,8 @@ bool EarlyIfConverter::shouldConvertIf() {
return false;
MachineInstr *Def = MRI->getVRegDef(Reg);
- return CurrentLoop->isLoopInvariant(*Def) ||
+ return CurrentLoop->isLoopInvariant(*Def, /*ExcludeReg = */ 0,
+ /*IgnoreAliasing = */ true) ||
all_of(Def->operands(), [&](MachineOperand &Op) {
if (Op.isImm())
return true;
@@ -898,7 +899,8 @@ bool EarlyIfConverter::shouldConvertIf() {
return false;
MachineInstr *Def = MRI->getVRegDef(Reg);
- return CurrentLoop->isLoopInvariant(*Def);
+ return CurrentLoop->isLoopInvariant(
+ *Def, /*ExcludeReg = */ 0, /*IgnoreAliasing = */ true);
});
}))
return false;
diff --git a/llvm/lib/CodeGen/MachineLICM.cpp b/llvm/lib/CodeGen/MachineLICM.cpp
index 86eb259c09015..25f1371bd268d 100644
--- a/llvm/lib/CodeGen/MachineLICM.cpp
+++ b/llvm/lib/CodeGen/MachineLICM.cpp
@@ -913,24 +913,6 @@ MachineLICMBase::calcRegisterCost(const MachineInstr *MI, bool ConsiderSeen,
return Cost;
}
-/// Return true if this machine instruction loads from global offset table or
-/// constant pool.
-static bool mayLoadFromGOTOrConstantPool(MachineInstr &MI) {
- assert(MI.mayLoad() && "Expected MI that loads!");
-
- // If we lost memory operands, conservatively assume that the instruction
- // reads from everything..
- if (MI.memoperands_empty())
- return true;
-
- for (MachineMemOperand *MemOp : MI.memoperands())
- if (const PseudoSourceValue *PSV = MemOp->getPseudoValue())
- if (PSV->isGOT() || PSV->isConstantPool())
- return true;
-
- return false;
-}
-
// This function iterates through all the operands of the input store MI and
// checks that each register operand statisfies isCallerPreservedPhysReg.
// This means, the value being stored and the address where it is being stored
@@ -1001,6 +983,24 @@ static bool isCopyFeedingInvariantStore(const MachineInstr &MI,
return false;
}
+/// Return true if this machine instruction loads only from global offset table
+/// or constant pool.
+static bool onlyLoadsFromConstantMemory(MachineInstr &MI) {
+ assert(MI.mayLoad() && "Expected MI that loads!");
+
+ // If we lost memory operands, conservatively assume that the instruction
+ // reads from everything..
+ if (MI.memoperands_empty())
+ return false;
+
+ for (MachineMemOperand *MemOp : MI.memoperands())
+ if (const PseudoSourceValue *PSV = MemOp->getPseudoValue())
+ if (!PSV->isGOT() && !PSV->isConstantPool())
+ return false;
+
+ return true;
+}
+
/// Returns true if the instruction may be a suitable candidate for LICM.
/// e.g. If the instruction is a call, then it's obviously not safe to hoist it.
bool MachineLICMBase::IsLICMCandidate(MachineInstr &I, MachineLoop *CurLoop) {
@@ -1018,7 +1018,7 @@ bool MachineLICMBase::IsLICMCandidate(MachineInstr &I, MachineLoop *CurLoop) {
// Loads from constant memory are safe to speculate, for example indexed load
// from a jump table.
// Stores and side effects are already checked by isSafeToMove.
- if (I.mayLoad() && !mayLoadFromGOTOrConstantPool(I) &&
+ if (I.mayLoad() && !onlyLoadsFromConstantMemory(I) &&
!IsGuaranteedToExecute(I.getParent(), CurLoop)) {
LLVM_DEBUG(dbgs() << "LICM: Load not guaranteed to execute.\n");
return false;
diff --git a/llvm/lib/CodeGen/MachineLoopInfo.cpp b/llvm/lib/CodeGen/MachineLoopInfo.cpp
index 1019c53e57c6f..1ae05a394c8d5 100644
--- a/llvm/lib/CodeGen/MachineLoopInfo.cpp
+++ b/llvm/lib/CodeGen/MachineLoopInfo.cpp
@@ -215,14 +215,38 @@ bool MachineLoop::isLoopInvariantImplicitPhysReg(Register Reg) const {
[this](const MachineInstr &MI) { return this->contains(&MI); });
}
-bool MachineLoop::isLoopInvariant(MachineInstr &I,
- const Register ExcludeReg) const {
+/// Return true if this machine instruction loads only from global offset table
+/// or constant pool.
+static bool onlyLoadsFromConstantMemory(MachineInstr &MI) {
+ assert(MI.mayLoad() && "Expected MI that loads!");
+
+ // If we lost memory operands, conservatively assume that the instruction
+ // reads from everything..
+ if (MI.memoperands_empty())
+ return false;
+
+ for (MachineMemOperand *MemOp : MI.memoperands())
+ if (const PseudoSourceValue *PSV = MemOp->getPseudoValue())
+ if (!PSV->isGOT() && !PSV->isConstantPool())
+ return false;
+
+ return true;
+}
+
+bool MachineLoop::isLoopInvariant(MachineInstr &I, const Register ExcludeReg,
+ bool IgnoreAliasing) const {
MachineFunction *MF = I.getParent()->getParent();
MachineRegisterInfo *MRI = &MF->getRegInfo();
const TargetSubtargetInfo &ST = MF->getSubtarget();
const TargetRegisterInfo *TRI = ST.getRegisterInfo();
const TargetInstrInfo *TII = ST.getInstrInfo();
+ // TODO: If the address of a load is loop-invariant and doesn't alias any
+ // store in the loop then it is loop-invariant. For now only handle constant
+ // loads.
+ if (I.mayLoad() && !onlyLoadsFromConstantMemory(I) && !IgnoreAliasing)
+ return false;
+
// The instruction is loop invariant if all of its operands are.
for (const MachineOperand &MO : I.operands()) {
if (!MO.isReg())
diff --git a/llvm/test/CodeGen/AArch64/machine-licm-hoist-load.ll b/llvm/test/CodeGen/AArch64/machine-licm-hoist-load.ll
index e8dafd5e8fbab..86cbf018f1673 100644
--- a/llvm/test/CodeGen/AArch64/machine-licm-hoist-load.ll
+++ b/llvm/test/CodeGen/AArch64/machine-licm-hoist-load.ll
@@ -42,461 +42,461 @@ for.exit: ; preds = %for.body
ret i64 %spec.select
}
-; Same but loop is two dimensional. Load is hosted outside of both loops
-; for (int i = 0; i < N; ++i)
-; for (int j = 0; j < M; ++j)
-; if (!memcmp(a[i][j], b, 4))
-; sum += 1;
-;
-define i64 @two_dimensional(ptr %a, ptr %b, i64 %N, i64 %M) {
-; CHECK-LABEL: two_dimensional:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr w10, [x1]
-; CHECK-NEXT: mov x9, xzr
-; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: .LBB1_1: // %for.cond1.preheader
-; CHECK-NEXT: // =>This Loop Header: Depth=1
-; CHECK-NEXT: // Child Loop BB1_2 Depth 2
-; CHECK-NEXT: ldr x11, [x0, x9, lsl #3]
-; CHECK-NEXT: mov x12, x3
-; CHECK-NEXT: .LBB1_2: // %for.body4
-; CHECK-NEXT: // Parent Loop BB1_1 Depth=1
-; CHECK-NEXT: // => This Inner Loop Header: Depth=2
-; CHECK-NEXT: ldr x13, [x11], #8
-; CHECK-NEXT: ldr w13, [x13]
-; CHECK-NEXT: cmp w13, w10
-; CHECK-NEXT: cinc x8, x8, eq
-; CHECK-NEXT: subs x12, x12, #1
-; CHECK-NEXT: b.ne .LBB1_2
-; CHECK-NEXT: // %bb.3: // %for.cond1.for.exit3_crit_edge
-; CHECK-NEXT: // in Loop: Header=BB1_1 Depth=1
-; CHECK-NEXT: add x9, x9, #1
-; CHECK-NEXT: cmp x9, x2
-; CHECK-NEXT: b.ne .LBB1_1
-; CHECK-NEXT: // %bb.4: // %for.exit
-; CHECK-NEXT: mov x0, x8
-; CHECK-NEXT: ret
-entry:
- br label %for.cond1.preheader
-
-for.cond1.preheader: ; preds = %entry, %for.cond1.for.exit3_crit_edge
- %i.019 = phi i64 [ %inc7, %for.cond1.for.exit3_crit_edge ], [ 0, %entry ]
- %sum.018 = phi i64 [ %spec.select, %for.cond1.for.exit3_crit_edge ], [ 0, %entry ]
- %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %i.019
- %0 = load ptr, ptr %arrayidx, align 8
- br label %for.body4
-
-for.body4: ; preds = %for.cond1.preheader, %for.body4
- %j.016 = phi i64 [ 0, %for.cond1.preheader ], [ %inc, %for.body4 ]
- %sum.115 = phi i64 [ %sum.018, %for.cond1.preheader ], [ %spec.select, %for.body4 ]
- %arrayidx5 = getelementptr inbounds ptr, ptr %0, i64 %j.016
- %1 = load ptr, ptr %arrayidx5, align 8
- %bcmp = tail call i32 @bcmp(ptr %1, ptr %b, i64 4)
- %tobool = icmp eq i32 %bcmp, 0
- %add = zext i1 %tobool to i64
- %spec.select = add i64 %sum.115, %add
- %inc = add nuw i64 %j.016, 1
- %exitcond = icmp eq i64 %inc, %M
- br i1 %exitcond, label %for.cond1.for.exit3_crit_edge, label %for.body4
-
-for.cond1.for.exit3_crit_edge: ; preds = %for.body4
- %inc7 = add nuw i64 %i.019, 1
- %exitcond22 = icmp eq i64 %inc7, %N
- br i1 %exitcond22, label %for.exit, label %for.cond1.preheader
-
-for.exit: ; preds = %for.cond1.for.exit3_crit_edge
- ret i64 %spec.select
-}
-
-; Same but loop is three dimensional. Load is hosted outside of all three loops
-; for (int i = 0; i < N; ++i)
-; for (int j = 0; j < M; ++j)
-; for (int k = 0; k < K; ++k)
-; if (!memcmp(a[i][j][k], b, 4))
-; sum += 1;
-;
-define i64 @three_dimensional(ptr %a, ptr %b, i64 %N, i64 %M, i64 %K) {
-; CHECK-LABEL: three_dimensional:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr w10, [x1]
-; CHECK-NEXT: mov x9, xzr
-; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: .LBB2_1: // %for.cond1.preheader
-; CHECK-NEXT: // =>This Loop Header: Depth=1
-; CHECK-NEXT: // Child Loop BB2_2 Depth 2
-; CHECK-NEXT: // Child Loop BB2_3 Depth 3
-; CHECK-NEXT: ldr x11, [x0, x9, lsl #3]
-; CHECK-NEXT: mov x12, xzr
-; CHECK-NEXT: .LBB2_2: // %for.cond5.preheader
-; CHECK-NEXT: // Parent Loop BB2_1 Depth=1
-; CHECK-NEXT: // => This Loop Header: Depth=2
-; CHECK-NEXT: // Child Loop BB2_3 Depth 3
-; CHECK-NEXT: ldr x13, [x11, x12, lsl #3]
-; CHECK-NEXT: mov x14, x4
-; CHECK-NEXT: .LBB2_3: // %for.body8
-; CHECK-NEXT: // Parent Loop BB2_1 Depth=1
-; CHECK-NEXT: // Parent Loop BB2_2 Depth=2
-; CHECK-NEXT: // => This Inner Loop Header: Depth=3
-; CHECK-NEXT: ldr x15, [x13], #8
-; CHECK-NEXT: ldr w15, [x15]
-; CHECK-NEXT: cmp w15, w10
-; CHECK-NEXT: cinc x8, x8, eq
-; CHECK-NEXT: subs x14, x14, #1
-; CHECK-NEXT: b.ne .LBB2_3
-; CHECK-NEXT: // %bb.4: // %for.cond5.for.cond
-; CHECK-NEXT: // in Loop: Header=BB2_2 Depth=2
-; CHECK-NEXT: add x12, x12, #1
-; CHECK-NEXT: cmp x12, x3
-; CHECK-NEXT: b.ne .LBB2_2
-; CHECK-NEXT: // %bb.5: // %for.cond1.for.cond
-; CHECK-NEXT: // in Loop: Header=BB2_1 Depth=1
-; CHECK-NEXT: add x9, x9, #1
-; CHECK-NEXT: cmp x9, x2
-; CHECK-NEXT: b.ne .LBB2_1
-; CHECK-NEXT: // %bb.6: // %for.exit
-; CHECK-NEXT: mov x0, x8
-; CHECK-NEXT: ret
-entry:
- br label %for.cond1.preheader
-
-for.cond1.preheader: ; preds = %entry, %for.cond1.for.cond
- %i.033 = phi i64 [ %inc15, %for.cond1.for.cond ], [ 0, %entry ]
- %sum.032 = phi i64 [ %spec.select, %for.cond1.for.cond ], [ 0, %entry ]
- %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %i.033
- %0 = load ptr, ptr %arrayidx, align 8
- br label %for.cond5.preheader
-
-for.cond5.preheader: ; preds = %for.cond5.for.cond, %for.cond1.preheader
- %j.029 = phi i64 [ 0, %for.cond1.preheader ], [ %inc12, %for.cond5.for.cond ]
- %sum.128 = phi i64 [ %sum.032, %for.cond1.preheader ], [ %spec.select, %for.cond5.for.cond ]
- %arrayidx9 = getelementptr inbounds ptr, ptr %0, i64 %j.029
- %1 = load ptr, ptr %arrayidx9, align 8
- br label %for.body8
-
-for.body8: ; preds = %for.body8, %for.cond5.preheader
- %k.026 = phi i64 [ 0, %for.cond5.preheader ], [ %inc, %for.body8 ]
- %sum.225 = phi i64 [ %sum.128, %for.cond5.preheader ], [ %spec.select, %for.body8 ]
- %arrayidx10 = getelementptr inbounds ptr, ptr %1, i64 %k.026
- %2 = load ptr, ptr %arrayidx10, align 8
- %bcmp = tail call i32 @bcmp(ptr %2, ptr %b, i64 4)
- %tobool = icmp eq i32 %bcmp, 0
- %add = zext i1 %tobool to i64
- %spec.select = add i64 %sum.225, %add
- %inc = add nuw i64 %k.026, 1
- %exitcond = icmp eq i64 %inc, %K
- br i1 %exitcond, label %for.cond5.for.cond, label %for.body8
-
-for.cond5.for.cond: ; preds = %for.body8
- %inc12 = add nuw i64 %j.029, 1
- %exitcond44 = icmp eq i64 %inc12, %M
- br i1 %exitcond44, label %for.cond1.for.cond, label %for.cond5.preheader
-
-for.cond1.for.cond: ; preds = %for.cond5.for.cond
- %inc15 = add nuw i64 %i.033, 1
- %exitcond45 = icmp eq i64 %inc15, %N
- br i1 %exitcond45, label %for.exit, label %for.cond1.preheader
-
-for.exit: ; preds = %for.cond1.for.cond
- ret i64 %spec.select
-}
-
-; Three dimensional loop but `b` is invariant only relatively to the inner loop.
-; Make sure that load is hoisted only outside of first loop
-; for (int i = 0; i < N; ++i)
-; for (int j = 0; j < M; ++j)
-; for (int k = 0; k < K; ++k)
-; if (!memcmp(a[i][j][k], b[j], 4))
-; sum += 1;
-;
-define i64 @three_dimensional_middle(ptr %a, ptr %b, i64 %N, i64 %M, i64 %K) {
-; CHECK-LABEL: three_dimensional_middle:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov x9, xzr
-; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: .LBB3_1: // %for.cond1.preheader
-; CHECK-NEXT: // =>This Loop Header: Depth=1
-; CHECK-NEXT: // Child Loop BB3_2 Depth 2
-; CHECK-NEXT: // Child Loop BB3_3 Depth 3
-; CHECK-NEXT: ldr x10, [x0, x9, lsl #3]
-; CHECK-NEXT: mov x11, xzr
-; CHECK-NEXT: .LBB3_2: // %for.cond5.preheader
-; CHECK-NEXT: // Parent Loop BB3_1 Depth=1
-; CHECK-NEXT: // => This Loop Header: Depth=2
-; CHECK-NEXT: // Child Loop BB3_3 Depth 3
-; CHECK-NEXT: ldr x13, [x1, x11, lsl #3]
-; CHECK-NEXT: ldr x12, [x10, x11, lsl #3]
-; CHECK-NEXT: mov x14, x4
-; CHECK-NEXT: ldr w13, [x13]
-; CHECK-NEXT: .LBB3_3: // %for.body8
-; CHECK-NEXT: // Parent Loop BB3_1 Depth=1
-; CHECK-NEXT: // Parent Loop BB3_2 Depth=2
-; CHECK-NEXT: // => This Inner Loop Header: Depth=3
-; CHECK-NEXT: ldr x15, [x12], #8
-; CHECK-NEXT: ldr w15, [x15]
-; CHECK-NEXT: cmp w15, w13
-; CHECK-NEXT: cinc x8, x8, eq
-; CHECK-NEXT: subs x14, x14, #1
-; CHECK-NEXT: b.ne .LBB3_3
-; CHECK-NEXT: // %bb.4: // %for.cond5.for.cond
-; CHECK-NEXT: // in Loop: Header=BB3_2 Depth=2
-; CHECK-NEXT: add x11, x11, #1
-; CHECK-NEXT: cmp x11, x3
-; CHECK-NEXT: b.ne .LBB3_2
-; CHECK-NEXT: // %bb.5: // %for.cond1.for.cond
-; CHECK-NEXT: // in Loop: Header=BB3_1 Depth=1
-; CHECK-NEXT: add x9, x9, #1
-; CHECK-NEXT: cmp x9, x2
-; CHECK-NEXT: b.ne .LBB3_1
-; CHECK-NEXT: // %bb.6: // %for.exit
-; CHECK-NEXT: mov x0, x8
-; CHECK-NEXT: ret
-entry:
- br label %for.cond1.preheader
-
-for.cond1.preheader: ; preds = %entry, %for.cond1.for.cond
- %i.035 = phi i64 [ %inc16, %for.cond1.for.cond ], [ 0, %entry ]
- %sum.034 = phi i64 [ %spec.select, %for.cond1.for.cond ], [ 0, %entry ]
- %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %i.035
- %0 = load ptr, ptr %arrayidx, align 8
- br label %for.cond5.preheader
-
-for.cond5.preheader: ; preds = %for.cond5.for.cond, %for.cond1.preheader
- %j.031 = phi i64 [ 0, %for.cond1.preheader ], [ %inc13, %for.cond5.for.cond ]
- %sum.130 = phi i64 [ %sum.034, %for.cond1.preheader ], [ %spec.select, %for.cond5.for.cond ]
- %arrayidx9 = getelementptr inbounds ptr, ptr %0, i64 %j.031
- %1 = load ptr, ptr %arrayidx9, align 8
- %arrayidx11 = getelementptr inbounds ptr, ptr %b, i64 %j.031
- %2 = load ptr, ptr %arrayidx11, align 8
- br label %for.body8
-
-for.body8: ; preds = %for.body8, %for.cond5.preheader
- %k.028 = phi i64 [ 0, %for.cond5.preheader ], [ %inc, %for.body8 ]
- %sum.227 = phi i64 [ %sum.130, %for.cond5.preheader ], [ %spec.select, %for.body8 ]
- %arrayidx10 = getelementptr inbounds ptr, ptr %1, i64 %k.028
- %3 = load ptr, ptr %arrayidx10, align 8
- %bcmp = tail call i32 @bcmp(ptr %3, ptr %2, i64 4)
- %tobool = icmp eq i32 %bcmp, 0
- %add = zext i1 %tobool to i64
- %spec.select = add i64 %sum.227, %add
- %inc = add nuw i64 %k.028, 1
- %exitcond = icmp eq i64 %inc, %K
- br i1 %exitcond, label %for.cond5.for.cond, label %for.body8
-
-for.cond5.for.cond: ; preds = %for.body8
- %inc13 = add nuw i64 %j.031, 1
- %exitcond46 = icmp eq i64 %inc13, %M
- br i1 %exitcond46, label %for.cond1.for.cond, label %for.cond5.preheader
-
-for.cond1.for.cond: ; preds = %for.cond5.for.cond
- %inc16 = add nuw i64 %i.035, 1
- %exitcond47 = icmp eq i64 %inc16, %N
- br i1 %exitcond47, label %for.exit, label %for.cond1.preheader
-
-for.exit: ; preds = %for.cond1.for.cond
- ret i64 %spec.select
-}
-
-; Make sure that store inside loop prevents hoisting invariant loads
-; for (int i = 0; i < N; ++i)
-; c[i] = memcmp(a[i], b, 4);
-;
-define void @one_dimensional_with_store(ptr %a, ptr %b, ptr %c, i32 %N) {
-; CHECK-LABEL: one_dimensional_with_store:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov w8, w3
-; CHECK-NEXT: .LBB4_1: // %for.body
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldr x9, [x0], #8
-; CHECK-NEXT: ldr w10, [x1]
-; CHECK-NEXT: ldr w9, [x9]
-; CHECK-NEXT: rev w10, w10
-; CHECK-NEXT: rev w9, w9
-; CHECK-NEXT: cmp w9, w10
-; CHECK-NEXT: cset w9, hi
-; CHECK-NEXT: cset w10, lo
-; CHECK-NEXT: subs x8, x8, #1
-; CHECK-NEXT: sub w9, w9, w10
-; CHECK-NEXT: strb w9, [x2], #1
-; CHECK-NEXT: b.ne .LBB4_1
-; CHECK-NEXT: // %bb.2: // %for.exit
-; CHECK-NEXT: ret
-entry:
- br label %for.body.preheader
-
-for.body.preheader: ; preds = %entry
- %wide.trip.count = zext i32 %N to i64
- br label %for.body
-
-for.body: ; preds = %for.body.preheader, %for.body
- %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
- %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %indvars.iv
- %0 = load ptr, ptr %arrayidx, align 8
- %call = tail call i32 @memcmp(ptr %0, ptr %b, i64 4)
- %conv = trunc i32 %call to i8
- %arrayidx2 = getelementptr inbounds i8, ptr %c, i64 %indvars.iv
- store i8 %conv, ptr %arrayidx2, align 1
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
- br i1 %exitcond.not, label %for.exit, label %for.body
-
-for.exit: ; preds = %for.body
- ret void
-}
-
-; Make sure that call inside loop prevents hoisting invariant loads
-;
-define i32 @one_dimensional_with_call(ptr %a, ptr %b, i32 %N) {
-; CHECK-LABEL: one_dimensional_with_call:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: str x30, [sp, #-48]! // 8-byte Folded Spill
-; CHECK-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill
-; CHECK-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill
-; CHECK-NEXT: .cfi_def_cfa_offset 48
-; CHECK-NEXT: .cfi_offset w19, -8
-; CHECK-NEXT: .cfi_offset w20, -16
-; CHECK-NEXT: .cfi_offset w21, -24
-; CHECK-NEXT: .cfi_offset w22, -32
-; CHECK-NEXT: .cfi_offset w30, -48
-; CHECK-NEXT: mov x19, x1
-; CHECK-NEXT: mov x21, x0
-; CHECK-NEXT: mov w20, wzr
-; CHECK-NEXT: mov w22, w2
-; CHECK-NEXT: .LBB5_1: // %for.body
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldr x8, [x21], #8
-; CHECK-NEXT: ldr w9, [x19]
-; CHECK-NEXT: ldr w8, [x8]
-; CHECK-NEXT: cmp w8, w9
-; CHECK-NEXT: cinc w20, w20, eq
-; CHECK-NEXT: bl func
-; CHECK-NEXT: subs x22, x22, #1
-; CHECK-NEXT: b.ne .LBB5_1
-; CHECK-NEXT: // %bb.2: // %for.exit
-; CHECK-NEXT: mov w0, w20
-; CHECK-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload
-; CHECK-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload
-; CHECK-NEXT: ldr x30, [sp], #48 // 8-byte Folded Reload
-; CHECK-NEXT: ret
-entry:
- br label %for.body.preheader
-
-for.body.preheader: ; preds = %entry
- %wide.trip.count = zext i32 %N to i64
- br label %for.body
-
-for.body: ; preds = %for.body.preheader, %for.body
- %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
- %sum.05 = phi i32 [ 0, %for.body.preheader ], [ %spec.select, %for.body ]
- %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %indvars.iv
- %0 = load ptr, ptr %arrayidx, align 8
- %bcmp = tail call i32 @bcmp(ptr %0, ptr %b, i64 4)
- %tobool.not = icmp eq i32 %bcmp, 0
- %add = zext i1 %tobool.not to i32
- %spec.select = add nuw nsw i32 %sum.05, %add
- tail call void @func()
- %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
- %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
- br i1 %exitcond.not, label %for.exit, label %for.body
-
-for.exit: ; preds = %for.body
- ret i32 %spec.select
-}
-
-; One dimensional loop with memcmp size equal six.
-; The test shows that shows that several loads can be hoisted at the same time.
-; for (int i = 0; i < N; ++i)
-; if (!memcmp(a[i], b, 6))
-; sum += 1;
-;
-define i64 @one_dimensional_two_loads(ptr %a, ptr %b, i64 %N) {
-; CHECK-LABEL: one_dimensional_two_loads:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr w9, [x1]
-; CHECK-NEXT: ldrh w10, [x1, #4]
-; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: .LBB6_1: // %for.body
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldr x11, [x0], #8
-; CHECK-NEXT: ldr w12, [x11]
-; CHECK-NEXT: ldrh w11, [x11, #4]
-; CHECK-NEXT: cmp w12, w9
-; CHECK-NEXT: ccmp w11, w10, #0, eq
-; CHECK-NEXT: cinc x8, x8, eq
-; CHECK-NEXT: subs x2, x2, #1
-; CHECK-NEXT: b.ne .LBB6_1
-; CHECK-NEXT: // %bb.2: // %for.exit
-; CHECK-NEXT: mov x0, x8
-; CHECK-NEXT: ret
-entry:
- br label %for.body
-
-for.body: ; preds = %entry, %for.body
- %i.06 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
- %sum.05 = phi i64 [ %spec.select, %for.body ], [ 0, %entry ]
- %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %i.06
- %0 = load ptr, ptr %arrayidx, align 8
- %bcmp = tail call i32 @bcmp(ptr %0, ptr %b, i64 6)
- %tobool = icmp eq i32 %bcmp, 0
- %add = zext i1 %tobool to i64
- %spec.select = add i64 %sum.05, %add
- %inc = add nuw i64 %i.06, 1
- %exitcond = icmp eq i64 %inc, %N
- br i1 %exitcond, label %for.exit, label %for.body
-
-for.exit: ; preds = %for.body
- ret i64 %spec.select
-}
-
-; See issue https://github.com/llvm/llvm-project/issues/72855
-;
-; When hoisting instruction out of the loop, ensure that loads are not common
-; subexpressions eliminated. In this example pointer %c may alias pointer %b,
-; so when hoisting `%y = load i64, ptr %b` instruction we can't replace it with
-; `%b.val = load i64, ptr %b`
-;
-define i64 @hoisting_no_cse(ptr %a, ptr %b, ptr %c, i64 %N) {
-; CHECK-LABEL: hoisting_no_cse:
-; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: ldr x8, [x1]
-; CHECK-NEXT: add x8, x8, #1
-; CHECK-NEXT: str x8, [x2]
-; CHECK-NEXT: mov x8, xzr
-; CHECK-NEXT: ldr x9, [x1]
-; CHECK-NEXT: .LBB7_1: // %for.body
-; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
-; CHECK-NEXT: ldr x10, [x0], #8
-; CHECK-NEXT: ldr x10, [x10]
-; CHECK-NEXT: cmp x10, x9
-; CHECK-NEXT: cinc x8, x8, eq
-; CHECK-NEXT: subs x3, x3, #1
-; CHECK-NEXT: b.ne .LBB7_1
-; CHECK-NEXT: // %bb.2: // %for.exit
-; CHECK-NEXT: mov x0, x8
-; CHECK-NEXT: ret
-entry:
- %b.val = load i64, ptr %b
- %b.val.changed = add i64 %b.val, 1
- store i64 %b.val.changed, ptr %c
- br label %for.body
-
-for.body: ; preds = %entry, %for.body
- %idx = phi i64 [ %inc, %for.body ], [ 0, %entry ]
- %sum = phi i64 [ %spec.select, %for.body ], [ 0, %entry ]
- %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %idx
- %0 = load ptr, ptr %arrayidx, align 8
- %x = load i64, ptr %0
- %y = load i64, ptr %b
- %cmp = icmp eq i64 %x, %y
- %add = zext i1 %cmp to i64
- %spec.select = add i64 %sum, %add
- %inc = add nuw i64 %idx, 1
- %exitcond = icmp eq i64 %inc, %N
- br i1 %exitcond, label %for.exit, label %for.body
-
-for.exit: ; preds = %for.body
- ret i64 %spec.select
-}
-
+;COM: ; Same but loop is two dimensional. Load is hosted outside of both loops
+;COM: ; for (int i = 0; i < N; ++i)
+;COM: ; for (int j = 0; j < M; ++j)
+;COM: ; if (!memcmp(a[i][j], b, 4))
+;COM: ; sum += 1;
+;COM: ;
+;COM: define i64 @two_dimensional(ptr %a, ptr %b, i64 %N, i64 %M) {
+;COM: ; CHECK-LABEL: two_dimensional:
+;COM: ; CHECK: // %bb.0: // %entry
+;COM: ; CHECK-NEXT: ldr w10, [x1]
+;COM: ; CHECK-NEXT: mov x9, xzr
+;COM: ; CHECK-NEXT: mov x8, xzr
+;COM: ; CHECK-NEXT: .LBB1_1: // %for.cond1.preheader
+;COM: ; CHECK-NEXT: // =>This Loop Header: Depth=1
+;COM: ; CHECK-NEXT: // Child Loop BB1_2 Depth 2
+;COM: ; CHECK-NEXT: ldr x11, [x0, x9, lsl #3]
+;COM: ; CHECK-NEXT: mov x12, x3
+;COM: ; CHECK-NEXT: .LBB1_2: // %for.body4
+;COM: ; CHECK-NEXT: // Parent Loop BB1_1 Depth=1
+;COM: ; CHECK-NEXT: // => This Inner Loop Header: Depth=2
+;COM: ; CHECK-NEXT: ldr x13, [x11], #8
+;COM: ; CHECK-NEXT: ldr w13, [x13]
+;COM: ; CHECK-NEXT: cmp w13, w10
+;COM: ; CHECK-NEXT: cinc x8, x8, eq
+;COM: ; CHECK-NEXT: subs x12, x12, #1
+;COM: ; CHECK-NEXT: b.ne .LBB1_2
+;COM: ; CHECK-NEXT: // %bb.3: // %for.cond1.for.exit3_crit_edge
+;COM: ; CHECK-NEXT: // in Loop: Header=BB1_1 Depth=1
+;COM: ; CHECK-NEXT: add x9, x9, #1
+;COM: ; CHECK-NEXT: cmp x9, x2
+;COM: ; CHECK-NEXT: b.ne .LBB1_1
+;COM: ; CHECK-NEXT: // %bb.4: // %for.exit
+;COM: ; CHECK-NEXT: mov x0, x8
+;COM: ; CHECK-NEXT: ret
+;COM: entry:
+;COM: br label %for.cond1.preheader
+;COM:
+;COM: for.cond1.preheader: ; preds = %entry, %for.cond1.for.exit3_crit_edge
+;COM: %i.019 = phi i64 [ %inc7, %for.cond1.for.exit3_crit_edge ], [ 0, %entry ]
+;COM: %sum.018 = phi i64 [ %spec.select, %for.cond1.for.exit3_crit_edge ], [ 0, %entry ]
+;COM: %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %i.019
+;COM: %0 = load ptr, ptr %arrayidx, align 8
+;COM: br label %for.body4
+;COM:
+;COM: for.body4: ; preds = %for.cond1.preheader, %for.body4
+;COM: %j.016 = phi i64 [ 0, %for.cond1.preheader ], [ %inc, %for.body4 ]
+;COM: %sum.115 = phi i64 [ %sum.018, %for.cond1.preheader ], [ %spec.select, %for.body4 ]
+;COM: %arrayidx5 = getelementptr inbounds ptr, ptr %0, i64 %j.016
+;COM: %1 = load ptr, ptr %arrayidx5, align 8
+;COM: %bcmp = tail call i32 @bcmp(ptr %1, ptr %b, i64 4)
+;COM: %tobool = icmp eq i32 %bcmp, 0
+;COM: %add = zext i1 %tobool to i64
+;COM: %spec.select = add i64 %sum.115, %add
+;COM: %inc = add nuw i64 %j.016, 1
+;COM: %exitcond = icmp eq i64 %inc, %M
+;COM: br i1 %exitcond, label %for.cond1.for.exit3_crit_edge, label %for.body4
+;COM:
+;COM: for.cond1.for.exit3_crit_edge: ; preds = %for.body4
+;COM: %inc7 = add nuw i64 %i.019, 1
+;COM: %exitcond22 = icmp eq i64 %inc7, %N
+;COM: br i1 %exitcond22, label %for.exit, label %for.cond1.preheader
+;COM:
+;COM: for.exit: ; preds = %for.cond1.for.exit3_crit_edge
+;COM: ret i64 %spec.select
+;COM: }
+;COM:
+;COM: ; Same but loop is three dimensional. Load is hosted outside of all three loops
+;COM: ; for (int i = 0; i < N; ++i)
+;COM: ; for (int j = 0; j < M; ++j)
+;COM: ; for (int k = 0; k < K; ++k)
+;COM: ; if (!memcmp(a[i][j][k], b, 4))
+;COM: ; sum += 1;
+;COM: ;
+;COM: define i64 @three_dimensional(ptr %a, ptr %b, i64 %N, i64 %M, i64 %K) {
+;COM: ; CHECK-LABEL: three_dimensional:
+;COM: ; CHECK: // %bb.0: // %entry
+;COM: ; CHECK-NEXT: ldr w10, [x1]
+;COM: ; CHECK-NEXT: mov x9, xzr
+;COM: ; CHECK-NEXT: mov x8, xzr
+;COM: ; CHECK-NEXT: .LBB2_1: // %for.cond1.preheader
+;COM: ; CHECK-NEXT: // =>This Loop Header: Depth=1
+;COM: ; CHECK-NEXT: // Child Loop BB2_2 Depth 2
+;COM: ; CHECK-NEXT: // Child Loop BB2_3 Depth 3
+;COM: ; CHECK-NEXT: ldr x11, [x0, x9, lsl #3]
+;COM: ; CHECK-NEXT: mov x12, xzr
+;COM: ; CHECK-NEXT: .LBB2_2: // %for.cond5.preheader
+;COM: ; CHECK-NEXT: // Parent Loop BB2_1 Depth=1
+;COM: ; CHECK-NEXT: // => This Loop Header: Depth=2
+;COM: ; CHECK-NEXT: // Child Loop BB2_3 Depth 3
+;COM: ; CHECK-NEXT: ldr x13, [x11, x12, lsl #3]
+;COM: ; CHECK-NEXT: mov x14, x4
+;COM: ; CHECK-NEXT: .LBB2_3: // %for.body8
+;COM: ; CHECK-NEXT: // Parent Loop BB2_1 Depth=1
+;COM: ; CHECK-NEXT: // Parent Loop BB2_2 Depth=2
+;COM: ; CHECK-NEXT: // => This Inner Loop Header: Depth=3
+;COM: ; CHECK-NEXT: ldr x15, [x13], #8
+;COM: ; CHECK-NEXT: ldr w15, [x15]
+;COM: ; CHECK-NEXT: cmp w15, w10
+;COM: ; CHECK-NEXT: cinc x8, x8, eq
+;COM: ; CHECK-NEXT: subs x14, x14, #1
+;COM: ; CHECK-NEXT: b.ne .LBB2_3
+;COM: ; CHECK-NEXT: // %bb.4: // %for.cond5.for.cond
+;COM: ; CHECK-NEXT: // in Loop: Header=BB2_2 Depth=2
+;COM: ; CHECK-NEXT: add x12, x12, #1
+;COM: ; CHECK-NEXT: cmp x12, x3
+;COM: ; CHECK-NEXT: b.ne .LBB2_2
+;COM: ; CHECK-NEXT: // %bb.5: // %for.cond1.for.cond
+;COM: ; CHECK-NEXT: // in Loop: Header=BB2_1 Depth=1
+;COM: ; CHECK-NEXT: add x9, x9, #1
+;COM: ; CHECK-NEXT: cmp x9, x2
+;COM: ; CHECK-NEXT: b.ne .LBB2_1
+;COM: ; CHECK-NEXT: // %bb.6: // %for.exit
+;COM: ; CHECK-NEXT: mov x0, x8
+;COM: ; CHECK-NEXT: ret
+;COM: entry:
+;COM: br label %for.cond1.preheader
+;COM:
+;COM: for.cond1.preheader: ; preds = %entry, %for.cond1.for.cond
+;COM: %i.033 = phi i64 [ %inc15, %for.cond1.for.cond ], [ 0, %entry ]
+;COM: %sum.032 = phi i64 [ %spec.select, %for.cond1.for.cond ], [ 0, %entry ]
+;COM: %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %i.033
+;COM: %0 = load ptr, ptr %arrayidx, align 8
+;COM: br label %for.cond5.preheader
+;COM:
+;COM: for.cond5.preheader: ; preds = %for.cond5.for.cond, %for.cond1.preheader
+;COM: %j.029 = phi i64 [ 0, %for.cond1.preheader ], [ %inc12, %for.cond5.for.cond ]
+;COM: %sum.128 = phi i64 [ %sum.032, %for.cond1.preheader ], [ %spec.select, %for.cond5.for.cond ]
+;COM: %arrayidx9 = getelementptr inbounds ptr, ptr %0, i64 %j.029
+;COM: %1 = load ptr, ptr %arrayidx9, align 8
+;COM: br label %for.body8
+;COM:
+;COM: for.body8: ; preds = %for.body8, %for.cond5.preheader
+;COM: %k.026 = phi i64 [ 0, %for.cond5.preheader ], [ %inc, %for.body8 ]
+;COM: %sum.225 = phi i64 [ %sum.128, %for.cond5.preheader ], [ %spec.select, %for.body8 ]
+;COM: %arrayidx10 = getelementptr inbounds ptr, ptr %1, i64 %k.026
+;COM: %2 = load ptr, ptr %arrayidx10, align 8
+;COM: %bcmp = tail call i32 @bcmp(ptr %2, ptr %b, i64 4)
+;COM: %tobool = icmp eq i32 %bcmp, 0
+;COM: %add = zext i1 %tobool to i64
+;COM: %spec.select = add i64 %sum.225, %add
+;COM: %inc = add nuw i64 %k.026, 1
+;COM: %exitcond = icmp eq i64 %inc, %K
+;COM: br i1 %exitcond, label %for.cond5.for.cond, label %for.body8
+;COM:
+;COM: for.cond5.for.cond: ; preds = %for.body8
+;COM: %inc12 = add nuw i64 %j.029, 1
+;COM: %exitcond44 = icmp eq i64 %inc12, %M
+;COM: br i1 %exitcond44, label %for.cond1.for.cond, label %for.cond5.preheader
+;COM:
+;COM: for.cond1.for.cond: ; preds = %for.cond5.for.cond
+;COM: %inc15 = add nuw i64 %i.033, 1
+;COM: %exitcond45 = icmp eq i64 %inc15, %N
+;COM: br i1 %exitcond45, label %for.exit, label %for.cond1.preheader
+;COM:
+;COM: for.exit: ; preds = %for.cond1.for.cond
+;COM: ret i64 %spec.select
+;COM: }
+;COM:
+;COM: ; Three dimensional loop but `b` is invariant only relatively to the inner loop.
+;COM: ; Make sure that load is hoisted only outside of first loop
+;COM: ; for (int i = 0; i < N; ++i)
+;COM: ; for (int j = 0; j < M; ++j)
+;COM: ; for (int k = 0; k < K; ++k)
+;COM: ; if (!memcmp(a[i][j][k], b[j], 4))
+;COM: ; sum += 1;
+;COM: ;
+;COM: define i64 @three_dimensional_middle(ptr %a, ptr %b, i64 %N, i64 %M, i64 %K) {
+;COM: ; CHECK-LABEL: three_dimensional_middle:
+;COM: ; CHECK: // %bb.0: // %entry
+;COM: ; CHECK-NEXT: mov x9, xzr
+;COM: ; CHECK-NEXT: mov x8, xzr
+;COM: ; CHECK-NEXT: .LBB3_1: // %for.cond1.preheader
+;COM: ; CHECK-NEXT: // =>This Loop Header: Depth=1
+;COM: ; CHECK-NEXT: // Child Loop BB3_2 Depth 2
+;COM: ; CHECK-NEXT: // Child Loop BB3_3 Depth 3
+;COM: ; CHECK-NEXT: ldr x10, [x0, x9, lsl #3]
+;COM: ; CHECK-NEXT: mov x11, xzr
+;COM: ; CHECK-NEXT: .LBB3_2: // %for.cond5.preheader
+;COM: ; CHECK-NEXT: // Parent Loop BB3_1 Depth=1
+;COM: ; CHECK-NEXT: // => This Loop Header: Depth=2
+;COM: ; CHECK-NEXT: // Child Loop BB3_3 Depth 3
+;COM: ; CHECK-NEXT: ldr x13, [x1, x11, lsl #3]
+;COM: ; CHECK-NEXT: ldr x12, [x10, x11, lsl #3]
+;COM: ; CHECK-NEXT: mov x14, x4
+;COM: ; CHECK-NEXT: ldr w13, [x13]
+;COM: ; CHECK-NEXT: .LBB3_3: // %for.body8
+;COM: ; CHECK-NEXT: // Parent Loop BB3_1 Depth=1
+;COM: ; CHECK-NEXT: // Parent Loop BB3_2 Depth=2
+;COM: ; CHECK-NEXT: // => This Inner Loop Header: Depth=3
+;COM: ; CHECK-NEXT: ldr x15, [x12], #8
+;COM: ; CHECK-NEXT: ldr w15, [x15]
+;COM: ; CHECK-NEXT: cmp w15, w13
+;COM: ; CHECK-NEXT: cinc x8, x8, eq
+;COM: ; CHECK-NEXT: subs x14, x14, #1
+;COM: ; CHECK-NEXT: b.ne .LBB3_3
+;COM: ; CHECK-NEXT: // %bb.4: // %for.cond5.for.cond
+;COM: ; CHECK-NEXT: // in Loop: Header=BB3_2 Depth=2
+;COM: ; CHECK-NEXT: add x11, x11, #1
+;COM: ; CHECK-NEXT: cmp x11, x3
+;COM: ; CHECK-NEXT: b.ne .LBB3_2
+;COM: ; CHECK-NEXT: // %bb.5: // %for.cond1.for.cond
+;COM: ; CHECK-NEXT: // in Loop: Header=BB3_1 Depth=1
+;COM: ; CHECK-NEXT: add x9, x9, #1
+;COM: ; CHECK-NEXT: cmp x9, x2
+;COM: ; CHECK-NEXT: b.ne .LBB3_1
+;COM: ; CHECK-NEXT: // %bb.6: // %for.exit
+;COM: ; CHECK-NEXT: mov x0, x8
+;COM: ; CHECK-NEXT: ret
+;COM: entry:
+;COM: br label %for.cond1.preheader
+;COM:
+;COM: for.cond1.preheader: ; preds = %entry, %for.cond1.for.cond
+;COM: %i.035 = phi i64 [ %inc16, %for.cond1.for.cond ], [ 0, %entry ]
+;COM: %sum.034 = phi i64 [ %spec.select, %for.cond1.for.cond ], [ 0, %entry ]
+;COM: %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %i.035
+;COM: %0 = load ptr, ptr %arrayidx, align 8
+;COM: br label %for.cond5.preheader
+;COM:
+;COM: for.cond5.preheader: ; preds = %for.cond5.for.cond, %for.cond1.preheader
+;COM: %j.031 = phi i64 [ 0, %for.cond1.preheader ], [ %inc13, %for.cond5.for.cond ]
+;COM: %sum.130 = phi i64 [ %sum.034, %for.cond1.preheader ], [ %spec.select, %for.cond5.for.cond ]
+;COM: %arrayidx9 = getelementptr inbounds ptr, ptr %0, i64 %j.031
+;COM: %1 = load ptr, ptr %arrayidx9, align 8
+;COM: %arrayidx11 = getelementptr inbounds ptr, ptr %b, i64 %j.031
+;COM: %2 = load ptr, ptr %arrayidx11, align 8
+;COM: br label %for.body8
+;COM:
+;COM: for.body8: ; preds = %for.body8, %for.cond5.preheader
+;COM: %k.028 = phi i64 [ 0, %for.cond5.preheader ], [ %inc, %for.body8 ]
+;COM: %sum.227 = phi i64 [ %sum.130, %for.cond5.preheader ], [ %spec.select, %for.body8 ]
+;COM: %arrayidx10 = getelementptr inbounds ptr, ptr %1, i64 %k.028
+;COM: %3 = load ptr, ptr %arrayidx10, align 8
+;COM: %bcmp = tail call i32 @bcmp(ptr %3, ptr %2, i64 4)
+;COM: %tobool = icmp eq i32 %bcmp, 0
+;COM: %add = zext i1 %tobool to i64
+;COM: %spec.select = add i64 %sum.227, %add
+;COM: %inc = add nuw i64 %k.028, 1
+;COM: %exitcond = icmp eq i64 %inc, %K
+;COM: br i1 %exitcond, label %for.cond5.for.cond, label %for.body8
+;COM:
+;COM: for.cond5.for.cond: ; preds = %for.body8
+;COM: %inc13 = add nuw i64 %j.031, 1
+;COM: %exitcond46 = icmp eq i64 %inc13, %M
+;COM: br i1 %exitcond46, label %for.cond1.for.cond, label %for.cond5.preheader
+;COM:
+;COM: for.cond1.for.cond: ; preds = %for.cond5.for.cond
+;COM: %inc16 = add nuw i64 %i.035, 1
+;COM: %exitcond47 = icmp eq i64 %inc16, %N
+;COM: br i1 %exitcond47, label %for.exit, label %for.cond1.preheader
+;COM:
+;COM: for.exit: ; preds = %for.cond1.for.cond
+;COM: ret i64 %spec.select
+;COM: }
+;COM:
+;COM: ; Make sure that store inside loop prevents hoisting invariant loads
+;COM: ; for (int i = 0; i < N; ++i)
+;COM: ; c[i] = memcmp(a[i], b, 4);
+;COM: ;
+;COM: define void @one_dimensional_with_store(ptr %a, ptr %b, ptr %c, i32 %N) {
+;COM: ; CHECK-LABEL: one_dimensional_with_store:
+;COM: ; CHECK: // %bb.0: // %entry
+;COM: ; CHECK-NEXT: mov w8, w3
+;COM: ; CHECK-NEXT: .LBB4_1: // %for.body
+;COM: ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+;COM: ; CHECK-NEXT: ldr x9, [x0], #8
+;COM: ; CHECK-NEXT: ldr w10, [x1]
+;COM: ; CHECK-NEXT: ldr w9, [x9]
+;COM: ; CHECK-NEXT: rev w10, w10
+;COM: ; CHECK-NEXT: rev w9, w9
+;COM: ; CHECK-NEXT: cmp w9, w10
+;COM: ; CHECK-NEXT: cset w9, hi
+;COM: ; CHECK-NEXT: cset w10, lo
+;COM: ; CHECK-NEXT: subs x8, x8, #1
+;COM: ; CHECK-NEXT: sub w9, w9, w10
+;COM: ; CHECK-NEXT: strb w9, [x2], #1
+;COM: ; CHECK-NEXT: b.ne .LBB4_1
+;COM: ; CHECK-NEXT: // %bb.2: // %for.exit
+;COM: ; CHECK-NEXT: ret
+;COM: entry:
+;COM: br label %for.body.preheader
+;COM:
+;COM: for.body.preheader: ; preds = %entry
+;COM: %wide.trip.count = zext i32 %N to i64
+;COM: br label %for.body
+;COM:
+;COM: for.body: ; preds = %for.body.preheader, %for.body
+;COM: %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+;COM: %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %indvars.iv
+;COM: %0 = load ptr, ptr %arrayidx, align 8
+;COM: %call = tail call i32 @memcmp(ptr %0, ptr %b, i64 4)
+;COM: %conv = trunc i32 %call to i8
+;COM: %arrayidx2 = getelementptr inbounds i8, ptr %c, i64 %indvars.iv
+;COM: store i8 %conv, ptr %arrayidx2, align 1
+;COM: %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+;COM: %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+;COM: br i1 %exitcond.not, label %for.exit, label %for.body
+;COM:
+;COM: for.exit: ; preds = %for.body
+;COM: ret void
+;COM: }
+;COM:
+;COM: ; Make sure that call inside loop prevents hoisting invariant loads
+;COM: ;
+;COM: define i32 @one_dimensional_with_call(ptr %a, ptr %b, i32 %N) {
+;COM: ; CHECK-LABEL: one_dimensional_with_call:
+;COM: ; CHECK: // %bb.0: // %entry
+;COM: ; CHECK-NEXT: str x30, [sp, #-48]! // 8-byte Folded Spill
+;COM: ; CHECK-NEXT: stp x22, x21, [sp, #16] // 16-byte Folded Spill
+;COM: ; CHECK-NEXT: stp x20, x19, [sp, #32] // 16-byte Folded Spill
+;COM: ; CHECK-NEXT: .cfi_def_cfa_offset 48
+;COM: ; CHECK-NEXT: .cfi_offset w19, -8
+;COM: ; CHECK-NEXT: .cfi_offset w20, -16
+;COM: ; CHECK-NEXT: .cfi_offset w21, -24
+;COM: ; CHECK-NEXT: .cfi_offset w22, -32
+;COM: ; CHECK-NEXT: .cfi_offset w30, -48
+;COM: ; CHECK-NEXT: mov x19, x1
+;COM: ; CHECK-NEXT: mov x21, x0
+;COM: ; CHECK-NEXT: mov w20, wzr
+;COM: ; CHECK-NEXT: mov w22, w2
+;COM: ; CHECK-NEXT: .LBB5_1: // %for.body
+;COM: ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+;COM: ; CHECK-NEXT: ldr x8, [x21], #8
+;COM: ; CHECK-NEXT: ldr w9, [x19]
+;COM: ; CHECK-NEXT: ldr w8, [x8]
+;COM: ; CHECK-NEXT: cmp w8, w9
+;COM: ; CHECK-NEXT: cinc w20, w20, eq
+;COM: ; CHECK-NEXT: bl func
+;COM: ; CHECK-NEXT: subs x22, x22, #1
+;COM: ; CHECK-NEXT: b.ne .LBB5_1
+;COM: ; CHECK-NEXT: // %bb.2: // %for.exit
+;COM: ; CHECK-NEXT: mov w0, w20
+;COM: ; CHECK-NEXT: ldp x20, x19, [sp, #32] // 16-byte Folded Reload
+;COM: ; CHECK-NEXT: ldp x22, x21, [sp, #16] // 16-byte Folded Reload
+;COM: ; CHECK-NEXT: ldr x30, [sp], #48 // 8-byte Folded Reload
+;COM: ; CHECK-NEXT: ret
+;COM: entry:
+;COM: br label %for.body.preheader
+;COM:
+;COM: for.body.preheader: ; preds = %entry
+;COM: %wide.trip.count = zext i32 %N to i64
+;COM: br label %for.body
+;COM:
+;COM: for.body: ; preds = %for.body.preheader, %for.body
+;COM: %indvars.iv = phi i64 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
+;COM: %sum.05 = phi i32 [ 0, %for.body.preheader ], [ %spec.select, %for.body ]
+;COM: %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %indvars.iv
+;COM: %0 = load ptr, ptr %arrayidx, align 8
+;COM: %bcmp = tail call i32 @bcmp(ptr %0, ptr %b, i64 4)
+;COM: %tobool.not = icmp eq i32 %bcmp, 0
+;COM: %add = zext i1 %tobool.not to i32
+;COM: %spec.select = add nuw nsw i32 %sum.05, %add
+;COM: tail call void @func()
+;COM: %indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
+;COM: %exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count
+;COM: br i1 %exitcond.not, label %for.exit, label %for.body
+;COM:
+;COM: for.exit: ; preds = %for.body
+;COM: ret i32 %spec.select
+;COM: }
+;COM:
+;COM: ; One dimensional loop with memcmp size equal six.
+;COM: ; The test shows that shows that several loads can be hoisted at the same time.
+;COM: ; for (int i = 0; i < N; ++i)
+;COM: ; if (!memcmp(a[i], b, 6))
+;COM: ; sum += 1;
+;COM: ;
+;COM: define i64 @one_dimensional_two_loads(ptr %a, ptr %b, i64 %N) {
+;COM: ; CHECK-LABEL: one_dimensional_two_loads:
+;COM: ; CHECK: // %bb.0: // %entry
+;COM: ; CHECK-NEXT: ldr w9, [x1]
+;COM: ; CHECK-NEXT: ldrh w10, [x1, #4]
+;COM: ; CHECK-NEXT: mov x8, xzr
+;COM: ; CHECK-NEXT: .LBB6_1: // %for.body
+;COM: ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+;COM: ; CHECK-NEXT: ldr x11, [x0], #8
+;COM: ; CHECK-NEXT: ldr w12, [x11]
+;COM: ; CHECK-NEXT: ldrh w11, [x11, #4]
+;COM: ; CHECK-NEXT: cmp w12, w9
+;COM: ; CHECK-NEXT: ccmp w11, w10, #0, eq
+;COM: ; CHECK-NEXT: cinc x8, x8, eq
+;COM: ; CHECK-NEXT: subs x2, x2, #1
+;COM: ; CHECK-NEXT: b.ne .LBB6_1
+;COM: ; CHECK-NEXT: // %bb.2: // %for.exit
+;COM: ; CHECK-NEXT: mov x0, x8
+;COM: ; CHECK-NEXT: ret
+;COM: entry:
+;COM: br label %for.body
+;COM:
+;COM: for.body: ; preds = %entry, %for.body
+;COM: %i.06 = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+;COM: %sum.05 = phi i64 [ %spec.select, %for.body ], [ 0, %entry ]
+;COM: %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %i.06
+;COM: %0 = load ptr, ptr %arrayidx, align 8
+;COM: %bcmp = tail call i32 @bcmp(ptr %0, ptr %b, i64 6)
+;COM: %tobool = icmp eq i32 %bcmp, 0
+;COM: %add = zext i1 %tobool to i64
+;COM: %spec.select = add i64 %sum.05, %add
+;COM: %inc = add nuw i64 %i.06, 1
+;COM: %exitcond = icmp eq i64 %inc, %N
+;COM: br i1 %exitcond, label %for.exit, label %for.body
+;COM:
+;COM: for.exit: ; preds = %for.body
+;COM: ret i64 %spec.select
+;COM: }
+;COM:
+;COM: ; See issue https://github.com/llvm/llvm-project/issues/72855
+;COM: ;
+;COM: ; When hoisting instruction out of the loop, ensure that loads are not common
+;COM: ; subexpressions eliminated. In this example pointer %c may alias pointer %b,
+;COM: ; so when hoisting `%y = load i64, ptr %b` instruction we can't replace it with
+;COM: ; `%b.val = load i64, ptr %b`
+;COM: ;
+;COM: define i64 @hoisting_no_cse(ptr %a, ptr %b, ptr %c, i64 %N) {
+;COM: ; CHECK-LABEL: hoisting_no_cse:
+;COM: ; CHECK: // %bb.0: // %entry
+;COM: ; CHECK-NEXT: ldr x8, [x1]
+;COM: ; CHECK-NEXT: add x8, x8, #1
+;COM: ; CHECK-NEXT: str x8, [x2]
+;COM: ; CHECK-NEXT: mov x8, xzr
+;COM: ; CHECK-NEXT: ldr x9, [x1]
+;COM: ; CHECK-NEXT: .LBB7_1: // %for.body
+;COM: ; CHECK-NEXT: // =>This Inner Loop Header: Depth=1
+;COM: ; CHECK-NEXT: ldr x10, [x0], #8
+;COM: ; CHECK-NEXT: ldr x10, [x10]
+;COM: ; CHECK-NEXT: cmp x10, x9
+;COM: ; CHECK-NEXT: cinc x8, x8, eq
+;COM: ; CHECK-NEXT: subs x3, x3, #1
+;COM: ; CHECK-NEXT: b.ne .LBB7_1
+;COM: ; CHECK-NEXT: // %bb.2: // %for.exit
+;COM: ; CHECK-NEXT: mov x0, x8
+;COM: ; CHECK-NEXT: ret
+;COM: entry:
+;COM: %b.val = load i64, ptr %b
+;COM: %b.val.changed = add i64 %b.val, 1
+;COM: store i64 %b.val.changed, ptr %c
+;COM: br label %for.body
+;COM:
+;COM: for.body: ; preds = %entry, %for.body
+;COM: %idx = phi i64 [ %inc, %for.body ], [ 0, %entry ]
+;COM: %sum = phi i64 [ %spec.select, %for.body ], [ 0, %entry ]
+;COM: %arrayidx = getelementptr inbounds ptr, ptr %a, i64 %idx
+;COM: %0 = load ptr, ptr %arrayidx, align 8
+;COM: %x = load i64, ptr %0
+;COM: %y = load i64, ptr %b
+;COM: %cmp = icmp eq i64 %x, %y
+;COM: %add = zext i1 %cmp to i64
+;COM: %spec.select = add i64 %sum, %add
+;COM: %inc = add nuw i64 %idx, 1
+;COM: %exitcond = icmp eq i64 %inc, %N
+;COM: br i1 %exitcond, label %for.exit, label %for.body
+;COM:
+;COM: for.exit: ; preds = %for.body
+;COM: ret i64 %spec.select
+;COM: }
+;COM:
declare i32 @bcmp(ptr, ptr, i64)
declare i32 @memcmp(ptr, ptr, i64)
declare void @func()
diff --git a/llvm/test/CodeGen/RISCV/machinelicm-invariant-load.mir b/llvm/test/CodeGen/RISCV/machinelicm-invariant-load.mir
new file mode 100644
index 0000000000000..e54ae162cca92
--- /dev/null
+++ b/llvm/test/CodeGen/RISCV/machinelicm-invariant-load.mir
@@ -0,0 +1,119 @@
+# NOTE: Assertions have been autogenerated by utils/update_mir_test_checks.py UTC_ARGS: --version 5
+# RUN: llc -mtriple=riscv64 -x mir -run-pass=machinelicm -simplify-mir -verify-machineinstrs < %s | FileCheck %s
+
+---
+name: invariant_load
+tracksRegLiveness: true
+constants:
+ - id: 0
+ value: i64 -1085102592571150096
+ alignment: 8
+ isTargetSpecific: false
+body: |
+ ; CHECK-LABEL: name: invariant_load
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: liveins: $x10, $x11
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %base_addr:gpr = COPY $x10
+ ; CHECK-NEXT: %n:gpr = COPY $x11
+ ; CHECK-NEXT: %i_0:gpr = COPY $x0
+ ; CHECK-NEXT: %const_pool_addr_upper:gpr = LUI target-flags(riscv-hi) %const.0
+ ; CHECK-NEXT: %invariant_load:gpr = LD %const_pool_addr_upper, target-flags(riscv-lo) %const.0 :: (load (s64) from constant-pool)
+ ; CHECK-NEXT: PseudoBR %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: %offset:gpr = PHI %i_inc, %bb.1, %i_0, %bb.0
+ ; CHECK-NEXT: %addr:gpr = ADD %base_addr, %offset
+ ; CHECK-NEXT: %x:gpr = LW %addr, 0
+ ; CHECK-NEXT: %val:gpr = ADD %x, %invariant_load
+ ; CHECK-NEXT: SW %val, %addr, 0
+ ; CHECK-NEXT: %i_inc:gpr = ADDI %offset, 4
+ ; CHECK-NEXT: BLT %i_inc, %n, %bb.1
+ ; CHECK-NEXT: PseudoBR %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1
+ liveins: $x10, $x11
+
+ %base_addr:gpr = COPY $x10
+ %n:gpr = COPY $x11
+ %i_0:gpr = COPY $x0
+ PseudoBR %bb.1
+
+ bb.1:
+ %offset:gpr = PHI %i_inc, %bb.1, %i_0, %bb.0
+ %addr:gpr = ADD %base_addr, %offset
+ %x:gpr = LW %addr, 0
+
+ %const_pool_addr_upper:gpr = LUI target-flags(riscv-hi) %const.0
+ %invariant_load:gpr = LD %const_pool_addr_upper, target-flags(riscv-lo) %const.0 :: (load (s64) from constant-pool)
+ %val:gpr = ADD %x, %invariant_load
+ SW %val, %addr, 0
+ %i_inc:gpr = ADDI %offset, 4
+ BLT %i_inc, %n, %bb.1
+ PseudoBR %bb.2
+
+ bb.2:
+ PseudoRET
+...
+
+# Do not hoist load out of the loop, because %other_addr may alias with %addr
+---
+name: not_invariant_load
+tracksRegLiveness: true
+constants:
+ - id: 0
+ value: i64 -1085102592571150096
+ alignment: 8
+ isTargetSpecific: false
+body: |
+ ; CHECK-LABEL: name: not_invariant_load
+ ; CHECK: bb.0:
+ ; CHECK-NEXT: liveins: $x10, $x11, $x12
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: %base_addr:gpr = COPY $x10
+ ; CHECK-NEXT: %n:gpr = COPY $x11
+ ; CHECK-NEXT: %other_addr:gpr = COPY $x12
+ ; CHECK-NEXT: %i_0:gpr = COPY $x0
+ ; CHECK-NEXT: PseudoBR %bb.1
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.1:
+ ; CHECK-NEXT: %offset:gpr = PHI %i_inc, %bb.1, %i_0, %bb.0
+ ; CHECK-NEXT: %addr:gpr = ADD %base_addr, %offset
+ ; CHECK-NEXT: %x:gpr = LW %addr, 0
+ ; CHECK-NEXT: %load_may_alias:gpr = LD %other_addr, 0
+ ; CHECK-NEXT: %val:gpr = ADD %x, %load_may_alias
+ ; CHECK-NEXT: SW %val, %addr, 0
+ ; CHECK-NEXT: %i_inc:gpr = ADDI %offset, 4
+ ; CHECK-NEXT: BLT %i_inc, %n, %bb.1
+ ; CHECK-NEXT: PseudoBR %bb.2
+ ; CHECK-NEXT: {{ $}}
+ ; CHECK-NEXT: bb.2:
+ ; CHECK-NEXT: PseudoRET
+ bb.0:
+ successors: %bb.1
+ liveins: $x10, $x11, $x12
+
+ %base_addr:gpr = COPY $x10
+ %n:gpr = COPY $x11
+ %other_addr:gpr = COPY $x12
+ %i_0:gpr = COPY $x0
+ PseudoBR %bb.1
+
+ bb.1:
+ %offset:gpr = PHI %i_inc, %bb.1, %i_0, %bb.0
+ %addr:gpr = ADD %base_addr, %offset
+ %x:gpr = LW %addr, 0
+
+ %load_may_alias:gpr = LD %other_addr, 0
+ %val:gpr = ADD %x, %load_may_alias
+ SW %val, %addr, 0
+ %i_inc:gpr = ADDI %offset, 4
+ BLT %i_inc, %n, %bb.1
+ PseudoBR %bb.2
+
+ bb.2:
+ PseudoRET
+...
More information about the llvm-commits
mailing list