[llvm] r259790 - [AArch64] Improve load/store optimizer to handle LDUR + LDR.
Chad Rosier via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 4 06:42:56 PST 2016
Author: mcrosier
Date: Thu Feb 4 08:42:55 2016
New Revision: 259790
URL: http://llvm.org/viewvc/llvm-project?rev=259790&view=rev
Log:
[AArch64] Improve load/store optimizer to handle LDUR + LDR.
This patch allows the mixing of scaled and unscaled load/stores to form
load/store pairs.
PR24465
http://reviews.llvm.org/D12116
Many thanks to Ahmed and Michael for fixes and code review.
This is a reapplication of r246769, which was reverted in r246782 due to a
test-suite failure. I'm unable to reproduce the issue at this time.
Added:
llvm/trunk/test/CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll
Modified:
llvm/trunk/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
Modified: llvm/trunk/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp?rev=259790&r1=259789&r2=259790&view=diff
==============================================================================
--- llvm/trunk/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp (original)
+++ llvm/trunk/lib/Target/AArch64/AArch64LoadStoreOptimizer.cpp Thu Feb 4 08:42:55 2016
@@ -645,10 +645,24 @@ AArch64LoadStoreOpt::mergePairedInsns(Ma
const MachineOperand &BaseRegOp =
MergeForward ? getLdStBaseOp(Paired) : getLdStBaseOp(I);
+ int Offset = getLdStOffsetOp(I).getImm();
+ int PairedOffset = getLdStOffsetOp(Paired).getImm();
+ bool PairedIsUnscaled = isUnscaledLdSt(Paired->getOpcode());
+
+ // We're trying to pair instructions that differ in how they are scaled.
+ // If I is scaled then scale the offset of Paired accordingly.
+ // Otherwise, do the opposite (i.e., make Paired's offset unscaled).
+ if (IsUnscaled != PairedIsUnscaled) {
+ int MemSize = getMemScale(Paired);
+ assert(!(PairedOffset % getMemScale(Paired)) &&
+ "Offset should be a multiple of the stride!");
+ PairedOffset =
+ PairedIsUnscaled ? PairedOffset / MemSize : PairedOffset * MemSize;
+ }
+
// Which register is Rt and which is Rt2 depends on the offset order.
MachineInstr *RtMI, *Rt2MI;
- if (getLdStOffsetOp(I).getImm() ==
- getLdStOffsetOp(Paired).getImm() + OffsetStride) {
+ if (Offset == PairedOffset + OffsetStride) {
RtMI = Paired;
Rt2MI = I;
// Here we swapped the assumption made for SExtIdx.
@@ -775,9 +789,12 @@ AArch64LoadStoreOpt::mergePairedInsns(Ma
.addImm(OffsetImm)
.setMemRefs(I->mergeMemRefsWith(*Paired));
} else {
- // Handle Unscaled
- if (IsUnscaled)
- OffsetImm /= OffsetStride;
+ // Scale the immediate offset, if necessary.
+ if (isUnscaledLdSt(RtMI->getOpcode())) {
+ assert(!(OffsetImm % getMemScale(RtMI)) &&
+ "Offset should be a multiple of the stride!");
+ OffsetImm /= getMemScale(RtMI);
+ }
MIB = BuildMI(*I->getParent(), InsertionPoint, I->getDebugLoc(),
TII->get(NewOpc))
.addOperand(getLdStRegOp(RtMI))
@@ -971,9 +988,13 @@ static void trackRegDefsUses(const Machi
static bool inBoundsForPair(bool IsUnscaled, int Offset, int OffsetStride) {
// Convert the byte-offset used by unscaled into an "element" offset used
// by the scaled pair load/store instructions.
- if (IsUnscaled)
+ if (IsUnscaled) {
+ // If the byte-offset isn't a multiple of the stride, there's no point
+ // trying to match it.
+ if (Offset % OffsetStride)
+ return false;
Offset /= OffsetStride;
-
+ }
return Offset <= 63 && Offset >= -64;
}
@@ -1063,6 +1084,34 @@ bool AArch64LoadStoreOpt::findMatchingSt
return false;
}
+
+static bool canMergeOpc(unsigned Opc, unsigned PairOpc, LdStPairFlags &Flags) {
+ // Opcodes match: nothing more to check.
+ if (Opc == PairOpc)
+ return true;
+
+ // Try to match a sign-extended load/store with a zero-extended load/store.
+ Flags.setSExtIdx(-1);
+ bool IsValidLdStrOpc, PairIsValidLdStrOpc;
+ unsigned NonSExtOpc = getMatchingNonSExtOpcode(Opc, &IsValidLdStrOpc);
+ assert(IsValidLdStrOpc &&
+ "Given Opc should be a Load or Store with an immediate");
+ // Opc will be the first instruction in the pair.
+ if (NonSExtOpc == getMatchingNonSExtOpcode(PairOpc, &PairIsValidLdStrOpc)) {
+ Flags.setSExtIdx(NonSExtOpc == (unsigned)Opc ? 1 : 0);
+ return true;
+ }
+
+ // FIXME: Can we also match a mixed sext/zext unscaled/scaled pair?
+
+ // If the second instruction isn't even a load/store, bail out.
+ if (!PairIsValidLdStrOpc)
+ return false;
+
+ // Try to match an unscaled load/store with a scaled load/store.
+ return isUnscaledLdSt(Opc) != isUnscaledLdSt(PairOpc) &&
+ getMatchingPairOpcode(Opc) == getMatchingPairOpcode(PairOpc);
+}
/// findMatchingInsn - Scan the instructions looking for a load/store that can
/// be combined with the current instruction into a load/store pair.
MachineBasicBlock::iterator
@@ -1116,19 +1165,8 @@ AArch64LoadStoreOpt::findMatchingInsn(Ma
// Now that we know this is a real instruction, count it.
++Count;
- bool CanMergeOpc = Opc == MI->getOpcode();
- Flags.setSExtIdx(-1);
- if (!CanMergeOpc) {
- bool IsValidLdStrOpc;
- unsigned NonSExtOpc = getMatchingNonSExtOpcode(Opc, &IsValidLdStrOpc);
- assert(IsValidLdStrOpc &&
- "Given Opc should be a Load or Store with an immediate");
- // Opc will be the first instruction in the pair.
- Flags.setSExtIdx(NonSExtOpc == (unsigned)Opc ? 1 : 0);
- CanMergeOpc = NonSExtOpc == getMatchingNonSExtOpcode(MI->getOpcode());
- }
-
- if (CanMergeOpc && getLdStOffsetOp(MI).isImm()) {
+ if (canMergeOpc(Opc, MI->getOpcode(), Flags) &&
+ getLdStOffsetOp(MI).isImm()) {
assert(MI->mayLoadOrStore() && "Expected memory operation.");
// If we've found another instruction with the same opcode, check to see
// if the base and offset are compatible with our starting instruction.
@@ -1142,6 +1180,24 @@ AArch64LoadStoreOpt::findMatchingInsn(Ma
// final offset must be in range.
unsigned MIBaseReg = getLdStBaseOp(MI).getReg();
int MIOffset = getLdStOffsetOp(MI).getImm();
+
+ // We're trying to pair instructions that differ in how they are scaled.
+ // If FirstMI is scaled then scale the offset of MI accordingly.
+ // Otherwise, do the opposite (i.e., make MI's offset unscaled).
+ bool MIIsUnscaled = isUnscaledLdSt(MI);
+ if (IsUnscaled != MIIsUnscaled) {
+ int MemSize = getMemScale(MI);
+ if (MIIsUnscaled) {
+ // If the unscaled offset isn't a multiple of the MemSize, we can't
+ // pair the operations together: bail and keep looking.
+ if (MIOffset % MemSize)
+ continue;
+ MIOffset /= MemSize;
+ } else {
+ MIOffset *= MemSize;
+ }
+ }
+
if (BaseReg == MIBaseReg && ((Offset == MIOffset + OffsetStride) ||
(Offset + OffsetStride == MIOffset))) {
int MinOffset = Offset < MIOffset ? Offset : MIOffset;
@@ -1152,10 +1208,9 @@ AArch64LoadStoreOpt::findMatchingInsn(Ma
return E;
// If the resultant immediate offset of merging these instructions
// is out of range for a pairwise instruction, bail and keep looking.
- bool MIIsUnscaled = isUnscaledLdSt(MI);
bool IsNarrowLoad = isNarrowLoad(MI->getOpcode());
if (!IsNarrowLoad &&
- !inBoundsForPair(MIIsUnscaled, MinOffset, OffsetStride)) {
+ !inBoundsForPair(IsUnscaled, MinOffset, OffsetStride)) {
trackRegDefsUses(MI, ModifiedRegs, UsedRegs, TRI);
MemInsns.push_back(MI);
continue;
Added: llvm/trunk/test/CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll?rev=259790&view=auto
==============================================================================
--- llvm/trunk/test/CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll (added)
+++ llvm/trunk/test/CodeGen/AArch64/ldp-stp-scaled-unscaled-pairs.ll Thu Feb 4 08:42:55 2016
@@ -0,0 +1,105 @@
+; RUN: llc < %s -march=aarch64 -aarch64-neon-syntax=apple -aarch64-stp-suppress=false -verify-machineinstrs -asm-verbose=false | FileCheck %s
+
+; CHECK-LABEL: test_strd_sturd:
+; CHECK-NEXT: stp d0, d1, [x0, #-8]
+; CHECK-NEXT: ret
+define void @test_strd_sturd(float* %ptr, <2 x float> %v1, <2 x float> %v2) #0 {
+ %tmp1 = bitcast float* %ptr to <2 x float>*
+ store <2 x float> %v2, <2 x float>* %tmp1, align 16
+ %add.ptr = getelementptr inbounds float, float* %ptr, i64 -2
+ %tmp = bitcast float* %add.ptr to <2 x float>*
+ store <2 x float> %v1, <2 x float>* %tmp, align 16
+ ret void
+}
+
+; CHECK-LABEL: test_sturd_strd:
+; CHECK-NEXT: stp d0, d1, [x0, #-8]
+; CHECK-NEXT: ret
+define void @test_sturd_strd(float* %ptr, <2 x float> %v1, <2 x float> %v2) #0 {
+ %add.ptr = getelementptr inbounds float, float* %ptr, i64 -2
+ %tmp = bitcast float* %add.ptr to <2 x float>*
+ store <2 x float> %v1, <2 x float>* %tmp, align 16
+ %tmp1 = bitcast float* %ptr to <2 x float>*
+ store <2 x float> %v2, <2 x float>* %tmp1, align 16
+ ret void
+}
+
+; CHECK-LABEL: test_strq_sturq:
+; CHECK-NEXT: stp q0, q1, [x0, #-16]
+; CHECK-NEXT: ret
+define void @test_strq_sturq(double* %ptr, <2 x double> %v1, <2 x double> %v2) #0 {
+ %tmp1 = bitcast double* %ptr to <2 x double>*
+ store <2 x double> %v2, <2 x double>* %tmp1, align 16
+ %add.ptr = getelementptr inbounds double, double* %ptr, i64 -2
+ %tmp = bitcast double* %add.ptr to <2 x double>*
+ store <2 x double> %v1, <2 x double>* %tmp, align 16
+ ret void
+}
+
+; CHECK-LABEL: test_sturq_strq:
+; CHECK-NEXT: stp q0, q1, [x0, #-16]
+; CHECK-NEXT: ret
+define void @test_sturq_strq(double* %ptr, <2 x double> %v1, <2 x double> %v2) #0 {
+ %add.ptr = getelementptr inbounds double, double* %ptr, i64 -2
+ %tmp = bitcast double* %add.ptr to <2 x double>*
+ store <2 x double> %v1, <2 x double>* %tmp, align 16
+ %tmp1 = bitcast double* %ptr to <2 x double>*
+ store <2 x double> %v2, <2 x double>* %tmp1, align 16
+ ret void
+}
+
+; CHECK-LABEL: test_ldrx_ldurx:
+; CHECK-NEXT: ldp [[V0:x[0-9]+]], [[V1:x[0-9]+]], [x0, #-8]
+; CHECK-NEXT: add x0, [[V0]], [[V1]]
+; CHECK-NEXT: ret
+define i64 @test_ldrx_ldurx(i64* %p) #0 {
+ %tmp = load i64, i64* %p, align 4
+ %add.ptr = getelementptr inbounds i64, i64* %p, i64 -1
+ %tmp1 = load i64, i64* %add.ptr, align 4
+ %add = add nsw i64 %tmp1, %tmp
+ ret i64 %add
+}
+
+; CHECK-LABEL: test_ldurx_ldrx:
+; CHECK-NEXT: ldp [[V0:x[0-9]+]], [[V1:x[0-9]+]], [x0, #-8]
+; CHECK-NEXT: add x0, [[V0]], [[V1]]
+; CHECK-NEXT: ret
+define i64 @test_ldurx_ldrx(i64* %p) #0 {
+ %add.ptr = getelementptr inbounds i64, i64* %p, i64 -1
+ %tmp1 = load i64, i64* %add.ptr, align 4
+ %tmp = load i64, i64* %p, align 4
+ %add = add nsw i64 %tmp1, %tmp
+ ret i64 %add
+}
+
+; CHECK-LABEL: test_ldrsw_ldursw:
+; CHECK-NEXT: ldpsw [[V0:x[0-9]+]], [[V1:x[0-9]+]], [x0, #-4]
+; CHECK-NEXT: add x0, [[V0]], [[V1]]
+; CHECK-NEXT: ret
+define i64 @test_ldrsw_ldursw(i32* %p) #0 {
+ %tmp = load i32, i32* %p, align 4
+ %add.ptr = getelementptr inbounds i32, i32* %p, i64 -1
+ %tmp1 = load i32, i32* %add.ptr, align 4
+ %sexttmp = sext i32 %tmp to i64
+ %sexttmp1 = sext i32 %tmp1 to i64
+ %add = add nsw i64 %sexttmp1, %sexttmp
+ ret i64 %add
+}
+
+; Also make sure we only match valid offsets.
+; CHECK-LABEL: test_ldrq_ldruq_invalidoffset:
+; CHECK-NEXT: ldr q[[V0:[0-9]+]], [x0]
+; CHECK-NEXT: ldur q[[V1:[0-9]+]], [x0, #24]
+; CHECK-NEXT: add.2d v0, v[[V0]], v[[V1]]
+; CHECK-NEXT: ret
+define <2 x i64> @test_ldrq_ldruq_invalidoffset(i64* %p) nounwind {
+ %a1 = bitcast i64* %p to <2 x i64>*
+ %tmp1 = load <2 x i64>, < 2 x i64>* %a1, align 8
+ %add.ptr2 = getelementptr inbounds i64, i64* %p, i64 3
+ %a2 = bitcast i64* %add.ptr2 to <2 x i64>*
+ %tmp2 = load <2 x i64>, <2 x i64>* %a2, align 8
+ %add = add nsw <2 x i64> %tmp1, %tmp2
+ ret <2 x i64> %add
+}
+
+attributes #0 = { nounwind }
More information about the llvm-commits
mailing list