[llvm] r328973 - [X86] Reduce Store Forward Block issues in HW - Recommit after fixing Bug 36346
Lama Saba via llvm-commits
llvm-commits at lists.llvm.org
Mon Apr 2 06:48:28 PDT 2018
Author: lsaba
Date: Mon Apr 2 06:48:28 2018
New Revision: 328973
URL: http://llvm.org/viewvc/llvm-project?rev=328973&view=rev
Log:
[X86] Reduce Store Forward Block issues in HW - Recommit after fixing Bug 36346
If a load follows a store and reloads data that the store has written to memory, Intel microarchitectures can in many cases forward the data directly from the store to the load, This "store forwarding" saves cycles by enabling the load to directly obtain the data instead of accessing the data from cache or memory.
A "store forward block" occurs in cases that a store cannot be forwarded to the load. The most typical case of store forward block on Intel Core microarchiticutre that a small store cannot be forwarded to a large load.
The estimated penalty for a store forward block is ~13 cycles.
This pass tries to recognize and handle cases where "store forward block" is created by the compiler when lowering memcpy calls to a sequence
of a load and a store.
The pass currently only handles cases where memcpy is lowered to XMM/YMM registers, it tries to break the memcpy into smaller copies.
breaking the memcpy should be possible since there is no atomicity guarantee for loads and stores to XMM/YMM.
Differential revision: https://reviews.llvm.org/D41330
Change-Id: Ib48836ccdf6005989f7d4466fa2035b7b04415d9
Added:
llvm/trunk/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
llvm/trunk/test/CodeGen/X86/avoid-sfb-overlaps.ll
llvm/trunk/test/CodeGen/X86/avoid-sfb.ll
Modified:
llvm/trunk/lib/Target/X86/CMakeLists.txt
llvm/trunk/lib/Target/X86/X86.h
llvm/trunk/lib/Target/X86/X86TargetMachine.cpp
llvm/trunk/test/CodeGen/X86/O3-pipeline.ll
Modified: llvm/trunk/lib/Target/X86/CMakeLists.txt
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/CMakeLists.txt?rev=328973&r1=328972&r2=328973&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/CMakeLists.txt (original)
+++ llvm/trunk/lib/Target/X86/CMakeLists.txt Mon Apr 2 06:48:28 2018
@@ -31,6 +31,7 @@ set(sources
X86FastISel.cpp
X86FixupBWInsts.cpp
X86FixupLEAs.cpp
+ X86AvoidStoreForwardingBlocks.cpp
X86FixupSetCC.cpp
X86FloatingPoint.cpp
X86FrameLowering.cpp
Modified: llvm/trunk/lib/Target/X86/X86.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86.h?rev=328973&r1=328972&r2=328973&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86.h (original)
+++ llvm/trunk/lib/Target/X86/X86.h Mon Apr 2 06:48:28 2018
@@ -70,6 +70,9 @@ FunctionPass *createX86OptimizeLEAs();
/// Return a pass that transforms setcc + movzx pairs into xor + setcc.
FunctionPass *createX86FixupSetCC();
+/// Return a pass that avoids creating store forward block issues in the hardware.
+FunctionPass *createX86AvoidStoreForwardingBlocks();
+
/// Return a pass that expands WinAlloca pseudo-instructions.
FunctionPass *createX86WinAllocaExpander();
Added: llvm/trunk/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp?rev=328973&view=auto
==============================================================================
--- llvm/trunk/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp (added)
+++ llvm/trunk/lib/Target/X86/X86AvoidStoreForwardingBlocks.cpp Mon Apr 2 06:48:28 2018
@@ -0,0 +1,722 @@
+//===- X86AvoidStoreForwardingBlockis.cpp - Avoid HW Store Forward Block --===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// If a load follows a store and reloads data that the store has written to
+// memory, Intel microarchitectures can in many cases forward the data directly
+// from the store to the load, This "store forwarding" saves cycles by enabling
+// the load to directly obtain the data instead of accessing the data from
+// cache or memory.
+// A "store forward block" occurs in cases that a store cannot be forwarded to
+// the load. The most typical case of store forward block on Intel Core
+// microarchitecture that a small store cannot be forwarded to a large load.
+// The estimated penalty for a store forward block is ~13 cycles.
+//
+// This pass tries to recognize and handle cases where "store forward block"
+// is created by the compiler when lowering memcpy calls to a sequence
+// of a load and a store.
+//
+// The pass currently only handles cases where memcpy is lowered to
+// XMM/YMM registers, it tries to break the memcpy into smaller copies.
+// breaking the memcpy should be possible since there is no atomicity
+// guarantee for loads and stores to XMM/YMM.
+//
+// It could be better for performance to solve the problem by loading
+// to XMM/YMM then inserting the partial store before storing back from XMM/YMM
+// to memory, but this will result in a more conservative optimization since it
+// requires we prove that all memory accesses between the blocking store and the
+// load must alias/don't alias before we can move the store, whereas the
+// transformation done here is correct regardless to other memory accesses.
+//===----------------------------------------------------------------------===//
+
+#include "X86InstrInfo.h"
+#include "X86Subtarget.h"
+#include "llvm/CodeGen/MachineBasicBlock.h"
+#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/CodeGen/MachineFunctionPass.h"
+#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
+#include "llvm/CodeGen/MachineOperand.h"
+#include "llvm/CodeGen/MachineRegisterInfo.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/DebugLoc.h"
+#include "llvm/IR/Function.h"
+#include "llvm/MC/MCInstrDesc.h"
+
+using namespace llvm;
+
+#define DEBUG_TYPE "x86-avoid-SFB"
+
+namespace llvm {
+void initializeX86AvoidSFBPassPass(PassRegistry &);
+} // end namespace llvm
+
+static cl::opt<bool> DisableX86AvoidStoreForwardBlocks(
+ "x86-disable-avoid-SFB", cl::Hidden,
+ cl::desc("X86: Disable Store Forwarding Blocks fixup."), cl::init(false));
+
+static cl::opt<unsigned> X86AvoidSFBInspectionLimit(
+ "x86-sfb-inspection-limit",
+ cl::desc("X86: Number of instructions backward to "
+ "inspect for store forwarding blocks."),
+ cl::init(20), cl::Hidden);
+
+namespace {
+
+using DisplacementSizeMap = std::map<int64_t, unsigned>;
+
+class X86AvoidSFBPass : public MachineFunctionPass {
+public:
+ static char ID;
+ X86AvoidSFBPass() : MachineFunctionPass(ID) {
+ initializeX86AvoidSFBPassPass(*PassRegistry::getPassRegistry());
+ }
+
+ StringRef getPassName() const override {
+ return "X86 Avoid Store Forwarding Blocks";
+ }
+
+ bool runOnMachineFunction(MachineFunction &MF) override;
+
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ MachineFunctionPass::getAnalysisUsage(AU);
+ AU.addRequired<AAResultsWrapperPass>();
+ }
+
+private:
+ MachineRegisterInfo *MRI;
+ const X86InstrInfo *TII;
+ const X86RegisterInfo *TRI;
+ SmallVector<std::pair<MachineInstr *, MachineInstr *>, 2>
+ BlockedLoadsStoresPairs;
+ SmallVector<MachineInstr *, 2> ForRemoval;
+ AliasAnalysis *AA;
+
+ /// \brief Returns couples of Load then Store to memory which look
+ /// like a memcpy.
+ void findPotentiallylBlockedCopies(MachineFunction &MF);
+ /// \brief Break the memcpy's load and store into smaller copies
+ /// such that each memory load that was blocked by a smaller store
+ /// would now be copied separately.
+ void breakBlockedCopies(MachineInstr *LoadInst, MachineInstr *StoreInst,
+ const DisplacementSizeMap &BlockingStoresDispSizeMap);
+ /// \brief Break a copy of size Size to smaller copies.
+ void buildCopies(int Size, MachineInstr *LoadInst, int64_t LdDispImm,
+ MachineInstr *StoreInst, int64_t StDispImm,
+ int64_t LMMOffset, int64_t SMMOffset);
+
+ void buildCopy(MachineInstr *LoadInst, unsigned NLoadOpcode, int64_t LoadDisp,
+ MachineInstr *StoreInst, unsigned NStoreOpcode,
+ int64_t StoreDisp, unsigned Size, int64_t LMMOffset,
+ int64_t SMMOffset);
+
+ bool alias(const MachineMemOperand &Op1, const MachineMemOperand &Op2) const;
+
+ unsigned getRegSizeInBytes(MachineInstr *Inst);
+};
+
+} // end anonymous namespace
+
+char X86AvoidSFBPass::ID = 0;
+
+INITIALIZE_PASS_BEGIN(X86AvoidSFBPass, DEBUG_TYPE, "Machine code sinking",
+ false, false)
+INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass)
+INITIALIZE_PASS_END(X86AvoidSFBPass, DEBUG_TYPE, "Machine code sinking", false,
+ false)
+
+FunctionPass *llvm::createX86AvoidStoreForwardingBlocks() {
+ return new X86AvoidSFBPass();
+}
+
+static bool isXMMLoadOpcode(unsigned Opcode) {
+ return Opcode == X86::MOVUPSrm || Opcode == X86::MOVAPSrm ||
+ Opcode == X86::VMOVUPSrm || Opcode == X86::VMOVAPSrm ||
+ Opcode == X86::VMOVUPDrm || Opcode == X86::VMOVAPDrm ||
+ Opcode == X86::VMOVDQUrm || Opcode == X86::VMOVDQArm ||
+ Opcode == X86::VMOVUPSZ128rm || Opcode == X86::VMOVAPSZ128rm ||
+ Opcode == X86::VMOVUPDZ128rm || Opcode == X86::VMOVAPDZ128rm ||
+ Opcode == X86::VMOVDQU64Z128rm || Opcode == X86::VMOVDQA64Z128rm ||
+ Opcode == X86::VMOVDQU32Z128rm || Opcode == X86::VMOVDQA32Z128rm;
+}
+static bool isYMMLoadOpcode(unsigned Opcode) {
+ return Opcode == X86::VMOVUPSYrm || Opcode == X86::VMOVAPSYrm ||
+ Opcode == X86::VMOVUPDYrm || Opcode == X86::VMOVAPDYrm ||
+ Opcode == X86::VMOVDQUYrm || Opcode == X86::VMOVDQAYrm ||
+ Opcode == X86::VMOVUPSZ256rm || Opcode == X86::VMOVAPSZ256rm ||
+ Opcode == X86::VMOVUPDZ256rm || Opcode == X86::VMOVAPDZ256rm ||
+ Opcode == X86::VMOVDQU64Z256rm || Opcode == X86::VMOVDQA64Z256rm ||
+ Opcode == X86::VMOVDQU32Z256rm || Opcode == X86::VMOVDQA32Z256rm;
+}
+
+static bool isPotentialBlockedMemCpyLd(unsigned Opcode) {
+ return isXMMLoadOpcode(Opcode) || isYMMLoadOpcode(Opcode);
+}
+
+static bool isPotentialBlockedMemCpyPair(int LdOpcode, int StOpcode) {
+ switch (LdOpcode) {
+ case X86::MOVUPSrm:
+ case X86::MOVAPSrm:
+ return StOpcode == X86::MOVUPSmr || StOpcode == X86::MOVAPSmr;
+ case X86::VMOVUPSrm:
+ case X86::VMOVAPSrm:
+ return StOpcode == X86::VMOVUPSmr || StOpcode == X86::VMOVAPSmr;
+ case X86::VMOVUPDrm:
+ case X86::VMOVAPDrm:
+ return StOpcode == X86::VMOVUPDmr || StOpcode == X86::VMOVAPDmr;
+ case X86::VMOVDQUrm:
+ case X86::VMOVDQArm:
+ return StOpcode == X86::VMOVDQUmr || StOpcode == X86::VMOVDQAmr;
+ case X86::VMOVUPSZ128rm:
+ case X86::VMOVAPSZ128rm:
+ return StOpcode == X86::VMOVUPSZ128mr || StOpcode == X86::VMOVAPSZ128mr;
+ case X86::VMOVUPDZ128rm:
+ case X86::VMOVAPDZ128rm:
+ return StOpcode == X86::VMOVUPDZ128mr || StOpcode == X86::VMOVAPDZ128mr;
+ case X86::VMOVUPSYrm:
+ case X86::VMOVAPSYrm:
+ return StOpcode == X86::VMOVUPSYmr || StOpcode == X86::VMOVAPSYmr;
+ case X86::VMOVUPDYrm:
+ case X86::VMOVAPDYrm:
+ return StOpcode == X86::VMOVUPDYmr || StOpcode == X86::VMOVAPDYmr;
+ case X86::VMOVDQUYrm:
+ case X86::VMOVDQAYrm:
+ return StOpcode == X86::VMOVDQUYmr || StOpcode == X86::VMOVDQAYmr;
+ case X86::VMOVUPSZ256rm:
+ case X86::VMOVAPSZ256rm:
+ return StOpcode == X86::VMOVUPSZ256mr || StOpcode == X86::VMOVAPSZ256mr;
+ case X86::VMOVUPDZ256rm:
+ case X86::VMOVAPDZ256rm:
+ return StOpcode == X86::VMOVUPDZ256mr || StOpcode == X86::VMOVAPDZ256mr;
+ case X86::VMOVDQU64Z128rm:
+ case X86::VMOVDQA64Z128rm:
+ return StOpcode == X86::VMOVDQU64Z128mr || StOpcode == X86::VMOVDQA64Z128mr;
+ case X86::VMOVDQU32Z128rm:
+ case X86::VMOVDQA32Z128rm:
+ return StOpcode == X86::VMOVDQU32Z128mr || StOpcode == X86::VMOVDQA32Z128mr;
+ case X86::VMOVDQU64Z256rm:
+ case X86::VMOVDQA64Z256rm:
+ return StOpcode == X86::VMOVDQU64Z256mr || StOpcode == X86::VMOVDQA64Z256mr;
+ case X86::VMOVDQU32Z256rm:
+ case X86::VMOVDQA32Z256rm:
+ return StOpcode == X86::VMOVDQU32Z256mr || StOpcode == X86::VMOVDQA32Z256mr;
+ default:
+ return false;
+ }
+}
+
+static bool isPotentialBlockingStoreInst(int Opcode, int LoadOpcode) {
+ bool PBlock = false;
+ PBlock |= Opcode == X86::MOV64mr || Opcode == X86::MOV64mi32 ||
+ Opcode == X86::MOV32mr || Opcode == X86::MOV32mi ||
+ Opcode == X86::MOV16mr || Opcode == X86::MOV16mi ||
+ Opcode == X86::MOV8mr || Opcode == X86::MOV8mi;
+ if (isYMMLoadOpcode(LoadOpcode))
+ PBlock |= Opcode == X86::VMOVUPSmr || Opcode == X86::VMOVAPSmr ||
+ Opcode == X86::VMOVUPDmr || Opcode == X86::VMOVAPDmr ||
+ Opcode == X86::VMOVDQUmr || Opcode == X86::VMOVDQAmr ||
+ Opcode == X86::VMOVUPSZ128mr || Opcode == X86::VMOVAPSZ128mr ||
+ Opcode == X86::VMOVUPDZ128mr || Opcode == X86::VMOVAPDZ128mr ||
+ Opcode == X86::VMOVDQU64Z128mr ||
+ Opcode == X86::VMOVDQA64Z128mr ||
+ Opcode == X86::VMOVDQU32Z128mr || Opcode == X86::VMOVDQA32Z128mr;
+ return PBlock;
+}
+
+static const int MOV128SZ = 16;
+static const int MOV64SZ = 8;
+static const int MOV32SZ = 4;
+static const int MOV16SZ = 2;
+static const int MOV8SZ = 1;
+
+static unsigned getYMMtoXMMLoadOpcode(unsigned LoadOpcode) {
+ switch (LoadOpcode) {
+ case X86::VMOVUPSYrm:
+ case X86::VMOVAPSYrm:
+ return X86::VMOVUPSrm;
+ case X86::VMOVUPDYrm:
+ case X86::VMOVAPDYrm:
+ return X86::VMOVUPDrm;
+ case X86::VMOVDQUYrm:
+ case X86::VMOVDQAYrm:
+ return X86::VMOVDQUrm;
+ case X86::VMOVUPSZ256rm:
+ case X86::VMOVAPSZ256rm:
+ return X86::VMOVUPSZ128rm;
+ case X86::VMOVUPDZ256rm:
+ case X86::VMOVAPDZ256rm:
+ return X86::VMOVUPDZ128rm;
+ case X86::VMOVDQU64Z256rm:
+ case X86::VMOVDQA64Z256rm:
+ return X86::VMOVDQU64Z128rm;
+ case X86::VMOVDQU32Z256rm:
+ case X86::VMOVDQA32Z256rm:
+ return X86::VMOVDQU32Z128rm;
+ default:
+ llvm_unreachable("Unexpected Load Instruction Opcode");
+ }
+ return 0;
+}
+
+static unsigned getYMMtoXMMStoreOpcode(unsigned StoreOpcode) {
+ switch (StoreOpcode) {
+ case X86::VMOVUPSYmr:
+ case X86::VMOVAPSYmr:
+ return X86::VMOVUPSmr;
+ case X86::VMOVUPDYmr:
+ case X86::VMOVAPDYmr:
+ return X86::VMOVUPDmr;
+ case X86::VMOVDQUYmr:
+ case X86::VMOVDQAYmr:
+ return X86::VMOVDQUmr;
+ case X86::VMOVUPSZ256mr:
+ case X86::VMOVAPSZ256mr:
+ return X86::VMOVUPSZ128mr;
+ case X86::VMOVUPDZ256mr:
+ case X86::VMOVAPDZ256mr:
+ return X86::VMOVUPDZ128mr;
+ case X86::VMOVDQU64Z256mr:
+ case X86::VMOVDQA64Z256mr:
+ return X86::VMOVDQU64Z128mr;
+ case X86::VMOVDQU32Z256mr:
+ case X86::VMOVDQA32Z256mr:
+ return X86::VMOVDQU32Z128mr;
+ default:
+ llvm_unreachable("Unexpected Load Instruction Opcode");
+ }
+ return 0;
+}
+
+static int getAddrOffset(MachineInstr *MI) {
+ const MCInstrDesc &Descl = MI->getDesc();
+ int AddrOffset = X86II::getMemoryOperandNo(Descl.TSFlags);
+ assert(AddrOffset != -1 && "Expected Memory Operand");
+ AddrOffset += X86II::getOperandBias(Descl);
+ return AddrOffset;
+}
+
+static MachineOperand &getBaseOperand(MachineInstr *MI) {
+ int AddrOffset = getAddrOffset(MI);
+ return MI->getOperand(AddrOffset + X86::AddrBaseReg);
+}
+
+static MachineOperand &getDispOperand(MachineInstr *MI) {
+ int AddrOffset = getAddrOffset(MI);
+ return MI->getOperand(AddrOffset + X86::AddrDisp);
+}
+
+// Relevant addressing modes contain only base register and immediate
+// displacement or frameindex and immediate displacement.
+// TODO: Consider expanding to other addressing modes in the future
+static bool isRelevantAddressingMode(MachineInstr *MI) {
+ int AddrOffset = getAddrOffset(MI);
+ MachineOperand &Base = getBaseOperand(MI);
+ MachineOperand &Disp = getDispOperand(MI);
+ MachineOperand &Scale = MI->getOperand(AddrOffset + X86::AddrScaleAmt);
+ MachineOperand &Index = MI->getOperand(AddrOffset + X86::AddrIndexReg);
+ MachineOperand &Segment = MI->getOperand(AddrOffset + X86::AddrSegmentReg);
+
+ if (!((Base.isReg() && Base.getReg() != X86::NoRegister) || Base.isFI()))
+ return false;
+ if (!Disp.isImm())
+ return false;
+ if (Scale.getImm() != 1)
+ return false;
+ if (!(Index.isReg() && Index.getReg() == X86::NoRegister))
+ return false;
+ if (!(Segment.isReg() && Segment.getReg() == X86::NoRegister))
+ return false;
+ return true;
+}
+
+// Collect potentially blocking stores.
+// Limit the number of instructions backwards we want to inspect
+// since the effect of store block won't be visible if the store
+// and load instructions have enough instructions in between to
+// keep the core busy.
+static SmallVector<MachineInstr *, 2>
+findPotentialBlockers(MachineInstr *LoadInst) {
+ SmallVector<MachineInstr *, 2> PotentialBlockers;
+ unsigned BlockCount = 0;
+ const unsigned InspectionLimit = X86AvoidSFBInspectionLimit;
+ for (auto PBInst = std::next(MachineBasicBlock::reverse_iterator(LoadInst)),
+ E = LoadInst->getParent()->rend();
+ PBInst != E; ++PBInst) {
+ BlockCount++;
+ if (BlockCount >= InspectionLimit)
+ break;
+ MachineInstr &MI = *PBInst;
+ if (MI.getDesc().isCall())
+ return PotentialBlockers;
+ PotentialBlockers.push_back(&MI);
+ }
+ // If we didn't get to the instructions limit try predecessing blocks.
+ // Ideally we should traverse the predecessor blocks in depth with some
+ // coloring algorithm, but for now let's just look at the first order
+ // predecessors.
+ if (BlockCount < InspectionLimit) {
+ MachineBasicBlock *MBB = LoadInst->getParent();
+ int LimitLeft = InspectionLimit - BlockCount;
+ for (MachineBasicBlock::pred_iterator PB = MBB->pred_begin(),
+ PE = MBB->pred_end();
+ PB != PE; ++PB) {
+ MachineBasicBlock *PMBB = *PB;
+ int PredCount = 0;
+ for (MachineBasicBlock::reverse_iterator PBInst = PMBB->rbegin(),
+ PME = PMBB->rend();
+ PBInst != PME; ++PBInst) {
+ PredCount++;
+ if (PredCount >= LimitLeft)
+ break;
+ if (PBInst->getDesc().isCall())
+ break;
+ PotentialBlockers.push_back(&*PBInst);
+ }
+ }
+ }
+ return PotentialBlockers;
+}
+
+void X86AvoidSFBPass::buildCopy(MachineInstr *LoadInst, unsigned NLoadOpcode,
+ int64_t LoadDisp, MachineInstr *StoreInst,
+ unsigned NStoreOpcode, int64_t StoreDisp,
+ unsigned Size, int64_t LMMOffset,
+ int64_t SMMOffset) {
+ MachineOperand &LoadBase = getBaseOperand(LoadInst);
+ MachineOperand &StoreBase = getBaseOperand(StoreInst);
+ MachineBasicBlock *MBB = LoadInst->getParent();
+ MachineMemOperand *LMMO = *LoadInst->memoperands_begin();
+ MachineMemOperand *SMMO = *StoreInst->memoperands_begin();
+
+ unsigned Reg1 = MRI->createVirtualRegister(
+ TII->getRegClass(TII->get(NLoadOpcode), 0, TRI, *(MBB->getParent())));
+ BuildMI(*MBB, LoadInst, LoadInst->getDebugLoc(), TII->get(NLoadOpcode), Reg1)
+ .add(LoadBase)
+ .addImm(1)
+ .addReg(X86::NoRegister)
+ .addImm(LoadDisp)
+ .addReg(X86::NoRegister)
+ .addMemOperand(
+ MBB->getParent()->getMachineMemOperand(LMMO, LMMOffset, Size));
+ DEBUG(LoadInst->getPrevNode()->dump());
+ // If the load and store are consecutive, use the loadInst location to
+ // reduce register pressure.
+ MachineInstr *StInst = StoreInst;
+ if (StoreInst->getPrevNode() == LoadInst)
+ StInst = LoadInst;
+ BuildMI(*MBB, StInst, StInst->getDebugLoc(), TII->get(NStoreOpcode))
+ .add(StoreBase)
+ .addImm(1)
+ .addReg(X86::NoRegister)
+ .addImm(StoreDisp)
+ .addReg(X86::NoRegister)
+ .addReg(Reg1)
+ .addMemOperand(
+ MBB->getParent()->getMachineMemOperand(SMMO, SMMOffset, Size));
+ DEBUG(StInst->getPrevNode()->dump());
+}
+
+void X86AvoidSFBPass::buildCopies(int Size, MachineInstr *LoadInst,
+ int64_t LdDispImm, MachineInstr *StoreInst,
+ int64_t StDispImm, int64_t LMMOffset,
+ int64_t SMMOffset) {
+ int LdDisp = LdDispImm;
+ int StDisp = StDispImm;
+ while (Size > 0) {
+ if ((Size - MOV128SZ >= 0) && isYMMLoadOpcode(LoadInst->getOpcode())) {
+ Size = Size - MOV128SZ;
+ buildCopy(LoadInst, getYMMtoXMMLoadOpcode(LoadInst->getOpcode()), LdDisp,
+ StoreInst, getYMMtoXMMStoreOpcode(StoreInst->getOpcode()),
+ StDisp, MOV128SZ, LMMOffset, SMMOffset);
+ LdDisp += MOV128SZ;
+ StDisp += MOV128SZ;
+ LMMOffset += MOV128SZ;
+ SMMOffset += MOV128SZ;
+ continue;
+ }
+ if (Size - MOV64SZ >= 0) {
+ Size = Size - MOV64SZ;
+ buildCopy(LoadInst, X86::MOV64rm, LdDisp, StoreInst, X86::MOV64mr, StDisp,
+ MOV64SZ, LMMOffset, SMMOffset);
+ LdDisp += MOV64SZ;
+ StDisp += MOV64SZ;
+ LMMOffset += MOV64SZ;
+ SMMOffset += MOV64SZ;
+ continue;
+ }
+ if (Size - MOV32SZ >= 0) {
+ Size = Size - MOV32SZ;
+ buildCopy(LoadInst, X86::MOV32rm, LdDisp, StoreInst, X86::MOV32mr, StDisp,
+ MOV32SZ, LMMOffset, SMMOffset);
+ LdDisp += MOV32SZ;
+ StDisp += MOV32SZ;
+ LMMOffset += MOV32SZ;
+ SMMOffset += MOV32SZ;
+ continue;
+ }
+ if (Size - MOV16SZ >= 0) {
+ Size = Size - MOV16SZ;
+ buildCopy(LoadInst, X86::MOV16rm, LdDisp, StoreInst, X86::MOV16mr, StDisp,
+ MOV16SZ, LMMOffset, SMMOffset);
+ LdDisp += MOV16SZ;
+ StDisp += MOV16SZ;
+ LMMOffset += MOV16SZ;
+ SMMOffset += MOV16SZ;
+ continue;
+ }
+ if (Size - MOV8SZ >= 0) {
+ Size = Size - MOV8SZ;
+ buildCopy(LoadInst, X86::MOV8rm, LdDisp, StoreInst, X86::MOV8mr, StDisp,
+ MOV8SZ, LMMOffset, SMMOffset);
+ LdDisp += MOV8SZ;
+ StDisp += MOV8SZ;
+ LMMOffset += MOV8SZ;
+ SMMOffset += MOV8SZ;
+ continue;
+ }
+ }
+ assert(Size == 0 && "Wrong size division");
+}
+
+static void updateKillStatus(MachineInstr *LoadInst, MachineInstr *StoreInst) {
+ MachineOperand &LoadBase = getBaseOperand(LoadInst);
+ MachineOperand &StoreBase = getBaseOperand(StoreInst);
+ if (LoadBase.isReg()) {
+ MachineInstr *LastLoad = LoadInst->getPrevNode();
+ // If the original load and store to xmm/ymm were consecutive
+ // then the partial copies were also created in
+ // a consecutive order to reduce register pressure,
+ // and the location of the last load is before the last store.
+ if (StoreInst->getPrevNode() == LoadInst)
+ LastLoad = LoadInst->getPrevNode()->getPrevNode();
+ getBaseOperand(LastLoad).setIsKill(LoadBase.isKill());
+ }
+ if (StoreBase.isReg()) {
+ MachineInstr *StInst = StoreInst;
+ if (StoreInst->getPrevNode() == LoadInst)
+ StInst = LoadInst;
+ getBaseOperand(StInst->getPrevNode()).setIsKill(StoreBase.isKill());
+ }
+}
+
+bool X86AvoidSFBPass::alias(const MachineMemOperand &Op1,
+ const MachineMemOperand &Op2) const {
+ if (!Op1.getValue() || !Op2.getValue())
+ return true;
+
+ int64_t MinOffset = std::min(Op1.getOffset(), Op2.getOffset());
+ int64_t Overlapa = Op1.getSize() + Op1.getOffset() - MinOffset;
+ int64_t Overlapb = Op2.getSize() + Op2.getOffset() - MinOffset;
+
+ AliasResult AAResult =
+ AA->alias(MemoryLocation(Op1.getValue(), Overlapa, Op1.getAAInfo()),
+ MemoryLocation(Op2.getValue(), Overlapb, Op2.getAAInfo()));
+ return AAResult != NoAlias;
+}
+
+void X86AvoidSFBPass::findPotentiallylBlockedCopies(MachineFunction &MF) {
+ for (auto &MBB : MF)
+ for (auto &MI : MBB) {
+ if (!isPotentialBlockedMemCpyLd(MI.getOpcode()))
+ continue;
+ int DefVR = MI.getOperand(0).getReg();
+ if (!MRI->hasOneUse(DefVR))
+ continue;
+ for (auto UI = MRI->use_nodbg_begin(DefVR), UE = MRI->use_nodbg_end();
+ UI != UE;) {
+ MachineOperand &StoreMO = *UI++;
+ MachineInstr &StoreMI = *StoreMO.getParent();
+ // Skip cases where the memcpy may overlap.
+ if (StoreMI.getParent() == MI.getParent() &&
+ isPotentialBlockedMemCpyPair(MI.getOpcode(), StoreMI.getOpcode()) &&
+ isRelevantAddressingMode(&MI) &&
+ isRelevantAddressingMode(&StoreMI)) {
+ assert(MI.hasOneMemOperand() &&
+ "Expected one memory operand for load instruction");
+ assert(StoreMI.hasOneMemOperand() &&
+ "Expected one memory operand for store instruction");
+ if (!alias(**MI.memoperands_begin(), **StoreMI.memoperands_begin()))
+ BlockedLoadsStoresPairs.push_back(std::make_pair(&MI, &StoreMI));
+ }
+ }
+ }
+}
+
+unsigned X86AvoidSFBPass::getRegSizeInBytes(MachineInstr *LoadInst) {
+ auto TRC = TII->getRegClass(TII->get(LoadInst->getOpcode()), 0, TRI,
+ *LoadInst->getParent()->getParent());
+ return TRI->getRegSizeInBits(*TRC) / 8;
+}
+
+void X86AvoidSFBPass::breakBlockedCopies(
+ MachineInstr *LoadInst, MachineInstr *StoreInst,
+ const DisplacementSizeMap &BlockingStoresDispSizeMap) {
+ int64_t LdDispImm = getDispOperand(LoadInst).getImm();
+ int64_t StDispImm = getDispOperand(StoreInst).getImm();
+ int64_t LMMOffset = (*LoadInst->memoperands_begin())->getOffset();
+ int64_t SMMOffset = (*StoreInst->memoperands_begin())->getOffset();
+
+ int64_t LdDisp1 = LdDispImm;
+ int64_t LdDisp2 = 0;
+ int64_t StDisp1 = StDispImm;
+ int64_t StDisp2 = 0;
+ unsigned Size1 = 0;
+ unsigned Size2 = 0;
+ int64_t LdStDelta = StDispImm - LdDispImm;
+
+ for (auto DispSizePair : BlockingStoresDispSizeMap) {
+ LdDisp2 = DispSizePair.first;
+ StDisp2 = DispSizePair.first + LdStDelta;
+ Size2 = DispSizePair.second;
+ // Avoid copying overlapping areas.
+ if (LdDisp2 < LdDisp1) {
+ int OverlapDelta = LdDisp1 - LdDisp2;
+ LdDisp2 += OverlapDelta;
+ StDisp2 += OverlapDelta;
+ Size2 -= OverlapDelta;
+ }
+ Size1 = std::abs(std::abs(LdDisp2) - std::abs(LdDisp1));
+
+ // Build a copy for the point until the current blocking store's
+ // displacement.
+ buildCopies(Size1, LoadInst, LdDisp1, StoreInst, StDisp1, LMMOffset,
+ SMMOffset);
+ // Build a copy for the current blocking store.
+ buildCopies(Size2, LoadInst, LdDisp2, StoreInst, StDisp2, LMMOffset + Size1,
+ SMMOffset + Size1);
+ LdDisp1 = LdDisp2 + Size2;
+ StDisp1 = StDisp2 + Size2;
+ LMMOffset += Size1 + Size2;
+ SMMOffset += Size1 + Size2;
+ }
+ unsigned Size3 = (LdDispImm + getRegSizeInBytes(LoadInst)) - LdDisp1;
+ buildCopies(Size3, LoadInst, LdDisp1, StoreInst, StDisp1, LMMOffset,
+ LMMOffset);
+}
+
+static bool hasSameBaseOpValue(MachineInstr *LoadInst,
+ MachineInstr *StoreInst) {
+ MachineOperand &LoadBase = getBaseOperand(LoadInst);
+ MachineOperand &StoreBase = getBaseOperand(StoreInst);
+ if (LoadBase.isReg() != StoreBase.isReg())
+ return false;
+ if (LoadBase.isReg())
+ return LoadBase.getReg() == StoreBase.getReg();
+ return LoadBase.getIndex() == StoreBase.getIndex();
+}
+
+static bool isBlockingStore(int64_t LoadDispImm, unsigned LoadSize,
+ int64_t StoreDispImm, unsigned StoreSize) {
+ return ((StoreDispImm >= LoadDispImm) &&
+ (StoreDispImm <= LoadDispImm + (LoadSize - StoreSize)));
+}
+
+// Keep track of all stores blocking a load
+static void
+updateBlockingStoresDispSizeMap(DisplacementSizeMap &BlockingStoresDispSizeMap,
+ int64_t DispImm, unsigned Size) {
+ if (BlockingStoresDispSizeMap.count(DispImm)) {
+ // Choose the smallest blocking store starting at this displacement.
+ if (BlockingStoresDispSizeMap[DispImm] > Size)
+ BlockingStoresDispSizeMap[DispImm] = Size;
+
+ } else
+ BlockingStoresDispSizeMap[DispImm] = Size;
+}
+
+// Remove blocking stores contained in each other.
+static void
+removeRedundantBlockingStores(DisplacementSizeMap &BlockingStoresDispSizeMap) {
+ if (BlockingStoresDispSizeMap.size() <= 1)
+ return;
+
+ int64_t PrevDisp = BlockingStoresDispSizeMap.begin()->first;
+ unsigned PrevSize = BlockingStoresDispSizeMap.begin()->second;
+ SmallVector<int64_t, 2> ForRemoval;
+ for (auto DispSizePair = std::next(BlockingStoresDispSizeMap.begin());
+ DispSizePair != BlockingStoresDispSizeMap.end(); ++DispSizePair) {
+ int64_t CurrDisp = DispSizePair->first;
+ unsigned CurrSize = DispSizePair->second;
+ if (CurrDisp + CurrSize <= PrevDisp + PrevSize) {
+ ForRemoval.push_back(PrevDisp);
+ }
+ PrevDisp = CurrDisp;
+ PrevSize = CurrSize;
+ }
+ for (auto Disp : ForRemoval)
+ BlockingStoresDispSizeMap.erase(Disp);
+}
+
+bool X86AvoidSFBPass::runOnMachineFunction(MachineFunction &MF) {
+ bool Changed = false;
+
+ if (DisableX86AvoidStoreForwardBlocks || skipFunction(MF.getFunction()) ||
+ !MF.getSubtarget<X86Subtarget>().is64Bit())
+ return false;
+
+ MRI = &MF.getRegInfo();
+ assert(MRI->isSSA() && "Expected MIR to be in SSA form");
+ TII = MF.getSubtarget<X86Subtarget>().getInstrInfo();
+ TRI = MF.getSubtarget<X86Subtarget>().getRegisterInfo();
+ AA = &getAnalysis<AAResultsWrapperPass>().getAAResults();
+ DEBUG(dbgs() << "Start X86AvoidStoreForwardBlocks\n";);
+ // Look for a load then a store to XMM/YMM which look like a memcpy
+ findPotentiallylBlockedCopies(MF);
+
+ for (auto LoadStoreInstPair : BlockedLoadsStoresPairs) {
+ MachineInstr *LoadInst = LoadStoreInstPair.first;
+ int64_t LdDispImm = getDispOperand(LoadInst).getImm();
+ DisplacementSizeMap BlockingStoresDispSizeMap;
+
+ SmallVector<MachineInstr *, 2> PotentialBlockers =
+ findPotentialBlockers(LoadInst);
+ for (auto PBInst : PotentialBlockers) {
+ if (!isPotentialBlockingStoreInst(PBInst->getOpcode(),
+ LoadInst->getOpcode()) ||
+ !isRelevantAddressingMode(PBInst))
+ continue;
+ int64_t PBstDispImm = getDispOperand(PBInst).getImm();
+ assert(PBInst->hasOneMemOperand() && "Expected One Memory Operand");
+ unsigned PBstSize = (*PBInst->memoperands_begin())->getSize();
+ // This check doesn't cover all cases, but it will suffice for now.
+ // TODO: take branch probability into consideration, if the blocking
+ // store is in an unreached block, breaking the memcopy could lose
+ // performance.
+ if (hasSameBaseOpValue(LoadInst, PBInst) &&
+ isBlockingStore(LdDispImm, getRegSizeInBytes(LoadInst), PBstDispImm,
+ PBstSize))
+ updateBlockingStoresDispSizeMap(BlockingStoresDispSizeMap, PBstDispImm,
+ PBstSize);
+ }
+
+ if (BlockingStoresDispSizeMap.empty())
+ continue;
+
+ // We found a store forward block, break the memcpy's load and store
+ // into smaller copies such that each smaller store that was causing
+ // a store block would now be copied separately.
+ MachineInstr *StoreInst = LoadStoreInstPair.second;
+ DEBUG(dbgs() << "Blocked load and store instructions: \n");
+ DEBUG(LoadInst->dump());
+ DEBUG(StoreInst->dump());
+ DEBUG(dbgs() << "Replaced with:\n");
+ removeRedundantBlockingStores(BlockingStoresDispSizeMap);
+ breakBlockedCopies(LoadInst, StoreInst, BlockingStoresDispSizeMap);
+ updateKillStatus(LoadInst, StoreInst);
+ ForRemoval.push_back(LoadInst);
+ ForRemoval.push_back(StoreInst);
+ }
+ for (auto RemovedInst : ForRemoval) {
+ RemovedInst->eraseFromParent();
+ }
+ ForRemoval.clear();
+ BlockedLoadsStoresPairs.clear();
+ DEBUG(dbgs() << "End X86AvoidStoreForwardBlocks\n";);
+
+ return Changed;
+}
Modified: llvm/trunk/lib/Target/X86/X86TargetMachine.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/X86/X86TargetMachine.cpp?rev=328973&r1=328972&r2=328973&view=diff
==============================================================================
--- llvm/trunk/lib/Target/X86/X86TargetMachine.cpp (original)
+++ llvm/trunk/lib/Target/X86/X86TargetMachine.cpp Mon Apr 2 06:48:28 2018
@@ -62,6 +62,7 @@ void initializeX86CallFrameOptimizationP
void initializeX86CmovConverterPassPass(PassRegistry &);
void initializeX86ExecutionDomainFixPass(PassRegistry &);
void initializeX86DomainReassignmentPass(PassRegistry &);
+void initializeX86AvoidSFBPassPass(PassRegistry &);
} // end namespace llvm
@@ -80,6 +81,7 @@ extern "C" void LLVMInitializeX86Target(
initializeX86CmovConverterPassPass(PR);
initializeX86ExecutionDomainFixPass(PR);
initializeX86DomainReassignmentPass(PR);
+ initializeX86AvoidSFBPassPass(PR);
}
static std::unique_ptr<TargetLoweringObjectFile> createTLOF(const Triple &TT) {
@@ -449,6 +451,7 @@ void X86PassConfig::addPreRegAlloc() {
addPass(createX86FixupSetCC());
addPass(createX86OptimizeLEAs());
addPass(createX86CallFrameOptimization());
+ addPass(createX86AvoidStoreForwardingBlocks());
}
addPass(createX86WinAllocaExpander());
Modified: llvm/trunk/test/CodeGen/X86/O3-pipeline.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/O3-pipeline.ll?rev=328973&r1=328972&r2=328973&view=diff
==============================================================================
--- llvm/trunk/test/CodeGen/X86/O3-pipeline.ll (original)
+++ llvm/trunk/test/CodeGen/X86/O3-pipeline.ll Mon Apr 2 06:48:28 2018
@@ -89,6 +89,7 @@
; CHECK-NEXT: X86 Fixup SetCC
; CHECK-NEXT: X86 LEA Optimize
; CHECK-NEXT: X86 Optimize Call Frame
+; CHECK-NEXT: X86 Avoid Store Forwarding Block
; CHECK-NEXT: X86 WinAlloca Expander
; CHECK-NEXT: Detect Dead Lanes
; CHECK-NEXT: Process Implicit Definitions
Added: llvm/trunk/test/CodeGen/X86/avoid-sfb-overlaps.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/avoid-sfb-overlaps.ll?rev=328973&view=auto
==============================================================================
--- llvm/trunk/test/CodeGen/X86/avoid-sfb-overlaps.ll (added)
+++ llvm/trunk/test/CodeGen/X86/avoid-sfb-overlaps.ll Mon Apr 2 06:48:28 2018
@@ -0,0 +1,521 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s -check-prefix=CHECK
+; RUN: llc < %s -mtriple=x86_64-linux --x86-disable-avoid-SFB | FileCheck %s --check-prefix=DISABLED
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=core-avx2 | FileCheck %s -check-prefix=CHECK-AVX2
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=skx | FileCheck %s -check-prefix=CHECK-AVX512
+
+; ModuleID = '../testSFB/testOverlapBlocks.c'
+source_filename = "../testSFB/testOverlapBlocks.c"
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+; Function Attrs: nounwind uwtable
+define dso_local void @test_overlap_1(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_overlap_1:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl $7, -8(%rdi)
+; CHECK-NEXT: movq -16(%rdi), %rax
+; CHECK-NEXT: movq %rax, (%rdi)
+; CHECK-NEXT: movl -8(%rdi), %eax
+; CHECK-NEXT: movl %eax, 8(%rdi)
+; CHECK-NEXT: movl -4(%rdi), %eax
+; CHECK-NEXT: movl %eax, 12(%rdi)
+; CHECK-NEXT: movslq %esi, %rax
+; CHECK-NEXT: movq %rax, -9(%rdi)
+; CHECK-NEXT: movq %rax, -16(%rdi)
+; CHECK-NEXT: movb $0, -1(%rdi)
+; CHECK-NEXT: movq -16(%rdi), %rax
+; CHECK-NEXT: movq %rax, 16(%rdi)
+; CHECK-NEXT: movl -8(%rdi), %eax
+; CHECK-NEXT: movl %eax, 24(%rdi)
+; CHECK-NEXT: movzwl -4(%rdi), %eax
+; CHECK-NEXT: movw %ax, 28(%rdi)
+; CHECK-NEXT: movb -2(%rdi), %al
+; CHECK-NEXT: movb %al, 30(%rdi)
+; CHECK-NEXT: movb -1(%rdi), %al
+; CHECK-NEXT: movb %al, 31(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_overlap_1:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl $7, -8(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rdi)
+; DISABLED-NEXT: movslq %esi, %rax
+; DISABLED-NEXT: movq %rax, -9(%rdi)
+; DISABLED-NEXT: movq %rax, -16(%rdi)
+; DISABLED-NEXT: movb $0, -1(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_overlap_1:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl $7, -8(%rdi)
+; CHECK-AVX2-NEXT: movq -16(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, (%rdi)
+; CHECK-AVX2-NEXT: movl -8(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 8(%rdi)
+; CHECK-AVX2-NEXT: movl -4(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 12(%rdi)
+; CHECK-AVX2-NEXT: movslq %esi, %rax
+; CHECK-AVX2-NEXT: movq %rax, -9(%rdi)
+; CHECK-AVX2-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX2-NEXT: movb $0, -1(%rdi)
+; CHECK-AVX2-NEXT: movq -16(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 16(%rdi)
+; CHECK-AVX2-NEXT: movl -8(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 24(%rdi)
+; CHECK-AVX2-NEXT: movzwl -4(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 28(%rdi)
+; CHECK-AVX2-NEXT: movb -2(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 30(%rdi)
+; CHECK-AVX2-NEXT: movb -1(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 31(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_overlap_1:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl $7, -8(%rdi)
+; CHECK-AVX512-NEXT: movq -16(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, (%rdi)
+; CHECK-AVX512-NEXT: movl -8(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 8(%rdi)
+; CHECK-AVX512-NEXT: movl -4(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 12(%rdi)
+; CHECK-AVX512-NEXT: movslq %esi, %rax
+; CHECK-AVX512-NEXT: movq %rax, -9(%rdi)
+; CHECK-AVX512-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX512-NEXT: movb $0, -1(%rdi)
+; CHECK-AVX512-NEXT: movq -16(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 16(%rdi)
+; CHECK-AVX512-NEXT: movl -8(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 24(%rdi)
+; CHECK-AVX512-NEXT: movzwl -4(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 28(%rdi)
+; CHECK-AVX512-NEXT: movb -2(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 30(%rdi)
+; CHECK-AVX512-NEXT: movb -1(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 31(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16
+ %add.ptr1 = getelementptr inbounds i8, i8* %A, i64 -8
+ %0 = bitcast i8* %add.ptr1 to i32*
+ store i32 7, i32* %0, align 4
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ %conv = sext i32 %x to i64
+ %add.ptr2 = getelementptr inbounds i8, i8* %A, i64 -9
+ %1 = bitcast i8* %add.ptr2 to i64*
+ store i64 %conv, i64* %1, align 8
+ %2 = bitcast i8* %add.ptr to i64*
+ store i64 %conv, i64* %2, align 8
+ %add.ptr5 = getelementptr inbounds i8, i8* %A, i64 -1
+ store i8 0, i8* %add.ptr5, align 1
+ %add.ptr6 = getelementptr inbounds i8, i8* %A, i64 16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr6, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ ret void
+}
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i1) #1
+
+; Function Attrs: nounwind uwtable
+define dso_local void @test_overlap_2(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_overlap_2:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movslq %esi, %rax
+; CHECK-NEXT: movq %rax, -16(%rdi)
+; CHECK-NEXT: movq -16(%rdi), %rcx
+; CHECK-NEXT: movq %rcx, (%rdi)
+; CHECK-NEXT: movq -8(%rdi), %rcx
+; CHECK-NEXT: movq %rcx, 8(%rdi)
+; CHECK-NEXT: movq %rax, -8(%rdi)
+; CHECK-NEXT: movl $7, -12(%rdi)
+; CHECK-NEXT: movl -16(%rdi), %eax
+; CHECK-NEXT: movl %eax, 16(%rdi)
+; CHECK-NEXT: movl -12(%rdi), %eax
+; CHECK-NEXT: movl %eax, 20(%rdi)
+; CHECK-NEXT: movq -8(%rdi), %rax
+; CHECK-NEXT: movq %rax, 24(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_overlap_2:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movslq %esi, %rax
+; DISABLED-NEXT: movq %rax, -16(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rdi)
+; DISABLED-NEXT: movq %rax, -8(%rdi)
+; DISABLED-NEXT: movl $7, -12(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_overlap_2:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movslq %esi, %rax
+; CHECK-AVX2-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX2-NEXT: movq -16(%rdi), %rcx
+; CHECK-AVX2-NEXT: movq %rcx, (%rdi)
+; CHECK-AVX2-NEXT: movq -8(%rdi), %rcx
+; CHECK-AVX2-NEXT: movq %rcx, 8(%rdi)
+; CHECK-AVX2-NEXT: movq %rax, -8(%rdi)
+; CHECK-AVX2-NEXT: movl $7, -12(%rdi)
+; CHECK-AVX2-NEXT: movl -16(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 16(%rdi)
+; CHECK-AVX2-NEXT: movl -12(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 20(%rdi)
+; CHECK-AVX2-NEXT: movq -8(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 24(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_overlap_2:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movslq %esi, %rax
+; CHECK-AVX512-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX512-NEXT: movq -16(%rdi), %rcx
+; CHECK-AVX512-NEXT: movq %rcx, (%rdi)
+; CHECK-AVX512-NEXT: movq -8(%rdi), %rcx
+; CHECK-AVX512-NEXT: movq %rcx, 8(%rdi)
+; CHECK-AVX512-NEXT: movq %rax, -8(%rdi)
+; CHECK-AVX512-NEXT: movl $7, -12(%rdi)
+; CHECK-AVX512-NEXT: movl -16(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 16(%rdi)
+; CHECK-AVX512-NEXT: movl -12(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 20(%rdi)
+; CHECK-AVX512-NEXT: movq -8(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 24(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16
+ %conv = sext i32 %x to i64
+ %0 = bitcast i8* %add.ptr to i64*
+ store i64 %conv, i64* %0, align 8
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ %add.ptr3 = getelementptr inbounds i8, i8* %A, i64 -8
+ %1 = bitcast i8* %add.ptr3 to i64*
+ store i64 %conv, i64* %1, align 8
+ %add.ptr4 = getelementptr inbounds i8, i8* %A, i64 -12
+ %2 = bitcast i8* %add.ptr4 to i32*
+ store i32 7, i32* %2, align 4
+ %add.ptr5 = getelementptr inbounds i8, i8* %A, i64 16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr5, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define dso_local void @test_overlap_3(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_overlap_3:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl $7, -10(%rdi)
+; CHECK-NEXT: movl -16(%rdi), %eax
+; CHECK-NEXT: movl %eax, (%rdi)
+; CHECK-NEXT: movzwl -12(%rdi), %eax
+; CHECK-NEXT: movw %ax, 4(%rdi)
+; CHECK-NEXT: movl -10(%rdi), %eax
+; CHECK-NEXT: movl %eax, 6(%rdi)
+; CHECK-NEXT: movl -6(%rdi), %eax
+; CHECK-NEXT: movl %eax, 10(%rdi)
+; CHECK-NEXT: movzwl -2(%rdi), %eax
+; CHECK-NEXT: movw %ax, 14(%rdi)
+; CHECK-NEXT: movslq %esi, %rax
+; CHECK-NEXT: movq %rax, -9(%rdi)
+; CHECK-NEXT: movq %rax, -16(%rdi)
+; CHECK-NEXT: movb $0, -1(%rdi)
+; CHECK-NEXT: movq -16(%rdi), %rax
+; CHECK-NEXT: movq %rax, 16(%rdi)
+; CHECK-NEXT: movzwl -8(%rdi), %eax
+; CHECK-NEXT: movw %ax, 24(%rdi)
+; CHECK-NEXT: movl -6(%rdi), %eax
+; CHECK-NEXT: movl %eax, 26(%rdi)
+; CHECK-NEXT: movb -2(%rdi), %al
+; CHECK-NEXT: movb %al, 30(%rdi)
+; CHECK-NEXT: movb -1(%rdi), %al
+; CHECK-NEXT: movb %al, 31(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_overlap_3:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl $7, -10(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rdi)
+; DISABLED-NEXT: movslq %esi, %rax
+; DISABLED-NEXT: movq %rax, -9(%rdi)
+; DISABLED-NEXT: movq %rax, -16(%rdi)
+; DISABLED-NEXT: movb $0, -1(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_overlap_3:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl $7, -10(%rdi)
+; CHECK-AVX2-NEXT: movl -16(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, (%rdi)
+; CHECK-AVX2-NEXT: movzwl -12(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 4(%rdi)
+; CHECK-AVX2-NEXT: movl -10(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 6(%rdi)
+; CHECK-AVX2-NEXT: movl -6(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 10(%rdi)
+; CHECK-AVX2-NEXT: movzwl -2(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 14(%rdi)
+; CHECK-AVX2-NEXT: movslq %esi, %rax
+; CHECK-AVX2-NEXT: movq %rax, -9(%rdi)
+; CHECK-AVX2-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX2-NEXT: movb $0, -1(%rdi)
+; CHECK-AVX2-NEXT: movq -16(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 16(%rdi)
+; CHECK-AVX2-NEXT: movzwl -8(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 24(%rdi)
+; CHECK-AVX2-NEXT: movl -6(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 26(%rdi)
+; CHECK-AVX2-NEXT: movb -2(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 30(%rdi)
+; CHECK-AVX2-NEXT: movb -1(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 31(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_overlap_3:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl $7, -10(%rdi)
+; CHECK-AVX512-NEXT: movl -16(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, (%rdi)
+; CHECK-AVX512-NEXT: movzwl -12(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 4(%rdi)
+; CHECK-AVX512-NEXT: movl -10(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 6(%rdi)
+; CHECK-AVX512-NEXT: movl -6(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 10(%rdi)
+; CHECK-AVX512-NEXT: movzwl -2(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 14(%rdi)
+; CHECK-AVX512-NEXT: movslq %esi, %rax
+; CHECK-AVX512-NEXT: movq %rax, -9(%rdi)
+; CHECK-AVX512-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX512-NEXT: movb $0, -1(%rdi)
+; CHECK-AVX512-NEXT: movq -16(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 16(%rdi)
+; CHECK-AVX512-NEXT: movzwl -8(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 24(%rdi)
+; CHECK-AVX512-NEXT: movl -6(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 26(%rdi)
+; CHECK-AVX512-NEXT: movb -2(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 30(%rdi)
+; CHECK-AVX512-NEXT: movb -1(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 31(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16
+ %add.ptr1 = getelementptr inbounds i8, i8* %A, i64 -10
+ %0 = bitcast i8* %add.ptr1 to i32*
+ store i32 7, i32* %0, align 4
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ %conv = sext i32 %x to i64
+ %add.ptr2 = getelementptr inbounds i8, i8* %A, i64 -9
+ %1 = bitcast i8* %add.ptr2 to i64*
+ store i64 %conv, i64* %1, align 8
+ %2 = bitcast i8* %add.ptr to i64*
+ store i64 %conv, i64* %2, align 8
+ %add.ptr5 = getelementptr inbounds i8, i8* %A, i64 -1
+ store i8 0, i8* %add.ptr5, align 1
+ %add.ptr6 = getelementptr inbounds i8, i8* %A, i64 16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr6, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define dso_local void @test_overlap_4(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_overlap_4:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movups -16(%rdi), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rdi)
+; CHECK-NEXT: movslq %esi, %rax
+; CHECK-NEXT: movq %rax, -8(%rdi)
+; CHECK-NEXT: movl %eax, -16(%rdi)
+; CHECK-NEXT: movl $0, -11(%rdi)
+; CHECK-NEXT: movl -16(%rdi), %eax
+; CHECK-NEXT: movl %eax, 16(%rdi)
+; CHECK-NEXT: movb -12(%rdi), %al
+; CHECK-NEXT: movb %al, 20(%rdi)
+; CHECK-NEXT: movl -11(%rdi), %eax
+; CHECK-NEXT: movl %eax, 21(%rdi)
+; CHECK-NEXT: movl -7(%rdi), %eax
+; CHECK-NEXT: movl %eax, 25(%rdi)
+; CHECK-NEXT: movzwl -3(%rdi), %eax
+; CHECK-NEXT: movw %ax, 29(%rdi)
+; CHECK-NEXT: movb -1(%rdi), %al
+; CHECK-NEXT: movb %al, 31(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_overlap_4:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rdi)
+; DISABLED-NEXT: movslq %esi, %rax
+; DISABLED-NEXT: movq %rax, -8(%rdi)
+; DISABLED-NEXT: movl %eax, -16(%rdi)
+; DISABLED-NEXT: movl $0, -11(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_overlap_4:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: vmovups -16(%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rdi)
+; CHECK-AVX2-NEXT: movslq %esi, %rax
+; CHECK-AVX2-NEXT: movq %rax, -8(%rdi)
+; CHECK-AVX2-NEXT: movl %eax, -16(%rdi)
+; CHECK-AVX2-NEXT: movl $0, -11(%rdi)
+; CHECK-AVX2-NEXT: movl -16(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 16(%rdi)
+; CHECK-AVX2-NEXT: movb -12(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 20(%rdi)
+; CHECK-AVX2-NEXT: movl -11(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 21(%rdi)
+; CHECK-AVX2-NEXT: movl -7(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 25(%rdi)
+; CHECK-AVX2-NEXT: movzwl -3(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 29(%rdi)
+; CHECK-AVX2-NEXT: movb -1(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 31(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_overlap_4:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: vmovups -16(%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rdi)
+; CHECK-AVX512-NEXT: movslq %esi, %rax
+; CHECK-AVX512-NEXT: movq %rax, -8(%rdi)
+; CHECK-AVX512-NEXT: movl %eax, -16(%rdi)
+; CHECK-AVX512-NEXT: movl $0, -11(%rdi)
+; CHECK-AVX512-NEXT: movl -16(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 16(%rdi)
+; CHECK-AVX512-NEXT: movb -12(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 20(%rdi)
+; CHECK-AVX512-NEXT: movl -11(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 21(%rdi)
+; CHECK-AVX512-NEXT: movl -7(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 25(%rdi)
+; CHECK-AVX512-NEXT: movzwl -3(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 29(%rdi)
+; CHECK-AVX512-NEXT: movb -1(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 31(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ %conv = sext i32 %x to i64
+ %add.ptr1 = getelementptr inbounds i8, i8* %A, i64 -8
+ %0 = bitcast i8* %add.ptr1 to i64*
+ store i64 %conv, i64* %0, align 8
+ %1 = bitcast i8* %add.ptr to i32*
+ store i32 %x, i32* %1, align 4
+ %add.ptr3 = getelementptr inbounds i8, i8* %A, i64 -11
+ %2 = bitcast i8* %add.ptr3 to i32*
+ store i32 0, i32* %2, align 4
+ %add.ptr4 = getelementptr inbounds i8, i8* %A, i64 16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr4, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define dso_local void @test_overlap_5(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_overlap_5:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movups -16(%rdi), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rdi)
+; CHECK-NEXT: movslq %esi, %rax
+; CHECK-NEXT: movq %rax, -16(%rdi)
+; CHECK-NEXT: movb %al, -14(%rdi)
+; CHECK-NEXT: movb $0, -11(%rdi)
+; CHECK-NEXT: movzwl -16(%rdi), %eax
+; CHECK-NEXT: movw %ax, 16(%rdi)
+; CHECK-NEXT: movb -14(%rdi), %al
+; CHECK-NEXT: movb %al, 18(%rdi)
+; CHECK-NEXT: movzwl -13(%rdi), %eax
+; CHECK-NEXT: movw %ax, 19(%rdi)
+; CHECK-NEXT: movb -11(%rdi), %al
+; CHECK-NEXT: movb %al, 21(%rdi)
+; CHECK-NEXT: movq -10(%rdi), %rax
+; CHECK-NEXT: movq %rax, 22(%rdi)
+; CHECK-NEXT: movzwl -2(%rdi), %eax
+; CHECK-NEXT: movw %ax, 30(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_overlap_5:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rdi)
+; DISABLED-NEXT: movslq %esi, %rax
+; DISABLED-NEXT: movq %rax, -16(%rdi)
+; DISABLED-NEXT: movb %al, -14(%rdi)
+; DISABLED-NEXT: movb $0, -11(%rdi)
+; DISABLED-NEXT: movups -16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_overlap_5:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: vmovups -16(%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rdi)
+; CHECK-AVX2-NEXT: movslq %esi, %rax
+; CHECK-AVX2-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX2-NEXT: movb %al, -14(%rdi)
+; CHECK-AVX2-NEXT: movb $0, -11(%rdi)
+; CHECK-AVX2-NEXT: movzwl -16(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 16(%rdi)
+; CHECK-AVX2-NEXT: movb -14(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 18(%rdi)
+; CHECK-AVX2-NEXT: movzwl -13(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 19(%rdi)
+; CHECK-AVX2-NEXT: movb -11(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 21(%rdi)
+; CHECK-AVX2-NEXT: movq -10(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 22(%rdi)
+; CHECK-AVX2-NEXT: movzwl -2(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 30(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_overlap_5:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: vmovups -16(%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rdi)
+; CHECK-AVX512-NEXT: movslq %esi, %rax
+; CHECK-AVX512-NEXT: movq %rax, -16(%rdi)
+; CHECK-AVX512-NEXT: movb %al, -14(%rdi)
+; CHECK-AVX512-NEXT: movb $0, -11(%rdi)
+; CHECK-AVX512-NEXT: movzwl -16(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 16(%rdi)
+; CHECK-AVX512-NEXT: movb -14(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 18(%rdi)
+; CHECK-AVX512-NEXT: movzwl -13(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 19(%rdi)
+; CHECK-AVX512-NEXT: movb -11(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 21(%rdi)
+; CHECK-AVX512-NEXT: movq -10(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 22(%rdi)
+; CHECK-AVX512-NEXT: movzwl -2(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 30(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 -16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* align 4 %A, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ %conv = sext i32 %x to i64
+ %0 = bitcast i8* %add.ptr to i64*
+ store i64 %conv, i64* %0, align 8
+ %conv2 = trunc i32 %x to i8
+ %add.ptr3 = getelementptr inbounds i8, i8* %A, i64 -14
+ store i8 %conv2, i8* %add.ptr3, align 1
+ %add.ptr4 = getelementptr inbounds i8, i8* %A, i64 -11
+ store i8 0, i8* %add.ptr4, align 1
+ %add.ptr5 = getelementptr inbounds i8, i8* %A, i64 16
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr5, i8* nonnull align 4 %add.ptr, i64 16, i1 false)
+ ret void
+}
+
+attributes #0 = { nounwind uwtable "correctly-rounded-divide-sqrt-fp-math"="false" "disable-tail-calls"="false" "less-precise-fpmad"="false" "no-frame-pointer-elim"="false" "no-infs-fp-math"="false" "no-jump-tables"="false" "no-nans-fp-math"="false" "no-signed-zeros-fp-math"="false" "no-trapping-math"="false" "stack-protector-buffer-size"="8" "target-cpu"="x86-64" "target-features"="+fxsr,+mmx,+sse,+sse2,+x87" "unsafe-fp-math"="false" "use-soft-float"="false" }
+attributes #1 = { argmemonly nounwind }
+
+
Added: llvm/trunk/test/CodeGen/X86/avoid-sfb.ll
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/test/CodeGen/X86/avoid-sfb.ll?rev=328973&view=auto
==============================================================================
--- llvm/trunk/test/CodeGen/X86/avoid-sfb.ll (added)
+++ llvm/trunk/test/CodeGen/X86/avoid-sfb.ll Mon Apr 2 06:48:28 2018
@@ -0,0 +1,1491 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-linux | FileCheck %s -check-prefix=CHECK
+; RUN: llc < %s -mtriple=x86_64-linux --x86-disable-avoid-SFB | FileCheck %s --check-prefix=DISABLED
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=core-avx2 | FileCheck %s -check-prefix=CHECK-AVX2
+; RUN: llc < %s -mtriple=x86_64-linux -mcpu=skx | FileCheck %s -check-prefix=CHECK-AVX512
+
+target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
+target triple = "x86_64-unknown-linux-gnu"
+
+%struct.S = type { i32, i32, i32, i32 }
+
+; Function Attrs: nounwind uwtable
+define void @test_conditional_block(%struct.S* nocapture noalias %s1 , %struct.S* nocapture noalias %s2, i32 %x, %struct.S* nocapture noalias %s3, %struct.S* nocapture noalias readonly %s4) local_unnamed_addr #0 {
+; CHECK-LABEL: test_conditional_block:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB0_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movl %edx, 4(%rdi)
+; CHECK-NEXT: .LBB0_2: # %if.end
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: movl %eax, (%rsi)
+; CHECK-NEXT: movl 4(%rdi), %eax
+; CHECK-NEXT: movl %eax, 4(%rsi)
+; CHECK-NEXT: movq 8(%rdi), %rax
+; CHECK-NEXT: movq %rax, 8(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_conditional_block:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB0_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movl %edx, 4(%rdi)
+; DISABLED-NEXT: .LBB0_2: # %if.end
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_conditional_block:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB0_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movl %edx, 4(%rdi)
+; CHECK-AVX2-NEXT: .LBB0_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX2-NEXT: movl (%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, (%rsi)
+; CHECK-AVX2-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX2-NEXT: movq 8(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 8(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_conditional_block:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB0_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movl %edx, 4(%rdi)
+; CHECK-AVX512-NEXT: .LBB0_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX512-NEXT: movl (%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, (%rsi)
+; CHECK-AVX512-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX512-NEXT: movq 8(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 8(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1
+ store i32 %x, i32* %b, align 4
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S* %s3 to i8*
+ %1 = bitcast %struct.S* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false)
+ %2 = bitcast %struct.S* %s2 to i8*
+ %3 = bitcast %struct.S* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @test_imm_store(%struct.S* nocapture noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3) local_unnamed_addr #0 {
+; CHECK-LABEL: test_imm_store:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl $0, (%rdi)
+; CHECK-NEXT: movl $1, (%rcx)
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: movl %eax, (%rsi)
+; CHECK-NEXT: movq 4(%rdi), %rax
+; CHECK-NEXT: movq %rax, 4(%rsi)
+; CHECK-NEXT: movl 12(%rdi), %eax
+; CHECK-NEXT: movl %eax, 12(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_imm_store:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl $0, (%rdi)
+; DISABLED-NEXT: movl $1, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_imm_store:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl $0, (%rdi)
+; CHECK-AVX2-NEXT: movl $1, (%rcx)
+; CHECK-AVX2-NEXT: movl (%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, (%rsi)
+; CHECK-AVX2-NEXT: movq 4(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 4(%rsi)
+; CHECK-AVX2-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_imm_store:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl $0, (%rdi)
+; CHECK-AVX512-NEXT: movl $1, (%rcx)
+; CHECK-AVX512-NEXT: movl (%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, (%rsi)
+; CHECK-AVX512-NEXT: movq 4(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 4(%rsi)
+; CHECK-AVX512-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %a = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 0
+ store i32 0, i32* %a, align 4
+ %a1 = getelementptr inbounds %struct.S, %struct.S* %s3, i64 0, i32 0
+ store i32 1, i32* %a1, align 4
+ %0 = bitcast %struct.S* %s2 to i8*
+ %1 = bitcast %struct.S* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @test_nondirect_br(%struct.S* nocapture noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3, %struct.S* nocapture readonly %s4, i32 %x2) local_unnamed_addr #0 {
+; CHECK-LABEL: test_nondirect_br:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB2_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movl %edx, 4(%rdi)
+; CHECK-NEXT: .LBB2_2: # %if.end
+; CHECK-NEXT: cmpl $14, %r9d
+; CHECK-NEXT: jl .LBB2_4
+; CHECK-NEXT: # %bb.3: # %if.then2
+; CHECK-NEXT: movl %r9d, 12(%rdi)
+; CHECK-NEXT: .LBB2_4: # %if.end3
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movq (%rdi), %rax
+; CHECK-NEXT: movq %rax, (%rsi)
+; CHECK-NEXT: movl 8(%rdi), %eax
+; CHECK-NEXT: movl %eax, 8(%rsi)
+; CHECK-NEXT: movl 12(%rdi), %eax
+; CHECK-NEXT: movl %eax, 12(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_nondirect_br:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB2_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movl %edx, 4(%rdi)
+; DISABLED-NEXT: .LBB2_2: # %if.end
+; DISABLED-NEXT: cmpl $14, %r9d
+; DISABLED-NEXT: jl .LBB2_4
+; DISABLED-NEXT: # %bb.3: # %if.then2
+; DISABLED-NEXT: movl %r9d, 12(%rdi)
+; DISABLED-NEXT: .LBB2_4: # %if.end3
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_nondirect_br:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB2_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movl %edx, 4(%rdi)
+; CHECK-AVX2-NEXT: .LBB2_2: # %if.end
+; CHECK-AVX2-NEXT: cmpl $14, %r9d
+; CHECK-AVX2-NEXT: jl .LBB2_4
+; CHECK-AVX2-NEXT: # %bb.3: # %if.then2
+; CHECK-AVX2-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX2-NEXT: .LBB2_4: # %if.end3
+; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX2-NEXT: movq (%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, (%rsi)
+; CHECK-AVX2-NEXT: movl 8(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 8(%rsi)
+; CHECK-AVX2-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_nondirect_br:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB2_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movl %edx, 4(%rdi)
+; CHECK-AVX512-NEXT: .LBB2_2: # %if.end
+; CHECK-AVX512-NEXT: cmpl $14, %r9d
+; CHECK-AVX512-NEXT: jl .LBB2_4
+; CHECK-AVX512-NEXT: # %bb.3: # %if.then2
+; CHECK-AVX512-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX512-NEXT: .LBB2_4: # %if.end3
+; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX512-NEXT: movq (%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, (%rsi)
+; CHECK-AVX512-NEXT: movl 8(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 8(%rsi)
+; CHECK-AVX512-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1
+ store i32 %x, i32* %b, align 4
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %cmp1 = icmp sgt i32 %x2, 13
+ br i1 %cmp1, label %if.then2, label %if.end3
+
+if.then2: ; preds = %if.end
+ %d = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 3
+ store i32 %x2, i32* %d, align 4
+ br label %if.end3
+
+if.end3: ; preds = %if.then2, %if.end
+ %0 = bitcast %struct.S* %s3 to i8*
+ %1 = bitcast %struct.S* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false)
+ %2 = bitcast %struct.S* %s2 to i8*
+ %3 = bitcast %struct.S* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @test_2preds_block(%struct.S* nocapture noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3, %struct.S* nocapture readonly %s4, i32 %x2) local_unnamed_addr #0 {
+; CHECK-LABEL: test_2preds_block:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %r9d, 12(%rdi)
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB3_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movl %edx, 4(%rdi)
+; CHECK-NEXT: .LBB3_2: # %if.end
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: movl %eax, (%rsi)
+; CHECK-NEXT: movl 4(%rdi), %eax
+; CHECK-NEXT: movl %eax, 4(%rsi)
+; CHECK-NEXT: movl 8(%rdi), %eax
+; CHECK-NEXT: movl %eax, 8(%rsi)
+; CHECK-NEXT: movl 12(%rdi), %eax
+; CHECK-NEXT: movl %eax, 12(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_2preds_block:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl %r9d, 12(%rdi)
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB3_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movl %edx, 4(%rdi)
+; DISABLED-NEXT: .LBB3_2: # %if.end
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_2preds_block:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB3_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movl %edx, 4(%rdi)
+; CHECK-AVX2-NEXT: .LBB3_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX2-NEXT: movl (%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, (%rsi)
+; CHECK-AVX2-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX2-NEXT: movl 8(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 8(%rsi)
+; CHECK-AVX2-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_2preds_block:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB3_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movl %edx, 4(%rdi)
+; CHECK-AVX512-NEXT: .LBB3_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX512-NEXT: movl (%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, (%rsi)
+; CHECK-AVX512-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX512-NEXT: movl 8(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 8(%rsi)
+; CHECK-AVX512-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %d = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 3
+ store i32 %x2, i32* %d, align 4
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1
+ store i32 %x, i32* %b, align 4
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S* %s3 to i8*
+ %1 = bitcast %struct.S* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false)
+ %2 = bitcast %struct.S* %s2 to i8*
+ %3 = bitcast %struct.S* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false)
+ ret void
+}
+%struct.S2 = type { i64, i64 }
+
+; Function Attrs: nounwind uwtable
+define void @test_type64(%struct.S2* nocapture noalias %s1, %struct.S2* nocapture %s2, i32 %x, %struct.S2* nocapture %s3, %struct.S2* nocapture readonly %s4) local_unnamed_addr #0 {
+; CHECK-LABEL: test_type64:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB4_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movslq %edx, %rax
+; CHECK-NEXT: movq %rax, 8(%rdi)
+; CHECK-NEXT: .LBB4_2: # %if.end
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movq (%rdi), %rax
+; CHECK-NEXT: movq %rax, (%rsi)
+; CHECK-NEXT: movq 8(%rdi), %rax
+; CHECK-NEXT: movq %rax, 8(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_type64:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB4_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movslq %edx, %rax
+; DISABLED-NEXT: movq %rax, 8(%rdi)
+; DISABLED-NEXT: .LBB4_2: # %if.end
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_type64:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB4_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movslq %edx, %rax
+; CHECK-AVX2-NEXT: movq %rax, 8(%rdi)
+; CHECK-AVX2-NEXT: .LBB4_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX2-NEXT: movq (%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, (%rsi)
+; CHECK-AVX2-NEXT: movq 8(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 8(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_type64:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB4_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movslq %edx, %rax
+; CHECK-AVX512-NEXT: movq %rax, 8(%rdi)
+; CHECK-AVX512-NEXT: .LBB4_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX512-NEXT: movq (%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, (%rsi)
+; CHECK-AVX512-NEXT: movq 8(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 8(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %conv = sext i32 %x to i64
+ %b = getelementptr inbounds %struct.S2, %struct.S2* %s1, i64 0, i32 1
+ store i64 %conv, i64* %b, align 8
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S2* %s3 to i8*
+ %1 = bitcast %struct.S2* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 8, i1 false)
+ %2 = bitcast %struct.S2* %s2 to i8*
+ %3 = bitcast %struct.S2* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 8, i1 false)
+ ret void
+}
+%struct.S3 = type { i64, i8, i8, i16, i32 }
+
+; Function Attrs: noinline nounwind uwtable
+define void @test_mixed_type(%struct.S3* nocapture noalias %s1, %struct.S3* nocapture %s2, i32 %x, %struct.S3* nocapture readnone %s3, %struct.S3* nocapture readnone %s4) local_unnamed_addr #0 {
+; CHECK-LABEL: test_mixed_type:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB5_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movslq %edx, %rax
+; CHECK-NEXT: movq %rax, (%rdi)
+; CHECK-NEXT: movb %dl, 8(%rdi)
+; CHECK-NEXT: .LBB5_2: # %if.end
+; CHECK-NEXT: movq (%rdi), %rax
+; CHECK-NEXT: movq %rax, (%rsi)
+; CHECK-NEXT: movb 8(%rdi), %al
+; CHECK-NEXT: movb %al, 8(%rsi)
+; CHECK-NEXT: movl 9(%rdi), %eax
+; CHECK-NEXT: movl %eax, 9(%rsi)
+; CHECK-NEXT: movzwl 13(%rdi), %eax
+; CHECK-NEXT: movw %ax, 13(%rsi)
+; CHECK-NEXT: movb 15(%rdi), %al
+; CHECK-NEXT: movb %al, 15(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_mixed_type:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB5_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movslq %edx, %rax
+; DISABLED-NEXT: movq %rax, (%rdi)
+; DISABLED-NEXT: movb %dl, 8(%rdi)
+; DISABLED-NEXT: .LBB5_2: # %if.end
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_mixed_type:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB5_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movslq %edx, %rax
+; CHECK-AVX2-NEXT: movq %rax, (%rdi)
+; CHECK-AVX2-NEXT: movb %dl, 8(%rdi)
+; CHECK-AVX2-NEXT: .LBB5_2: # %if.end
+; CHECK-AVX2-NEXT: movq (%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, (%rsi)
+; CHECK-AVX2-NEXT: movb 8(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 8(%rsi)
+; CHECK-AVX2-NEXT: movl 9(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 9(%rsi)
+; CHECK-AVX2-NEXT: movzwl 13(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 13(%rsi)
+; CHECK-AVX2-NEXT: movb 15(%rdi), %al
+; CHECK-AVX2-NEXT: movb %al, 15(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_mixed_type:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB5_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movslq %edx, %rax
+; CHECK-AVX512-NEXT: movq %rax, (%rdi)
+; CHECK-AVX512-NEXT: movb %dl, 8(%rdi)
+; CHECK-AVX512-NEXT: .LBB5_2: # %if.end
+; CHECK-AVX512-NEXT: movq (%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, (%rsi)
+; CHECK-AVX512-NEXT: movb 8(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 8(%rsi)
+; CHECK-AVX512-NEXT: movl 9(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 9(%rsi)
+; CHECK-AVX512-NEXT: movzwl 13(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 13(%rsi)
+; CHECK-AVX512-NEXT: movb 15(%rdi), %al
+; CHECK-AVX512-NEXT: movb %al, 15(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %conv = sext i32 %x to i64
+ %a = getelementptr inbounds %struct.S3, %struct.S3* %s1, i64 0, i32 0
+ store i64 %conv, i64* %a, align 8
+ %conv1 = trunc i32 %x to i8
+ %b = getelementptr inbounds %struct.S3, %struct.S3* %s1, i64 0, i32 1
+ store i8 %conv1, i8* %b, align 8
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S3* %s2 to i8*
+ %1 = bitcast %struct.S3* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 8, i1 false)
+ ret void
+}
+%struct.S4 = type { i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32, i32 }
+
+; Function Attrs: nounwind uwtable
+define void @test_multiple_blocks(%struct.S4* nocapture noalias %s1, %struct.S4* nocapture %s2) local_unnamed_addr #0 {
+; CHECK-LABEL: test_multiple_blocks:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl $0, 4(%rdi)
+; CHECK-NEXT: movl $0, 36(%rdi)
+; CHECK-NEXT: movups 16(%rdi), %xmm0
+; CHECK-NEXT: movups %xmm0, 16(%rsi)
+; CHECK-NEXT: movl 32(%rdi), %eax
+; CHECK-NEXT: movl %eax, 32(%rsi)
+; CHECK-NEXT: movl 36(%rdi), %eax
+; CHECK-NEXT: movl %eax, 36(%rsi)
+; CHECK-NEXT: movq 40(%rdi), %rax
+; CHECK-NEXT: movq %rax, 40(%rsi)
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: movl %eax, (%rsi)
+; CHECK-NEXT: movl 4(%rdi), %eax
+; CHECK-NEXT: movl %eax, 4(%rsi)
+; CHECK-NEXT: movq 8(%rdi), %rax
+; CHECK-NEXT: movq %rax, 8(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_multiple_blocks:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl $0, 4(%rdi)
+; DISABLED-NEXT: movl $0, 36(%rdi)
+; DISABLED-NEXT: movups 16(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rsi)
+; DISABLED-NEXT: movups 32(%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 32(%rsi)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_multiple_blocks:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl $0, 4(%rdi)
+; CHECK-AVX2-NEXT: movl $0, 36(%rdi)
+; CHECK-AVX2-NEXT: vmovups 16(%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, 16(%rsi)
+; CHECK-AVX2-NEXT: movl 32(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 32(%rsi)
+; CHECK-AVX2-NEXT: movl 36(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 36(%rsi)
+; CHECK-AVX2-NEXT: movq 40(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 40(%rsi)
+; CHECK-AVX2-NEXT: movl (%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, (%rsi)
+; CHECK-AVX2-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX2-NEXT: vmovups 8(%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, 8(%rsi)
+; CHECK-AVX2-NEXT: movq 24(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 24(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_multiple_blocks:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl $0, 4(%rdi)
+; CHECK-AVX512-NEXT: movl $0, 36(%rdi)
+; CHECK-AVX512-NEXT: vmovups 16(%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, 16(%rsi)
+; CHECK-AVX512-NEXT: movl 32(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 32(%rsi)
+; CHECK-AVX512-NEXT: movl 36(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 36(%rsi)
+; CHECK-AVX512-NEXT: movq 40(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 40(%rsi)
+; CHECK-AVX512-NEXT: movl (%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, (%rsi)
+; CHECK-AVX512-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX512-NEXT: vmovups 8(%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, 8(%rsi)
+; CHECK-AVX512-NEXT: movq 24(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 24(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %b = getelementptr inbounds %struct.S4, %struct.S4* %s1, i64 0, i32 1
+ store i32 0, i32* %b, align 4
+ %b3 = getelementptr inbounds %struct.S4, %struct.S4* %s1, i64 0, i32 9
+ store i32 0, i32* %b3, align 4
+ %0 = bitcast %struct.S4* %s2 to i8*
+ %1 = bitcast %struct.S4* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 48, i32 4, i1 false)
+ ret void
+}
+%struct.S5 = type { i16, i16, i16, i16, i16, i16, i16, i16 }
+
+; Function Attrs: nounwind uwtable
+define void @test_type16(%struct.S5* nocapture noalias %s1, %struct.S5* nocapture %s2, i32 %x, %struct.S5* nocapture %s3, %struct.S5* nocapture readonly %s4) local_unnamed_addr #0 {
+; CHECK-LABEL: test_type16:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB7_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movw %dx, 2(%rdi)
+; CHECK-NEXT: .LBB7_2: # %if.end
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movzwl (%rdi), %eax
+; CHECK-NEXT: movw %ax, (%rsi)
+; CHECK-NEXT: movzwl 2(%rdi), %eax
+; CHECK-NEXT: movw %ax, 2(%rsi)
+; CHECK-NEXT: movq 4(%rdi), %rax
+; CHECK-NEXT: movq %rax, 4(%rsi)
+; CHECK-NEXT: movl 12(%rdi), %eax
+; CHECK-NEXT: movl %eax, 12(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_type16:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB7_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movw %dx, 2(%rdi)
+; DISABLED-NEXT: .LBB7_2: # %if.end
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_type16:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB7_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movw %dx, 2(%rdi)
+; CHECK-AVX2-NEXT: .LBB7_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX2-NEXT: movzwl (%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, (%rsi)
+; CHECK-AVX2-NEXT: movzwl 2(%rdi), %eax
+; CHECK-AVX2-NEXT: movw %ax, 2(%rsi)
+; CHECK-AVX2-NEXT: movq 4(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 4(%rsi)
+; CHECK-AVX2-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_type16:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB7_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movw %dx, 2(%rdi)
+; CHECK-AVX512-NEXT: .LBB7_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r8), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rcx)
+; CHECK-AVX512-NEXT: movzwl (%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, (%rsi)
+; CHECK-AVX512-NEXT: movzwl 2(%rdi), %eax
+; CHECK-AVX512-NEXT: movw %ax, 2(%rsi)
+; CHECK-AVX512-NEXT: movq 4(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 4(%rsi)
+; CHECK-AVX512-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 12(%rsi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %conv = trunc i32 %x to i16
+ %b = getelementptr inbounds %struct.S5, %struct.S5* %s1, i64 0, i32 1
+ store i16 %conv, i16* %b, align 2
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S5* %s3 to i8*
+ %1 = bitcast %struct.S5* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 2, i1 false)
+ %2 = bitcast %struct.S5* %s2 to i8*
+ %3 = bitcast %struct.S5* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 2, i1 false)
+ ret void
+}
+
+%struct.S6 = type { [4 x i32], i32, i32, i32, i32 }
+
+; Function Attrs: nounwind uwtable
+define void @test_stack(%struct.S6* noalias nocapture sret %agg.result, %struct.S6* byval nocapture readnone align 8 %s1, %struct.S6* byval nocapture align 8 %s2, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_stack:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %esi, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
+; CHECK-NEXT: movups %xmm0, (%rdi)
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: movq %rax, 16(%rdi)
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT: movl %eax, 24(%rdi)
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-NEXT: movl %eax, 28(%rdi)
+; CHECK-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %ecx
+; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %edx
+; CHECK-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movl %ecx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movl %edx, {{[0-9]+}}(%rsp)
+; CHECK-NEXT: movq %rdi, %rax
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_stack:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl %esi, {{[0-9]+}}(%rsp)
+; DISABLED-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%rdi)
+; DISABLED-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
+; DISABLED-NEXT: movups %xmm0, 16(%rdi)
+; DISABLED-NEXT: movaps {{[0-9]+}}(%rsp), %xmm0
+; DISABLED-NEXT: movaps {{[0-9]+}}(%rsp), %xmm1
+; DISABLED-NEXT: movaps %xmm0, {{[0-9]+}}(%rsp)
+; DISABLED-NEXT: movaps %xmm1, {{[0-9]+}}(%rsp)
+; DISABLED-NEXT: movq %rdi, %rax
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_stack:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl %esi, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%rdi)
+; CHECK-AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-AVX2-NEXT: movq %rax, 16(%rdi)
+; CHECK-AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX2-NEXT: movl %eax, 24(%rdi)
+; CHECK-AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX2-NEXT: movl %eax, 28(%rdi)
+; CHECK-AVX2-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-AVX2-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX2-NEXT: movl %eax, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX2-NEXT: movl %eax, {{[0-9]+}}(%rsp)
+; CHECK-AVX2-NEXT: movq %rdi, %rax
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_stack:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl %esi, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%rdi)
+; CHECK-AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-AVX512-NEXT: movq %rax, 16(%rdi)
+; CHECK-AVX512-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX512-NEXT: movl %eax, 24(%rdi)
+; CHECK-AVX512-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX512-NEXT: movl %eax, 28(%rdi)
+; CHECK-AVX512-NEXT: vmovups {{[0-9]+}}(%rsp), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; CHECK-AVX512-NEXT: movq %rax, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX512-NEXT: movl %eax, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; CHECK-AVX512-NEXT: movl %eax, {{[0-9]+}}(%rsp)
+; CHECK-AVX512-NEXT: movq %rdi, %rax
+; CHECK-AVX512-NEXT: retq
+entry:
+ %s6.sroa.0.0..sroa_cast1 = bitcast %struct.S6* %s2 to i8*
+ %s6.sroa.3.0..sroa_idx4 = getelementptr inbounds %struct.S6, %struct.S6* %s2, i64 0, i32 3
+ store i32 %x, i32* %s6.sroa.3.0..sroa_idx4, align 8
+ %0 = bitcast %struct.S6* %agg.result to i8*
+ %s6.sroa.0.0..sroa_cast2 = bitcast %struct.S6* %s1 to i8*
+ call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* nonnull %s6.sroa.0.0..sroa_cast1, i64 32, i32 4, i1 false)
+ call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull %s6.sroa.0.0..sroa_cast2, i8* nonnull %s6.sroa.0.0..sroa_cast1, i64 32, i32 4, i1 false)
+
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @test_limit_all(%struct.S* noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3, %struct.S* nocapture readonly %s4, i32 %x2) local_unnamed_addr #0 {
+; CHECK-LABEL: test_limit_all:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: pushq %rbp
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: pushq %r15
+; CHECK-NEXT: .cfi_def_cfa_offset 24
+; CHECK-NEXT: pushq %r14
+; CHECK-NEXT: .cfi_def_cfa_offset 32
+; CHECK-NEXT: pushq %r12
+; CHECK-NEXT: .cfi_def_cfa_offset 40
+; CHECK-NEXT: pushq %rbx
+; CHECK-NEXT: .cfi_def_cfa_offset 48
+; CHECK-NEXT: .cfi_offset %rbx, -48
+; CHECK-NEXT: .cfi_offset %r12, -40
+; CHECK-NEXT: .cfi_offset %r14, -32
+; CHECK-NEXT: .cfi_offset %r15, -24
+; CHECK-NEXT: .cfi_offset %rbp, -16
+; CHECK-NEXT: movq %r8, %r15
+; CHECK-NEXT: movq %rcx, %r14
+; CHECK-NEXT: movl %edx, %ebp
+; CHECK-NEXT: movq %rsi, %r12
+; CHECK-NEXT: movq %rdi, %rbx
+; CHECK-NEXT: movl %r9d, 12(%rdi)
+; CHECK-NEXT: callq bar
+; CHECK-NEXT: cmpl $18, %ebp
+; CHECK-NEXT: jl .LBB9_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movl %ebp, 4(%rbx)
+; CHECK-NEXT: movq %rbx, %rdi
+; CHECK-NEXT: callq bar
+; CHECK-NEXT: .LBB9_2: # %if.end
+; CHECK-NEXT: movups (%r15), %xmm0
+; CHECK-NEXT: movups %xmm0, (%r14)
+; CHECK-NEXT: movups (%rbx), %xmm0
+; CHECK-NEXT: movups %xmm0, (%r12)
+; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: popq %r12
+; CHECK-NEXT: popq %r14
+; CHECK-NEXT: popq %r15
+; CHECK-NEXT: popq %rbp
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_limit_all:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: pushq %rbp
+; DISABLED-NEXT: .cfi_def_cfa_offset 16
+; DISABLED-NEXT: pushq %r15
+; DISABLED-NEXT: .cfi_def_cfa_offset 24
+; DISABLED-NEXT: pushq %r14
+; DISABLED-NEXT: .cfi_def_cfa_offset 32
+; DISABLED-NEXT: pushq %r12
+; DISABLED-NEXT: .cfi_def_cfa_offset 40
+; DISABLED-NEXT: pushq %rbx
+; DISABLED-NEXT: .cfi_def_cfa_offset 48
+; DISABLED-NEXT: .cfi_offset %rbx, -48
+; DISABLED-NEXT: .cfi_offset %r12, -40
+; DISABLED-NEXT: .cfi_offset %r14, -32
+; DISABLED-NEXT: .cfi_offset %r15, -24
+; DISABLED-NEXT: .cfi_offset %rbp, -16
+; DISABLED-NEXT: movq %r8, %r15
+; DISABLED-NEXT: movq %rcx, %r14
+; DISABLED-NEXT: movl %edx, %ebp
+; DISABLED-NEXT: movq %rsi, %r12
+; DISABLED-NEXT: movq %rdi, %rbx
+; DISABLED-NEXT: movl %r9d, 12(%rdi)
+; DISABLED-NEXT: callq bar
+; DISABLED-NEXT: cmpl $18, %ebp
+; DISABLED-NEXT: jl .LBB9_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movl %ebp, 4(%rbx)
+; DISABLED-NEXT: movq %rbx, %rdi
+; DISABLED-NEXT: callq bar
+; DISABLED-NEXT: .LBB9_2: # %if.end
+; DISABLED-NEXT: movups (%r15), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%r14)
+; DISABLED-NEXT: movups (%rbx), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%r12)
+; DISABLED-NEXT: popq %rbx
+; DISABLED-NEXT: popq %r12
+; DISABLED-NEXT: popq %r14
+; DISABLED-NEXT: popq %r15
+; DISABLED-NEXT: popq %rbp
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_limit_all:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: pushq %rbp
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 16
+; CHECK-AVX2-NEXT: pushq %r15
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 24
+; CHECK-AVX2-NEXT: pushq %r14
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 32
+; CHECK-AVX2-NEXT: pushq %r12
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 40
+; CHECK-AVX2-NEXT: pushq %rbx
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 48
+; CHECK-AVX2-NEXT: .cfi_offset %rbx, -48
+; CHECK-AVX2-NEXT: .cfi_offset %r12, -40
+; CHECK-AVX2-NEXT: .cfi_offset %r14, -32
+; CHECK-AVX2-NEXT: .cfi_offset %r15, -24
+; CHECK-AVX2-NEXT: .cfi_offset %rbp, -16
+; CHECK-AVX2-NEXT: movq %r8, %r15
+; CHECK-AVX2-NEXT: movq %rcx, %r14
+; CHECK-AVX2-NEXT: movl %edx, %ebp
+; CHECK-AVX2-NEXT: movq %rsi, %r12
+; CHECK-AVX2-NEXT: movq %rdi, %rbx
+; CHECK-AVX2-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX2-NEXT: callq bar
+; CHECK-AVX2-NEXT: cmpl $18, %ebp
+; CHECK-AVX2-NEXT: jl .LBB9_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movl %ebp, 4(%rbx)
+; CHECK-AVX2-NEXT: movq %rbx, %rdi
+; CHECK-AVX2-NEXT: callq bar
+; CHECK-AVX2-NEXT: .LBB9_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r15), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%r14)
+; CHECK-AVX2-NEXT: vmovups (%rbx), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%r12)
+; CHECK-AVX2-NEXT: popq %rbx
+; CHECK-AVX2-NEXT: popq %r12
+; CHECK-AVX2-NEXT: popq %r14
+; CHECK-AVX2-NEXT: popq %r15
+; CHECK-AVX2-NEXT: popq %rbp
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_limit_all:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: pushq %rbp
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 16
+; CHECK-AVX512-NEXT: pushq %r15
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 24
+; CHECK-AVX512-NEXT: pushq %r14
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 32
+; CHECK-AVX512-NEXT: pushq %r12
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 40
+; CHECK-AVX512-NEXT: pushq %rbx
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 48
+; CHECK-AVX512-NEXT: .cfi_offset %rbx, -48
+; CHECK-AVX512-NEXT: .cfi_offset %r12, -40
+; CHECK-AVX512-NEXT: .cfi_offset %r14, -32
+; CHECK-AVX512-NEXT: .cfi_offset %r15, -24
+; CHECK-AVX512-NEXT: .cfi_offset %rbp, -16
+; CHECK-AVX512-NEXT: movq %r8, %r15
+; CHECK-AVX512-NEXT: movq %rcx, %r14
+; CHECK-AVX512-NEXT: movl %edx, %ebp
+; CHECK-AVX512-NEXT: movq %rsi, %r12
+; CHECK-AVX512-NEXT: movq %rdi, %rbx
+; CHECK-AVX512-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX512-NEXT: callq bar
+; CHECK-AVX512-NEXT: cmpl $18, %ebp
+; CHECK-AVX512-NEXT: jl .LBB9_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movl %ebp, 4(%rbx)
+; CHECK-AVX512-NEXT: movq %rbx, %rdi
+; CHECK-AVX512-NEXT: callq bar
+; CHECK-AVX512-NEXT: .LBB9_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r15), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%r14)
+; CHECK-AVX512-NEXT: vmovups (%rbx), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%r12)
+; CHECK-AVX512-NEXT: popq %rbx
+; CHECK-AVX512-NEXT: popq %r12
+; CHECK-AVX512-NEXT: popq %r14
+; CHECK-AVX512-NEXT: popq %r15
+; CHECK-AVX512-NEXT: popq %rbp
+; CHECK-AVX512-NEXT: retq
+entry:
+ %d = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 3
+ store i32 %x2, i32* %d, align 4
+ tail call void @bar(%struct.S* %s1) #3
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1
+ store i32 %x, i32* %b, align 4
+ tail call void @bar(%struct.S* nonnull %s1) #3
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S* %s3 to i8*
+ %1 = bitcast %struct.S* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false)
+ %2 = bitcast %struct.S* %s2 to i8*
+ %3 = bitcast %struct.S* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define void @test_limit_one_pred(%struct.S* noalias %s1, %struct.S* nocapture %s2, i32 %x, %struct.S* nocapture %s3, %struct.S* nocapture readonly %s4, i32 %x2) local_unnamed_addr #0 {
+; CHECK-LABEL: test_limit_one_pred:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: pushq %r15
+; CHECK-NEXT: .cfi_def_cfa_offset 16
+; CHECK-NEXT: pushq %r14
+; CHECK-NEXT: .cfi_def_cfa_offset 24
+; CHECK-NEXT: pushq %r12
+; CHECK-NEXT: .cfi_def_cfa_offset 32
+; CHECK-NEXT: pushq %rbx
+; CHECK-NEXT: .cfi_def_cfa_offset 40
+; CHECK-NEXT: pushq %rax
+; CHECK-NEXT: .cfi_def_cfa_offset 48
+; CHECK-NEXT: .cfi_offset %rbx, -40
+; CHECK-NEXT: .cfi_offset %r12, -32
+; CHECK-NEXT: .cfi_offset %r14, -24
+; CHECK-NEXT: .cfi_offset %r15, -16
+; CHECK-NEXT: movq %r8, %r12
+; CHECK-NEXT: movq %rcx, %r15
+; CHECK-NEXT: movq %rsi, %r14
+; CHECK-NEXT: movq %rdi, %rbx
+; CHECK-NEXT: movl %r9d, 12(%rdi)
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB10_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movl %edx, 4(%rbx)
+; CHECK-NEXT: movq %rbx, %rdi
+; CHECK-NEXT: callq bar
+; CHECK-NEXT: .LBB10_2: # %if.end
+; CHECK-NEXT: movups (%r12), %xmm0
+; CHECK-NEXT: movups %xmm0, (%r15)
+; CHECK-NEXT: movq (%rbx), %rax
+; CHECK-NEXT: movq %rax, (%r14)
+; CHECK-NEXT: movl 8(%rbx), %eax
+; CHECK-NEXT: movl %eax, 8(%r14)
+; CHECK-NEXT: movl 12(%rbx), %eax
+; CHECK-NEXT: movl %eax, 12(%r14)
+; CHECK-NEXT: addq $8, %rsp
+; CHECK-NEXT: popq %rbx
+; CHECK-NEXT: popq %r12
+; CHECK-NEXT: popq %r14
+; CHECK-NEXT: popq %r15
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_limit_one_pred:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: pushq %r15
+; DISABLED-NEXT: .cfi_def_cfa_offset 16
+; DISABLED-NEXT: pushq %r14
+; DISABLED-NEXT: .cfi_def_cfa_offset 24
+; DISABLED-NEXT: pushq %r12
+; DISABLED-NEXT: .cfi_def_cfa_offset 32
+; DISABLED-NEXT: pushq %rbx
+; DISABLED-NEXT: .cfi_def_cfa_offset 40
+; DISABLED-NEXT: pushq %rax
+; DISABLED-NEXT: .cfi_def_cfa_offset 48
+; DISABLED-NEXT: .cfi_offset %rbx, -40
+; DISABLED-NEXT: .cfi_offset %r12, -32
+; DISABLED-NEXT: .cfi_offset %r14, -24
+; DISABLED-NEXT: .cfi_offset %r15, -16
+; DISABLED-NEXT: movq %r8, %r15
+; DISABLED-NEXT: movq %rcx, %r14
+; DISABLED-NEXT: movq %rsi, %r12
+; DISABLED-NEXT: movq %rdi, %rbx
+; DISABLED-NEXT: movl %r9d, 12(%rdi)
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB10_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movl %edx, 4(%rbx)
+; DISABLED-NEXT: movq %rbx, %rdi
+; DISABLED-NEXT: callq bar
+; DISABLED-NEXT: .LBB10_2: # %if.end
+; DISABLED-NEXT: movups (%r15), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%r14)
+; DISABLED-NEXT: movups (%rbx), %xmm0
+; DISABLED-NEXT: movups %xmm0, (%r12)
+; DISABLED-NEXT: addq $8, %rsp
+; DISABLED-NEXT: popq %rbx
+; DISABLED-NEXT: popq %r12
+; DISABLED-NEXT: popq %r14
+; DISABLED-NEXT: popq %r15
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_limit_one_pred:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: pushq %r15
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 16
+; CHECK-AVX2-NEXT: pushq %r14
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 24
+; CHECK-AVX2-NEXT: pushq %r12
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 32
+; CHECK-AVX2-NEXT: pushq %rbx
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 40
+; CHECK-AVX2-NEXT: pushq %rax
+; CHECK-AVX2-NEXT: .cfi_def_cfa_offset 48
+; CHECK-AVX2-NEXT: .cfi_offset %rbx, -40
+; CHECK-AVX2-NEXT: .cfi_offset %r12, -32
+; CHECK-AVX2-NEXT: .cfi_offset %r14, -24
+; CHECK-AVX2-NEXT: .cfi_offset %r15, -16
+; CHECK-AVX2-NEXT: movq %r8, %r12
+; CHECK-AVX2-NEXT: movq %rcx, %r15
+; CHECK-AVX2-NEXT: movq %rsi, %r14
+; CHECK-AVX2-NEXT: movq %rdi, %rbx
+; CHECK-AVX2-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB10_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movl %edx, 4(%rbx)
+; CHECK-AVX2-NEXT: movq %rbx, %rdi
+; CHECK-AVX2-NEXT: callq bar
+; CHECK-AVX2-NEXT: .LBB10_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r12), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, (%r15)
+; CHECK-AVX2-NEXT: movq (%rbx), %rax
+; CHECK-AVX2-NEXT: movq %rax, (%r14)
+; CHECK-AVX2-NEXT: movl 8(%rbx), %eax
+; CHECK-AVX2-NEXT: movl %eax, 8(%r14)
+; CHECK-AVX2-NEXT: movl 12(%rbx), %eax
+; CHECK-AVX2-NEXT: movl %eax, 12(%r14)
+; CHECK-AVX2-NEXT: addq $8, %rsp
+; CHECK-AVX2-NEXT: popq %rbx
+; CHECK-AVX2-NEXT: popq %r12
+; CHECK-AVX2-NEXT: popq %r14
+; CHECK-AVX2-NEXT: popq %r15
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_limit_one_pred:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: pushq %r15
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 16
+; CHECK-AVX512-NEXT: pushq %r14
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 24
+; CHECK-AVX512-NEXT: pushq %r12
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 32
+; CHECK-AVX512-NEXT: pushq %rbx
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 40
+; CHECK-AVX512-NEXT: pushq %rax
+; CHECK-AVX512-NEXT: .cfi_def_cfa_offset 48
+; CHECK-AVX512-NEXT: .cfi_offset %rbx, -40
+; CHECK-AVX512-NEXT: .cfi_offset %r12, -32
+; CHECK-AVX512-NEXT: .cfi_offset %r14, -24
+; CHECK-AVX512-NEXT: .cfi_offset %r15, -16
+; CHECK-AVX512-NEXT: movq %r8, %r12
+; CHECK-AVX512-NEXT: movq %rcx, %r15
+; CHECK-AVX512-NEXT: movq %rsi, %r14
+; CHECK-AVX512-NEXT: movq %rdi, %rbx
+; CHECK-AVX512-NEXT: movl %r9d, 12(%rdi)
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB10_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movl %edx, 4(%rbx)
+; CHECK-AVX512-NEXT: movq %rbx, %rdi
+; CHECK-AVX512-NEXT: callq bar
+; CHECK-AVX512-NEXT: .LBB10_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r12), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, (%r15)
+; CHECK-AVX512-NEXT: movq (%rbx), %rax
+; CHECK-AVX512-NEXT: movq %rax, (%r14)
+; CHECK-AVX512-NEXT: movl 8(%rbx), %eax
+; CHECK-AVX512-NEXT: movl %eax, 8(%r14)
+; CHECK-AVX512-NEXT: movl 12(%rbx), %eax
+; CHECK-AVX512-NEXT: movl %eax, 12(%r14)
+; CHECK-AVX512-NEXT: addq $8, %rsp
+; CHECK-AVX512-NEXT: popq %rbx
+; CHECK-AVX512-NEXT: popq %r12
+; CHECK-AVX512-NEXT: popq %r14
+; CHECK-AVX512-NEXT: popq %r15
+; CHECK-AVX512-NEXT: retq
+entry:
+ %d = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 3
+ store i32 %x2, i32* %d, align 4
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S, %struct.S* %s1, i64 0, i32 1
+ store i32 %x, i32* %b, align 4
+ tail call void @bar(%struct.S* nonnull %s1) #3
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S* %s3 to i8*
+ %1 = bitcast %struct.S* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 16, i32 4, i1 false)
+ %2 = bitcast %struct.S* %s2 to i8*
+ %3 = bitcast %struct.S* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 16, i32 4, i1 false)
+ ret void
+}
+
+
+declare void @bar(%struct.S*) local_unnamed_addr #1
+
+
+; Function Attrs: argmemonly nounwind
+declare void @llvm.memcpy.p0i8.p0i8.i64(i8* nocapture writeonly, i8* nocapture readonly, i64, i32, i1) #1
+
+attributes #0 = { nounwind uwtable "target-cpu"="x86-64" }
+
+%struct.S7 = type { float, float, float , float, float, float, float, float }
+
+; Function Attrs: nounwind uwtable
+define void @test_conditional_block_float(%struct.S7* nocapture noalias %s1, %struct.S7* nocapture %s2, i32 %x, %struct.S7* nocapture %s3, %struct.S7* nocapture readonly %s4, float %y) local_unnamed_addr #0 {
+; CHECK-LABEL: test_conditional_block_float:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB11_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movl $1065353216, 4(%rdi) # imm = 0x3F800000
+; CHECK-NEXT: .LBB11_2: # %if.end
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups 16(%r8), %xmm1
+; CHECK-NEXT: movups %xmm1, 16(%rcx)
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: movl 4(%rdi), %ecx
+; CHECK-NEXT: movq 8(%rdi), %rdx
+; CHECK-NEXT: movups 16(%rdi), %xmm0
+; CHECK-NEXT: movups %xmm0, 16(%rsi)
+; CHECK-NEXT: movl %eax, (%rsi)
+; CHECK-NEXT: movl %ecx, 4(%rsi)
+; CHECK-NEXT: movq %rdx, 8(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_conditional_block_float:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB11_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movl $1065353216, 4(%rdi) # imm = 0x3F800000
+; DISABLED-NEXT: .LBB11_2: # %if.end
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups 16(%r8), %xmm1
+; DISABLED-NEXT: movups %xmm1, 16(%rcx)
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups 16(%rdi), %xmm1
+; DISABLED-NEXT: movups %xmm1, 16(%rsi)
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_conditional_block_float:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB11_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movl $1065353216, 4(%rdi) # imm = 0x3F800000
+; CHECK-AVX2-NEXT: .LBB11_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r8), %ymm0
+; CHECK-AVX2-NEXT: vmovups %ymm0, (%rcx)
+; CHECK-AVX2-NEXT: movl (%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, (%rsi)
+; CHECK-AVX2-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX2-NEXT: vmovups 8(%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, 8(%rsi)
+; CHECK-AVX2-NEXT: movq 24(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 24(%rsi)
+; CHECK-AVX2-NEXT: vzeroupper
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_conditional_block_float:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB11_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movl $1065353216, 4(%rdi) # imm = 0x3F800000
+; CHECK-AVX512-NEXT: .LBB11_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r8), %ymm0
+; CHECK-AVX512-NEXT: vmovups %ymm0, (%rcx)
+; CHECK-AVX512-NEXT: movl (%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, (%rsi)
+; CHECK-AVX512-NEXT: movl 4(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 4(%rsi)
+; CHECK-AVX512-NEXT: vmovups 8(%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, 8(%rsi)
+; CHECK-AVX512-NEXT: movq 24(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 24(%rsi)
+; CHECK-AVX512-NEXT: vzeroupper
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S7, %struct.S7* %s1, i64 0, i32 1
+ store float 1.0, float* %b, align 4
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S7* %s3 to i8*
+ %1 = bitcast %struct.S7* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 32, i32 4, i1 false)
+ %2 = bitcast %struct.S7* %s2 to i8*
+ %3 = bitcast %struct.S7* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 32, i32 4, i1 false)
+ ret void
+}
+
+%struct.S8 = type { i64, i64, i64, i64, i64, i64 }
+
+; Function Attrs: nounwind uwtable
+define void @test_conditional_block_ymm(%struct.S8* nocapture noalias %s1, %struct.S8* nocapture %s2, i32 %x, %struct.S8* nocapture %s3, %struct.S8* nocapture readonly %s4) local_unnamed_addr #0 {
+; CHECK-LABEL: test_conditional_block_ymm:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: cmpl $18, %edx
+; CHECK-NEXT: jl .LBB12_2
+; CHECK-NEXT: # %bb.1: # %if.then
+; CHECK-NEXT: movq $1, 8(%rdi)
+; CHECK-NEXT: .LBB12_2: # %if.end
+; CHECK-NEXT: movups (%r8), %xmm0
+; CHECK-NEXT: movups 16(%r8), %xmm1
+; CHECK-NEXT: movups %xmm1, 16(%rcx)
+; CHECK-NEXT: movups %xmm0, (%rcx)
+; CHECK-NEXT: movq (%rdi), %rax
+; CHECK-NEXT: movq 8(%rdi), %rcx
+; CHECK-NEXT: movups 16(%rdi), %xmm0
+; CHECK-NEXT: movups %xmm0, 16(%rsi)
+; CHECK-NEXT: movq %rax, (%rsi)
+; CHECK-NEXT: movq %rcx, 8(%rsi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_conditional_block_ymm:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: cmpl $18, %edx
+; DISABLED-NEXT: jl .LBB12_2
+; DISABLED-NEXT: # %bb.1: # %if.then
+; DISABLED-NEXT: movq $1, 8(%rdi)
+; DISABLED-NEXT: .LBB12_2: # %if.end
+; DISABLED-NEXT: movups (%r8), %xmm0
+; DISABLED-NEXT: movups 16(%r8), %xmm1
+; DISABLED-NEXT: movups %xmm1, 16(%rcx)
+; DISABLED-NEXT: movups %xmm0, (%rcx)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups 16(%rdi), %xmm1
+; DISABLED-NEXT: movups %xmm1, 16(%rsi)
+; DISABLED-NEXT: movups %xmm0, (%rsi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_conditional_block_ymm:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: cmpl $18, %edx
+; CHECK-AVX2-NEXT: jl .LBB12_2
+; CHECK-AVX2-NEXT: # %bb.1: # %if.then
+; CHECK-AVX2-NEXT: movq $1, 8(%rdi)
+; CHECK-AVX2-NEXT: .LBB12_2: # %if.end
+; CHECK-AVX2-NEXT: vmovups (%r8), %ymm0
+; CHECK-AVX2-NEXT: vmovups %ymm0, (%rcx)
+; CHECK-AVX2-NEXT: movq (%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, (%rsi)
+; CHECK-AVX2-NEXT: movq 8(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 8(%rsi)
+; CHECK-AVX2-NEXT: vmovups 16(%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, 16(%rsi)
+; CHECK-AVX2-NEXT: vzeroupper
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_conditional_block_ymm:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: cmpl $18, %edx
+; CHECK-AVX512-NEXT: jl .LBB12_2
+; CHECK-AVX512-NEXT: # %bb.1: # %if.then
+; CHECK-AVX512-NEXT: movq $1, 8(%rdi)
+; CHECK-AVX512-NEXT: .LBB12_2: # %if.end
+; CHECK-AVX512-NEXT: vmovups (%r8), %ymm0
+; CHECK-AVX512-NEXT: vmovups %ymm0, (%rcx)
+; CHECK-AVX512-NEXT: movq (%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, (%rsi)
+; CHECK-AVX512-NEXT: movq 8(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 8(%rsi)
+; CHECK-AVX512-NEXT: vmovups 16(%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, 16(%rsi)
+; CHECK-AVX512-NEXT: vzeroupper
+; CHECK-AVX512-NEXT: retq
+entry:
+ %cmp = icmp sgt i32 %x, 17
+ br i1 %cmp, label %if.then, label %if.end
+
+if.then: ; preds = %entry
+ %b = getelementptr inbounds %struct.S8, %struct.S8* %s1, i64 0, i32 1
+ store i64 1, i64* %b, align 4
+ br label %if.end
+
+if.end: ; preds = %if.then, %entry
+ %0 = bitcast %struct.S8* %s3 to i8*
+ %1 = bitcast %struct.S8* %s4 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %0, i8* %1, i64 32, i32 4, i1 false)
+ %2 = bitcast %struct.S8* %s2 to i8*
+ %3 = bitcast %struct.S8* %s1 to i8*
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* %2, i8* %3, i64 32, i32 4, i1 false)
+ ret void
+}
+
+define dso_local void @test_alias(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_alias:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %esi, (%rdi)
+; CHECK-NEXT: movups (%rdi), %xmm0
+; CHECK-NEXT: movups %xmm0, 4(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_alias:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl %esi, (%rdi)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 4(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_alias:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl %esi, (%rdi)
+; CHECK-AVX2-NEXT: vmovups (%rdi), %xmm0
+; CHECK-AVX2-NEXT: vmovups %xmm0, 4(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_alias:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl %esi, (%rdi)
+; CHECK-AVX512-NEXT: vmovups (%rdi), %xmm0
+; CHECK-AVX512-NEXT: vmovups %xmm0, 4(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %a = bitcast i8* %A to i32*
+ store i32 %x, i32* %a, align 4
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 4
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr, i8* align 4 %A, i64 16, i32 4, i1 false)
+ ret void
+}
+
+; Function Attrs: nounwind uwtable
+define dso_local void @test_noalias(i8* nocapture %A, i32 %x) local_unnamed_addr #0 {
+; CHECK-LABEL: test_noalias:
+; CHECK: # %bb.0: # %entry
+; CHECK-NEXT: movl %esi, (%rdi)
+; CHECK-NEXT: movl (%rdi), %eax
+; CHECK-NEXT: movl %eax, 20(%rdi)
+; CHECK-NEXT: movq 4(%rdi), %rax
+; CHECK-NEXT: movq %rax, 24(%rdi)
+; CHECK-NEXT: movl 12(%rdi), %eax
+; CHECK-NEXT: movl %eax, 32(%rdi)
+; CHECK-NEXT: retq
+;
+; DISABLED-LABEL: test_noalias:
+; DISABLED: # %bb.0: # %entry
+; DISABLED-NEXT: movl %esi, (%rdi)
+; DISABLED-NEXT: movups (%rdi), %xmm0
+; DISABLED-NEXT: movups %xmm0, 20(%rdi)
+; DISABLED-NEXT: retq
+;
+; CHECK-AVX2-LABEL: test_noalias:
+; CHECK-AVX2: # %bb.0: # %entry
+; CHECK-AVX2-NEXT: movl %esi, (%rdi)
+; CHECK-AVX2-NEXT: movl (%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 20(%rdi)
+; CHECK-AVX2-NEXT: movq 4(%rdi), %rax
+; CHECK-AVX2-NEXT: movq %rax, 24(%rdi)
+; CHECK-AVX2-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX2-NEXT: movl %eax, 32(%rdi)
+; CHECK-AVX2-NEXT: retq
+;
+; CHECK-AVX512-LABEL: test_noalias:
+; CHECK-AVX512: # %bb.0: # %entry
+; CHECK-AVX512-NEXT: movl %esi, (%rdi)
+; CHECK-AVX512-NEXT: movl (%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 20(%rdi)
+; CHECK-AVX512-NEXT: movq 4(%rdi), %rax
+; CHECK-AVX512-NEXT: movq %rax, 24(%rdi)
+; CHECK-AVX512-NEXT: movl 12(%rdi), %eax
+; CHECK-AVX512-NEXT: movl %eax, 32(%rdi)
+; CHECK-AVX512-NEXT: retq
+entry:
+ %a = bitcast i8* %A to i32*
+ store i32 %x, i32* %a, align 4
+ %add.ptr = getelementptr inbounds i8, i8* %A, i64 20
+ tail call void @llvm.memcpy.p0i8.p0i8.i64(i8* nonnull align 4 %add.ptr, i8* align 4 %A, i64 16, i32 4, i1 false)
+ ret void
+}
+
+
+
More information about the llvm-commits
mailing list