<div dir="ltr">This commit broke buildbots. For one example, see <a href="http://lab.llvm.org:8011/builders/clang-with-thin-lto-ubuntu/builds/5290/">http://lab.llvm.org:8011/builders/clang-with-thin-lto-ubuntu/builds/5290/</a>.<div><br></div><div>Reverted by r313352.</div></div><div class="gmail_extra"><br><div class="gmail_quote">On Fri, Sep 15, 2017 at 8:56 AM, Dinar Temirbulatov via llvm-commits <span dir="ltr"><<a href="mailto:llvm-commits@lists.llvm.org" target="_blank">llvm-commits@lists.llvm.org</a>></span> wrote:<br><blockquote class="gmail_quote" style="margin:0 0 0 .8ex;border-left:1px #ccc solid;padding-left:1ex">Author: dinar<br>
Date: Thu Sep 14 23:56:39 2017<br>
New Revision: 313348<br>
<br>
URL: <a href="http://llvm.org/viewvc/llvm-project?rev=313348&view=rev" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-<wbr>project?rev=313348&view=rev</a><br>
Log:<br>
[SLPVectorizer] Failure to beneficially vectorize 'copyable' elements in integer binary ops.<br>
<br>
Patch tries to improve vectorization of the following code:<br>
<br>
void add1(int * __restrict dst, const int * __restrict src) {<br>
*dst++ = *src++;<br>
*dst++ = *src++ + 1;<br>
*dst++ = *src++ + 2;<br>
*dst++ = *src++ + 3;<br>
}<br>
Allows to vectorize even if the very first operation is not a binary add, but just a load.<br>
<br>
Reviewers: spatel, mzolotukhin, mkuper, hfinkel, RKSimon, filcab, ABataev, davide<br>
<br>
Subscribers: llvm-commits, RKSimon<br>
<br>
Differential Revision: <a href="https://reviews.llvm.org/D28907" rel="noreferrer" target="_blank">https://reviews.llvm.org/<wbr>D28907</a><br>
<br>
Modified:<br>
llvm/trunk/lib/Transforms/<wbr>Vectorize/SLPVectorizer.cpp<br>
llvm/trunk/test/Transforms/<wbr>SLPVectorizer/X86/vect_<wbr>copyable_in_binops.ll<br>
<br>
Modified: llvm/trunk/lib/Transforms/<wbr>Vectorize/SLPVectorizer.cpp<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Transforms/Vectorize/SLPVectorizer.cpp?rev=313348&r1=313347&r2=313348&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-<wbr>project/llvm/trunk/lib/<wbr>Transforms/Vectorize/<wbr>SLPVectorizer.cpp?rev=313348&<wbr>r1=313347&r2=313348&view=diff</a><br>
==============================<wbr>==============================<wbr>==================<br>
--- llvm/trunk/lib/Transforms/<wbr>Vectorize/SLPVectorizer.cpp (original)<br>
+++ llvm/trunk/lib/Transforms/<wbr>Vectorize/SLPVectorizer.cpp Thu Sep 14 23:56:39 2017<br>
@@ -332,7 +332,7 @@ static unsigned getAltOpcode(unsigned Op<br>
case Instruction::Sub:<br>
return Instruction::Add;<br>
default:<br>
- return 0;<br>
+ return Op;<br>
}<br>
}<br>
<br>
@@ -345,6 +345,20 @@ static bool sameOpcodeOrAlt(unsigned Opc<br>
return Opcode == CheckedOpcode || AltOpcode == CheckedOpcode;<br>
}<br>
<br>
+/// Checks if the \p Opcode can be considered as an operand of a (possibly)<br>
+/// binary operation \p I.<br>
+/// \returns The code of the binary operation of instruction \p I if the<br>
+/// instruction with \p Opcode can be considered as an operand of \p I with the<br>
+/// default value.<br>
+static unsigned tryToRepresentAsInstArg(<wbr>unsigned Opcode, Instruction *I) {<br>
+ assert(!sameOpcodeOrAlt(<wbr>Opcode, getAltOpcode(Opcode), I->getOpcode())<br>
+ && "Invalid Opcode");<br>
+ if (Opcode != Instruction::PHI && isa<BinaryOperator>(I) &&<br>
+ (I->getType()->isIntegerTy() || I->hasUnsafeAlgebra()))<br>
+ return I->getOpcode();<br>
+ return 0;<br>
+}<br>
+<br>
/// Chooses the correct key for scheduling data. If \p Op has the same (or<br>
/// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p<br>
/// OpValue.<br>
@@ -365,6 +379,12 @@ namespace {<br>
struct RawInstructionsData {<br>
/// Main Opcode of the instructions going to be vectorized.<br>
unsigned Opcode = 0;<br>
+ /// Position of the first instruction with the \a Opcode.<br>
+ unsigned OpcodePos = 0;<br>
+ /// Need an additional analysis (if at least one of the instruction is not<br>
+ /// same instruction kind as an instruction at OpcodePos position in the<br>
+ /// list).<br>
+ bool NeedAnalysis = false;<br>
/// The list of instructions have some instructions with alternate opcodes.<br>
bool HasAltOpcodes = false;<br>
};<br>
@@ -378,16 +398,38 @@ static RawInstructionsData getMainOpcode<br>
return {};<br>
RawInstructionsData Res;<br>
unsigned Opcode = I0->getOpcode();<br>
+ unsigned AltOpcode = getAltOpcode(Opcode);<br>
+ unsigned NewOpcodePos = 0;<br>
// Walk through the list of the vectorized instructions<br>
// in order to check its structure described by RawInstructionsData.<br>
for (unsigned Cnt = 0, E = VL.size(); Cnt != E; ++Cnt) {<br>
auto *I = dyn_cast<Instruction>(VL[Cnt])<wbr>;<br>
if (!I)<br>
return {};<br>
- if (Opcode != I->getOpcode())<br>
- Res.HasAltOpcodes = true;<br>
+ if (sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode())) {<br>
+ if (Opcode != I->getOpcode()) {<br>
+ Res.HasAltOpcodes = true;<br>
+ if (Res.NeedAnalysis && isOdd(NewOpcodePos))<br>
+ std::swap(Opcode, AltOpcode);<br>
+ }<br>
+ continue;<br>
+ }<br>
+ if (unsigned NewOpcode = tryToRepresentAsInstArg(<wbr>Opcode, I)) {<br>
+ if (!Instruction::isBinaryOp(<wbr>Opcode) ||<br>
+ !Instruction::isCommutative(<wbr>Opcode)) {<br>
+ NewOpcodePos = Cnt;<br>
+ Opcode = NewOpcode;<br>
+ AltOpcode = getAltOpcode(Opcode);<br>
+ Res.NeedAnalysis = true;<br>
+ }<br>
+ } else if (tryToRepresentAsInstArg(I-><wbr>getOpcode(),<br>
+ cast<Instruction>(VL[<wbr>NewOpcodePos])))<br>
+ Res.NeedAnalysis = true;<br>
+ else<br>
+ return {};<br>
}<br>
Res.Opcode = Opcode;<br>
+ Res.OpcodePos = NewOpcodePos;<br>
return Res;<br>
}<br>
<br>
@@ -412,16 +454,20 @@ struct InstructionsState {<br>
static InstructionsState getSameOpcode(ArrayRef<Value *> VL) {<br>
auto Res = getMainOpcode(VL);<br>
unsigned Opcode = Res.Opcode;<br>
- if (!Res.HasAltOpcodes)<br>
- return InstructionsState(VL[0], Opcode, false);<br>
- auto *OpInst = cast<Instruction>(VL[0]);<br>
+ if (!Res.NeedAnalysis && !Res.HasAltOpcodes)<br>
+ return InstructionsState(VL[Res.<wbr>OpcodePos], Opcode, false);<br>
+ auto *OpInst = cast<Instruction>(VL[Res.<wbr>OpcodePos]);<br>
unsigned AltOpcode = getAltOpcode(Opcode);<br>
// Examine each element in the list instructions VL to determine<br>
// if some operations there could be considered as an alternative<br>
- // (for example as subtraction relates to addition operation).<br>
+ // (for example as subtraction relates to addition operation) or<br>
+ // operation could be an operand of a (possibly) binary operation.<br>
for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {<br>
auto *I = cast<Instruction>(VL[Cnt]);<br>
unsigned InstOpcode = I->getOpcode();<br>
+ if (Res.NeedAnalysis && !sameOpcodeOrAlt(Opcode, AltOpcode, InstOpcode))<br>
+ if (tryToRepresentAsInstArg(<wbr>InstOpcode, OpInst))<br>
+ InstOpcode = (Res.HasAltOpcodes && isOdd(Cnt)) ? AltOpcode : Opcode;<br>
if ((Res.HasAltOpcodes &&<br>
InstOpcode != (isOdd(Cnt) ? AltOpcode : Opcode)) ||<br>
(!Res.HasAltOpcodes && InstOpcode != Opcode)) {<br>
@@ -574,6 +620,7 @@ public:<br>
void deleteTree() {<br>
VectorizableTree.clear();<br>
ScalarToTreeEntry.clear();<br>
+ ExtraScalarToTreeEntry.clear()<wbr>;<br>
MustGather.clear();<br>
ExternalUses.clear();<br>
NumLoadsWantToKeepOrder = 0;<br>
@@ -713,22 +760,40 @@ private:<br>
/// The TreeEntry index containing the user of this entry. We can actually<br>
/// have multiple users so the data structure is not truly a tree.<br>
SmallVector<int, 1> UserTreeIndices;<br>
+<br>
+ /// Info about instruction in this tree entry.<br>
+ InstructionsState State;<br>
};<br>
<br>
/// Create a new VectorizableTree entry.<br>
TreeEntry *newTreeEntry(ArrayRef<Value *> VL, bool Vectorized,<br>
- int &UserTreeIdx) {<br>
+ int &UserTreeIdx, const InstructionsState &S) {<br>
+ assert((!Vectorized || S.Opcode != 0) &&<br>
+ "Vectorized TreeEntry without opcode");<br>
VectorizableTree.emplace_back(<wbr>VectorizableTree);<br>
int idx = VectorizableTree.size() - 1;<br>
TreeEntry *Last = &VectorizableTree[idx];<br>
Last->Scalars.insert(Last-><wbr>Scalars.begin(), VL.begin(), VL.end());<br>
Last->NeedToGather = !Vectorized;<br>
if (Vectorized) {<br>
+ Last->State = S;<br>
+ unsigned AltOpcode = getAltOpcode(S.Opcode);<br>
for (int i = 0, e = VL.size(); i != e; ++i) {<br>
- assert(!getTreeEntry(VL[i]) && "Scalar already in tree!");<br>
- ScalarToTreeEntry[VL[i]] = idx;<br>
+ unsigned RealOpcode =<br>
+ (S.IsAltShuffle && isOdd(i)) ? AltOpcode : S.Opcode;<br>
+ Value *Key = (cast<Instruction>(VL[i])-><wbr>getOpcode() == RealOpcode)<br>
+ ? VL[i]<br>
+ : S.OpValue;<br>
+ assert(!getTreeEntry(VL[i], Key) && "Scalar already in tree!");<br>
+ if (VL[i] == Key)<br>
+ ScalarToTreeEntry[Key] = idx;<br>
+ else<br>
+ ExtraScalarToTreeEntry[VL[i]][<wbr>Key] = idx;<br>
}<br>
} else {<br>
+ Last->State.Opcode = 0;<br>
+ Last->State.OpValue = VL[0];<br>
+ Last->State.IsAltShuffle = false;<br>
MustGather.insert(VL.begin(), VL.end());<br>
}<br>
<br>
@@ -756,9 +821,25 @@ private:<br>
return nullptr;<br>
}<br>
<br>
+ TreeEntry *getTreeEntry(Value *V, Value *OpValue) {<br>
+ if (V == OpValue)<br>
+ return getTreeEntry(V);<br>
+ auto I = ExtraScalarToTreeEntry.find(V)<wbr>;<br>
+ if (I != ExtraScalarToTreeEntry.end()) {<br>
+ auto &STT = I->second;<br>
+ auto STTI = STT.find(OpValue);<br>
+ if (STTI != STT.end())<br>
+ return &VectorizableTree[STTI-><wbr>second];<br>
+ }<br>
+ return nullptr;<br>
+ }<br>
+<br>
/// Maps a specific scalar to its tree entry.<br>
SmallDenseMap<Value*, int> ScalarToTreeEntry;<br>
<br>
+ /// Maps a specific scalar to its tree entry(s) with leading scalar.<br>
+ SmallDenseMap<Value*, SmallDenseMap<Value*, int>> ExtraScalarToTreeEntry;<br>
+<br>
/// A list of scalars that we found that we need to keep as scalars.<br>
ValueSet MustGather;<br>
<br>
@@ -1327,9 +1408,15 @@ void BoUpSLP::buildTree(ArrayRef<<wbr>Value *<br>
continue;<br>
<br>
// For each lane:<br>
+ const unsigned Opcode = Entry->State.Opcode;<br>
+ const unsigned AltOpcode = getAltOpcode(Opcode);<br>
for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {<br>
Value *Scalar = Entry->Scalars[Lane];<br>
<br>
+ if (!sameOpcodeOrAlt(Opcode, AltOpcode,<br>
+ cast<Instruction>(Scalar)-><wbr>getOpcode()))<br>
+ continue;<br>
+<br>
// Check if the scalar is externally used as an extra arg.<br>
auto ExtI = ExternallyUsedValues.find(<wbr>Scalar);<br>
if (ExtI != ExternallyUsedValues.end()) {<br>
@@ -1372,6 +1459,38 @@ void BoUpSLP::buildTree(ArrayRef<<wbr>Value *<br>
}<br>
}<br>
<br>
+static Value *getDefaultConstantForOpcode(<wbr>unsigned Opcode, Type *Ty) {<br>
+ switch(Opcode) {<br>
+ case Instruction::Add:<br>
+ case Instruction::Sub:<br>
+ case Instruction::Or:<br>
+ case Instruction::Xor:<br>
+ return ConstantInt::getNullValue(Ty);<br>
+ case Instruction::Mul:<br>
+ case Instruction::UDiv:<br>
+ case Instruction::SDiv:<br>
+ case Instruction::URem:<br>
+ case Instruction::SRem:<br>
+ return ConstantInt::get(Ty, /*V=*/1);<br>
+ case Instruction::FAdd:<br>
+ case Instruction::FSub:<br>
+ return ConstantFP::get(Ty, /*V=*/0.0);<br>
+ case Instruction::FMul:<br>
+ case Instruction::FDiv:<br>
+ case Instruction::FRem:<br>
+ return ConstantFP::get(Ty, /*V=*/1.0);<br>
+ case Instruction::And:<br>
+ return ConstantInt::getAllOnesValue(<wbr>Ty);<br>
+ case Instruction::Shl:<br>
+ case Instruction::LShr:<br>
+ case Instruction::AShr:<br>
+ return ConstantInt::getNullValue(<wbr>Type::getInt32Ty(Ty-><wbr>getContext()));<br>
+ default:<br>
+ break;<br>
+ }<br>
+ llvm_unreachable("unknown binop for default constant value");<br>
+}<br>
+<br>
void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Value *> VL, unsigned Depth,<br>
int UserTreeIdx) {<br>
assert((allConstant(VL) || allSameType(VL)) && "Invalid types!");<br>
@@ -1379,28 +1498,28 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
InstructionsState S = getSameOpcode(VL);<br>
if (Depth == RecursionMaxDepth) {<br>
DEBUG(dbgs() << "SLP: Gathering due to max recursion depth.\n");<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
<br>
// Don't handle vectors.<br>
if (S.OpValue->getType()-><wbr>isVectorTy()) {<br>
DEBUG(dbgs() << "SLP: Gathering due to vector type.\n");<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
<br>
if (StoreInst *SI = dyn_cast<StoreInst>(S.OpValue)<wbr>)<br>
if (SI->getValueOperand()-><wbr>getType()->isVectorTy()) {<br>
DEBUG(dbgs() << "SLP: Gathering due to store vector type.\n");<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
<br>
// If all of the operands are identical or constant we have a simple solution.<br>
if (allConstant(VL) || isSplat(VL) || !allSameBlock(VL) || !S.Opcode) {<br>
DEBUG(dbgs() << "SLP: Gathering due to C,S,B,O. \n");<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
<br>
@@ -1412,7 +1531,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
if (EphValues.count(VL[i])) {<br>
DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<<br>
") is ephemeral.\n");<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
}<br>
@@ -1423,7 +1542,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
DEBUG(dbgs() << "SLP: \tChecking bundle: " << *VL[i] << ".\n");<br>
if (E->Scalars[i] != VL[i]) {<br>
DEBUG(dbgs() << "SLP: Gathering due to partial overlap.\n");<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
}<br>
@@ -1435,14 +1554,17 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
}<br>
<br>
// Check that none of the instructions in the bundle are already in the tree.<br>
+ unsigned AltOpcode = getAltOpcode(S.Opcode);<br>
for (unsigned i = 0, e = VL.size(); i != e; ++i) {<br>
+ unsigned RealOpcode = (S.IsAltShuffle && isOdd(i)) ? AltOpcode : S.Opcode;<br>
auto *I = dyn_cast<Instruction>(VL[i]);<br>
if (!I)<br>
continue;<br>
- if (getTreeEntry(I)) {<br>
+ Value *Key = (I->getOpcode() == RealOpcode) ? I : S.OpValue;<br>
+ if (getTreeEntry(I, Key)) {<br>
DEBUG(dbgs() << "SLP: The instruction (" << *VL[i] <<<br>
") is already in tree.\n");<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
}<br>
@@ -1452,7 +1574,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
for (unsigned i = 0, e = VL.size(); i != e; ++i) {<br>
if (MustGather.count(VL[i])) {<br>
DEBUG(dbgs() << "SLP: Gathering due to gathered scalar.\n");<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
}<br>
@@ -1466,7 +1588,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
// Don't go into unreachable blocks. They may contain instructions with<br>
// dependency cycles which confuse the final scheduling.<br>
DEBUG(dbgs() << "SLP: bundle in unreachable block.\n");<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
<br>
@@ -1475,7 +1597,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
for (unsigned j = i+1; j < e; ++j)<br>
if (VL[i] == VL[j]) {<br>
DEBUG(dbgs() << "SLP: Scalar used twice in bundle.\n");<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
<br>
@@ -1490,7 +1612,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
assert((!BS.getScheduleData(<wbr>VL0) ||<br>
!BS.getScheduleData(VL0)-><wbr>isPartOfBundle()) &&<br>
"tryScheduleBundle should cancelScheduling on failure");<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");<br>
@@ -1509,12 +1631,12 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
if (Term) {<br>
DEBUG(dbgs() << "SLP: Need to swizzle PHINodes (TerminatorInst use).\n");<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
}<br>
<br>
- newTreeEntry(VL, true, UserTreeIdx);<br>
+ newTreeEntry(VL, true, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: added a vector of PHINodes.\n");<br>
<br>
for (unsigned i = 0, e = PH->getNumIncomingValues(); i < e; ++i) {<br>
@@ -1536,7 +1658,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
} else {<br>
BS.cancelScheduling(VL, VL0);<br>
}<br>
- newTreeEntry(VL, Reuse, UserTreeIdx);<br>
+ newTreeEntry(VL, Reuse, UserTreeIdx, S);<br>
return;<br>
}<br>
case Instruction::Load: {<br>
@@ -1552,7 +1674,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
if (DL->getTypeSizeInBits(<wbr>ScalarTy) !=<br>
DL->getTypeAllocSizeInBits(<wbr>ScalarTy)) {<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: Gathering loads of non-packed type.\n");<br>
return;<br>
}<br>
@@ -1563,7 +1685,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
LoadInst *L = cast<LoadInst>(VL[i]);<br>
if (!L->isSimple()) {<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: Gathering non-simple loads.\n");<br>
return;<br>
}<br>
@@ -1585,7 +1707,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
<br>
if (Consecutive) {<br>
++NumLoadsWantToKeepOrder;<br>
- newTreeEntry(VL, true, UserTreeIdx);<br>
+ newTreeEntry(VL, true, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: added a vector of loads.\n");<br>
return;<br>
}<br>
@@ -1600,7 +1722,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
}<br>
<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
<br>
if (ReverseConsecutive) {<br>
++NumLoadsWantToChangeOrder;<br>
@@ -1627,12 +1749,12 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
Type *Ty = cast<Instruction>(VL[i])-><wbr>getOperand(0)->getType();<br>
if (Ty != SrcTy || !isValidElementType(Ty)) {<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: Gathering casts with different src types.\n");<br>
return;<br>
}<br>
}<br>
- newTreeEntry(VL, true, UserTreeIdx);<br>
+ newTreeEntry(VL, true, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: added a vector of casts.\n");<br>
<br>
for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {<br>
@@ -1655,13 +1777,13 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
if (Cmp->getPredicate() != P0 ||<br>
Cmp->getOperand(0)->getType() != ComparedTy) {<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: Gathering cmp with different predicate.\n");<br>
return;<br>
}<br>
}<br>
<br>
- newTreeEntry(VL, true, UserTreeIdx);<br>
+ newTreeEntry(VL, true, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: added a vector of compares.\n");<br>
<br>
for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {<br>
@@ -1693,7 +1815,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
case Instruction::And:<br>
case Instruction::Or:<br>
case Instruction::Xor:<br>
- newTreeEntry(VL, true, UserTreeIdx);<br>
+ newTreeEntry(VL, true, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: added a vector of bin op.\n");<br>
<br>
// Sort operands of the instructions so that each side is more likely to<br>
@@ -1709,8 +1831,19 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {<br>
ValueList Operands;<br>
// Prepare the operand vector.<br>
- for (Value *j : VL)<br>
- Operands.push_back(cast<<wbr>Instruction>(j)->getOperand(i)<wbr>);<br>
+ for (Value *VecOp : VL) {<br>
+ auto *I = cast<Instruction>(VecOp);<br>
+ if (I->getOpcode() == S.Opcode) {<br>
+ Operands.push_back(I-><wbr>getOperand(i));<br>
+ continue;<br>
+ }<br>
+ assert(Instruction::<wbr>isBinaryOp(S.Opcode) &&<br>
+ "Expected a binary operation.");<br>
+ Value *Operand = isOdd(i)<br>
+ ? getDefaultConstantForOpcode(S.<wbr>Opcode, I->getType())<br>
+ : VecOp;<br>
+ Operands.push_back(Operand);<br>
+ }<br>
<br>
buildTree_rec(Operands, Depth + 1, UserTreeIdx);<br>
}<br>
@@ -1722,7 +1855,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
if (cast<Instruction>(VL[j])-><wbr>getNumOperands() != 2) {<br>
DEBUG(dbgs() << "SLP: not-vectorizable GEP (nested indexes).\n");<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
}<br>
@@ -1735,7 +1868,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
if (Ty0 != CurTy) {<br>
DEBUG(dbgs() << "SLP: not-vectorizable GEP (different types).\n");<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
}<br>
@@ -1747,12 +1880,12 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
DEBUG(<br>
dbgs() << "SLP: not-vectorizable GEP (non-constant indexes).\n");<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
return;<br>
}<br>
}<br>
<br>
- newTreeEntry(VL, true, UserTreeIdx);<br>
+ newTreeEntry(VL, true, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: added a vector of GEPs.\n");<br>
for (unsigned i = 0, e = 2; i < e; ++i) {<br>
ValueList Operands;<br>
@@ -1769,12 +1902,12 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
for (unsigned i = 0, e = VL.size() - 1; i < e; ++i)<br>
if (!isConsecutiveAccess(VL[i], VL[i + 1], *DL, *SE)) {<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: Non-consecutive store.\n");<br>
return;<br>
}<br>
<br>
- newTreeEntry(VL, true, UserTreeIdx);<br>
+ newTreeEntry(VL, true, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: added a vector of stores.\n");<br>
<br>
ValueList Operands;<br>
@@ -1792,7 +1925,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
Intrinsic::ID ID = getVectorIntrinsicIDForCall(<wbr>CI, TLI);<br>
if (!isTriviallyVectorizable(ID)) {<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: Non-vectorizable call.\n");<br>
return;<br>
}<br>
@@ -1806,7 +1939,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
getVectorIntrinsicIDForCall(<wbr>CI2, TLI) != ID ||<br>
!CI-><wbr>hasIdenticalOperandBundleSchem<wbr>a(*CI2)) {<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: mismatched calls:" << *CI << "!=" << *VL[i]<br>
<< "\n");<br>
return;<br>
@@ -1817,7 +1950,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
Value *A1J = CI2->getArgOperand(1);<br>
if (A1I != A1J) {<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: mismatched arguments in call:" << *CI<br>
<< " argument "<< A1I<<"!=" << A1J<br>
<< "\n");<br>
@@ -1830,14 +1963,14 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
CI->op_begin() + CI->getBundleOperandsEndIndex(<wbr>),<br>
CI2->op_begin() + CI2-><wbr>getBundleOperandsStartIndex())<wbr>) {<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: mismatched bundle operands in calls:" << *CI << "!="<br>
<< *VL[i] << '\n');<br>
return;<br>
}<br>
}<br>
<br>
- newTreeEntry(VL, true, UserTreeIdx);<br>
+ newTreeEntry(VL, true, UserTreeIdx, S);<br>
for (unsigned i = 0, e = CI->getNumArgOperands(); i != e; ++i) {<br>
ValueList Operands;<br>
// Prepare the operand vector.<br>
@@ -1854,11 +1987,11 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
// then do not vectorize this instruction.<br>
if (!S.IsAltShuffle) {<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");<br>
return;<br>
}<br>
- newTreeEntry(VL, true, UserTreeIdx);<br>
+ newTreeEntry(VL, true, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n");<br>
<br>
// Reorder operands if reordering would enable vectorization.<br>
@@ -1873,8 +2006,19 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {<br>
ValueList Operands;<br>
// Prepare the operand vector.<br>
- for (Value *j : VL)<br>
- Operands.push_back(cast<<wbr>Instruction>(j)->getOperand(i)<wbr>);<br>
+ for (Value *VecOp : VL) {<br>
+ auto *I = cast<Instruction>(VecOp);<br>
+ if (sameOpcodeOrAlt(S.Opcode, AltOpcode, I->getOpcode())) {<br>
+ Operands.push_back(I-><wbr>getOperand(i));<br>
+ continue;<br>
+ }<br>
+ assert(Instruction::<wbr>isBinaryOp(S.Opcode) &&<br>
+ "Expected a binary operation.");<br>
+ Value *Operand = isOdd(i)<br>
+ ? getDefaultConstantForOpcode(S.<wbr>Opcode, I->getType())<br>
+ : VecOp;<br>
+ Operands.push_back(Operand);<br>
+ }<br>
<br>
buildTree_rec(Operands, Depth + 1, UserTreeIdx);<br>
}<br>
@@ -1882,7 +2026,7 @@ void BoUpSLP::buildTree_rec(<wbr>ArrayRef<Val<br>
<br>
default:<br>
BS.cancelScheduling(VL, VL0);<br>
- newTreeEntry(VL, false, UserTreeIdx);<br>
+ newTreeEntry(VL, false, UserTreeIdx, S);<br>
DEBUG(dbgs() << "SLP: Gathering unknown instruction.\n");<br>
return;<br>
}<br>
@@ -2003,18 +2147,17 @@ int BoUpSLP::getEntryCost(<wbr>TreeEntry *E)<br>
}<br>
return getGatherCost(E->Scalars);<br>
}<br>
- InstructionsState S = getSameOpcode(VL);<br>
- assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL");<br>
- Instruction *VL0 = cast<Instruction>(S.OpValue);<br>
- unsigned ShuffleOrOp = S.IsAltShuffle ?<br>
- (unsigned) Instruction::ShuffleVector : S.Opcode;<br>
+ assert(E->State.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL");<br>
+ auto *VL0 = cast<Instruction>(E->State.<wbr>OpValue);<br>
+ unsigned ShuffleOrOp = E->State.IsAltShuffle ?<br>
+ (unsigned) Instruction::ShuffleVector : E->State.Opcode;<br>
switch (ShuffleOrOp) {<br>
case Instruction::PHI:<br>
return 0;<br>
<br>
case Instruction::ExtractValue:<br>
case Instruction::ExtractElement:<br>
- if (canReuseExtract(VL, S.OpValue)) {<br>
+ if (canReuseExtract(VL, E->State.OpValue)) {<br>
int DeadCost = 0;<br>
for (unsigned i = 0, e = VL.size(); i < e; ++i) {<br>
Instruction *E = cast<Instruction>(VL[i]);<br>
@@ -2058,8 +2201,8 @@ int BoUpSLP::getEntryCost(<wbr>TreeEntry *E)<br>
// Calculate the cost of this instruction.<br>
VectorType *MaskTy = VectorType::get(Builder.<wbr>getInt1Ty(), VL.size());<br>
int ScalarCost = VecTy->getNumElements() *<br>
- TTI->getCmpSelInstrCost(S.<wbr>Opcode, ScalarTy, Builder.getInt1Ty(), VL0);<br>
- int VecCost = TTI->getCmpSelInstrCost(S.<wbr>Opcode, VecTy, MaskTy, VL0);<br>
+ TTI->getCmpSelInstrCost(<wbr>ShuffleOrOp, ScalarTy, Builder.getInt1Ty(), VL0);<br>
+ int VecCost = TTI->getCmpSelInstrCost(<wbr>ShuffleOrOp, VecTy, MaskTy, VL0);<br>
return VecCost - ScalarCost;<br>
}<br>
case Instruction::Add:<br>
@@ -2085,7 +2228,7 @@ int BoUpSLP::getEntryCost(<wbr>TreeEntry *E)<br>
TargetTransformInfo::<wbr>OperandValueKind Op1VK =<br>
TargetTransformInfo::OK_<wbr>AnyValue;<br>
TargetTransformInfo::<wbr>OperandValueKind Op2VK =<br>
- TargetTransformInfo::OK_<wbr>UniformConstantValue;<br>
+ TargetTransformInfo::OK_<wbr>AnyValue;<br>
TargetTransformInfo::<wbr>OperandValueProperties Op1VP =<br>
TargetTransformInfo::OP_None;<br>
TargetTransformInfo::<wbr>OperandValueProperties Op2VP =<br>
@@ -2096,34 +2239,33 @@ int BoUpSLP::getEntryCost(<wbr>TreeEntry *E)<br>
// If instead not all operands are constants, then set the operand kind<br>
// to OK_AnyValue. If all operands are constants but not the same,<br>
// then set the operand kind to OK_NonUniformConstantValue.<br>
- ConstantInt *CInt = nullptr;<br>
- for (unsigned i = 0; i < VL.size(); ++i) {<br>
- const Instruction *I = cast<Instruction>(VL[i]);<br>
- if (!isa<ConstantInt>(I-><wbr>getOperand(1))) {<br>
- Op2VK = TargetTransformInfo::OK_<wbr>AnyValue;<br>
- break;<br>
- }<br>
- if (i == 0) {<br>
- CInt = cast<ConstantInt>(I-><wbr>getOperand(1));<br>
- continue;<br>
+ if (auto *CInt = dyn_cast<ConstantInt>(VL0-><wbr>getOperand(1))) {<br>
+ Op2VK = TargetTransformInfo::OK_<wbr>UniformConstantValue;<br>
+ const unsigned Opcode = E->State.Opcode;<br>
+ for (auto *V : VL) {<br>
+ auto *I = cast<Instruction>(V);<br>
+ if (I == VL0 || Opcode != I->getOpcode())<br>
+ continue;<br>
+ if (!isa<ConstantInt>(I-><wbr>getOperand(1))) {<br>
+ Op2VK = TargetTransformInfo::OK_<wbr>AnyValue;<br>
+ break;<br>
+ }<br>
+ if (Op2VK == TargetTransformInfo::OK_<wbr>UniformConstantValue &&<br>
+ CInt != cast<ConstantInt>(I-><wbr>getOperand(1)))<br>
+ Op2VK = TargetTransformInfo::OK_<wbr>NonUniformConstantValue;<br>
}<br>
+ // FIXME: Currently cost of model modification for division by power of<br>
+ // 2 is handled for X86 and AArch64. Add support for other targets.<br>
if (Op2VK == TargetTransformInfo::OK_<wbr>UniformConstantValue &&<br>
- CInt != cast<ConstantInt>(I-><wbr>getOperand(1)))<br>
- Op2VK = TargetTransformInfo::OK_<wbr>NonUniformConstantValue;<br>
+ CInt->getValue().isPowerOf2())<br>
+ Op2VP = TargetTransformInfo::OP_<wbr>PowerOf2;<br>
}<br>
- // FIXME: Currently cost of model modification for division by power of<br>
- // 2 is handled for X86 and AArch64. Add support for other targets.<br>
- if (Op2VK == TargetTransformInfo::OK_<wbr>UniformConstantValue && CInt &&<br>
- CInt->getValue().isPowerOf2())<br>
- Op2VP = TargetTransformInfo::OP_<wbr>PowerOf2;<br>
<br>
- SmallVector<const Value *, 4> Operands(VL0->operand_values()<wbr>);<br>
- int ScalarCost =<br>
- VecTy->getNumElements() *<br>
- TTI->getArithmeticInstrCost(S.<wbr>Opcode, ScalarTy, Op1VK, Op2VK, Op1VP,<br>
- Op2VP, Operands);<br>
- int VecCost = TTI->getArithmeticInstrCost(S.<wbr>Opcode, VecTy, Op1VK, Op2VK,<br>
- Op1VP, Op2VP, Operands);<br>
+ int ScalarCost = VecTy->getNumElements() *<br>
+ TTI->getArithmeticInstrCost(E-<wbr>>State.Opcode, ScalarTy,<br>
+ Op1VK, Op2VK, Op1VP, Op2VP);<br>
+ int VecCost = TTI->getArithmeticInstrCost(E-<wbr>>State.Opcode, VecTy, Op1VK,<br>
+ Op2VK, Op1VP, Op2VP);<br>
return VecCost - ScalarCost;<br>
}<br>
case Instruction::GetElementPtr: {<br>
@@ -2189,23 +2331,18 @@ int BoUpSLP::getEntryCost(<wbr>TreeEntry *E)<br>
TargetTransformInfo::OK_<wbr>AnyValue;<br>
TargetTransformInfo::<wbr>OperandValueKind Op2VK =<br>
TargetTransformInfo::OK_<wbr>AnyValue;<br>
- int ScalarCost = 0;<br>
- int VecCost = 0;<br>
- for (Value *i : VL) {<br>
- Instruction *I = cast<Instruction>(i);<br>
- if (!I)<br>
- break;<br>
- ScalarCost +=<br>
- TTI->getArithmeticInstrCost(I-<wbr>>getOpcode(), ScalarTy, Op1VK, Op2VK);<br>
- }<br>
+ unsigned AltOpcode = getAltOpcode(E->State.Opcode);<br>
+ int ScalarCost =<br>
+ TTI->getArithmeticInstrCost(E-<wbr>>State.Opcode, ScalarTy, Op1VK, Op2VK) *<br>
+ VL.size() / 2;<br>
+ ScalarCost +=<br>
+ TTI->getArithmeticInstrCost(<wbr>AltOpcode, ScalarTy, Op1VK, Op2VK) *<br>
+ VL.size() / 2;<br>
// VecCost is equal to sum of the cost of creating 2 vectors<br>
// and the cost of creating shuffle.<br>
- Instruction *I0 = cast<Instruction>(VL[0]);<br>
- VecCost =<br>
- TTI->getArithmeticInstrCost(<wbr>I0->getOpcode(), VecTy, Op1VK, Op2VK);<br>
- Instruction *I1 = cast<Instruction>(VL[1]);<br>
- VecCost +=<br>
- TTI->getArithmeticInstrCost(<wbr>I1->getOpcode(), VecTy, Op1VK, Op2VK);<br>
+ int VecCost =<br>
+ TTI->getArithmeticInstrCost(E-<wbr>>State.Opcode, VecTy, Op1VK, Op2VK);<br>
+ VecCost += TTI->getArithmeticInstrCost(<wbr>AltOpcode, VecTy, Op1VK, Op2VK);<br>
VecCost +=<br>
TTI->getShuffleCost(<wbr>TargetTransformInfo::SK_<wbr>Alternate, VecTy, 0);<br>
return VecCost - ScalarCost;<br>
@@ -2271,7 +2408,7 @@ int BoUpSLP::getSpillCost() {<br>
Instruction *PrevInst = nullptr;<br>
<br>
for (const auto &N : VectorizableTree) {<br>
- Instruction *Inst = dyn_cast<Instruction>(N.<wbr>Scalars[0]);<br>
+ Instruction *Inst = dyn_cast<Instruction>(N.State.<wbr>OpValue);<br>
if (!Inst)<br>
continue;<br>
<br>
@@ -2331,7 +2468,7 @@ int BoUpSLP::getTreeCost() {<br>
for (TreeEntry &TE : VectorizableTree) {<br>
int C = getEntryCost(&TE);<br>
DEBUG(dbgs() << "SLP: Adding cost " << C << " for bundle that starts with "<br>
- << *TE.Scalars[0] << ".\n");<br>
+ << *TE.State.OpValue << ".\n");<br>
Cost += C;<br>
}<br>
<br>
@@ -2352,7 +2489,7 @@ int BoUpSLP::getTreeCost() {<br>
// extend the extracted value back to the original type. Here, we account<br>
// for the extract and the added cost of the sign extend if needed.<br>
auto *VecTy = VectorType::get(EU.Scalar-><wbr>getType(), BundleWidth);<br>
- auto *ScalarRoot = VectorizableTree[0].Scalars[0]<wbr>;<br>
+ auto *ScalarRoot = VectorizableTree[0].State.<wbr>OpValue;<br>
if (MinBWs.count(ScalarRoot)) {<br>
auto *MinTy = IntegerType::get(F-><wbr>getContext(), MinBWs[ScalarRoot].first);<br>
auto Extend =<br>
@@ -2415,13 +2552,15 @@ void BoUpSLP::<wbr>reorderAltShuffleOperands(<br>
SmallVectorImpl<Value *> &Right) {<br>
// Push left and right operands of binary operation into Left and Right<br>
unsigned AltOpcode = getAltOpcode(Opcode);<br>
- (void)AltOpcode;<br>
for (Value *V : VL) {<br>
auto *I = cast<Instruction>(V);<br>
- assert(sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode()) &&<br>
- "Incorrect instruction in vector");<br>
- Left.push_back(I->getOperand(<wbr>0));<br>
- Right.push_back(I->getOperand(<wbr>1));<br>
+ if (sameOpcodeOrAlt(Opcode, AltOpcode, I->getOpcode())) {<br>
+ Left.push_back(I->getOperand(<wbr>0));<br>
+ Right.push_back(I->getOperand(<wbr>1));<br>
+ } else {<br>
+ Left.push_back(I);<br>
+ Right.push_back(<wbr>getDefaultConstantForOpcode(<wbr>Opcode, I->getType()));<br>
+ }<br>
}<br>
<br>
// Reorder if we have a commutative operation and consecutive access<br>
@@ -2470,8 +2609,13 @@ static bool shouldReorderOperands(<br>
int i, unsigned Opcode, Instruction &I, ArrayRef<Value *> Left,<br>
ArrayRef<Value *> Right, bool AllSameOpcodeLeft, bool AllSameOpcodeRight,<br>
bool SplatLeft, bool SplatRight, Value *&VLeft, Value *&VRight) {<br>
- VLeft = I.getOperand(0);<br>
- VRight = I.getOperand(1);<br>
+ if (I.getOpcode() == Opcode) {<br>
+ VLeft = I.getOperand(0);<br>
+ VRight = I.getOperand(1);<br>
+ } else {<br>
+ VLeft = &I;<br>
+ VRight = getDefaultConstantForOpcode(<wbr>Opcode, I.getType());<br>
+ }<br>
// If we have "SplatRight", try to see if commuting is needed to preserve it.<br>
if (SplatRight) {<br>
if (VRight == Right[i - 1])<br>
@@ -2535,8 +2679,15 @@ void BoUpSLP::<wbr>reorderInputsAccordingToOp<br>
// Peel the first iteration out of the loop since there's nothing<br>
// interesting to do anyway and it simplifies the checks in the loop.<br>
auto *I = cast<Instruction>(VL[0]);<br>
- Value *VLeft = I->getOperand(0);<br>
- Value *VRight = I->getOperand(1);<br>
+ Value *VLeft;<br>
+ Value *VRight;<br>
+ if (I->getOpcode() == Opcode) {<br>
+ VLeft = I->getOperand(0);<br>
+ VRight = I->getOperand(1);<br>
+ } else {<br>
+ VLeft = I;<br>
+ VRight = getDefaultConstantForOpcode(<wbr>Opcode, I->getType());<br>
+ }<br>
if (!isa<Instruction>(VRight) && isa<Instruction>(VLeft))<br>
// Favor having instruction to the right. FIXME: why?<br>
std::swap(VLeft, VRight);<br>
@@ -2741,12 +2892,11 @@ Value *BoUpSLP::vectorizeTree(<wbr>TreeEntry<br>
IRBuilder<>::InsertPointGuard Guard(Builder);<br>
<br>
if (E->VectorizedValue) {<br>
- DEBUG(dbgs() << "SLP: Diamond merged for " << *E->Scalars[0] << ".\n");<br>
+ DEBUG(dbgs() << "SLP: Diamond merged for " << *E->State.OpValue << ".\n");<br>
return E->VectorizedValue;<br>
}<br>
<br>
- InstructionsState S = getSameOpcode(E->Scalars);<br>
- Instruction *VL0 = cast<Instruction>(E->Scalars[<wbr>0]);<br>
+ Instruction *VL0 = cast<Instruction>(E->State.<wbr>OpValue);<br>
Type *ScalarTy = VL0->getType();<br>
if (StoreInst *SI = dyn_cast<StoreInst>(VL0))<br>
ScalarTy = SI->getValueOperand()-><wbr>getType();<br>
@@ -2759,8 +2909,8 @@ Value *BoUpSLP::vectorizeTree(<wbr>TreeEntry<br>
return V;<br>
}<br>
<br>
- unsigned ShuffleOrOp = S.IsAltShuffle ?<br>
- (unsigned) Instruction::ShuffleVector : S.Opcode;<br>
+ unsigned ShuffleOrOp = E->State.IsAltShuffle ?<br>
+ (unsigned) Instruction::ShuffleVector : E->State.Opcode;<br>
switch (ShuffleOrOp) {<br>
case Instruction::PHI: {<br>
PHINode *PH = dyn_cast<PHINode>(VL0);<br>
@@ -2870,7 +3020,7 @@ Value *BoUpSLP::vectorizeTree(<wbr>TreeEntry<br>
<br>
CmpInst::Predicate P0 = cast<CmpInst>(VL0)-><wbr>getPredicate();<br>
Value *V;<br>
- if (S.Opcode == Instruction::FCmp)<br>
+ if (E->State.Opcode == Instruction::FCmp)<br>
V = Builder.CreateFCmp(P0, L, R);<br>
else<br>
V = Builder.CreateICmp(P0, L, R);<br>
@@ -2922,13 +3072,19 @@ Value *BoUpSLP::vectorizeTree(<wbr>TreeEntry<br>
case Instruction::Xor: {<br>
ValueList LHSVL, RHSVL;<br>
if (isa<BinaryOperator>(VL0) && VL0->isCommutative())<br>
- reorderInputsAccordingToOpcode<wbr>(S.Opcode, E->Scalars, LHSVL,<br>
+ reorderInputsAccordingToOpcode<wbr>(E->State.Opcode, E->Scalars, LHSVL,<br>
RHSVL);<br>
else<br>
for (Value *V : E->Scalars) {<br>
auto *I = cast<Instruction>(V);<br>
- LHSVL.push_back(I->getOperand(<wbr>0));<br>
- RHSVL.push_back(I->getOperand(<wbr>1));<br>
+ if (I->getOpcode() == E->State.Opcode) {<br>
+ LHSVL.push_back(I->getOperand(<wbr>0));<br>
+ RHSVL.push_back(I->getOperand(<wbr>1));<br>
+ } else {<br>
+ LHSVL.push_back(V);<br>
+ RHSVL.push_back(<br>
+ getDefaultConstantForOpcode(E-<wbr>>State.Opcode, I->getType()));<br>
+ }<br>
}<br>
<br>
setInsertPointAfterBundle(E-><wbr>Scalars, VL0);<br>
@@ -2940,7 +3096,7 @@ Value *BoUpSLP::vectorizeTree(<wbr>TreeEntry<br>
return V;<br>
<br>
Value *V = Builder.CreateBinOp(<br>
- static_cast<Instruction::<wbr>BinaryOps>(S.Opcode), LHS, RHS);<br>
+ static_cast<Instruction::<wbr>BinaryOps>(E->State.Opcode), LHS, RHS);<br>
E->VectorizedValue = V;<br>
propagateIRFlags(E-><wbr>VectorizedValue, E->Scalars, VL0);<br>
++NumVectorInstructions;<br>
@@ -3091,9 +3247,9 @@ Value *BoUpSLP::vectorizeTree(<wbr>TreeEntry<br>
}<br>
case Instruction::ShuffleVector: {<br>
ValueList LHSVL, RHSVL;<br>
- assert(Instruction::<wbr>isBinaryOp(S.Opcode) &&<br>
+ assert(Instruction::<wbr>isBinaryOp(E->State.Opcode) &&<br>
"Invalid Shuffle Vector Operand");<br>
- reorderAltShuffleOperands(S.<wbr>Opcode, E->Scalars, LHSVL, RHSVL);<br>
+ reorderAltShuffleOperands(E-><wbr>State.Opcode, E->Scalars, LHSVL, RHSVL);<br>
setInsertPointAfterBundle(E-><wbr>Scalars, VL0);<br>
<br>
Value *LHS = vectorizeTree(LHSVL);<br>
@@ -3104,9 +3260,9 @@ Value *BoUpSLP::vectorizeTree(<wbr>TreeEntry<br>
<br>
// Create a vector of LHS op1 RHS<br>
Value *V0 = Builder.CreateBinOp(<br>
- static_cast<Instruction::<wbr>BinaryOps>(S.Opcode), LHS, RHS);<br>
+ static_cast<Instruction::<wbr>BinaryOps>(E->State.Opcode), LHS, RHS);<br>
<br>
- unsigned AltOpcode = getAltOpcode(S.Opcode);<br>
+ unsigned AltOpcode = getAltOpcode(E->State.Opcode);<br>
// Create a vector of LHS op2 RHS<br>
Value *V1 = Builder.CreateBinOp(<br>
static_cast<Instruction::<wbr>BinaryOps>(AltOpcode), LHS, RHS);<br>
@@ -3128,8 +3284,13 @@ Value *BoUpSLP::vectorizeTree(<wbr>TreeEntry<br>
}<br>
<br>
Value *ShuffleMask = ConstantVector::get(Mask);<br>
- propagateIRFlags(V0, EvenScalars);<br>
- propagateIRFlags(V1, OddScalars);<br>
+ InstructionsState S = getSameOpcode(EvenScalars);<br>
+ assert(!S.IsAltShuffle && "Unexpected alternate opcode");<br>
+ propagateIRFlags(V0, EvenScalars, S.OpValue);<br>
+<br>
+ S = getSameOpcode(OddScalars);<br>
+ assert(!S.IsAltShuffle && "Unexpected alternate opcode");<br>
+ propagateIRFlags(V1, OddScalars, S.OpValue);<br>
<br>
Value *V = Builder.CreateShuffleVector(<wbr>V0, V1, ShuffleMask);<br>
E->VectorizedValue = V;<br>
@@ -3163,7 +3324,7 @@ BoUpSLP::vectorizeTree(<wbr>ExtraValueToDebug<br>
// If the vectorized tree can be rewritten in a smaller type, we truncate the<br>
// vectorized root. InstCombine will then rewrite the entire expression. We<br>
// sign extend the extracted values below.<br>
- auto *ScalarRoot = VectorizableTree[0].Scalars[0]<wbr>;<br>
+ auto *ScalarRoot = VectorizableTree[0].State.<wbr>OpValue;<br>
if (MinBWs.count(ScalarRoot)) {<br>
if (auto *I = dyn_cast<Instruction>(<wbr>VectorRoot))<br>
Builder.SetInsertPoint(&*++<wbr>BasicBlock::iterator(I));<br>
@@ -3274,9 +3435,15 @@ BoUpSLP::vectorizeTree(<wbr>ExtraValueToDebug<br>
assert(Entry->VectorizedValue && "Can't find vectorizable value");<br>
<br>
// For each lane:<br>
+ const unsigned Opcode = Entry->State.Opcode;<br>
+ const unsigned AltOpcode = getAltOpcode(Opcode);<br>
for (int Lane = 0, LE = Entry->Scalars.size(); Lane != LE; ++Lane) {<br>
Value *Scalar = Entry->Scalars[Lane];<br>
<br>
+ if (!sameOpcodeOrAlt(Opcode, AltOpcode,<br>
+ cast<Instruction>(Scalar)-><wbr>getOpcode()))<br>
+ continue;<br>
+<br>
Type *Ty = Scalar->getType();<br>
if (!Ty->isVoidTy()) {<br>
#ifndef NDEBUG<br>
@@ -3408,7 +3575,7 @@ bool BoUpSLP::BlockScheduling::<wbr>trySchedu<br>
}<br>
<br>
for (Value *V : VL) {<br>
- ScheduleData *BundleMember = getScheduleData(V);<br>
+ ScheduleData *BundleMember = getScheduleData(V, isOneOf(OpValue, V));<br>
assert(BundleMember &&<br>
"no ScheduleData for bundle member (maybe not in same basic block)");<br>
if (BundleMember->IsScheduled) {<br>
@@ -3481,7 +3648,7 @@ void BoUpSLP::BlockScheduling::<wbr>cancelSch<br>
if (isa<PHINode>(OpValue))<br>
return;<br>
<br>
- ScheduleData *Bundle = getScheduleData(OpValue);<br>
+ ScheduleData *Bundle = getScheduleData(OpValue)-><wbr>FirstInBundle;<br>
DEBUG(dbgs() << "SLP: cancel scheduling of " << *Bundle << "\n");<br>
assert(!Bundle->IsScheduled &&<br>
"Can't cancel bundle which is already scheduled");<br>
@@ -3784,7 +3951,7 @@ void BoUpSLP::scheduleBlock(<wbr>BlockSchedul<br>
I = I->getNextNode()) {<br>
BS->doForAllOpcodes(I, [this, &Idx, &NumToSchedule, BS](ScheduleData *SD) {<br>
assert(SD->isPartOfBundle() ==<br>
- (getTreeEntry(SD->Inst) != nullptr) &&<br>
+ (getTreeEntry(SD->Inst, SD->OpValue) != nullptr) &&<br>
"scheduler and vectorizer bundle mismatch");<br>
SD->FirstInBundle-><wbr>SchedulingPriority = Idx++;<br>
if (SD->isSchedulingEntity()) {<br>
@@ -3807,15 +3974,15 @@ void BoUpSLP::scheduleBlock(<wbr>BlockSchedul<br>
ScheduleData *BundleMember = picked;<br>
while (BundleMember) {<br>
Instruction *pickedInst = BundleMember->Inst;<br>
- if (LastScheduledInst-><wbr>getNextNode() != pickedInst) {<br>
- BS->BB->getInstList().remove(<wbr>pickedInst);<br>
- BS->BB->getInstList().insert(<wbr>LastScheduledInst-><wbr>getIterator(),<br>
- pickedInst);<br>
+ if (pickedInst == BundleMember->OpValue) {<br>
+ if (LastScheduledInst-><wbr>getNextNode() != pickedInst) {<br>
+ BS->BB->getInstList().remove(<wbr>pickedInst);<br>
+ BS->BB->getInstList().insert(<wbr>LastScheduledInst-><wbr>getIterator(), pickedInst);<br>
+ }<br>
+ LastScheduledInst = pickedInst;<br>
}<br>
- LastScheduledInst = pickedInst;<br>
BundleMember = BundleMember->NextInBundle;<br>
}<br>
-<br>
BS->schedule(picked, ReadyInsts);<br>
NumToSchedule--;<br>
}<br>
@@ -5146,7 +5313,9 @@ public:<br>
VectorizedTree, ReducedSubTree,<br>
ReductionData.getKind());<br>
VectorizedTree = VectReductionData.createOp(<wbr>Builder, "op.rdx");<br>
- propagateIRFlags(<wbr>VectorizedTree, ReductionOps);<br>
+ InstructionsState S = getSameOpcode(ReductionOps);<br>
+ assert(!S.IsAltShuffle && "Unexpected alternate opcode");<br>
+ propagateIRFlags(<wbr>VectorizedTree, ReductionOps, S.OpValue);<br>
} else<br>
VectorizedTree = ReducedSubTree;<br>
i += ReduxWidth;<br>
@@ -5162,7 +5331,9 @@ public:<br>
VectorizedTree, I,<br>
ReductionData.getKind());<br>
VectorizedTree = VectReductionData.createOp(<wbr>Builder);<br>
- propagateIRFlags(<wbr>VectorizedTree, ReductionOps);<br>
+ InstructionsState S = getSameOpcode(ReductionOps);<br>
+ assert(!S.IsAltShuffle && "Unexpected alternate opcode");<br>
+ propagateIRFlags(<wbr>VectorizedTree, ReductionOps, S.OpValue);<br>
}<br>
for (auto &Pair : ExternallyUsedValues) {<br>
assert(!Pair.second.empty() &&<br>
@@ -5174,7 +5345,9 @@ public:<br>
VectorizedTree, Pair.first,<br>
ReductionData.getKind());<br>
VectorizedTree = VectReductionData.createOp(<wbr>Builder, "op.extra");<br>
- propagateIRFlags(<wbr>VectorizedTree, I);<br>
+ InstructionsState S = getSameOpcode(I);<br>
+ assert(!S.IsAltShuffle && "Unexpected alternate opcode");<br>
+ propagateIRFlags(<wbr>VectorizedTree, I, S.OpValue);<br>
}<br>
}<br>
// Update users.<br>
@@ -5284,7 +5457,9 @@ private:<br>
OperationData VectReductionData(<wbr>ReductionData.getOpcode(), LeftShuf,<br>
RightShuf, ReductionData.getKind());<br>
TmpVec = VectReductionData.createOp(<wbr>Builder, "op.rdx");<br>
- propagateIRFlags(TmpVec, RedOps);<br>
+ InstructionsState S = getSameOpcode(RedOps);<br>
+ assert(!S.IsAltShuffle && "Unexpected alternate opcode");<br>
+ propagateIRFlags(TmpVec, RedOps, S.OpValue);<br>
}<br>
<br>
// The result is in the first element of the vector.<br>
<br>
Modified: llvm/trunk/test/Transforms/<wbr>SLPVectorizer/X86/vect_<wbr>copyable_in_binops.ll<br>
URL: <a href="http://llvm.org/viewvc/llvm-project/llvm/trunk/test/Transforms/SLPVectorizer/X86/vect_copyable_in_binops.ll?rev=313348&r1=313347&r2=313348&view=diff" rel="noreferrer" target="_blank">http://llvm.org/viewvc/llvm-<wbr>project/llvm/trunk/test/<wbr>Transforms/SLPVectorizer/X86/<wbr>vect_copyable_in_binops.ll?<wbr>rev=313348&r1=313347&r2=<wbr>313348&view=diff</a><br>
==============================<wbr>==============================<wbr>==================<br>
--- llvm/trunk/test/Transforms/<wbr>SLPVectorizer/X86/vect_<wbr>copyable_in_binops.ll (original)<br>
+++ llvm/trunk/test/Transforms/<wbr>SLPVectorizer/X86/vect_<wbr>copyable_in_binops.ll Thu Sep 14 23:56:39 2017<br>
@@ -43,22 +43,16 @@ define void @add1(i32* noalias %dst, i32<br>
; CHECK-LABEL: @add1(<br>
; CHECK-NEXT: entry:<br>
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1<br>
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[SRC]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1<br>
-; CHECK-NEXT: store i32 [[TMP0]], i32* [[DST]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2<br>
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4<br>
-; CHECK-NEXT: [[ADD3:%.*]] = add nsw i32 [[TMP1]], 1<br>
; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2<br>
-; CHECK-NEXT: store i32 [[ADD3]], i32* [[INCDEC_PTR1]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3<br>
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4<br>
-; CHECK-NEXT: [[ADD6:%.*]] = add nsw i32 [[TMP2]], 2<br>
; CHECK-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3<br>
-; CHECK-NEXT: store i32 [[ADD6]], i32* [[INCDEC_PTR4]], align 4<br>
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR5]], align 4<br>
-; CHECK-NEXT: [[ADD9:%.*]] = add nsw i32 [[TMP3]], 3<br>
-; CHECK-NEXT: store i32 [[ADD9]], i32* [[INCDEC_PTR7]], align 4<br>
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast i32* [[SRC]] to <4 x i32>*<br>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4<br>
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> <i32 0, i32 1, i32 2, i32 3>, [[TMP1]]<br>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[DST]] to <4 x i32>*<br>
+; CHECK-NEXT: store <4 x i32> [[TMP2]], <4 x i32>* [[TMP3]], align 4<br>
; CHECK-NEXT: ret void<br>
;<br>
entry:<br>
@@ -86,22 +80,16 @@ define void @sub0(i32* noalias %dst, i32<br>
; CHECK-LABEL: @sub0(<br>
; CHECK-NEXT: entry:<br>
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1<br>
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[SRC]], align 4<br>
-; CHECK-NEXT: [[SUB:%.*]] = add nsw i32 [[TMP0]], -1<br>
; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1<br>
-; CHECK-NEXT: store i32 [[SUB]], i32* [[DST]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2<br>
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2<br>
-; CHECK-NEXT: store i32 [[TMP1]], i32* [[INCDEC_PTR1]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3<br>
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4<br>
-; CHECK-NEXT: [[SUB5:%.*]] = add nsw i32 [[TMP2]], -2<br>
; CHECK-NEXT: [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3<br>
-; CHECK-NEXT: store i32 [[SUB5]], i32* [[INCDEC_PTR3]], align 4<br>
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4<br>
-; CHECK-NEXT: [[SUB8:%.*]] = add nsw i32 [[TMP3]], -3<br>
-; CHECK-NEXT: store i32 [[SUB8]], i32* [[INCDEC_PTR6]], align 4<br>
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast i32* [[SRC]] to <4 x i32>*<br>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4<br>
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> <i32 -1, i32 0, i32 -2, i32 -3>, [[TMP1]]<br>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[DST]] to <4 x i32>*<br>
+; CHECK-NEXT: store <4 x i32> [[TMP2]], <4 x i32>* [[TMP3]], align 4<br>
; CHECK-NEXT: ret void<br>
;<br>
entry:<br>
@@ -205,22 +193,18 @@ define void @addsub0(i32* noalias %dst,<br>
; CHECK-LABEL: @addsub0(<br>
; CHECK-NEXT: entry:<br>
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1<br>
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[SRC]], align 4<br>
-; CHECK-NEXT: [[SUB:%.*]] = add nsw i32 [[TMP0]], -1<br>
; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1<br>
-; CHECK-NEXT: store i32 [[SUB]], i32* [[DST]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2<br>
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2<br>
-; CHECK-NEXT: store i32 [[TMP1]], i32* [[INCDEC_PTR1]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3<br>
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4<br>
-; CHECK-NEXT: [[SUB5:%.*]] = add nsw i32 [[TMP2]], -2<br>
; CHECK-NEXT: [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3<br>
-; CHECK-NEXT: store i32 [[SUB5]], i32* [[INCDEC_PTR3]], align 4<br>
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4<br>
-; CHECK-NEXT: [[SUB8:%.*]] = sub nsw i32 [[TMP3]], -3<br>
-; CHECK-NEXT: store i32 [[SUB8]], i32* [[INCDEC_PTR6]], align 4<br>
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast i32* [[SRC]] to <4 x i32>*<br>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4<br>
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 -1, i32 0, i32 -2, i32 -3><br>
+; CHECK-NEXT: [[TMP3:%.*]] = sub nsw <4 x i32> [[TMP1]], <i32 -1, i32 0, i32 -2, i32 -3><br>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7><br>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i32* [[DST]] to <4 x i32>*<br>
+; CHECK-NEXT: store <4 x i32> [[TMP4]], <4 x i32>* [[TMP5]], align 4<br>
; CHECK-NEXT: ret void<br>
;<br>
entry:<br>
@@ -248,22 +232,18 @@ define void @addsub1(i32* noalias %dst,<br>
; CHECK-LABEL: @addsub1(<br>
; CHECK-NEXT: entry:<br>
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1<br>
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[SRC]], align 4<br>
-; CHECK-NEXT: [[SUB:%.*]] = add nsw i32 [[TMP0]], -1<br>
; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1<br>
-; CHECK-NEXT: store i32 [[SUB]], i32* [[DST]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2<br>
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4<br>
-; CHECK-NEXT: [[SUB1:%.*]] = sub nsw i32 [[TMP1]], -1<br>
; CHECK-NEXT: [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2<br>
-; CHECK-NEXT: store i32 [[SUB1]], i32* [[INCDEC_PTR1]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3<br>
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3<br>
-; CHECK-NEXT: store i32 [[TMP2]], i32* [[INCDEC_PTR3]], align 4<br>
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4<br>
-; CHECK-NEXT: [[SUB8:%.*]] = sub nsw i32 [[TMP3]], -3<br>
-; CHECK-NEXT: store i32 [[SUB8]], i32* [[INCDEC_PTR6]], align 4<br>
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast i32* [[SRC]] to <4 x i32>*<br>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4<br>
+; CHECK-NEXT: [[TMP2:%.*]] = add nsw <4 x i32> [[TMP1]], <i32 -1, i32 -1, i32 0, i32 -3><br>
+; CHECK-NEXT: [[TMP3:%.*]] = sub nsw <4 x i32> [[TMP1]], <i32 -1, i32 -1, i32 0, i32 -3><br>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7><br>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast i32* [[DST]] to <4 x i32>*<br>
+; CHECK-NEXT: store <4 x i32> [[TMP4]], <4 x i32>* [[TMP5]], align 4<br>
; CHECK-NEXT: ret void<br>
;<br>
entry:<br>
@@ -291,22 +271,16 @@ define void @mul(i32* noalias %dst, i32*<br>
; CHECK-LABEL: @mul(<br>
; CHECK-NEXT: entry:<br>
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1<br>
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[SRC]], align 4<br>
-; CHECK-NEXT: [[MUL:%.*]] = mul nsw i32 [[TMP0]], 257<br>
; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1<br>
-; CHECK-NEXT: store i32 [[MUL]], i32* [[DST]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2<br>
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4<br>
-; CHECK-NEXT: [[MUL3:%.*]] = mul nsw i32 [[TMP1]], -3<br>
; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2<br>
-; CHECK-NEXT: store i32 [[MUL3]], i32* [[INCDEC_PTR1]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3<br>
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3<br>
-; CHECK-NEXT: store i32 [[TMP2]], i32* [[INCDEC_PTR4]], align 4<br>
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR5]], align 4<br>
-; CHECK-NEXT: [[MUL9:%.*]] = mul nsw i32 [[TMP3]], -9<br>
-; CHECK-NEXT: store i32 [[MUL9]], i32* [[INCDEC_PTR7]], align 4<br>
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast i32* [[SRC]] to <4 x i32>*<br>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4<br>
+; CHECK-NEXT: [[TMP2:%.*]] = mul nsw <4 x i32> <i32 257, i32 -3, i32 1, i32 -9>, [[TMP1]]<br>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[DST]] to <4 x i32>*<br>
+; CHECK-NEXT: store <4 x i32> [[TMP2]], <4 x i32>* [[TMP3]], align 4<br>
; CHECK-NEXT: ret void<br>
;<br>
entry:<br>
@@ -334,22 +308,16 @@ define void @shl0(i32* noalias %dst, i32<br>
; CHECK-LABEL: @shl0(<br>
; CHECK-NEXT: entry:<br>
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds i32, i32* [[SRC:%.*]], i64 1<br>
-; CHECK-NEXT: [[TMP0:%.*]] = load i32, i32* [[SRC]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds i32, i32* [[DST:%.*]], i64 1<br>
-; CHECK-NEXT: store i32 [[TMP0]], i32* [[DST]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 2<br>
-; CHECK-NEXT: [[TMP1:%.*]] = load i32, i32* [[INCDEC_PTR]], align 4<br>
-; CHECK-NEXT: [[SHL:%.*]] = shl i32 [[TMP1]], 1<br>
; CHECK-NEXT: [[INCDEC_PTR3:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 2<br>
-; CHECK-NEXT: store i32 [[SHL]], i32* [[INCDEC_PTR1]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds i32, i32* [[SRC]], i64 3<br>
-; CHECK-NEXT: [[TMP2:%.*]] = load i32, i32* [[INCDEC_PTR2]], align 4<br>
-; CHECK-NEXT: [[SHL5:%.*]] = shl i32 [[TMP2]], 2<br>
; CHECK-NEXT: [[INCDEC_PTR6:%.*]] = getelementptr inbounds i32, i32* [[DST]], i64 3<br>
-; CHECK-NEXT: store i32 [[SHL5]], i32* [[INCDEC_PTR3]], align 4<br>
-; CHECK-NEXT: [[TMP3:%.*]] = load i32, i32* [[INCDEC_PTR4]], align 4<br>
-; CHECK-NEXT: [[SHL8:%.*]] = shl i32 [[TMP3]], 3<br>
-; CHECK-NEXT: store i32 [[SHL8]], i32* [[INCDEC_PTR6]], align 4<br>
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast i32* [[SRC]] to <4 x i32>*<br>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, <4 x i32>* [[TMP0]], align 4<br>
+; CHECK-NEXT: [[TMP2:%.*]] = shl <4 x i32> [[TMP1]], <i32 0, i32 1, i32 2, i32 3><br>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast i32* [[DST]] to <4 x i32>*<br>
+; CHECK-NEXT: store <4 x i32> [[TMP2]], <4 x i32>* [[TMP3]], align 4<br>
; CHECK-NEXT: ret void<br>
;<br>
entry:<br>
@@ -453,22 +421,16 @@ define void @add1f(float* noalias %dst,<br>
; CHECK-LABEL: @add1f(<br>
; CHECK-NEXT: entry:<br>
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1<br>
-; CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[SRC]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1<br>
-; CHECK-NEXT: store float [[TMP0]], float* [[DST]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2<br>
-; CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4<br>
-; CHECK-NEXT: [[ADD3:%.*]] = fadd fast float [[TMP1]], 1.000000e+00<br>
; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2<br>
-; CHECK-NEXT: store float [[ADD3]], float* [[INCDEC_PTR1]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3<br>
-; CHECK-NEXT: [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4<br>
-; CHECK-NEXT: [[ADD6:%.*]] = fadd fast float [[TMP2]], 2.000000e+00<br>
; CHECK-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3<br>
-; CHECK-NEXT: store float [[ADD6]], float* [[INCDEC_PTR4]], align 4<br>
-; CHECK-NEXT: [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4<br>
-; CHECK-NEXT: [[ADD9:%.*]] = fadd fast float [[TMP3]], 3.000000e+00<br>
-; CHECK-NEXT: store float [[ADD9]], float* [[INCDEC_PTR7]], align 4<br>
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast float* [[SRC]] to <4 x float>*<br>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4<br>
+; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> <float 0.000000e+00, float 1.000000e+00, float 2.000000e+00, float 3.000000e+00>, [[TMP1]]<br>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float* [[DST]] to <4 x float>*<br>
+; CHECK-NEXT: store <4 x float> [[TMP2]], <4 x float>* [[TMP3]], align 4<br>
; CHECK-NEXT: ret void<br>
;<br>
entry:<br>
@@ -496,22 +458,16 @@ define void @sub0f(float* noalias %dst,<br>
; CHECK-LABEL: @sub0f(<br>
; CHECK-NEXT: entry:<br>
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1<br>
-; CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[SRC]], align 4<br>
-; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP0]], -1.000000e+00<br>
; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1<br>
-; CHECK-NEXT: store float [[ADD]], float* [[DST]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2<br>
-; CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2<br>
-; CHECK-NEXT: store float [[TMP1]], float* [[INCDEC_PTR1]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3<br>
-; CHECK-NEXT: [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4<br>
-; CHECK-NEXT: [[ADD6:%.*]] = fadd fast float [[TMP2]], -2.000000e+00<br>
; CHECK-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3<br>
-; CHECK-NEXT: store float [[ADD6]], float* [[INCDEC_PTR4]], align 4<br>
-; CHECK-NEXT: [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4<br>
-; CHECK-NEXT: [[ADD9:%.*]] = fadd fast float [[TMP3]], -3.000000e+00<br>
-; CHECK-NEXT: store float [[ADD9]], float* [[INCDEC_PTR7]], align 4<br>
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast float* [[SRC]] to <4 x float>*<br>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4<br>
+; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> <float -1.000000e+00, float 0.000000e+00, float -2.000000e+00, float -3.000000e+00>, [[TMP1]]<br>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float* [[DST]] to <4 x float>*<br>
+; CHECK-NEXT: store <4 x float> [[TMP2]], <4 x float>* [[TMP3]], align 4<br>
; CHECK-NEXT: ret void<br>
;<br>
entry:<br>
@@ -615,22 +571,18 @@ define void @addsub0f(float* noalias %ds<br>
; CHECK-LABEL: @addsub0f(<br>
; CHECK-NEXT: entry:<br>
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1<br>
-; CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[SRC]], align 4<br>
-; CHECK-NEXT: [[SUB:%.*]] = fadd fast float [[TMP0]], -1.000000e+00<br>
; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1<br>
-; CHECK-NEXT: store float [[SUB]], float* [[DST]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2<br>
-; CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR3:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2<br>
-; CHECK-NEXT: store float [[TMP1]], float* [[INCDEC_PTR1]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3<br>
-; CHECK-NEXT: [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4<br>
-; CHECK-NEXT: [[SUB5:%.*]] = fadd fast float [[TMP2]], -2.000000e+00<br>
; CHECK-NEXT: [[INCDEC_PTR6:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3<br>
-; CHECK-NEXT: store float [[SUB5]], float* [[INCDEC_PTR3]], align 4<br>
-; CHECK-NEXT: [[TMP3:%.*]] = load float, float* [[INCDEC_PTR4]], align 4<br>
-; CHECK-NEXT: [[SUB8:%.*]] = fsub fast float [[TMP3]], -3.000000e+00<br>
-; CHECK-NEXT: store float [[SUB8]], float* [[INCDEC_PTR6]], align 4<br>
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast float* [[SRC]] to <4 x float>*<br>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4<br>
+; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> [[TMP1]], <float -1.000000e+00, float 0.000000e+00, float -2.000000e+00, float -3.000000e+00><br>
+; CHECK-NEXT: [[TMP3:%.*]] = fsub fast <4 x float> [[TMP1]], <float -1.000000e+00, float 0.000000e+00, float -2.000000e+00, float -3.000000e+00><br>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7><br>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast float* [[DST]] to <4 x float>*<br>
+; CHECK-NEXT: store <4 x float> [[TMP4]], <4 x float>* [[TMP5]], align 4<br>
; CHECK-NEXT: ret void<br>
;<br>
entry:<br>
@@ -658,22 +610,18 @@ define void @addsub1f(float* noalias %ds<br>
; CHECK-LABEL: @addsub1f(<br>
; CHECK-NEXT: entry:<br>
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1<br>
-; CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[SRC]], align 4<br>
-; CHECK-NEXT: [[SUB:%.*]] = fadd fast float [[TMP0]], -1.000000e+00<br>
; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1<br>
-; CHECK-NEXT: store float [[SUB]], float* [[DST]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2<br>
-; CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4<br>
-; CHECK-NEXT: [[SUB1:%.*]] = fsub fast float [[TMP1]], -1.000000e+00<br>
; CHECK-NEXT: [[INCDEC_PTR3:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2<br>
-; CHECK-NEXT: store float [[SUB1]], float* [[INCDEC_PTR1]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3<br>
-; CHECK-NEXT: [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR6:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3<br>
-; CHECK-NEXT: store float [[TMP2]], float* [[INCDEC_PTR3]], align 4<br>
-; CHECK-NEXT: [[TMP3:%.*]] = load float, float* [[INCDEC_PTR4]], align 4<br>
-; CHECK-NEXT: [[SUB8:%.*]] = fsub fast float [[TMP3]], -3.000000e+00<br>
-; CHECK-NEXT: store float [[SUB8]], float* [[INCDEC_PTR6]], align 4<br>
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast float* [[SRC]] to <4 x float>*<br>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4<br>
+; CHECK-NEXT: [[TMP2:%.*]] = fadd fast <4 x float> [[TMP1]], <float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -3.000000e+00><br>
+; CHECK-NEXT: [[TMP3:%.*]] = fsub fast <4 x float> [[TMP1]], <float -1.000000e+00, float -1.000000e+00, float 0.000000e+00, float -3.000000e+00><br>
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <4 x float> [[TMP2]], <4 x float> [[TMP3]], <4 x i32> <i32 0, i32 5, i32 2, i32 7><br>
+; CHECK-NEXT: [[TMP5:%.*]] = bitcast float* [[DST]] to <4 x float>*<br>
+; CHECK-NEXT: store <4 x float> [[TMP4]], <4 x float>* [[TMP5]], align 4<br>
; CHECK-NEXT: ret void<br>
;<br>
entry:<br>
@@ -701,22 +649,16 @@ define void @mulf(float* noalias %dst, f<br>
; CHECK-LABEL: @mulf(<br>
; CHECK-NEXT: entry:<br>
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1<br>
-; CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[SRC]], align 4<br>
-; CHECK-NEXT: [[SUB:%.*]] = fmul fast float [[TMP0]], 2.570000e+02<br>
; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1<br>
-; CHECK-NEXT: store float [[SUB]], float* [[DST]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2<br>
-; CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4<br>
-; CHECK-NEXT: [[SUB3:%.*]] = fmul fast float [[TMP1]], -3.000000e+00<br>
; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2<br>
-; CHECK-NEXT: store float [[SUB3]], float* [[INCDEC_PTR1]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3<br>
-; CHECK-NEXT: [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3<br>
-; CHECK-NEXT: store float [[TMP2]], float* [[INCDEC_PTR4]], align 4<br>
-; CHECK-NEXT: [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4<br>
-; CHECK-NEXT: [[SUB9:%.*]] = fmul fast float [[TMP3]], -9.000000e+00<br>
-; CHECK-NEXT: store float [[SUB9]], float* [[INCDEC_PTR7]], align 4<br>
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast float* [[SRC]] to <4 x float>*<br>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4<br>
+; CHECK-NEXT: [[TMP2:%.*]] = fmul fast <4 x float> <float 2.570000e+02, float -3.000000e+00, float 1.000000e+00, float -9.000000e+00>, [[TMP1]]<br>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float* [[DST]] to <4 x float>*<br>
+; CHECK-NEXT: store <4 x float> [[TMP2]], <4 x float>* [[TMP3]], align 4<br>
; CHECK-NEXT: ret void<br>
;<br>
entry:<br>
@@ -825,22 +767,16 @@ define void @sub0fn(float* noalias %dst,<br>
; CHECK-LABEL: @sub0fn(<br>
; CHECK-NEXT: entry:<br>
; CHECK-NEXT: [[INCDEC_PTR:%.*]] = getelementptr inbounds float, float* [[SRC:%.*]], i64 1<br>
-; CHECK-NEXT: [[TMP0:%.*]] = load float, float* [[SRC]], align 4<br>
-; CHECK-NEXT: [[ADD:%.*]] = fadd fast float [[TMP0]], -1.000000e+00<br>
; CHECK-NEXT: [[INCDEC_PTR1:%.*]] = getelementptr inbounds float, float* [[DST:%.*]], i64 1<br>
-; CHECK-NEXT: store float [[ADD]], float* [[DST]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR2:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 2<br>
-; CHECK-NEXT: [[TMP1:%.*]] = load float, float* [[INCDEC_PTR]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR4:%.*]] = getelementptr inbounds float, float* [[DST]], i64 2<br>
-; CHECK-NEXT: store float [[TMP1]], float* [[INCDEC_PTR1]], align 4<br>
; CHECK-NEXT: [[INCDEC_PTR5:%.*]] = getelementptr inbounds float, float* [[SRC]], i64 3<br>
-; CHECK-NEXT: [[TMP2:%.*]] = load float, float* [[INCDEC_PTR2]], align 4<br>
-; CHECK-NEXT: [[ADD6:%.*]] = fadd float [[TMP2]], -2.000000e+00<br>
; CHECK-NEXT: [[INCDEC_PTR7:%.*]] = getelementptr inbounds float, float* [[DST]], i64 3<br>
-; CHECK-NEXT: store float [[ADD6]], float* [[INCDEC_PTR4]], align 4<br>
-; CHECK-NEXT: [[TMP3:%.*]] = load float, float* [[INCDEC_PTR5]], align 4<br>
-; CHECK-NEXT: [[ADD9:%.*]] = fadd float [[TMP3]], -3.000000e+00<br>
-; CHECK-NEXT: store float [[ADD9]], float* [[INCDEC_PTR7]], align 4<br>
+; CHECK-NEXT: [[TMP0:%.*]] = bitcast float* [[SRC]] to <4 x float>*<br>
+; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, <4 x float>* [[TMP0]], align 4<br>
+; CHECK-NEXT: [[TMP2:%.*]] = fadd <4 x float> <float -1.000000e+00, float 0.000000e+00, float -2.000000e+00, float -3.000000e+00>, [[TMP1]]<br>
+; CHECK-NEXT: [[TMP3:%.*]] = bitcast float* [[DST]] to <4 x float>*<br>
+; CHECK-NEXT: store <4 x float> [[TMP2]], <4 x float>* [[TMP3]], align 4<br>
; CHECK-NEXT: ret void<br>
;<br>
entry:<br>
<br>
<br>
______________________________<wbr>_________________<br>
llvm-commits mailing list<br>
<a href="mailto:llvm-commits@lists.llvm.org">llvm-commits@lists.llvm.org</a><br>
<a href="http://lists.llvm.org/cgi-bin/mailman/listinfo/llvm-commits" rel="noreferrer" target="_blank">http://lists.llvm.org/cgi-bin/<wbr>mailman/listinfo/llvm-commits</a><br>
</blockquote></div><br><br clear="all"><div><br></div>-- <br><div class="gmail_signature" data-smartmail="gmail_signature"><div dir="ltr"><div><div dir="ltr"><div>Regards,</div><div>Ilya Biryukov</div></div></div></div></div>
</div>