[llvm] r322499 - [Hexagon] Implement signed and unsigned multiply-high for vectors
Krzysztof Parzyszek via llvm-commits
llvm-commits at lists.llvm.org
Mon Jan 15 10:43:55 PST 2018
Author: kparzysz
Date: Mon Jan 15 10:43:55 2018
New Revision: 322499
URL: http://llvm.org/viewvc/llvm-project?rev=322499&view=rev
Log:
[Hexagon] Implement signed and unsigned multiply-high for vectors
Modified:
llvm/trunk/lib/Target/Hexagon/HexagonISelLowering.cpp
llvm/trunk/lib/Target/Hexagon/HexagonISelLowering.h
llvm/trunk/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
llvm/trunk/lib/Target/Hexagon/HexagonPatterns.td
Modified: llvm/trunk/lib/Target/Hexagon/HexagonISelLowering.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/Hexagon/HexagonISelLowering.cpp?rev=322499&r1=322498&r2=322499&view=diff
==============================================================================
--- llvm/trunk/lib/Target/Hexagon/HexagonISelLowering.cpp (original)
+++ llvm/trunk/lib/Target/Hexagon/HexagonISelLowering.cpp Mon Jan 15 10:43:55 2018
@@ -2095,6 +2095,8 @@ HexagonTargetLowering::HexagonTargetLowe
}
setOperationAction(ISD::MUL, T, Custom);
+ setOperationAction(ISD::MULHS, T, Custom);
+ setOperationAction(ISD::MULHU, T, Custom);
setOperationAction(ISD::SETCC, T, Custom);
setOperationAction(ISD::BUILD_VECTOR, T, Custom);
setOperationAction(ISD::INSERT_SUBVECTOR, T, Custom);
@@ -3018,6 +3020,11 @@ HexagonTargetLowering::LowerOperation(SD
if (Subtarget.useHVXOps())
return LowerHvxMul(Op, DAG);
break;
+ case ISD::MULHS:
+ case ISD::MULHU:
+ if (Subtarget.useHVXOps())
+ return LowerHvxMulh(Op, DAG);
+ break;
}
return SDValue();
}
Modified: llvm/trunk/lib/Target/Hexagon/HexagonISelLowering.h
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/Hexagon/HexagonISelLowering.h?rev=322499&r1=322498&r2=322499&view=diff
==============================================================================
--- llvm/trunk/lib/Target/Hexagon/HexagonISelLowering.h (original)
+++ llvm/trunk/lib/Target/Hexagon/HexagonISelLowering.h Mon Jan 15 10:43:55 2018
@@ -360,6 +360,7 @@ namespace HexagonISD {
SDValue LowerHvxExtractSubvector(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerHvxInsertSubvector(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerHvxMul(SDValue Op, SelectionDAG &DAG) const;
+ SDValue LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerHvxSetCC(SDValue Op, SelectionDAG &DAG) const;
SDValue LowerHvxExtend(SDValue Op, SelectionDAG &DAG) const;
Modified: llvm/trunk/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp?rev=322499&r1=322498&r2=322499&view=diff
==============================================================================
--- llvm/trunk/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp (original)
+++ llvm/trunk/lib/Target/Hexagon/HexagonISelLoweringHVX.cpp Mon Jan 15 10:43:55 2018
@@ -412,8 +412,7 @@ HexagonTargetLowering::LowerHvxInsertSub
SDValue
HexagonTargetLowering::LowerHvxMul(SDValue Op, SelectionDAG &DAG) const {
MVT ResTy = ty(Op);
- if (!ResTy.isVector())
- return SDValue();
+ assert(ResTy.isVector());
const SDLoc &dl(Op);
SmallVector<int,256> ShuffMask;
@@ -424,7 +423,7 @@ HexagonTargetLowering::LowerHvxMul(SDVal
switch (ElemTy.SimpleTy) {
case MVT::i8:
- case MVT::i16: {
+ case MVT::i16: { // V6_vmpyih
// For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
// V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
// where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
@@ -464,6 +463,112 @@ HexagonTargetLowering::LowerHvxMul(SDVal
}
SDValue
+HexagonTargetLowering::LowerHvxMulh(SDValue Op, SelectionDAG &DAG) const {
+ MVT ResTy = ty(Op);
+ assert(ResTy.isVector());
+ const SDLoc &dl(Op);
+ SmallVector<int,256> ShuffMask;
+
+ MVT ElemTy = ResTy.getVectorElementType();
+ unsigned VecLen = ResTy.getVectorNumElements();
+ SDValue Vs = Op.getOperand(0);
+ SDValue Vt = Op.getOperand(1);
+ bool IsSigned = Op.getOpcode() == ISD::MULHS;
+
+ if (ElemTy == MVT::i8 || ElemTy == MVT::i16) {
+ // For i8 vectors Vs = (a0, a1, ...), Vt = (b0, b1, ...),
+ // V6_vmpybv Vs, Vt produces a pair of i16 vectors Hi:Lo,
+ // where Lo = (a0*b0, a2*b2, ...), Hi = (a1*b1, a3*b3, ...).
+ // For i16, use V6_vmpyhv, which behaves in an analogous way to
+ // V6_vmpybv: results Lo and Hi are products of even/odd elements
+ // respectively.
+ MVT ExtTy = typeExtElem(ResTy, 2);
+ unsigned MpyOpc = ElemTy == MVT::i8
+ ? (IsSigned ? Hexagon::V6_vmpybv : Hexagon::V6_vmpyubv)
+ : (IsSigned ? Hexagon::V6_vmpyhv : Hexagon::V6_vmpyuhv);
+ SDValue M = getNode(MpyOpc, dl, ExtTy, {Vs, Vt}, DAG);
+
+ // Discard low halves of the resulting values, collect the high halves.
+ for (unsigned I = 0; I < VecLen; I += 2) {
+ ShuffMask.push_back(I+1); // Pick even element.
+ ShuffMask.push_back(I+VecLen+1); // Pick odd element.
+ }
+ VectorPair P = opSplit(opCastElem(M, ElemTy, DAG), dl, DAG);
+ SDValue BS = getByteShuffle(dl, P.first, P.second, ShuffMask, DAG);
+ return DAG.getBitcast(ResTy, BS);
+ }
+
+ assert(ElemTy == MVT::i32);
+ SDValue S16 = DAG.getConstant(16, dl, MVT::i32);
+
+ if (IsSigned) {
+ // mulhs(Vs,Vt) =
+ // = [(Hi(Vs)*2^16 + Lo(Vs)) *s (Hi(Vt)*2^16 + Lo(Vt))] >> 32
+ // = [Hi(Vs)*2^16 *s Hi(Vt)*2^16 + Hi(Vs) *su Lo(Vt)*2^16
+ // + Lo(Vs) *us (Hi(Vt)*2^16 + Lo(Vt))] >> 32
+ // = [Hi(Vs) *s Hi(Vt)*2^32 + Hi(Vs) *su Lo(Vt)*2^16
+ // + Lo(Vs) *us Vt] >> 32
+ // The low half of Lo(Vs)*Lo(Vt) will be discarded (it's not added to
+ // anything, so it cannot produce any carry over to higher bits),
+ // so everything in [] can be shifted by 16 without loss of precision.
+ // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + Lo(Vs)*Vt >> 16] >> 16
+ // = [Hi(Vs) *s Hi(Vt)*2^16 + Hi(Vs)*su Lo(Vt) + V6_vmpyewuh(Vs,Vt)] >> 16
+ // Denote Hi(Vs) = Vs':
+ // = [Vs'*s Hi(Vt)*2^16 + Vs' *su Lo(Vt) + V6_vmpyewuh(Vt,Vs)] >> 16
+ // = Vs'*s Hi(Vt) + (V6_vmpyiewuh(Vs',Vt) + V6_vmpyewuh(Vt,Vs)) >> 16
+ SDValue T0 = getNode(Hexagon::V6_vmpyewuh, dl, ResTy, {Vt, Vs}, DAG);
+ // Get Vs':
+ SDValue S0 = getNode(Hexagon::V6_vasrw, dl, ResTy, {Vs, S16}, DAG);
+ SDValue T1 = getNode(Hexagon::V6_vmpyiewuh_acc, dl, ResTy,
+ {T0, S0, Vt}, DAG);
+ // Shift by 16:
+ SDValue S2 = getNode(Hexagon::V6_vasrw, dl, ResTy, {T1, S16}, DAG);
+ // Get Vs'*Hi(Vt):
+ SDValue T2 = getNode(Hexagon::V6_vmpyiowh, dl, ResTy, {S0, Vt}, DAG);
+ // Add:
+ SDValue T3 = DAG.getNode(ISD::ADD, dl, ResTy, {S2, T2});
+ return T3;
+ }
+
+ // Unsigned mulhw. (Would expansion using signed mulhw be better?)
+
+ auto LoVec = [&DAG,ResTy,dl] (SDValue Pair) {
+ return DAG.getTargetExtractSubreg(Hexagon::vsub_lo, dl, ResTy, Pair);
+ };
+ auto HiVec = [&DAG,ResTy,dl] (SDValue Pair) {
+ return DAG.getTargetExtractSubreg(Hexagon::vsub_hi, dl, ResTy, Pair);
+ };
+
+ MVT PairTy = typeJoin({ResTy, ResTy});
+ SDValue P = getNode(Hexagon::V6_lvsplatw, dl, ResTy,
+ {DAG.getConstant(0x02020202, dl, MVT::i32)}, DAG);
+ // Multiply-unsigned halfwords:
+ // LoVec = Vs.uh[2i] * Vt.uh[2i],
+ // HiVec = Vs.uh[2i+1] * Vt.uh[2i+1]
+ SDValue T0 = getNode(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, Vt}, DAG);
+ // The low halves in the LoVec of the pair can be discarded. They are
+ // not added to anything (in the full-precision product), so they cannot
+ // produce a carry into the higher bits.
+ SDValue T1 = getNode(Hexagon::V6_vlsrw, dl, ResTy, {LoVec(T0), S16}, DAG);
+ // Swap low and high halves in Vt, and do the halfword multiplication
+ // to get products Vs.uh[2i] * Vt.uh[2i+1] and Vs.uh[2i+1] * Vt.uh[2i].
+ SDValue D0 = getNode(Hexagon::V6_vdelta, dl, ResTy, {Vt, P}, DAG);
+ SDValue T2 = getNode(Hexagon::V6_vmpyuhv, dl, PairTy, {Vs, D0}, DAG);
+ // T2 has mixed products of halfwords: Lo(Vt)*Hi(Vs) and Hi(Vt)*Lo(Vs).
+ // These products are words, but cannot be added directly because the
+ // sums could overflow. Add these products, by halfwords, where each sum
+ // of a pair of halfwords gives a word.
+ SDValue T3 = getNode(Hexagon::V6_vadduhw, dl, PairTy,
+ {LoVec(T2), HiVec(T2)}, DAG);
+ // Add the high halfwords from the products of the low halfwords.
+ SDValue T4 = DAG.getNode(ISD::ADD, dl, ResTy, {T1, LoVec(T3)});
+ SDValue T5 = getNode(Hexagon::V6_vlsrw, dl, ResTy, {T4, S16}, DAG);
+ SDValue T6 = DAG.getNode(ISD::ADD, dl, ResTy, {HiVec(T0), HiVec(T3)});
+ SDValue T7 = DAG.getNode(ISD::ADD, dl, ResTy, {T5, T6});
+ return T7;
+}
+
+SDValue
HexagonTargetLowering::LowerHvxSetCC(SDValue Op, SelectionDAG &DAG) const {
MVT VecTy = ty(Op.getOperand(0));
assert(VecTy == ty(Op.getOperand(1)));
Modified: llvm/trunk/lib/Target/Hexagon/HexagonPatterns.td
URL: http://llvm.org/viewvc/llvm-project/llvm/trunk/lib/Target/Hexagon/HexagonPatterns.td?rev=322499&r1=322498&r2=322499&view=diff
==============================================================================
--- llvm/trunk/lib/Target/Hexagon/HexagonPatterns.td (original)
+++ llvm/trunk/lib/Target/Hexagon/HexagonPatterns.td Mon Jan 15 10:43:55 2018
@@ -1274,6 +1274,56 @@ def: AccRRI_pat<M2_naccii, Sub, Su<Ad
def: AccRRI_pat<M2_accii, Add, Su<Add>, I32, s32_0ImmPred>;
def: AccRRR_pat<M2_acci, Add, Su<Add>, I32, I32>;
+// Mulh for vectors
+//
+def: Pat<(v2i32 (mulhu V2I32:$Rss, V2I32:$Rtt)),
+ (Combinew (M2_mpyu_up (HiReg $Rss), (HiReg $Rtt)),
+ (M2_mpyu_up (LoReg $Rss), (LoReg $Rtt)))>;
+
+def: Pat<(v2i32 (mulhs V2I32:$Rs, V2I32:$Rt)),
+ (Combinew (M2_mpy_up (HiReg $Rs), (HiReg $Rt)),
+ (M2_mpy_up (LoReg $Rt), (LoReg $Rt)))>;
+
+def Mulhub:
+ OutPatFrag<(ops node:$Rss, node:$Rtt),
+ (Combinew (S2_vtrunohb (M5_vmpybuu (HiReg $Rss), (HiReg $Rtt))),
+ (S2_vtrunohb (M5_vmpybuu (LoReg $Rss), (LoReg $Rtt))))>;
+
+// Equivalent of byte-wise arithmetic shift right by 7 in v8i8.
+def Asr7:
+ OutPatFrag<(ops node:$Rss), (C2_mask (C2_not (A4_vcmpbgti $Rss, 0)))>;
+
+def: Pat<(v8i8 (mulhu V8I8:$Rss, V8I8:$Rtt)),
+ (Mulhub $Rss, $Rtt)>;
+
+def: Pat<(v8i8 (mulhs V8I8:$Rss, V8I8:$Rtt)),
+ (A2_vsubub
+ (Mulhub $Rss, $Rtt),
+ (A2_vaddub (A2_andp V8I8:$Rss, (Asr7 $Rtt)),
+ (A2_andp V8I8:$Rtt, (Asr7 $Rss))))>;
+
+def Mpysh:
+ OutPatFrag<(ops node:$Rs, node:$Rt), (M2_vmpy2s_s0 $Rs, $Rt)>;
+def Mpyshh:
+ OutPatFrag<(ops node:$Rss, node:$Rtt), (Mpysh (HiReg $Rss), (HiReg $Rtt))>;
+def Mpyshl:
+ OutPatFrag<(ops node:$Rss, node:$Rtt), (Mpysh (LoReg $Rss), (LoReg $Rtt))>;
+
+def Mulhsh:
+ OutPatFrag<(ops node:$Rss, node:$Rtt),
+ (Combinew (A2_combine_hh (HiReg (Mpyshh $Rss, $Rtt)),
+ (LoReg (Mpyshh $Rss, $Rtt))),
+ (A2_combine_hh (HiReg (Mpyshl $Rss, $Rtt)),
+ (LoReg (Mpyshl $Rss, $Rtt))))>;
+
+def: Pat<(v4i16 (mulhs V4I16:$Rss, V4I16:$Rtt)), (Mulhsh $Rss, $Rtt)>;
+
+def: Pat<(v4i16 (mulhu V4I16:$Rss, V4I16:$Rtt)),
+ (A2_vaddh
+ (Mulhsh $Rss, $Rtt),
+ (A2_vaddh (A2_andp V4I16:$Rss, (S2_asr_i_vh $Rtt, 15)),
+ (A2_andp V4I16:$Rtt, (S2_asr_i_vh $Rss, 15))))>;
+
def: Pat<(ineg (mul I32:$Rs, u8_0ImmPred:$u8)),
(M2_mpysin IntRegs:$Rs, imm:$u8)>;
More information about the llvm-commits
mailing list