[PATCH 3/3] R600: Use optimized 24bit path in udivrem
Jan Vesely
jan.vesely at rutgers.edu
Tue Jul 29 12:10:18 PDT 2014
On Mon, 2014-07-28 at 22:40 -0700, Matt Arsenault wrote:
> On Jul 26, 2014, at 5:15 PM, Jan Vesely <jan.vesely at rutgers.edu> wrote:
>
> > Signed-off-by: Jan Vesely <jan.vesely at rutgers.edu>
> > ---
> > No piglit regressions on my TURKS gpu. Matt's extended tests modified for uint pass too.
> >
> > lib/Target/R600/AMDGPUISelLowering.cpp | 23 +++-
> > lib/Target/R600/AMDGPUISelLowering.h | 2 +-
> > test/CodeGen/R600/udivrem24.ll | 244 +++++++++++++++++++++++++++++++++
> > 3 files changed, 263 insertions(+), 6 deletions(-)
> > create mode 100644 test/CodeGen/R600/udivrem24.ll
> >
> > diff --git a/lib/Target/R600/AMDGPUISelLowering.cpp b/lib/Target/R600/AMDGPUISelLowering.cpp
> > index 3996093..290ea8c 100644
> > --- a/lib/Target/R600/AMDGPUISelLowering.cpp
> > +++ b/lib/Target/R600/AMDGPUISelLowering.cpp
> > @@ -1381,7 +1381,7 @@ SDValue AMDGPUTargetLowering::LowerSTORE(SDValue Op, SelectionDAG &DAG) const {
> > // This is a shortcut for integer division because we have fast i32<->f32
> > // conversions, and fast f32 reciprocal instructions. The fractional part of a
> > // float is enough to accurately represent up to a 24-bit integer.
> > -SDValue AMDGPUTargetLowering::LowerSDIVREM24(SDValue Op, SelectionDAG &DAG) const {
> > +SDValue AMDGPUTargetLowering::LowerDIVREM24(SDValue Op, SelectionDAG &DAG, bool sign) const {
> > SDLoc DL(Op);
> > EVT VT = Op.getValueType();
> > SDValue LHS = Op.getOperand(0);
> > @@ -1389,6 +1389,9 @@ SDValue AMDGPUTargetLowering::LowerSDIVREM24(SDValue Op, SelectionDAG &DAG) cons
> > MVT IntVT = MVT::i32;
> > MVT FltVT = MVT::f32;
> >
> > + enum ISD::NodeType ToFp = sign ? ISD::SINT_TO_FP : ISD::UINT_TO_FP;
> > + enum ISD::NodeType ToInt = sign ? ISD::FP_TO_SINT : ISD::FP_TO_UINT;
>
> Don’t need ‘enum’ here
Fixed in v2 with some other changes
>
> > +
> > if (VT.isVector()) {
> > unsigned NElts = VT.getVectorNumElements();
> > IntVT = MVT::getVectorVT(MVT::i32, NElts);
> > @@ -1416,10 +1419,10 @@ SDValue AMDGPUTargetLowering::LowerSDIVREM24(SDValue Op, SelectionDAG &DAG) cons
> > SDValue ib = DAG.getSExtOrTrunc(RHS, DL, IntVT);
> >
> > // float fa = (float)ia;
> > - SDValue fa = DAG.getNode(ISD::SINT_TO_FP, DL, FltVT, ia);
> > + SDValue fa = DAG.getNode(ToFp, DL, FltVT, ia);
> >
> > // float fb = (float)ib;
> > - SDValue fb = DAG.getNode(ISD::SINT_TO_FP, DL, FltVT, ib);
> > + SDValue fb = DAG.getNode(ToFp, DL, FltVT, ib);
> >
> > // float fq = native_divide(fa, fb);
> > SDValue fq = DAG.getNode(ISD::FMUL, DL, FltVT,
> > @@ -1436,7 +1439,7 @@ SDValue AMDGPUTargetLowering::LowerSDIVREM24(SDValue Op, SelectionDAG &DAG) cons
> > DAG.getNode(ISD::FMUL, DL, FltVT, fqneg, fb), fa);
> >
> > // int iq = (int)fq;
> > - SDValue iq = DAG.getNode(ISD::FP_TO_SINT, DL, IntVT, fq);
> > + SDValue iq = DAG.getNode(ToInt, DL, IntVT, fq);
> >
> > // fr = fabs(fr);
> > fr = DAG.getNode(ISD::FABS, DL, FltVT, fr);
> > @@ -1475,6 +1478,16 @@ SDValue AMDGPUTargetLowering::LowerUDIVREM(SDValue Op,
> > SDValue Num = Op.getOperand(0);
> > SDValue Den = Op.getOperand(1);
> >
> > + if (VT == MVT::i32) {
> > + if (DAG.MaskedValueIsZero(Op.getOperand(0), APInt(32, 0xff << 24)) &&
> > + DAG.MaskedValueIsZero(Op.getOperand(1), APInt(32, 0xff << 24))) {
> > + // TODO: We technically could do this for i64, but shouldn't that just be
> > + // handled by something generally reducing 64-bit division on 32-bit
> > + // values to 32-bit?
> > + return LowerDIVREM24(Op, DAG, false);
> > + }
> > + }
> > +
> > // RCP = URECIP(Den) = 2^32 / Den + e
> > // e is rounding error.
> > SDValue RCP = DAG.getNode(AMDGPUISD::URECIP, DL, VT, Den);
> > @@ -1585,7 +1598,7 @@ SDValue AMDGPUTargetLowering::LowerSDIVREM(SDValue Op,
> > // TODO: We technically could do this for i64, but shouldn't that just be
> > // handled by something generally reducing 64-bit division on 32-bit
> > // values to 32-bit?
> > - return LowerSDIVREM24(Op, DAG);
> > + return LowerDIVREM24(Op, DAG, true);
> > }
> > }
> >
> > diff --git a/lib/Target/R600/AMDGPUISelLowering.h b/lib/Target/R600/AMDGPUISelLowering.h
> > index 574e9b2..fe576a3 100644
> > --- a/lib/Target/R600/AMDGPUISelLowering.h
> > +++ b/lib/Target/R600/AMDGPUISelLowering.h
> > @@ -82,7 +82,7 @@ protected:
> > SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const;
> > SDValue LowerSTORE(SDValue Op, SelectionDAG &DAG) const;
> > SDValue LowerSDIVREM(SDValue Op, SelectionDAG &DAG) const;
> > - SDValue LowerSDIVREM24(SDValue Op, SelectionDAG &DAG) const;
> > + SDValue LowerDIVREM24(SDValue Op, SelectionDAG &DAG, bool sign) const;
> > bool isHWTrueValue(SDValue Op) const;
> > bool isHWFalseValue(SDValue Op) const;
> >
> > diff --git a/test/CodeGen/R600/udivrem24.ll b/test/CodeGen/R600/udivrem24.ll
> > new file mode 100644
> > index 0000000..219c662
> > --- /dev/null
> > +++ b/test/CodeGen/R600/udivrem24.ll
> > @@ -0,0 +1,244 @@
> > +; RUN: llc -march=r600 -mcpu=SI < %s | FileCheck -check-prefix=SI -check-prefix=FUNC %s
> > +; RUN: llc -march=r600 -mcpu=redwood < %s | FileCheck -check-prefix=EG -check-prefix=FUNC %s
> > +
> > +; FUNC-LABEL: @udiv24_i8
> > +; SI: V_CVT_F32_UBYTE
> > +; SI: V_CVT_F32_UBYTE
> > +; SI: V_RCP_F32
> > +; SI: V_CVT_U32_F32
> > +
> > +; EG: UINT_TO_FLT
> > +; EG-DAG: UINT_TO_FLT
> > +; EG-DAG: RECIP_IEEE
> These don’t need to be EG-DAG since the conversions do need to happen before the recipe
Is there something I missed that's preventing the following sequence?:
fb <- int2flt b
recfb <- rcp fb
fa <- int2ftl a
... <- mul fa recfb
>
> > +; EG: FLT_TO_UINT
> > +define void @udiv24_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %in) {
> > + %den_ptr = getelementptr i8 addrspace(1)* %in, i8 1
> > + %num = load i8 addrspace(1) * %in
> > + %den = load i8 addrspace(1) * %den_ptr
> > + %result = udiv i8 %num, %den
> > + store i8 %result, i8 addrspace(1)* %out
> > + ret void
> > +}
> > +
> > +; FUNC-LABEL: @udiv24_i16
> > +; SI: V_CVT_F32_U32
> > +; SI: V_CVT_F32_U32
> > +; SI: V_RCP_F32
> > +; SI: V_CVT_U32_F32
> > +
> > +; EG: UINT_TO_FLT
> > +; EG-DAG: UINT_TO_FLT
> > +; EG-DAG: RECIP_IEEE
> > +; EG: FLT_TO_UINT
> > +define void @udiv24_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in) {
> > + %den_ptr = getelementptr i16 addrspace(1)* %in, i16 1
> > + %num = load i16 addrspace(1) * %in, align 2
> > + %den = load i16 addrspace(1) * %den_ptr, align 2
> > + %result = udiv i16 %num, %den
> > + store i16 %result, i16 addrspace(1)* %out, align 2
> > + ret void
> > +}
> > +
> > +; FUNC-LABEL: @udiv24_i32
> > +; SI: V_CVT_F32_U32
> > +; SI-DAG: V_CVT_F32_U32
> > +; SI-DAG: V_RCP_F32
> > +; SI: V_CVT_U32_F32
> > +
> > +; EG: UINT_TO_FLT
> > +; EG-DAG: UINT_TO_FLT
> > +; EG-DAG: RECIP_IEEE
> > +; EG: FLT_TO_UINT
> > +define void @udiv24_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
> > + %den_ptr = getelementptr i32 addrspace(1)* %in, i32 1
> > + %num = load i32 addrspace(1) * %in, align 4
> > + %den = load i32 addrspace(1) * %den_ptr, align 4
> > + %num.i24.0 = shl i32 %num, 8
> > + %den.i24.0 = shl i32 %den, 8
> > + %num.i24 = lshr i32 %num.i24.0, 8
> > + %den.i24 = lshr i32 %den.i24.0, 8
> > + %result = udiv i32 %num.i24, %den.i24
> > + store i32 %result, i32 addrspace(1)* %out, align 4
> > + ret void
> > +}
> > +
> > +; FUNC-LABEL: @udiv25_i32
> > +; RCP_IFLAG is for URECIP in the full 32b alg
> > +; SI: V_RCP_IFLAG
> > +; SI-NOT: V_RCP_F32
> > +
> > +; EG-NOT: UINT_TO_FLT
> > +; EG-NOT: RECIP_IEEE
> > +define void @udiv25_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
> > + %den_ptr = getelementptr i32 addrspace(1)* %in, i32 1
> > + %num = load i32 addrspace(1) * %in, align 4
> > + %den = load i32 addrspace(1) * %den_ptr, align 4
> > + %num.i24.0 = shl i32 %num, 7
> > + %den.i24.0 = shl i32 %den, 7
> > + %num.i24 = lshr i32 %num.i24.0, 7
> > + %den.i24 = lshr i32 %den.i24.0, 7
> > + %result = udiv i32 %num.i24, %den.i24
> > + store i32 %result, i32 addrspace(1)* %out, align 4
> > + ret void
> > +}
> > +
> > +; FUNC-LABEL: @test_no_udiv24_i32_1
> > +; RCP_IFLAG is for URECIP in the full 32b alg
> > +; SI: V_RCP_IFLAG
> > +; SI-NOT: V_RCP_F32
> > +
> > +; EG-NOT: UINT_TO_FLT
> > +; EG-NOT: RECIP_IEEE
> > +define void @test_no_udiv24_i32_1(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
> > + %den_ptr = getelementptr i32 addrspace(1)* %in, i32 1
> > + %num = load i32 addrspace(1) * %in, align 4
> > + %den = load i32 addrspace(1) * %den_ptr, align 4
> > + %num.i24.0 = shl i32 %num, 8
> > + %den.i24.0 = shl i32 %den, 7
> > + %num.i24 = lshr i32 %num.i24.0, 8
> > + %den.i24 = lshr i32 %den.i24.0, 7
> > + %result = udiv i32 %num.i24, %den.i24
> > + store i32 %result, i32 addrspace(1)* %out, align 4
> > + ret void
> > +}
> > +
> > +; FUNC-LABEL: @test_no_udiv24_i32_2
> > +; RCP_IFLAG is for URECIP in the full 32b alg
> > +; SI: V_RCP_IFLAG
> > +; SI-NOT: V_RCP_F32
> > +
> > +; EG-NOT: UINT_TO_FLT
> > +; EG-NOT: RECIP_IEEE
> > +define void @test_no_udiv24_i32_2(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
> > + %den_ptr = getelementptr i32 addrspace(1)* %in, i32 1
> > + %num = load i32 addrspace(1) * %in, align 4
> > + %den = load i32 addrspace(1) * %den_ptr, align 4
> > + %num.i24.0 = shl i32 %num, 7
> > + %den.i24.0 = shl i32 %den, 8
> > + %num.i24 = lshr i32 %num.i24.0, 7
> > + %den.i24 = lshr i32 %den.i24.0, 8
> > + %result = udiv i32 %num.i24, %den.i24
> > + store i32 %result, i32 addrspace(1)* %out, align 4
> > + ret void
> > +}
> > +
> > +; FUNC-LABEL: @urem24_i8
> > +; SI: V_CVT_F32_UBYTE
> > +; SI: V_CVT_F32_UBYTE
> > +; SI: V_RCP_F32
> > +; SI: V_CVT_U32_F32
> > +
> > +; EG: UINT_TO_FLT
> > +; EG-DAG: UINT_TO_FLT
> > +; EG-DAG: RECIP_IEEE
> > +; EG: FLT_TO_UINT
> > +define void @urem24_i8(i8 addrspace(1)* %out, i8 addrspace(1)* %in) {
> > + %den_ptr = getelementptr i8 addrspace(1)* %in, i8 1
> > + %num = load i8 addrspace(1) * %in
> > + %den = load i8 addrspace(1) * %den_ptr
> > + %result = urem i8 %num, %den
> > + store i8 %result, i8 addrspace(1)* %out
> > + ret void
> > +}
> > +
> > +; FUNC-LABEL: @urem24_i16
> > +; SI: V_CVT_F32_U32
> > +; SI: V_CVT_F32_U32
> > +; SI: V_RCP_F32
> > +; SI: V_CVT_U32_F32
> > +
> > +; EG: UINT_TO_FLT
> > +; EG-DAG: UINT_TO_FLT
> > +; EG-DAG: RECIP_IEEE
> > +; EG: FLT_TO_UINT
> > +define void @urem24_i16(i16 addrspace(1)* %out, i16 addrspace(1)* %in) {
> > + %den_ptr = getelementptr i16 addrspace(1)* %in, i16 1
> > + %num = load i16 addrspace(1) * %in, align 2
> > + %den = load i16 addrspace(1) * %den_ptr, align 2
> > + %result = urem i16 %num, %den
> > + store i16 %result, i16 addrspace(1)* %out, align 2
> > + ret void
> > +}
> > +
> > +; FUNC-LABEL: @urem24_i32
> > +; SI: V_CVT_F32_U32
> > +; SI: V_CVT_F32_U32
> > +; SI: V_RCP_F32
> > +; SI: V_CVT_U32_F32
> > +
> > +; EG: UINT_TO_FLT
> > +; EG-DAG: UINT_TO_FLT
> > +; EG-DAG: RECIP_IEEE
> > +; EG: FLT_TO_UINT
> > +define void @urem24_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
> > + %den_ptr = getelementptr i32 addrspace(1)* %in, i32 1
> > + %num = load i32 addrspace(1) * %in, align 4
> > + %den = load i32 addrspace(1) * %den_ptr, align 4
> > + %num.i24.0 = shl i32 %num, 8
> > + %den.i24.0 = shl i32 %den, 8
> > + %num.i24 = lshr i32 %num.i24.0, 8
> > + %den.i24 = lshr i32 %den.i24.0, 8
> > + %result = urem i32 %num.i24, %den.i24
> > + store i32 %result, i32 addrspace(1)* %out, align 4
> > + ret void
> > +}
> > +
> > +; FUNC-LABEL: @urem25_i32
> > +; RCP_IFLAG is for URECIP in the full 32b alg
> > +; SI: V_RCP_IFLAG
> > +; SI-NOT: V_RCP_F32
> > +
> > +; EG-NOT: UINT_TO_FLT
> > +; EG-NOT: RECIP_IEEE
> > +define void @urem25_i32(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
> > + %den_ptr = getelementptr i32 addrspace(1)* %in, i32 1
> > + %num = load i32 addrspace(1) * %in, align 4
> > + %den = load i32 addrspace(1) * %den_ptr, align 4
> > + %num.i24.0 = shl i32 %num, 7
> > + %den.i24.0 = shl i32 %den, 7
> > + %num.i24 = lshr i32 %num.i24.0, 7
> > + %den.i24 = lshr i32 %den.i24.0, 7
> > + %result = urem i32 %num.i24, %den.i24
> > + store i32 %result, i32 addrspace(1)* %out, align 4
> > + ret void
> > +}
> > +
> > +; FUNC-LABEL: @test_no_urem24_i32_1
> > +; RCP_IFLAG is for URECIP in the full 32b alg
> > +; SI: V_RCP_IFLAG
> > +; SI-NOT: V_RCP_F32
> > +
> > +; EG-NOT: UINT_TO_FLT
> > +; EG-NOT: RECIP_IEEE
> > +define void @test_no_urem24_i32_1(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
> > + %den_ptr = getelementptr i32 addrspace(1)* %in, i32 1
> > + %num = load i32 addrspace(1) * %in, align 4
> > + %den = load i32 addrspace(1) * %den_ptr, align 4
> > + %num.i24.0 = shl i32 %num, 8
> > + %den.i24.0 = shl i32 %den, 7
> > + %num.i24 = lshr i32 %num.i24.0, 8
> > + %den.i24 = lshr i32 %den.i24.0, 7
> > + %result = urem i32 %num.i24, %den.i24
> > + store i32 %result, i32 addrspace(1)* %out, align 4
> > + ret void
> > +}
> > +
> > +; FUNC-LABEL: @test_no_urem24_i32_2
> > +; RCP_IFLAG is for URECIP in the full 32b alg
> > +; SI: V_RCP_IFLAG
> > +; SI-NOT: V_RCP_F32
> > +
> > +; EG-NOT: UINT_TO_FLT
> > +; EG-NOT: RECIP_IEEE
> > +define void @test_no_urem24_i32_2(i32 addrspace(1)* %out, i32 addrspace(1)* %in) {
> > + %den_ptr = getelementptr i32 addrspace(1)* %in, i32 1
> > + %num = load i32 addrspace(1) * %in, align 4
> > + %den = load i32 addrspace(1) * %den_ptr, align 4
> > + %num.i24.0 = shl i32 %num, 7
> > + %den.i24.0 = shl i32 %den, 8
> > + %num.i24 = lshr i32 %num.i24.0, 7
> > + %den.i24 = lshr i32 %den.i24.0, 8
> > + %result = urem i32 %num.i24, %den.i24
> > + store i32 %result, i32 addrspace(1)* %out, align 4
> > + ret void
> > +}
> > --
> > 1.9.3
> >
>
--
Jan Vesely <jan.vesely at rutgers.edu>
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 819 bytes
Desc: This is a digitally signed message part
URL: <http://lists.llvm.org/pipermail/llvm-commits/attachments/20140729/12814a66/attachment.sig>
More information about the llvm-commits
mailing list