[llvm] [AMDGPU] Hoist readlane/readfirst through unary/binary operands (PR #129037)
Pierre van Houtryve via llvm-commits
llvm-commits at lists.llvm.org
Thu Feb 27 03:42:59 PST 2025
https://github.com/Pierre-vh updated https://github.com/llvm/llvm-project/pull/129037
>From ba3e6f63083379ca539a3d0b0f4281ebd7c55dcb Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Thu, 27 Feb 2025 12:11:44 +0100
Subject: [PATCH 1/2] [AMDGPU] Hoist readlane/readfirst through unary/binary
operands
When a read(first)lane is used on a binary operator and the intrinsic is the only user of the operator, we can move the read(first)lane into the operand if the other operand is uniform.
Unfortunately IC doesn't let us access UniformityAnalysis and thus we can't truly check uniformity, we have to do with a basic uniformity check which only allows constants or trivially uniform intrinsics calls.
We can also do the same for simple unary operations.
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 59 +++
.../Target/AMDGPU/AMDGPUTargetTransformInfo.h | 3 +
.../AMDGPU/llvm.amdgcn.readfirstlane.ll | 461 ++++++++++++++++++
.../AMDGPU/llvm.amdgcn.readlane.ll | 143 ++++++
4 files changed, 666 insertions(+)
create mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readfirstlane.ll
create mode 100644 llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readlane.ll
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index ebc00e59584ac..c2a977d7168fe 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -481,6 +481,59 @@ bool GCNTTIImpl::simplifyDemandedLaneMaskArg(InstCombiner &IC,
return false;
}
+Instruction *GCNTTIImpl::hoistReadLaneThroughOperand(InstCombiner &IC,
+ IntrinsicInst &II) const {
+ Instruction *Op = dyn_cast<Instruction>(II.getOperand(0));
+
+ // Only do this if both instructions are in the same block
+ // (so the exec mask won't change) and the readlane is the only user of its
+ // operand.
+ if (!Op || !Op->hasOneUser() || Op->getParent() != II.getParent())
+ return nullptr;
+
+ const bool IsReadLane = (II.getIntrinsicID() == Intrinsic::amdgcn_readlane);
+
+ // If this is a readlane, check that the second operand is a constant, or is
+ // defined before Op so we know it's safe to move this intrinsic higher.
+ Value *LaneID = nullptr;
+ if (IsReadLane) {
+ LaneID = II.getOperand(1);
+ if (!isa<Constant>(LaneID) && !(isa<Instruction>(LaneID) &&
+ cast<Instruction>(LaneID)->comesBefore(Op)))
+ return nullptr;
+ }
+
+ const auto DoIt = [&](unsigned OpIdx) -> Instruction * {
+ SmallVector<Value *, 2> Ops{Op->getOperand(OpIdx)};
+ if (IsReadLane)
+ Ops.push_back(LaneID);
+
+ Instruction *NewII =
+ IC.Builder.CreateIntrinsic(II.getType(), II.getIntrinsicID(), Ops);
+
+ Instruction &NewOp = *Op->clone();
+ NewOp.setOperand(OpIdx, NewII);
+ return &NewOp;
+ };
+
+ // TODO: Are any operations more expensive on the SALU than VALU, and thus
+ // need to be excluded here?
+
+ if (isa<UnaryOperator>(Op))
+ return DoIt(0);
+
+ if (isa<BinaryOperator>(Op)) {
+ // FIXME: If we had access to UniformityInfo here we could just check
+ // if the operand is uniform.
+ if (isTriviallyUniform(Op->getOperandUse(0)))
+ return DoIt(1);
+ if (isTriviallyUniform(Op->getOperandUse(1)))
+ return DoIt(0);
+ }
+
+ return nullptr;
+}
+
std::optional<Instruction *>
GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
Intrinsic::ID IID = II.getIntrinsicID();
@@ -1128,6 +1181,12 @@ GCNTTIImpl::instCombineIntrinsic(InstCombiner &IC, IntrinsicInst &II) const {
simplifyDemandedLaneMaskArg(IC, II, 1))
return &II;
+ // If the readfirstlane reads the result of an operation that exists
+ // both in the SALU and VALU, we may be able to hoist it higher in order
+ // to scalarize the expression.
+ if (Instruction *Res = hoistReadLaneThroughOperand(IC, II))
+ return Res;
+
return std::nullopt;
}
case Intrinsic::amdgcn_writelane: {
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
index a0d62008d9ddc..4f1ae82739d16 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.h
@@ -224,6 +224,9 @@ class GCNTTIImpl final : public BasicTTIImplBase<GCNTTIImpl> {
bool simplifyDemandedLaneMaskArg(InstCombiner &IC, IntrinsicInst &II,
unsigned LaneAgIdx) const;
+ Instruction *hoistReadLaneThroughOperand(InstCombiner &IC,
+ IntrinsicInst &II) const;
+
std::optional<Instruction *> instCombineIntrinsic(InstCombiner &IC,
IntrinsicInst &II) const;
std::optional<Value *> simplifyDemandedVectorEltsIntrinsic(
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readfirstlane.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readfirstlane.ll
new file mode 100644
index 0000000000000..9f27fda591382
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readfirstlane.ll
@@ -0,0 +1,461 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1030 -passes=instcombine -S < %s | FileCheck %s
+
+; test unary
+
+define float @hoist_fneg_f32(float %arg) {
+; CHECK-LABEL: define float @hoist_fneg_f32(
+; CHECK-SAME: float [[ARG:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = fneg float [[TMP0]]
+; CHECK-NEXT: ret float [[RFL]]
+;
+bb:
+ %val = fneg float %arg
+ %rfl = call float @llvm.amdgcn.readfirstlane.f32(float %val)
+ ret float %rfl
+}
+
+define double @hoist_fneg_f64(double %arg) {
+; CHECK-LABEL: define double @hoist_fneg_f64(
+; CHECK-SAME: double [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = fneg double [[TMP0]]
+; CHECK-NEXT: ret double [[RFL]]
+;
+bb:
+ %val = fneg double %arg
+ %rfl = call double @llvm.amdgcn.readfirstlane.f64(double %val)
+ ret double %rfl
+}
+
+; test binary i32
+
+define i32 @hoist_add_i32(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_add_i32(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = add i32 [[TMP0]], 16777215
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = add i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define float @hoist_fadd_f32(float %arg) {
+; CHECK-LABEL: define float @hoist_fadd_f32(
+; CHECK-SAME: float [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = fadd float [[TMP0]], 1.280000e+02
+; CHECK-NEXT: ret float [[RFL]]
+;
+bb:
+ %val = fadd float %arg, 128.0
+ %rfl = call float @llvm.amdgcn.readfirstlane.f32(float %val)
+ ret float %rfl
+}
+
+define i32 @hoist_sub_i32(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_sub_i32(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = add i32 [[TMP0]], -16777215
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = sub i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define float @hoist_fsub_f32(float %arg) {
+; CHECK-LABEL: define float @hoist_fsub_f32(
+; CHECK-SAME: float [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = fadd float [[TMP0]], -1.280000e+02
+; CHECK-NEXT: ret float [[RFL]]
+;
+bb:
+ %val = fsub float %arg, 128.0
+ %rfl = call float @llvm.amdgcn.readfirstlane.f32(float %val)
+ ret float %rfl
+}
+
+define i32 @hoist_mul_i32(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_mul_i32(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = mul i32 [[TMP0]], 16777215
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = mul i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define float @hoist_fmul_f32(float %arg) {
+; CHECK-LABEL: define float @hoist_fmul_f32(
+; CHECK-SAME: float [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = fmul float [[TMP0]], 1.280000e+02
+; CHECK-NEXT: ret float [[RFL]]
+;
+bb:
+ %val = fmul float %arg, 128.0
+ %rfl = call float @llvm.amdgcn.readfirstlane.f32(float %val)
+ ret float %rfl
+}
+
+define i32 @hoist_udiv_i32(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_udiv_i32(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = udiv i32 [[TMP0]], 16777215
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = udiv i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define i32 @hoist_sdiv_i32(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_sdiv_i32(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = sdiv i32 [[TMP0]], 16777215
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = sdiv i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define float @hoist_fdiv_f32(float %arg) {
+; CHECK-LABEL: define float @hoist_fdiv_f32(
+; CHECK-SAME: float [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = fmul float [[TMP0]], 7.812500e-03
+; CHECK-NEXT: ret float [[RFL]]
+;
+bb:
+ %val = fdiv float %arg, 128.0
+ %rfl = call float @llvm.amdgcn.readfirstlane.f32(float %val)
+ ret float %rfl
+}
+
+define i32 @hoist_urem_i32(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_urem_i32(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = urem i32 [[TMP0]], 16777215
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = urem i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define i32 @hoist_srem_i32(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_srem_i32(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = srem i32 [[TMP0]], 16777215
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = srem i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define float @hoist_frem_f32(float %arg) {
+; CHECK-LABEL: define float @hoist_frem_f32(
+; CHECK-SAME: float [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = frem float [[TMP0]], 1.280000e+02
+; CHECK-NEXT: ret float [[RFL]]
+;
+bb:
+ %val = frem float %arg, 128.0
+ %rfl = call float @llvm.amdgcn.readfirstlane.f32(float %val)
+ ret float %rfl
+}
+
+define i32 @hoist_shl_i32(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_shl_i32(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = shl i32 [[TMP0]], 4
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = shl i32 %arg, 4
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define i32 @hoist_lshr_i32(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_lshr_i32(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = lshr i32 [[TMP0]], 4
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = lshr i32 %arg, 4
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define i32 @hoist_ashr_i32(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_ashr_i32(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = ashr i32 [[TMP0]], 4
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = ashr i32 %arg, 4
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+
+define i32 @hoist_and_i32(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_and_i32(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = and i32 [[TMP0]], 16777215
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = and i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define i32 @hoist_or_i32(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_or_i32(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = or i32 [[TMP0]], 16777215
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = or i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define i32 @hoist_xor_i32(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_xor_i32(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = xor i32 [[TMP0]], 16777215
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = xor i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+; test binary i64
+
+define i64 @hoist_and_i64(i64 %arg) {
+; CHECK-LABEL: define i64 @hoist_and_i64(
+; CHECK-SAME: i64 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.readfirstlane.i64(i64 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = and i64 [[TMP0]], 16777215
+; CHECK-NEXT: ret i64 [[RFL]]
+;
+bb:
+ %val = and i64 %arg, 16777215
+ %rfl = call i64 @llvm.amdgcn.readfirstlane.i32(i64 %val)
+ ret i64 %rfl
+}
+
+define double @hoist_fadd_f64(double %arg) {
+; CHECK-LABEL: define double @hoist_fadd_f64(
+; CHECK-SAME: double [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call double @llvm.amdgcn.readfirstlane.f64(double [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = fadd double [[TMP0]], 1.280000e+02
+; CHECK-NEXT: ret double [[RFL]]
+;
+bb:
+ %val = fadd double %arg, 128.0
+ %rfl = call double @llvm.amdgcn.readfirstlane.f64(double %val)
+ ret double %rfl
+}
+
+; test constant on LHS
+
+define i32 @hoist_sub_i32_lhs(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_sub_i32_lhs(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = sub i32 16777215, [[TMP0]]
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = sub i32 16777215, %arg
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define float @hoist_fsub_f32_lhs(float %arg) {
+; CHECK-LABEL: define float @hoist_fsub_f32_lhs(
+; CHECK-SAME: float [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call float @llvm.amdgcn.readfirstlane.f32(float [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = fsub float 1.280000e+02, [[TMP0]]
+; CHECK-NEXT: ret float [[RFL]]
+;
+bb:
+ %val = fsub float 128.0, %arg
+ %rfl = call float @llvm.amdgcn.readfirstlane.f32(float %val)
+ ret float %rfl
+}
+
+; test other operand is trivially uniform
+
+define i32 @hoist_add_i32_trivially_uniform_rhs(i32 %arg, i32 %v.other) {
+; CHECK-LABEL: define i32 @hoist_add_i32_trivially_uniform_rhs(
+; CHECK-SAME: i32 [[ARG:%.*]], i32 [[V_OTHER:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[OTHER:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[V_OTHER]])
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = add i32 [[TMP0]], [[OTHER]]
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %other = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %v.other)
+ %val = add i32 %arg, %other
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define i32 @hoist_add_i32_trivially_uniform_lhs(i32 %arg, i32 %v.other) {
+; CHECK-LABEL: define i32 @hoist_add_i32_trivially_uniform_lhs(
+; CHECK-SAME: i32 [[ARG:%.*]], i32 [[V_OTHER:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[OTHER:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[V_OTHER]])
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[RFL:%.*]] = sub i32 [[OTHER]], [[TMP0]]
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %other = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %v.other)
+ %val = sub i32 %other, %arg
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+; test multiple iterations
+
+define i32 @hoist_multiple_times(i32 %arg) {
+; CHECK-LABEL: define i32 @hoist_multiple_times(
+; CHECK-SAME: i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]])
+; CHECK-NEXT: [[TMP1:%.*]] = shl i32 [[TMP0]], 2
+; CHECK-NEXT: [[TMP2:%.*]] = sub i32 16777215, [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = xor i32 [[TMP2]], 4242
+; CHECK-NEXT: [[RFL:%.*]] = add i32 [[TMP3]], 6
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val.0 = shl i32 %arg, 2
+ %val.1 = sub i32 16777215, %val.0
+ %val.2 = xor i32 %val.1, 4242
+ %val.3 = add i32 %val.2, 6
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val.3)
+ ret i32 %rfl
+}
+
+; test cases where hoisting isn't possible
+
+define i32 @cross_block_hoisting(i1 %cond, i32 %arg) {
+; CHECK-LABEL: define i32 @cross_block_hoisting(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[ARG:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*]]:
+; CHECK-NEXT: [[VAL:%.*]] = add i32 [[ARG]], 16777215
+; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[END:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[RFL:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[VAL]])
+; CHECK-NEXT: br label %[[END]]
+; CHECK: [[END]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RFL]], %[[THEN]] ], [ [[VAL]], %[[BB]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+bb:
+ %val = add i32 %arg, 16777215
+ br i1 %cond, label %then, label %end
+
+then:
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ br label %end
+
+end:
+ %res = phi i32 [%rfl, %then], [%val, %bb]
+ ret i32 %res
+}
+
+define i32 @operand_is_instr(i32 %arg, ptr %src) {
+; CHECK-LABEL: define i32 @operand_is_instr(
+; CHECK-SAME: i32 [[ARG:%.*]], ptr [[SRC:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[OTHER:%.*]] = load i32, ptr [[SRC]], align 4
+; CHECK-NEXT: [[VAL:%.*]] = add i32 [[ARG]], [[OTHER]]
+; CHECK-NEXT: [[RFL:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[VAL]])
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %other = load i32, ptr %src
+ %val = add i32 %arg, %other
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
+
+define i32 @operand_is_arg(i32 %arg, i32 %other) {
+; CHECK-LABEL: define i32 @operand_is_arg(
+; CHECK-SAME: i32 [[ARG:%.*]], i32 [[OTHER:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = add i32 [[ARG]], [[OTHER]]
+; CHECK-NEXT: [[RFL:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[VAL]])
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = add i32 %arg, %other
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
+ ret i32 %rfl
+}
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readlane.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readlane.ll
new file mode 100644
index 0000000000000..6ac65f5c70337
--- /dev/null
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readlane.ll
@@ -0,0 +1,143 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -mtriple=amdgcn-- -mcpu=gfx1030 -passes=instcombine -S < %s | FileCheck %s
+
+; The readfirstlane version of this test covers all the interesting cases of the
+; shared logic. This testcase focuses on readlane specific pitfalls.
+
+; test unary
+
+define float @hoist_fneg_f32(float %arg, i32 %lane) {
+; CHECK-LABEL: define float @hoist_fneg_f32(
+; CHECK-SAME: float [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = fneg float [[ARG]]
+; CHECK-NEXT: [[RFL:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: ret float [[RFL]]
+;
+bb:
+ %val = fneg float %arg
+ %rfl = call float @llvm.amdgcn.readlane.f32(float %val, i32 %lane)
+ ret float %rfl
+}
+
+define double @hoist_fneg_f64(double %arg, i32 %lane) {
+; CHECK-LABEL: define double @hoist_fneg_f64(
+; CHECK-SAME: double [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = fneg double [[ARG]]
+; CHECK-NEXT: [[RFL:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: ret double [[RFL]]
+;
+bb:
+ %val = fneg double %arg
+ %rfl = call double @llvm.amdgcn.readlane.f64(double %val, i32 %lane)
+ ret double %rfl
+}
+
+; test binary i32
+
+define i32 @hoist_add_i32(i32 %arg, i32 %lane) {
+; CHECK-LABEL: define i32 @hoist_add_i32(
+; CHECK-SAME: i32 [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = add i32 [[ARG]], 16777215
+; CHECK-NEXT: [[RFL:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = add i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readlane.i32(i32 %val, i32 %lane)
+ ret i32 %rfl
+}
+
+define float @hoist_fadd_f32(float %arg, i32 %lane) {
+; CHECK-LABEL: define float @hoist_fadd_f32(
+; CHECK-SAME: float [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = fadd float [[ARG]], 1.280000e+02
+; CHECK-NEXT: [[RFL:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: ret float [[RFL]]
+;
+bb:
+ %val = fadd float %arg, 128.0
+ %rfl = call float @llvm.amdgcn.readlane.f32(float %val, i32 %lane)
+ ret float %rfl
+}
+
+; test binary i64
+
+define i64 @hoist_and_i64(i64 %arg, i32 %lane) {
+; CHECK-LABEL: define i64 @hoist_and_i64(
+; CHECK-SAME: i64 [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = and i64 [[ARG]], 16777215
+; CHECK-NEXT: [[RFL:%.*]] = call i64 @llvm.amdgcn.readlane.i64(i64 [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: ret i64 [[RFL]]
+;
+bb:
+ %val = and i64 %arg, 16777215
+ %rfl = call i64 @llvm.amdgcn.readlane.i32(i64 %val, i32 %lane)
+ ret i64 %rfl
+}
+
+define double @hoist_fadd_f64(double %arg, i32 %lane) {
+; CHECK-LABEL: define double @hoist_fadd_f64(
+; CHECK-SAME: double [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = fadd double [[ARG]], 1.280000e+02
+; CHECK-NEXT: [[RFL:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: ret double [[RFL]]
+;
+bb:
+ %val = fadd double %arg, 128.0
+ %rfl = call double @llvm.amdgcn.readlane.f64(double %val, i32 %lane)
+ ret double %rfl
+}
+
+; test constant on LHS
+
+define i32 @hoist_sub_i32_lhs(i32 %arg, i32 %lane) {
+; CHECK-LABEL: define i32 @hoist_sub_i32_lhs(
+; CHECK-SAME: i32 [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = sub i32 16777215, [[ARG]]
+; CHECK-NEXT: [[RFL:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: ret i32 [[RFL]]
+;
+bb:
+ %val = sub i32 16777215, %arg
+ %rfl = call i32 @llvm.amdgcn.readlane.i32(i32 %val, i32 %lane)
+ ret i32 %rfl
+}
+
+define float @hoist_fsub_f32_lhs(float %arg, i32 %lane) {
+; CHECK-LABEL: define float @hoist_fsub_f32_lhs(
+; CHECK-SAME: float [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = fsub float 1.280000e+02, [[ARG]]
+; CHECK-NEXT: [[RFL:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: ret float [[RFL]]
+;
+bb:
+ %val = fsub float 128.0, %arg
+ %rfl = call float @llvm.amdgcn.readlane.f32(float %val, i32 %lane)
+ ret float %rfl
+}
+
+; Check cases where we can't move the readlane higher
+
+define float @cannot_move_readlane(float %arg, i32 %base) {
+; CHECK-LABEL: define float @cannot_move_readlane(
+; CHECK-SAME: float [[ARG:%.*]], i32 [[BASE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*:]]
+; CHECK-NEXT: [[VAL:%.*]] = fsub float 1.280000e+02, [[ARG]]
+; CHECK-NEXT: [[LANE:%.*]] = add i32 [[BASE]], 2
+; CHECK-NEXT: [[RFL:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: ret float [[RFL]]
+;
+bb:
+ %val = fsub float 128.0, %arg
+ %lane = add i32 %base, 2
+ %rfl = call float @llvm.amdgcn.readlane.f32(float %val, i32 %lane)
+ ret float %rfl
+}
>From 8d661f7cd8525e953579c6a9053f742fd20797f4 Mon Sep 17 00:00:00 2001
From: pvanhout <pierre.vanhoutryve at amd.com>
Date: Thu, 27 Feb 2025 12:42:37 +0100
Subject: [PATCH 2/2] Fix comesBefore check
---
.../AMDGPU/AMDGPUInstCombineIntrinsic.cpp | 15 +++-
.../AMDGPU/llvm.amdgcn.readfirstlane.ll | 30 +++++++
.../AMDGPU/llvm.amdgcn.readlane.ll | 90 +++++++++++++++----
3 files changed, 115 insertions(+), 20 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
index c2a977d7168fe..92a479f8c5ff7 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUInstCombineIntrinsic.cpp
@@ -18,6 +18,7 @@
#include "AMDGPUTargetTransformInfo.h"
#include "GCNSubtarget.h"
#include "llvm/ADT/FloatingPointMode.h"
+#include "llvm/IR/Dominators.h"
#include "llvm/IR/IntrinsicsAMDGPU.h"
#include "llvm/Transforms/InstCombine/InstCombiner.h"
#include <optional>
@@ -498,8 +499,9 @@ Instruction *GCNTTIImpl::hoistReadLaneThroughOperand(InstCombiner &IC,
Value *LaneID = nullptr;
if (IsReadLane) {
LaneID = II.getOperand(1);
- if (!isa<Constant>(LaneID) && !(isa<Instruction>(LaneID) &&
- cast<Instruction>(LaneID)->comesBefore(Op)))
+ // Check LaneID is available at Op, otherwise we can't move the readlane
+ // higher.
+ if (!IC.getDominatorTree().dominates(LaneID, Op))
return nullptr;
}
@@ -508,8 +510,13 @@ Instruction *GCNTTIImpl::hoistReadLaneThroughOperand(InstCombiner &IC,
if (IsReadLane)
Ops.push_back(LaneID);
- Instruction *NewII =
- IC.Builder.CreateIntrinsic(II.getType(), II.getIntrinsicID(), Ops);
+ // Make sure convergence tokens are preserved.
+ // TODO: CreateIntrinsic should allow directly copying bundles
+ SmallVector<OperandBundleDef, 2> OpBundles;
+ II.getOperandBundlesAsDefs(OpBundles);
+
+ CallInst *NewII =
+ IC.Builder.CreateCall(II.getCalledFunction(), Ops, OpBundles);
Instruction &NewOp = *Op->clone();
NewOp.setOperand(OpIdx, NewII);
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readfirstlane.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readfirstlane.ll
index 9f27fda591382..bc750bea4beea 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readfirstlane.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readfirstlane.ll
@@ -459,3 +459,33 @@ bb:
%rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val)
ret i32 %rfl
}
+
+; test that convergence tokens are preserved
+
+define i32 @hoist_preserves_convergence_token(i1 %cond, i32 %arg) convergent {
+; CHECK-LABEL: define i32 @hoist_preserves_convergence_token(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[ARG:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[BB:.*]]:
+; CHECK-NEXT: [[ENTRY:%.*]] = call token @llvm.experimental.convergence.entry()
+; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[END:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readfirstlane.i32(i32 [[ARG]]) [ "convergencectrl"(token [[ENTRY]]) ]
+; CHECK-NEXT: [[RFL:%.*]] = add i32 [[TMP0]], 16777215
+; CHECK-NEXT: br label %[[END]]
+; CHECK: [[END]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RFL]], %[[THEN]] ], [ [[ARG]], %[[BB]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+bb:
+ %entry = call token @llvm.experimental.convergence.entry()
+ br i1 %cond, label %then, label %end
+
+then:
+ %val = add i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readfirstlane.i32(i32 %val) [ "convergencectrl"(token %entry)]
+ br label %end
+
+end:
+ %res = phi i32 [%rfl, %then], [%arg, %bb]
+ ret i32 %res
+}
diff --git a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readlane.ll b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readlane.ll
index 6ac65f5c70337..ffd0327209ae0 100644
--- a/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readlane.ll
+++ b/llvm/test/Transforms/InstCombine/AMDGPU/llvm.amdgcn.readlane.ll
@@ -10,8 +10,8 @@ define float @hoist_fneg_f32(float %arg, i32 %lane) {
; CHECK-LABEL: define float @hoist_fneg_f32(
; CHECK-SAME: float [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[BB:.*:]]
-; CHECK-NEXT: [[VAL:%.*]] = fneg float [[ARG]]
-; CHECK-NEXT: [[RFL:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: [[TMP0:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[ARG]], i32 [[LANE]])
+; CHECK-NEXT: [[RFL:%.*]] = fneg float [[TMP0]]
; CHECK-NEXT: ret float [[RFL]]
;
bb:
@@ -24,8 +24,8 @@ define double @hoist_fneg_f64(double %arg, i32 %lane) {
; CHECK-LABEL: define double @hoist_fneg_f64(
; CHECK-SAME: double [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[BB:.*:]]
-; CHECK-NEXT: [[VAL:%.*]] = fneg double [[ARG]]
-; CHECK-NEXT: [[RFL:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: [[TMP0:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[ARG]], i32 [[LANE]])
+; CHECK-NEXT: [[RFL:%.*]] = fneg double [[TMP0]]
; CHECK-NEXT: ret double [[RFL]]
;
bb:
@@ -40,8 +40,8 @@ define i32 @hoist_add_i32(i32 %arg, i32 %lane) {
; CHECK-LABEL: define i32 @hoist_add_i32(
; CHECK-SAME: i32 [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[BB:.*:]]
-; CHECK-NEXT: [[VAL:%.*]] = add i32 [[ARG]], 16777215
-; CHECK-NEXT: [[RFL:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 [[LANE]])
+; CHECK-NEXT: [[RFL:%.*]] = add i32 [[TMP0]], 16777215
; CHECK-NEXT: ret i32 [[RFL]]
;
bb:
@@ -54,8 +54,8 @@ define float @hoist_fadd_f32(float %arg, i32 %lane) {
; CHECK-LABEL: define float @hoist_fadd_f32(
; CHECK-SAME: float [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[BB:.*:]]
-; CHECK-NEXT: [[VAL:%.*]] = fadd float [[ARG]], 1.280000e+02
-; CHECK-NEXT: [[RFL:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: [[TMP0:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[ARG]], i32 [[LANE]])
+; CHECK-NEXT: [[RFL:%.*]] = fadd float [[TMP0]], 1.280000e+02
; CHECK-NEXT: ret float [[RFL]]
;
bb:
@@ -70,8 +70,8 @@ define i64 @hoist_and_i64(i64 %arg, i32 %lane) {
; CHECK-LABEL: define i64 @hoist_and_i64(
; CHECK-SAME: i64 [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[BB:.*:]]
-; CHECK-NEXT: [[VAL:%.*]] = and i64 [[ARG]], 16777215
-; CHECK-NEXT: [[RFL:%.*]] = call i64 @llvm.amdgcn.readlane.i64(i64 [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: [[TMP0:%.*]] = call i64 @llvm.amdgcn.readlane.i64(i64 [[ARG]], i32 [[LANE]])
+; CHECK-NEXT: [[RFL:%.*]] = and i64 [[TMP0]], 16777215
; CHECK-NEXT: ret i64 [[RFL]]
;
bb:
@@ -84,8 +84,8 @@ define double @hoist_fadd_f64(double %arg, i32 %lane) {
; CHECK-LABEL: define double @hoist_fadd_f64(
; CHECK-SAME: double [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[BB:.*:]]
-; CHECK-NEXT: [[VAL:%.*]] = fadd double [[ARG]], 1.280000e+02
-; CHECK-NEXT: [[RFL:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: [[TMP0:%.*]] = call double @llvm.amdgcn.readlane.f64(double [[ARG]], i32 [[LANE]])
+; CHECK-NEXT: [[RFL:%.*]] = fadd double [[TMP0]], 1.280000e+02
; CHECK-NEXT: ret double [[RFL]]
;
bb:
@@ -100,8 +100,8 @@ define i32 @hoist_sub_i32_lhs(i32 %arg, i32 %lane) {
; CHECK-LABEL: define i32 @hoist_sub_i32_lhs(
; CHECK-SAME: i32 [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[BB:.*:]]
-; CHECK-NEXT: [[VAL:%.*]] = sub i32 16777215, [[ARG]]
-; CHECK-NEXT: [[RFL:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 [[LANE]])
+; CHECK-NEXT: [[RFL:%.*]] = sub i32 16777215, [[TMP0]]
; CHECK-NEXT: ret i32 [[RFL]]
;
bb:
@@ -114,8 +114,8 @@ define float @hoist_fsub_f32_lhs(float %arg, i32 %lane) {
; CHECK-LABEL: define float @hoist_fsub_f32_lhs(
; CHECK-SAME: float [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[BB:.*:]]
-; CHECK-NEXT: [[VAL:%.*]] = fsub float 1.280000e+02, [[ARG]]
-; CHECK-NEXT: [[RFL:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[VAL]], i32 [[LANE]])
+; CHECK-NEXT: [[TMP0:%.*]] = call float @llvm.amdgcn.readlane.f32(float [[ARG]], i32 [[LANE]])
+; CHECK-NEXT: [[RFL:%.*]] = fsub float 1.280000e+02, [[TMP0]]
; CHECK-NEXT: ret float [[RFL]]
;
bb:
@@ -141,3 +141,61 @@ bb:
%rfl = call float @llvm.amdgcn.readlane.f32(float %val, i32 %lane)
ret float %rfl
}
+
+define i32 @readlane_lane_op_in_other_block(i1 %cond, i32 %arg, i32 %base) {
+; CHECK-LABEL: define i32 @readlane_lane_op_in_other_block(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[ARG:%.*]], i32 [[BASE:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[BB:.*]]:
+; CHECK-NEXT: [[LANE:%.*]] = add i32 [[BASE]], 2
+; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[END:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 [[LANE]])
+; CHECK-NEXT: [[RFL:%.*]] = add i32 [[TMP0]], 16777215
+; CHECK-NEXT: br label %[[END]]
+; CHECK: [[END]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RFL]], %[[THEN]] ], [ [[LANE]], %[[BB]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+bb:
+ %lane = add i32 %base, 2
+ br i1 %cond, label %then, label %end
+
+then:
+ %val = add i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readlane.i32(i32 %val, i32 %lane)
+ br label %end
+
+end:
+ %res = phi i32 [%rfl, %then], [%lane, %bb]
+ ret i32 %res
+}
+
+; test that convergence tokens are preserved
+
+define i32 @hoist_preserves_convergence_token(i1 %cond, i32 %arg, i32 %lane) convergent {
+; CHECK-LABEL: define i32 @hoist_preserves_convergence_token(
+; CHECK-SAME: i1 [[COND:%.*]], i32 [[ARG:%.*]], i32 [[LANE:%.*]]) #[[ATTR1:[0-9]+]] {
+; CHECK-NEXT: [[BB:.*]]:
+; CHECK-NEXT: [[ENTRY:%.*]] = call token @llvm.experimental.convergence.entry()
+; CHECK-NEXT: br i1 [[COND]], label %[[THEN:.*]], label %[[END:.*]]
+; CHECK: [[THEN]]:
+; CHECK-NEXT: [[TMP0:%.*]] = call i32 @llvm.amdgcn.readlane.i32(i32 [[ARG]], i32 [[LANE]]) [ "convergencectrl"(token [[ENTRY]]) ]
+; CHECK-NEXT: [[RFL:%.*]] = add i32 [[TMP0]], 16777215
+; CHECK-NEXT: br label %[[END]]
+; CHECK: [[END]]:
+; CHECK-NEXT: [[RES:%.*]] = phi i32 [ [[RFL]], %[[THEN]] ], [ [[ARG]], %[[BB]] ]
+; CHECK-NEXT: ret i32 [[RES]]
+;
+bb:
+ %entry = call token @llvm.experimental.convergence.entry()
+ br i1 %cond, label %then, label %end
+
+then:
+ %val = add i32 %arg, 16777215
+ %rfl = call i32 @llvm.amdgcn.readlane.i32(i32 %val, i32 %lane) [ "convergencectrl"(token %entry)]
+ br label %end
+
+end:
+ %res = phi i32 [%rfl, %then], [%arg, %bb]
+ ret i32 %res
+}
More information about the llvm-commits
mailing list