[llvm] 08c2f4e - [CVP] When expanding `urem`, always freeze the nominator
Roman Lebedev via llvm-commits
llvm-commits at lists.llvm.org
Fri Dec 30 18:01:42 PST 2022
Author: Roman Lebedev
Date: 2022-12-31T05:00:43+03:00
New Revision: 08c2f4eb7ab3e75eb9d2048f73ea9874c663fb5a
URL: https://github.com/llvm/llvm-project/commit/08c2f4eb7ab3e75eb9d2048f73ea9874c663fb5a
DIFF: https://github.com/llvm/llvm-project/commit/08c2f4eb7ab3e75eb9d2048f73ea9874c663fb5a.diff
LOG: [CVP] When expanding `urem`, always freeze the nominator
As per the post-commit feedback - that was not the correct precondition
to avoid it here. I think we should generally start changing mentality
about `freeze`, the fact that we have been conditioned to be afraid of it
(or of anything in LLVM in general) is the key problem here.
Added:
Modified:
llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
llvm/test/Transforms/CorrelatedValuePropagation/urem.ll
llvm/test/Transforms/PhaseOrdering/cmp-logic.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
index 1093f4e23254c..fdebd6fff03c4 100644
--- a/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
+++ b/llvm/lib/Transforms/Scalar/CorrelatedValuePropagation.cpp
@@ -801,21 +801,16 @@ static bool expandURem(BinaryOperator *Instr, LazyValueInfo *LVI) {
// R = X* % Y iff X* u< 2*Y (w/ unsigned saturation)
// but that does not seem profitable here.
- bool XIsBound =
- XCR.icmp(ICmpInst::ICMP_ULT, YCR.umul_sat(APInt(YCR.getBitWidth(), 2)));
// Even if we don't know X's range, the divisor may be so large, X can't ever
// be 2x larger than that. I.e. if divisor is always negative.
- if (!XIsBound && !YCR.isAllNegative())
+ if (!XCR.icmp(ICmpInst::ICMP_ULT,
+ YCR.umul_sat(APInt(YCR.getBitWidth(), 2))) &&
+ !YCR.isAllNegative())
return false;
-
IRBuilder<> B{Instr};
- if (!XIsBound) {
- // NOTE: this transformation increases use count on X, but that is fine
- // unless the transformation is valid because the divisor is negative,
- // and is non-variable, and thus we didn't have any extra uses.
- if (auto *Ycst = dyn_cast<ConstantInt>(Y); Ycst && Ycst->isNegative())
- X = B.CreateFreeze(X, X->getName() + ".frozen");
- }
+ // NOTE: this transformation introduces two uses of X,
+ // but it may be undef so we must freeze it first.
+ X = B.CreateFreeze(X, X->getName() + ".frozen");
auto *AdjX = B.CreateNUWSub(X, Y, Instr->getName() + ".urem");
auto *Cmp = B.CreateICmp(ICmpInst::ICMP_ULT, X, Y, Instr->getName() + ".cmp");
auto *ExpandedURem = B.CreateSelect(Cmp, X, AdjX);
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
index 36d9771392c0c..620cd5e0651ca 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/urem-expansion.ll
@@ -20,9 +20,10 @@ define i8 @constant.divisor.v4(i8 %x) {
; CHECK-LABEL: @constant.divisor.v4(
; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X:%.*]], 4
; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]])
-; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X]], 3
-; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X]], 3
-; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], 3
+; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], 3
+; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
; CHECK-NEXT: ret i8 [[REM]]
;
%cmp.x.upper = icmp ult i8 %x, 4
@@ -33,9 +34,10 @@ define i8 @constant.divisor.v4(i8 %x) {
define i8 @constant.divisor.x.range.v4(ptr %x.ptr) {
; CHECK-LABEL: @constant.divisor.x.range.v4(
; CHECK-NEXT: [[X:%.*]] = load i8, ptr [[X_PTR:%.*]], align 1, !range [[RNG0:![0-9]+]]
-; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X]], 3
-; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X]], 3
-; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], 3
+; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], 3
+; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
; CHECK-NEXT: ret i8 [[REM]]
;
%x = load i8, ptr %x.ptr, !range !{ i8 0, i8 4 }
@@ -45,9 +47,10 @@ define i8 @constant.divisor.x.range.v4(ptr %x.ptr) {
define i8 @constant.divisor.x.mask.v4(i8 %x) {
; CHECK-LABEL: @constant.divisor.x.mask.v4(
; CHECK-NEXT: [[X_MASKED:%.*]] = and i8 [[X:%.*]], 3
-; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_MASKED]], 3
-; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_MASKED]], 3
-; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_MASKED]], i8 [[REM_UREM]]
+; CHECK-NEXT: [[X_MASKED_FROZEN:%.*]] = freeze i8 [[X_MASKED]]
+; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_MASKED_FROZEN]], 3
+; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_MASKED_FROZEN]], 3
+; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_MASKED_FROZEN]], i8 [[REM_UREM]]
; CHECK-NEXT: ret i8 [[REM]]
;
%x.masked = and i8 %x, 3
@@ -58,9 +61,10 @@ define i8 @constant.divisor.v5(i8 %x) {
; CHECK-LABEL: @constant.divisor.v5(
; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X:%.*]], 5
; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]])
-; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X]], 3
-; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X]], 3
-; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], 3
+; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], 3
+; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
; CHECK-NEXT: ret i8 [[REM]]
;
%cmp.x.upper = icmp ult i8 %x, 5
@@ -72,9 +76,10 @@ define i8 @constant.divisor.v6(i8 %x) {
; CHECK-LABEL: @constant.divisor.v6(
; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X:%.*]], 6
; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]])
-; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X]], 3
-; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X]], 3
-; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], 3
+; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], 3
+; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
; CHECK-NEXT: ret i8 [[REM]]
;
%cmp.x.upper = icmp ult i8 %x, 6
@@ -124,9 +129,10 @@ define i8 @variable.v4(i8 %x, i8 %y) {
; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_Y_LOWER]])
; CHECK-NEXT: [[CMP_Y_UPPER:%.*]] = icmp ule i8 [[Y]], 4
; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_Y_UPPER]])
-; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X]], [[Y]]
-; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X]], [[Y]]
-; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], [[Y]]
+; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], [[Y]]
+; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
; CHECK-NEXT: ret i8 [[REM]]
;
%cmp.x = icmp ult i8 %x, 4
@@ -142,9 +148,10 @@ define i8 @variable.v4.range(ptr %x.ptr, ptr %y.ptr) {
; CHECK-LABEL: @variable.v4.range(
; CHECK-NEXT: [[X:%.*]] = load i8, ptr [[X_PTR:%.*]], align 1, !range [[RNG0]]
; CHECK-NEXT: [[Y:%.*]] = load i8, ptr [[Y_PTR:%.*]], align 1, !range [[RNG1:![0-9]+]]
-; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X]], [[Y]]
-; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X]], [[Y]]
-; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], [[Y]]
+; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], [[Y]]
+; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
; CHECK-NEXT: ret i8 [[REM]]
;
%x = load i8, ptr %x.ptr, !range !{ i8 0, i8 4 }
@@ -160,9 +167,10 @@ define i8 @variable.v5(i8 %x, i8 %y) {
; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_Y_LOWER]])
; CHECK-NEXT: [[CMP_Y_UPPER:%.*]] = icmp ule i8 [[Y]], 4
; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_Y_UPPER]])
-; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X]], [[Y]]
-; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X]], [[Y]]
-; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], [[Y]]
+; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], [[Y]]
+; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
; CHECK-NEXT: ret i8 [[REM]]
;
%cmp.x = icmp ult i8 %x, 5
@@ -182,9 +190,10 @@ define i8 @variable.v6(i8 %x, i8 %y) {
; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_Y_LOWER]])
; CHECK-NEXT: [[CMP_Y_UPPER:%.*]] = icmp ule i8 [[Y]], 4
; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_Y_UPPER]])
-; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X]], [[Y]]
-; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X]], [[Y]]
-; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], [[Y]]
+; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], [[Y]]
+; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
; CHECK-NEXT: ret i8 [[REM]]
;
%cmp.x = icmp ult i8 %x, 6
@@ -234,9 +243,10 @@ define i8 @large.divisor.v1(i8 %x) {
; CHECK-LABEL: @large.divisor.v1(
; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X:%.*]], -128
; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]])
-; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X]], 127
-; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X]], 127
-; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], 127
+; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], 127
+; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
; CHECK-NEXT: ret i8 [[REM]]
;
%cmp.x.upper = icmp ult i8 %x, 128
@@ -247,9 +257,10 @@ define i8 @large.divisor.v1(i8 %x) {
define i8 @large.divisor.v1.range(ptr %x.ptr) {
; CHECK-LABEL: @large.divisor.v1.range(
; CHECK-NEXT: [[X:%.*]] = load i8, ptr [[X_PTR:%.*]], align 1, !range [[RNG2:![0-9]+]]
-; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X]], 127
-; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X]], 127
-; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], 127
+; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], 127
+; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
; CHECK-NEXT: ret i8 [[REM]]
;
%x = load i8, ptr %x.ptr, !range !{ i8 0, i8 128 }
@@ -280,9 +291,10 @@ define i8 @large.divisor.with.overflow.v1(i8 %x) {
; CHECK-LABEL: @large.divisor.with.overflow.v1(
; CHECK-NEXT: [[CMP_X_UPPER:%.*]] = icmp ult i8 [[X:%.*]], -127
; CHECK-NEXT: call void @llvm.assume(i1 [[CMP_X_UPPER]])
-; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X]], -128
-; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X]], -128
-; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], -128
+; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], -128
+; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
; CHECK-NEXT: ret i8 [[REM]]
;
%cmp.x.upper = icmp ult i8 %x, 129
@@ -293,9 +305,10 @@ define i8 @large.divisor.with.overflow.v1(i8 %x) {
define i8 @large.divisor.with.overflow.v1.range(ptr %x.ptr) {
; CHECK-LABEL: @large.divisor.with.overflow.v1.range(
; CHECK-NEXT: [[X:%.*]] = load i8, ptr [[X_PTR:%.*]], align 1, !range [[RNG3:![0-9]+]]
-; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X]], -128
-; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X]], -128
-; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X]], i8 [[REM_UREM]]
+; CHECK-NEXT: [[X_FROZEN:%.*]] = freeze i8 [[X]]
+; CHECK-NEXT: [[REM_UREM:%.*]] = sub nuw i8 [[X_FROZEN]], -128
+; CHECK-NEXT: [[REM_CMP:%.*]] = icmp ult i8 [[X_FROZEN]], -128
+; CHECK-NEXT: [[REM:%.*]] = select i1 [[REM_CMP]], i8 [[X_FROZEN]], i8 [[REM_UREM]]
; CHECK-NEXT: ret i8 [[REM]]
;
%x = load i8, ptr %x.ptr, !range !{ i8 0, i8 129 }
diff --git a/llvm/test/Transforms/CorrelatedValuePropagation/urem.ll b/llvm/test/Transforms/CorrelatedValuePropagation/urem.ll
index 19dfc681c9111..b9c112c69e1a6 100644
--- a/llvm/test/Transforms/CorrelatedValuePropagation/urem.ll
+++ b/llvm/test/Transforms/CorrelatedValuePropagation/urem.ll
@@ -117,9 +117,10 @@ exit:
define void @test5(i32 %n) {
; CHECK-LABEL: @test5(
; CHECK-NEXT: [[TRUNC:%.*]] = and i32 [[N:%.*]], 63
-; CHECK-NEXT: [[DIV_UREM:%.*]] = sub nuw i32 [[TRUNC]], 42
-; CHECK-NEXT: [[DIV_CMP:%.*]] = icmp ult i32 [[TRUNC]], 42
-; CHECK-NEXT: [[DIV:%.*]] = select i1 [[DIV_CMP]], i32 [[TRUNC]], i32 [[DIV_UREM]]
+; CHECK-NEXT: [[TRUNC_FROZEN:%.*]] = freeze i32 [[TRUNC]]
+; CHECK-NEXT: [[DIV_UREM:%.*]] = sub nuw i32 [[TRUNC_FROZEN]], 42
+; CHECK-NEXT: [[DIV_CMP:%.*]] = icmp ult i32 [[TRUNC_FROZEN]], 42
+; CHECK-NEXT: [[DIV:%.*]] = select i1 [[DIV_CMP]], i32 [[TRUNC_FROZEN]], i32 [[DIV_UREM]]
; CHECK-NEXT: ret void
;
%trunc = and i32 %n, 63
diff --git a/llvm/test/Transforms/PhaseOrdering/cmp-logic.ll b/llvm/test/Transforms/PhaseOrdering/cmp-logic.ll
index 008e88e8dbe52..04eae7d2941d8 100644
--- a/llvm/test/Transforms/PhaseOrdering/cmp-logic.ll
+++ b/llvm/test/Transforms/PhaseOrdering/cmp-logic.ll
@@ -123,7 +123,8 @@ define i32 @PR56119(i32 %e.coerce) {
;
; OZ-LABEL: @PR56119(
; OZ-NEXT: entry:
-; OZ-NEXT: [[CONV2:%.*]] = and i32 [[E_COERCE:%.*]], 255
+; OZ-NEXT: [[E_COERCE_FR:%.*]] = freeze i32 [[E_COERCE:%.*]]
+; OZ-NEXT: [[CONV2:%.*]] = and i32 [[E_COERCE_FR]], 255
; OZ-NEXT: [[CMP1:%.*]] = icmp eq i32 [[CONV2]], 7
; OZ-NEXT: br i1 [[CMP1]], label [[IF_THEN:%.*]], label [[IF_END:%.*]]
; OZ: if.then:
More information about the llvm-commits
mailing list