[polly] ceee3c3 - [Polly] Restate array sizes and subscripts when the element type shrinks (#221496)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 24 04:59:28 PDT 2026
Author: Timur Baidusenov
Date: 2026-09-24T13:59:23+02:00
New Revision: ceee3c39e1be9692557a22aedc7b87ae88ef5811
URL: https://github.com/llvm/llvm-project/commit/ceee3c39e1be9692557a22aedc7b87ae88ef5811
DIFF: https://github.com/llvm/llvm-project/commit/ceee3c39e1be9692557a22aedc7b87ae88ef5811.diff
LOG: [Polly] Restate array sizes and subscripts when the element type shrinks (#221496)
Polly uses the smallest type to an array with element type of its
ScopArrayInfo. So a byte-wise access, such as a memset, can make the
element type smaller after the sizes of the array dimensions have
already been stored. Those sizes were not updated when the element type
changed: a row of a two-dimensional array of double kept the size. Every
access to the array then looks like it goes past the end of its row, the
inbounds assumption becomes impossible to satisfy, and the SCoP is
dropped without any message. Finally, this patch multiplies the size of
the innermost dimension.
Assisted-by: Claude (Anthropic)
Added:
polly/test/ScopInfo/multidim-shrunk-element-type.ll
Modified:
polly/lib/Analysis/ScopInfo.cpp
Removed:
################################################################################
diff --git a/polly/lib/Analysis/ScopInfo.cpp b/polly/lib/Analysis/ScopInfo.cpp
index e47d8602547c8b..166740b57407ad 100644
--- a/polly/lib/Analysis/ScopInfo.cpp
+++ b/polly/lib/Analysis/ScopInfo.cpp
@@ -280,6 +280,22 @@ bool ScopArrayInfo::isCompatibleWith(const ScopArrayInfo *Array) const {
return true;
}
+/// Multiply the innermost of @p Sizes by @p Factor.
+///
+/// Dimension sizes count elements of an array's canonical element type, so a
+/// row holds @p Factor times as many of them once that type becomes @p Factor
+/// times smaller. Only the innermost size changes: the outer ones count rows,
+/// and a row grows together with the innermost dimension.
+static void stretchInnermostSize(SmallVectorImpl<const SCEV *> &Sizes,
+ uint64_t Factor, ScalarEvolution &SE) {
+ if (Factor == 1 || Sizes.empty() || !Sizes.back())
+ return;
+
+ const SCEV *Innermost = Sizes.back();
+ Sizes.back() =
+ SE.getMulExpr(Innermost, SE.getConstant(Innermost->getType(), Factor));
+}
+
void ScopArrayInfo::updateElementType(Type *NewElementType) {
if (NewElementType == ElementType)
return;
@@ -290,12 +306,38 @@ void ScopArrayInfo::updateElementType(Type *NewElementType) {
if (NewElementSize == OldElementSize || NewElementSize == 0)
return;
+ Type *CanonicalType;
if (NewElementSize % OldElementSize == 0 && NewElementSize < OldElementSize) {
- ElementType = NewElementType;
+ CanonicalType = NewElementType;
} else {
auto GCD = std::gcd((uint64_t)NewElementSize, (uint64_t)OldElementSize);
- ElementType = IntegerType::get(ElementType->getContext(), GCD);
+ CanonicalType = IntegerType::get(ElementType->getContext(), GCD);
}
+
+ // The sizes on record count elements of the type being replaced, so they no
+ // longer describe the same memory once it changes. Restate them in the new
+ // element. Leaving them alone would shrink every row along with the element
+ // type and model in-bounds accesses as running past its end, which makes the
+ // inbounds assumption infeasible and drops the SCoP.
+ //
+ // The canonical type is an integer of the greatest common divisor of two
+ // sizes, and rounding that up to its allocation size can leave it not
+ // dividing the type it replaces. There is then no whole number of new
+ // elements per old one to restate the sizes in. Give up before touching
+ // anything rather than leave the element type and the sizes disagreeing.
+ uint64_t CanonicalSize = DL.getTypeAllocSizeInBits(CanonicalType);
+ if (CanonicalSize == 0 || (uint64_t)OldElementSize % CanonicalSize != 0)
+ return;
+
+ ElementType = CanonicalType;
+
+ uint64_t Factor = (uint64_t)OldElementSize / CanonicalSize;
+ if (Factor == 1)
+ return;
+
+ SmallVector<const SCEV *, 4> Stretched(DimensionSizes);
+ stretchInnermostSize(Stretched, Factor, *S.getSE());
+ updateSizes(Stretched, false /* CheckConsistency */);
}
bool ScopArrayInfo::updateSizes(ArrayRef<const SCEV *> NewSizes,
@@ -494,7 +536,14 @@ void MemoryAccess::updateDimensionality() {
// access is larger than the canonical element type of the array.
//
// An access ((float *)A)[i] to an array char *A is modeled as
- // {[i] -> A[o] : 4 i <= o <= 4 i + 3
+ // {[i] -> A[o] : 4 i <= o <= 4 i + 3}
+ //
+ // The subscript of a non-delinearized access was divided by ArrayElemSize
+ // above, which already stated it in canonical elements. A delinearized one
+ // still counts elements of the type it reads or writes, so it is scaled
+ // here instead. Only the innermost subscript is scaled: the outer ones count
+ // rows and are already stated in the sizes the access was delinearized
+ // against.
if (ElemBytes > ArrayElemSize) {
assert(ElemBytes % ArrayElemSize == 0 &&
"Loaded element size should be multiple of canonical element size");
@@ -509,15 +558,18 @@ void MemoryAccess::updateDimensionality() {
LS = isl::local_space(Map.get_space());
int Num = ElemBytes / getScopArrayInfo()->getElemSizeInBytes();
+ int Scale = DimsAccess == 1 ? 1 : Num;
+ // Scale * i - o + (Num - 1) >= 0, that is o <= Scale * i + Num - 1.
C = isl::constraint::alloc_inequality(LS);
C = C.set_constant_val(isl::val(Ctx, Num - 1));
- C = C.set_coefficient_si(isl::dim::in, DimsArray - 1, 1);
+ C = C.set_coefficient_si(isl::dim::in, DimsArray - 1, Scale);
C = C.set_coefficient_si(isl::dim::out, DimsArray - 1, -1);
Map = Map.add_constraint(C);
+ // o - Scale * i >= 0, that is o >= Scale * i.
C = isl::constraint::alloc_inequality(LS);
- C = C.set_coefficient_si(isl::dim::in, DimsArray - 1, -1);
+ C = C.set_coefficient_si(isl::dim::in, DimsArray - 1, -Scale);
C = C.set_coefficient_si(isl::dim::out, DimsArray - 1, 1);
C = C.set_constant_val(isl::val(Ctx, 0));
Map = Map.add_constraint(C);
@@ -1759,9 +1811,23 @@ ScopArrayInfo *Scop::getOrCreateScopArrayInfo(Value *BasePtr, Type *ElementType,
ScopArrayInfoSet.insert(SAI.get());
} else {
SAI->updateElementType(ElementType);
+
+ // The sizes handed in count elements of ElementType, which is larger than
+ // the canonical element type of the array whenever some other access to it
+ // uses a smaller one. Restate them in the canonical element, so that they
+ // are compared against, and stored next to, sizes in the same unit.
+ auto &DL = getFunction().getParent()->getDataLayout();
+ uint64_t AccessElemSize = DL.getTypeAllocSize(ElementType);
+ uint64_t CanonicalElemSize = SAI->getElemSizeInBytes();
+
+ SmallVector<const SCEV *, 4> CanonicalSizes(Sizes);
+ if (CanonicalElemSize != 0 && AccessElemSize % CanonicalElemSize == 0)
+ stretchInnermostSize(CanonicalSizes, AccessElemSize / CanonicalElemSize,
+ *getSE());
+
// In case of mismatching array sizes, we bail out by setting the run-time
// context to false.
- if (!SAI->updateSizes(Sizes))
+ if (!SAI->updateSizes(CanonicalSizes))
invalidate(DELINEARIZATION, DebugLoc());
}
return SAI.get();
diff --git a/polly/test/ScopInfo/multidim-shrunk-element-type.ll b/polly/test/ScopInfo/multidim-shrunk-element-type.ll
new file mode 100644
index 00000000000000..667223e53ab78d
--- /dev/null
+++ b/polly/test/ScopInfo/multidim-shrunk-element-type.ll
@@ -0,0 +1,208 @@
+; RUN: opt %loadNPMPolly -polly-only-func=shrink_after_sizes \
+; RUN: '-passes=polly-custom<scops>' -polly-print-scops -disable-output \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=AFTER
+; RUN: opt %loadNPMPolly -polly-only-func=sizes_after_shrink \
+; RUN: '-passes=polly-custom<scops>' -polly-print-scops -disable-output \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=BEFORE
+; RUN: opt %loadNPMPolly -polly-only-func=three_dimensions \
+; RUN: '-passes=polly-custom<scops>' -polly-print-scops -disable-output \
+; RUN: < %s 2>&1 | FileCheck %s --check-prefix=THREE
+
+target datalayout = "e-p:64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:64:64-f32:32:32-f64:64:64-v64:64:64-v128:128:128-a0:0:64-s0:64:64-f80:128:128-n8:16:32:64-S128"
+
+; A memset shrinks the canonical element type of A from double to i8. Dimension
+; sizes and subscripts both count canonical elements, so the innermost of each
+; has to grow by the same factor. Left in double units, a row would hold m
+; bytes rather than 8m, the stores would be modelled as running past its end,
+; and the resulting inbounds assumption would be infeasible and drop the SCoP.
+
+; The memset follows the loop nest, so the sizes are recorded while double is
+; still the canonical element type and have to be restated when it shrinks.
+;
+; void shrink_after_sizes(long n, long m, double A[n][m]) {
+; for (long i = 0; i < 100; i++)
+; for (long j = 0; j < 150; j++)
+; A[i][j] = 1.0;
+; memset(A, 0, m * sizeof(double));
+; }
+
+; AFTER: Assumed Context:
+; AFTER-NEXT: [m] -> { : m >= 150 }
+
+; AFTER: Arrays {
+; AFTER-NEXT: i8 MemRef_A[*][(8 * %m)]; // Element size 1
+; AFTER-NEXT: }
+
+; AFTER: Statements {
+; AFTER-NEXT: Stmt_for_j
+; AFTER: MustWriteAccess := [Reduction Type: NONE] [Scalar: 0]
+; AFTER-NEXT: [m] -> { Stmt_for_j[i0, i1] -> MemRef_A[i0, o1] : 8i1 <= o1 <= 7 + 8i1 };
+; AFTER-NEXT: Stmt_memset_bb
+; AFTER: MustWriteAccess := [Reduction Type: NONE] [Scalar: 0]
+; AFTER-NEXT: [m] -> { Stmt_memset_bb[] -> MemRef_A[0, o1] : 0 <= o1 < 8m };
+; AFTER-NEXT: }
+
+define void @shrink_after_sizes(i64 %n, i64 %m, ptr %A) {
+entry:
+ %len = shl nsw i64 %m, 3
+ br label %for.i
+
+for.i:
+ %i = phi i64 [ 0, %entry ], [ %i.inc, %for.i.inc ]
+ %tmp = mul nsw i64 %i, %m
+ br label %for.j
+
+for.j:
+ %j = phi i64 [ 0, %for.i ], [ %j.inc, %for.j ]
+ %vlaarrayidx.sum = add i64 %j, %tmp
+ %arrayidx = getelementptr inbounds double, ptr %A, i64 %vlaarrayidx.sum
+ store double 1.0, ptr %arrayidx
+ %j.inc = add nsw i64 %j, 1
+ %j.exitcond = icmp eq i64 %j.inc, 150
+ br i1 %j.exitcond, label %for.i.inc, label %for.j
+
+for.i.inc:
+ %i.inc = add nsw i64 %i, 1
+ %i.exitcond = icmp eq i64 %i.inc, 100
+ br i1 %i.exitcond, label %memset.bb, label %for.i
+
+memset.bb:
+ call void @llvm.memset.p0.i64(ptr %A, i8 0, i64 %len, i1 false)
+ br label %end
+
+end:
+ ret void
+}
+
+; The memset comes first, so the array is created with i8 as its element type
+; and the sizes of the loop nest arrive afterwards, stated in doubles. They
+; have to be restated before they are compared against the ones on record.
+;
+; void sizes_after_shrink(long n, long m, double A[n][m]) {
+; for (long i = 0; i < 100; i++) {
+; memset(A, 0, m * sizeof(double));
+; for (long j = 0; j < 150; j++)
+; A[i][j] = 1.0;
+; }
+; }
+
+; BEFORE: Assumed Context:
+; BEFORE-NEXT: [m] -> { : m >= 150 }
+
+; BEFORE: Arrays {
+; BEFORE-NEXT: i8 MemRef_A[*][(8 * %m)]; // Element size 1
+; BEFORE-NEXT: }
+
+; BEFORE: Statements {
+; BEFORE-NEXT: Stmt_memset_bb
+; BEFORE: MustWriteAccess := [Reduction Type: NONE] [Scalar: 0]
+; BEFORE-NEXT: [m] -> { Stmt_memset_bb[i0] -> MemRef_A[0, o1] : 0 <= o1 < 8m };
+; BEFORE-NEXT: Stmt_for_j
+; BEFORE: MustWriteAccess := [Reduction Type: NONE] [Scalar: 0]
+; BEFORE-NEXT: [m] -> { Stmt_for_j[i0, i1] -> MemRef_A[i0, o1] : 8i1 <= o1 <= 7 + 8i1 };
+; BEFORE-NEXT: }
+
+define void @sizes_after_shrink(i64 %n, i64 %m, ptr %A) {
+entry:
+ %len = shl nsw i64 %m, 3
+ br label %for.i
+
+for.i:
+ %i = phi i64 [ 0, %entry ], [ %i.inc, %for.i.inc ]
+ %tmp = mul nsw i64 %i, %m
+ br label %memset.bb
+
+memset.bb:
+ call void @llvm.memset.p0.i64(ptr %A, i8 0, i64 %len, i1 false)
+ br label %for.j
+
+for.j:
+ %j = phi i64 [ 0, %memset.bb ], [ %j.inc, %for.j ]
+ %vlaarrayidx.sum = add i64 %j, %tmp
+ %arrayidx = getelementptr inbounds double, ptr %A, i64 %vlaarrayidx.sum
+ store double 1.0, ptr %arrayidx
+ %j.inc = add nsw i64 %j, 1
+ %j.exitcond = icmp eq i64 %j.inc, 150
+ br i1 %j.exitcond, label %for.i.inc, label %for.j
+
+for.i.inc:
+ %i.inc = add nsw i64 %i, 1
+ %i.exitcond = icmp eq i64 %i.inc, 100
+ br i1 %i.exitcond, label %end, label %for.i
+
+end:
+ ret void
+}
+
+; Only the innermost dimension is stretched. The outer ones count rows, and a
+; row grows together with the innermost dimension, so q stays as it is while r
+; becomes 8r.
+;
+; void three_dimensions(long q, long r, double A[100][q][r]) {
+; for (long i = 0; i < 100; i++)
+; for (long j = 0; j < 150; j++)
+; for (long k = 0; k < 200; k++)
+; A[i][j][k] = 1.0;
+; memset(A, 0, r * sizeof(double));
+; }
+
+; THREE: Assumed Context:
+; THREE-NEXT: [q, r] -> { : q >= 150 and r >= 200 }
+
+; THREE: Arrays {
+; THREE-NEXT: i8 MemRef_A[*][%q][(8 * %r)]; // Element size 1
+; THREE-NEXT: }
+
+; THREE: Statements {
+; THREE-NEXT: Stmt_for_k
+; THREE: MustWriteAccess := [Reduction Type: NONE] [Scalar: 0]
+; THREE-NEXT: [q, r] -> { Stmt_for_k[i0, i1, i2] -> MemRef_A[i0, i1, o2] : 8i2 <= o2 <= 7 + 8i2 };
+; THREE-NEXT: Stmt_memset_bb
+; THREE: MustWriteAccess := [Reduction Type: NONE] [Scalar: 0]
+; THREE-NEXT: [q, r] -> { Stmt_memset_bb[] -> MemRef_A[0, 0, o2] : 0 <= o2 < 8r };
+; THREE-NEXT: }
+
+define void @three_dimensions(i64 %q, i64 %r, ptr %A) {
+entry:
+ %len = shl nsw i64 %r, 3
+ br label %for.i
+
+for.i:
+ %i = phi i64 [ 0, %entry ], [ %i.inc, %for.i.inc ]
+ %t1 = mul nsw i64 %i, %q
+ br label %for.j
+
+for.j:
+ %j = phi i64 [ 0, %for.i ], [ %j.inc, %for.j.inc ]
+ %t2 = add nsw i64 %t1, %j
+ %t3 = mul nsw i64 %t2, %r
+ br label %for.k
+
+for.k:
+ %k = phi i64 [ 0, %for.j ], [ %k.inc, %for.k ]
+ %idx = add nsw i64 %t3, %k
+ %arrayidx = getelementptr inbounds double, ptr %A, i64 %idx
+ store double 1.0, ptr %arrayidx
+ %k.inc = add nsw i64 %k, 1
+ %k.exitcond = icmp eq i64 %k.inc, 200
+ br i1 %k.exitcond, label %for.j.inc, label %for.k
+
+for.j.inc:
+ %j.inc = add nsw i64 %j, 1
+ %j.exitcond = icmp eq i64 %j.inc, 150
+ br i1 %j.exitcond, label %for.i.inc, label %for.j
+
+for.i.inc:
+ %i.inc = add nsw i64 %i, 1
+ %i.exitcond = icmp eq i64 %i.inc, 100
+ br i1 %i.exitcond, label %memset.bb, label %for.i
+
+memset.bb:
+ call void @llvm.memset.p0.i64(ptr %A, i8 0, i64 %len, i1 false)
+ br label %end
+
+end:
+ ret void
+}
+
+declare void @llvm.memset.p0.i64(ptr nocapture writeonly, i8, i64, i1)
More information about the llvm-commits
mailing list