[flang-commits] [flang] [flang] Let temporaries inherit the alignment of the variable they copy (PR #234900)
Eugene Epshteyn via flang-commits
flang-commits at lists.llvm.org
Fri Oct 23 03:26:07 PDT 2026
https://github.com/eugeneepshteyn created https://github.com/llvm/llvm-project/pull/234900
Give the FIRBuilder temporary helpers (`createTemporaryAlloc`, `createTemporary`, `createHeapTemporary`, `createAndDeclareTemp`, `createArrayTemp`) an optional trailing alignment that is applied to the alloca or allocmem and recorded on the temporary's declare. `createTempFromMold` and `createStackTempFromMold` read it from the mold's declare, and the copy-in buffers made by `InlineHLFIRCopy` and `ConvertToFIR`'s `CopyInOpConversion` read it from the source variable.
With this, a temporary that the compiler creates as a copy of a variable whose declare carries `alignment(N)` is itself N-aligned, whether it lives on the stack or on the heap. Without a declared alignment nothing changes.
Stacked on #234897, #234898 and #234899.
>From c86795ec5ba55102235a4a28876baa7cb5955a4e Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 22 Oct 2026 19:27:48 -0700
Subject: [PATCH 1/4] [flang] Add an alignment attribute to fir.alloca
`fir.allocmem` and `fir.global` can carry a requested alignment, but
`fir.alloca` could not, so there was no way to represent an aligned
stack object in FIR. Add an optional `alignment` attribute to
`fir.alloca`, verified to be a positive power of two like the one on
`fir.allocmem`, and forward it to `llvm.alloca` in codegen.
The requested value is a minimum: codegen raises it to the ABI alignment
of the allocated type when it is smaller, so an alloca never ends up
less aligned than one without the attribute. The same clamp is applied
to `fir.global`, which until now forwarded the attribute verbatim.
This is the first step towards a directive that sets the alignment of
local, automatic and global variables; passes that convert between
stack and heap allocations learn to preserve the attribute in the
following change.
---
.../include/flang/Optimizer/Dialect/FIROps.td | 8 ++++-
flang/lib/Optimizer/CodeGen/CodeGen.cpp | 15 +++++++--
flang/lib/Optimizer/Dialect/FIROps.cpp | 16 ++++++----
flang/test/Fir/convert-to-llvm.fir | 32 +++++++++++++++++++
flang/test/Fir/fir-ops.fir | 2 ++
flang/test/Fir/invalid.fir | 8 +++++
6 files changed, 72 insertions(+), 9 deletions(-)
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index fc00daac1695b1..0b727b46dc4ce4 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -205,6 +205,11 @@ def fir_AllocaOp : fir_Op<"alloca", [
The pinned attribute is used to flag fir.alloca operation in a specific
region and avoid them being hoisted in an alloca hoisting pass.
+
+ The optional alignment attribute requests a minimum byte alignment for
+ the allocation, as `fir.allocmem` and `fir.global` do. It must be a
+ positive power of two. Code generation never lowers the alignment below
+ the ABI alignment of the allocated type.
}];
let arguments = (ins
@@ -213,7 +218,8 @@ def fir_AllocaOp : fir_Op<"alloca", [
OptionalAttr<StrAttr>:$bindc_name,
UnitAttr:$pinned,
Variadic<AnyIntegerType>:$typeparams,
- Variadic<AnyIntegerType>:$shape
+ Variadic<AnyIntegerType>:$shape,
+ OptionalAttr<I64Attr>:$alignment
);
let results =
diff --git a/flang/lib/Optimizer/CodeGen/CodeGen.cpp b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
index d5097aa6f76035..63cc61193d1f65 100644
--- a/flang/lib/Optimizer/CodeGen/CodeGen.cpp
+++ b/flang/lib/Optimizer/CodeGen/CodeGen.cpp
@@ -480,13 +480,21 @@ struct AllocaOpConversion : public fir::FIROpConversion<fir::AllocaOp> {
rewriter.setInsertionPointAfter(size.getDefiningOp());
}
+ // A requested alignment is a minimum: never go below the ABI alignment of
+ // the allocated type, which is what an alloca without an explicit
+ // alignment gets.
+ unsigned alignment = 0;
+ if (std::optional<std::int64_t> requested = alloc.getAlignment())
+ alignment = static_cast<unsigned>(std::max<std::uint64_t>(
+ *requested, getDataLayout().getTypeABIAlignment(llvmObjectType)));
+
// NOTE: we used to pass alloc->getAttrs() in the builder for non opaque
// pointers! Only propagate pinned and bindc_name to help debugging, but
// this should have no functional purpose (and passing the operand segment
// attribute like before is certainly bad).
auto llvmAlloc = mlir::LLVM::AllocaOp::create(
rewriter, loc, ::getLlvmPtrType(alloc.getContext(), allocaAs),
- llvmObjectType, size);
+ llvmObjectType, size, alignment);
if (alloc.getPinned())
llvmAlloc->setDiscardableAttr(alloc.getPinnedAttrName(),
alloc.getPinnedAttr());
@@ -3848,8 +3856,11 @@ struct GlobalOpConversion : public fir::FIROpConversion<fir::GlobalOp> {
0, getGlobalAddressSpace(rewriter), false,
mlir::LLVM::ThreadLocalMode::NotThreadLocal, comdat, attrs, dbgExprs);
+ // A requested alignment is a minimum: never go below the ABI alignment of
+ // the global's type.
if (global.getAlignment() && *global.getAlignment() > 0)
- g.setAlignment(*global.getAlignment());
+ g.setAlignment(std::max<std::uint64_t>(
+ *global.getAlignment(), getDataLayout().getTypeABIAlignment(tyAttr)));
if (auto section = global.getSection())
g.setSection(*section);
diff --git a/flang/lib/Optimizer/Dialect/FIROps.cpp b/flang/lib/Optimizer/Dialect/FIROps.cpp
index 3d079afa0d1a51..97bca969e96b7b 100644
--- a/flang/lib/Optimizer/Dialect/FIROps.cpp
+++ b/flang/lib/Optimizer/Dialect/FIROps.cpp
@@ -408,7 +408,7 @@ void fir::AllocaOp::build(mlir::OpBuilder &builder,
llvm::ArrayRef<mlir::NamedAttribute> attributes) {
auto nameAttr = builder.getStringAttr(uniqName);
build(builder, result, wrapAllocaResultType(inType), inType, nameAttr, {},
- /*pinned=*/false, typeparams, shape);
+ /*pinned=*/false, typeparams, shape, /*alignment=*/{});
result.addAttributes(attributes);
}
@@ -419,7 +419,7 @@ void fir::AllocaOp::build(mlir::OpBuilder &builder,
llvm::ArrayRef<mlir::NamedAttribute> attributes) {
auto nameAttr = builder.getStringAttr(uniqName);
build(builder, result, wrapAllocaResultType(inType), inType, nameAttr, {},
- pinned, typeparams, shape);
+ pinned, typeparams, shape, /*alignment=*/{});
result.addAttributes(attributes);
}
@@ -433,7 +433,7 @@ void fir::AllocaOp::build(mlir::OpBuilder &builder,
auto bindcAttr =
bindcName.empty() ? mlir::StringAttr{} : builder.getStringAttr(bindcName);
build(builder, result, wrapAllocaResultType(inType), inType, nameAttr,
- bindcAttr, /*pinned=*/false, typeparams, shape);
+ bindcAttr, /*pinned=*/false, typeparams, shape, /*alignment=*/{});
result.addAttributes(attributes);
}
@@ -448,7 +448,7 @@ void fir::AllocaOp::build(mlir::OpBuilder &builder,
auto bindcAttr =
bindcName.empty() ? mlir::StringAttr{} : builder.getStringAttr(bindcName);
build(builder, result, wrapAllocaResultType(inType), inType, nameAttr,
- bindcAttr, pinned, typeparams, shape);
+ bindcAttr, pinned, typeparams, shape, /*alignment=*/{});
result.addAttributes(attributes);
}
@@ -457,7 +457,7 @@ void fir::AllocaOp::build(mlir::OpBuilder &builder,
mlir::ValueRange typeparams, mlir::ValueRange shape,
llvm::ArrayRef<mlir::NamedAttribute> attributes) {
build(builder, result, wrapAllocaResultType(inType), inType, {}, {},
- /*pinned=*/false, typeparams, shape);
+ /*pinned=*/false, typeparams, shape, /*alignment=*/{});
result.addAttributes(attributes);
}
@@ -467,7 +467,7 @@ void fir::AllocaOp::build(mlir::OpBuilder &builder,
mlir::ValueRange shape,
llvm::ArrayRef<mlir::NamedAttribute> attributes) {
build(builder, result, wrapAllocaResultType(inType), inType, {}, {}, pinned,
- typeparams, shape);
+ typeparams, shape, /*alignment=*/{});
result.addAttributes(attributes);
}
@@ -492,6 +492,10 @@ llvm::LogicalResult fir::AllocaOp::verify() {
mlir::Type outType = getType();
if (!mlir::isa<fir::ReferenceType>(outType))
return emitOpError("must be a !fir.ref type");
+ if (std::optional<std::int64_t> alignment = getAlignment()) {
+ if (*alignment <= 0 || (*alignment & (*alignment - 1)) != 0)
+ return emitOpError("alignment must be a positive power of two");
+ }
return mlir::success();
}
diff --git a/flang/test/Fir/convert-to-llvm.fir b/flang/test/Fir/convert-to-llvm.fir
index 73b930b3803285..a0c4d0f6db0f46 100644
--- a/flang/test/Fir/convert-to-llvm.fir
+++ b/flang/test/Fir/convert-to-llvm.fir
@@ -1115,6 +1115,38 @@ func.func @alloca_several() -> !fir.ref<i32> {
// -----
+// Test fir.alloca with a requested alignment
+
+func.func @alloca_aligned() -> !fir.ref<!fir.array<10xf32>> {
+ %1 = fir.alloca !fir.array<10xf32> {alignment = 64 : i64}
+ return %1 : !fir.ref<!fir.array<10xf32>>
+}
+
+// CHECK-LABEL: llvm.func @alloca_aligned() -> !llvm.ptr
+// CHECK: [[N:%.*]] = llvm.mlir.constant(1 : i64) : i64
+// GENERIC: [[A:%.*]] = llvm.alloca [[N]] x !llvm.array<10 x f32> {alignment = 64 : i64}
+// AMDGPU: [[AA:%.*]] = llvm.alloca [[N]] x !llvm.array<10 x f32> {alignment = 64 : i64} : (i64) -> !llvm.ptr<5>
+// AMDGPU: [[A:%.*]] = llvm.addrspacecast [[AA]] : !llvm.ptr<5> to !llvm.ptr
+// CHECK: llvm.return [[A]] : !llvm.ptr
+
+// -----
+
+// A requested alignment below the ABI alignment of the type is raised to it.
+
+func.func @alloca_underaligned() -> !fir.ref<i32> {
+ %1 = fir.alloca i32 {alignment = 2 : i64}
+ return %1 : !fir.ref<i32>
+}
+
+// CHECK-LABEL: llvm.func @alloca_underaligned() -> !llvm.ptr
+// CHECK: [[N:%.*]] = llvm.mlir.constant(1 : i64) : i64
+// GENERIC: [[A:%.*]] = llvm.alloca [[N]] x i32 {alignment = 4 : i64}
+// AMDGPU: [[AA:%.*]] = llvm.alloca [[N]] x i32 {alignment = 4 : i64} : (i64) -> !llvm.ptr<5>
+// AMDGPU: [[A:%.*]] = llvm.addrspacecast [[AA]] : !llvm.ptr<5> to !llvm.ptr
+// CHECK: llvm.return [[A]] : !llvm.ptr
+
+// -----
+
// Test fir.alloca of pointer to array
func.func @alloca_ptr_to_array() -> !fir.ref<!fir.ptr<!fir.array<?xi32>>> {
diff --git a/flang/test/Fir/fir-ops.fir b/flang/test/Fir/fir-ops.fir
index 704cd2ab217fa6..bf42a9c1ae79e1 100644
--- a/flang/test/Fir/fir-ops.fir
+++ b/flang/test/Fir/fir-ops.fir
@@ -37,10 +37,12 @@ func.func @instructions() {
// CHECK: [[VAL_0:%.*]] = fir.alloca !fir.array<10xi32>
// CHECK: [[VAL_1:%.*]] = fir.load [[VAL_0]] : !fir.ref<!fir.array<10xi32>>
// CHECK: [[VAL_2:%.*]] = fir.alloca i32
+// CHECK: [[VAL_ALIGNED:%.*]] = fir.alloca !fir.array<10xf32> <{alignment = 64 : i64}>
// CHECK: [[VAL_3:%.*]] = arith.constant 22 : i32
%0 = fir.alloca !fir.array<10xi32>
%1 = fir.load %0 : !fir.ref<!fir.array<10xi32>>
%2 = fir.alloca i32
+ %aligned = fir.alloca !fir.array<10xf32> {alignment = 64 : i64}
%3 = arith.constant 22 : i32
// CHECK: fir.store [[VAL_3]] to [[VAL_2]] : !fir.ref<i32>
diff --git a/flang/test/Fir/invalid.fir b/flang/test/Fir/invalid.fir
index 8964df1d2e93e6..8bf414f86173f9 100644
--- a/flang/test/Fir/invalid.fir
+++ b/flang/test/Fir/invalid.fir
@@ -1400,3 +1400,11 @@ func.func @allocmem_bad_alignment() -> !fir.heap<!fir.array<10xf32>> {
%0 = fir.allocmem !fir.array<10xf32> {alignment = 3 : i64}
return %0 : !fir.heap<!fir.array<10xf32>>
}
+
+// -----
+
+func.func @alloca_bad_alignment() -> !fir.ref<!fir.array<10xf32>> {
+ // expected-error at +1 {{'fir.alloca' op alignment must be a positive power of two}}
+ %0 = fir.alloca !fir.array<10xf32> {alignment = 24 : i64}
+ return %0 : !fir.ref<!fir.array<10xf32>>
+}
>From ef50769176c1ba9d520ae422a551ef49e6e3ffb3 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 22 Oct 2026 19:27:49 -0700
Subject: [PATCH 2/4] [flang] Preserve the alignment attribute across
stack/heap conversions
The two helpers that convert allocations between the stack and the heap
rebuilt the new allocation from the type, names and operands only, so a
requested alignment was lost in both directions:
- `fir::createAllocMemFromAlloca` (used by MemoryAllocation,
AllocationPlacement and the CUDA heap promotion) now copies the
`fir.alloca` alignment onto the `fir.allocmem` it creates;
- `AllocMemConversion::insertAlloca` (StackArrays, also reused by
AllocationPlacement) now copies the `fir.allocmem` alignment onto the
`fir.alloca` it creates.
AllocationPlacement's stack budget also accounts for the worst-case
padding an over-aligned stack object may need.
Today the only producer of the `fir.allocmem` alignment attribute is
the allocatable/pointer allocation path, which StackArrays does not
touch by design, so this fixes a latent rather than an observable drop;
it becomes live as soon as temporaries and user variables carry an
alignment.
---
.../Transforms/AllocationPlacement.cpp | 10 ++++++--
.../lib/Optimizer/Transforms/MemoryUtils.cpp | 12 ++++++----
.../lib/Optimizer/Transforms/StackArrays.cpp | 3 +++
flang/test/Fir/memory-allocation-opt.fir | 14 +++++++++++
.../test/Transforms/allocation-placement.fir | 24 +++++++++++++++++++
flang/test/Transforms/stack-arrays.fir | 15 ++++++++++++
6 files changed, 72 insertions(+), 6 deletions(-)
diff --git a/flang/lib/Optimizer/Transforms/AllocationPlacement.cpp b/flang/lib/Optimizer/Transforms/AllocationPlacement.cpp
index 89783d78148316..a59c7181a1fef4 100644
--- a/flang/lib/Optimizer/Transforms/AllocationPlacement.cpp
+++ b/flang/lib/Optimizer/Transforms/AllocationPlacement.cpp
@@ -215,9 +215,15 @@ void AllocationPlacementPass::runOnOperation() {
? placementHook(info, basePolicy, stackBytesUsed)
: fir::decideAllocationPlacement(info, basePolicy, stackBytesUsed);
- // Account for the decision in the running stack budget.
- if (endsUpOnStack(placement, info.isCurrentlyOnStack) && info.byteSize)
+ // Account for the decision in the running stack budget, including the
+ // worst-case padding a requested alignment may need.
+ if (endsUpOnStack(placement, info.isCurrentlyOnStack) && info.byteSize) {
stackBytesUsed += static_cast<std::size_t>(*info.byteSize);
+ std::optional<std::int64_t> alignment =
+ alloca ? alloca.getAlignment() : allocmem.getAlignment();
+ if (alignment && *alignment > 1)
+ stackBytesUsed += static_cast<std::size_t>(*alignment - 1);
+ }
LLVM_DEBUG({
llvm::dbgs() << "allocation-placement: "
diff --git a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
index 00a25a7326f2dd..6da0c3141b8c44 100644
--- a/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
+++ b/flang/lib/Optimizer/Transforms/MemoryUtils.cpp
@@ -322,10 +322,14 @@ fir::AllocMemOp fir::createAllocMemFromAlloca(mlir::OpBuilder &builder,
return *opt;
return {};
};
- return fir::AllocMemOp::create(builder, alloca.getLoc(), alloca.getInType(),
- unpackName(alloca.getUniqName()),
- unpackName(alloca.getBindcName()),
- alloca.getTypeparams(), alloca.getShape());
+ auto heap = fir::AllocMemOp::create(
+ builder, alloca.getLoc(), alloca.getInType(),
+ unpackName(alloca.getUniqName()), unpackName(alloca.getBindcName()),
+ alloca.getTypeparams(), alloca.getShape());
+ // Keep a requested alignment across the stack-to-heap conversion.
+ if (std::optional<std::int64_t> alignment = alloca.getAlignment())
+ heap.setAlignment(*alignment);
+ return heap;
}
/// Device code keeps its stack allocations: the unified/managed entry points
diff --git a/flang/lib/Optimizer/Transforms/StackArrays.cpp b/flang/lib/Optimizer/Transforms/StackArrays.cpp
index 0674cc24319606..6b8edde0c2d927 100644
--- a/flang/lib/Optimizer/Transforms/StackArrays.cpp
+++ b/flang/lib/Optimizer/Transforms/StackArrays.cpp
@@ -711,6 +711,9 @@ fir::AllocMemConversion::insertAlloca(fir::AllocMemOp &oldAlloc,
auto alloca =
fir::AllocaOp::create(rewriter, loc, varTy, uniqName, bindcName,
oldAlloc.getTypeparams(), oldAlloc.getShape());
+ // Keep a requested alignment across the heap-to-stack conversion.
+ if (std::optional<std::int64_t> alignment = oldAlloc.getAlignment())
+ alloca.setAlignment(*alignment);
if (emitLifetimeMarkers)
insertLifetimeMarkers(oldAlloc, alloca, rewriter);
diff --git a/flang/test/Fir/memory-allocation-opt.fir b/flang/test/Fir/memory-allocation-opt.fir
index 610a3f0b3072a6..9993ebc6b7e502 100644
--- a/flang/test/Fir/memory-allocation-opt.fir
+++ b/flang/test/Fir/memory-allocation-opt.fir
@@ -32,3 +32,17 @@ func.func @_QPs2(%arg0: !fir.ref<i32>) {
}
func.func private @_QPs3(!fir.ref<!fir.array<1000123xi32>>)
+// A requested alignment survives the move to the heap.
+
+// CHECK-LABEL: func @_QPs4(
+// CHECK: %[[mem:.*]] = fir.allocmem !fir.array<1000123xi32> <{alignment = 64 : i64, bindc_name = "array", uniq_name = "_QFs4Earray"}>
+// CHECK: fir.call @_QPs3(
+// CHECK: fir.freemem %[[mem]]
+// CHECK-NEXT: return
+
+func.func @_QPs4() {
+ %0 = fir.alloca !fir.array<1000123xi32> {alignment = 64 : i64, bindc_name = "array", uniq_name = "_QFs4Earray"}
+ fir.call @_QPs3(%0) : (!fir.ref<!fir.array<1000123xi32>>) -> ()
+ return
+}
+
diff --git a/flang/test/Transforms/allocation-placement.fir b/flang/test/Transforms/allocation-placement.fir
index aa09d830fd5a5f..c9c8f6d65e6374 100644
--- a/flang/test/Transforms/allocation-placement.fir
+++ b/flang/test/Transforms/allocation-placement.fir
@@ -63,6 +63,30 @@ func.func @big_temp_alloca() {
return
}
+// A requested alignment survives heap-to-stack ...
+// CHECK-LABEL: func.func @small_temp_aligned
+// CHECK: fir.alloca !fir.array<10xi32> <{alignment = 64 : i64}>
+// CHECK-NOT: fir.allocmem
+func.func @small_temp_aligned() {
+ %0 = fir.allocmem !fir.array<10xi32> {alignment = 64 : i64}
+ %c0 = arith.constant 0 : index
+ %v = arith.constant 0 : i32
+ %r = fir.convert %0 : (!fir.heap<!fir.array<10xi32>>) -> !fir.ref<!fir.array<10xi32>>
+ %e = fir.coordinate_of %r, %c0 : (!fir.ref<!fir.array<10xi32>>, index) -> !fir.ref<i32>
+ fir.store %v to %e : !fir.ref<i32>
+ fir.freemem %0 : !fir.heap<!fir.array<10xi32>>
+ return
+}
+
+// ... and stack-to-heap.
+// CHECK-LABEL: func.func @big_temp_alloca_aligned
+// CHECK: fir.allocmem !fir.array<100xi32> <{alignment = 64 : i64
+// CHECK: fir.freemem
+func.func @big_temp_alloca_aligned() {
+ %0 = fir.alloca !fir.array<100xi32> {alignment = 64 : i64}
+ return
+}
+
// Runtime-sized user variable (automatic array) -> heap.
// CHECK-LABEL: func.func @dyn_user
// CHECK: fir.allocmem !fir.array<?xi32>
diff --git a/flang/test/Transforms/stack-arrays.fir b/flang/test/Transforms/stack-arrays.fir
index 5b149bfd8b6216..08d893d5679ea3 100644
--- a/flang/test/Transforms/stack-arrays.fir
+++ b/flang/test/Transforms/stack-arrays.fir
@@ -15,6 +15,21 @@ func.func @simple() {
// CHECK: fir.alloca !fir.array<42xi32>
// CHECK: return
+// A requested alignment survives the move to the stack
+func.func @aligned() {
+ %0 = fir.allocmem !fir.array<42xi32> {alignment = 64 : i64}
+ %c0_a = arith.constant 0 : index
+ %c0_i32_a = arith.constant 0 : i32
+ %ref_a = fir.convert %0 : (!fir.heap<!fir.array<42xi32>>) -> !fir.ref<!fir.array<42xi32>>
+ %elt_a = fir.coordinate_of %ref_a, %c0_a : (!fir.ref<!fir.array<42xi32>>, index) -> !fir.ref<i32>
+ fir.store %c0_i32_a to %elt_a : !fir.ref<i32>
+ fir.freemem %0 : !fir.heap<!fir.array<42xi32>>
+ return
+}
+// CHECK-LABEL: func.func @aligned()
+// CHECK: fir.alloca !fir.array<42xi32> <{alignment = 64 : i64}>
+// CHECK: return
+
// Check fir.must_be_heap allocations are not moved
func.func @must_be_heap() {
%0 = fir.allocmem !fir.array<42xi32> {fir.must_be_heap = true}
>From 7e82d41da1abbe5cd038aa5ebd45e127bce8edac Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 22 Oct 2026 19:36:47 -0700
Subject: [PATCH 3/4] [flang] Carry a requested alignment on fir.declare and
hlfir.declare
Add an optional `alignment` attribute to `fir.declare` and
`hlfir.declare`, printed as `alignment(N)`, and propagate it from the
HLFIR op to the FIR op in the HLFIR-to-FIR conversion.
The attribute is the durable per-variable carrier for an alignment the
user requested on a variable: temporaries created from the variable and
the HLFIR rewrites that re-allocate storage see SSA values, not front-end
symbols, so they need the information on the declare operation. For a
variable whose storage is allocated by the compiler the value is also a
fact about that storage; for a descriptor-based variable it is only the
policy for future allocations, and the op documentation says so.
Two helpers, `hlfir::getDeclaredAlignment` and
`hlfir::setDeclaredAlignment`, read and record the attribute through
either declare op; the getter looks through `hlfir.designate` to the
designated variable.
---
.../flang/Optimizer/Builder/HLFIRTools.h | 9 +++++++
.../include/flang/Optimizer/Dialect/FIROps.td | 10 +++++++-
.../include/flang/Optimizer/HLFIR/HLFIROps.td | 4 ++-
flang/lib/Optimizer/Builder/FIRBuilder.cpp | 2 +-
flang/lib/Optimizer/Builder/HLFIRTools.cpp | 25 +++++++++++++++++++
flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp | 3 ++-
.../HLFIR/Transforms/ConvertToFIR.cpp | 2 +-
flang/test/Fir/declare.fir | 8 ++++++
flang/test/HLFIR/declare-codegen.fir | 8 ++++++
flang/test/HLFIR/declare.fir | 8 ++++++
.../Optimizer/FortranVariableTest.cpp | 11 ++++----
11 files changed, 80 insertions(+), 10 deletions(-)
diff --git a/flang/include/flang/Optimizer/Builder/HLFIRTools.h b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
index ce99a370526213..59f6e13d3594d4 100644
--- a/flang/include/flang/Optimizer/Builder/HLFIRTools.h
+++ b/flang/include/flang/Optimizer/Builder/HLFIRTools.h
@@ -555,6 +555,15 @@ hlfir::Entity createStackTempFromMold(mlir::Location loc,
fir::FirOpBuilder &builder,
hlfir::Entity mold);
+/// Return the alignment recorded on the hlfir.declare or fir.declare that
+/// defines \p var, if any. A designator of a variable (hlfir.designate) is
+/// looked through to the variable it designates.
+std::optional<std::int64_t> getDeclaredAlignment(mlir::Value var);
+
+/// Record \p alignment on the hlfir.declare or fir.declare defining \p var.
+/// No-op when \p var is not defined by a declare operation.
+void setDeclaredAlignment(mlir::Value var, std::int64_t alignment);
+
hlfir::EntityWithAttributes convertCharacterKind(mlir::Location loc,
fir::FirOpBuilder &builder,
hlfir::Entity scalarChar,
diff --git a/flang/include/flang/Optimizer/Dialect/FIROps.td b/flang/include/flang/Optimizer/Dialect/FIROps.td
index 0b727b46dc4ce4..8d6f6419221bfe 100644
--- a/flang/include/flang/Optimizer/Dialect/FIROps.td
+++ b/flang/include/flang/Optimizer/Dialect/FIROps.td
@@ -3393,6 +3393,12 @@ def fir_DeclareOp
If these operands are absent, then the storage of the declared variable
is only known to start where the memref operand points to.
+ The optional alignment attribute records the byte alignment requested
+ for the variable (e.g. by a compiler directive). For a variable whose
+ storage is allocated by the compiler it is a fact about that storage;
+ for a variable represented by a descriptor it is only the policy for
+ future allocations and must not be turned into an alignment assumption.
+
Example:
CHARACTER(n), OPTIONAL, TARGET :: c(10:, 20:)
@@ -3419,7 +3425,8 @@ def fir_DeclareOp
Builtin_StringAttr:$uniq_name,
OptionalAttr<fir_FortranVariableFlagsAttr>:$fortran_attrs,
OptionalAttr<cuf_DataAttributeAttr>:$data_attr,
- OptionalAttr<UI32Attr>:$dummy_arg_no);
+ OptionalAttr<UI32Attr>:$dummy_arg_no,
+ OptionalAttr<I64Attr>:$alignment);
let results = (outs AnyRefOrBox);
@@ -3430,6 +3437,7 @@ def fir_DeclareOp
`uniq_name` `(` $uniq_name `)`
(`fortran_attrs` `` $fortran_attrs^)?
(`data_attr` `(` qualified($data_attr)^ `)`)?
+ (`alignment` `(` $alignment^ `)`)?
attr-dict `:` functional-type(operands, results)
}];
diff --git a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
index 4c531c5bddf0b8..fee42bcaad5d16 100644
--- a/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
+++ b/flang/include/flang/Optimizer/HLFIR/HLFIROps.td
@@ -104,7 +104,8 @@ def hlfir_DeclareOp
Builtin_StringAttr:$uniq_name,
OptionalAttr<fir_FortranVariableFlagsAttr>:$fortran_attrs,
OptionalAttr<cuf_DataAttributeAttr>:$data_attr,
- OptionalAttr<UnitAttr>:$skip_rebox, OptionalAttr<UI32Attr>:$dummy_arg_no);
+ OptionalAttr<UnitAttr>:$skip_rebox, OptionalAttr<UI32Attr>:$dummy_arg_no,
+ OptionalAttr<I64Attr>:$alignment);
let results = (outs AnyFortranVariable, AnyRefOrBoxLike);
@@ -116,6 +117,7 @@ def hlfir_DeclareOp
`uniq_name` `(` $uniq_name `)`
(`fortran_attrs` `` $fortran_attrs^)?
(`data_attr` `(` qualified($data_attr)^ `)`)?
+ (`alignment` `(` $alignment^ `)`)?
attr-dict `:` functional-type(operands, results)
}];
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index 7f03f3f050d5fd..2386a94a0374d4 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -463,7 +463,7 @@ mlir::Value fir::FirOpBuilder::genTempDeclareOp(
/*dummy_scope=*/nullptr,
/*storage=*/nullptr,
/*storage_offset=*/0, nameAttr, fortranAttrs, cuf::DataAttributeAttr{},
- /*dummy_arg_no=*/mlir::IntegerAttr{});
+ /*dummy_arg_no=*/mlir::IntegerAttr{}, /*alignment=*/mlir::IntegerAttr{});
}
mlir::Value fir::FirOpBuilder::genStackSave(mlir::Location loc) {
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index 231c8f282cdcb3..03e5bf9c163660 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -1536,6 +1536,31 @@ hlfir::Entity hlfir::createStackTempFromMold(mlir::Location loc,
return hlfir::Entity{declareOp.getBase()};
}
+std::optional<std::int64_t> hlfir::getDeclaredAlignment(mlir::Value var) {
+ while (var) {
+ mlir::Operation *defOp = var.getDefiningOp();
+ if (!defOp)
+ return std::nullopt;
+ if (auto declare = mlir::dyn_cast<hlfir::DeclareOp>(defOp))
+ return declare.getAlignment();
+ if (auto declare = mlir::dyn_cast<fir::DeclareOp>(defOp))
+ return declare.getAlignment();
+ if (auto designate = mlir::dyn_cast<hlfir::DesignateOp>(defOp)) {
+ var = designate.getMemref();
+ continue;
+ }
+ return std::nullopt;
+ }
+ return std::nullopt;
+}
+
+void hlfir::setDeclaredAlignment(mlir::Value var, std::int64_t alignment) {
+ if (auto declare = var.getDefiningOp<hlfir::DeclareOp>())
+ declare.setAlignment(alignment);
+ else if (auto declare = var.getDefiningOp<fir::DeclareOp>())
+ declare.setAlignment(alignment);
+}
+
hlfir::EntityWithAttributes
hlfir::convertCharacterKind(mlir::Location loc, fir::FirOpBuilder &builder,
hlfir::Entity scalarChar, int toKind) {
diff --git a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
index 9503d64bd5d374..710409d2b038f1 100644
--- a/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
+++ b/flang/lib/Optimizer/HLFIR/IR/HLFIROps.cpp
@@ -281,7 +281,8 @@ void hlfir::DeclareOp::build(
argNoAttr = builder.getUI32IntegerAttr(dummy_arg_no);
build(builder, result, {hlfirVariableType, firVarType}, memref, shape,
typeparams, dummy_scope, storage, storage_offset, nameAttr,
- fortran_attrs, data_attr, /*skip_rebox=*/mlir::UnitAttr{}, argNoAttr);
+ fortran_attrs, data_attr, /*skip_rebox=*/mlir::UnitAttr{}, argNoAttr,
+ /*alignment=*/mlir::IntegerAttr{});
}
llvm::LogicalResult hlfir::DeclareOp::verify() {
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
index c223b07c4bf583..11d9bc3a3377c6 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
@@ -435,7 +435,7 @@ class DeclareOpConversion : public mlir::OpRewritePattern<hlfir::DeclareOp> {
/*storage=*/declareOp.getStorage(),
/*storage_offset=*/declareOp.getStorageOffset(),
declareOp.getUniqName(), fortranAttrs, dataAttr,
- declareOp.getDummyArgNoAttr());
+ declareOp.getDummyArgNoAttr(), declareOp.getAlignmentAttr());
// Propagate other attributes from hlfir.declare to fir.declare.
// OpenACC's acc.declare is one example. Right now, the propagation
diff --git a/flang/test/Fir/declare.fir b/flang/test/Fir/declare.fir
index b30c38d7b66e31..75ddde6ef3942e 100644
--- a/flang/test/Fir/declare.fir
+++ b/flang/test/Fir/declare.fir
@@ -162,3 +162,11 @@ func.func @vars_within_physical_storage() {
%8 = fir.declare %7 storage (%1[4]) uniq_name("b") : (!fir.ref<f32>, !fir.ref<!fir.array<8xi8>>) -> !fir.ref<f32>
return
}
+
+func.func @aligned_declare(%arg0: !fir.ref<f32>) {
+ %0 = fir.declare %arg0 uniq_name("a") alignment(64) : (!fir.ref<f32>) -> !fir.ref<f32>
+ return
+}
+// CHECK-LABEL: func.func @aligned_declare(
+// CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<f32>) {
+// CHECK: %[[VAL_1:.*]] = fir.declare %[[VAL_0]] uniq_name("a") alignment(64) : (!fir.ref<f32>) -> !fir.ref<f32>
diff --git a/flang/test/HLFIR/declare-codegen.fir b/flang/test/HLFIR/declare-codegen.fir
index 4908a19d488c51..b535bc971a4137 100644
--- a/flang/test/HLFIR/declare-codegen.fir
+++ b/flang/test/HLFIR/declare-codegen.fir
@@ -294,3 +294,11 @@ func.func @vars_within_physical_storage() {
// CHECK: %[[VAL_9:.*]] = fir.declare %[[VAL_8]] storage(%[[VAL_2]][4]) uniq_name("b") : (!fir.ref<f32>, !fir.ref<!fir.array<8xi8>>) -> !fir.ref<f32>
// CHECK: return
// CHECK: }
+
+func.func @aligned_declare(%arg0: !fir.ref<f32>) {
+ %0:2 = hlfir.declare %arg0 uniq_name("a") alignment(64) : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+ return
+}
+// CHECK-LABEL: func.func @aligned_declare(
+// CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<f32>) {
+// CHECK: %[[VAL_1:.*]] = fir.declare %[[VAL_0]] uniq_name("a") alignment(64) : (!fir.ref<f32>) -> !fir.ref<f32>
diff --git a/flang/test/HLFIR/declare.fir b/flang/test/HLFIR/declare.fir
index f7c5a5c3c52fae..6603dd53a0bdd6 100644
--- a/flang/test/HLFIR/declare.fir
+++ b/flang/test/HLFIR/declare.fir
@@ -179,3 +179,11 @@ func.func @vars_within_physical_storage() {
// CHECK: %[[VAL_2:.*]] = fir.address_of(@block_) : !fir.ref<!fir.array<8xi8>>
// CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %{{.*}} storage(%[[VAL_2]][0]) uniq_name("a") : (!fir.ref<f32>, !fir.ref<!fir.array<8xi8>>) -> (!fir.ref<f32>, !fir.ref<f32>)
// CHECK: %[[VAL_9:.*]]:2 = hlfir.declare %{{.*}} storage(%[[VAL_2]][4]) uniq_name("b") : (!fir.ref<f32>, !fir.ref<!fir.array<8xi8>>) -> (!fir.ref<f32>, !fir.ref<f32>)
+
+func.func @aligned_declare(%arg0: !fir.ref<f32>) {
+ %0:2 = hlfir.declare %arg0 uniq_name("a") alignment(64) : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
+ return
+}
+// CHECK-LABEL: func.func @aligned_declare(
+// CHECK-SAME: %[[VAL_0:.*]]: !fir.ref<f32>) {
+// CHECK: %[[VAL_1:.*]]:2 = hlfir.declare %[[VAL_0]] uniq_name("a") alignment(64) : (!fir.ref<f32>) -> (!fir.ref<f32>, !fir.ref<f32>)
diff --git a/flang/unittests/Optimizer/FortranVariableTest.cpp b/flang/unittests/Optimizer/FortranVariableTest.cpp
index d2ed5547357622..7b31b6ec1766ec 100644
--- a/flang/unittests/Optimizer/FortranVariableTest.cpp
+++ b/flang/unittests/Optimizer/FortranVariableTest.cpp
@@ -53,7 +53,7 @@ TEST_F(FortranVariableTest, SimpleScalar) {
/*dummy_scope=*/nullptr, /*storage=*/nullptr, /*storage_offset=*/0, name,
/*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
/*data_attr=*/cuf::DataAttributeAttr{},
- /*dummy_arg_no=*/mlir::IntegerAttr{});
+ /*dummy_arg_no=*/mlir::IntegerAttr{}, /*alignment=*/mlir::IntegerAttr{});
fir::FortranVariableOpInterface fortranVariable = declare;
EXPECT_FALSE(fortranVariable.isArray());
@@ -81,7 +81,7 @@ TEST_F(FortranVariableTest, CharacterScalar) {
/*storage=*/nullptr, /*storage_offset=*/0, name,
/*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
/*data_attr=*/cuf::DataAttributeAttr{},
- /*dummy_arg_no=*/mlir::IntegerAttr{});
+ /*dummy_arg_no=*/mlir::IntegerAttr{}, /*alignment=*/mlir::IntegerAttr{});
fir::FortranVariableOpInterface fortranVariable = declare;
EXPECT_FALSE(fortranVariable.isArray());
@@ -114,7 +114,7 @@ TEST_F(FortranVariableTest, SimpleArray) {
/*storage=*/nullptr, /*storage_offset=*/0, name,
/*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
/*data_attr=*/cuf::DataAttributeAttr{},
- /*dummy_arg_no=*/mlir::IntegerAttr{});
+ /*dummy_arg_no=*/mlir::IntegerAttr{}, /*alignment=*/mlir::IntegerAttr{});
fir::FortranVariableOpInterface fortranVariable = declare;
EXPECT_TRUE(fortranVariable.isArray());
@@ -147,7 +147,7 @@ TEST_F(FortranVariableTest, CharacterArray) {
/*storage_offset=*/0, name,
/*fortran_attrs=*/fir::FortranVariableFlagsAttr{},
/*data_attr=*/cuf::DataAttributeAttr{},
- /*dummy_arg_no=*/mlir::IntegerAttr{});
+ /*dummy_arg_no=*/mlir::IntegerAttr{}, /*alignment=*/mlir::IntegerAttr{});
fir::FortranVariableOpInterface fortranVariable = declare;
EXPECT_TRUE(fortranVariable.isArray());
@@ -219,7 +219,8 @@ TEST_F(FortranVariableTest, OptionalBoxThroughAssert) {
builder->getStringAttr("x"),
fir::FortranVariableFlagsAttr::get(&context, flags),
/*data_attr=*/cuf::DataAttributeAttr{},
- /*dummy_arg_no=*/mlir::IntegerAttr{});
+ /*dummy_arg_no=*/mlir::IntegerAttr{},
+ /*alignment=*/mlir::IntegerAttr{});
auto guard = fir::AssertOp::create(*builder, loc, condition,
fir::RuntimeCheckKind::conformance, "mismatch", mlir::ValueRange{},
mlir::ValueRange{declare});
>From a31c1a36036b796aa4ee56bf757b751aecf541c0 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 22 Oct 2026 19:36:48 -0700
Subject: [PATCH 4/4] [flang] Let temporaries inherit the alignment of the
variable they copy
A temporary that the compiler creates from a variable should keep the
alignment requested for that variable, otherwise a library contract the
variable satisfies is silently broken by the copy.
Give the builder's temporary helpers (`createTemporaryAlloc`,
`createTemporary`, `createHeapTemporary`, `createAndDeclareTemp`,
`createArrayTemp`) a trailing optional alignment that is set on the
`fir.alloca`/`fir.allocmem` and recorded on the temporary's declare, and
use it where a temporary is molded from a variable:
- `hlfir::createTempFromMold` and `createStackTempFromMold` (hlfir.as_expr
copies, assignment temporaries) read the mold's declared alignment;
- the copy-in buffer, both in the loop-inlined form (InlineHLFIRCopy) and
in the hlfir.copy_in codegen (ConvertToFIR), reads the alignment of the
variable being copied, looking through the section designator.
Temporaries built from a type or a shape only (elemental expression
temporaries, FORALL/WHERE temporaries) have no mold variable and are
unchanged; polymorphic temporaries allocated by the runtime are unchanged
as well.
---
.../flang/Optimizer/Builder/FIRBuilder.h | 31 +++++++++----
flang/lib/Optimizer/Builder/FIRBuilder.cpp | 44 +++++++++++++------
flang/lib/Optimizer/Builder/HLFIRTools.cpp | 16 +++++--
.../HLFIR/Transforms/ConvertToFIR.cpp | 14 ++++--
.../HLFIR/Transforms/InlineHLFIRCopy.cpp | 17 +++++--
flang/test/HLFIR/as_expr-codegen.fir | 15 +++++++
flang/test/HLFIR/copy-in-out-codegen.fir | 11 +++++
flang/test/HLFIR/inline-hlfir-copy-stack.fir | 33 ++++++++++++++
8 files changed, 147 insertions(+), 34 deletions(-)
diff --git a/flang/include/flang/Optimizer/Builder/FIRBuilder.h b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
index 3857d2b73c4ee1..2d797803ff3ccd 100644
--- a/flang/include/flang/Optimizer/Builder/FIRBuilder.h
+++ b/flang/include/flang/Optimizer/Builder/FIRBuilder.h
@@ -251,21 +251,25 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
/// Create a temporary using `fir.alloca`. This function does not hoist.
/// It is the callers responsibility to set the insertion point if
- /// hoisting is required.
+ /// hoisting is required. \p alignment, when given, is the requested byte
+ /// alignment of the allocation.
mlir::Value createTemporaryAlloc(
mlir::Location loc, mlir::Type type, llvm::StringRef name,
mlir::ValueRange lenParams = {}, mlir::ValueRange shape = {},
llvm::ArrayRef<mlir::NamedAttribute> attrs = {},
- std::optional<Fortran::common::CUDADataAttr> cudaAttr = std::nullopt);
+ std::optional<Fortran::common::CUDADataAttr> cudaAttr = std::nullopt,
+ std::optional<std::int64_t> alignment = std::nullopt);
/// Create a temporary. A temp is allocated using `fir.alloca` and can be read
/// and written using `fir.load` and `fir.store`, resp. The temporary can be
- /// given a name via a front-end `Symbol` or a `StringRef`.
+ /// given a name via a front-end `Symbol` or a `StringRef`. \p alignment,
+ /// when given, is the requested byte alignment of the allocation.
mlir::Value createTemporary(
mlir::Location loc, mlir::Type type, llvm::StringRef name = {},
mlir::ValueRange shape = {}, mlir::ValueRange lenParams = {},
llvm::ArrayRef<mlir::NamedAttribute> attrs = {},
- std::optional<Fortran::common::CUDADataAttr> cudaAttr = std::nullopt);
+ std::optional<Fortran::common::CUDADataAttr> cudaAttr = std::nullopt,
+ std::optional<std::int64_t> alignment = std::nullopt);
/// Create an unnamed and untracked temporary on the stack.
mlir::Value createTemporary(mlir::Location loc, mlir::Type type,
@@ -284,12 +288,14 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
return createTemporary(loc, type, name, {}, {}, attrs);
}
- /// Create a temporary on the heap.
+ /// Create a temporary on the heap. \p alignment, when given, is the
+ /// requested byte alignment of the allocation.
mlir::Value
createHeapTemporary(mlir::Location loc, mlir::Type type,
llvm::StringRef name = {}, mlir::ValueRange shape = {},
mlir::ValueRange lenParams = {},
- llvm::ArrayRef<mlir::NamedAttribute> attrs = {});
+ llvm::ArrayRef<mlir::NamedAttribute> attrs = {},
+ std::optional<std::int64_t> alignment = std::nullopt);
/// Sample genDeclare callback for createArrayTemp() below.
/// It creates fir.declare operation using the given operands.
@@ -313,6 +319,10 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
/// just passes through the base of the temporary.
/// If \p useStack is true, the function will try to do the allocation
/// in stack memory (which is not always possible currently).
+ /// \p alignment, when given, is the requested byte alignment of the
+ /// allocation; it is also recorded on the declare operation of the
+ /// temporary (polymorphic temporaries, allocated by the runtime, do not
+ /// honor it yet).
/// The first return value is the base of the temporary object,
/// which may be !fir.ref<!fir.array<>> or !fir.box/class<>.
/// The second return value is true, if the actual allocation
@@ -322,7 +332,8 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
llvm::ArrayRef<mlir::Value> extents,
llvm::ArrayRef<mlir::Value> typeParams,
const std::function<decltype(genTempDeclareOp)> &genDeclare,
- mlir::Value polymorphicMold, bool useStack, llvm::StringRef tmpName);
+ mlir::Value polymorphicMold, bool useStack, llvm::StringRef tmpName,
+ std::optional<std::int64_t> alignment = std::nullopt);
/// Create and declare an array temporary.
std::pair<mlir::Value, bool>
createArrayTemp(mlir::Location loc, fir::SequenceType arrayType,
@@ -330,9 +341,11 @@ class FirOpBuilder : public mlir::OpBuilder, public mlir::OpBuilder::Listener {
llvm::ArrayRef<mlir::Value> typeParams,
const std::function<decltype(genTempDeclareOp)> &genDeclare,
mlir::Value polymorphicMold, bool useStack = false,
- llvm::StringRef tmpName = ".tmp.array") {
+ llvm::StringRef tmpName = ".tmp.array",
+ std::optional<std::int64_t> alignment = std::nullopt) {
return createAndDeclareTemp(loc, arrayType, shape, extents, typeParams,
- genDeclare, polymorphicMold, useStack, tmpName);
+ genDeclare, polymorphicMold, useStack, tmpName,
+ alignment);
}
/// Create an LLVM stack save intrinsic op. Returns the saved stack pointer.
diff --git a/flang/lib/Optimizer/Builder/FIRBuilder.cpp b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
index 2386a94a0374d4..666ada7612d24d 100644
--- a/flang/lib/Optimizer/Builder/FIRBuilder.cpp
+++ b/flang/lib/Optimizer/Builder/FIRBuilder.cpp
@@ -12,6 +12,7 @@
#include "flang/Optimizer/Builder/BoxValue.h"
#include "flang/Optimizer/Builder/Character.h"
#include "flang/Optimizer/Builder/Complex.h"
+#include "flang/Optimizer/Builder/HLFIRTools.h"
#include "flang/Optimizer/Builder/MutableBox.h"
#include "flang/Optimizer/Builder/Runtime/Allocatable.h"
#include "flang/Optimizer/Builder/Runtime/Assign.h"
@@ -365,7 +366,8 @@ mlir::Value fir::FirOpBuilder::createTemporaryAlloc(
mlir::Location loc, mlir::Type type, llvm::StringRef name,
mlir::ValueRange lenParams, mlir::ValueRange shape,
llvm::ArrayRef<mlir::NamedAttribute> attrs,
- std::optional<Fortran::common::CUDADataAttr> cudaAttr) {
+ std::optional<Fortran::common::CUDADataAttr> cudaAttr,
+ std::optional<std::int64_t> alignment) {
assert(!mlir::isa<fir::ReferenceType>(type) && "cannot be a reference");
// If the alloca is inside an OpenMP Op which will be outlined then pin
// the alloca here.
@@ -377,9 +379,12 @@ mlir::Value fir::FirOpBuilder::createTemporaryAlloc(
/*unique_name=*/llvm::StringRef{}, name, attr,
lenParams, shape, attrs);
} else {
- return fir::AllocaOp::create(*this, loc, type,
- /*unique_name=*/llvm::StringRef{}, name,
- pinned, lenParams, shape, attrs);
+ auto alloca = fir::AllocaOp::create(*this, loc, type,
+ /*unique_name=*/llvm::StringRef{}, name,
+ pinned, lenParams, shape, attrs);
+ if (alignment)
+ alloca.setAlignment(*alignment);
+ return alloca;
}
}
@@ -389,7 +394,8 @@ mlir::Value fir::FirOpBuilder::createTemporary(
mlir::Location loc, mlir::Type type, llvm::StringRef name,
mlir::ValueRange shape, mlir::ValueRange lenParams,
llvm::ArrayRef<mlir::NamedAttribute> attrs,
- std::optional<Fortran::common::CUDADataAttr> cudaAttr) {
+ std::optional<Fortran::common::CUDADataAttr> cudaAttr,
+ std::optional<std::int64_t> alignment) {
llvm::SmallVector<mlir::Value> dynamicShape =
fir::factory::elideExtentsAlreadyInType(type, shape);
llvm::SmallVector<mlir::Value> dynamicLength =
@@ -401,8 +407,8 @@ mlir::Value fir::FirOpBuilder::createTemporary(
setInsertionPointToStart(getAllocaBlock());
}
- mlir::Value ae = createTemporaryAlloc(loc, type, name, dynamicLength,
- dynamicShape, attrs, cudaAttr);
+ mlir::Value ae = createTemporaryAlloc(
+ loc, type, name, dynamicLength, dynamicShape, attrs, cudaAttr, alignment);
if (hoistAlloc)
restoreInsertionPoint(insPt);
@@ -412,23 +418,28 @@ mlir::Value fir::FirOpBuilder::createTemporary(
mlir::Value fir::FirOpBuilder::createHeapTemporary(
mlir::Location loc, mlir::Type type, llvm::StringRef name,
mlir::ValueRange shape, mlir::ValueRange lenParams,
- llvm::ArrayRef<mlir::NamedAttribute> attrs) {
+ llvm::ArrayRef<mlir::NamedAttribute> attrs,
+ std::optional<std::int64_t> alignment) {
llvm::SmallVector<mlir::Value> dynamicShape =
fir::factory::elideExtentsAlreadyInType(type, shape);
llvm::SmallVector<mlir::Value> dynamicLength =
fir::factory::elideLengthsAlreadyInType(type, lenParams);
assert(!mlir::isa<fir::ReferenceType>(type) && "cannot be a reference");
- return fir::AllocMemOp::create(*this, loc, type,
- /*unique_name=*/llvm::StringRef{}, name,
- dynamicLength, dynamicShape, attrs);
+ auto heap = fir::AllocMemOp::create(*this, loc, type,
+ /*unique_name=*/llvm::StringRef{}, name,
+ dynamicLength, dynamicShape, attrs);
+ if (alignment)
+ heap.setAlignment(*alignment);
+ return heap;
}
std::pair<mlir::Value, bool> fir::FirOpBuilder::createAndDeclareTemp(
mlir::Location loc, mlir::Type baseType, mlir::Value shape,
llvm::ArrayRef<mlir::Value> extents, llvm::ArrayRef<mlir::Value> typeParams,
const std::function<decltype(FirOpBuilder::genTempDeclareOp)> &genDeclare,
- mlir::Value polymorphicMold, bool useStack, llvm::StringRef tmpName) {
+ mlir::Value polymorphicMold, bool useStack, llvm::StringRef tmpName,
+ std::optional<std::int64_t> alignment) {
if (polymorphicMold) {
// Create *allocated* polymorphic temporary using the dynamic type
// of the mold and the provided shape/extents.
@@ -444,11 +455,16 @@ std::pair<mlir::Value, bool> fir::FirOpBuilder::createAndDeclareTemp(
}
mlir::Value allocmem;
if (useStack)
- allocmem = createTemporary(loc, baseType, tmpName, extents, typeParams);
+ allocmem =
+ createTemporary(loc, baseType, tmpName, extents, typeParams,
+ /*attrs=*/{}, /*cudaAttr=*/std::nullopt, alignment);
else
- allocmem = createHeapTemporary(loc, baseType, tmpName, extents, typeParams);
+ allocmem = createHeapTemporary(loc, baseType, tmpName, extents, typeParams,
+ /*attrs=*/{}, alignment);
mlir::Value base = genDeclare(*this, loc, allocmem, tmpName, shape,
typeParams, fir::FortranVariableFlagsAttr{});
+ if (alignment)
+ hlfir::setDeclaredAlignment(base, *alignment);
return {base, !useStack};
}
diff --git a/flang/lib/Optimizer/Builder/HLFIRTools.cpp b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
index 03e5bf9c163660..51ea9e40069bef 100644
--- a/flang/lib/Optimizer/Builder/HLFIRTools.cpp
+++ b/flang/lib/Optimizer/Builder/HLFIRTools.cpp
@@ -1498,10 +1498,12 @@ hlfir::createTempFromMold(mlir::Location loc, fir::FirOpBuilder &builder,
return declareOp.getBase();
};
+ // A temporary created from an aligned variable keeps its alignment.
+ std::optional<std::int64_t> alignment = hlfir::getDeclaredAlignment(mold);
auto [base, isHeapAlloc] = builder.createAndDeclareTemp(
loc, mold.getElementOrSequenceType(), shape, extents, lenParams,
genTempDeclareOp, mold.isPolymorphic() ? mold.getBase() : nullptr,
- useStack, tmpName);
+ useStack, tmpName, alignment);
return {hlfir::Entity{base}, isHeapAlloc};
}
@@ -1514,6 +1516,8 @@ hlfir::Entity hlfir::createStackTempFromMold(mlir::Location loc,
mlir::Value alloc;
mlir::Value shape{};
fir::FortranVariableFlagsAttr declAttrs;
+ // A temporary created from an aligned variable keeps its alignment.
+ std::optional<std::int64_t> alignment = hlfir::getDeclaredAlignment(mold);
if (mold.isPolymorphic()) {
// genAllocatableApplyMold does heap allocation
@@ -1523,16 +1527,20 @@ hlfir::Entity hlfir::createStackTempFromMold(mlir::Location loc,
hlfir::getFortranElementOrSequenceType(mold.getType());
shape = hlfir::genShape(loc, builder, mold);
auto extents = hlfir::getIndexExtents(loc, builder, shape);
- alloc =
- builder.createTemporary(loc, sequenceType, tmpName, extents, lenParams);
+ alloc = builder.createTemporary(loc, sequenceType, tmpName, extents,
+ lenParams, /*attrs=*/{},
+ /*cudaAttr=*/std::nullopt, alignment);
} else {
alloc = builder.createTemporary(loc, mold.getFortranElementType(), tmpName,
- /*shape=*/{}, lenParams);
+ /*shape=*/{}, lenParams, /*attrs=*/{},
+ /*cudaAttr=*/std::nullopt, alignment);
}
auto declareOp =
hlfir::DeclareOp::create(builder, loc, alloc, tmpName, shape, lenParams,
/*dummy_scope=*/nullptr, /*storage=*/nullptr,
/*storage_offset=*/0, declAttrs);
+ if (alignment)
+ declareOp.setAlignment(*alignment);
return hlfir::Entity{declareOp.getBase()};
}
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
index 11d9bc3a3377c6..006e4f0896d2d2 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/ConvertToFIR.cpp
@@ -292,6 +292,10 @@ class CopyInOpConversion : public mlir::OpRewritePattern<hlfir::CopyInOp> {
hlfir::genLengthParameters(loc, builder, inputEntity, typeParams);
const bool useStack =
fir::shouldUseStackForCopyin(loc, sequenceType, policy, sizeContext);
+ // A buffer copied from an aligned variable keeps that variable's
+ // alignment.
+ std::optional<std::int64_t> alignment =
+ hlfir::getDeclaredAlignment(inputVariable);
mlir::Value addr =
builder
@@ -303,13 +307,17 @@ class CopyInOpConversion : public mlir::OpRewritePattern<hlfir::CopyInOp> {
llvm::StringRef tmpName{".tmp.copy_in"};
mlir::Value allocation =
useStack
- ? builder.createTemporary(loc, sequenceType, tmpName,
- extents, typeParams)
+ ? builder.createTemporary(
+ loc, sequenceType, tmpName, extents, typeParams,
+ /*attrs=*/{}, /*cudaAttr=*/std::nullopt, alignment)
: builder.createHeapTemporary(loc, sequenceType, tmpName,
- extents, typeParams);
+ extents, typeParams,
+ /*attrs=*/{}, alignment);
mlir::Value declared = fir::FirOpBuilder::genTempDeclareOp(
builder, loc, allocation, tmpName, shape, typeParams,
fir::FortranVariableFlagsAttr{});
+ if (alignment)
+ hlfir::setDeclaredAlignment(declared, *alignment);
bool useDynamicType =
fir::isBoxedRecordType(inputVariable.getType());
diff --git a/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopy.cpp b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopy.cpp
index cf401db47a86ec..9896ee0a923f4d 100644
--- a/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopy.cpp
+++ b/flang/lib/Optimizer/HLFIR/Transforms/InlineHLFIRCopy.cpp
@@ -114,6 +114,10 @@ InlineCopyInConversion::matchAndRewrite(hlfir::CopyInOp copyIn,
if (copyOut.getVar())
return rewriter.notifyMatchFailure(copyIn, "CopyIn needs a copy-out");
+ // A buffer copied from an aligned variable keeps that variable's alignment.
+ std::optional<std::int64_t> alignment =
+ hlfir::getDeclaredAlignment(copyIn.getVar());
+
inputVariable =
hlfir::derefPointersAndAllocatables(loc, builder, inputVariable);
mlir::Type sequenceType =
@@ -151,16 +155,21 @@ InlineCopyInConversion::matchAndRewrite(hlfir::CopyInOp copyIn,
llvm::StringRef tmpName{".tmp.copy_in"};
llvm::SmallVector<mlir::Value> lenParams;
mlir::Value alloc =
- useStack ? builder.createTemporary(loc, sequenceType, tmpName,
- extents, lenParams)
- : builder.createHeapTemporary(
- loc, sequenceType, tmpName, extents, lenParams);
+ useStack
+ ? builder.createTemporary(
+ loc, sequenceType, tmpName, extents, lenParams,
+ /*attrs=*/{}, /*cudaAttr=*/std::nullopt, alignment)
+ : builder.createHeapTemporary(loc, sequenceType, tmpName,
+ extents, lenParams,
+ /*attrs=*/{}, alignment);
auto declareOp = hlfir::DeclareOp::create(builder, loc, alloc,
tmpName, shape, lenParams,
/*dummy_scope=*/nullptr,
/*storage=*/nullptr,
/*storage_offset=*/0);
+ if (alignment)
+ declareOp.setAlignment(*alignment);
hlfir::Entity temp{declareOp.getBase()};
hlfir::LoopNest loopNest =
hlfir::genLoopNest(loc, builder, extents, /*isUnordered=*/true,
diff --git a/flang/test/HLFIR/as_expr-codegen.fir b/flang/test/HLFIR/as_expr-codegen.fir
index 61824c3019dbd9..f470bf28a54585 100644
--- a/flang/test/HLFIR/as_expr-codegen.fir
+++ b/flang/test/HLFIR/as_expr-codegen.fir
@@ -54,6 +54,21 @@ func.func @shape_from_type(%arg0 : !fir.ref<!fir.array<10x20xi32>>) {
// CHECK: %[[VAL_9:.*]] = fir.insert_value %[[VAL_8]], %[[VAL_6]]#0, [0 : index] : (tuple<!fir.heap<!fir.array<10x20xi32>>, i1>, !fir.heap<!fir.array<10x20xi32>>) -> tuple<!fir.heap<!fir.array<10x20xi32>>, i1>
+// The temporary copied from an aligned variable inherits its alignment, on the
+// allocation and on its own declare.
+func.func @aligned_mold(%arg0 : !fir.ref<!fir.array<10x20xi32>>) {
+ %c10 = arith.constant 10 : index
+ %c20 = arith.constant 20 : index
+ %sh = fir.shape %c10, %c20 : (index, index) -> !fir.shape<2>
+ %0:2 = hlfir.declare %arg0(%sh) uniq_name("a") alignment(64) : (!fir.ref<!fir.array<10x20xi32>>, !fir.shape<2>) -> (!fir.ref<!fir.array<10x20xi32>>, !fir.ref<!fir.array<10x20xi32>>)
+ %expr = hlfir.as_expr %0#0 : (!fir.ref<!fir.array<10x20xi32>>) -> !hlfir.expr<10x20xi32>
+ return
+}
+// CHECK-LABEL: func.func @aligned_mold(
+// CHECK: %[[VAL_4:.*]] = fir.allocmem !fir.array<10x20xi32> <{alignment = 64 : i64, bindc_name = ".tmp", uniq_name = ""}>
+// CHECK: %[[VAL_6:.*]]:2 = hlfir.declare %[[VAL_4]](%{{.*}}) uniq_name(".tmp") alignment(64) : (!fir.heap<!fir.array<10x20xi32>>, !fir.shape<2>) -> (!fir.heap<!fir.array<10x20xi32>>, !fir.heap<!fir.array<10x20xi32>>)
+// CHECK: hlfir.assign %{{.*}} to %[[VAL_6]]#0 temporary_lhs
+
func.func @shape_from_box(%arg0 : !fir.box<!fir.array<10x?xi32>>) {
%expr = hlfir.as_expr %arg0 : (!fir.box<!fir.array<10x?xi32>>) -> !hlfir.expr<10x?xi32>
return
diff --git a/flang/test/HLFIR/copy-in-out-codegen.fir b/flang/test/HLFIR/copy-in-out-codegen.fir
index c73db2c342871a..6cf31f099ccfe6 100644
--- a/flang/test/HLFIR/copy-in-out-codegen.fir
+++ b/flang/test/HLFIR/copy-in-out-codegen.fir
@@ -13,6 +13,17 @@ func.func @test_copy_in(%box: !fir.box<!fir.array<?xf64>>, %temp: !fir.ref<!fir.
// CHECK: fir.store %{{.*}} to %{{.*}} : !fir.ref<!fir.box<!fir.array<?xf64>>>
// CHECK-NOT: fir.call @_FortranACopyInAssign
+// The buffer copied from an aligned variable carries its alignment.
+func.func @test_copy_in_aligned(%box: !fir.box<!fir.array<?xf64>>, %temp: !fir.ref<!fir.box<!fir.array<?xf64>>>) {
+ %decl = fir.declare %box uniq_name("a") alignment(64) : (!fir.box<!fir.array<?xf64>>) -> !fir.box<!fir.array<?xf64>>
+ %0:3 = hlfir.copy_in %decl to %temp : (!fir.box<!fir.array<?xf64>>, !fir.ref<!fir.box<!fir.array<?xf64>>>) -> (!fir.box<!fir.array<?xf64>>, i1, i1)
+ return
+}
+// CHECK-LABEL: func.func @test_copy_in_aligned(
+// CHECK: %[[BUF:.*]] = fir.allocmem !fir.array<?xf64>, %{{.*}} <{alignment = 64 : i64, bindc_name = ".tmp.copy_in"
+// CHECK: fir.declare %[[BUF]]({{.*}}) uniq_name(".tmp.copy_in") alignment(64)
+// CHECK: fir.call @_FortranAShallowCopyDirect
+
func.func @test_copy_in_poly_pair(%poly : !fir.class<!fir.array<?x!fir.type<test_copy_in_poly_pairTt1{i:i32}>>>, %temp: !fir.ref<!fir.class<!fir.array<?x!fir.type<test_copy_in_poly_pairTt1{i:i32}>>>>) {
%0:3 = hlfir.copy_in %poly to %temp : (!fir.class<!fir.array<?x!fir.type<test_copy_in_poly_pairTt1{i:i32}>>>, !fir.ref<!fir.class<!fir.array<?x!fir.type<test_copy_in_poly_pairTt1{i:i32}>>>>) -> (!fir.class<!fir.array<?x!fir.type<test_copy_in_poly_pairTt1{i:i32}>>>, i1, i1)
hlfir.copy_out %temp, %0#1, %0#2 : (!fir.ref<!fir.class<!fir.array<?x!fir.type<test_copy_in_poly_pairTt1{i:i32}>>>>, i1, i1) -> ()
diff --git a/flang/test/HLFIR/inline-hlfir-copy-stack.fir b/flang/test/HLFIR/inline-hlfir-copy-stack.fir
index cf3a07608bd496..d1d030ef4b34ea 100644
--- a/flang/test/HLFIR/inline-hlfir-copy-stack.fir
+++ b/flang/test/HLFIR/inline-hlfir-copy-stack.fir
@@ -46,6 +46,26 @@ func.func @big_constant_buffer(%arg0: !fir.box<!fir.array<?x?xf64>>, %arg1: !fir
return
}
+// A section of an aligned variable: the buffer inherits the alignment, both
+// on the stack and on the heap.
+func.func @aligned_source(%arg0: !fir.ref<!fir.array<20x20xf64>>, %arg1: !fir.ref<i32>) {
+ %tmpBox = fir.alloca !fir.box<!fir.array<8xf64>>
+ %c1 = arith.constant 1 : index
+ %c8 = arith.constant 8 : index
+ %c20 = arith.constant 20 : index
+ %sh = fir.shape %c20, %c20 : (index, index) -> !fir.shape<2>
+ %0:2 = hlfir.declare %arg0(%sh) uniq_name("a") alignment(64) : (!fir.ref<!fir.array<20x20xf64>>, !fir.shape<2>) -> (!fir.ref<!fir.array<20x20xf64>>, !fir.ref<!fir.array<20x20xf64>>)
+ %1 = fir.load %arg1 : !fir.ref<i32>
+ %2 = fir.convert %1 : (i32) -> i64
+ %3 = fir.shape %c8 : (index) -> !fir.shape<1>
+ %4 = hlfir.designate %0#0 (%2, %c1:%c8:%c1) shape %3 : (!fir.ref<!fir.array<20x20xf64>>, i64, index, index, index, !fir.shape<1>) -> !fir.box<!fir.array<8xf64>>
+ %5:3 = hlfir.copy_in %4 to %tmpBox : (!fir.box<!fir.array<8xf64>>, !fir.ref<!fir.box<!fir.array<8xf64>>>) -> (!fir.box<!fir.array<8xf64>>, i1, i1)
+ %6 = fir.box_addr %5#0 : (!fir.box<!fir.array<8xf64>>) -> !fir.ref<!fir.array<8xf64>>
+ fir.call @callee8(%6) : (!fir.ref<!fir.array<8xf64>>) -> ()
+ hlfir.copy_out %tmpBox, %5#1, %5#2 : (!fir.ref<!fir.box<!fir.array<8xf64>>>, i1, i1) -> ()
+ return
+}
+
// Runtime extent: the buffer stays on the heap whatever the threshold is,
// because a dynamically sized fir.alloca cannot be hoisted out of the copy-in
// branch and would grow the stack if the copy-in were inside a loop.
@@ -103,6 +123,16 @@ func.func @dynamic_buffer(%arg0: !fir.box<!fir.array<?x?xf64>>, %arg1: !fir.ref<
// CHECK: %[[FREE_HEAP:.*]] = fir.convert %[[FREE_ADDR]]
// CHECK: fir.freemem %[[FREE_HEAP]]
+// The buffer copied from a section of an aligned variable carries the
+// variable's alignment, on the allocation and on its declare.
+
+// CHECK-LABEL: func.func @aligned_source(
+// CHECK: %[[BUF:.*]] = fir.alloca !fir.array<8xf64> <{alignment = 64 : i64, bindc_name = ".tmp.copy_in"}>
+// CHECK: } else {
+// CHECK: %[[DECL:.*]]:2 = hlfir.declare %[[BUF]]({{.*}}) uniq_name(".tmp.copy_in") alignment(64)
+// CHECK: fir.call @callee8
+// CHECK-NOT: fir.allocmem
+
// A runtime extent always stays on the heap.
// CHECK-LABEL: func.func @dynamic_buffer(
@@ -125,6 +155,9 @@ func.func @dynamic_buffer(%arg0: !fir.box<!fir.array<?x?xf64>>, %arg1: !fir.ref<
// BIGTHRESHOLD-NOT: fir.allocmem
// BIGTHRESHOLD-NOT: fir.freemem
+// BIGTHRESHOLD-LABEL: func.func @aligned_source(
+// BIGTHRESHOLD: fir.alloca !fir.array<8xf64> <{alignment = 64 : i64, bindc_name = ".tmp.copy_in"}>
+
// BIGTHRESHOLD-LABEL: func.func @dynamic_buffer(
// BIGTHRESHOLD: fir.allocmem !fir.array<?xf64>, %{{.*}} <{bindc_name = ".tmp.copy_in"
// BIGTHRESHOLD: fir.freemem
More information about the flang-commits
mailing list