[flang-commits] [flang] [llvm] [flang-rt] Copy out only the modified suffix of an argument temporary (PR #222101)
Eugene Epshteyn via flang-commits
flang-commits at lists.llvm.org
Tue Sep 15 12:03:39 PDT 2026
https://github.com/eugeneepshteyn updated https://github.com/llvm/llvm-project/pull/222101
>From 5abbfcd8f41dc264f5497dd4afb7b20f1f0e8626 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 8 Sep 2026 11:22:12 -0700
Subject: [PATCH 1/6] [flang-rt] Copy back only modified elements in
CopyOutAssign
CopyOutAssign() used to copy the whole temporary back into the original
variable unconditionally. Since CopyInAssign() creates the temporary as a
bitwise copy of the variable, an element the callee never modified is
still bitwise-identical to the original, and storing it back is
unnecessary.
Compare each element bitwise and store only the elements that differ.
This avoids compiler-generated stores into storage that must not be
written -- e.g. an actual argument that is not definable and was,
conformingly, never modified by the callee -- which is a prerequisite for
passing non-definable actual arguments (such as named constants) without
defensive temporaries, as discussed in #187348.
The comparison is bitwise (memcmp), not a value comparison, so unmodified
NaN elements and derived-type padding do not produce spurious stores. The
helpers mirror the ShallowCopy* type/rank specialization structure so the
per-element comparison and copy inline to fixed-size operations.
---
.../include/flang-rt/runtime/environment.h | 4 +
flang-rt/include/flang-rt/runtime/tools.h | 11 ++
flang-rt/lib/runtime/assign.cpp | 15 ++-
flang-rt/lib/runtime/environment.cpp | 13 ++
flang-rt/lib/runtime/tools.cpp | 122 ++++++++++++++++++
flang-rt/unittests/Runtime/Assign.cpp | 122 ++++++++++++++++++
flang/docs/RuntimeEnvironment.md | 11 ++
7 files changed, 297 insertions(+), 1 deletion(-)
diff --git a/flang-rt/include/flang-rt/runtime/environment.h b/flang-rt/include/flang-rt/runtime/environment.h
index 8ae441c12b3f8..a4dabb2ca20fb 100644
--- a/flang-rt/include/flang-rt/runtime/environment.h
+++ b/flang-rt/include/flang-rt/runtime/environment.h
@@ -74,6 +74,10 @@ struct ExecutionEnvironment {
bool checkPointerDeallocation{true}; // FORT_CHECK_POINTER_DEALLOCATION
bool truncateStream{true}; // FORT_TRUNCATE_STREAM
bool noEmptyAllocation{false}; // FORT_NO_EMPTY_ALLOCATION
+ // FLANG_RT_COPYOUT_MODIFIED_ONLY=0 restores the unconditional copy-out
+ // of argument temporaries (CopyOutAssign copies every element back
+ // instead of only the elements the callee modified).
+ bool copyOutModifiedOnly{true}; // FLANG_RT_COPYOUT_MODIFIED_ONLY
enum InternalDebugging { WorkQueue = 1 };
int internalDebugging{0}; // FLANG_RT_DEBUG
diff --git a/flang-rt/include/flang-rt/runtime/tools.h b/flang-rt/include/flang-rt/runtime/tools.h
index a45c2ac98f2fa..411e5b2221702 100644
--- a/flang-rt/include/flang-rt/runtime/tools.h
+++ b/flang-rt/include/flang-rt/runtime/tools.h
@@ -525,6 +525,17 @@ RT_API_ATTRS void ShallowCopy(const Descriptor &to, const Descriptor &from,
bool toIsContiguous, bool fromIsContiguous);
RT_API_ATTRS void ShallowCopy(const Descriptor &to, const Descriptor &from);
+// Copies payload data like ShallowCopy(), but stores only to the elements of
+// 'to' whose bit patterns differ from the corresponding elements of 'from'.
+// Used by copy-out to avoid stores when the callee never modified the data,
+// so that an original that lives in read-only memory (e.g. a named constant)
+// is not written to by an unmodifying copy-out. The comparison is bitwise, so
+// it is exact when 'from' was originally produced from 'to' by ShallowCopy()
+// (as CopyInAssign() does): unmodified elements compare equal even for NaNs
+// and padding bytes, which a value comparison would misjudge.
+RT_API_ATTRS void ShallowCopyModifiedElements(
+ const Descriptor &to, const Descriptor &from);
+
// Ensures that a character string is null-terminated, allocating a /p length +1
// size memory for null-terminator if necessary. Returns the original or a newly
// allocated null-terminated string (responsibility for deallocation is on the
diff --git a/flang-rt/lib/runtime/assign.cpp b/flang-rt/lib/runtime/assign.cpp
index 2fc584d2569e7..1f7e7c688875f 100644
--- a/flang-rt/lib/runtime/assign.cpp
+++ b/flang-rt/lib/runtime/assign.cpp
@@ -10,6 +10,7 @@
#include "flang-rt/runtime/assign-impl.h"
#include "flang-rt/runtime/derived.h"
#include "flang-rt/runtime/descriptor.h"
+#include "flang-rt/runtime/environment.h"
#include "flang-rt/runtime/memory.h"
#include "flang-rt/runtime/stat.h"
#include "flang-rt/runtime/terminator.h"
@@ -838,8 +839,20 @@ void RTDEF(CopyOutAssign)(
Terminator terminator{sourceFile, sourceLine};
// Copyout from the temporary must not cause any finalizations
// for LHS. The variable must be properly initialized already.
+ // Copy back only the elements that were modified through the temporary:
+ // the temporary was created as a bitwise copy of the variable (see
+ // CopyInAssign above), so an element the callee never assigned is still
+ // bit-identical to the original and must not be stored to. This keeps a
+ // compiler-generated copy-out from writing into read-only storage when the
+ // effective argument is not definable (e.g., a named constant) and the
+ // callee, conformingly, never modified it.
+ // FLANG_RT_COPYOUT_MODIFIED_ONLY=0 restores the unconditional copy-out.
if (var) {
- ShallowCopy(*var, temp);
+ if (executionEnvironment.copyOutModifiedOnly) {
+ ShallowCopyModifiedElements(*var, temp);
+ } else {
+ ShallowCopy(*var, temp);
+ }
}
temp.Deallocate();
}
diff --git a/flang-rt/lib/runtime/environment.cpp b/flang-rt/lib/runtime/environment.cpp
index ea98eb4634293..fc19c8c3973e7 100644
--- a/flang-rt/lib/runtime/environment.cpp
+++ b/flang-rt/lib/runtime/environment.cpp
@@ -205,6 +205,19 @@ void ExecutionEnvironment::Configure(int ac, const char *av[],
}
}
+ if (auto *x{std::getenv("FLANG_RT_COPYOUT_MODIFIED_ONLY")}) {
+ char *end;
+ auto n{std::strtol(x, &end, 10)};
+ if (n >= 0 && n <= 1 && *end == '\0') {
+ copyOutModifiedOnly = n != 0;
+ } else {
+ std::fprintf(stderr,
+ "Fortran runtime: FLANG_RT_COPYOUT_MODIFIED_ONLY=%s is invalid; "
+ "ignored\n",
+ x);
+ }
+ }
+
if (auto *x{std::getenv("FLANG_RT_DEBUG")}) {
internalDebugging = std::strtol(x, nullptr, 10);
}
diff --git a/flang-rt/lib/runtime/tools.cpp b/flang-rt/lib/runtime/tools.cpp
index b7408c81f83f4..56890770c253f 100644
--- a/flang-rt/lib/runtime/tools.cpp
+++ b/flang-rt/lib/runtime/tools.cpp
@@ -271,6 +271,128 @@ RT_API_ATTRS void ShallowCopy(const Descriptor &to, const Descriptor &from) {
ShallowCopy(to, from, to.IsContiguous(), from.IsContiguous());
}
+// Compares one element bitwise and copies it only when it differs. As in the
+// ShallowCopy* helpers above, the compile-time element size lets the compiler
+// inline both the comparison and the copy.
+template <typename P>
+static inline RT_API_ATTRS void CopyElementIfModified(
+ char *toAt, const char *fromAt, std::size_t elementBytes) {
+ constexpr std::size_t typeElementBytes{sizeof(P)};
+ if constexpr (typeElementBytes != 1) {
+ if (runtime::memcmp(toAt, fromAt, typeElementBytes) != 0) {
+ runtime::memcpy(toAt, fromAt, typeElementBytes);
+ }
+ } else {
+ if (runtime::memcmp(toAt, fromAt, elementBytes) != 0) {
+ runtime::memcpy(toAt, fromAt, elementBytes);
+ }
+ }
+}
+
+template <typename P, int RANK = -1>
+static RT_API_ATTRS void ShallowCopyModifiedInner(const Descriptor &to,
+ const Descriptor &from, bool toIsContiguous, bool fromIsContiguous) {
+ std::size_t elementBytes{to.ElementBytes()};
+ if (toIsContiguous) {
+ char *toAt{to.OffsetElement()};
+ if (fromIsContiguous) {
+ const char *fromAt{from.OffsetElement()};
+ for (std::size_t n{to.Elements()}; n-- > 0;
+ toAt += elementBytes, fromAt += elementBytes) {
+ CopyElementIfModified<P>(toAt, fromAt, elementBytes);
+ }
+ } else {
+ DescriptorIterator<RANK> fromIt{from};
+ for (std::size_t n{to.Elements()}; n-- > 0;
+ toAt += elementBytes, fromIt.Advance()) {
+ CopyElementIfModified<P>(
+ toAt, fromIt.template Get<char>(), elementBytes);
+ }
+ }
+ } else {
+ DescriptorIterator<RANK> toIt{to};
+ if (fromIsContiguous) {
+ const char *fromAt{from.OffsetElement()};
+ for (std::size_t n{to.Elements()}; n-- > 0;
+ toIt.Advance(), fromAt += elementBytes) {
+ CopyElementIfModified<P>(
+ toIt.template Get<char>(), fromAt, elementBytes);
+ }
+ } else {
+ DescriptorIterator<RANK> fromIt{from};
+ for (std::size_t n{to.Elements()}; n-- > 0;
+ toIt.Advance(), fromIt.Advance()) {
+ CopyElementIfModified<P>(toIt.template Get<char>(),
+ fromIt.template Get<char>(), elementBytes);
+ }
+ }
+ }
+}
+
+template <typename P>
+static RT_API_ATTRS void ShallowCopyModifiedRank(const Descriptor &to,
+ const Descriptor &from, bool toIsContiguous, bool fromIsContiguous) {
+ INTERNAL_CHECK(to.rank() == from.rank());
+ // Mirror ShallowCopyRank's rank specialization policy.
+ switch (to.rank()) {
+ case 1:
+ ShallowCopyModifiedInner<P, 1>(to, from, toIsContiguous, fromIsContiguous);
+ return;
+ case 2:
+ ShallowCopyModifiedInner<P, 2>(to, from, toIsContiguous, fromIsContiguous);
+ return;
+ case 3:
+ ShallowCopyModifiedInner<P, 3>(to, from, toIsContiguous, fromIsContiguous);
+ return;
+ case 4:
+ ShallowCopyModifiedInner<P, 4>(to, from, toIsContiguous, fromIsContiguous);
+ return;
+ default:
+ ShallowCopyModifiedInner<P>(to, from, toIsContiguous, fromIsContiguous);
+ return;
+ }
+}
+
+RT_API_ATTRS void ShallowCopyModifiedElements(
+ const Descriptor &to, const Descriptor &from) {
+ bool toIsContiguous{to.IsContiguous()};
+ bool fromIsContiguous{from.IsContiguous()};
+ std::size_t elementBytes{to.ElementBytes()};
+ // Same type-based dispatch as ShallowCopy() above, so the per-element
+ // comparison and copy inline to fixed-size operations.
+ if (to.type().IsInteger()) {
+ if (elementBytes == sizeof(int64_t)) {
+ ShallowCopyModifiedRank<int64_t>(
+ to, from, toIsContiguous, fromIsContiguous);
+ } else if (elementBytes == sizeof(int32_t)) {
+ ShallowCopyModifiedRank<int32_t>(
+ to, from, toIsContiguous, fromIsContiguous);
+ } else if (elementBytes == sizeof(int16_t)) {
+ ShallowCopyModifiedRank<int16_t>(
+ to, from, toIsContiguous, fromIsContiguous);
+#if defined USING_NATIVE_INT128_T
+ } else if (elementBytes == sizeof(__int128_t)) {
+ ShallowCopyModifiedRank<__int128_t>(
+ to, from, toIsContiguous, fromIsContiguous);
+#endif
+ } else {
+ ShallowCopyModifiedRank<char>(to, from, toIsContiguous, fromIsContiguous);
+ }
+ } else if (to.type().IsReal()) {
+ if (elementBytes == sizeof(double)) {
+ ShallowCopyModifiedRank<double>(
+ to, from, toIsContiguous, fromIsContiguous);
+ } else if (elementBytes == sizeof(float)) {
+ ShallowCopyModifiedRank<float>(
+ to, from, toIsContiguous, fromIsContiguous);
+ } else {
+ ShallowCopyModifiedRank<char>(to, from, toIsContiguous, fromIsContiguous);
+ }
+ } else {
+ ShallowCopyModifiedRank<char>(to, from, toIsContiguous, fromIsContiguous);
+ }
+}
+
RT_API_ATTRS char *EnsureNullTerminated(
char *str, std::size_t length, Terminator &terminator) {
if (runtime::memchr(str, '\0', length) == nullptr) {
diff --git a/flang-rt/unittests/Runtime/Assign.cpp b/flang-rt/unittests/Runtime/Assign.cpp
index 8e9b89d7f661f..e9b8de52ffde2 100644
--- a/flang-rt/unittests/Runtime/Assign.cpp
+++ b/flang-rt/unittests/Runtime/Assign.cpp
@@ -10,7 +10,14 @@
#include "CrashHandlerFixture.h"
#include "tools.h"
#include "gtest/gtest.h"
+#include "flang-rt/runtime/environment.h"
+#include <cstdint>
+#include <cstring>
#include <vector>
+#if defined(__unix__) || defined(__APPLE__)
+#include <sys/mman.h>
+#include <unistd.h>
+#endif
using namespace Fortran::runtime;
using Fortran::common::TypeCategory;
@@ -446,3 +453,118 @@ TEST(AssignSimpleCrash, NonAllocatableElementCountMismatch) {
ASSERT_DEATH(RTNAME(AssignSimple)(dest, source, __FILE__, __LINE__),
"AssignSimple: mismatching element counts");
}
+
+TEST(Assign, RTNAME(CopyOutAssign)) {
+ // Copy-out writes back the elements the callee modified through the
+ // temporary and performs no stores for the elements it never touched.
+ // Discontiguous var: stride-2 view (elements 1,3,5,7) of an 8-element
+ // backing array, as copy-in/copy-out creates for a non-contiguous actual.
+ int data[8] = {1, 2, 3, 4, 5, 6, 7, 8};
+ TypeCode intType{TypeCategory::Integer, 4};
+ StaticDescriptor<1> staticVar;
+ Descriptor &var{staticVar.descriptor()};
+ SubscriptValue extent[1]{4};
+ var.Establish(intType, sizeof(int), data, 1, extent);
+ var.GetDimension(0).SetLowerBound(1);
+ var.GetDimension(0).SetByteStride(sizeof(int) * 2);
+
+ StaticDescriptor<1> staticTemp;
+ Descriptor &temp{staticTemp.descriptor()};
+ RTNAME(CopyInAssign)(temp, var, __FILE__, __LINE__);
+ ASSERT_TRUE(temp.IsAllocated());
+ ASSERT_TRUE(temp.IsContiguous());
+
+ // The "callee" modifies the first and third elements of the temporary.
+ *temp.OffsetElement<int>(0 * sizeof(int)) = 100;
+ *temp.OffsetElement<int>(2 * sizeof(int)) = 300;
+
+ RTNAME(CopyOutAssign)(&var, temp, __FILE__, __LINE__);
+
+ int expected[8] = {100, 2, 3, 4, 300, 6, 7, 8};
+ EXPECT_EQ(std::memcmp(data, expected, 8 * sizeof(int)), 0);
+}
+
+#if defined(__unix__) || defined(__APPLE__)
+TEST(Assign, RTNAME(CopyOutAssignReadOnlyUnmodified)) {
+ // An unmodified copy-out must perform no stores at all: the original may
+ // live in read-only memory (e.g. a named constant's storage). The array
+ // includes a NaN element to verify that the comparison is bitwise — a
+ // value comparison would consider the unmodified NaN element "changed"
+ // and store to it, faulting on the read-only page.
+ std::size_t pageSize{static_cast<std::size_t>(sysconf(_SC_PAGESIZE))};
+ void *page{mmap(nullptr, pageSize, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)};
+ ASSERT_NE(page, MAP_FAILED);
+ double *data{static_cast<double *>(page)};
+ for (int j{0}; j < 8; ++j) {
+ data[j] = j + 1;
+ }
+ std::uint64_t quietNaN{0x7FF8000000000000ULL};
+ std::memcpy(&data[2], &quietNaN, sizeof(double));
+ ASSERT_EQ(mprotect(page, pageSize, PROT_READ), 0);
+
+ // Discontiguous read-only var: stride-2 view (elements 1,NaN,5,7).
+ StaticDescriptor<1> staticVar;
+ Descriptor &var{staticVar.descriptor()};
+ SubscriptValue extent[1]{4};
+ var.Establish(
+ TypeCode{TypeCategory::Real, 8}, sizeof(double), data, 1, extent);
+ var.GetDimension(0).SetLowerBound(1);
+ var.GetDimension(0).SetByteStride(sizeof(double) * 2);
+
+ StaticDescriptor<1> staticTemp;
+ Descriptor &temp{staticTemp.descriptor()};
+ RTNAME(CopyInAssign)(temp, var, __FILE__, __LINE__);
+ ASSERT_TRUE(temp.IsAllocated());
+
+ // The "callee" only reads the temporary; copying out into the read-only
+ // original must not fault.
+ RTNAME(CopyOutAssign)(&var, temp, __FILE__, __LINE__);
+
+ std::uint64_t elem2Bits;
+ std::memcpy(&elem2Bits, &data[2], sizeof(double));
+ EXPECT_EQ(elem2Bits, quietNaN);
+ EXPECT_EQ(data[0], 1.0);
+ EXPECT_EQ(data[4], 5.0);
+ ASSERT_EQ(munmap(page, pageSize), 0);
+}
+#endif
+
+#if defined(__unix__) || defined(__APPLE__)
+TEST(Assign, RTNAME(CopyOutAssignUnconditionalEnvVar)) {
+ // With FLANG_RT_COPYOUT_MODIFIED_ONLY=0 semantics (unconditional copy-out),
+ // even an unmodified copy-out stores every element, so a read-only original
+ // faults. This proves the environment control selects the legacy path.
+ std::size_t pageSize{static_cast<std::size_t>(sysconf(_SC_PAGESIZE))};
+ void *page{mmap(nullptr, pageSize, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)};
+ ASSERT_NE(page, MAP_FAILED);
+ double *data{static_cast<double *>(page)};
+ for (int j{0}; j < 8; ++j) {
+ data[j] = j + 1;
+ }
+ ASSERT_EQ(mprotect(page, pageSize, PROT_READ), 0);
+
+ StaticDescriptor<1> staticVar;
+ Descriptor &var{staticVar.descriptor()};
+ SubscriptValue extent[1]{4};
+ var.Establish(
+ TypeCode{TypeCategory::Real, 8}, sizeof(double), data, 1, extent);
+ var.GetDimension(0).SetLowerBound(1);
+ var.GetDimension(0).SetByteStride(sizeof(double) * 2);
+
+ StaticDescriptor<1> staticTemp;
+ Descriptor &temp{staticTemp.descriptor()};
+ RTNAME(CopyInAssign)(temp, var, __FILE__, __LINE__);
+ ASSERT_TRUE(temp.IsAllocated());
+
+ executionEnvironment.copyOutModifiedOnly = false;
+ EXPECT_DEATH(RTNAME(CopyOutAssign)(&var, temp, __FILE__, __LINE__), "");
+ executionEnvironment.copyOutModifiedOnly = true;
+
+ // The parent's temp is still allocated (the death happened in the child);
+ // clean it up through the default path.
+ RTNAME(CopyOutAssign)(&var, temp, __FILE__, __LINE__);
+ ASSERT_EQ(munmap(page, pageSize), 0);
+}
+#endif
diff --git a/flang/docs/RuntimeEnvironment.md b/flang/docs/RuntimeEnvironment.md
index 0414ea1e0d59c..a24da8e6a6a62 100644
--- a/flang/docs/RuntimeEnvironment.md
+++ b/flang/docs/RuntimeEnvironment.md
@@ -34,6 +34,17 @@ Determines data conversions applied to unformatted I/O.
* `BIG_ENDIAN`: assume input is big-endian; emit big-endian output
* `SWAP`: reverse endianness (always convert)
+## `FLANG_RT_COPYOUT_MODIFIED_ONLY`
+
+When the compiler passes a copy of an actual argument to a procedure
+(copy-in/copy-out), the runtime copies back only the elements whose bit
+patterns were changed through the temporary copy. This avoids stores to
+the original argument when the callee never modified the data -- in
+particular, stores into read-only storage backing a non-definable actual
+argument.
+Set `FLANG_RT_COPYOUT_MODIFIED_ONLY=0` to restore the unconditional
+copy-out of every element.
+
## `FORT_CHECK_POINTER_DEALLOCATION`
Fortran requires that a pointer that appears in a `DEALLOCATE` statement
>From 56179cfbdec339a9f27d60471502e82dbb8d6931 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Wed, 9 Sep 2026 08:21:34 -0700
Subject: [PATCH 2/6] [flang-rt] Fall back to whole-object copy-out when the
temporary was modified
Benchmarking the per-element compare-and-store across modification
percentages showed a large penalty on partially modified data: with 50%
of the elements modified at random positions, the per-element conditional
store is an unpredictable branch, measuring up to 3.6x the cost of the
unconditional copy on x86-64 and 2.7x on AArch64, while 0% and 100% were
near parity.
Replace it with an all-or-nothing scheme: scan for the first bitwise
difference between the temporary and the variable; when there is none,
store nothing (the no-store guarantee for unmodified copy-out is
unchanged); when any element was modified, perform the plain whole-object
ShallowCopy. A modified temporary implies the variable is legally
definable, and storing unmodified elements alongside modified ones is
exactly the pre-existing behavior, so this changes nothing observable for
correct programs. With the fallback, copy-out measures at parity with the
unconditional copy at every modification percentage, and the unmodified
case keeps its no-store property.
This also replaces the ShallowCopyModifiedElements template family with a
smaller ElementsBitwiseEqual scan (early exit at the first difference),
and adds a death test documenting that a modified temporary copied out
into read-only storage still faults.
---
flang-rt/include/flang-rt/runtime/tools.h | 18 ++---
flang-rt/lib/runtime/assign.cpp | 19 +++--
flang-rt/lib/runtime/tools.cpp | 91 ++++++++++++-----------
flang-rt/unittests/Runtime/Assign.cpp | 40 +++++++++-
flang/docs/RuntimeEnvironment.md | 9 ++-
5 files changed, 113 insertions(+), 64 deletions(-)
diff --git a/flang-rt/include/flang-rt/runtime/tools.h b/flang-rt/include/flang-rt/runtime/tools.h
index 411e5b2221702..6038157369f3c 100644
--- a/flang-rt/include/flang-rt/runtime/tools.h
+++ b/flang-rt/include/flang-rt/runtime/tools.h
@@ -525,15 +525,15 @@ RT_API_ATTRS void ShallowCopy(const Descriptor &to, const Descriptor &from,
bool toIsContiguous, bool fromIsContiguous);
RT_API_ATTRS void ShallowCopy(const Descriptor &to, const Descriptor &from);
-// Copies payload data like ShallowCopy(), but stores only to the elements of
-// 'to' whose bit patterns differ from the corresponding elements of 'from'.
-// Used by copy-out to avoid stores when the callee never modified the data,
-// so that an original that lives in read-only memory (e.g. a named constant)
-// is not written to by an unmodifying copy-out. The comparison is bitwise, so
-// it is exact when 'from' was originally produced from 'to' by ShallowCopy()
-// (as CopyInAssign() does): unmodified elements compare equal even for NaNs
-// and padding bytes, which a value comparison would misjudge.
-RT_API_ATTRS void ShallowCopyModifiedElements(
+// Returns true when every element of 'from' is bitwise-identical to the
+// corresponding element of 'to' (returns false at the first difference).
+// Used by copy-out to skip all stores when the callee never modified the
+// data, so that an original that lives in read-only memory (e.g. a named
+// constant) is not written to by an unmodifying copy-out. The comparison is
+// bitwise, so it is exact when 'from' was originally produced from 'to' by
+// ShallowCopy() (as CopyInAssign() does): unmodified elements compare equal
+// even for NaNs and padding bytes, which a value comparison would misjudge.
+RT_API_ATTRS bool ElementsBitwiseEqual(
const Descriptor &to, const Descriptor &from);
// Ensures that a character string is null-terminated, allocating a /p length +1
diff --git a/flang-rt/lib/runtime/assign.cpp b/flang-rt/lib/runtime/assign.cpp
index 1f7e7c688875f..0377c00c24f32 100644
--- a/flang-rt/lib/runtime/assign.cpp
+++ b/flang-rt/lib/runtime/assign.cpp
@@ -839,18 +839,21 @@ void RTDEF(CopyOutAssign)(
Terminator terminator{sourceFile, sourceLine};
// Copyout from the temporary must not cause any finalizations
// for LHS. The variable must be properly initialized already.
- // Copy back only the elements that were modified through the temporary:
- // the temporary was created as a bitwise copy of the variable (see
- // CopyInAssign above), so an element the callee never assigned is still
- // bit-identical to the original and must not be stored to. This keeps a
+ // Skip the copy-out entirely when the temporary is still bitwise-identical
+ // to the variable: the temporary was created as a bitwise copy (see
+ // CopyInAssign above), so it can only differ if the callee modified it, and
+ // an unmodifying copy-out must not store at all. This keeps a
// compiler-generated copy-out from writing into read-only storage when the
// effective argument is not definable (e.g., a named constant) and the
- // callee, conformingly, never modified it.
+ // callee, conformingly, never modified it. When any element was modified,
+ // fall back to the plain whole-object copy: a per-element conditional
+ // store measures far slower on partially-modified data (branch
+ // misprediction), and a modified temporary means the variable is legally
+ // writable anyway.
// FLANG_RT_COPYOUT_MODIFIED_ONLY=0 restores the unconditional copy-out.
if (var) {
- if (executionEnvironment.copyOutModifiedOnly) {
- ShallowCopyModifiedElements(*var, temp);
- } else {
+ if (!executionEnvironment.copyOutModifiedOnly ||
+ !ElementsBitwiseEqual(*var, temp)) {
ShallowCopy(*var, temp);
}
}
diff --git a/flang-rt/lib/runtime/tools.cpp b/flang-rt/lib/runtime/tools.cpp
index 56890770c253f..2cc4a9da9d5bb 100644
--- a/flang-rt/lib/runtime/tools.cpp
+++ b/flang-rt/lib/runtime/tools.cpp
@@ -271,42 +271,41 @@ RT_API_ATTRS void ShallowCopy(const Descriptor &to, const Descriptor &from) {
ShallowCopy(to, from, to.IsContiguous(), from.IsContiguous());
}
-// Compares one element bitwise and copies it only when it differs. As in the
-// ShallowCopy* helpers above, the compile-time element size lets the compiler
-// inline both the comparison and the copy.
+// Compares one element bitwise. As in the ShallowCopy* helpers above, the
+// compile-time element size lets the compiler inline the comparison.
template <typename P>
-static inline RT_API_ATTRS void CopyElementIfModified(
- char *toAt, const char *fromAt, std::size_t elementBytes) {
+static inline RT_API_ATTRS bool ElementIsModified(
+ const char *toAt, const char *fromAt, std::size_t elementBytes) {
constexpr std::size_t typeElementBytes{sizeof(P)};
if constexpr (typeElementBytes != 1) {
- if (runtime::memcmp(toAt, fromAt, typeElementBytes) != 0) {
- runtime::memcpy(toAt, fromAt, typeElementBytes);
- }
+ return runtime::memcmp(toAt, fromAt, typeElementBytes) != 0;
} else {
- if (runtime::memcmp(toAt, fromAt, elementBytes) != 0) {
- runtime::memcpy(toAt, fromAt, elementBytes);
- }
+ return runtime::memcmp(toAt, fromAt, elementBytes) != 0;
}
}
template <typename P, int RANK = -1>
-static RT_API_ATTRS void ShallowCopyModifiedInner(const Descriptor &to,
+static RT_API_ATTRS bool ElementsBitwiseEqualInner(const Descriptor &to,
const Descriptor &from, bool toIsContiguous, bool fromIsContiguous) {
std::size_t elementBytes{to.ElementBytes()};
if (toIsContiguous) {
- char *toAt{to.OffsetElement()};
+ const char *toAt{to.OffsetElement()};
if (fromIsContiguous) {
const char *fromAt{from.OffsetElement()};
for (std::size_t n{to.Elements()}; n-- > 0;
toAt += elementBytes, fromAt += elementBytes) {
- CopyElementIfModified<P>(toAt, fromAt, elementBytes);
+ if (ElementIsModified<P>(toAt, fromAt, elementBytes)) {
+ return false;
+ }
}
} else {
DescriptorIterator<RANK> fromIt{from};
for (std::size_t n{to.Elements()}; n-- > 0;
toAt += elementBytes, fromIt.Advance()) {
- CopyElementIfModified<P>(
- toAt, fromIt.template Get<char>(), elementBytes);
+ if (ElementIsModified<P>(
+ toAt, fromIt.template Get<char>(), elementBytes)) {
+ return false;
+ }
}
}
} else {
@@ -315,81 +314,89 @@ static RT_API_ATTRS void ShallowCopyModifiedInner(const Descriptor &to,
const char *fromAt{from.OffsetElement()};
for (std::size_t n{to.Elements()}; n-- > 0;
toIt.Advance(), fromAt += elementBytes) {
- CopyElementIfModified<P>(
- toIt.template Get<char>(), fromAt, elementBytes);
+ if (ElementIsModified<P>(
+ toIt.template Get<char>(), fromAt, elementBytes)) {
+ return false;
+ }
}
} else {
DescriptorIterator<RANK> fromIt{from};
for (std::size_t n{to.Elements()}; n-- > 0;
toIt.Advance(), fromIt.Advance()) {
- CopyElementIfModified<P>(toIt.template Get<char>(),
- fromIt.template Get<char>(), elementBytes);
+ if (ElementIsModified<P>(toIt.template Get<char>(),
+ fromIt.template Get<char>(), elementBytes)) {
+ return false;
+ }
}
}
}
+ return true;
}
template <typename P>
-static RT_API_ATTRS void ShallowCopyModifiedRank(const Descriptor &to,
+static RT_API_ATTRS bool ElementsBitwiseEqualRank(const Descriptor &to,
const Descriptor &from, bool toIsContiguous, bool fromIsContiguous) {
INTERNAL_CHECK(to.rank() == from.rank());
// Mirror ShallowCopyRank's rank specialization policy.
switch (to.rank()) {
case 1:
- ShallowCopyModifiedInner<P, 1>(to, from, toIsContiguous, fromIsContiguous);
- return;
+ return ElementsBitwiseEqualInner<P, 1>(
+ to, from, toIsContiguous, fromIsContiguous);
case 2:
- ShallowCopyModifiedInner<P, 2>(to, from, toIsContiguous, fromIsContiguous);
- return;
+ return ElementsBitwiseEqualInner<P, 2>(
+ to, from, toIsContiguous, fromIsContiguous);
case 3:
- ShallowCopyModifiedInner<P, 3>(to, from, toIsContiguous, fromIsContiguous);
- return;
+ return ElementsBitwiseEqualInner<P, 3>(
+ to, from, toIsContiguous, fromIsContiguous);
case 4:
- ShallowCopyModifiedInner<P, 4>(to, from, toIsContiguous, fromIsContiguous);
- return;
+ return ElementsBitwiseEqualInner<P, 4>(
+ to, from, toIsContiguous, fromIsContiguous);
default:
- ShallowCopyModifiedInner<P>(to, from, toIsContiguous, fromIsContiguous);
- return;
+ return ElementsBitwiseEqualInner<P>(
+ to, from, toIsContiguous, fromIsContiguous);
}
}
-RT_API_ATTRS void ShallowCopyModifiedElements(
+RT_API_ATTRS bool ElementsBitwiseEqual(
const Descriptor &to, const Descriptor &from) {
bool toIsContiguous{to.IsContiguous()};
bool fromIsContiguous{from.IsContiguous()};
std::size_t elementBytes{to.ElementBytes()};
// Same type-based dispatch as ShallowCopy() above, so the per-element
- // comparison and copy inline to fixed-size operations.
+ // comparison inlines to fixed-size operations.
if (to.type().IsInteger()) {
if (elementBytes == sizeof(int64_t)) {
- ShallowCopyModifiedRank<int64_t>(
+ return ElementsBitwiseEqualRank<int64_t>(
to, from, toIsContiguous, fromIsContiguous);
} else if (elementBytes == sizeof(int32_t)) {
- ShallowCopyModifiedRank<int32_t>(
+ return ElementsBitwiseEqualRank<int32_t>(
to, from, toIsContiguous, fromIsContiguous);
} else if (elementBytes == sizeof(int16_t)) {
- ShallowCopyModifiedRank<int16_t>(
+ return ElementsBitwiseEqualRank<int16_t>(
to, from, toIsContiguous, fromIsContiguous);
#if defined USING_NATIVE_INT128_T
} else if (elementBytes == sizeof(__int128_t)) {
- ShallowCopyModifiedRank<__int128_t>(
+ return ElementsBitwiseEqualRank<__int128_t>(
to, from, toIsContiguous, fromIsContiguous);
#endif
} else {
- ShallowCopyModifiedRank<char>(to, from, toIsContiguous, fromIsContiguous);
+ return ElementsBitwiseEqualRank<char>(
+ to, from, toIsContiguous, fromIsContiguous);
}
} else if (to.type().IsReal()) {
if (elementBytes == sizeof(double)) {
- ShallowCopyModifiedRank<double>(
+ return ElementsBitwiseEqualRank<double>(
to, from, toIsContiguous, fromIsContiguous);
} else if (elementBytes == sizeof(float)) {
- ShallowCopyModifiedRank<float>(
+ return ElementsBitwiseEqualRank<float>(
to, from, toIsContiguous, fromIsContiguous);
} else {
- ShallowCopyModifiedRank<char>(to, from, toIsContiguous, fromIsContiguous);
+ return ElementsBitwiseEqualRank<char>(
+ to, from, toIsContiguous, fromIsContiguous);
}
} else {
- ShallowCopyModifiedRank<char>(to, from, toIsContiguous, fromIsContiguous);
+ return ElementsBitwiseEqualRank<char>(
+ to, from, toIsContiguous, fromIsContiguous);
}
}
diff --git a/flang-rt/unittests/Runtime/Assign.cpp b/flang-rt/unittests/Runtime/Assign.cpp
index e9b8de52ffde2..cf2dba619766e 100644
--- a/flang-rt/unittests/Runtime/Assign.cpp
+++ b/flang-rt/unittests/Runtime/Assign.cpp
@@ -456,7 +456,8 @@ TEST(AssignSimpleCrash, NonAllocatableElementCountMismatch) {
TEST(Assign, RTNAME(CopyOutAssign)) {
// Copy-out writes back the elements the callee modified through the
- // temporary and performs no stores for the elements it never touched.
+ // temporary (when nothing was modified, it performs no stores at all;
+ // see the read-only test below).
// Discontiguous var: stride-2 view (elements 1,3,5,7) of an 8-element
// backing array, as copy-in/copy-out creates for a non-contiguous actual.
int data[8] = {1, 2, 3, 4, 5, 6, 7, 8};
@@ -567,4 +568,41 @@ TEST(Assign, RTNAME(CopyOutAssignUnconditionalEnvVar)) {
RTNAME(CopyOutAssign)(&var, temp, __FILE__, __LINE__);
ASSERT_EQ(munmap(page, pageSize), 0);
}
+
+TEST(Assign, RTNAME(CopyOutAssignReadOnlyModifiedDies)) {
+ // When the callee DID modify the temporary, copy-out falls back to the
+ // whole-object copy; storing into a read-only original then faults, which
+ // is the intended behavior for a program that modifies a non-definable
+ // actual argument.
+ std::size_t pageSize{static_cast<std::size_t>(sysconf(_SC_PAGESIZE))};
+ void *page{mmap(nullptr, pageSize, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)};
+ ASSERT_NE(page, MAP_FAILED);
+ double *data{static_cast<double *>(page)};
+ for (int j{0}; j < 8; ++j) {
+ data[j] = j + 1;
+ }
+ ASSERT_EQ(mprotect(page, pageSize, PROT_READ), 0);
+
+ StaticDescriptor<1> staticVar;
+ Descriptor &var{staticVar.descriptor()};
+ SubscriptValue extent[1]{4};
+ var.Establish(
+ TypeCode{TypeCategory::Real, 8}, sizeof(double), data, 1, extent);
+ var.GetDimension(0).SetLowerBound(1);
+ var.GetDimension(0).SetByteStride(sizeof(double) * 2);
+
+ StaticDescriptor<1> staticTemp;
+ Descriptor &temp{staticTemp.descriptor()};
+ RTNAME(CopyInAssign)(temp, var, __FILE__, __LINE__);
+ ASSERT_TRUE(temp.IsAllocated());
+ *temp.OffsetElement<double>(1 * sizeof(double)) = -1.0;
+
+ EXPECT_DEATH(RTNAME(CopyOutAssign)(&var, temp, __FILE__, __LINE__), "");
+
+ // Clean up the parent's still-allocated temp against writable storage.
+ ASSERT_EQ(mprotect(page, pageSize, PROT_READ | PROT_WRITE), 0);
+ RTNAME(CopyOutAssign)(&var, temp, __FILE__, __LINE__);
+ ASSERT_EQ(munmap(page, pageSize), 0);
+}
#endif
diff --git a/flang/docs/RuntimeEnvironment.md b/flang/docs/RuntimeEnvironment.md
index a24da8e6a6a62..02128f4430de0 100644
--- a/flang/docs/RuntimeEnvironment.md
+++ b/flang/docs/RuntimeEnvironment.md
@@ -37,13 +37,14 @@ Determines data conversions applied to unformatted I/O.
## `FLANG_RT_COPYOUT_MODIFIED_ONLY`
When the compiler passes a copy of an actual argument to a procedure
-(copy-in/copy-out), the runtime copies back only the elements whose bit
-patterns were changed through the temporary copy. This avoids stores to
-the original argument when the callee never modified the data -- in
+(copy-in/copy-out), the runtime skips the copy-out entirely when the
+temporary copy is still bitwise-identical to the original, and performs
+the normal whole-object copy-out otherwise. This avoids stores to the
+original argument when the callee never modified the data -- in
particular, stores into read-only storage backing a non-definable actual
argument.
Set `FLANG_RT_COPYOUT_MODIFIED_ONLY=0` to restore the unconditional
-copy-out of every element.
+copy-out.
## `FORT_CHECK_POINTER_DEALLOCATION`
>From 31bf02f94c880403f181622cb31808756cbd9423 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Wed, 9 Sep 2026 10:14:52 -0700
Subject: [PATCH 3/6] [flang-rt] Copy only the modified suffix in CopyOutAssign
The scan-then-copy-whole-object fallback re-traverses the entire object
when the first modified element sits late in the array: with the
modifications clustered at the tail, the scan reads everything before
finding a difference and the copy then traverses everything again,
measuring up to 2.3x the unconditional copy.
Fuse the two phases: scan to the first bitwise difference and copy from
that position to the end, reusing the scan's position (one pass, no
repositioning; the copy phase remains unconditional, so no data-dependent
branching is reintroduced). Elements before the first difference are
bitwise-identical, so skipping them changes nothing observable; the
unmodified case still performs no stores; and copy-out now never
traverses the data more than once nor stores more elements than the
unconditional copy would, for any modification layout.
Adds a unit test pinning the prefix skip: the variable spans a read-only
page followed by a writable page, the callee modifies only an element on
the writable page, and copy-out must not fault while delivering the
modification.
---
flang-rt/include/flang-rt/runtime/tools.h | 22 +++---
flang-rt/lib/runtime/assign.cpp | 22 +++---
flang-rt/lib/runtime/tools.cpp | 94 +++++++++++++++--------
flang-rt/unittests/Runtime/Assign.cpp | 43 +++++++++++
4 files changed, 129 insertions(+), 52 deletions(-)
diff --git a/flang-rt/include/flang-rt/runtime/tools.h b/flang-rt/include/flang-rt/runtime/tools.h
index 6038157369f3c..01977ebe11912 100644
--- a/flang-rt/include/flang-rt/runtime/tools.h
+++ b/flang-rt/include/flang-rt/runtime/tools.h
@@ -525,15 +525,19 @@ RT_API_ATTRS void ShallowCopy(const Descriptor &to, const Descriptor &from,
bool toIsContiguous, bool fromIsContiguous);
RT_API_ATTRS void ShallowCopy(const Descriptor &to, const Descriptor &from);
-// Returns true when every element of 'from' is bitwise-identical to the
-// corresponding element of 'to' (returns false at the first difference).
-// Used by copy-out to skip all stores when the callee never modified the
-// data, so that an original that lives in read-only memory (e.g. a named
-// constant) is not written to by an unmodifying copy-out. The comparison is
-// bitwise, so it is exact when 'from' was originally produced from 'to' by
-// ShallowCopy() (as CopyInAssign() does): unmodified elements compare equal
-// even for NaNs and padding bytes, which a value comparison would misjudge.
-RT_API_ATTRS bool ElementsBitwiseEqual(
+// Scans for the first element of 'from' whose bit pattern differs from the
+// corresponding element of 'to', then copies that element and every element
+// after it (one fused pass). Elements before the first difference are
+// bitwise-identical and are not stored to. Used by copy-out so that an
+// unmodifying copy-out performs no stores at all — an original that lives in
+// read-only memory (e.g. a named constant) is never written to unless it was
+// actually modified — while a modifying copy-out never traverses the data
+// more than once nor stores more than the unconditional copy would. The
+// comparison is bitwise, so it is exact when 'from' was originally produced
+// from 'to' by ShallowCopy() (as CopyInAssign() does): unmodified elements
+// compare equal even for NaNs and padding bytes, which a value comparison
+// would misjudge.
+RT_API_ATTRS void ShallowCopyModifiedSuffix(
const Descriptor &to, const Descriptor &from);
// Ensures that a character string is null-terminated, allocating a /p length +1
diff --git a/flang-rt/lib/runtime/assign.cpp b/flang-rt/lib/runtime/assign.cpp
index 0377c00c24f32..e044d672c6e05 100644
--- a/flang-rt/lib/runtime/assign.cpp
+++ b/flang-rt/lib/runtime/assign.cpp
@@ -839,21 +839,21 @@ void RTDEF(CopyOutAssign)(
Terminator terminator{sourceFile, sourceLine};
// Copyout from the temporary must not cause any finalizations
// for LHS. The variable must be properly initialized already.
- // Skip the copy-out entirely when the temporary is still bitwise-identical
- // to the variable: the temporary was created as a bitwise copy (see
- // CopyInAssign above), so it can only differ if the callee modified it, and
- // an unmodifying copy-out must not store at all. This keeps a
+ // Scan for the first bitwise difference and copy from there to the end
+ // (fused, one pass): the temporary was created as a bitwise copy (see
+ // CopyInAssign above), so it can only differ where the callee modified it,
+ // and an unmodifying copy-out must not store at all. This keeps a
// compiler-generated copy-out from writing into read-only storage when the
// effective argument is not definable (e.g., a named constant) and the
- // callee, conformingly, never modified it. When any element was modified,
- // fall back to the plain whole-object copy: a per-element conditional
- // store measures far slower on partially-modified data (branch
- // misprediction), and a modified temporary means the variable is legally
- // writable anyway.
+ // callee, conformingly, never modified it. From the first difference
+ // onward the copy is unconditional: a per-element conditional store
+ // measures far slower on partially-modified data (branch misprediction),
+ // and a modified temporary means the variable is legally writable anyway.
// FLANG_RT_COPYOUT_MODIFIED_ONLY=0 restores the unconditional copy-out.
if (var) {
- if (!executionEnvironment.copyOutModifiedOnly ||
- !ElementsBitwiseEqual(*var, temp)) {
+ if (executionEnvironment.copyOutModifiedOnly) {
+ ShallowCopyModifiedSuffix(*var, temp);
+ } else {
ShallowCopy(*var, temp);
}
}
diff --git a/flang-rt/lib/runtime/tools.cpp b/flang-rt/lib/runtime/tools.cpp
index 2cc4a9da9d5bb..cb3a36b4d74fc 100644
--- a/flang-rt/lib/runtime/tools.cpp
+++ b/flang-rt/lib/runtime/tools.cpp
@@ -284,118 +284,148 @@ static inline RT_API_ATTRS bool ElementIsModified(
}
}
+template <typename P>
+static inline RT_API_ATTRS void CopyElement(
+ char *toAt, const char *fromAt, std::size_t elementBytes) {
+ constexpr std::size_t typeElementBytes{sizeof(P)};
+ if constexpr (typeElementBytes != 1) {
+ runtime::memcpy(toAt, fromAt, typeElementBytes);
+ } else {
+ runtime::memcpy(toAt, fromAt, elementBytes);
+ }
+}
+
+// Scans for the first bitwise difference; when one is found, copies that
+// element and everything after it, reusing the scan's position (fused, one
+// pass). Elements before the first difference are bitwise-identical and are
+// not stored to, so an unmodified copy-out performs no stores at all, and a
+// copy-out never traverses the data more than once nor stores more elements
+// than the unconditional copy would.
template <typename P, int RANK = -1>
-static RT_API_ATTRS bool ElementsBitwiseEqualInner(const Descriptor &to,
+static RT_API_ATTRS void ShallowCopyModifiedSuffixInner(const Descriptor &to,
const Descriptor &from, bool toIsContiguous, bool fromIsContiguous) {
std::size_t elementBytes{to.ElementBytes()};
+ std::size_t n{to.Elements()};
if (toIsContiguous) {
- const char *toAt{to.OffsetElement()};
+ char *toAt{to.OffsetElement()};
if (fromIsContiguous) {
const char *fromAt{from.OffsetElement()};
- for (std::size_t n{to.Elements()}; n-- > 0;
- toAt += elementBytes, fromAt += elementBytes) {
+ for (; n > 0; --n, toAt += elementBytes, fromAt += elementBytes) {
if (ElementIsModified<P>(toAt, fromAt, elementBytes)) {
- return false;
+ // Copy the remaining elements, including this one, in one block.
+ runtime::memcpy(toAt, fromAt, n * elementBytes);
+ return;
}
}
} else {
DescriptorIterator<RANK> fromIt{from};
- for (std::size_t n{to.Elements()}; n-- > 0;
- toAt += elementBytes, fromIt.Advance()) {
+ for (; n > 0; --n, toAt += elementBytes, fromIt.Advance()) {
if (ElementIsModified<P>(
toAt, fromIt.template Get<char>(), elementBytes)) {
- return false;
+ break;
}
}
+ for (; n > 0; --n, toAt += elementBytes, fromIt.Advance()) {
+ CopyElement<P>(toAt, fromIt.template Get<char>(), elementBytes);
+ }
}
} else {
DescriptorIterator<RANK> toIt{to};
if (fromIsContiguous) {
const char *fromAt{from.OffsetElement()};
- for (std::size_t n{to.Elements()}; n-- > 0;
- toIt.Advance(), fromAt += elementBytes) {
+ for (; n > 0; --n, toIt.Advance(), fromAt += elementBytes) {
if (ElementIsModified<P>(
toIt.template Get<char>(), fromAt, elementBytes)) {
- return false;
+ break;
}
}
+ for (; n > 0; --n, toIt.Advance(), fromAt += elementBytes) {
+ CopyElement<P>(toIt.template Get<char>(), fromAt, elementBytes);
+ }
} else {
DescriptorIterator<RANK> fromIt{from};
- for (std::size_t n{to.Elements()}; n-- > 0;
- toIt.Advance(), fromIt.Advance()) {
+ for (; n > 0; --n, toIt.Advance(), fromIt.Advance()) {
if (ElementIsModified<P>(toIt.template Get<char>(),
fromIt.template Get<char>(), elementBytes)) {
- return false;
+ break;
}
}
+ for (; n > 0; --n, toIt.Advance(), fromIt.Advance()) {
+ CopyElement<P>(toIt.template Get<char>(), fromIt.template Get<char>(),
+ elementBytes);
+ }
}
}
- return true;
}
template <typename P>
-static RT_API_ATTRS bool ElementsBitwiseEqualRank(const Descriptor &to,
+static RT_API_ATTRS void ShallowCopyModifiedSuffixRank(const Descriptor &to,
const Descriptor &from, bool toIsContiguous, bool fromIsContiguous) {
INTERNAL_CHECK(to.rank() == from.rank());
// Mirror ShallowCopyRank's rank specialization policy.
switch (to.rank()) {
case 1:
- return ElementsBitwiseEqualInner<P, 1>(
+ ShallowCopyModifiedSuffixInner<P, 1>(
to, from, toIsContiguous, fromIsContiguous);
+ return;
case 2:
- return ElementsBitwiseEqualInner<P, 2>(
+ ShallowCopyModifiedSuffixInner<P, 2>(
to, from, toIsContiguous, fromIsContiguous);
+ return;
case 3:
- return ElementsBitwiseEqualInner<P, 3>(
+ ShallowCopyModifiedSuffixInner<P, 3>(
to, from, toIsContiguous, fromIsContiguous);
+ return;
case 4:
- return ElementsBitwiseEqualInner<P, 4>(
+ ShallowCopyModifiedSuffixInner<P, 4>(
to, from, toIsContiguous, fromIsContiguous);
+ return;
default:
- return ElementsBitwiseEqualInner<P>(
+ ShallowCopyModifiedSuffixInner<P>(
to, from, toIsContiguous, fromIsContiguous);
+ return;
}
}
-RT_API_ATTRS bool ElementsBitwiseEqual(
+RT_API_ATTRS void ShallowCopyModifiedSuffix(
const Descriptor &to, const Descriptor &from) {
bool toIsContiguous{to.IsContiguous()};
bool fromIsContiguous{from.IsContiguous()};
std::size_t elementBytes{to.ElementBytes()};
// Same type-based dispatch as ShallowCopy() above, so the per-element
- // comparison inlines to fixed-size operations.
+ // comparison and copy inline to fixed-size operations.
if (to.type().IsInteger()) {
if (elementBytes == sizeof(int64_t)) {
- return ElementsBitwiseEqualRank<int64_t>(
+ ShallowCopyModifiedSuffixRank<int64_t>(
to, from, toIsContiguous, fromIsContiguous);
} else if (elementBytes == sizeof(int32_t)) {
- return ElementsBitwiseEqualRank<int32_t>(
+ ShallowCopyModifiedSuffixRank<int32_t>(
to, from, toIsContiguous, fromIsContiguous);
} else if (elementBytes == sizeof(int16_t)) {
- return ElementsBitwiseEqualRank<int16_t>(
+ ShallowCopyModifiedSuffixRank<int16_t>(
to, from, toIsContiguous, fromIsContiguous);
#if defined USING_NATIVE_INT128_T
} else if (elementBytes == sizeof(__int128_t)) {
- return ElementsBitwiseEqualRank<__int128_t>(
+ ShallowCopyModifiedSuffixRank<__int128_t>(
to, from, toIsContiguous, fromIsContiguous);
#endif
} else {
- return ElementsBitwiseEqualRank<char>(
+ ShallowCopyModifiedSuffixRank<char>(
to, from, toIsContiguous, fromIsContiguous);
}
} else if (to.type().IsReal()) {
if (elementBytes == sizeof(double)) {
- return ElementsBitwiseEqualRank<double>(
+ ShallowCopyModifiedSuffixRank<double>(
to, from, toIsContiguous, fromIsContiguous);
} else if (elementBytes == sizeof(float)) {
- return ElementsBitwiseEqualRank<float>(
+ ShallowCopyModifiedSuffixRank<float>(
to, from, toIsContiguous, fromIsContiguous);
} else {
- return ElementsBitwiseEqualRank<char>(
+ ShallowCopyModifiedSuffixRank<char>(
to, from, toIsContiguous, fromIsContiguous);
}
} else {
- return ElementsBitwiseEqualRank<char>(
+ ShallowCopyModifiedSuffixRank<char>(
to, from, toIsContiguous, fromIsContiguous);
}
}
diff --git a/flang-rt/unittests/Runtime/Assign.cpp b/flang-rt/unittests/Runtime/Assign.cpp
index cf2dba619766e..0449fd4f652ad 100644
--- a/flang-rt/unittests/Runtime/Assign.cpp
+++ b/flang-rt/unittests/Runtime/Assign.cpp
@@ -569,6 +569,49 @@ TEST(Assign, RTNAME(CopyOutAssignUnconditionalEnvVar)) {
ASSERT_EQ(munmap(page, pageSize), 0);
}
+TEST(Assign, RTNAME(CopyOutAssignSkipsUnmodifiedPrefix)) {
+ // When the first modification lies beyond a read-only prefix, copy-out
+ // must not store into the unmodified prefix: two adjacent pages, the first
+ // read-only, the second writable; the variable spans both; the callee
+ // modifies only an element on the second page. Copy-out must not fault and
+ // must deliver the modification.
+ std::size_t pageSize{static_cast<std::size_t>(sysconf(_SC_PAGESIZE))};
+ void *pages{mmap(nullptr, 2 * pageSize, PROT_READ | PROT_WRITE,
+ MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)};
+ ASSERT_NE(pages, MAP_FAILED);
+ // Element stride 2*sizeof(double); place 'count' elements so that the
+ // first ones sit on page 1 and the last ones on page 2.
+ std::size_t perPage{pageSize / (2 * sizeof(double))};
+ std::size_t count{perPage + 4};
+ double *data{static_cast<double *>(pages)};
+ for (std::size_t j{0}; j < 2 * count; ++j) {
+ data[j] = static_cast<double>(j);
+ }
+ ASSERT_EQ(mprotect(pages, pageSize, PROT_READ), 0); // page 1 read-only
+
+ StaticDescriptor<1> staticVar;
+ Descriptor &var{staticVar.descriptor()};
+ SubscriptValue extent[1]{static_cast<SubscriptValue>(count)};
+ var.Establish(
+ TypeCode{TypeCategory::Real, 8}, sizeof(double), data, 1, extent);
+ var.GetDimension(0).SetLowerBound(1);
+ var.GetDimension(0).SetByteStride(sizeof(double) * 2);
+
+ StaticDescriptor<1> staticTemp;
+ Descriptor &temp{staticTemp.descriptor()};
+ RTNAME(CopyInAssign)(temp, var, __FILE__, __LINE__);
+ ASSERT_TRUE(temp.IsAllocated());
+ // Modify only the last element; its storage is on the writable page 2.
+ *temp.OffsetElement<double>((count - 1) * sizeof(double)) = -123.0;
+
+ RTNAME(CopyOutAssign)(&var, temp, __FILE__, __LINE__); // must not fault
+ EXPECT_EQ(data[2 * (count - 1)], -123.0);
+ EXPECT_EQ(data[0], 0.0);
+
+ ASSERT_EQ(mprotect(pages, pageSize, PROT_READ | PROT_WRITE), 0);
+ ASSERT_EQ(munmap(pages, 2 * pageSize), 0);
+}
+
TEST(Assign, RTNAME(CopyOutAssignReadOnlyModifiedDies)) {
// When the callee DID modify the temporary, copy-out falls back to the
// whole-object copy; storing into a read-only original then faults, which
>From 9ac27f9664daa6e1431666832702739295eb649d Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Wed, 9 Sep 2026 11:25:09 -0700
Subject: [PATCH 4/6] [flang-rt] Test FLANG_RT_COPYOUT_MODIFIED_ONLY parsing
Add a unit test that exercises the environment-variable parsing path in
ExecutionEnvironment::Configure() for FLANG_RT_COPYOUT_MODIFIED_ONLY:
"0" disables, "1" enables, an invalid value warns and leaves the default
enabled, and an absent variable leaves the default. The existing control
test sets the parsed field directly and does not cover the parsing itself.
---
flang-rt/unittests/Runtime/Assign.cpp | 30 +++++++++++++++++++++++++++
1 file changed, 30 insertions(+)
diff --git a/flang-rt/unittests/Runtime/Assign.cpp b/flang-rt/unittests/Runtime/Assign.cpp
index 0449fd4f652ad..a53d6c7e07f07 100644
--- a/flang-rt/unittests/Runtime/Assign.cpp
+++ b/flang-rt/unittests/Runtime/Assign.cpp
@@ -532,6 +532,36 @@ TEST(Assign, RTNAME(CopyOutAssignReadOnlyUnmodified)) {
#endif
#if defined(__unix__) || defined(__APPLE__)
+#if defined(__unix__) || defined(__APPLE__)
+TEST(Assign, RTNAME(CopyOutAssignEnvVarParsing)) {
+ // Exercise the FLANG_RT_COPYOUT_MODIFIED_ONLY parsing path in
+ // ExecutionEnvironment::Configure(), rather than setting the field
+ // directly: "0" disables, "1" enables, an invalid value warns and leaves
+ // the default (enabled), and an absent variable leaves the default.
+ bool saved{executionEnvironment.copyOutModifiedOnly};
+
+ ASSERT_EQ(setenv("FLANG_RT_COPYOUT_MODIFIED_ONLY", "0", 1), 0);
+ executionEnvironment.Configure(0, nullptr, nullptr, nullptr);
+ EXPECT_FALSE(executionEnvironment.copyOutModifiedOnly);
+
+ ASSERT_EQ(setenv("FLANG_RT_COPYOUT_MODIFIED_ONLY", "1", 1), 0);
+ executionEnvironment.Configure(0, nullptr, nullptr, nullptr);
+ EXPECT_TRUE(executionEnvironment.copyOutModifiedOnly);
+
+ // Invalid value: warns, leaves the default (enabled).
+ ASSERT_EQ(setenv("FLANG_RT_COPYOUT_MODIFIED_ONLY", "2", 1), 0);
+ executionEnvironment.Configure(0, nullptr, nullptr, nullptr);
+ EXPECT_TRUE(executionEnvironment.copyOutModifiedOnly);
+
+ // Absent: default (enabled).
+ ASSERT_EQ(unsetenv("FLANG_RT_COPYOUT_MODIFIED_ONLY"), 0);
+ executionEnvironment.Configure(0, nullptr, nullptr, nullptr);
+ EXPECT_TRUE(executionEnvironment.copyOutModifiedOnly);
+
+ executionEnvironment.copyOutModifiedOnly = saved;
+}
+#endif
+
TEST(Assign, RTNAME(CopyOutAssignUnconditionalEnvVar)) {
// With FLANG_RT_COPYOUT_MODIFIED_ONLY=0 semantics (unconditional copy-out),
// even an unmodified copy-out stores every element, so a read-only original
>From 17bb79f715eecda0e7326156a7a5585370392fd7 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Thu, 10 Sep 2026 13:49:01 -0700
Subject: [PATCH 5/6] [flang-rt] Name FLANG_RT_COPYOUT_MODIFIED_ONLY as a
system environment variable
Refer to FLANG_RT_COPYOUT_MODIFIED_ONLY explicitly as a system environment
variable in the documentation and in the comments that mention it, so a
reader can tell what it is without consulting the runtime sources.
Documentation and comments only; no functional change.
---
flang-rt/include/flang-rt/runtime/environment.h | 3 ++-
flang-rt/lib/runtime/assign.cpp | 3 ++-
flang/docs/RuntimeEnvironment.md | 7 +++++--
3 files changed, 9 insertions(+), 4 deletions(-)
diff --git a/flang-rt/include/flang-rt/runtime/environment.h b/flang-rt/include/flang-rt/runtime/environment.h
index a4dabb2ca20fb..31478bcf3e4f1 100644
--- a/flang-rt/include/flang-rt/runtime/environment.h
+++ b/flang-rt/include/flang-rt/runtime/environment.h
@@ -74,7 +74,8 @@ struct ExecutionEnvironment {
bool checkPointerDeallocation{true}; // FORT_CHECK_POINTER_DEALLOCATION
bool truncateStream{true}; // FORT_TRUNCATE_STREAM
bool noEmptyAllocation{false}; // FORT_NO_EMPTY_ALLOCATION
- // FLANG_RT_COPYOUT_MODIFIED_ONLY=0 restores the unconditional copy-out
+ // The system environment variable FLANG_RT_COPYOUT_MODIFIED_ONLY=0
+ // restores the unconditional copy-out
// of argument temporaries (CopyOutAssign copies every element back
// instead of only the elements the callee modified).
bool copyOutModifiedOnly{true}; // FLANG_RT_COPYOUT_MODIFIED_ONLY
diff --git a/flang-rt/lib/runtime/assign.cpp b/flang-rt/lib/runtime/assign.cpp
index e044d672c6e05..6f8f246d8e3c1 100644
--- a/flang-rt/lib/runtime/assign.cpp
+++ b/flang-rt/lib/runtime/assign.cpp
@@ -849,7 +849,8 @@ void RTDEF(CopyOutAssign)(
// onward the copy is unconditional: a per-element conditional store
// measures far slower on partially-modified data (branch misprediction),
// and a modified temporary means the variable is legally writable anyway.
- // FLANG_RT_COPYOUT_MODIFIED_ONLY=0 restores the unconditional copy-out.
+ // Setting the system environment variable FLANG_RT_COPYOUT_MODIFIED_ONLY=0
+ // restores the unconditional copy-out.
if (var) {
if (executionEnvironment.copyOutModifiedOnly) {
ShallowCopyModifiedSuffix(*var, temp);
diff --git a/flang/docs/RuntimeEnvironment.md b/flang/docs/RuntimeEnvironment.md
index 02128f4430de0..0d3e274328a8a 100644
--- a/flang/docs/RuntimeEnvironment.md
+++ b/flang/docs/RuntimeEnvironment.md
@@ -36,6 +36,9 @@ Determines data conversions applied to unformatted I/O.
## `FLANG_RT_COPYOUT_MODIFIED_ONLY`
+The system environment variable `FLANG_RT_COPYOUT_MODIFIED_ONLY` selects how
+the runtime performs copy-out.
+
When the compiler passes a copy of an actual argument to a procedure
(copy-in/copy-out), the runtime skips the copy-out entirely when the
temporary copy is still bitwise-identical to the original, and performs
@@ -43,8 +46,8 @@ the normal whole-object copy-out otherwise. This avoids stores to the
original argument when the callee never modified the data -- in
particular, stores into read-only storage backing a non-definable actual
argument.
-Set `FLANG_RT_COPYOUT_MODIFIED_ONLY=0` to restore the unconditional
-copy-out.
+Set the system environment variable `FLANG_RT_COPYOUT_MODIFIED_ONLY=0` to
+restore the unconditional copy-out.
## `FORT_CHECK_POINTER_DEALLOCATION`
>From d810757b5046eccf051f086b1c1eb0deccb79569 Mon Sep 17 00:00:00 2001
From: Eugene Epshteyn <eepshteyn at nvidia.com>
Date: Tue, 15 Sep 2026 12:03:24 -0700
Subject: [PATCH 6/6] [flang-rt] Address review findings: documentation,
comments, and device instantiation
Correct the documentation and comments that still described a superseded
design in which an unmodified copy-out skipped stores but a modified one
fell back to a whole-object copy: the shipped algorithm scans for the
first bitwise difference and copies only from that element through the
end, in one fused pass. Document that behavior, and the
FLANG_RT_COPYOUT_MODIFIED_ONLY=0 escape hatch, in the public contract of
CopyOutAssign in flang/Runtime/assign.h, and note in the documentation
that the escape hatch restores stores of unmodified data (including into
read-only storage) and is not a safer mode.
Move the ShallowCopyModifiedSuffix family below RT_OFFLOAD_API_GROUP_END:
its only caller is CopyOutAssign, which is host-only, so the device
instantiations of the family were dead code with link-time-optimization
cost.
Also drop an incorrect branch-misprediction attribution in a comment
(measurement attributes the cost difference to inhibited loop unrolling),
and deduplicate a doubled preprocessor guard around the environment
variable parsing test.
---
.../include/flang-rt/runtime/environment.h | 6 +-
flang-rt/include/flang-rt/runtime/tools.h | 4 +-
flang-rt/lib/runtime/assign.cpp | 6 +-
flang-rt/lib/runtime/tools.cpp | 204 +++++++++---------
flang-rt/unittests/Runtime/Assign.cpp | 10 +-
flang/docs/RuntimeEnvironment.md | 16 +-
flang/include/flang/Runtime/assign.h | 8 +-
7 files changed, 131 insertions(+), 123 deletions(-)
diff --git a/flang-rt/include/flang-rt/runtime/environment.h b/flang-rt/include/flang-rt/runtime/environment.h
index 31478bcf3e4f1..a75ac261b61e8 100644
--- a/flang-rt/include/flang-rt/runtime/environment.h
+++ b/flang-rt/include/flang-rt/runtime/environment.h
@@ -75,9 +75,9 @@ struct ExecutionEnvironment {
bool truncateStream{true}; // FORT_TRUNCATE_STREAM
bool noEmptyAllocation{false}; // FORT_NO_EMPTY_ALLOCATION
// The system environment variable FLANG_RT_COPYOUT_MODIFIED_ONLY=0
- // restores the unconditional copy-out
- // of argument temporaries (CopyOutAssign copies every element back
- // instead of only the elements the callee modified).
+ // restores the unconditional copy-out of argument temporaries
+ // (CopyOutAssign then copies every element back instead of only the
+ // suffix from the first modified element through the end).
bool copyOutModifiedOnly{true}; // FLANG_RT_COPYOUT_MODIFIED_ONLY
enum InternalDebugging { WorkQueue = 1 };
diff --git a/flang-rt/include/flang-rt/runtime/tools.h b/flang-rt/include/flang-rt/runtime/tools.h
index 01977ebe11912..245d2829f8ee5 100644
--- a/flang-rt/include/flang-rt/runtime/tools.h
+++ b/flang-rt/include/flang-rt/runtime/tools.h
@@ -537,8 +537,8 @@ RT_API_ATTRS void ShallowCopy(const Descriptor &to, const Descriptor &from);
// from 'to' by ShallowCopy() (as CopyInAssign() does): unmodified elements
// compare equal even for NaNs and padding bytes, which a value comparison
// would misjudge.
-RT_API_ATTRS void ShallowCopyModifiedSuffix(
- const Descriptor &to, const Descriptor &from);
+// (Host-only: the sole caller is CopyOutAssign, outside the offload group.)
+void ShallowCopyModifiedSuffix(const Descriptor &to, const Descriptor &from);
// Ensures that a character string is null-terminated, allocating a /p length +1
// size memory for null-terminator if necessary. Returns the original or a newly
diff --git a/flang-rt/lib/runtime/assign.cpp b/flang-rt/lib/runtime/assign.cpp
index 6f8f246d8e3c1..0ae105d9df671 100644
--- a/flang-rt/lib/runtime/assign.cpp
+++ b/flang-rt/lib/runtime/assign.cpp
@@ -846,9 +846,9 @@ void RTDEF(CopyOutAssign)(
// compiler-generated copy-out from writing into read-only storage when the
// effective argument is not definable (e.g., a named constant) and the
// callee, conformingly, never modified it. From the first difference
- // onward the copy is unconditional: a per-element conditional store
- // measures far slower on partially-modified data (branch misprediction),
- // and a modified temporary means the variable is legally writable anyway.
+ // onward the copy is unconditional: the fused scan-then-copy traverses the
+ // data only once, and a modified temporary means the variable is legally
+ // writable anyway.
// Setting the system environment variable FLANG_RT_COPYOUT_MODIFIED_ONLY=0
// restores the unconditional copy-out.
if (var) {
diff --git a/flang-rt/lib/runtime/tools.cpp b/flang-rt/lib/runtime/tools.cpp
index cb3a36b4d74fc..603a646048da8 100644
--- a/flang-rt/lib/runtime/tools.cpp
+++ b/flang-rt/lib/runtime/tools.cpp
@@ -271,10 +271,109 @@ RT_API_ATTRS void ShallowCopy(const Descriptor &to, const Descriptor &from) {
ShallowCopy(to, from, to.IsContiguous(), from.IsContiguous());
}
+RT_API_ATTRS char *EnsureNullTerminated(
+ char *str, std::size_t length, Terminator &terminator) {
+ if (runtime::memchr(str, '\0', length) == nullptr) {
+ char *newCmd{(char *)AllocateMemoryOrCrash(terminator, length + 1)};
+ runtime::memcpy(newCmd, str, length);
+ newCmd[length] = '\0';
+ return newCmd;
+ } else {
+ return str;
+ }
+}
+
+RT_API_ATTRS bool IsValidCharDescriptor(const Descriptor *value) {
+ return value && value->IsAllocated() &&
+ value->type() == TypeCode(TypeCategory::Character, 1) &&
+ value->rank() == 0;
+}
+
+RT_API_ATTRS bool IsValidIntDescriptor(const Descriptor *intVal) {
+ // Check that our descriptor is allocated and is a scalar integer with
+ // kind != 1 (i.e. with a large enough decimal exponent range).
+ return intVal && intVal->IsAllocated() && intVal->rank() == 0 &&
+ intVal->type().IsInteger() && intVal->type().GetCategoryAndKind() &&
+ intVal->type().GetCategoryAndKind()->second != 1;
+}
+
+RT_API_ATTRS std::int32_t CopyCharsToDescriptor(const Descriptor &value,
+ const char *rawValue, std::size_t rawValueLength, const Descriptor *errmsg,
+ std::size_t offset) {
+
+ const std::int64_t toCopy{std::min(static_cast<std::int64_t>(rawValueLength),
+ static_cast<std::int64_t>(value.ElementBytes() - offset))};
+ if (toCopy < 0) {
+ return ToErrmsg(errmsg, StatValueTooShort);
+ }
+
+ runtime::memcpy(value.OffsetElement(offset), rawValue, toCopy);
+
+ if (static_cast<std::int64_t>(rawValueLength) > toCopy) {
+ return ToErrmsg(errmsg, StatValueTooShort);
+ }
+
+ return StatOk;
+}
+
+RT_API_ATTRS void StoreIntToDescriptor(
+ const Descriptor *length, std::int64_t value, Terminator &terminator) {
+ auto typeCode{length->type().GetCategoryAndKind()};
+ int kind{typeCode->second};
+ ApplyIntegerKind<StoreIntegerAt, void>(
+ kind, terminator, *length, /* atIndex = */ 0, value);
+}
+
+template <int KIND> struct FitsInIntegerKind {
+ RT_API_ATTRS bool operator()([[maybe_unused]] std::int64_t value) {
+ if constexpr (KIND >= 8) {
+ return true;
+ } else {
+ return value <=
+ std::numeric_limits<
+ CppTypeFor<Fortran::common::TypeCategory::Integer, KIND>>::max();
+ }
+ }
+};
+
+// Utility: establishes & allocates the result array for a partial
+// reduction (i.e., one with DIM=).
+RT_API_ATTRS void CreatePartialReductionResult(Descriptor &result,
+ const Descriptor &x, std::size_t resultElementSize, int dim,
+ Terminator &terminator, const char *intrinsic, TypeCode typeCode) {
+ int xRank{x.rank()};
+ if (dim < 1 || dim > xRank) {
+ terminator.Crash(
+ "%s: bad DIM=%d for ARRAY with rank %d", intrinsic, dim, xRank);
+ }
+ int zeroBasedDim{dim - 1};
+ SubscriptValue resultExtent[maxRank];
+ for (int j{0}; j < zeroBasedDim; ++j) {
+ resultExtent[j] = x.GetDimension(j).Extent();
+ }
+ for (int j{zeroBasedDim + 1}; j < xRank; ++j) {
+ resultExtent[j - 1] = x.GetDimension(j).Extent();
+ }
+ result.Establish(typeCode, resultElementSize, nullptr, xRank - 1,
+ resultExtent, CFI_attribute_allocatable);
+ for (int j{0}; j + 1 < xRank; ++j) {
+ result.GetDimension(j).SetBounds(1, resultExtent[j]);
+ }
+ if (int stat{result.Allocate(kNoAsyncObject)}) {
+ terminator.Crash(
+ "%s: could not allocate memory for result; STAT=%d", intrinsic, stat);
+ }
+}
+
+RT_OFFLOAD_API_GROUP_END
+
+// The ShallowCopyModifiedSuffix family is deliberately outside the offload
+// API group: its only caller is CopyOutAssign, which is host-only, and
+// instantiating it for the device would only add dead device code.
// Compares one element bitwise. As in the ShallowCopy* helpers above, the
// compile-time element size lets the compiler inline the comparison.
template <typename P>
-static inline RT_API_ATTRS bool ElementIsModified(
+static inline bool ElementIsModified(
const char *toAt, const char *fromAt, std::size_t elementBytes) {
constexpr std::size_t typeElementBytes{sizeof(P)};
if constexpr (typeElementBytes != 1) {
@@ -302,7 +401,7 @@ static inline RT_API_ATTRS void CopyElement(
// copy-out never traverses the data more than once nor stores more elements
// than the unconditional copy would.
template <typename P, int RANK = -1>
-static RT_API_ATTRS void ShallowCopyModifiedSuffixInner(const Descriptor &to,
+static void ShallowCopyModifiedSuffixInner(const Descriptor &to,
const Descriptor &from, bool toIsContiguous, bool fromIsContiguous) {
std::size_t elementBytes{to.ElementBytes()};
std::size_t n{to.Elements()};
@@ -359,7 +458,7 @@ static RT_API_ATTRS void ShallowCopyModifiedSuffixInner(const Descriptor &to,
}
template <typename P>
-static RT_API_ATTRS void ShallowCopyModifiedSuffixRank(const Descriptor &to,
+static void ShallowCopyModifiedSuffixRank(const Descriptor &to,
const Descriptor &from, bool toIsContiguous, bool fromIsContiguous) {
INTERNAL_CHECK(to.rank() == from.rank());
// Mirror ShallowCopyRank's rank specialization policy.
@@ -387,8 +486,7 @@ static RT_API_ATTRS void ShallowCopyModifiedSuffixRank(const Descriptor &to,
}
}
-RT_API_ATTRS void ShallowCopyModifiedSuffix(
- const Descriptor &to, const Descriptor &from) {
+void ShallowCopyModifiedSuffix(const Descriptor &to, const Descriptor &from) {
bool toIsContiguous{to.IsContiguous()};
bool fromIsContiguous{from.IsContiguous()};
std::size_t elementBytes{to.ElementBytes()};
@@ -429,100 +527,4 @@ RT_API_ATTRS void ShallowCopyModifiedSuffix(
to, from, toIsContiguous, fromIsContiguous);
}
}
-
-RT_API_ATTRS char *EnsureNullTerminated(
- char *str, std::size_t length, Terminator &terminator) {
- if (runtime::memchr(str, '\0', length) == nullptr) {
- char *newCmd{(char *)AllocateMemoryOrCrash(terminator, length + 1)};
- runtime::memcpy(newCmd, str, length);
- newCmd[length] = '\0';
- return newCmd;
- } else {
- return str;
- }
-}
-
-RT_API_ATTRS bool IsValidCharDescriptor(const Descriptor *value) {
- return value && value->IsAllocated() &&
- value->type() == TypeCode(TypeCategory::Character, 1) &&
- value->rank() == 0;
-}
-
-RT_API_ATTRS bool IsValidIntDescriptor(const Descriptor *intVal) {
- // Check that our descriptor is allocated and is a scalar integer with
- // kind != 1 (i.e. with a large enough decimal exponent range).
- return intVal && intVal->IsAllocated() && intVal->rank() == 0 &&
- intVal->type().IsInteger() && intVal->type().GetCategoryAndKind() &&
- intVal->type().GetCategoryAndKind()->second != 1;
-}
-
-RT_API_ATTRS std::int32_t CopyCharsToDescriptor(const Descriptor &value,
- const char *rawValue, std::size_t rawValueLength, const Descriptor *errmsg,
- std::size_t offset) {
-
- const std::int64_t toCopy{std::min(static_cast<std::int64_t>(rawValueLength),
- static_cast<std::int64_t>(value.ElementBytes() - offset))};
- if (toCopy < 0) {
- return ToErrmsg(errmsg, StatValueTooShort);
- }
-
- runtime::memcpy(value.OffsetElement(offset), rawValue, toCopy);
-
- if (static_cast<std::int64_t>(rawValueLength) > toCopy) {
- return ToErrmsg(errmsg, StatValueTooShort);
- }
-
- return StatOk;
-}
-
-RT_API_ATTRS void StoreIntToDescriptor(
- const Descriptor *length, std::int64_t value, Terminator &terminator) {
- auto typeCode{length->type().GetCategoryAndKind()};
- int kind{typeCode->second};
- ApplyIntegerKind<StoreIntegerAt, void>(
- kind, terminator, *length, /* atIndex = */ 0, value);
-}
-
-template <int KIND> struct FitsInIntegerKind {
- RT_API_ATTRS bool operator()([[maybe_unused]] std::int64_t value) {
- if constexpr (KIND >= 8) {
- return true;
- } else {
- return value <=
- std::numeric_limits<
- CppTypeFor<Fortran::common::TypeCategory::Integer, KIND>>::max();
- }
- }
-};
-
-// Utility: establishes & allocates the result array for a partial
-// reduction (i.e., one with DIM=).
-RT_API_ATTRS void CreatePartialReductionResult(Descriptor &result,
- const Descriptor &x, std::size_t resultElementSize, int dim,
- Terminator &terminator, const char *intrinsic, TypeCode typeCode) {
- int xRank{x.rank()};
- if (dim < 1 || dim > xRank) {
- terminator.Crash(
- "%s: bad DIM=%d for ARRAY with rank %d", intrinsic, dim, xRank);
- }
- int zeroBasedDim{dim - 1};
- SubscriptValue resultExtent[maxRank];
- for (int j{0}; j < zeroBasedDim; ++j) {
- resultExtent[j] = x.GetDimension(j).Extent();
- }
- for (int j{zeroBasedDim + 1}; j < xRank; ++j) {
- resultExtent[j - 1] = x.GetDimension(j).Extent();
- }
- result.Establish(typeCode, resultElementSize, nullptr, xRank - 1,
- resultExtent, CFI_attribute_allocatable);
- for (int j{0}; j + 1 < xRank; ++j) {
- result.GetDimension(j).SetBounds(1, resultExtent[j]);
- }
- if (int stat{result.Allocate(kNoAsyncObject)}) {
- terminator.Crash(
- "%s: could not allocate memory for result; STAT=%d", intrinsic, stat);
- }
-}
-
-RT_OFFLOAD_API_GROUP_END
} // namespace Fortran::runtime
diff --git a/flang-rt/unittests/Runtime/Assign.cpp b/flang-rt/unittests/Runtime/Assign.cpp
index a53d6c7e07f07..4f2e6f30c1a09 100644
--- a/flang-rt/unittests/Runtime/Assign.cpp
+++ b/flang-rt/unittests/Runtime/Assign.cpp
@@ -531,7 +531,6 @@ TEST(Assign, RTNAME(CopyOutAssignReadOnlyUnmodified)) {
}
#endif
-#if defined(__unix__) || defined(__APPLE__)
#if defined(__unix__) || defined(__APPLE__)
TEST(Assign, RTNAME(CopyOutAssignEnvVarParsing)) {
// Exercise the FLANG_RT_COPYOUT_MODIFIED_ONLY parsing path in
@@ -560,7 +559,6 @@ TEST(Assign, RTNAME(CopyOutAssignEnvVarParsing)) {
executionEnvironment.copyOutModifiedOnly = saved;
}
-#endif
TEST(Assign, RTNAME(CopyOutAssignUnconditionalEnvVar)) {
// With FLANG_RT_COPYOUT_MODIFIED_ONLY=0 semantics (unconditional copy-out),
@@ -643,10 +641,10 @@ TEST(Assign, RTNAME(CopyOutAssignSkipsUnmodifiedPrefix)) {
}
TEST(Assign, RTNAME(CopyOutAssignReadOnlyModifiedDies)) {
- // When the callee DID modify the temporary, copy-out falls back to the
- // whole-object copy; storing into a read-only original then faults, which
- // is the intended behavior for a program that modifies a non-definable
- // actual argument.
+ // When the callee DID modify the temporary, copy-out stores the modified
+ // suffix -- from the first differing element through the end; storing into
+ // a read-only original then faults, which is the intended behavior for a
+ // program that modifies a non-definable actual argument.
std::size_t pageSize{static_cast<std::size_t>(sysconf(_SC_PAGESIZE))};
void *page{mmap(nullptr, pageSize, PROT_READ | PROT_WRITE,
MAP_PRIVATE | MAP_ANONYMOUS, -1, 0)};
diff --git a/flang/docs/RuntimeEnvironment.md b/flang/docs/RuntimeEnvironment.md
index 0d3e274328a8a..16e75d3ab1e9c 100644
--- a/flang/docs/RuntimeEnvironment.md
+++ b/flang/docs/RuntimeEnvironment.md
@@ -40,14 +40,16 @@ The system environment variable `FLANG_RT_COPYOUT_MODIFIED_ONLY` selects how
the runtime performs copy-out.
When the compiler passes a copy of an actual argument to a procedure
-(copy-in/copy-out), the runtime skips the copy-out entirely when the
-temporary copy is still bitwise-identical to the original, and performs
-the normal whole-object copy-out otherwise. This avoids stores to the
-original argument when the callee never modified the data -- in
-particular, stores into read-only storage backing a non-definable actual
-argument.
+(copy-in/copy-out), the runtime scans the temporary for the first element
+whose bit pattern differs from the original and copies back only from that
+element through the end; when the callee never modified the copy, nothing
+is stored at all. This avoids stores to the original argument when the
+callee never modified the data -- in particular, stores into read-only
+storage backing a non-definable actual argument.
Set the system environment variable `FLANG_RT_COPYOUT_MODIFIED_ONLY=0` to
-restore the unconditional copy-out.
+restore the unconditional whole-object copy-out. Note that this restores
+stores of unmodified data as well (including into read-only storage), so it
+is an escape hatch and A/B-comparison aid, not a safer mode.
## `FORT_CHECK_POINTER_DEALLOCATION`
diff --git a/flang/include/flang/Runtime/assign.h b/flang/include/flang/Runtime/assign.h
index bf050ff890068..172453334ef15 100644
--- a/flang/include/flang/Runtime/assign.h
+++ b/flang/include/flang/Runtime/assign.h
@@ -66,7 +66,13 @@ void RTDECL(AssignTemporary)(Descriptor &to, const Descriptor &from,
void RTDECL(CopyInAssign)(Descriptor &temp, const Descriptor &var,
const char *sourceFile = nullptr, int sourceLine = 0);
// When "var" is provided, copy "temp" to it assuming "var" is already
-// initialized. Destroy and deallocate "temp" in all cases.
+// initialized. The copy is performed only from the first element of "temp"
+// whose bit pattern differs from the corresponding element of "var" through
+// the last element; when "temp" is bitwise identical to "var", nothing is
+// stored, so a "var" backed by read-only storage is not written to unless
+// it was actually modified. Setting the system environment variable
+// FLANG_RT_COPYOUT_MODIFIED_ONLY=0 restores the unconditional whole-object
+// copy. Destroy and deallocate "temp" in all cases.
void RTDECL(CopyOutAssign)(Descriptor *var, Descriptor &temp,
const char *sourceFile = nullptr, int sourceLine = 0);
// This variant is for assignments to explicit-length CHARACTER left-hand
More information about the flang-commits
mailing list