[flang-commits] [flang] [llvm] [flang-rt] Split large unformatted sequential records into subrecords (PR #213003)

[via flang-commits]

================
@@ -607,67 +711,130 @@ void ExternalFileUnit::EndIoStatement() {
 void ExternalFileUnit::BeginSequentialVariableUnformattedInputRecord(
     IoErrorHandler &handler) {
   RUNTIME_CHECK(handler, access == Access::Sequential);
-  std::uint32_t header{0}, footer{0};
-  std::size_t need{recordOffsetInFrame_ + sizeof header};
-  std::size_t got{ReadFrame(frameOffsetInFile_, need, handler)};
-  // Try to emit informative errors to help debug corrupted files.
+  unformattedDiskExtent_ = 0;
+  streamRead_ = false;
+  FileOffset recordStart{
+      frameOffsetInFile_ + static_cast<FileOffset>(recordOffsetInFrame_)};
   const char *error{nullptr};
-  if (got < need) {
-    if (got == recordOffsetInFrame_) {
-      HitEndOnRead(handler);
+  std::int32_t header{0}, footer{0}, firstHeader{0};
+  std::size_t totalPayload{0};
+  FileOffset filePos{recordStart};
+
+  enum class MarkerRead { Ok, Eof, Short };
+  enum class ScanOutcome { Ok, Eof, Fail };
+  const char *scanError{nullptr};
+
+  auto readMarkerAt = [&](FileOffset &pos, std::int32_t &marker, bool swap) {
+    char buf[kRecMarkSize];
+    auto got{Read(pos, buf, kRecMarkSize, kRecMarkSize, handler)};
+    if (got < kRecMarkSize) {
+      return got == 0 ? MarkerRead::Eof : MarkerRead::Short;
+    }
+    pos += kRecMarkSize;
+    marker = LoadRecordMarker(buf, swap);
+    return MarkerRead::Ok;
+  };
+
+  auto scanRecord = [&](bool swap, std::int32_t &outHeader,
+                        std::int32_t &outFooter, std::int32_t &outFirstHeader,
+                        std::size_t &outTotalPayload,
+                        FileOffset &outFilePos) -> ScanOutcome {
+    scanError = nullptr;
+    outFilePos = recordStart;
+    outTotalPayload = 0;
+    switch (readMarkerAt(outFilePos, outHeader, swap)) {
+    case MarkerRead::Ok:
+      break;
+    case MarkerRead::Eof:
+      return ScanOutcome::Eof;
+    case MarkerRead::Short:
+      scanError = "Unformatted variable-length sequential file input failed at "
+                  "record #%jd (file offset %jd): truncated record header";
+      return ScanOutcome::Fail;
+    }
+    outFirstHeader = outHeader;
+    do {
+      outTotalPayload += SubrecordPayloadBytes(outHeader);
+      outFilePos += static_cast<FileOffset>(SubrecordPayloadBytes(outHeader));
+      if (readMarkerAt(outFilePos, outFooter, swap) != MarkerRead::Ok) {
----------------
ejose02 wrote:

Fixed: single-marker path for ordinary records; chain/streaming only when needed. ~27 reads again on the 100k-record case.

https://github.com/llvm/llvm-project/pull/213003