[flang-commits] [flang] [llvm] [flang-rt] Split large unformatted sequential records into subrecords (PR #213003)

[via flang-commits]

================
@@ -607,67 +711,130 @@ void ExternalFileUnit::EndIoStatement() {
 void ExternalFileUnit::BeginSequentialVariableUnformattedInputRecord(
     IoErrorHandler &handler) {
   RUNTIME_CHECK(handler, access == Access::Sequential);
-  std::uint32_t header{0}, footer{0};
-  std::size_t need{recordOffsetInFrame_ + sizeof header};
-  std::size_t got{ReadFrame(frameOffsetInFile_, need, handler)};
-  // Try to emit informative errors to help debug corrupted files.
+  unformattedDiskExtent_ = 0;
+  streamRead_ = false;
+  FileOffset recordStart{
+      frameOffsetInFile_ + static_cast<FileOffset>(recordOffsetInFrame_)};
   const char *error{nullptr};
-  if (got < need) {
-    if (got == recordOffsetInFrame_) {
-      HitEndOnRead(handler);
+  std::int32_t header{0}, footer{0}, firstHeader{0};
+  std::size_t totalPayload{0};
+  FileOffset filePos{recordStart};
+
+  enum class MarkerRead { Ok, Eof, Short };
+  enum class ScanOutcome { Ok, Eof, Fail };
+  const char *scanError{nullptr};
+
+  auto readMarkerAt = [&](FileOffset &pos, std::int32_t &marker, bool swap) {
+    char buf[kRecMarkSize];
+    auto got{Read(pos, buf, kRecMarkSize, kRecMarkSize, handler)};
+    if (got < kRecMarkSize) {
+      return got == 0 ? MarkerRead::Eof : MarkerRead::Short;
+    }
+    pos += kRecMarkSize;
+    marker = LoadRecordMarker(buf, swap);
+    return MarkerRead::Ok;
+  };
+
+  auto scanRecord = [&](bool swap, std::int32_t &outHeader,
+                        std::int32_t &outFooter, std::int32_t &outFirstHeader,
+                        std::size_t &outTotalPayload,
+                        FileOffset &outFilePos) -> ScanOutcome {
+    scanError = nullptr;
+    outFilePos = recordStart;
+    outTotalPayload = 0;
+    switch (readMarkerAt(outFilePos, outHeader, swap)) {
+    case MarkerRead::Ok:
+      break;
+    case MarkerRead::Eof:
+      return ScanOutcome::Eof;
+    case MarkerRead::Short:
+      scanError = "Unformatted variable-length sequential file input failed at "
+                  "record #%jd (file offset %jd): truncated record header";
+      return ScanOutcome::Fail;
+    }
+    outFirstHeader = outHeader;
+    do {
+      outTotalPayload += SubrecordPayloadBytes(outHeader);
+      outFilePos += static_cast<FileOffset>(SubrecordPayloadBytes(outHeader));
+      if (readMarkerAt(outFilePos, outFooter, swap) != MarkerRead::Ok) {
+        scanError =
+            "Unformatted variable-length sequential file input failed at "
+            "record #%jd (file offset %jd): truncated record footer";
+        return ScanOutcome::Fail;
+      }
+      if (SubrecordPayloadBytes(outFooter) !=
+          SubrecordPayloadBytes(outHeader)) {
+        scanError =
+            "Unformatted variable-length sequential file input failed at "
+            "record #%jd (file offset %jd): record header has length %jd "
+            "that does not match record footer (%jd)";
+        return ScanOutcome::Fail;
+      }
+      if (outHeader >= 0) {
+        return ScanOutcome::Ok;
+      }
+      if (readMarkerAt(outFilePos, outHeader, swap) != MarkerRead::Ok) {
+        scanError =
+            "Unformatted variable-length sequential file input failed at "
+            "record #%jd (file offset %jd): truncated record header";
+        return ScanOutcome::Fail;
+      }
+    } while (true);
+  };
+
+  auto outcome{scanRecord(
+      swapEndianness_, header, footer, firstHeader, totalPayload, filePos)};
+  if (outcome == ScanOutcome::Eof) {
+    HitEndOnRead(handler);
+    return;
+  }
+  if (outcome == ScanOutcome::Fail) {
+    const char *primaryError{scanError};
+    if (frameOffsetInFile_ == 0 && recordOffsetInFrame_ == 0 && [&]() {
+          std::int32_t swappedHeader{0}, swappedFooter{0},
+              swappedFirstHeader{0};
+          std::size_t swappedTotalPayload{0};
+          FileOffset swappedFilePos{recordStart};
+          return scanRecord(!swapEndianness_, swappedHeader, swappedFooter,
+                     swappedFirstHeader, swappedTotalPayload,
+                     swappedFilePos) == ScanOutcome::Ok;
+        }()) {
+      error =
+          "Unformatted variable-length sequential file input failed on the "
+          "first record, probably due to a need for byte order data "
+          "conversion; consider adding CONVERT='SWAP' to the OPEN statement "
+          "or adding FORT_CONVERT=SWAP to the execution environment";
     } else {
-      error = "Unformatted variable-length sequential file input failed at "
-              "record #%jd (file offset %jd): truncated record header";
+      error = primaryError
+          ? primaryError
+          : "Unformatted variable-length sequential file input "
+            "failed at record #%jd (file offset %jd): truncated "
+            "record header";
     }
----------------
ejose02 wrote:

Done. set recordLength before footer check so we advance after a bad record.

https://github.com/llvm/llvm-project/pull/213003