[compiler-rt] [rtsan] Fix va_args handling in open functions (PR #108291)

[Chris Apple via llvm-commits]

https://github.com/cjappl updated https://github.com/llvm/llvm-project/pull/108291

>From 1363f42967ea729026fbccdbadd63ea1916bb002 Mon Sep 17 00:00:00 2001
From: Chris Apple <cja-private at pm.me>
Date: Wed, 11 Sep 2024 14:41:14 -0600
Subject: [PATCH 1/2] [rtsan] Fix va_args handling in open functions

---
 compiler-rt/lib/rtsan/rtsan_interceptors.cpp | 28 +++++++++++---------
 1 file changed, 16 insertions(+), 12 deletions(-)

diff --git a/compiler-rt/lib/rtsan/rtsan_interceptors.cpp b/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
index 409e27c3ad3234..a9398cf284d827 100644
--- a/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
@@ -64,13 +64,15 @@ INTERCEPTOR(int, open, const char *path, int oflag, ...) {
   // O_NONBLOCK
   __rtsan_expect_not_realtime("open");
 
-  va_list args;
-  va_start(args, oflag);
-  const mode_t mode = va_arg(args, int);
-  va_end(args);
+  if (oflag & O_CREAT) {
+    va_list args;
+    va_start(args, oflag);
+    const mode_t mode = va_arg(args, int);
+    va_end(args);
+    return REAL(open)(path, oflag, mode);
+  }
 
-  const int result = REAL(open)(path, oflag, mode);
-  return result;
+  return REAL(open)(path, oflag);
 }
 
 INTERCEPTOR(int, openat, int fd, const char *path, int oflag, ...) {
@@ -78,13 +80,15 @@ INTERCEPTOR(int, openat, int fd, const char *path, int oflag, ...) {
   // O_NONBLOCK
   __rtsan_expect_not_realtime("openat");
 
-  va_list args;
-  va_start(args, oflag);
-  mode_t mode = va_arg(args, int);
-  va_end(args);
+  if (oflag & O_CREAT) {
+    va_list args;
+    va_start(args, oflag);
+    const mode_t mode = va_arg(args, int);
+    va_end(args);
+    return REAL(openat)(fd, path, oflag, mode);
+  }
 
-  const int result = REAL(openat)(fd, path, oflag, mode);
-  return result;
+  return REAL(openat)(fd, path, oflag);
 }
 
 INTERCEPTOR(int, creat, const char *path, mode_t mode) {

>From e372168b52c5d2344cd004c2122a7c7ae00d2d31 Mon Sep 17 00:00:00 2001
From: Chris Apple <cja-private at pm.me>
Date: Sat, 14 Sep 2024 06:48:31 -0600
Subject: [PATCH 2/2] [PR] vitalybuka - Ensure we read O_TMPFILE as well

---
 compiler-rt/lib/rtsan/rtsan_interceptors.cpp | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/compiler-rt/lib/rtsan/rtsan_interceptors.cpp b/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
index a9398cf284d827..0724bd7cb17457 100644
--- a/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
+++ b/compiler-rt/lib/rtsan/rtsan_interceptors.cpp
@@ -58,13 +58,20 @@ struct DlsymAlloc : public DlSymAllocator<DlsymAlloc> {
 } // namespace
 
 // Filesystem
+static bool open_reads_va_args(int oflag) {
+#ifdef O_TMPFILE
+  return (oflag & (O_CREAT | O_TMPFILE)) != 0;
+#else
+  return (oflag & O_CREAT) != 0;
+#endif
+}
 
 INTERCEPTOR(int, open, const char *path, int oflag, ...) {
   // TODO Establish whether we should intercept here if the flag contains
   // O_NONBLOCK
   __rtsan_expect_not_realtime("open");
 
-  if (oflag & O_CREAT) {
+  if (open_reads_va_args(oflag)) {
     va_list args;
     va_start(args, oflag);
     const mode_t mode = va_arg(args, int);
@@ -80,7 +87,7 @@ INTERCEPTOR(int, openat, int fd, const char *path, int oflag, ...) {
   // O_NONBLOCK
   __rtsan_expect_not_realtime("openat");
 
-  if (oflag & O_CREAT) {
+  if (open_reads_va_args(oflag)) {
     va_list args;
     va_start(args, oflag);
     const mode_t mode = va_arg(args, int);