[clang] Fix preprocessed block comment newline on windows containing extra carriage return (PR #205084)
Joe Kirchoff via cfe-commits
cfe-commits at lists.llvm.org
Mon Jul 13 03:47:49 PDT 2026
https://github.com/Rinn updated https://github.com/llvm/llvm-project/pull/205084
>From 25db0f0051754ebc88a1b42438d4d593b486f934 Mon Sep 17 00:00:00 2001
From: Joe Kirchoff <joe.kirchoff at epicgames.com>
Date: Mon, 22 Jun 2026 12:29:50 +0100
Subject: [PATCH 1/7] Fix preprocessed block comment newline on windows
When preprocessing a source file on windows with \r\n newlines while preserving comments, any newlines in a block comment are written directly which is then later normalized to \r\r\n. Fix this by only writing out \n in PrintPreprocessedTokens for those comments by copying the logic from PrintPPOutputPPCallbacks::HandleNewlinesInToken
---
.../lib/Frontend/PrintPreprocessedOutput.cpp | 30 +++++++++++++++++--
1 file changed, 28 insertions(+), 2 deletions(-)
diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
index 02266882c4c4a..16c09d1263e30 100644
--- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -889,6 +889,26 @@ struct UnknownPragmaHandler : public PragmaHandler {
};
} // end anonymous namespace
+static void PrintPreprocessedComment(raw_ostream *OS, const char *TokStr,
+ unsigned Len) {
+ for (; Len; --Len, ++TokStr) {
+ if (*TokStr != '\n' &&
+ *TokStr != '\r') {
+ *OS << *TokStr;
+ continue;
+ }
+
+ *OS << '\n';
+
+ // If we have \n\r or \r\n, skip both and emit one newline.
+ if (Len != 1 &&
+ (TokStr[1] == '\n' || TokStr[1] == '\r') &&
+ TokStr[0] != TokStr[1]) {
+ ++TokStr;
+ --Len;
+ }
+ }
+}
static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
PrintPPOutputPPCallbacks *Callbacks) {
@@ -1022,7 +1042,10 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
} else if (Tok.getLength() < std::size(Buffer)) {
const char *TokPtr = Buffer;
unsigned Len = PP.getSpelling(Tok, TokPtr);
- Callbacks->OS->write(TokPtr, Len);
+ if (Tok.is(tok::comment))
+ PrintPreprocessedComment(Callbacks->OS, TokPtr, Len);
+ else
+ Callbacks->OS->write(TokPtr, Len);
// Tokens that can contain embedded newlines need to adjust our current
// line number.
@@ -1039,7 +1062,10 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
}
} else {
std::string S = PP.getSpelling(Tok);
- Callbacks->OS->write(S.data(), S.size());
+ if (Tok.is(tok::comment))
+ PrintPreprocessedComment(Callbacks->OS, S.data(), S.size());
+ else
+ Callbacks->OS->write(S.data(), S.size());
// Tokens that can contain embedded newlines need to adjust our current
// line number.
>From 515cd83647efe645fe4ece3c3e2e306cd57c09ec Mon Sep 17 00:00:00 2001
From: Joe Kirchoff <joe.kirchoff at epicgames.com>
Date: Mon, 22 Jun 2026 12:48:29 +0100
Subject: [PATCH 2/7] clang-format
---
clang/lib/Frontend/PrintPreprocessedOutput.cpp | 6 ++----
1 file changed, 2 insertions(+), 4 deletions(-)
diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
index 16c09d1263e30..40c1b89366a7c 100644
--- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -892,8 +892,7 @@ struct UnknownPragmaHandler : public PragmaHandler {
static void PrintPreprocessedComment(raw_ostream *OS, const char *TokStr,
unsigned Len) {
for (; Len; --Len, ++TokStr) {
- if (*TokStr != '\n' &&
- *TokStr != '\r') {
+ if (*TokStr != '\n' && *TokStr != '\r') {
*OS << *TokStr;
continue;
}
@@ -901,8 +900,7 @@ static void PrintPreprocessedComment(raw_ostream *OS, const char *TokStr,
*OS << '\n';
// If we have \n\r or \r\n, skip both and emit one newline.
- if (Len != 1 &&
- (TokStr[1] == '\n' || TokStr[1] == '\r') &&
+ if (Len != 1 && (TokStr[1] == '\n' || TokStr[1] == '\r') &&
TokStr[0] != TokStr[1]) {
++TokStr;
--Len;
>From dec4c02c62716739b9969321f191970f02a6e046 Mon Sep 17 00:00:00 2001
From: Joe Kirchoff <Kirchoff.Joseph.P at gmail.com>
Date: Fri, 26 Jun 2026 09:13:35 +0100
Subject: [PATCH 3/7] Apply suggestions from code review
Co-authored-by: Aaron Ballman <aaron at aaronballman.com>
---
clang/lib/Frontend/PrintPreprocessedOutput.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
index 40c1b89366a7c..38a802dd7614b 100644
--- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -1040,7 +1040,7 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
} else if (Tok.getLength() < std::size(Buffer)) {
const char *TokPtr = Buffer;
unsigned Len = PP.getSpelling(Tok, TokPtr);
- if (Tok.is(tok::comment))
+ if (Tok.is(tok::comment, tok::unknown))
PrintPreprocessedComment(Callbacks->OS, TokPtr, Len);
else
Callbacks->OS->write(TokPtr, Len);
@@ -1060,7 +1060,7 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
}
} else {
std::string S = PP.getSpelling(Tok);
- if (Tok.is(tok::comment))
+ if (Tok.is(tok::comment, tok::unknown))
PrintPreprocessedComment(Callbacks->OS, S.data(), S.size());
else
Callbacks->OS->write(S.data(), S.size());
>From b9afb477f1e82741f15275046d45176428cffb1e Mon Sep 17 00:00:00 2001
From: Joe Kirchoff <joe.kirchoff at epicgames.com>
Date: Fri, 26 Jun 2026 10:12:25 +0100
Subject: [PATCH 4/7] Fix Token.is only accepting one paramter, rename
PrintPreprocessedComment
---
clang/lib/Frontend/PrintPreprocessedOutput.cpp | 12 ++++++------
1 file changed, 6 insertions(+), 6 deletions(-)
diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
index 38a802dd7614b..57de5126bcd71 100644
--- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -889,8 +889,8 @@ struct UnknownPragmaHandler : public PragmaHandler {
};
} // end anonymous namespace
-static void PrintPreprocessedComment(raw_ostream *OS, const char *TokStr,
- unsigned Len) {
+static void WriteMultilineToken(raw_ostream *OS, const char *TokStr,
+ unsigned Len) {
for (; Len; --Len, ++TokStr) {
if (*TokStr != '\n' && *TokStr != '\r') {
*OS << *TokStr;
@@ -1040,8 +1040,8 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
} else if (Tok.getLength() < std::size(Buffer)) {
const char *TokPtr = Buffer;
unsigned Len = PP.getSpelling(Tok, TokPtr);
- if (Tok.is(tok::comment, tok::unknown))
- PrintPreprocessedComment(Callbacks->OS, TokPtr, Len);
+ if (Tok.is(tok::comment) || Tok.is(tok::unknown))
+ WriteMultilineToken(Callbacks->OS, TokPtr, Len);
else
Callbacks->OS->write(TokPtr, Len);
@@ -1060,8 +1060,8 @@ static void PrintPreprocessedTokens(Preprocessor &PP, Token &Tok,
}
} else {
std::string S = PP.getSpelling(Tok);
- if (Tok.is(tok::comment, tok::unknown))
- PrintPreprocessedComment(Callbacks->OS, S.data(), S.size());
+ if (Tok.is(tok::comment) || Tok.is(tok::unknown))
+ WriteMultilineToken(Callbacks->OS, S.data(), S.size());
else
Callbacks->OS->write(S.data(), S.size());
>From 18546015bba9055ea3d90a00b45da6e9ce8a0e98 Mon Sep 17 00:00:00 2001
From: Joe Kirchoff <joe.kirchoff at epicgames.com>
Date: Wed, 1 Jul 2026 10:58:41 +0100
Subject: [PATCH 5/7] Add test
---
clang/test/.gitattributes | 1 +
clang/test/Preprocessor/Inputs/comment_save_crlf.h | 5 +++++
clang/test/Preprocessor/comment_save_crlf.c | 10 ++++++++++
3 files changed, 16 insertions(+)
create mode 100644 clang/test/.gitattributes
create mode 100644 clang/test/Preprocessor/Inputs/comment_save_crlf.h
create mode 100644 clang/test/Preprocessor/comment_save_crlf.c
diff --git a/clang/test/.gitattributes b/clang/test/.gitattributes
new file mode 100644
index 0000000000000..0e7628b9abfda
--- /dev/null
+++ b/clang/test/.gitattributes
@@ -0,0 +1 @@
+Preprocessor/Inputs/comment_save_crlf.h -text
diff --git a/clang/test/Preprocessor/Inputs/comment_save_crlf.h b/clang/test/Preprocessor/Inputs/comment_save_crlf.h
new file mode 100644
index 0000000000000..6cd956eff14ff
--- /dev/null
+++ b/clang/test/Preprocessor/Inputs/comment_save_crlf.h
@@ -0,0 +1,5 @@
+int a;
+/* block comment
+ spanning multiple
+ CRLF-terminated lines */
+int b;
diff --git a/clang/test/Preprocessor/comment_save_crlf.c b/clang/test/Preprocessor/comment_save_crlf.c
new file mode 100644
index 0000000000000..7e8a01e3d1b02
--- /dev/null
+++ b/clang/test/Preprocessor/comment_save_crlf.c
@@ -0,0 +1,10 @@
+// REQUIRES: system-windows
+
+// RUN: %clang_cc1 -E -C -o %t %S/Inputs/comment_save_crlf.h
+// RUN: FileCheck --strict-whitespace --match-full-lines --input-file=%t %s
+
+// CHECK:int a;
+// CHECK-NEXT:/* block comment
+// CHECK-NEXT: spanning multiple
+// CHECK-NEXT: CRLF-terminated lines */
+// CHECK-NEXT:int b;
>From b92e042469b25b01d7618b9af7526b579902f33e Mon Sep 17 00:00:00 2001
From: Joe Kirchoff <joe.kirchoff at epicgames.com>
Date: Mon, 13 Jul 2026 11:08:51 +0100
Subject: [PATCH 6/7] Revert "Add test"
This reverts commit 18546015bba9055ea3d90a00b45da6e9ce8a0e98.
---
clang/test/.gitattributes | 1 -
clang/test/Preprocessor/Inputs/comment_save_crlf.h | 5 -----
clang/test/Preprocessor/comment_save_crlf.c | 10 ----------
3 files changed, 16 deletions(-)
delete mode 100644 clang/test/.gitattributes
delete mode 100644 clang/test/Preprocessor/Inputs/comment_save_crlf.h
delete mode 100644 clang/test/Preprocessor/comment_save_crlf.c
diff --git a/clang/test/.gitattributes b/clang/test/.gitattributes
deleted file mode 100644
index 0e7628b9abfda..0000000000000
--- a/clang/test/.gitattributes
+++ /dev/null
@@ -1 +0,0 @@
-Preprocessor/Inputs/comment_save_crlf.h -text
diff --git a/clang/test/Preprocessor/Inputs/comment_save_crlf.h b/clang/test/Preprocessor/Inputs/comment_save_crlf.h
deleted file mode 100644
index 6cd956eff14ff..0000000000000
--- a/clang/test/Preprocessor/Inputs/comment_save_crlf.h
+++ /dev/null
@@ -1,5 +0,0 @@
-int a;
-/* block comment
- spanning multiple
- CRLF-terminated lines */
-int b;
diff --git a/clang/test/Preprocessor/comment_save_crlf.c b/clang/test/Preprocessor/comment_save_crlf.c
deleted file mode 100644
index 7e8a01e3d1b02..0000000000000
--- a/clang/test/Preprocessor/comment_save_crlf.c
+++ /dev/null
@@ -1,10 +0,0 @@
-// REQUIRES: system-windows
-
-// RUN: %clang_cc1 -E -C -o %t %S/Inputs/comment_save_crlf.h
-// RUN: FileCheck --strict-whitespace --match-full-lines --input-file=%t %s
-
-// CHECK:int a;
-// CHECK-NEXT:/* block comment
-// CHECK-NEXT: spanning multiple
-// CHECK-NEXT: CRLF-terminated lines */
-// CHECK-NEXT:int b;
>From 819433176693f59fe4ebcc923ad57e958d639a7e Mon Sep 17 00:00:00 2001
From: Joe Kirchoff <joe.kirchoff at epicgames.com>
Date: Mon, 13 Jul 2026 11:43:50 +0100
Subject: [PATCH 7/7] Add test
---
clang/test/Preprocessor/Inputs/comment_save_crlf.h | 5 +++++
clang/test/Preprocessor/comment_save_crlf.c | 6 ++++++
clang/test/lit.cfg.py | 7 +++++++
3 files changed, 18 insertions(+)
create mode 100644 clang/test/Preprocessor/Inputs/comment_save_crlf.h
create mode 100644 clang/test/Preprocessor/comment_save_crlf.c
diff --git a/clang/test/Preprocessor/Inputs/comment_save_crlf.h b/clang/test/Preprocessor/Inputs/comment_save_crlf.h
new file mode 100644
index 0000000000000..d942ddf0b1c64
--- /dev/null
+++ b/clang/test/Preprocessor/Inputs/comment_save_crlf.h
@@ -0,0 +1,5 @@
+int a;
+/* multi-line
+ block
+ comment */
+int b;
diff --git a/clang/test/Preprocessor/comment_save_crlf.c b/clang/test/Preprocessor/comment_save_crlf.c
new file mode 100644
index 0000000000000..1aa4d967b2ff3
--- /dev/null
+++ b/clang/test/Preprocessor/comment_save_crlf.c
@@ -0,0 +1,6 @@
+// REQUIRES: system-windows
+
+// RUN: %{to-crlf} %S/Inputs/comment_save_crlf.h > %t.h
+// RUN: %clang_cc1 -E -C -o %t.i %t.h
+// RUN: %{reveal-cr} %t.i | FileCheck %s
+// CHECK-NOT: <CR>
diff --git a/clang/test/lit.cfg.py b/clang/test/lit.cfg.py
index f7b3a77266cb8..42d5ccaee3daa 100644
--- a/clang/test/lit.cfg.py
+++ b/clang/test/lit.cfg.py
@@ -112,6 +112,13 @@
),
)
)
+
+# Rewrite LF to CRLF
+config.substitutions.append(("%{to-crlf}", f"{sed_cmd} -e 's/\r//g' -e 's/$/\r/'"))
+
+# Filtering command for testing for carriage return
+config.substitutions.append(("%{reveal-cr}", f"{sed_cmd} -e 's/\r/<CR>/g'"))
+
# For each occurrence of a clang tool name, replace it with the full path to
# the build directory holding that tool. We explicitly specify the directories
# to search to ensure that we get the tools just built and not some random
More information about the cfe-commits
mailing list