[clang] [llvm] Alisonz/filetag conversion (PR #206519)
Alison Zhang via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 3 06:01:10 PDT 2026
https://github.com/azhan92 updated https://github.com/llvm/llvm-project/pull/206519
>From 792ce0ab221b94a0cc51f34f86b977c2c1c5d3c0 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Sun, 28 Jun 2026 15:47:43 -0400
Subject: [PATCH 1/8] Get canonical names for encodings
---
llvm/include/llvm/Support/TextEncoding.h | 11 +++++++++++
llvm/lib/Support/TextEncoding.cpp | 13 ++++++++++++-
2 files changed, 23 insertions(+), 1 deletion(-)
diff --git a/llvm/include/llvm/Support/TextEncoding.h b/llvm/include/llvm/Support/TextEncoding.h
index 8a304910aa5dd..09e24000594db 100644
--- a/llvm/include/llvm/Support/TextEncoding.h
+++ b/llvm/include/llvm/Support/TextEncoding.h
@@ -105,6 +105,17 @@ class TextEncodingConverter {
LLVM_ABI static ErrorOr<TextEncodingConverter> create(StringRef From,
StringRef To);
+ /// Maps the encoding name to enum constant if possible.
+ /// Uses normalized charset name matching.
+ /// \param[in] Name the character encoding name
+ /// \return the TextEncoding enum value if known, std::nullopt otherwise
+ LLVM_ABI static std::optional<TextEncoding> getKnownEncoding(StringRef Name);
+
+ /// Returns the canonical name for a known encoding.
+ /// \param[in] Encoding the TextEncoding enum value
+ /// \return the canonical name for the encoding (e.g., "UTF-8" or "IBM-1047")
+ LLVM_ABI static StringRef getKnownEncodingName(TextEncoding Encoding);
+
TextEncodingConverter(const TextEncodingConverter &) = delete;
TextEncodingConverter &operator=(const TextEncodingConverter &) = delete;
diff --git a/llvm/lib/Support/TextEncoding.cpp b/llvm/lib/Support/TextEncoding.cpp
index d36f02c1300b9..8e9653dab38ec 100644
--- a/llvm/lib/Support/TextEncoding.cpp
+++ b/llvm/lib/Support/TextEncoding.cpp
@@ -48,7 +48,7 @@ static void normalizeCharSetName(StringRef CSName,
}
// Maps the encoding name to enum constant if possible.
-static std::optional<TextEncoding> getKnownEncoding(StringRef Name) {
+std::optional<TextEncoding> TextEncodingConverter::getKnownEncoding(StringRef Name) {
SmallString<16> Normalized;
normalizeCharSetName(Name, Normalized);
if (Normalized.equals("utf8"))
@@ -58,6 +58,17 @@ static std::optional<TextEncoding> getKnownEncoding(StringRef Name) {
return std::nullopt;
}
+// Returns the canonical name for a known encoding.
+StringRef TextEncodingConverter::getKnownEncodingName(TextEncoding Encoding) {
+ switch (Encoding) {
+ case TextEncoding::UTF8:
+ return "UTF-8";
+ case TextEncoding::IBM1047:
+ return "IBM-1047";
+ }
+ llvm_unreachable("Invalid TextEncoding value");
+}
+
[[maybe_unused]] static void HandleOverflow(size_t &Capacity, char *&Output,
size_t &OutputLength,
SmallVectorImpl<char> &Result) {
>From 41a96501462d2ff871731a727f923ec89ea81268 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Sun, 28 Jun 2026 15:50:57 -0400
Subject: [PATCH 2/8] Text mode mismatch checking
---
clang/lib/Basic/FileManager.cpp | 17 +++++++++-----
llvm/include/llvm/Support/VirtualFileSystem.h | 9 ++++++++
llvm/lib/Support/VirtualFileSystem.cpp | 22 ++++++++++++++++---
3 files changed, 40 insertions(+), 8 deletions(-)
diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp
index 8fb3ba0a27aad..94fc4c15f51d2 100644
--- a/clang/lib/Basic/FileManager.cpp
+++ b/clang/lib/Basic/FileManager.cpp
@@ -539,15 +539,22 @@ FileManager::getBufferForFile(FileEntryRef FE, bool isVolatile,
FileSize = -1;
StringRef Filename = FE.getName();
- // If the file is already open, use the open file descriptor.
+ // If the file is already open, check if the mode matches.
if (Entry->File) {
- auto Result = Entry->File->getBuffer(Filename, FileSize,
- RequiresNullTerminator, isVolatile);
+ // Check if the cached file's mode matches the requested mode
+ // Only perform mismatch recovery for real files
+ if (!Entry->File->realFileCheckTextModeMismatch(IsText)) {
+ // Mode matches, use the cached file descriptor
+ auto Result = Entry->File->getBuffer(Filename, FileSize,
+ RequiresNullTerminator, isVolatile);
+ Entry->closeFile();
+ return Result;
+ }
+ // Mode mismatch - close the cached file and reopen with correct mode
Entry->closeFile();
- return Result;
}
- // Otherwise, open the file.
+ // Open the file with the requested mode.
return getBufferForFileImpl(Filename, FileSize, isVolatile,
RequiresNullTerminator, IsText);
}
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index d22c534228331..9cc35b6f5fb6d 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -137,6 +137,15 @@ class LLVM_ABI File {
/// Closes the file.
virtual std::error_code close() = 0;
+ /// Checks if this is a real file and the requested text mode differs
+ /// from the current mode. For real files with a text mode mismatch where
+ /// the buffer was previously requested, this will call llvm::report_fatal_error.
+ /// Always returns false for non-real files.
+ /// Default implementation returns false for non-real files.
+ virtual bool realFileCheckTextModeMismatch(bool RequestedIsText) const {
+ return false;
+ }
+
// Get the same file with a different path.
static ErrorOr<std::unique_ptr<File>>
getWithPath(ErrorOr<std::unique_ptr<File>> Result, const Twine &P);
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 42e8bb4f9958e..1c1f9e9dd15b6 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -194,11 +194,15 @@ class RealFile : public File {
file_t FD;
Status S;
std::string RealName;
+ bool IsTextMode;
+ bool BufferWasRequested;
- RealFile(file_t RawFD, StringRef NewName, StringRef NewRealPathName)
+ RealFile(file_t RawFD, StringRef NewName, StringRef NewRealPathName,
+ bool IsText)
: FD(RawFD), S(NewName, {}, {}, {}, {}, {},
llvm::sys::fs::file_type::status_error, {}),
- RealName(NewRealPathName.str()) {
+ RealName(NewRealPathName.str()), IsTextMode(IsText),
+ BufferWasRequested(false) {
assert(FD != kInvalidFile && "Invalid or inactive file descriptor");
}
@@ -213,6 +217,16 @@ class RealFile : public File {
bool IsVolatile) override;
std::error_code close() override;
void setPath(const Twine &Path) override;
+ bool realFileCheckTextModeMismatch(bool RequestedIsText) const override {
+ bool HasMismatch = IsTextMode != RequestedIsText;
+ if (HasMismatch && BufferWasRequested) {
+ llvm::report_fatal_error(
+ "Text mode mismatch: file was previously opened with " +
+ Twine(IsTextMode ? "text" : "binary") + " mode, now requested with " +
+ Twine(RequestedIsText ? "text" : "binary") + " mode");
+ }
+ return HasMismatch;
+ }
};
} // namespace
@@ -242,6 +256,7 @@ RealFile::getBuffer(const Twine &Name, int64_t FileSize,
auto BypassSandbox = sys::sandbox::scopedDisable();
assert(FD != kInvalidFile && "cannot get buffer for closed file");
+ BufferWasRequested = true;
return MemoryBuffer::getOpenFile(FD, Name, FileSize, RequiresNullTerminator,
IsVolatile);
}
@@ -320,8 +335,9 @@ class RealFileSystem : public FileSystem {
adjustPath(Name, Storage), Flags, &RealName);
if (!FDOrErr)
return errorToErrorCode(FDOrErr.takeError());
+ bool IsText = (Flags & sys::fs::OF_Text) != sys::fs::OF_None;
return std::unique_ptr<File>(
- new RealFile(*FDOrErr, Name.str(), RealName.str()));
+ new RealFile(*FDOrErr, Name.str(), RealName.str(), IsText));
}
struct WorkingDirectory {
>From 568ba3dc979a4ffb60bb48279568daba5c84bf0f Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Sun, 28 Jun 2026 15:51:57 -0400
Subject: [PATCH 3/8] getEncodingNameFromFileTag
---
llvm/include/llvm/Support/AutoConvert.h | 26 +++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/llvm/include/llvm/Support/AutoConvert.h b/llvm/include/llvm/Support/AutoConvert.h
index d68b0e8b515e0..b437b157b7725 100644
--- a/llvm/include/llvm/Support/AutoConvert.h
+++ b/llvm/include/llvm/Support/AutoConvert.h
@@ -18,6 +18,7 @@
#include <_Ccsid.h>
#endif
#ifdef __cplusplus
+#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Support/Error.h"
#include <system_error>
@@ -105,6 +106,31 @@ inline ErrorOr<bool> needConversion(const Twine &FileName, const int FD = -1) {
return false;
}
+inline ErrorOr<SmallString<32>>
+getEncodingNameFromFileTag(const Twine &FileName, const int FD = -1) {
+#ifdef __MVS__
+ ErrorOr<__ccsid_t> TagOrErr = getzOSFileTag(FileName, FD);
+ if (!TagOrErr)
+ return TagOrErr.getError();
+
+ __ccsid_t Tag = *TagOrErr;
+ if (Tag == 0)
+ return SmallString<32>(); // Return empty string for no tag
+
+ if (Tag == 1208)
+ return SmallString<32>("utf-8");
+
+ if (Tag == 1047)
+ return SmallString<32>("ibm-1047");
+
+ SmallString<32> Result;
+ raw_svector_ostream(Result) << Tag;
+ return Result;
+#else
+ return SmallString<32>(); // Return empty string for non-MVS platforms
+#endif
+}
+
} /* namespace llvm */
#endif /* __cplusplus */
>From e76575e1dec9f16899b9e339a929e639ebb86df3 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Sun, 28 Jun 2026 16:02:38 -0400
Subject: [PATCH 4/8] Add cache for text encoding converters
---
clang/include/clang/Basic/SourceManager.h | 15 ++++++++++
clang/lib/Basic/SourceManager.cpp | 34 +++++++++++++++++++++++
2 files changed, 49 insertions(+)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 1939d1aa4915e..2b5cf0b1e084e 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -50,6 +50,7 @@
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/TextEncoding.h"
#include <cassert>
#include <cstddef>
#include <map>
@@ -846,6 +847,11 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// we can add a cc1-level option to do so.
SmallVector<std::pair<std::string, FullSourceLoc>, 2> StoredModuleBuildStack;
+ /// Cache of all text encoding converters used by this SourceManager.
+ /// This includes both the input charset converter and file tag converters.
+ /// Maps from "source_encoding:target_encoding" to the converter.
+ llvm::StringMap<std::unique_ptr<llvm::TextEncodingConverter>> ConverterCache;
+
public:
SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr,
bool UserFilesAreVolatile = false);
@@ -863,6 +869,15 @@ class SourceManager : public RefCountedBase<SourceManager> {
FileManager &getFileManager() const { return FileMgr; }
+ /// Get or create a text encoding converter from the cache.
+ /// This method manages all converters (input charset and file tag converters)
+ /// in a single cache owned by SourceManager.
+ /// \param SourceEncoding the source character encoding name
+ /// \return pointer to the converter or an error code
+ /// The target encoding is always UTF-8.
+ llvm::ErrorOr<llvm::TextEncodingConverter *>
+ getOrCreateConverter(llvm::StringRef SourceEncoding);
+
/// Set true if the SourceManager should report the original file name
/// for contents of files that were overridden by other files. Defaults to
/// true.
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 5540aade05ef5..2b2c58ef851d8 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -448,6 +448,40 @@ ContentCache &SourceManager::createMemBufferContentCache(
return *Entry;
}
+llvm::ErrorOr<llvm::TextEncodingConverter *>
+SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) {
+ // Use getKnownEncoding to get normalized encoding names
+ std::optional<llvm::TextEncoding> SourceKnown =
+ llvm::TextEncodingConverter::getKnownEncoding(SourceEncoding);
+
+ if (SourceKnown && *SourceKnown == llvm::TextEncoding::UTF8)
+ return nullptr;
+
+ // Create a cache key using canonical encoding name
+ llvm::StringRef CacheKey = SourceKnown
+ ? llvm::TextEncodingConverter::getKnownEncodingName(*SourceKnown)
+ : SourceEncoding;
+
+ // Check if converter already exists in cache
+ auto It = ConverterCache.find(CacheKey);
+ if (It != ConverterCache.end())
+ return It->second.get();
+
+ // Create a new converter
+ llvm::ErrorOr<llvm::TextEncodingConverter> NewConverter =
+ llvm::TextEncodingConverter::create(SourceEncoding, "UTF-8");
+
+ if (!NewConverter)
+ return NewConverter.getError();
+
+ // Store the converter in the cache
+ auto Inserted = ConverterCache.insert(
+ std::make_pair(CacheKey, std::make_unique<llvm::TextEncodingConverter>(
+ std::move(*NewConverter))));
+
+ return Inserted.first->second.get();
+}
+
const SrcMgr::SLocEntry &SourceManager::loadSLocEntry(unsigned Index,
bool *Invalid) const {
return const_cast<SourceManager *>(this)->loadSLocEntry(Index, Invalid);
>From add43de1f7cd34d08f758f6f104c5951bd09701e Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Sun, 28 Jun 2026 16:20:56 -0400
Subject: [PATCH 5/8] Add CLANG_DEFAULT_INPUT_ENCODING_IBM1047 flag
---
clang/CMakeLists.txt | 3 +++
clang/include/clang/Config/config.h.cmake | 3 +++
clang/lib/Driver/ToolChains/Clang.cpp | 15 ++++++++++++++-
3 files changed, 20 insertions(+), 1 deletion(-)
diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index e920e83a537d4..0d56dcc200563 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -275,6 +275,9 @@ set(ENABLE_X86_RELAX_RELOCATIONS ON CACHE BOOL
set(PPC_LINUX_DEFAULT_IEEELONGDOUBLE OFF CACHE BOOL
"Enable IEEE binary128 as default long double format on PowerPC Linux.")
+set(CLANG_DEFAULT_INPUT_ENCODING_IBM1047 OFF CACHE BOOL
+ "Set IBM-1047 as the default input encoding")
+
set(CLANG_SPAWN_CC1 OFF CACHE BOOL
"Whether clang should use a new process for the CC1 invocation")
diff --git a/clang/include/clang/Config/config.h.cmake b/clang/include/clang/Config/config.h.cmake
index 11b4096726f67..fbafafc710afe 100644
--- a/clang/include/clang/Config/config.h.cmake
+++ b/clang/include/clang/Config/config.h.cmake
@@ -75,6 +75,9 @@
/* Enable IEEE binary128 as default long double format on PowerPC Linux. */
#cmakedefine01 PPC_LINUX_DEFAULT_IEEELONGDOUBLE
+/* Set IBM-1047 as the default input encoding */
+#cmakedefine01 CLANG_DEFAULT_INPUT_ENCODING_IBM1047
+
/* Enable each functionality of modules */
#cmakedefine01 CLANG_ENABLE_OBJC_REWRITER
#cmakedefine01 CLANG_ENABLE_STATIC_ANALYZER
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index a3a3954bc464e..103efece4f30a 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -7973,10 +7973,23 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
// -finput_charset=UTF-8 is default. Reject others
if (Arg *inputCharset = Args.getLastArg(options::OPT_finput_charset_EQ)) {
StringRef value = inputCharset->getValue();
- if (!value.equals_insensitive("utf-8"))
+#if CLANG_DEFAULT_INPUT_ENCODING_IBM1047
+ bool isValid = value.equals_insensitive("ibm-1047") ||
+ value.equals_insensitive("ibm1047");
+#else
+ bool isValid = value.equals_insensitive("utf-8");
+#endif
+ if (!isValid)
D.Diag(diag::err_drv_invalid_value) << inputCharset->getAsString(Args)
<< value;
}
+#if CLANG_DEFAULT_INPUT_ENCODING_IBM1047
+ else {
+ // When IBM-1047 default is enabled and no explicit charset is specified,
+ // set IBM-1047 as the default
+ CmdArgs.push_back("-finput-charset=IBM-1047");
+ }
+#endif
// -fexec_charset=UTF-8 is default. Reject others
if (Arg *execCharset = Args.getLastArg(options::OPT_fexec_charset_EQ)) {
>From e9fdd11488aa8150948f790f539d2477aa4fb654 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Sun, 28 Jun 2026 22:29:36 -0400
Subject: [PATCH 6/8] Changes for finput-charset
---
.../clang/Basic/DiagnosticCommonKinds.td | 5 +
clang/include/clang/Basic/LangOptions.h | 3 +
clang/include/clang/Basic/SourceManager.h | 41 +++++-
.../include/clang/Frontend/CompilerInstance.h | 1 +
clang/lib/Basic/SourceManager.cpp | 121 ++++++++++++++----
clang/lib/Frontend/CompilerInstance.cpp | 10 +-
.../lib/Frontend/VerifyDiagnosticConsumer.cpp | 4 +-
clang/lib/Lex/ModuleMap.cpp | 7 +-
clang/lib/Lex/PPDirectives.cpp | 6 +-
clang/lib/Lex/Preprocessor.cpp | 4 +-
clang/lib/Serialization/ASTReader.cpp | 5 +-
11 files changed, 175 insertions(+), 32 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td
index f2ed2f4698b8d..55e71020c9c91 100644
--- a/clang/include/clang/Basic/DiagnosticCommonKinds.td
+++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td
@@ -417,6 +417,11 @@ def note_file_sloc_usage : Note<
"%plural{0:|: plus %2B (%human2B) for macro expansions}2">;
def note_file_misc_sloc_usage : Note<
"%0 additional files entered using a total of %1B (%human1B) of space">;
+def warn_encoding_conversion_failed : Warning<
+ "conversion from source encoding failed for '%0': %1; interpreting as %2">,
+ InGroup<DiagGroup<"encoding-conversion-failed">>;
+def err_encoding_conversion_failed : Error<
+ "conversion from source encoding failed for '%0': %1">;
// Modules
def err_module_format_unhandled : Error<
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index 53c4c1084784a..5256e55380236 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -621,6 +621,9 @@ class LangOptions : public LangOptionsBase {
/// The allocation token mode.
std::optional<llvm::AllocTokenMode> AllocTokenMode;
+ /// Name of the input encoding to convert to the internal encoding.
+ std::string InputEncoding;
+
LangOptions();
/// Set language defaults for the given input language and
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 2b5cf0b1e084e..fbb6a3be69c1b 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -157,6 +157,13 @@ class alignas(8) ContentCache {
/// FIXME: Remove this once OrigEntry is a FileEntryRef with a stable name.
StringRef Filename;
+ /// Information on whether this is associated with a FileID for a file (as
+ /// opposed to a buffer) and, if so, what conversion (if any) was requested.
+ /// The integer part uses 2 bits: bit 0 indicates if used by FileID,
+ /// bit 1 indicates if the file was tagged.
+ llvm::PointerIntPair<llvm::TextEncodingConverter *, 2u, unsigned>
+ FileIDConverterInfo;
+
/// A bump pointer allocated array of offsets for each source line.
///
/// This is lazily computed. The lines are owned by the SourceManager
@@ -273,6 +280,36 @@ class alignas(8) ContentCache {
// If BufStr has an invalid BOM, returns the BOM name; otherwise, returns
// nullptr
static const char *getInvalidBOM(StringRef BufStr);
+
+ /// Helper methods for FileIDConverterInfo bit manipulation.
+ /// Bit 0: Used by FileID flag
+ /// Bit 1: File tagged flag
+
+ bool isUsedByFileID() const {
+ return FileIDConverterInfo.getInt() & 0x1;
+ }
+
+ void setUsedByFileID(bool Used) {
+ unsigned Flags = FileIDConverterInfo.getInt();
+ if (Used)
+ Flags |= 0x1;
+ else
+ Flags &= ~0x1;
+ FileIDConverterInfo.setInt(Flags);
+ }
+
+ bool isFileTagged() const {
+ return FileIDConverterInfo.getInt() & 0x2;
+ }
+
+ void setFileTagged(bool Tagged) {
+ unsigned Flags = FileIDConverterInfo.getInt();
+ if (Tagged)
+ Flags |= 0x2;
+ else
+ Flags &= ~0x2;
+ FileIDConverterInfo.setInt(Flags);
+ }
};
// Assert that the \c ContentCache objects will always be 8-byte aligned so
@@ -939,6 +976,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// being \#included from the specified IncludePosition.
FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos,
SrcMgr::CharacteristicKind FileCharacter,
+ llvm::StringRef InputEncodingName = {},
int LoadedID = 0,
SourceLocation::UIntTy LoadedOffset = 0);
@@ -963,7 +1001,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// Get the FileID for \p SourceFile if it exists. Otherwise, create a
/// new FileID for the \p SourceFile.
FileID getOrCreateFileID(FileEntryRef SourceFile,
- SrcMgr::CharacteristicKind FileCharacter);
+ SrcMgr::CharacteristicKind FileCharacter,
+ llvm::StringRef InputEncodingName = {});
/// Creates an expansion SLocEntry for the substitution of an argument into a
/// function-like macro's body. Returns the start of the expansion.
diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h
index 24488e053c628..6bc5661d605d0 100644
--- a/clang/include/clang/Frontend/CompilerInstance.h
+++ b/clang/include/clang/Frontend/CompilerInstance.h
@@ -869,6 +869,7 @@ class CompilerInstance : public ModuleLoader {
///
/// \return True on success.
static bool InitializeSourceManager(const FrontendInputFile &Input,
+ llvm::StringRef InputEncodingName,
DiagnosticsEngine &Diags,
FileManager &FileMgr,
SourceManager &SourceMgr);
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 2b2c58ef851d8..e30537c751005 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -10,6 +10,7 @@
//
//===----------------------------------------------------------------------===//
+#include "clang/Config/config.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/FileManager.h"
@@ -32,6 +33,7 @@
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/SmallVectorMemoryBuffer.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
@@ -136,8 +138,6 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
// return paths.
IsBufferInvalid = true;
- auto BufferOrError = FM.getBufferForFile(*ContentsEntry, IsFileVolatile);
-
// If we were unable to open the file, then we are in an inconsistent
// situation where the content cache referenced a file which no longer
// exists. Most likely, we were using a stat cache with an invalid entry but
@@ -152,6 +152,55 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
Buffer = std::move(*BufferOrError);
+ // Unless this is a named pipe (in which case we can handle a mismatch),
+ // check that the file's size is the same as in the file entry (which may
+ // have come from a stat cache).
+ assert(Buffer->getBufferSize() >= (size_t)ContentsEntry->getSize());
+ if (!ContentsEntry->isNamedPipe() &&
+ Buffer->getBufferSize() < (size_t)ContentsEntry->getSize()) {
+ Diag.Report(Loc, diag::err_file_modified) << ContentsEntry->getName();
+
+ return std::nullopt;
+ }
+
+ // Convert source from the input charset to UTF-8 if necessary.
+ llvm::TextEncodingConverter *Converter = FileIDConverterInfo.getPointer();
+ if (Converter) {
+ StringRef OriginalBuf = Buffer->getBuffer();
+ llvm::SmallString<0> UTF8Buf;
+ UTF8Buf.reserve(OriginalBuf.size() + 1);
+
+ std::error_code EC = Converter->convert(OriginalBuf, UTF8Buf);
+ if (EC) {
+ // For tagged files, conversion failure is an error and we don't fall back
+ if (isFileTagged()) {
+ Diag.Report(Loc, diag::err_encoding_conversion_failed)
+ << ContentsEntry->getName() << EC.message();
+ return std::nullopt;
+ }
+
+ // If conversion fails, emit a warning and fall back to interpreting the
+ // file as the default encoding.
+ //
+ // This allows the compiler to accept system or third-party headers that
+ // are encoded in the default encoding even if conversion to the
+ // option-specified input encoding failed.
+ //
+ // TODO: Add input byte offset information.
+ //
+ // TODO: Consider adjusting the message to omit the recovery description
+ // if the warning has been upgraded to an error.
+ const char *FallbackEncoding = CLANG_DEFAULT_INPUT_ENCODING_IBM1047 ? "IBM-1047" : "UTF-8";
+ Diag.Report(Loc, diag::warn_encoding_conversion_failed)
+ << ContentsEntry->getName() << EC.message() << FallbackEncoding;
+ } else {
+ // TODO: Reclaim memory if the buffer size exceeds the content.
+ auto NewBuf = std::make_unique<llvm::SmallVectorMemoryBuffer>(
+ std::move(UTF8Buf), Buffer->getBufferIdentifier());
+ Buffer = std::move(NewBuf);
+ }
+ }
+
// Check that the file's size fits in an 'unsigned' (with room for a
// past-the-end value). This is deeply regrettable, but various parts of
// Clang (including elsewhere in this file!) use 'unsigned' to represent file
@@ -167,22 +216,15 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
return std::nullopt;
}
- // Unless this is a named pipe (in which case we can handle a mismatch),
- // check that the file's size is the same as in the file entry (which may
- // have come from a stat cache).
- // The buffer will always be larger than the file size on z/OS in the presence
- // of characters outside the base character set.
- assert(Buffer->getBufferSize() >= (size_t)ContentsEntry->getSize());
- if (!ContentsEntry->isNamedPipe() &&
- Buffer->getBufferSize() < (size_t)ContentsEntry->getSize()) {
- Diag.Report(Loc, diag::err_file_modified) << ContentsEntry->getName();
-
- return std::nullopt;
- }
-
- // If the buffer is valid, check to see if it has a UTF Byte Order Mark
- // (BOM). We only support UTF-8 with and without a BOM right now. See
- // http://en.wikipedia.org/wiki/Byte_order_mark for more information.
+ // If the buffer is valid, check to see if it has a UTF Byte Order Mark (BOM)
+ // Note that any conversion requested using `-finput-charset` (if successful)
+ // has already occurred, so we are expecting UTF-8 with or without a BOM.
+ //
+ // In theory, if we see a non-UTF-8 BOM, we can assume that an appropriate
+ // conversion was not supplied via `-finput-charset` and we could try to
+ // convert based on the BOM.
+ //
+ // See http://en.wikipedia.org/wiki/Byte_order_mark for more information.
StringRef BufStr = Buffer->getBuffer();
const char *InvalidBOM = getInvalidBOM(BufStr);
@@ -591,6 +633,7 @@ FileID SourceManager::getNextFileID(FileID FID) const {
FileID SourceManager::createFileID(FileEntryRef SourceFile,
SourceLocation IncludePos,
SrcMgr::CharacteristicKind FileCharacter,
+ llvm::StringRef InputEncodingName,
int LoadedID,
SourceLocation::UIntTy LoadedOffset) {
SrcMgr::ContentCache &IR = getOrCreateContentCache(SourceFile,
@@ -604,9 +647,38 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
FileIDContentCaches.push_back(Cache);
}
+ llvm::ErrorOr<llvm::TextEncodingConverter *> Converter = nullptr;
+ if (!InputEncodingName.empty()) {
+ // No file tag but -finput-charset conversion is desired.
+ // Get the converter from the cache using the input encoding name.
+ Converter = getOrCreateConverter(InputEncodingName);
+ if (!Converter) {
+ llvm::report_fatal_error(
+ "Cannot create converter for file '" + SourceFile.getName() + "': " +
+ Converter.getError().message());
+ }
+ }
+
+ #ifndef NDEBUG
+ // Either the content cache has never been used for a FileID (and, if we are
+ // being asked to use a converter, there should be no valid buffer set up for
+ // it) or the conversion (or lack thereof) should be the same as that used
+ // previously.
+ llvm::TextEncodingConverter *CacheConverter = IR.FileIDConverterInfo.getPointer();
+ bool CacheUsedByFileID = IR.isUsedByFileID();
+ llvm::TextEncodingConverter *ConverterPtr = Converter ? *Converter : nullptr;
+ if (CacheUsedByFileID)
+ assert(CacheConverter == ConverterPtr);
+ else
+ assert(!ConverterPtr || IR.IsBufferInvalid || !IR.getBufferIfLoaded());
+#endif
+ IR.setUsedByFileID(true);
+ IR.FileIDConverterInfo.setPointer(Converter ? *Converter : nullptr);
+
// If this is a named pipe, immediately load the buffer to ensure subsequent
// calls to ContentCache::getSize() are accurate.
- if (Cache->ContentsEntry->isNamedPipe())
+ // Do the same if character-encoding conversion was requested.
+ if (Cache->ContentsEntry->isNamedPipe() || Converter)
(void)Cache->getBufferOrNone(Diag, getFileManager(), SourceLocation());
return createFileIDImpl(*Cache, Filename, IncludePos, FileCharacter, LoadedID,
@@ -644,10 +716,12 @@ FileID SourceManager::createFileID(const llvm::MemoryBufferRef &Buffer,
/// new FileID for the \p SourceFile.
FileID
SourceManager::getOrCreateFileID(FileEntryRef SourceFile,
- SrcMgr::CharacteristicKind FileCharacter) {
+ SrcMgr::CharacteristicKind FileCharacter,
+ llvm::StringRef InputEncodingName) {
FileID ID = translateFile(SourceFile);
- return ID.isValid() ? ID : createFileID(SourceFile, SourceLocation(),
- FileCharacter);
+ return ID.isValid() ? ID
+ : createFileID(SourceFile, SourceLocation(), FileCharacter,
+ InputEncodingName);
}
/// createFileID - Create a new FileID for the specified ContentCache and
@@ -2403,7 +2477,8 @@ SourceManagerForFile::SourceManagerForFile(StringRef FileName,
SourceMgr = std::make_unique<SourceManager>(*Diagnostics, *FileMgr);
FileEntryRef FE = llvm::cantFail(FileMgr->getFileRef(FileName));
FileID ID =
- SourceMgr->createFileID(FE, SourceLocation(), clang::SrcMgr::C_User);
+ SourceMgr->createFileID(FE, SourceLocation(), clang::SrcMgr::C_User,
+ /*InputEncodingName=*/{});
assert(ID.isValid());
SourceMgr->setMainFileID(ID);
}
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 8aee45b5dc644..6be01e4563e44 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -912,12 +912,15 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary,
// Initialization Utilities
bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){
- return InitializeSourceManager(Input, getDiagnostics(), getFileManager(),
- getSourceManager());
+ StringRef InputEncodingName =
+ hasPreprocessor() ? llvm::StringRef(getLangOpts().InputEncoding) : llvm::StringRef();
+ return InitializeSourceManager(Input, InputEncodingName, getDiagnostics(),
+ getFileManager(), getSourceManager());
}
// static
bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input,
+ llvm::StringRef InputEncodingName,
DiagnosticsEngine &Diags,
FileManager &FileMgr,
SourceManager &SourceMgr) {
@@ -950,7 +953,8 @@ bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input,
}
SourceMgr.setMainFileID(
- SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind));
+ SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind,
+ InputEncodingName));
assert(SourceMgr.getMainFileID().isValid() &&
"Couldn't establish MainFileID!");
diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
index 1bfe644b2525a..691bc5a5fd31d 100644
--- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
+++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
@@ -610,8 +610,10 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
}
FileID FID = SM.translateFile(*File);
+ // FIXME: Figure out character-encoding converter treatment.
if (FID.isInvalid())
- FID = SM.createFileID(*File, Pos, SrcMgr::C_User);
+ FID = SM.createFileID(*File, Pos, SrcMgr::C_User,
+ /*InputEncodingName=*/{});
if (PH.Next(Line) && Line > 0)
ExpectedLoc = SM.translateLineCol(FID, Line, 1);
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index 6c07386f89010..3e5c5b317d580 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -1473,7 +1473,12 @@ bool ModuleMap::parseModuleMapFile(FileEntryRef File, bool IsSystem,
if (LocalFID.isInvalid()) {
auto FileCharacter =
IsSystem ? SrcMgr::C_System_ModuleMap : SrcMgr::C_User_ModuleMap;
- LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter);
+ // Module map files are textual "source files". Use input charset converter
+ // if available, and file tag converters are handled by SourceManager's cache.
+ // Get input encoding from LangOptions for charset conversion
+ llvm::StringRef InputEncodingName = LangOpts.InputEncoding;
+ LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter,
+ InputEncodingName);
}
ID = LocalFID;
}
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index eb21a510dcf83..f989c2d1d4b96 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -2796,7 +2796,11 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
// position on the file where it will be included and after the expansions.
if (IncludePos.isMacroID())
IncludePos = SourceMgr.getExpansionRange(IncludePos).getEnd();
- FileID FID = SourceMgr.createFileID(*File, IncludePos, FileCharacter);
+ // Use the SourceManager's input charset converter for non-tagged files
+ // by passing the input encoding name
+ llvm::StringRef InputEncodingName = getLangOpts().InputEncoding;
+ FileID FID = SourceMgr.createFileID(*File, IncludePos, FileCharacter,
+ InputEncodingName);
if (!FID.isValid()) {
TheModuleLoader.HadFatalFailure = true;
return ImportAction::Failure;
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index c69d084d6514f..610d3c703c693 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -651,8 +651,10 @@ void Preprocessor::EnterMainSourceFile() {
<< PPOpts.PCHThroughHeader;
return;
}
+ // FIXME: Figure out character-encoding converter treatment.
setPCHThroughHeaderFileID(
- SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User));
+ SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User,
+ /*InputEncodingName=*/{}));
}
// Skip tokens from the Predefines and if needed the main file.
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index f8a6a38bb9b5c..52b60df62977d 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -2002,7 +2002,10 @@ bool ASTReader::ReadSLocEntry(int ID) {
}
SrcMgr::CharacteristicKind
FileCharacter = (SrcMgr::CharacteristicKind)Record[2];
- FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter, ID,
+ // Note: If conversion was originally necessary, OverriddenBuffer should be
+ // true and the associated handling will trigger.
+ FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter,
+ /*InputEncodingName=*/{}, ID,
BaseOffset + Record[0]);
SrcMgr::FileInfo &FileInfo = SourceMgr.getSLocEntry(FID).getFile();
FileInfo.NumCreatedFIDs = Record[5];
>From 236c44859bad044a9e9a8810a3e66d67e81d3ae0 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Sun, 28 Jun 2026 21:53:49 -0400
Subject: [PATCH 7/8] Use converters for tagged files
---
clang/lib/Basic/SourceManager.cpp | 31 ++++++++++++++++++++++++++++++-
1 file changed, 30 insertions(+), 1 deletion(-)
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index e30537c751005..1a186a6a4d98e 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -138,6 +138,18 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
// return paths.
IsBufferInvalid = true;
+ // If a converter is set, open the file in binary mode to get raw bytes
+ // and avoid platform-specific auto-conversion (e.g., EBCDIC->ASCII on z/OS,
+ // CRLF->LF on Windows). The explicit converter will handle all transformations.
+ bool NeedsExplicitConversion = FileIDConverterInfo.getPointer() != nullptr;
+ bool IsText = !NeedsExplicitConversion;
+
+ auto BufferOrError = FM.getBufferForFile(*ContentsEntry, IsFileVolatile,
+ /*RequiresNullTerminator=*/true,
+ /*MaybeLimit=*/std::nullopt,
+ IsText);
+
+>>>>>>> de18010595aa (Use converters for tagged files)
// If we were unable to open the file, then we are in an inconsistent
// situation where the content cache referenced a file which no longer
// exists. Most likely, we were using a stat cache with an invalid entry but
@@ -648,7 +660,24 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
}
llvm::ErrorOr<llvm::TextEncodingConverter *> Converter = nullptr;
- if (!InputEncodingName.empty()) {
+ llvm::ErrorOr<llvm::SmallString<32>> Ccsid =
+ llvm::getEncodingNameFromFileTag(SourceFile.getName());
+ if (!Ccsid) {
+ Diag.Report(SourceLocation(), diag::err_cannot_open_file)
+ << SourceFile.getName() << Ccsid.getError().message();
+ return FileID();
+ }
+ if (!Ccsid->empty()) {
+ // File has a tag, use the converter from SourceManager's cache
+ Converter = getOrCreateConverter(*Ccsid);
+ if (!Converter) {
+ Diag.Report(SourceLocation(), diag::err_cannot_open_file)
+ << SourceFile.getName()
+ << (llvm::Twine("cannot create converter from encoding '") + *Ccsid + "'");
+ return FileID();
+ }
+ IR.setFileTagged(true);
+ } else if (!InputEncodingName.empty()) {
// No file tag but -finput-charset conversion is desired.
// Get the converter from the cache using the input encoding name.
Converter = getOrCreateConverter(InputEncodingName);
>From 32671c0c189d99830d4084999a92f0ccb32011ee Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Fri, 3 Jul 2026 09:00:47 -0400
Subject: [PATCH 8/8] Clang format
---
clang/include/clang/Basic/LangOptions.h | 2 +-
clang/include/clang/Basic/SourceManager.h | 13 ++---
.../include/clang/Frontend/CompilerInstance.h | 2 +-
clang/lib/Basic/SourceManager.cpp | 52 ++++++++++---------
clang/lib/Frontend/CompilerInstance.cpp | 20 ++++---
.../lib/Frontend/VerifyDiagnosticConsumer.cpp | 4 +-
clang/lib/Lex/ModuleMap.cpp | 9 ++--
clang/lib/Lex/Preprocessor.cpp | 6 +--
clang/lib/Serialization/ASTReader.cpp | 4 +-
llvm/include/llvm/Support/VirtualFileSystem.h | 6 +--
llvm/lib/Support/TextEncoding.cpp | 3 +-
11 files changed, 60 insertions(+), 61 deletions(-)
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index 5256e55380236..3200d73f0381f 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -623,7 +623,7 @@ class LangOptions : public LangOptionsBase {
/// Name of the input encoding to convert to the internal encoding.
std::string InputEncoding;
-
+
LangOptions();
/// Set language defaults for the given input language and
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index fbb6a3be69c1b..16dd446371540 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -285,9 +285,7 @@ class alignas(8) ContentCache {
/// Bit 0: Used by FileID flag
/// Bit 1: File tagged flag
- bool isUsedByFileID() const {
- return FileIDConverterInfo.getInt() & 0x1;
- }
+ bool isUsedByFileID() const { return FileIDConverterInfo.getInt() & 0x1; }
void setUsedByFileID(bool Used) {
unsigned Flags = FileIDConverterInfo.getInt();
@@ -298,9 +296,7 @@ class alignas(8) ContentCache {
FileIDConverterInfo.setInt(Flags);
}
- bool isFileTagged() const {
- return FileIDConverterInfo.getInt() & 0x2;
- }
+ bool isFileTagged() const { return FileIDConverterInfo.getInt() & 0x2; }
void setFileTagged(bool Tagged) {
unsigned Flags = FileIDConverterInfo.getInt();
@@ -976,8 +972,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// being \#included from the specified IncludePosition.
FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos,
SrcMgr::CharacteristicKind FileCharacter,
- llvm::StringRef InputEncodingName = {},
- int LoadedID = 0,
+ llvm::StringRef InputEncodingName = {}, int LoadedID = 0,
SourceLocation::UIntTy LoadedOffset = 0);
/// Create a new FileID that represents the specified memory buffer.
@@ -1002,7 +997,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// new FileID for the \p SourceFile.
FileID getOrCreateFileID(FileEntryRef SourceFile,
SrcMgr::CharacteristicKind FileCharacter,
- llvm::StringRef InputEncodingName = {});
+ llvm::StringRef InputEncodingName = {});
/// Creates an expansion SLocEntry for the substitution of an argument into a
/// function-like macro's body. Returns the start of the expansion.
diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h
index 6bc5661d605d0..0ebc50d99b86d 100644
--- a/clang/include/clang/Frontend/CompilerInstance.h
+++ b/clang/include/clang/Frontend/CompilerInstance.h
@@ -869,7 +869,7 @@ class CompilerInstance : public ModuleLoader {
///
/// \return True on success.
static bool InitializeSourceManager(const FrontendInputFile &Input,
- llvm::StringRef InputEncodingName,
+ llvm::StringRef InputEncodingName,
DiagnosticsEngine &Diags,
FileManager &FileMgr,
SourceManager &SourceMgr);
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 1a186a6a4d98e..bf4016531a7c5 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -10,13 +10,13 @@
//
//===----------------------------------------------------------------------===//
-#include "clang/Config/config.h"
#include "clang/Basic/SourceManager.h"
#include "clang/Basic/Diagnostic.h"
#include "clang/Basic/FileManager.h"
#include "clang/Basic/LLVM.h"
#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/SourceManagerInternals.h"
+#include "clang/Config/config.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
@@ -32,8 +32,8 @@
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
-#include "llvm/Support/raw_ostream.h"
#include "llvm/Support/SmallVectorMemoryBuffer.h"
+#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
@@ -140,14 +140,14 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
// If a converter is set, open the file in binary mode to get raw bytes
// and avoid platform-specific auto-conversion (e.g., EBCDIC->ASCII on z/OS,
- // CRLF->LF on Windows). The explicit converter will handle all transformations.
+ // CRLF->LF on Windows). The explicit converter will handle all
+ // transformations.
bool NeedsExplicitConversion = FileIDConverterInfo.getPointer() != nullptr;
bool IsText = !NeedsExplicitConversion;
auto BufferOrError = FM.getBufferForFile(*ContentsEntry, IsFileVolatile,
/*RequiresNullTerminator=*/true,
- /*MaybeLimit=*/std::nullopt,
- IsText);
+ /*MaybeLimit=*/std::nullopt, IsText);
>>>>>>> de18010595aa (Use converters for tagged files)
// If we were unable to open the file, then we are in an inconsistent
@@ -202,14 +202,15 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
//
// TODO: Consider adjusting the message to omit the recovery description
// if the warning has been upgraded to an error.
- const char *FallbackEncoding = CLANG_DEFAULT_INPUT_ENCODING_IBM1047 ? "IBM-1047" : "UTF-8";
+ const char *FallbackEncoding =
+ CLANG_DEFAULT_INPUT_ENCODING_IBM1047 ? "IBM-1047" : "UTF-8";
Diag.Report(Loc, diag::warn_encoding_conversion_failed)
<< ContentsEntry->getName() << EC.message() << FallbackEncoding;
} else {
// TODO: Reclaim memory if the buffer size exceeds the content.
auto NewBuf = std::make_unique<llvm::SmallVectorMemoryBuffer>(
std::move(UTF8Buf), Buffer->getBufferIdentifier());
- Buffer = std::move(NewBuf);
+ Buffer = std::move(NewBuf);
}
}
@@ -512,9 +513,10 @@ SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) {
return nullptr;
// Create a cache key using canonical encoding name
- llvm::StringRef CacheKey = SourceKnown
- ? llvm::TextEncodingConverter::getKnownEncodingName(*SourceKnown)
- : SourceEncoding;
+ llvm::StringRef CacheKey =
+ SourceKnown
+ ? llvm::TextEncodingConverter::getKnownEncodingName(*SourceKnown)
+ : SourceEncoding;
// Check if converter already exists in cache
auto It = ConverterCache.find(CacheKey);
@@ -529,9 +531,9 @@ SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) {
return NewConverter.getError();
// Store the converter in the cache
- auto Inserted = ConverterCache.insert(
- std::make_pair(CacheKey, std::make_unique<llvm::TextEncodingConverter>(
- std::move(*NewConverter))));
+ auto Inserted = ConverterCache.insert(std::make_pair(
+ CacheKey,
+ std::make_unique<llvm::TextEncodingConverter>(std::move(*NewConverter))));
return Inserted.first->second.get();
}
@@ -645,7 +647,7 @@ FileID SourceManager::getNextFileID(FileID FID) const {
FileID SourceManager::createFileID(FileEntryRef SourceFile,
SourceLocation IncludePos,
SrcMgr::CharacteristicKind FileCharacter,
- llvm::StringRef InputEncodingName,
+ llvm::StringRef InputEncodingName,
int LoadedID,
SourceLocation::UIntTy LoadedOffset) {
SrcMgr::ContentCache &IR = getOrCreateContentCache(SourceFile,
@@ -673,7 +675,8 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
if (!Converter) {
Diag.Report(SourceLocation(), diag::err_cannot_open_file)
<< SourceFile.getName()
- << (llvm::Twine("cannot create converter from encoding '") + *Ccsid + "'");
+ << (llvm::Twine("cannot create converter from encoding '") + *Ccsid +
+ "'");
return FileID();
}
IR.setFileTagged(true);
@@ -682,18 +685,19 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
// Get the converter from the cache using the input encoding name.
Converter = getOrCreateConverter(InputEncodingName);
if (!Converter) {
- llvm::report_fatal_error(
- "Cannot create converter for file '" + SourceFile.getName() + "': " +
- Converter.getError().message());
+ llvm::report_fatal_error("Cannot create converter for file '" +
+ SourceFile.getName() +
+ "': " + Converter.getError().message());
}
}
- #ifndef NDEBUG
+#ifndef NDEBUG
// Either the content cache has never been used for a FileID (and, if we are
// being asked to use a converter, there should be no valid buffer set up for
// it) or the conversion (or lack thereof) should be the same as that used
// previously.
- llvm::TextEncodingConverter *CacheConverter = IR.FileIDConverterInfo.getPointer();
+ llvm::TextEncodingConverter *CacheConverter =
+ IR.FileIDConverterInfo.getPointer();
bool CacheUsedByFileID = IR.isUsedByFileID();
llvm::TextEncodingConverter *ConverterPtr = Converter ? *Converter : nullptr;
if (CacheUsedByFileID)
@@ -746,11 +750,11 @@ FileID SourceManager::createFileID(const llvm::MemoryBufferRef &Buffer,
FileID
SourceManager::getOrCreateFileID(FileEntryRef SourceFile,
SrcMgr::CharacteristicKind FileCharacter,
- llvm::StringRef InputEncodingName) {
+ llvm::StringRef InputEncodingName) {
FileID ID = translateFile(SourceFile);
return ID.isValid() ? ID
- : createFileID(SourceFile, SourceLocation(), FileCharacter,
- InputEncodingName);
+ : createFileID(SourceFile, SourceLocation(),
+ FileCharacter, InputEncodingName);
}
/// createFileID - Create a new FileID for the specified ContentCache and
@@ -2507,7 +2511,7 @@ SourceManagerForFile::SourceManagerForFile(StringRef FileName,
FileEntryRef FE = llvm::cantFail(FileMgr->getFileRef(FileName));
FileID ID =
SourceMgr->createFileID(FE, SourceLocation(), clang::SrcMgr::C_User,
- /*InputEncodingName=*/{});
+ /*InputEncodingName=*/{});
assert(ID.isValid());
SourceMgr->setMainFileID(ID);
}
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 6be01e4563e44..ccb987df1083e 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -913,17 +913,16 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary,
bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){
StringRef InputEncodingName =
- hasPreprocessor() ? llvm::StringRef(getLangOpts().InputEncoding) : llvm::StringRef();
- return InitializeSourceManager(Input, InputEncodingName, getDiagnostics(),
- getFileManager(), getSourceManager());
+ hasPreprocessor() ? llvm::StringRef(getLangOpts().InputEncoding)
+ : llvm::StringRef();
+ return InitializeSourceManager(Input, InputEncodingName, getDiagnostics(),
+ getFileManager(), getSourceManager());
}
// static
-bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input,
- llvm::StringRef InputEncodingName,
- DiagnosticsEngine &Diags,
- FileManager &FileMgr,
- SourceManager &SourceMgr) {
+bool CompilerInstance::InitializeSourceManager(
+ const FrontendInputFile &Input, llvm::StringRef InputEncodingName,
+ DiagnosticsEngine &Diags, FileManager &FileMgr, SourceManager &SourceMgr) {
SrcMgr::CharacteristicKind Kind =
Input.getKind().getFormat() == InputKind::ModuleMap
? Input.isSystem() ? SrcMgr::C_System_ModuleMap
@@ -952,9 +951,8 @@ bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input,
return false;
}
- SourceMgr.setMainFileID(
- SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind,
- InputEncodingName));
+ SourceMgr.setMainFileID(SourceMgr.createFileID(*FileOrErr, SourceLocation(),
+ Kind, InputEncodingName));
assert(SourceMgr.getMainFileID().isValid() &&
"Couldn't establish MainFileID!");
diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
index 691bc5a5fd31d..1ae2337adbb1f 100644
--- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
+++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
@@ -610,10 +610,10 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
}
FileID FID = SM.translateFile(*File);
- // FIXME: Figure out character-encoding converter treatment.
+ // FIXME: Figure out character-encoding converter treatment.
if (FID.isInvalid())
FID = SM.createFileID(*File, Pos, SrcMgr::C_User,
- /*InputEncodingName=*/{});
+ /*InputEncodingName=*/{});
if (PH.Next(Line) && Line > 0)
ExpectedLoc = SM.translateLineCol(FID, Line, 1);
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index 3e5c5b317d580..b9750e7a8c197 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -1473,12 +1473,13 @@ bool ModuleMap::parseModuleMapFile(FileEntryRef File, bool IsSystem,
if (LocalFID.isInvalid()) {
auto FileCharacter =
IsSystem ? SrcMgr::C_System_ModuleMap : SrcMgr::C_User_ModuleMap;
- // Module map files are textual "source files". Use input charset converter
- // if available, and file tag converters are handled by SourceManager's cache.
- // Get input encoding from LangOptions for charset conversion
+ // Module map files are textual "source files". Use input charset
+ // converter if available, and file tag converters are handled by
+ // SourceManager's cache. Get input encoding from LangOptions for charset
+ // conversion
llvm::StringRef InputEncodingName = LangOpts.InputEncoding;
LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter,
- InputEncodingName);
+ InputEncodingName);
}
ID = LocalFID;
}
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 610d3c703c693..6ff3652760de7 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -652,9 +652,9 @@ void Preprocessor::EnterMainSourceFile() {
return;
}
// FIXME: Figure out character-encoding converter treatment.
- setPCHThroughHeaderFileID(
- SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User,
- /*InputEncodingName=*/{}));
+ setPCHThroughHeaderFileID(SourceMgr.createFileID(*File, SourceLocation(),
+ SrcMgr::C_User,
+ /*InputEncodingName=*/{}));
}
// Skip tokens from the Predefines and if needed the main file.
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 52b60df62977d..998cea077d836 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -2004,8 +2004,8 @@ bool ASTReader::ReadSLocEntry(int ID) {
FileCharacter = (SrcMgr::CharacteristicKind)Record[2];
// Note: If conversion was originally necessary, OverriddenBuffer should be
// true and the associated handling will trigger.
- FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter,
- /*InputEncodingName=*/{}, ID,
+ FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter,
+ /*InputEncodingName=*/{}, ID,
BaseOffset + Record[0]);
SrcMgr::FileInfo &FileInfo = SourceMgr.getSLocEntry(FID).getFile();
FileInfo.NumCreatedFIDs = Record[5];
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index 9cc35b6f5fb6d..274886f920fc6 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -139,9 +139,9 @@ class LLVM_ABI File {
/// Checks if this is a real file and the requested text mode differs
/// from the current mode. For real files with a text mode mismatch where
- /// the buffer was previously requested, this will call llvm::report_fatal_error.
- /// Always returns false for non-real files.
- /// Default implementation returns false for non-real files.
+ /// the buffer was previously requested, this will call
+ /// llvm::report_fatal_error. Always returns false for non-real files. Default
+ /// implementation returns false for non-real files.
virtual bool realFileCheckTextModeMismatch(bool RequestedIsText) const {
return false;
}
diff --git a/llvm/lib/Support/TextEncoding.cpp b/llvm/lib/Support/TextEncoding.cpp
index 8e9653dab38ec..99f2c8099d966 100644
--- a/llvm/lib/Support/TextEncoding.cpp
+++ b/llvm/lib/Support/TextEncoding.cpp
@@ -48,7 +48,8 @@ static void normalizeCharSetName(StringRef CSName,
}
// Maps the encoding name to enum constant if possible.
-std::optional<TextEncoding> TextEncodingConverter::getKnownEncoding(StringRef Name) {
+std::optional<TextEncoding>
+TextEncodingConverter::getKnownEncoding(StringRef Name) {
SmallString<16> Normalized;
normalizeCharSetName(Name, Normalized);
if (Normalized.equals("utf8"))
More information about the llvm-commits
mailing list