[clang] [llvm] Replace autoconversion with TextEncodingConverter converters (PR #206519)
Alison Zhang via cfe-commits
cfe-commits at lists.llvm.org
Tue Jul 14 11:52:36 PDT 2026
https://github.com/azhan92 updated https://github.com/llvm/llvm-project/pull/206519
>From 41ef2c9d4b6d0a36682ed67dfa853897115d9ddd Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Tue, 14 Jul 2026 13:14:46 -0400
Subject: [PATCH 1/7] Get canonical names for encodings
---
llvm/include/llvm/Support/TextEncoding.h | 11 +++++++++++
llvm/lib/Support/TextEncoding.cpp | 14 +++++++++++++-
2 files changed, 24 insertions(+), 1 deletion(-)
diff --git a/llvm/include/llvm/Support/TextEncoding.h b/llvm/include/llvm/Support/TextEncoding.h
index 6e7eb95464eff..f1889e1dfd04d 100644
--- a/llvm/include/llvm/Support/TextEncoding.h
+++ b/llvm/include/llvm/Support/TextEncoding.h
@@ -104,6 +104,17 @@ class TextEncodingConverter {
LLVM_ABI static ErrorOr<TextEncodingConverter> create(StringRef From,
StringRef To);
+ /// Maps the encoding name to enum constant if possible.
+ /// Uses normalized charset name matching.
+ /// \param[in] Name the character encoding name
+ /// \return the TextEncoding enum value if known, std::nullopt otherwise
+ LLVM_ABI static std::optional<TextEncoding> getKnownEncoding(StringRef Name);
+
+ /// Returns the canonical name for a known encoding.
+ /// \param[in] Encoding the TextEncoding enum value
+ /// \return the canonical name for the encoding (e.g., "UTF-8" or "IBM-1047")
+ LLVM_ABI static StringRef getKnownEncodingName(TextEncoding Encoding);
+
TextEncodingConverter(const TextEncodingConverter &) = delete;
TextEncodingConverter &operator=(const TextEncodingConverter &) = delete;
diff --git a/llvm/lib/Support/TextEncoding.cpp b/llvm/lib/Support/TextEncoding.cpp
index c44bf78ea9fba..838956b9636f5 100644
--- a/llvm/lib/Support/TextEncoding.cpp
+++ b/llvm/lib/Support/TextEncoding.cpp
@@ -49,7 +49,8 @@ static void normalizeCharSetName(StringRef CSName,
}
// Maps the encoding name to enum constant if possible.
-static std::optional<TextEncoding> getKnownEncoding(StringRef Name) {
+std::optional<TextEncoding>
+TextEncodingConverter::getKnownEncoding(StringRef Name) {
SmallString<16> Normalized;
normalizeCharSetName(Name, Normalized);
if (Normalized.equals("utf8"))
@@ -59,6 +60,17 @@ static std::optional<TextEncoding> getKnownEncoding(StringRef Name) {
return std::nullopt;
}
+// Returns the canonical name for a known encoding.
+StringRef TextEncodingConverter::getKnownEncodingName(TextEncoding Encoding) {
+ switch (Encoding) {
+ case TextEncoding::UTF8:
+ return "UTF-8";
+ case TextEncoding::IBM1047:
+ return "IBM-1047";
+ }
+ llvm_unreachable("Invalid TextEncoding value");
+}
+
[[maybe_unused]] static void HandleOverflow(size_t &Capacity, char *&Output,
size_t &OutputLength,
SmallVectorImpl<char> &Result) {
>From 280a4c9970103740005ed6149ad862ddae560601 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Tue, 14 Jul 2026 13:15:41 -0400
Subject: [PATCH 2/7] Text mode mismatch checking
---
clang/lib/Basic/FileManager.cpp | 17 +++++++++-----
llvm/include/llvm/Support/VirtualFileSystem.h | 9 ++++++++
llvm/lib/Support/VirtualFileSystem.cpp | 22 ++++++++++++++++---
3 files changed, 40 insertions(+), 8 deletions(-)
diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp
index 8fb3ba0a27aad..94fc4c15f51d2 100644
--- a/clang/lib/Basic/FileManager.cpp
+++ b/clang/lib/Basic/FileManager.cpp
@@ -539,15 +539,22 @@ FileManager::getBufferForFile(FileEntryRef FE, bool isVolatile,
FileSize = -1;
StringRef Filename = FE.getName();
- // If the file is already open, use the open file descriptor.
+ // If the file is already open, check if the mode matches.
if (Entry->File) {
- auto Result = Entry->File->getBuffer(Filename, FileSize,
- RequiresNullTerminator, isVolatile);
+ // Check if the cached file's mode matches the requested mode
+ // Only perform mismatch recovery for real files
+ if (!Entry->File->realFileCheckTextModeMismatch(IsText)) {
+ // Mode matches, use the cached file descriptor
+ auto Result = Entry->File->getBuffer(Filename, FileSize,
+ RequiresNullTerminator, isVolatile);
+ Entry->closeFile();
+ return Result;
+ }
+ // Mode mismatch - close the cached file and reopen with correct mode
Entry->closeFile();
- return Result;
}
- // Otherwise, open the file.
+ // Open the file with the requested mode.
return getBufferForFileImpl(Filename, FileSize, isVolatile,
RequiresNullTerminator, IsText);
}
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index d22c534228331..274886f920fc6 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -137,6 +137,15 @@ class LLVM_ABI File {
/// Closes the file.
virtual std::error_code close() = 0;
+ /// Checks if this is a real file and the requested text mode differs
+ /// from the current mode. For real files with a text mode mismatch where
+ /// the buffer was previously requested, this will call
+ /// llvm::report_fatal_error. Always returns false for non-real files. Default
+ /// implementation returns false for non-real files.
+ virtual bool realFileCheckTextModeMismatch(bool RequestedIsText) const {
+ return false;
+ }
+
// Get the same file with a different path.
static ErrorOr<std::unique_ptr<File>>
getWithPath(ErrorOr<std::unique_ptr<File>> Result, const Twine &P);
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 42e8bb4f9958e..1c1f9e9dd15b6 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -194,11 +194,15 @@ class RealFile : public File {
file_t FD;
Status S;
std::string RealName;
+ bool IsTextMode;
+ bool BufferWasRequested;
- RealFile(file_t RawFD, StringRef NewName, StringRef NewRealPathName)
+ RealFile(file_t RawFD, StringRef NewName, StringRef NewRealPathName,
+ bool IsText)
: FD(RawFD), S(NewName, {}, {}, {}, {}, {},
llvm::sys::fs::file_type::status_error, {}),
- RealName(NewRealPathName.str()) {
+ RealName(NewRealPathName.str()), IsTextMode(IsText),
+ BufferWasRequested(false) {
assert(FD != kInvalidFile && "Invalid or inactive file descriptor");
}
@@ -213,6 +217,16 @@ class RealFile : public File {
bool IsVolatile) override;
std::error_code close() override;
void setPath(const Twine &Path) override;
+ bool realFileCheckTextModeMismatch(bool RequestedIsText) const override {
+ bool HasMismatch = IsTextMode != RequestedIsText;
+ if (HasMismatch && BufferWasRequested) {
+ llvm::report_fatal_error(
+ "Text mode mismatch: file was previously opened with " +
+ Twine(IsTextMode ? "text" : "binary") + " mode, now requested with " +
+ Twine(RequestedIsText ? "text" : "binary") + " mode");
+ }
+ return HasMismatch;
+ }
};
} // namespace
@@ -242,6 +256,7 @@ RealFile::getBuffer(const Twine &Name, int64_t FileSize,
auto BypassSandbox = sys::sandbox::scopedDisable();
assert(FD != kInvalidFile && "cannot get buffer for closed file");
+ BufferWasRequested = true;
return MemoryBuffer::getOpenFile(FD, Name, FileSize, RequiresNullTerminator,
IsVolatile);
}
@@ -320,8 +335,9 @@ class RealFileSystem : public FileSystem {
adjustPath(Name, Storage), Flags, &RealName);
if (!FDOrErr)
return errorToErrorCode(FDOrErr.takeError());
+ bool IsText = (Flags & sys::fs::OF_Text) != sys::fs::OF_None;
return std::unique_ptr<File>(
- new RealFile(*FDOrErr, Name.str(), RealName.str()));
+ new RealFile(*FDOrErr, Name.str(), RealName.str(), IsText));
}
struct WorkingDirectory {
>From 55c2f56ea77865e05d70376898c715595af6bcaf Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Sun, 28 Jun 2026 15:51:57 -0400
Subject: [PATCH 3/7] getEncodingNameFromFileTag
---
llvm/include/llvm/Support/AutoConvert.h | 26 +++++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git a/llvm/include/llvm/Support/AutoConvert.h b/llvm/include/llvm/Support/AutoConvert.h
index 3c025cb605f2c..10a296c81a7a7 100644
--- a/llvm/include/llvm/Support/AutoConvert.h
+++ b/llvm/include/llvm/Support/AutoConvert.h
@@ -18,6 +18,7 @@
#include <_Ccsid.h>
#endif
#ifdef __cplusplus
+#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/Twine.h"
#include "llvm/Support/Error.h"
#include <system_error>
@@ -114,6 +115,31 @@ inline ErrorOr<bool> needConversion(const Twine &FileName, const int FD = -1) {
return false;
}
+inline ErrorOr<SmallString<32>>
+getEncodingNameFromFileTag(const Twine &FileName, const int FD = -1) {
+#ifdef __MVS__
+ ErrorOr<__ccsid_t> TagOrErr = getzOSFileTag(FileName, FD);
+ if (!TagOrErr)
+ return TagOrErr.getError();
+
+ __ccsid_t Tag = *TagOrErr;
+ if (Tag == 0)
+ return SmallString<32>(); // Return empty string for no tag
+
+ if (Tag == 1208)
+ return SmallString<32>("utf-8");
+
+ if (Tag == 1047)
+ return SmallString<32>("ibm-1047");
+
+ SmallString<32> Result;
+ raw_svector_ostream(Result) << Tag;
+ return Result;
+#else
+ return SmallString<32>(); // Return empty string for non-MVS platforms
+#endif
+}
+
} /* namespace llvm */
#endif /* __cplusplus */
>From 1b21117d5f620db589ba6badcd922914fd4c3922 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Tue, 14 Jul 2026 13:17:22 -0400
Subject: [PATCH 4/7] Add cache for text encoding converters
---
clang/include/clang/Basic/SourceManager.h | 15 ++++++++++
clang/lib/Basic/SourceManager.cpp | 35 +++++++++++++++++++++++
2 files changed, 50 insertions(+)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 1939d1aa4915e..2b5cf0b1e084e 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -50,6 +50,7 @@
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/TextEncoding.h"
#include <cassert>
#include <cstddef>
#include <map>
@@ -846,6 +847,11 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// we can add a cc1-level option to do so.
SmallVector<std::pair<std::string, FullSourceLoc>, 2> StoredModuleBuildStack;
+ /// Cache of all text encoding converters used by this SourceManager.
+ /// This includes both the input charset converter and file tag converters.
+ /// Maps from "source_encoding:target_encoding" to the converter.
+ llvm::StringMap<std::unique_ptr<llvm::TextEncodingConverter>> ConverterCache;
+
public:
SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr,
bool UserFilesAreVolatile = false);
@@ -863,6 +869,15 @@ class SourceManager : public RefCountedBase<SourceManager> {
FileManager &getFileManager() const { return FileMgr; }
+ /// Get or create a text encoding converter from the cache.
+ /// This method manages all converters (input charset and file tag converters)
+ /// in a single cache owned by SourceManager.
+ /// \param SourceEncoding the source character encoding name
+ /// \return pointer to the converter or an error code
+ /// The target encoding is always UTF-8.
+ llvm::ErrorOr<llvm::TextEncodingConverter *>
+ getOrCreateConverter(llvm::StringRef SourceEncoding);
+
/// Set true if the SourceManager should report the original file name
/// for contents of files that were overridden by other files. Defaults to
/// true.
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 5540aade05ef5..6175d09881f56 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -448,6 +448,41 @@ ContentCache &SourceManager::createMemBufferContentCache(
return *Entry;
}
+llvm::ErrorOr<llvm::TextEncodingConverter *>
+SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) {
+ // Use getKnownEncoding to get normalized encoding names
+ std::optional<llvm::TextEncoding> SourceKnown =
+ llvm::TextEncodingConverter::getKnownEncoding(SourceEncoding);
+
+ if (SourceKnown && *SourceKnown == llvm::TextEncoding::UTF8)
+ return nullptr;
+
+ // Create a cache key using canonical encoding name
+ llvm::StringRef CacheKey =
+ SourceKnown
+ ? llvm::TextEncodingConverter::getKnownEncodingName(*SourceKnown)
+ : SourceEncoding;
+
+ // Check if converter already exists in cache
+ auto It = ConverterCache.find(CacheKey);
+ if (It != ConverterCache.end())
+ return It->second.get();
+
+ // Create a new converter
+ llvm::ErrorOr<llvm::TextEncodingConverter> NewConverter =
+ llvm::TextEncodingConverter::create(SourceEncoding, "UTF-8");
+
+ if (!NewConverter)
+ return NewConverter.getError();
+
+ // Store the converter in the cache
+ auto Inserted = ConverterCache.insert(std::make_pair(
+ CacheKey,
+ std::make_unique<llvm::TextEncodingConverter>(std::move(*NewConverter))));
+
+ return Inserted.first->second.get();
+}
+
const SrcMgr::SLocEntry &SourceManager::loadSLocEntry(unsigned Index,
bool *Invalid) const {
return const_cast<SourceManager *>(this)->loadSLocEntry(Index, Invalid);
>From 60fbd87edab238e8503c9dc2d6ed71130b4dc8c0 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Tue, 14 Jul 2026 14:52:00 -0400
Subject: [PATCH 5/7] Add CLANG_DEFAULT_INPUT_ENCODING_IBM1047 flag
---
clang/CMakeLists.txt | 3 +++
clang/include/clang/Config/config.h.cmake | 3 +++
clang/lib/Driver/ToolChains/Clang.cpp | 10 +++++++++-
3 files changed, 15 insertions(+), 1 deletion(-)
diff --git a/clang/CMakeLists.txt b/clang/CMakeLists.txt
index 89f584c5dceaa..e8c2c8ac65c67 100644
--- a/clang/CMakeLists.txt
+++ b/clang/CMakeLists.txt
@@ -283,6 +283,9 @@ set(ENABLE_X86_RELAX_RELOCATIONS ON CACHE BOOL
set(PPC_LINUX_DEFAULT_IEEELONGDOUBLE OFF CACHE BOOL
"Enable IEEE binary128 as default long double format on PowerPC Linux.")
+set(CLANG_DEFAULT_INPUT_ENCODING_IBM1047 OFF CACHE BOOL
+ "Set IBM-1047 as the default input encoding")
+
set(CLANG_SPAWN_CC1 OFF CACHE BOOL
"Whether clang should use a new process for the CC1 invocation")
diff --git a/clang/include/clang/Config/config.h.cmake b/clang/include/clang/Config/config.h.cmake
index 11b4096726f67..fbafafc710afe 100644
--- a/clang/include/clang/Config/config.h.cmake
+++ b/clang/include/clang/Config/config.h.cmake
@@ -75,6 +75,9 @@
/* Enable IEEE binary128 as default long double format on PowerPC Linux. */
#cmakedefine01 PPC_LINUX_DEFAULT_IEEELONGDOUBLE
+/* Set IBM-1047 as the default input encoding */
+#cmakedefine01 CLANG_DEFAULT_INPUT_ENCODING_IBM1047
+
/* Enable each functionality of modules */
#cmakedefine01 CLANG_ENABLE_OBJC_REWRITER
#cmakedefine01 CLANG_ENABLE_STATIC_ANALYZER
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index 1d6877ffbd8a7..7a59745b17e13 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -7987,7 +7987,15 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA,
// -finput_charset=UTF-8 is default. Reject others
if (Arg *inputCharset = Args.getLastArg(options::OPT_finput_charset_EQ)) {
StringRef value = inputCharset->getValue();
- if (!value.equals_insensitive("utf-8"))
+#if CLANG_DEFAULT_INPUT_ENCODING_IBM1047
+ // -finput_charset=IBM-1047 is default
+ bool isValid = value.equals_insensitive("ibm-1047");
+ CmdArgs.push_back("-finput-charset");
+ CmdArgs.push_back("IBM-1047");
+#else
+ bool isValid = value.equals_insensitive("utf-8");
+#endif
+ if (!isValid)
D.Diag(diag::err_drv_invalid_value) << inputCharset->getAsString(Args)
<< value;
}
>From b00a260e9ec47e496f0476fd37fb691e85e5caf2 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Tue, 14 Jul 2026 13:36:17 -0400
Subject: [PATCH 6/7] Changes related to finput-charset support
---
.../clang/Basic/DiagnosticCommonKinds.td | 5 +
clang/include/clang/Basic/LangOptions.h | 3 +
clang/include/clang/Basic/SourceManager.h | 38 ++++-
.../include/clang/Frontend/CompilerInstance.h | 1 +
clang/lib/Basic/SourceManager.cpp | 149 +++++++++++++++---
clang/lib/Frontend/CompilerInstance.cpp | 20 ++-
.../lib/Frontend/VerifyDiagnosticConsumer.cpp | 4 +-
clang/lib/Lex/ModuleMap.cpp | 8 +-
clang/lib/Lex/PPDirectives.cpp | 6 +-
clang/lib/Lex/Preprocessor.cpp | 6 +-
clang/lib/Serialization/ASTReader.cpp | 5 +-
11 files changed, 208 insertions(+), 37 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td
index f2ed2f4698b8d..55e71020c9c91 100644
--- a/clang/include/clang/Basic/DiagnosticCommonKinds.td
+++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td
@@ -417,6 +417,11 @@ def note_file_sloc_usage : Note<
"%plural{0:|: plus %2B (%human2B) for macro expansions}2">;
def note_file_misc_sloc_usage : Note<
"%0 additional files entered using a total of %1B (%human1B) of space">;
+def warn_encoding_conversion_failed : Warning<
+ "conversion from source encoding failed for '%0': %1; interpreting as %2">,
+ InGroup<DiagGroup<"encoding-conversion-failed">>;
+def err_encoding_conversion_failed : Error<
+ "conversion from source encoding failed for '%0': %1">;
// Modules
def err_module_format_unhandled : Error<
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index f21131622d03d..989069fd0b98b 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -623,6 +623,9 @@ class LangOptions : public LangOptionsBase {
/// Name of the literal encoding to convert the internal encoding to.
std::string LiteralEncoding;
+
+ /// Name of the input encoding to convert to the internal encoding.
+ std::string InputEncoding;
LangOptions();
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 2b5cf0b1e084e..16dd446371540 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -157,6 +157,13 @@ class alignas(8) ContentCache {
/// FIXME: Remove this once OrigEntry is a FileEntryRef with a stable name.
StringRef Filename;
+ /// Information on whether this is associated with a FileID for a file (as
+ /// opposed to a buffer) and, if so, what conversion (if any) was requested.
+ /// The integer part uses 2 bits: bit 0 indicates if used by FileID,
+ /// bit 1 indicates if the file was tagged.
+ llvm::PointerIntPair<llvm::TextEncodingConverter *, 2u, unsigned>
+ FileIDConverterInfo;
+
/// A bump pointer allocated array of offsets for each source line.
///
/// This is lazily computed. The lines are owned by the SourceManager
@@ -273,6 +280,32 @@ class alignas(8) ContentCache {
// If BufStr has an invalid BOM, returns the BOM name; otherwise, returns
// nullptr
static const char *getInvalidBOM(StringRef BufStr);
+
+ /// Helper methods for FileIDConverterInfo bit manipulation.
+ /// Bit 0: Used by FileID flag
+ /// Bit 1: File tagged flag
+
+ bool isUsedByFileID() const { return FileIDConverterInfo.getInt() & 0x1; }
+
+ void setUsedByFileID(bool Used) {
+ unsigned Flags = FileIDConverterInfo.getInt();
+ if (Used)
+ Flags |= 0x1;
+ else
+ Flags &= ~0x1;
+ FileIDConverterInfo.setInt(Flags);
+ }
+
+ bool isFileTagged() const { return FileIDConverterInfo.getInt() & 0x2; }
+
+ void setFileTagged(bool Tagged) {
+ unsigned Flags = FileIDConverterInfo.getInt();
+ if (Tagged)
+ Flags |= 0x2;
+ else
+ Flags &= ~0x2;
+ FileIDConverterInfo.setInt(Flags);
+ }
};
// Assert that the \c ContentCache objects will always be 8-byte aligned so
@@ -939,7 +972,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// being \#included from the specified IncludePosition.
FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos,
SrcMgr::CharacteristicKind FileCharacter,
- int LoadedID = 0,
+ llvm::StringRef InputEncodingName = {}, int LoadedID = 0,
SourceLocation::UIntTy LoadedOffset = 0);
/// Create a new FileID that represents the specified memory buffer.
@@ -963,7 +996,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// Get the FileID for \p SourceFile if it exists. Otherwise, create a
/// new FileID for the \p SourceFile.
FileID getOrCreateFileID(FileEntryRef SourceFile,
- SrcMgr::CharacteristicKind FileCharacter);
+ SrcMgr::CharacteristicKind FileCharacter,
+ llvm::StringRef InputEncodingName = {});
/// Creates an expansion SLocEntry for the substitution of an argument into a
/// function-like macro's body. Returns the start of the expansion.
diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h
index 24488e053c628..0ebc50d99b86d 100644
--- a/clang/include/clang/Frontend/CompilerInstance.h
+++ b/clang/include/clang/Frontend/CompilerInstance.h
@@ -869,6 +869,7 @@ class CompilerInstance : public ModuleLoader {
///
/// \return True on success.
static bool InitializeSourceManager(const FrontendInputFile &Input,
+ llvm::StringRef InputEncodingName,
DiagnosticsEngine &Diags,
FileManager &FileMgr,
SourceManager &SourceMgr);
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 6175d09881f56..5b12c6177de6c 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -16,6 +16,7 @@
#include "clang/Basic/LLVM.h"
#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/SourceManagerInternals.h"
+#include "clang/Config/config.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
@@ -31,6 +32,7 @@
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
+#include "llvm/Support/SmallVectorMemoryBuffer.h"
#include "llvm/Support/raw_ostream.h"
#include <algorithm>
#include <cassert>
@@ -44,6 +46,7 @@
#include <vector>
using namespace clang;
+
using namespace SrcMgr;
using llvm::MemoryBuffer;
@@ -152,6 +155,83 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
Buffer = std::move(*BufferOrError);
+ // Unless this is a named pipe (in which case we can handle a mismatch),
+ // check that the file's size is the same as in the file entry (which may
+ // have come from a stat cache).
+ assert(Buffer->getBufferSize() >= (size_t)ContentsEntry->getSize());
+ if (!ContentsEntry->isNamedPipe() &&
+ Buffer->getBufferSize() < (size_t)ContentsEntry->getSize()) {
+ Diag.Report(Loc, diag::err_file_modified) << ContentsEntry->getName();
+
+ return std::nullopt;
+ }
+
+ // Convert source from the input charset to UTF-8 if necessary.
+ llvm::TextEncodingConverter *Converter = FileIDConverterInfo.getPointer();
+ if (Converter) {
+ StringRef OriginalBuf = Buffer->getBuffer();
+ llvm::SmallString<0> UTF8Buf;
+ UTF8Buf.reserve(OriginalBuf.size() + 1);
+
+ std::error_code EC = Converter->convert(OriginalBuf, UTF8Buf);
+ if (EC) {
+ // For tagged files, conversion failure is an error and we don't fall back
+ if (isFileTagged()) {
+ Diag.Report(Loc, diag::err_encoding_conversion_failed)
+ << ContentsEntry->getName() << EC.message();
+ return std::nullopt;
+ }
+
+ // Only fall back to the default encoding if the file is untagged.
+ // If conversion fails, emit a warning and attempt to fall back to the
+ // default encoding.
+ //
+ // This allows the compiler to accept system or third-party headers that
+ // are encoded in the default encoding even if conversion to the
+ // option-specified input encoding failed.
+ //
+ // TODO: Add input byte offset information.
+ // FIXME: Need to fallback to IBM-1047 if that is the default even if the
+ // file is being read as UTF-8.
+ const char *FallbackEncoding =
+ CLANG_DEFAULT_INPUT_ENCODING_IBM1047 ? "IBM-1047" : "UTF-8";
+ Diag.Report(Loc, diag::warn_encoding_conversion_failed)
+ << ContentsEntry->getName() << EC.message() << FallbackEncoding;
+
+ // If the default fallback encoding is IBM-1047, attempt conversion using
+ // that converter
+ if (CLANG_DEFAULT_INPUT_ENCODING_IBM1047) {
+ // Create fallback converter in place
+ llvm::ErrorOr<llvm::TextEncodingConverter> FallbackConverter =
+ llvm::TextEncodingConverter::create("IBM-1047", "UTF-8");
+ // Only attempt conversion if the fallback converter was created
+ // successfully
+ if (FallbackConverter) {
+ UTF8Buf.clear();
+ UTF8Buf.reserve(OriginalBuf.size() + 1);
+ std::error_code FallbackEC =
+ FallbackConverter->convert(OriginalBuf, UTF8Buf);
+ if (!FallbackEC) {
+ // Conversion with fallback encoding succeeded
+ auto NewBuf = std::make_unique<llvm::SmallVectorMemoryBuffer>(
+ std::move(UTF8Buf), Buffer->getBufferIdentifier());
+ Buffer = std::move(NewBuf);
+ } else {
+ // Fallback conversion also failed, emit an error
+ Diag.Report(Loc, diag::err_encoding_conversion_failed)
+ << ContentsEntry->getName() << FallbackEC.message();
+ return std::nullopt;
+ }
+ }
+ }
+ } else {
+ // TODO: Reclaim memory if the buffer size exceeds the content.
+ auto NewBuf = std::make_unique<llvm::SmallVectorMemoryBuffer>(
+ std::move(UTF8Buf), Buffer->getBufferIdentifier());
+ Buffer = std::move(NewBuf);
+ }
+ }
+
// Check that the file's size fits in an 'unsigned' (with room for a
// past-the-end value). This is deeply regrettable, but various parts of
// Clang (including elsewhere in this file!) use 'unsigned' to represent file
@@ -167,22 +247,15 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
return std::nullopt;
}
- // Unless this is a named pipe (in which case we can handle a mismatch),
- // check that the file's size is the same as in the file entry (which may
- // have come from a stat cache).
- // The buffer will always be larger than the file size on z/OS in the presence
- // of characters outside the base character set.
- assert(Buffer->getBufferSize() >= (size_t)ContentsEntry->getSize());
- if (!ContentsEntry->isNamedPipe() &&
- Buffer->getBufferSize() < (size_t)ContentsEntry->getSize()) {
- Diag.Report(Loc, diag::err_file_modified) << ContentsEntry->getName();
-
- return std::nullopt;
- }
-
- // If the buffer is valid, check to see if it has a UTF Byte Order Mark
- // (BOM). We only support UTF-8 with and without a BOM right now. See
- // http://en.wikipedia.org/wiki/Byte_order_mark for more information.
+ // If the buffer is valid, check to see if it has a UTF Byte Order Mark (BOM)
+ // Note that any conversion requested using `-finput-charset` (if successful)
+ // has already occurred, so we are expecting UTF-8 with or without a BOM.
+ //
+ // In theory, if we see a non-UTF-8 BOM, we can assume that an appropriate
+ // conversion was not supplied via `-finput-charset` and we could try to
+ // convert based on the BOM.
+ //
+ // See http://en.wikipedia.org/wiki/Byte_order_mark for more information.
StringRef BufStr = Buffer->getBuffer();
const char *InvalidBOM = getInvalidBOM(BufStr);
@@ -592,6 +665,7 @@ FileID SourceManager::getNextFileID(FileID FID) const {
FileID SourceManager::createFileID(FileEntryRef SourceFile,
SourceLocation IncludePos,
SrcMgr::CharacteristicKind FileCharacter,
+ llvm::StringRef InputEncodingName,
int LoadedID,
SourceLocation::UIntTy LoadedOffset) {
SrcMgr::ContentCache &IR = getOrCreateContentCache(SourceFile,
@@ -605,9 +679,39 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
FileIDContentCaches.push_back(Cache);
}
+ llvm::ErrorOr<llvm::TextEncodingConverter *> Converter = nullptr;
+ if (!InputEncodingName.empty()) {
+ // No file tag but -finput-charset conversion is desired.
+ // Get the converter from the cache using the input encoding name.
+ Converter = getOrCreateConverter(InputEncodingName);
+ if (!Converter) {
+ llvm::report_fatal_error("Cannot create converter for file '" +
+ SourceFile.getName() +
+ "': " + Converter.getError().message());
+ }
+ }
+
+#ifndef NDEBUG
+ // Either the content cache has never been used for a FileID (and, if we are
+ // being asked to use a converter, there should be no valid buffer set up for
+ // it) or the conversion (or lack thereof) should be the same as that used
+ // previously.
+ llvm::TextEncodingConverter *CacheConverter =
+ IR.FileIDConverterInfo.getPointer();
+ bool CacheUsedByFileID = IR.isUsedByFileID();
+ llvm::TextEncodingConverter *ConverterPtr = Converter ? *Converter : nullptr;
+ if (CacheUsedByFileID)
+ assert(CacheConverter == ConverterPtr);
+ else
+ assert(!ConverterPtr || IR.IsBufferInvalid || !IR.getBufferIfLoaded());
+#endif
+ IR.setUsedByFileID(true);
+ IR.FileIDConverterInfo.setPointer(Converter ? *Converter : nullptr);
+
// If this is a named pipe, immediately load the buffer to ensure subsequent
// calls to ContentCache::getSize() are accurate.
- if (Cache->ContentsEntry->isNamedPipe())
+ // Do the same if character-encoding conversion was requested.
+ if (Cache->ContentsEntry->isNamedPipe() || Converter)
(void)Cache->getBufferOrNone(Diag, getFileManager(), SourceLocation());
return createFileIDImpl(*Cache, Filename, IncludePos, FileCharacter, LoadedID,
@@ -645,10 +749,12 @@ FileID SourceManager::createFileID(const llvm::MemoryBufferRef &Buffer,
/// new FileID for the \p SourceFile.
FileID
SourceManager::getOrCreateFileID(FileEntryRef SourceFile,
- SrcMgr::CharacteristicKind FileCharacter) {
+ SrcMgr::CharacteristicKind FileCharacter,
+ llvm::StringRef InputEncodingName) {
FileID ID = translateFile(SourceFile);
- return ID.isValid() ? ID : createFileID(SourceFile, SourceLocation(),
- FileCharacter);
+ return ID.isValid() ? ID
+ : createFileID(SourceFile, SourceLocation(),
+ FileCharacter, InputEncodingName);
}
/// createFileID - Create a new FileID for the specified ContentCache and
@@ -2404,7 +2510,8 @@ SourceManagerForFile::SourceManagerForFile(StringRef FileName,
SourceMgr = std::make_unique<SourceManager>(*Diagnostics, *FileMgr);
FileEntryRef FE = llvm::cantFail(FileMgr->getFileRef(FileName));
FileID ID =
- SourceMgr->createFileID(FE, SourceLocation(), clang::SrcMgr::C_User);
+ SourceMgr->createFileID(FE, SourceLocation(), clang::SrcMgr::C_User,
+ /*InputEncodingName=*/{});
assert(ID.isValid());
SourceMgr->setMainFileID(ID);
}
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 941c9e40f0c1d..70839e8fa6607 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -918,15 +918,19 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary,
// Initialization Utilities
bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){
- return InitializeSourceManager(Input, getDiagnostics(), getFileManager(),
- getSourceManager());
+ // Only get InputEncoding from LangOpts if we have a preprocessor, because
+ // LangOpts is only initialized when preprocessing is enabled.
+ StringRef InputEncodingName =
+ hasPreprocessor() ? llvm::StringRef(getLangOpts().InputEncoding)
+ : llvm::StringRef();
+ return InitializeSourceManager(Input, InputEncodingName, getDiagnostics(),
+ getFileManager(), getSourceManager());
}
// static
-bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input,
- DiagnosticsEngine &Diags,
- FileManager &FileMgr,
- SourceManager &SourceMgr) {
+bool CompilerInstance::InitializeSourceManager(
+ const FrontendInputFile &Input, llvm::StringRef InputEncodingName,
+ DiagnosticsEngine &Diags, FileManager &FileMgr, SourceManager &SourceMgr) {
SrcMgr::CharacteristicKind Kind =
Input.getKind().getFormat() == InputKind::ModuleMap
? Input.isSystem() ? SrcMgr::C_System_ModuleMap
@@ -955,8 +959,8 @@ bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input,
return false;
}
- SourceMgr.setMainFileID(
- SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind));
+ SourceMgr.setMainFileID(SourceMgr.createFileID(*FileOrErr, SourceLocation(),
+ Kind, InputEncodingName));
assert(SourceMgr.getMainFileID().isValid() &&
"Couldn't establish MainFileID!");
diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
index 1bfe644b2525a..1ae2337adbb1f 100644
--- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
+++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
@@ -610,8 +610,10 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
}
FileID FID = SM.translateFile(*File);
+ // FIXME: Figure out character-encoding converter treatment.
if (FID.isInvalid())
- FID = SM.createFileID(*File, Pos, SrcMgr::C_User);
+ FID = SM.createFileID(*File, Pos, SrcMgr::C_User,
+ /*InputEncodingName=*/{});
if (PH.Next(Line) && Line > 0)
ExpectedLoc = SM.translateLineCol(FID, Line, 1);
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index 6c07386f89010..b9750e7a8c197 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -1473,7 +1473,13 @@ bool ModuleMap::parseModuleMapFile(FileEntryRef File, bool IsSystem,
if (LocalFID.isInvalid()) {
auto FileCharacter =
IsSystem ? SrcMgr::C_System_ModuleMap : SrcMgr::C_User_ModuleMap;
- LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter);
+ // Module map files are textual "source files". Use input charset
+ // converter if available, and file tag converters are handled by
+ // SourceManager's cache. Get input encoding from LangOptions for charset
+ // conversion
+ llvm::StringRef InputEncodingName = LangOpts.InputEncoding;
+ LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter,
+ InputEncodingName);
}
ID = LocalFID;
}
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index c161f6a03593e..a96873059f610 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -2800,7 +2800,11 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
// position on the file where it will be included and after the expansions.
if (IncludePos.isMacroID())
IncludePos = SourceMgr.getExpansionRange(IncludePos).getEnd();
- FileID FID = SourceMgr.createFileID(*File, IncludePos, FileCharacter);
+ // Use the SourceManager's input charset converter for non-tagged files
+ // by passing the input encoding name
+ llvm::StringRef InputEncodingName = getLangOpts().InputEncoding;
+ FileID FID = SourceMgr.createFileID(*File, IncludePos, FileCharacter,
+ InputEncodingName);
if (!FID.isValid()) {
TheModuleLoader.HadFatalFailure = true;
return ImportAction::Failure;
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 427a3826b299a..c1ac6252f93fc 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -655,8 +655,10 @@ void Preprocessor::EnterMainSourceFile() {
<< PPOpts.PCHThroughHeader;
return;
}
- setPCHThroughHeaderFileID(
- SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User));
+ // FIXME: Figure out character-encoding converter treatment.
+ setPCHThroughHeaderFileID(SourceMgr.createFileID(*File, SourceLocation(),
+ SrcMgr::C_User,
+ /*InputEncodingName=*/{}));
}
// Skip tokens from the Predefines and if needed the main file.
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 080cb7df04406..575fad0e029e3 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -2005,7 +2005,10 @@ bool ASTReader::ReadSLocEntry(int ID) {
}
SrcMgr::CharacteristicKind
FileCharacter = (SrcMgr::CharacteristicKind)Record[2];
- FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter, ID,
+ // Note: If conversion was originally necessary, OverriddenBuffer should be
+ // true and the associated handling will trigger.
+ FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter,
+ /*InputEncodingName=*/{}, ID,
BaseOffset + Record[0]);
SrcMgr::FileInfo &FileInfo = SourceMgr.getSLocEntry(FID).getFile();
FileInfo.NumCreatedFIDs = Record[5];
>From 244b900f70e4156e4d8f7de299e5fcdd962472e7 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Tue, 14 Jul 2026 13:28:50 -0400
Subject: [PATCH 7/7] Use explicit converters
---
clang/lib/Basic/SourceManager.cpp | 31 +++++++++++++++++++++++++++++--
1 file changed, 29 insertions(+), 2 deletions(-)
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 5b12c6177de6c..ea389ad56bc4d 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -139,7 +139,16 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
// return paths.
IsBufferInvalid = true;
- auto BufferOrError = FM.getBufferForFile(*ContentsEntry, IsFileVolatile);
+ // If a converter is set, open the file in binary mode to get raw bytes
+ // and avoid platform-specific auto-conversion (e.g., EBCDIC->ASCII on z/OS,
+ // CRLF->LF on Windows). The explicit converter will handle all
+ // transformations.
+ bool NeedsExplicitConversion = FileIDConverterInfo.getPointer() != nullptr;
+ bool IsText = !NeedsExplicitConversion;
+
+ auto BufferOrError = FM.getBufferForFile(*ContentsEntry, IsFileVolatile,
+ /*RequiresNullTerminator=*/true,
+ /*MaybeLimit=*/std::nullopt, IsText);
// If we were unable to open the file, then we are in an inconsistent
// situation where the content cache referenced a file which no longer
@@ -680,7 +689,25 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
}
llvm::ErrorOr<llvm::TextEncodingConverter *> Converter = nullptr;
- if (!InputEncodingName.empty()) {
+ llvm::ErrorOr<llvm::SmallString<32>> Ccsid =
+ llvm::getEncodingNameFromFileTag(SourceFile.getName());
+ if (!Ccsid) {
+ Diag.Report(SourceLocation(), diag::err_cannot_open_file)
+ << SourceFile.getName() << Ccsid.getError().message();
+ return FileID();
+ }
+ if (!Ccsid->empty()) {
+ // File has a tag, use the converter from SourceManager's cache
+ Converter = getOrCreateConverter(*Ccsid);
+ if (!Converter) {
+ Diag.Report(SourceLocation(), diag::err_cannot_open_file)
+ << SourceFile.getName()
+ << (llvm::Twine("cannot create converter from encoding '") + *Ccsid +
+ "'");
+ return FileID();
+ }
+ IR.setFileTagged(true);
+ } else if (!InputEncodingName.empty()) {
// No file tag but -finput-charset conversion is desired.
// Get the converter from the cache using the input encoding name.
Converter = getOrCreateConverter(InputEncodingName);
More information about the cfe-commits
mailing list