[clang] [llvm] Alisonz/tmp1 (PR #205163)
Alison Zhang via cfe-commits
cfe-commits at lists.llvm.org
Mon Jun 22 12:54:21 PDT 2026
https://github.com/azhan92 updated https://github.com/llvm/llvm-project/pull/205163
>From 785f4a51a8e7696a273ac12a6ae1fbbf30f04c1c Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Thu, 18 Jun 2026 12:00:14 -0400
Subject: [PATCH 01/29] Changes from fexec-charset PR
---
clang/include/clang/Lex/Preprocessor.h | 4 ++++
clang/include/clang/Lex/TextEncoding.h | 23 +++++++++++++++++++++++
clang/lib/Lex/CMakeLists.txt | 1 +
clang/lib/Lex/TextEncoding.cpp | 18 ++++++++++++++++++
4 files changed, 46 insertions(+)
create mode 100644 clang/include/clang/Lex/TextEncoding.h
create mode 100644 clang/lib/Lex/TextEncoding.cpp
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 8b684e85eb1c1..28a14eb76f8ef 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -30,6 +30,7 @@
#include "clang/Lex/ModuleMap.h"
#include "clang/Lex/PPCallbacks.h"
#include "clang/Lex/PPEmbedParameters.h"
+#include "clang/Lex/TextEncoding.h"
#include "clang/Lex/Token.h"
#include "clang/Lex/TokenLexer.h"
#include "clang/Support/Compiler.h"
@@ -198,6 +199,7 @@ class Preprocessor {
std::unique_ptr<ScratchBuffer> ScratchBuf;
HeaderSearch &HeaderInfo;
ModuleLoader &TheModuleLoader;
+ TextEncoding TE;
/// External source of macros.
ExternalPreprocessorSource *ExternalSource;
@@ -1265,6 +1267,8 @@ class Preprocessor {
Builtin::Context &getBuiltinInfo() { return *BuiltinInfo; }
llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
+ TextEncoding &getTextEncoding() { return TE; }
+
void setExternalSource(ExternalPreprocessorSource *Source) {
ExternalSource = Source;
}
diff --git a/clang/include/clang/Lex/TextEncoding.h b/clang/include/clang/Lex/TextEncoding.h
new file mode 100644
index 0000000000000..ef718aec4d6d6
--- /dev/null
+++ b/clang/include/clang/Lex/TextEncoding.h
@@ -0,0 +1,23 @@
+//===-- clang/Lex/TextEncoding.h - Text Conversion Config -*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_CLANG_LEX_TEXTENCODING_H
+#define LLVM_CLANG_LEX_TEXTENCODING_H
+
+#include "clang/Basic/LangOptions.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Support/TextEncoding.h"
+
+enum ConversionAction { CA_NoConversion };
+
+class TextEncoding {
+public:
+ llvm::TextEncodingConverter *getConverter(ConversionAction Action) const;
+};
+
+#endif
diff --git a/clang/lib/Lex/CMakeLists.txt b/clang/lib/Lex/CMakeLists.txt
index f61737cd68021..7b0be7249cd99 100644
--- a/clang/lib/Lex/CMakeLists.txt
+++ b/clang/lib/Lex/CMakeLists.txt
@@ -29,6 +29,7 @@ add_clang_library(clangLex
Preprocessor.cpp
PreprocessorLexer.cpp
ScratchBuffer.cpp
+ TextEncoding.cpp
TokenConcatenation.cpp
TokenLexer.cpp
diff --git a/clang/lib/Lex/TextEncoding.cpp b/clang/lib/Lex/TextEncoding.cpp
new file mode 100644
index 0000000000000..33e5436367014
--- /dev/null
+++ b/clang/lib/Lex/TextEncoding.cpp
@@ -0,0 +1,18 @@
+//===--- TextEncoding.cpp -------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/TextEncoding.h"
+#include "clang/Basic/DiagnosticDriver.h"
+
+llvm::TextEncodingConverter *
+TextEncoding::getConverter(ConversionAction Action) const {
+ switch (Action) {
+ default:
+ return nullptr;
+ }
+}
>From e8270826b0c143247bf62459e20e9643d7468c29 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Thu, 18 Jun 2026 12:05:21 -0400
Subject: [PATCH 02/29] Changes from finput-charset PR
---
.../clang/Basic/DiagnosticCommonKinds.td | 3 +
clang/include/clang/Basic/SourceManager.h | 10 +-
.../include/clang/Frontend/CompilerInstance.h | 1 +
clang/include/clang/Lex/TextEncoding.h | 4 +-
clang/lib/Basic/SourceManager.cpp | 104 ++++++++++++++----
clang/lib/Frontend/CompilerInstance.cpp | 20 ++--
.../lib/Frontend/VerifyDiagnosticConsumer.cpp | 4 +-
clang/lib/Lex/ModuleMap.cpp | 5 +-
clang/lib/Lex/PPDirectives.cpp | 7 +-
clang/lib/Lex/Preprocessor.cpp | 5 +-
clang/lib/Lex/TextEncoding.cpp | 2 +
clang/lib/Serialization/ASTReader.cpp | 7 +-
12 files changed, 132 insertions(+), 40 deletions(-)
diff --git a/clang/include/clang/Basic/DiagnosticCommonKinds.td b/clang/include/clang/Basic/DiagnosticCommonKinds.td
index f2ed2f4698b8d..8ebac3908b465 100644
--- a/clang/include/clang/Basic/DiagnosticCommonKinds.td
+++ b/clang/include/clang/Basic/DiagnosticCommonKinds.td
@@ -417,6 +417,9 @@ def note_file_sloc_usage : Note<
"%plural{0:|: plus %2B (%human2B) for macro expansions}2">;
def note_file_misc_sloc_usage : Note<
"%0 additional files entered using a total of %1B (%human1B) of space">;
+def warn_charset_conversion_failed : Warning<
+ "conversion from source encoding failed for '%0': %1; interpreting as IBM-1047">,
+ InGroup<DiagGroup<"charset-conversion-failed">>;
// Modules
def err_module_format_unhandled : Error<
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 4217b8683da1e..f7d91d612e4ab 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -50,6 +50,7 @@
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/TextEncoding.h"
#include <cassert>
#include <cstddef>
#include <map>
@@ -156,6 +157,11 @@ class alignas(8) ContentCache {
/// FIXME: Remove this once OrigEntry is a FileEntryRef with a stable name.
StringRef Filename;
+ /// Information on whether this is associated with a FileID for a file (as
+ /// opposed to a buffer) and, if so, what conversion (if any) was requested.
+ llvm::PointerIntPair<llvm::TextEncodingConverter *, 1u, bool>
+ FileIDConverterInfo;
+
/// A bump pointer allocated array of offsets for each source line.
///
/// This is lazily computed. The lines are owned by the SourceManager
@@ -918,6 +924,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// being \#included from the specified IncludePosition.
FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos,
SrcMgr::CharacteristicKind FileCharacter,
+ llvm::TextEncodingConverter *Converter = nullptr,
int LoadedID = 0,
SourceLocation::UIntTy LoadedOffset = 0);
@@ -942,7 +949,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// Get the FileID for \p SourceFile if it exists. Otherwise, create a
/// new FileID for the \p SourceFile.
FileID getOrCreateFileID(FileEntryRef SourceFile,
- SrcMgr::CharacteristicKind FileCharacter);
+ SrcMgr::CharacteristicKind FileCharacter,
+ llvm::TextEncodingConverter *Converter = nullptr);
/// Creates an expansion SLocEntry for the substitution of an argument into a
/// function-like macro's body. Returns the start of the expansion.
diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h
index bb0eddb918623..89a0d066afd6f 100644
--- a/clang/include/clang/Frontend/CompilerInstance.h
+++ b/clang/include/clang/Frontend/CompilerInstance.h
@@ -864,6 +864,7 @@ class CompilerInstance : public ModuleLoader {
///
/// \return True on success.
static bool InitializeSourceManager(const FrontendInputFile &Input,
+ llvm::TextEncodingConverter *,
DiagnosticsEngine &Diags,
FileManager &FileMgr,
SourceManager &SourceMgr);
diff --git a/clang/include/clang/Lex/TextEncoding.h b/clang/include/clang/Lex/TextEncoding.h
index ef718aec4d6d6..3e7653580e994 100644
--- a/clang/include/clang/Lex/TextEncoding.h
+++ b/clang/include/clang/Lex/TextEncoding.h
@@ -13,9 +13,11 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/Support/TextEncoding.h"
-enum ConversionAction { CA_NoConversion };
+enum ConversionAction { CA_NoConversion, CA_FromInputEncoding };
class TextEncoding {
+std::unique_ptr<llvm::TextEncodingConverter> FromInputEncodingConverter;
+
public:
llvm::TextEncodingConverter *getConverter(ConversionAction Action) const;
};
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index b6cc6ec9365f5..950186866c86d 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -16,6 +16,7 @@
#include "clang/Basic/LLVM.h"
#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/SourceManagerInternals.h"
+#include "clang/Lex/TextEncoding.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
@@ -31,6 +32,7 @@
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/SmallVectorMemoryBuffer.h"
#include <algorithm>
#include <cassert>
#include <cstddef>
@@ -136,7 +138,51 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
Buffer = std::move(*BufferOrError);
- // Check that the file's size fits in an 'unsigned' (with room for a
+ // Unless this is a named pipe (in which case we can handle a mismatch),
+ // check that the file's size is the same as in the file entry (which may
+ // have come from a stat cache).
+ assert(Buffer->getBufferSize() >= (size_t)ContentsEntry->getSize());
+ if (!ContentsEntry->isNamedPipe() &&
+ Buffer->getBufferSize() < (size_t)ContentsEntry->getSize()) {
+ Diag.Report(Loc, diag::err_file_modified) << ContentsEntry->getName();
+
+ return std::nullopt;
+ }
+
+ // Convert source from the input charset to UTF-8 if necessary.
+ llvm::TextEncodingConverter *Converter = FileIDConverterInfo.getPointer();
+ if (Converter) {
+ StringRef OriginalBuf = Buffer->getBuffer();
+ llvm::SmallString<0> UTF8Buf;
+ UTF8Buf.reserve(OriginalBuf.size() + 1);
+
+ std::error_code EC = Converter->convert(OriginalBuf, UTF8Buf);
+ if (EC) {
+ // If conversion fails, emit a warning and fall back to interpreting the
+ // file as UTF-8 without conversion.
+ //
+ // This allows the compiler to accept system or third-party headers that
+ // are encoded in UTF-8 even if conversion to the option-specified input
+ // charset failed.
+ //
+ // Diagnostics already exist when files are not well-formed UTF-8.
+ //
+ // TODO: Add input byte offset information.
+ //
+ // TODO: Consider adjusting the message to omit the "interpreting as
+ // UTF-8" recovery description if the warning has been upgraded to an
+ // error.
+ Diag.Report(Loc, diag::warn_charset_conversion_failed)
+ << ContentsEntry->getName() << EC.message();
+ } else {
+ // TODO: Reclaim memory if the buffer size exceeds the content.
+ auto NewBuf = std::make_unique<llvm::SmallVectorMemoryBuffer>(
+ std::move(UTF8Buf), Buffer->getBufferIdentifier());
+ Buffer = std::move(NewBuf);
+ }
+ }
+
+ // Check that the buffer's size fits in an 'unsigned' (with room for a
// past-the-end value). This is deeply regrettable, but various parts of
// Clang (including elsewhere in this file!) use 'unsigned' to represent file
// offsets, line numbers, string literal lengths, and so on, and fail
@@ -151,22 +197,15 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
return std::nullopt;
}
- // Unless this is a named pipe (in which case we can handle a mismatch),
- // check that the file's size is the same as in the file entry (which may
- // have come from a stat cache).
- // The buffer will always be larger than the file size on z/OS in the presence
- // of characters outside the base character set.
- assert(Buffer->getBufferSize() >= (size_t)ContentsEntry->getSize());
- if (!ContentsEntry->isNamedPipe() &&
- Buffer->getBufferSize() < (size_t)ContentsEntry->getSize()) {
- Diag.Report(Loc, diag::err_file_modified) << ContentsEntry->getName();
-
- return std::nullopt;
- }
-
- // If the buffer is valid, check to see if it has a UTF Byte Order Mark
- // (BOM). We only support UTF-8 with and without a BOM right now. See
- // http://en.wikipedia.org/wiki/Byte_order_mark for more information.
+ // If the buffer is valid, check to see if it has a UTF Byte Order Mark (BOM)
+ // Note that any conversion requested using `-finput-charset` (if successful)
+ // has already occurred, so we are expecting UTF-8 with or without a BOM.
+ //
+ // In theory, if we see a non-UTF-8 BOM, we can assume that an appropriate
+ // conversion was not supplied via `-finput-charset` and we could try to
+ // convert based on the BOM.
+ //
+ // See http://en.wikipedia.org/wiki/Byte_order_mark for more information.
StringRef BufStr = Buffer->getBuffer();
const char *InvalidBOM = getInvalidBOM(BufStr);
@@ -537,15 +576,30 @@ FileID SourceManager::getNextFileID(FileID FID) const {
/// being \#included from the specified IncludePosition.
FileID SourceManager::createFileID(FileEntryRef SourceFile,
SourceLocation IncludePos,
- SrcMgr::CharacteristicKind FileCharacter,
+ SrcMgr::CharacteristicKind FileCharacter,
+ llvm::TextEncodingConverter *Converter,
int LoadedID,
SourceLocation::UIntTy LoadedOffset) {
SrcMgr::ContentCache &IR = getOrCreateContentCache(SourceFile,
isSystem(FileCharacter));
+ #ifndef NDEBUG
+ // Either the content cache has never been used for a FileID (and, if we are
+ // being asked to use a converter, there should be no valid buffer set up for
+ // it) or the conversion (or lack thereof) should be the same as that used
+ // previously.
+ auto [CacheConverter, CacheUsedByFileID] = IR.FileIDConverterInfo;
+ if (CacheUsedByFileID)
+ assert(CacheConverter == Converter);
+ else
+ assert(!Converter || IR.IsBufferInvalid || !IR.getBufferIfLoaded());
+#endif
+ IR.FileIDConverterInfo.setPointerAndInt(Converter, true);
+
// If this is a named pipe, immediately load the buffer to ensure subsequent
// calls to ContentCache::getSize() are accurate.
- if (IR.ContentsEntry->isNamedPipe())
+ // Do the same if character-encoding conversion was requested.
+ if (IR.ContentsEntry->isNamedPipe() || Converter)
(void)IR.getBufferOrNone(Diag, getFileManager(), SourceLocation());
return createFileIDImpl(IR, SourceFile.getName(), IncludePos, FileCharacter,
@@ -583,10 +637,12 @@ FileID SourceManager::createFileID(const llvm::MemoryBufferRef &Buffer,
/// new FileID for the \p SourceFile.
FileID
SourceManager::getOrCreateFileID(FileEntryRef SourceFile,
- SrcMgr::CharacteristicKind FileCharacter) {
+ SrcMgr::CharacteristicKind FileCharacter,
+ llvm::TextEncodingConverter *Converter) {
FileID ID = translateFile(SourceFile);
- return ID.isValid() ? ID : createFileID(SourceFile, SourceLocation(),
- FileCharacter);
+ return ID.isValid() ? ID
+ : createFileID(SourceFile, SourceLocation(),
+ FileCharacter, Converter);
}
/// createFileID - Create a new FileID for the specified ContentCache and
@@ -2340,8 +2396,8 @@ SourceManagerForFile::SourceManagerForFile(StringRef FileName,
std::make_unique<DiagnosticsEngine>(DiagnosticIDs::create(), *DiagOpts);
SourceMgr = std::make_unique<SourceManager>(*Diagnostics, *FileMgr);
FileEntryRef FE = llvm::cantFail(FileMgr->getFileRef(FileName));
- FileID ID =
- SourceMgr->createFileID(FE, SourceLocation(), clang::SrcMgr::C_User);
+ FileID ID = SourceMgr->createFileID(
+ FE, SourceLocation(), clang::SrcMgr::C_User, /*Converter=*/nullptr);
assert(ID.isValid());
SourceMgr->setMainFileID(ID);
}
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 8aee45b5dc644..008bdb5bdeb0d 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -34,6 +34,7 @@
#include "clang/Lex/HeaderSearch.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/PreprocessorOptions.h"
+#include "clang/Lex/TextEncoding.h"
#include "clang/Sema/CodeCompleteConsumer.h"
#include "clang/Sema/ParsedAttr.h"
#include "clang/Sema/Sema.h"
@@ -912,15 +913,20 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary,
// Initialization Utilities
bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){
- return InitializeSourceManager(Input, getDiagnostics(), getFileManager(),
- getSourceManager());
+ // Retrieve the converter to the internal charset if it exists.
+ llvm::TextEncodingConverter *Converter =
+ hasPreprocessor() ? getPreprocessor().getTextEncoding().getConverter(
+ CA_FromInputEncoding)
+ : nullptr;
+
+ return InitializeSourceManager(Input, Converter, getDiagnostics(),
+ getFileManager(), getSourceManager());
}
// static
-bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input,
- DiagnosticsEngine &Diags,
- FileManager &FileMgr,
- SourceManager &SourceMgr) {
+bool CompilerInstance::InitializeSourceManager(
+ const FrontendInputFile &Input, llvm::TextEncodingConverter *Converter,
+ DiagnosticsEngine &Diags, FileManager &FileMgr, SourceManager &SourceMgr) {
SrcMgr::CharacteristicKind Kind =
Input.getKind().getFormat() == InputKind::ModuleMap
? Input.isSystem() ? SrcMgr::C_System_ModuleMap
@@ -950,7 +956,7 @@ bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input,
}
SourceMgr.setMainFileID(
- SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind));
+ SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind, Converter));
assert(SourceMgr.getMainFileID().isValid() &&
"Couldn't establish MainFileID!");
diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
index 1bfe644b2525a..01e3b20e0c7cb 100644
--- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
+++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
@@ -610,8 +610,10 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
}
FileID FID = SM.translateFile(*File);
+ // FIXME: Figure out character-encoding converter treatment.
if (FID.isInvalid())
- FID = SM.createFileID(*File, Pos, SrcMgr::C_User);
+ FID = SM.createFileID(*File, Pos, SrcMgr::C_User,
+ /*Converter=*/nullptr);
if (PH.Next(Line) && Line > 0)
ExpectedLoc = SM.translateLineCol(FID, Line, 1);
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index 6c07386f89010..c7c1e04b76ea1 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -1473,7 +1473,10 @@ bool ModuleMap::parseModuleMapFile(FileEntryRef File, bool IsSystem,
if (LocalFID.isInvalid()) {
auto FileCharacter =
IsSystem ? SrcMgr::C_System_ModuleMap : SrcMgr::C_User_ModuleMap;
- LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter);
+ // FIXME: Module map files are also textual "source files". For consistency,
+ // conversion should occur.
+ LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter,
+ /*Converter=*/nullptr);
}
ID = LocalFID;
}
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index eb21a510dcf83..9f42ad12655e1 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -2796,7 +2796,12 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
// position on the file where it will be included and after the expansions.
if (IncludePos.isMacroID())
IncludePos = SourceMgr.getExpansionRange(IncludePos).getEnd();
- FileID FID = SourceMgr.createFileID(*File, IncludePos, FileCharacter);
+ // Retrieve the converter to the internal charset if it exists.
+ llvm::TextEncodingConverter *Converter =
+ getTextEncoding().getConverter(CA_FromInputEncoding);
+
+ FileID FID =
+ SourceMgr.createFileID(*File, IncludePos, FileCharacter, Converter);
if (!FID.isValid()) {
TheModuleLoader.HadFatalFailure = true;
return ImportAction::Failure;
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 1e21b4a94cea3..c5e32468bcc7e 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -649,8 +649,9 @@ void Preprocessor::EnterMainSourceFile() {
<< PPOpts.PCHThroughHeader;
return;
}
- setPCHThroughHeaderFileID(
- SourceMgr.createFileID(*File, SourceLocation(), SrcMgr::C_User));
+ // FIXME: Figure out character-encoding converter treatment.
+ setPCHThroughHeaderFileID(SourceMgr.createFileID(
+ *File, SourceLocation(), SrcMgr::C_User, /*Converter=*/nullptr));
}
// Skip tokens from the Predefines and if needed the main file.
diff --git a/clang/lib/Lex/TextEncoding.cpp b/clang/lib/Lex/TextEncoding.cpp
index 33e5436367014..7a0fea3798277 100644
--- a/clang/lib/Lex/TextEncoding.cpp
+++ b/clang/lib/Lex/TextEncoding.cpp
@@ -12,6 +12,8 @@
llvm::TextEncodingConverter *
TextEncoding::getConverter(ConversionAction Action) const {
switch (Action) {
+ case CA_FromInputEncoding:
+ return FromInputEncodingConverter.get();
default:
return nullptr;
}
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index f8a6a38bb9b5c..379622e92984b 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -2002,8 +2002,11 @@ bool ASTReader::ReadSLocEntry(int ID) {
}
SrcMgr::CharacteristicKind
FileCharacter = (SrcMgr::CharacteristicKind)Record[2];
- FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter, ID,
- BaseOffset + Record[0]);
+ // Note: If conversion was originally necessary, OverriddenBuffer should be
+ // true and the associated handling will trigger.
+ FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter,
+ /*Converter=*/nullptr, ID,
+ BaseOffset + Record[0]);
SrcMgr::FileInfo &FileInfo = SourceMgr.getSLocEntry(FID).getFile();
FileInfo.NumCreatedFIDs = Record[5];
if (Record[3])
>From 569cd4065cdf76720c6636f9ff4af975f25f5d7d Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Thu, 18 Jun 2026 11:18:31 -0400
Subject: [PATCH 03/29] Add getEncodingNameFromFileTag function
---
llvm/include/llvm/Support/AutoConvert.h | 19 +++++++++++++++++++
1 file changed, 19 insertions(+)
diff --git a/llvm/include/llvm/Support/AutoConvert.h b/llvm/include/llvm/Support/AutoConvert.h
index d68b0e8b515e0..337befec1b352 100644
--- a/llvm/include/llvm/Support/AutoConvert.h
+++ b/llvm/include/llvm/Support/AutoConvert.h
@@ -105,6 +105,25 @@ inline ErrorOr<bool> needConversion(const Twine &FileName, const int FD = -1) {
return false;
}
+inline ErrorOr<std::string>
+getEncodingNameFromFileTag(const Twine &FileName, const int FD = -1) {
+#ifdef __MVS__
+ ErrorOr<__ccsid_t> TagOrErr = getzOSFileTag(FileName, FD);
+ if (!TagOrErr)
+ return TagOrErr.getError();
+
+ __ccsid_t Tag = *TagOrErr;
+ if (Tag == 0)
+ return std::string(); // Return empty string for no tag
+
+ char Buffer[16];
+ snprintf(Buffer, sizeof(Buffer), "%03d", Tag);
+ return std::string(Buffer);
+#else
+ return std::string(); // Return empty string for non-MVS platforms
+#endif
+}
+
} /* namespace llvm */
#endif /* __cplusplus */
>From a578243ef2a3f3a4bde452703a3186c38d182949 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Sun, 21 Jun 2026 13:49:51 -0400
Subject: [PATCH 04/29] Add global converter cache
---
llvm/include/llvm/Support/TextEncoding.h | 15 ++++++
llvm/lib/Support/TextEncoding.cpp | 60 ++++++++++++++++++++++++
2 files changed, 75 insertions(+)
diff --git a/llvm/include/llvm/Support/TextEncoding.h b/llvm/include/llvm/Support/TextEncoding.h
index 8a304910aa5dd..6ca37ce6e7a4e 100644
--- a/llvm/include/llvm/Support/TextEncoding.h
+++ b/llvm/include/llvm/Support/TextEncoding.h
@@ -21,8 +21,10 @@
#include "llvm/Support/Compiler.h"
#include "llvm/Support/ErrorOr.h"
+#include <memory>
#include <string>
#include <system_error>
+#include <utility>
namespace llvm {
@@ -137,6 +139,19 @@ class TextEncodingConverter {
}
};
+/// Cache for TextEncodingConverter instances.
+class TextEncodingConverterCache {
+ public:
+ /// Get or create a cached TextEncodingConverter.
+ /// If the converter exists in the cache, returns it. Otherwise, creates a new
+ /// converter, caches it, and returns it.
+ /// \param[in] SourceEncoding the source character encoding name
+ /// \param[in] TargetEncoding the target character encoding name
+ /// \return pointer to the converter or an error code
+ LLVM_ABI static ErrorOr<TextEncodingConverter *>
+ getOrCreateConverter(StringRef SourceEncoding, StringRef TargetEncoding);
+};
+
} // namespace llvm
#endif
diff --git a/llvm/lib/Support/TextEncoding.cpp b/llvm/lib/Support/TextEncoding.cpp
index d36f02c1300b9..7ec3ed037dbc7 100644
--- a/llvm/lib/Support/TextEncoding.cpp
+++ b/llvm/lib/Support/TextEncoding.cpp
@@ -16,8 +16,12 @@
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/StringExtras.h"
+#include "llvm/ADT/StringMap.h"
#include "llvm/Support/ConvertEBCDIC.h"
+#include "llvm/Support/ManagedStatic.h"
+#include "llvm/Support/RWMutex.h"
#include <system_error>
+#include <utility>
#if HAVE_ICU
#if HAVE_WINDOWS_ICU
@@ -356,3 +360,59 @@ ErrorOr<TextEncodingConverter> TextEncodingConverter::create(StringRef From,
return std::make_error_code(std::errc::invalid_argument);
#endif
}
+
+namespace {
+// Global cache for TextEncodingConverter instances
+// Use StringMap which is designed for string keys
+using ConverterCache = StringMap<std::unique_ptr<TextEncodingConverter>>;
+
+struct ConverterCacheData {
+ ConverterCache Cache;
+ llvm::sys::RWMutex Mutex;
+};
+
+static ManagedStatic<ConverterCacheData> GlobalConverterCache;
+} // namespace
+
+ErrorOr<TextEncodingConverter *>
+TextEncodingConverterCache::getOrCreateConverter(StringRef SourceEncoding,
+ StringRef TargetEncoding) {
+ // Don't create a converter if source and target are the same
+ if (SourceEncoding == TargetEncoding)
+ return nullptr;
+
+ // Create cache key by concatenating source and target with a separator
+ SmallString<64> Key;
+ Key = SourceEncoding;
+ Key += " -> ";
+ Key += TargetEncoding;
+
+ // First, try to find existing converter with shared lock (allows concurrent reads)
+ {
+ llvm::sys::ScopedReader ReadLock(GlobalConverterCache->Mutex);
+ auto Iter = GlobalConverterCache->Cache.find(Key);
+ if (Iter != GlobalConverterCache->Cache.end())
+ return Iter->second.get();
+ }
+
+ // Not found, need to create - acquire unique lock for writing
+ llvm::sys::ScopedWriter WriteLock(GlobalConverterCache->Mutex);
+
+ // Double-check: another thread might have created it while we were waiting
+ auto Iter = GlobalConverterCache->Cache.find(Key);
+ if (Iter != GlobalConverterCache->Cache.end())
+ return Iter->second.get();
+
+ // Create a new converter
+ ErrorOr<TextEncodingConverter> ErrorOrConverter =
+ TextEncodingConverter::create(SourceEncoding, TargetEncoding);
+ if (!ErrorOrConverter)
+ return ErrorOrConverter.getError();
+
+ // Insert into cache and return pointer
+ auto NewConverter =
+ std::make_unique<TextEncodingConverter>(std::move(*ErrorOrConverter));
+ TextEncodingConverter *Result = NewConverter.get();
+ GlobalConverterCache->Cache.try_emplace(Key, std::move(NewConverter));
+ return Result;
+}
>From 4fa714631715a9c2b3cd0bc5b5bee196cacaead3 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Thu, 18 Jun 2026 11:31:00 -0400
Subject: [PATCH 05/29] Get filetag and create converter
---
clang/lib/Basic/SourceManager.cpp | 14 ++++++++++++++
1 file changed, 14 insertions(+)
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 950186866c86d..945bc3e4177e2 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -583,6 +583,20 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
SrcMgr::ContentCache &IR = getOrCreateContentCache(SourceFile,
isSystem(FileCharacter));
+ llvm::ErrorOr<std::string> Ccsid =
+ llvm::getEncodingNameFromFileTag(SourceFile.getName());
+ if (!Ccsid) {
+ Diag.Report(SourceLocation(), diag::err_cannot_open_file)
+ << SourceFile.getName() << Ccsid.getError().message();
+ return FileID();
+ }
+ if (!Ccsid->empty()) {
+ llvm::ErrorOr<llvm::TextEncodingConverter *> FileTagConverter =
+ llvm::TextEncodingConverterCache::getOrCreateConverter(*Ccsid, "UTF-8");
+ if (FileTagConverter)
+ Converter = *FileTagConverter;
+ }
+
#ifndef NDEBUG
// Either the content cache has never been used for a FileID (and, if we are
// being asked to use a converter, there should be no valid buffer set up for
>From c14c18c154a5266d549df3a891b50170a495f7b8 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 10:40:16 -0400
Subject: [PATCH 06/29] some more changes
---
clang/include/clang/Basic/SourceManager.h | 39 +++++++++++++-
.../include/clang/Frontend/CompilerInstance.h | 1 -
clang/lib/Basic/SourceManager.cpp | 51 ++++++++++++++++---
clang/lib/Frontend/CompilerInstance.cpp | 23 +++++----
clang/lib/Lex/PPDirectives.cpp | 13 ++---
5 files changed, 102 insertions(+), 25 deletions(-)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index f7d91d612e4ab..4e619e717f802 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -846,6 +846,16 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// we can add a cc1-level option to do so.
SmallVector<std::pair<std::string, FullSourceLoc>, 2> StoredModuleBuildStack;
+ /// Converter for -finput-charset conversion to UTF-8.
+ /// Stored here to avoid global cache and allow SourceManager to manage
+ /// converters for non-tagged files.
+ std::unique_ptr<llvm::TextEncodingConverter> InputCharsetConverter;
+
+ /// Cache of all text encoding converters used by this SourceManager.
+ /// This includes both the input charset converter and file tag converters.
+ /// Maps from "source_encoding:target_encoding" to the converter.
+ llvm::StringMap<std::unique_ptr<llvm::TextEncodingConverter>> ConverterCache;
+
public:
SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr,
bool UserFilesAreVolatile = false);
@@ -863,6 +873,28 @@ class SourceManager : public RefCountedBase<SourceManager> {
FileManager &getFileManager() const { return FileMgr; }
+ /// Set the input charset converter for -finput-charset conversion.
+ void setInputCharsetConverter(
+ std::unique_ptr<llvm::TextEncodingConverter> Converter) {
+ InputCharsetConverter = std::move(Converter);
+ }
+
+ /// Get the input charset converter for -finput-charset conversion.
+ /// Returns nullptr if no converter is set.
+ llvm::TextEncodingConverter *getInputCharsetConverter() const {
+ return InputCharsetConverter.get();
+ }
+
+ /// Get or create a text encoding converter from the cache.
+ /// This method manages all converters (input charset and file tag converters)
+ /// in a single cache owned by SourceManager.
+ /// \param SourceEncoding the source character encoding name
+ /// \param TargetEncoding the target character encoding name
+ /// \return pointer to the converter or an error code
+ llvm::ErrorOr<llvm::TextEncodingConverter *>
+ getOrCreateConverter(llvm::StringRef SourceEncoding,
+ llvm::StringRef TargetEncoding);
+
/// Set true if the SourceManager should report the original file name
/// for contents of files that were overridden by other files. Defaults to
/// true.
@@ -922,11 +954,14 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// Create a new FileID that represents the specified file
/// being \#included from the specified IncludePosition.
+ /// \param Converter Optional converter to use. If nullptr and
+ /// UseInputCharsetConverter is true, will use the SourceManager's
+ /// input charset converter for non-tagged files.
FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos,
SrcMgr::CharacteristicKind FileCharacter,
- llvm::TextEncodingConverter *Converter = nullptr,
int LoadedID = 0,
- SourceLocation::UIntTy LoadedOffset = 0);
+ SourceLocation::UIntTy LoadedOffset = 0,
+ bool UseInputCharsetConverter = false);
/// Create a new FileID that represents the specified memory buffer.
///
diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h
index 89a0d066afd6f..bb0eddb918623 100644
--- a/clang/include/clang/Frontend/CompilerInstance.h
+++ b/clang/include/clang/Frontend/CompilerInstance.h
@@ -864,7 +864,6 @@ class CompilerInstance : public ModuleLoader {
///
/// \return True on success.
static bool InitializeSourceManager(const FrontendInputFile &Input,
- llvm::TextEncodingConverter *,
DiagnosticsEngine &Diags,
FileManager &FileMgr,
SourceManager &SourceMgr);
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 945bc3e4177e2..cdb68e82aa9f2 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -466,6 +466,35 @@ ContentCache &SourceManager::createMemBufferContentCache(
MemBufferInfos.push_back(Entry);
Entry->setBuffer(std::move(Buffer));
return *Entry;
+
+llvm::ErrorOr<llvm::TextEncodingConverter *>
+SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding,
+ llvm::StringRef TargetEncoding) {
+ // Create a cache key from source and target encodings
+ llvm::SmallString<64> CacheKey;
+ CacheKey = SourceEncoding;
+ CacheKey += ":";
+ CacheKey += TargetEncoding;
+
+ // Check if converter already exists in cache
+ auto It = ConverterCache.find(CacheKey);
+ if (It != ConverterCache.end())
+ return It->second.get();
+
+ // Create a new converter
+ llvm::ErrorOr<llvm::TextEncodingConverter> NewConverter =
+ llvm::TextEncodingConverter::create(SourceEncoding, TargetEncoding);
+
+ if (!NewConverter)
+ return NewConverter.getError();
+
+ // Store the converter in the cache
+ auto Inserted = ConverterCache.insert(
+ std::make_pair(CacheKey, std::make_unique<llvm::TextEncodingConverter>(
+ std::move(*NewConverter))));
+
+ return Inserted.first->second.get();
+}
}
const SrcMgr::SLocEntry &SourceManager::loadSLocEntry(unsigned Index,
@@ -576,13 +605,14 @@ FileID SourceManager::getNextFileID(FileID FID) const {
/// being \#included from the specified IncludePosition.
FileID SourceManager::createFileID(FileEntryRef SourceFile,
SourceLocation IncludePos,
- SrcMgr::CharacteristicKind FileCharacter,
- llvm::TextEncodingConverter *Converter,
+ SrcMgr::CharacteristicKind FileCharacter,
int LoadedID,
- SourceLocation::UIntTy LoadedOffset) {
+ SourceLocation::UIntTy LoadedOffset,
+ bool UseInputCharsetConverter) {
SrcMgr::ContentCache &IR = getOrCreateContentCache(SourceFile,
isSystem(FileCharacter));
+ llvm::ErrorOr<llvm::TextEncodingConverter *> Converter = nullptr;
llvm::ErrorOr<std::string> Ccsid =
llvm::getEncodingNameFromFileTag(SourceFile.getName());
if (!Ccsid) {
@@ -591,10 +621,17 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
return FileID();
}
if (!Ccsid->empty()) {
- llvm::ErrorOr<llvm::TextEncodingConverter *> FileTagConverter =
- llvm::TextEncodingConverterCache::getOrCreateConverter(*Ccsid, "UTF-8");
- if (FileTagConverter)
- Converter = *FileTagConverter;
+ // File has a tag, use the converter from SourceManager's cache
+ Converter = getOrCreateConverter(*Ccsid, "UTF-8");
+ if (!Converter) {
+ Diag.Report(SourceLocation(), diag::err_cannot_open_file)
+ << SourceFile.getName() << "Failed to create converter";
+ return FileID();
+ }
+ } else if (UseInputCharsetConverter) {
+ // No file tag but -finput-charset conversion is desired. Use the converter
+ // from SourceManager.
+ Converter = getInputCharsetConverter();
}
#ifndef NDEBUG
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 008bdb5bdeb0d..28ba5ceded1a9 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -913,19 +913,20 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary,
// Initialization Utilities
bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){
- // Retrieve the converter to the internal charset if it exists.
- llvm::TextEncodingConverter *Converter =
- hasPreprocessor() ? getPreprocessor().getTextEncoding().getConverter(
- CA_FromInputEncoding)
- : nullptr;
-
- return InitializeSourceManager(Input, Converter, getDiagnostics(),
+ // Note: The input charset converter is now managed by SourceManager's
+ // converter cache along with file tag converters. The Preprocessor's
+ // TextEncoding still has its own converter for backward compatibility,
+ // but SourceManager owns and caches all converters used during file loading.
+ // When createFileID is called with UseInputCharsetConverter=true, it will
+ // use the converter from SourceManager's InputCharsetConverter field.
+
+ return InitializeSourceManager(Input, getDiagnostics(),
getFileManager(), getSourceManager());
}
// static
bool CompilerInstance::InitializeSourceManager(
- const FrontendInputFile &Input, llvm::TextEncodingConverter *Converter,
+ const FrontendInputFile &Input,
DiagnosticsEngine &Diags, FileManager &FileMgr, SourceManager &SourceMgr) {
SrcMgr::CharacteristicKind Kind =
Input.getKind().getFormat() == InputKind::ModuleMap
@@ -955,8 +956,12 @@ bool CompilerInstance::InitializeSourceManager(
return false;
}
+ // Use UseInputCharsetConverter=true to get converter from SourceManager
SourceMgr.setMainFileID(
- SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind, Converter));
+ SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind,
+ /*Converter=*/nullptr, /*LoadedID=*/0,
+ /*LoadedOffset=*/0,
+ /*UseInputCharsetConverter=*/true));
assert(SourceMgr.getMainFileID().isValid() &&
"Couldn't establish MainFileID!");
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 9f42ad12655e1..0e7604d4ab324 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -2796,12 +2796,13 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
// position on the file where it will be included and after the expansions.
if (IncludePos.isMacroID())
IncludePos = SourceMgr.getExpansionRange(IncludePos).getEnd();
- // Retrieve the converter to the internal charset if it exists.
- llvm::TextEncodingConverter *Converter =
- getTextEncoding().getConverter(CA_FromInputEncoding);
-
- FileID FID =
- SourceMgr.createFileID(*File, IncludePos, FileCharacter, Converter);
+
+ // Use the SourceManager's input charset converter for non-tagged files
+ // by passing UseInputCharsetConverter=true
+ FileID FID = SourceMgr.createFileID(*File, IncludePos, FileCharacter,
+ /*Converter=*/nullptr, /*LoadedID=*/0,
+ /*LoadedOffset=*/0,
+ /*UseInputCharsetConverter=*/true);
if (!FID.isValid()) {
TheModuleLoader.HadFatalFailure = true;
return ImportAction::Failure;
>From f8960168adeb2686d5ca9eb13bfc8c3bdb7fe3d7 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 10:58:17 -0400
Subject: [PATCH 07/29] some updates
---
clang/include/clang/Basic/SourceManager.h | 4 +---
.../include/clang/Frontend/CompilerInstance.h | 3 ++-
clang/lib/Basic/SourceManager.cpp | 9 ++++----
clang/lib/Frontend/CompilerInstance.cpp | 23 +++++++++++--------
4 files changed, 20 insertions(+), 19 deletions(-)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 4e619e717f802..cbcc8f6159cfa 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -847,8 +847,6 @@ class SourceManager : public RefCountedBase<SourceManager> {
SmallVector<std::pair<std::string, FullSourceLoc>, 2> StoredModuleBuildStack;
/// Converter for -finput-charset conversion to UTF-8.
- /// Stored here to avoid global cache and allow SourceManager to manage
- /// converters for non-tagged files.
std::unique_ptr<llvm::TextEncodingConverter> InputCharsetConverter;
/// Cache of all text encoding converters used by this SourceManager.
@@ -985,7 +983,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// new FileID for the \p SourceFile.
FileID getOrCreateFileID(FileEntryRef SourceFile,
SrcMgr::CharacteristicKind FileCharacter,
- llvm::TextEncodingConverter *Converter = nullptr);
+ bool UseInputCharsetConverter = false);
/// Creates an expansion SLocEntry for the substitution of an argument into a
/// function-like macro's body. Returns the start of the expansion.
diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h
index bb0eddb918623..347d5807c4fc6 100644
--- a/clang/include/clang/Frontend/CompilerInstance.h
+++ b/clang/include/clang/Frontend/CompilerInstance.h
@@ -866,7 +866,8 @@ class CompilerInstance : public ModuleLoader {
static bool InitializeSourceManager(const FrontendInputFile &Input,
DiagnosticsEngine &Diags,
FileManager &FileMgr,
- SourceManager &SourceMgr);
+ SourceManager &SourceMgr,
+ bool UseInputCharsetConverter = false);
/// @}
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index cdb68e82aa9f2..69e3672a5e25f 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -495,7 +495,6 @@ SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding,
return Inserted.first->second.get();
}
-}
const SrcMgr::SLocEntry &SourceManager::loadSLocEntry(unsigned Index,
bool *Invalid) const {
@@ -625,7 +624,7 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
Converter = getOrCreateConverter(*Ccsid, "UTF-8");
if (!Converter) {
Diag.Report(SourceLocation(), diag::err_cannot_open_file)
- << SourceFile.getName() << "Failed to create converter";
+ << SourceFile.getName() << Converter.getError().message();
return FileID();
}
} else if (UseInputCharsetConverter) {
@@ -689,11 +688,11 @@ FileID SourceManager::createFileID(const llvm::MemoryBufferRef &Buffer,
FileID
SourceManager::getOrCreateFileID(FileEntryRef SourceFile,
SrcMgr::CharacteristicKind FileCharacter,
- llvm::TextEncodingConverter *Converter) {
+ bool UseInputCharsetConverter) {
FileID ID = translateFile(SourceFile);
return ID.isValid() ? ID
: createFileID(SourceFile, SourceLocation(),
- FileCharacter, Converter);
+ FileCharacter, UseInputCharsetConverter);
}
/// createFileID - Create a new FileID for the specified ContentCache and
@@ -2448,7 +2447,7 @@ SourceManagerForFile::SourceManagerForFile(StringRef FileName,
SourceMgr = std::make_unique<SourceManager>(*Diagnostics, *FileMgr);
FileEntryRef FE = llvm::cantFail(FileMgr->getFileRef(FileName));
FileID ID = SourceMgr->createFileID(
- FE, SourceLocation(), clang::SrcMgr::C_User, /*Converter=*/nullptr);
+ FE, SourceLocation(), clang::SrcMgr::C_User, /*UseInputCharsetConverter=*/false);
assert(ID.isValid());
SourceMgr->setMainFileID(ID);
}
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 28ba5ceded1a9..fbf26af128f6d 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -913,21 +913,24 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary,
// Initialization Utilities
bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){
- // Note: The input charset converter is now managed by SourceManager's
- // converter cache along with file tag converters. The Preprocessor's
- // TextEncoding still has its own converter for backward compatibility,
- // but SourceManager owns and caches all converters used during file loading.
- // When createFileID is called with UseInputCharsetConverter=true, it will
- // use the converter from SourceManager's InputCharsetConverter field.
+ // Check if we have an input charset converter from the preprocessor
+ bool UseInputCharsetConverter = false;
+ if (hasPreprocessor()) {
+ llvm::TextEncodingConverter *Converter =
+ getPreprocessor().getTextEncoding().getConverter(CA_FromInputEncoding);
+ UseInputCharsetConverter = (Converter != nullptr);
+ }
return InitializeSourceManager(Input, getDiagnostics(),
- getFileManager(), getSourceManager());
+ getFileManager(), getSourceManager(),
+ UseInputCharsetConverter);
}
// static
bool CompilerInstance::InitializeSourceManager(
const FrontendInputFile &Input,
- DiagnosticsEngine &Diags, FileManager &FileMgr, SourceManager &SourceMgr) {
+ DiagnosticsEngine &Diags, FileManager &FileMgr, SourceManager &SourceMgr,
+ bool UseInputCharsetConverter) {
SrcMgr::CharacteristicKind Kind =
Input.getKind().getFormat() == InputKind::ModuleMap
? Input.isSystem() ? SrcMgr::C_System_ModuleMap
@@ -956,12 +959,12 @@ bool CompilerInstance::InitializeSourceManager(
return false;
}
- // Use UseInputCharsetConverter=true to get converter from SourceManager
+ // Use the UseInputCharsetConverter parameter passed from the caller
SourceMgr.setMainFileID(
SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind,
/*Converter=*/nullptr, /*LoadedID=*/0,
/*LoadedOffset=*/0,
- /*UseInputCharsetConverter=*/true));
+ UseInputCharsetConverter));
assert(SourceMgr.getMainFileID().isValid() &&
"Couldn't establish MainFileID!");
>From 3310618fe96e3fce3c803290236579ae5aabe25b Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 11:03:04 -0400
Subject: [PATCH 08/29] update createfileID
---
clang/lib/Frontend/VerifyDiagnosticConsumer.cpp | 7 +++++--
clang/lib/Lex/ModuleMap.cpp | 8 +++++---
clang/lib/Lex/Preprocessor.cpp | 7 +++++--
3 files changed, 15 insertions(+), 7 deletions(-)
diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
index 01e3b20e0c7cb..cac28c2733092 100644
--- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
+++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
@@ -610,10 +610,13 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
}
FileID FID = SM.translateFile(*File);
- // FIXME: Figure out character-encoding converter treatment.
+ // Use input charset converter if available, and file tag converters
+ // are handled by SourceManager's cache.
if (FID.isInvalid())
FID = SM.createFileID(*File, Pos, SrcMgr::C_User,
- /*Converter=*/nullptr);
+ /*Converter=*/nullptr, /*LoadedID=*/0,
+ /*LoadedOffset=*/0,
+ /*UseInputCharsetConverter=*/true);
if (PH.Next(Line) && Line > 0)
ExpectedLoc = SM.translateLineCol(FID, Line, 1);
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index c7c1e04b76ea1..ecfca9fc7ee51 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -1473,10 +1473,12 @@ bool ModuleMap::parseModuleMapFile(FileEntryRef File, bool IsSystem,
if (LocalFID.isInvalid()) {
auto FileCharacter =
IsSystem ? SrcMgr::C_System_ModuleMap : SrcMgr::C_User_ModuleMap;
- // FIXME: Module map files are also textual "source files". For consistency,
- // conversion should occur.
+ // Module map files are textual "source files". Use input charset converter
+ // if available, and file tag converters are handled by SourceManager's cache.
LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter,
- /*Converter=*/nullptr);
+ /*Converter=*/nullptr, /*LoadedID=*/0,
+ /*LoadedOffset=*/0,
+ /*UseInputCharsetConverter=*/true);
}
ID = LocalFID;
}
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index c5e32468bcc7e..bac9dc2ef912d 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -649,9 +649,12 @@ void Preprocessor::EnterMainSourceFile() {
<< PPOpts.PCHThroughHeader;
return;
}
- // FIXME: Figure out character-encoding converter treatment.
+ // Use input charset converter if available, and file tag converters
+ // are handled by SourceManager's cache.
setPCHThroughHeaderFileID(SourceMgr.createFileID(
- *File, SourceLocation(), SrcMgr::C_User, /*Converter=*/nullptr));
+ *File, SourceLocation(), SrcMgr::C_User, /*Converter=*/nullptr,
+ /*LoadedID=*/0, /*LoadedOffset=*/0,
+ /*UseInputCharsetConverter=*/true));
}
// Skip tokens from the Predefines and if needed the main file.
>From 478dc9fe071865b63b02c9952da94dcb42d976b5 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 11:07:44 -0400
Subject: [PATCH 09/29] Update createFileID again
---
clang/include/clang/Basic/SourceManager.h | 4 ++--
clang/lib/Basic/SourceManager.cpp | 5 ++---
clang/lib/Frontend/CompilerInstance.cpp | 4 +---
clang/lib/Frontend/VerifyDiagnosticConsumer.cpp | 3 +--
clang/lib/Lex/ModuleMap.cpp | 3 +--
clang/lib/Lex/PPDirectives.cpp | 3 +--
clang/lib/Lex/Preprocessor.cpp | 4 ++--
clang/lib/Serialization/ASTReader.cpp | 4 ++--
8 files changed, 12 insertions(+), 18 deletions(-)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index cbcc8f6159cfa..dce778f623991 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -958,8 +958,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos,
SrcMgr::CharacteristicKind FileCharacter,
int LoadedID = 0,
- SourceLocation::UIntTy LoadedOffset = 0,
- bool UseInputCharsetConverter = false);
+ bool UseInputCharsetConverter = false,
+ SourceLocation::UIntTy LoadedOffset = 0);
/// Create a new FileID that represents the specified memory buffer.
///
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 69e3672a5e25f..344abcff46484 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -16,7 +16,6 @@
#include "clang/Basic/LLVM.h"
#include "clang/Basic/SourceLocation.h"
#include "clang/Basic/SourceManagerInternals.h"
-#include "clang/Lex/TextEncoding.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/MapVector.h"
#include "llvm/ADT/STLExtras.h"
@@ -606,8 +605,8 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
SourceLocation IncludePos,
SrcMgr::CharacteristicKind FileCharacter,
int LoadedID,
- SourceLocation::UIntTy LoadedOffset,
- bool UseInputCharsetConverter) {
+ bool UseInputCharsetConverter,
+ SourceLocation::UIntTy LoadedOffset) {
SrcMgr::ContentCache &IR = getOrCreateContentCache(SourceFile,
isSystem(FileCharacter));
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index fbf26af128f6d..442b9d2b644e2 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -34,7 +34,6 @@
#include "clang/Lex/HeaderSearch.h"
#include "clang/Lex/Preprocessor.h"
#include "clang/Lex/PreprocessorOptions.h"
-#include "clang/Lex/TextEncoding.h"
#include "clang/Sema/CodeCompleteConsumer.h"
#include "clang/Sema/ParsedAttr.h"
#include "clang/Sema/Sema.h"
@@ -962,8 +961,7 @@ bool CompilerInstance::InitializeSourceManager(
// Use the UseInputCharsetConverter parameter passed from the caller
SourceMgr.setMainFileID(
SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind,
- /*Converter=*/nullptr, /*LoadedID=*/0,
- /*LoadedOffset=*/0,
+ /*LoadedID=*/0,
UseInputCharsetConverter));
assert(SourceMgr.getMainFileID().isValid() &&
diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
index cac28c2733092..c3f8b024b1ff9 100644
--- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
+++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
@@ -614,8 +614,7 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
// are handled by SourceManager's cache.
if (FID.isInvalid())
FID = SM.createFileID(*File, Pos, SrcMgr::C_User,
- /*Converter=*/nullptr, /*LoadedID=*/0,
- /*LoadedOffset=*/0,
+ /*LoadedID=*/0,
/*UseInputCharsetConverter=*/true);
if (PH.Next(Line) && Line > 0)
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index ecfca9fc7ee51..11c1f6e76bdb5 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -1476,8 +1476,7 @@ bool ModuleMap::parseModuleMapFile(FileEntryRef File, bool IsSystem,
// Module map files are textual "source files". Use input charset converter
// if available, and file tag converters are handled by SourceManager's cache.
LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter,
- /*Converter=*/nullptr, /*LoadedID=*/0,
- /*LoadedOffset=*/0,
+ /*LoadedID=*/0,
/*UseInputCharsetConverter=*/true);
}
ID = LocalFID;
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 0e7604d4ab324..8cee4d5db57d1 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -2800,8 +2800,7 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
// Use the SourceManager's input charset converter for non-tagged files
// by passing UseInputCharsetConverter=true
FileID FID = SourceMgr.createFileID(*File, IncludePos, FileCharacter,
- /*Converter=*/nullptr, /*LoadedID=*/0,
- /*LoadedOffset=*/0,
+ /*LoadedID=*/0,
/*UseInputCharsetConverter=*/true);
if (!FID.isValid()) {
TheModuleLoader.HadFatalFailure = true;
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index bac9dc2ef912d..6db4d704c88c7 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -652,8 +652,8 @@ void Preprocessor::EnterMainSourceFile() {
// Use input charset converter if available, and file tag converters
// are handled by SourceManager's cache.
setPCHThroughHeaderFileID(SourceMgr.createFileID(
- *File, SourceLocation(), SrcMgr::C_User, /*Converter=*/nullptr,
- /*LoadedID=*/0, /*LoadedOffset=*/0,
+ *File, SourceLocation(), SrcMgr::C_User,
+ /*LoadedID=*/0,
/*UseInputCharsetConverter=*/true));
}
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index 379622e92984b..e1287dda4ea55 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -2005,8 +2005,8 @@ bool ASTReader::ReadSLocEntry(int ID) {
// Note: If conversion was originally necessary, OverriddenBuffer should be
// true and the associated handling will trigger.
FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter,
- /*Converter=*/nullptr, ID,
- BaseOffset + Record[0]);
+ ID, /*UseInputCharsetConverter=*/false,
+ BaseOffset + Record[0]);
SrcMgr::FileInfo &FileInfo = SourceMgr.getSLocEntry(FID).getFile();
FileInfo.NumCreatedFIDs = Record[5];
if (Record[3])
>From a9f63f4a664678d4f61a54f6af29658a6e18daa8 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 11:14:10 -0400
Subject: [PATCH 10/29] make cache thread safe
---
clang/include/clang/Basic/SourceManager.h | 6 ++++++
clang/lib/Basic/SourceManager.cpp | 13 ++++++++++++-
2 files changed, 18 insertions(+), 1 deletion(-)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index dce778f623991..bbedaefee12c1 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -55,7 +55,9 @@
#include <cstddef>
#include <map>
#include <memory>
+#include <mutex>
#include <optional>
+#include <shared_mutex>
#include <string>
#include <utility>
#include <vector>
@@ -853,6 +855,10 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// This includes both the input charset converter and file tag converters.
/// Maps from "source_encoding:target_encoding" to the converter.
llvm::StringMap<std::unique_ptr<llvm::TextEncodingConverter>> ConverterCache;
+
+ /// Shared mutex to protect ConverterCache for thread-safe access.
+ /// Uses shared_mutex to allow multiple concurrent readers.
+ mutable std::shared_mutex ConverterCacheMutex;
public:
SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr,
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 344abcff46484..16b08027161cf 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -475,7 +475,18 @@ SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding,
CacheKey += ":";
CacheKey += TargetEncoding;
- // Check if converter already exists in cache
+ // First, try to find the converter with a shared (read) lock
+ {
+ std::shared_lock<std::shared_mutex> ReadLock(ConverterCacheMutex);
+ auto It = ConverterCache.find(CacheKey);
+ if (It != ConverterCache.end())
+ return It->second.get();
+ }
+
+ // Converter not found, acquire exclusive (write) lock to create it
+ std::unique_lock<std::shared_mutex> WriteLock(ConverterCacheMutex);
+
+ // Double-check that another thread didn't create it while we were waiting
auto It = ConverterCache.find(CacheKey);
if (It != ConverterCache.end())
return It->second.get();
>From c6c747e30d8276b4e2b135afafdb49ef26557c6d Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 11:16:45 -0400
Subject: [PATCH 11/29] change type
---
clang/include/clang/Basic/SourceManager.h | 9 ++++-----
clang/lib/Basic/SourceManager.cpp | 8 ++++----
2 files changed, 8 insertions(+), 9 deletions(-)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index bbedaefee12c1..fe0003cfc9d8a 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -50,14 +50,13 @@
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/MemoryBuffer.h"
+#include "llvm/Support/RWMutex.h"
#include "llvm/Support/TextEncoding.h"
#include <cassert>
#include <cstddef>
#include <map>
#include <memory>
-#include <mutex>
#include <optional>
-#include <shared_mutex>
#include <string>
#include <utility>
#include <vector>
@@ -856,9 +855,9 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// Maps from "source_encoding:target_encoding" to the converter.
llvm::StringMap<std::unique_ptr<llvm::TextEncodingConverter>> ConverterCache;
- /// Shared mutex to protect ConverterCache for thread-safe access.
- /// Uses shared_mutex to allow multiple concurrent readers.
- mutable std::shared_mutex ConverterCacheMutex;
+ /// Read-write mutex to protect ConverterCache for thread-safe access.
+ /// Allows multiple concurrent readers while ensuring exclusive write access.
+ mutable llvm::sys::RWMutex ConverterCacheMutex;
public:
SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr,
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 16b08027161cf..b861c3c19c7f7 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -475,16 +475,16 @@ SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding,
CacheKey += ":";
CacheKey += TargetEncoding;
- // First, try to find the converter with a shared (read) lock
+ // First, try to find the converter with a read lock
{
- std::shared_lock<std::shared_mutex> ReadLock(ConverterCacheMutex);
+ llvm::sys::ScopedReader ReadLock(ConverterCacheMutex);
auto It = ConverterCache.find(CacheKey);
if (It != ConverterCache.end())
return It->second.get();
}
- // Converter not found, acquire exclusive (write) lock to create it
- std::unique_lock<std::shared_mutex> WriteLock(ConverterCacheMutex);
+ // Converter not found, acquire write lock to create it
+ llvm::sys::ScopedWriter WriteLock(ConverterCacheMutex);
// Double-check that another thread didn't create it while we were waiting
auto It = ConverterCache.find(CacheKey);
>From 88a3d9a9d06195b2fda9e3e2de00f4b76bfa3a93 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 11:17:29 -0400
Subject: [PATCH 12/29] File mismatch checking
---
clang/lib/Basic/FileManager.cpp | 17 ++++++++++++-----
llvm/include/llvm/Support/VirtualFileSystem.h | 12 ++++++++++++
llvm/lib/Support/VirtualFileSystem.cpp | 13 ++++++++++---
3 files changed, 34 insertions(+), 8 deletions(-)
diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp
index 8fb3ba0a27aad..717a692661588 100644
--- a/clang/lib/Basic/FileManager.cpp
+++ b/clang/lib/Basic/FileManager.cpp
@@ -539,15 +539,22 @@ FileManager::getBufferForFile(FileEntryRef FE, bool isVolatile,
FileSize = -1;
StringRef Filename = FE.getName();
- // If the file is already open, use the open file descriptor.
+ // If the file is already open, check if the mode matches.
if (Entry->File) {
- auto Result = Entry->File->getBuffer(Filename, FileSize,
- RequiresNullTerminator, isVolatile);
+ // Check if the cached file's mode matches the requested mode
+ // Only perform mismatch recovery for real files
+ if (!Entry->File->realFileTextMismatch(IsText)) {
+ // Mode matches, use the cached file descriptor
+ auto Result = Entry->File->getBuffer(Filename, FileSize,
+ RequiresNullTerminator, isVolatile);
+ Entry->closeFile();
+ return Result;
+ }
+ // Mode mismatch - close the cached file and reopen with correct mode
Entry->closeFile();
- return Result;
}
- // Otherwise, open the file.
+ // Open the file with the requested mode.
return getBufferForFileImpl(Filename, FileSize, isVolatile,
RequiresNullTerminator, IsText);
}
diff --git a/llvm/include/llvm/Support/VirtualFileSystem.h b/llvm/include/llvm/Support/VirtualFileSystem.h
index d22c534228331..a3ef38fe552a7 100644
--- a/llvm/include/llvm/Support/VirtualFileSystem.h
+++ b/llvm/include/llvm/Support/VirtualFileSystem.h
@@ -137,6 +137,18 @@ class LLVM_ABI File {
/// Closes the file.
virtual std::error_code close() = 0;
+ /// Returns true if this file was opened in text mode (with potential
+ /// encoding conversions), false if opened in binary mode.
+ /// Default implementation returns true for backward compatibility.
+ virtual bool isText() const { return true; }
+
+ /// Returns true if this is a real file and the requested text mode differs
+ /// from the current mode. Always returns false for non-real files.
+ /// Default implementation returns false for non-real files.
+ virtual bool realFileTextMismatch(bool RequestedIsText) const {
+ return false;
+ }
+
// Get the same file with a different path.
static ErrorOr<std::unique_ptr<File>>
getWithPath(ErrorOr<std::unique_ptr<File>> Result, const Twine &P);
diff --git a/llvm/lib/Support/VirtualFileSystem.cpp b/llvm/lib/Support/VirtualFileSystem.cpp
index 42e8bb4f9958e..2def668e63cb3 100644
--- a/llvm/lib/Support/VirtualFileSystem.cpp
+++ b/llvm/lib/Support/VirtualFileSystem.cpp
@@ -194,11 +194,13 @@ class RealFile : public File {
file_t FD;
Status S;
std::string RealName;
+ bool IsTextMode;
- RealFile(file_t RawFD, StringRef NewName, StringRef NewRealPathName)
+ RealFile(file_t RawFD, StringRef NewName, StringRef NewRealPathName,
+ bool IsText)
: FD(RawFD), S(NewName, {}, {}, {}, {}, {},
llvm::sys::fs::file_type::status_error, {}),
- RealName(NewRealPathName.str()) {
+ RealName(NewRealPathName.str()), IsTextMode(IsText) {
assert(FD != kInvalidFile && "Invalid or inactive file descriptor");
}
@@ -213,6 +215,10 @@ class RealFile : public File {
bool IsVolatile) override;
std::error_code close() override;
void setPath(const Twine &Path) override;
+ bool isText() const override { return IsTextMode; }
+ bool realFileTextMismatch(bool RequestedIsText) const override {
+ return IsTextMode != RequestedIsText;
+ }
};
} // namespace
@@ -320,8 +326,9 @@ class RealFileSystem : public FileSystem {
adjustPath(Name, Storage), Flags, &RealName);
if (!FDOrErr)
return errorToErrorCode(FDOrErr.takeError());
+ bool IsText = (Flags & sys::fs::OF_Text) != sys::fs::OF_None;
return std::unique_ptr<File>(
- new RealFile(*FDOrErr, Name.str(), RealName.str()));
+ new RealFile(*FDOrErr, Name.str(), RealName.str(), IsText));
}
struct WorkingDirectory {
>From 1e1753f163404f3cb11daafd7cded4a82374afc3 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 12:08:27 -0400
Subject: [PATCH 13/29] change some things
---
clang/lib/Basic/SourceManager.cpp | 23 ++++++++++++++++---
.../lib/Frontend/VerifyDiagnosticConsumer.cpp | 2 +-
llvm/include/llvm/Support/TextEncoding.h | 6 +++++
llvm/lib/Support/TextEncoding.cpp | 2 +-
4 files changed, 28 insertions(+), 5 deletions(-)
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index b861c3c19c7f7..fc423416c66a7 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -469,11 +469,28 @@ ContentCache &SourceManager::createMemBufferContentCache(
llvm::ErrorOr<llvm::TextEncodingConverter *>
SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding,
llvm::StringRef TargetEncoding) {
- // Create a cache key from source and target encodings
+ // Use getKnownEncoding to get normalized encoding names
+ std::optional<llvm::TextEncoding> SourceKnown =
+ llvm::TextEncodingConverter::getKnownEncoding(SourceEncoding);
+ std::optional<llvm::TextEncoding> TargetKnown =
+ llvm::TextEncodingConverter::getKnownEncoding(TargetEncoding);
+
+ if (SourceKnown == TargetKnown)
+ return nullptr;
+
+ // Create a cache key - use enum values for known encodings, raw names otherwise
llvm::SmallString<64> CacheKey;
- CacheKey = SourceEncoding;
+ if (SourceKnown) {
+ CacheKey += llvm::Twine(static_cast<int>(*SourceKnown)).str();
+ } else {
+ CacheKey += SourceEncoding;
+ }
CacheKey += ":";
- CacheKey += TargetEncoding;
+ if (TargetKnown) {
+ CacheKey += llvm::Twine(static_cast<int>(*TargetKnown)).str();
+ } else {
+ CacheKey += TargetEncoding;
+ }
// First, try to find the converter with a read lock
{
diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
index c3f8b024b1ff9..851a2d4b1cdca 100644
--- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
+++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
@@ -614,7 +614,7 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
// are handled by SourceManager's cache.
if (FID.isInvalid())
FID = SM.createFileID(*File, Pos, SrcMgr::C_User,
- /*LoadedID=*/0,
+ /*LoadedID=*/0,
/*UseInputCharsetConverter=*/true);
if (PH.Next(Line) && Line > 0)
diff --git a/llvm/include/llvm/Support/TextEncoding.h b/llvm/include/llvm/Support/TextEncoding.h
index 6ca37ce6e7a4e..b9c851b8c8a0a 100644
--- a/llvm/include/llvm/Support/TextEncoding.h
+++ b/llvm/include/llvm/Support/TextEncoding.h
@@ -107,6 +107,12 @@ class TextEncodingConverter {
LLVM_ABI static ErrorOr<TextEncodingConverter> create(StringRef From,
StringRef To);
+ /// Maps the encoding name to enum constant if possible.
+ /// Uses normalized charset name matching.
+ /// \param[in] Name the character encoding name
+ /// \return the TextEncoding enum value if known, std::nullopt otherwise
+ LLVM_ABI static std::optional<TextEncoding> getKnownEncoding(StringRef Name);
+
TextEncodingConverter(const TextEncodingConverter &) = delete;
TextEncodingConverter &operator=(const TextEncodingConverter &) = delete;
diff --git a/llvm/lib/Support/TextEncoding.cpp b/llvm/lib/Support/TextEncoding.cpp
index 7ec3ed037dbc7..7bcca72d937d2 100644
--- a/llvm/lib/Support/TextEncoding.cpp
+++ b/llvm/lib/Support/TextEncoding.cpp
@@ -52,7 +52,7 @@ static void normalizeCharSetName(StringRef CSName,
}
// Maps the encoding name to enum constant if possible.
-static std::optional<TextEncoding> getKnownEncoding(StringRef Name) {
+std::optional<TextEncoding> TextEncodingConverter::getKnownEncoding(StringRef Name) {
SmallString<16> Normalized;
normalizeCharSetName(Name, Normalized);
if (Normalized.equals("utf8"))
>From 069bda3688e70ef1b61ed135e30eaa15c0def11e Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 12:27:36 -0400
Subject: [PATCH 14/29] bob
---
clang/include/clang/Basic/LangOptions.h | 3 +++
clang/include/clang/Basic/SourceManager.h | 18 ++++++++----------
clang/lib/Basic/SourceManager.cpp | 13 ++++++++++---
clang/lib/Frontend/CompilerInstance.cpp | 18 ++++++++++++++----
4 files changed, 35 insertions(+), 17 deletions(-)
diff --git a/clang/include/clang/Basic/LangOptions.h b/clang/include/clang/Basic/LangOptions.h
index 9af036156b1ad..d04974a88beb1 100644
--- a/clang/include/clang/Basic/LangOptions.h
+++ b/clang/include/clang/Basic/LangOptions.h
@@ -618,6 +618,9 @@ class LangOptions : public LangOptionsBase {
/// The allocation token mode.
std::optional<llvm::AllocTokenMode> AllocTokenMode;
+ /// Name of the input encoding to convert to the internal encoding.
+ std::string InputEncoding;
+
LangOptions();
/// Set language defaults for the given input language and
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index fe0003cfc9d8a..dbe6b35fd4ecc 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -847,8 +847,8 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// we can add a cc1-level option to do so.
SmallVector<std::pair<std::string, FullSourceLoc>, 2> StoredModuleBuildStack;
- /// Converter for -finput-charset conversion to UTF-8.
- std::unique_ptr<llvm::TextEncodingConverter> InputCharsetConverter;
+ /// Name of the input encoding for -finput-charset conversion.
+ std::string InputEncodingName;
/// Cache of all text encoding converters used by this SourceManager.
/// This includes both the input charset converter and file tag converters.
@@ -876,16 +876,14 @@ class SourceManager : public RefCountedBase<SourceManager> {
FileManager &getFileManager() const { return FileMgr; }
- /// Set the input charset converter for -finput-charset conversion.
- void setInputCharsetConverter(
- std::unique_ptr<llvm::TextEncodingConverter> Converter) {
- InputCharsetConverter = std::move(Converter);
+ /// Set the input encoding name for -finput-charset conversion.
+ void setInputEncodingName(llvm::StringRef Name) {
+ InputEncodingName = Name.str();
}
- /// Get the input charset converter for -finput-charset conversion.
- /// Returns nullptr if no converter is set.
- llvm::TextEncodingConverter *getInputCharsetConverter() const {
- return InputCharsetConverter.get();
+ /// Get the input encoding name for -finput-charset conversion.
+ llvm::StringRef getInputEncodingName() const {
+ return InputEncodingName;
}
/// Get or create a text encoding converter from the cache.
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index fc423416c66a7..b37bd96e95f3b 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -655,9 +655,16 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
return FileID();
}
} else if (UseInputCharsetConverter) {
- // No file tag but -finput-charset conversion is desired. Use the converter
- // from SourceManager.
- Converter = getInputCharsetConverter();
+ // No file tag but -finput-charset conversion is desired.
+ // Get the converter from the cache using the input encoding name.
+ if (!InputEncodingName.empty()) {
+ Converter = getOrCreateConverter(InputEncodingName, "UTF-8");
+ if (!Converter) {
+ Diag.Report(SourceLocation(), diag::err_cannot_open_file)
+ << SourceFile.getName() << Converter.getError().message();
+ return FileID();
+ }
+ }
}
#ifndef NDEBUG
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 442b9d2b644e2..0994da1263786 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -912,12 +912,22 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary,
// Initialization Utilities
bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){
- // Check if we have an input charset converter from the preprocessor
+ // Check if we have an input charset converter
bool UseInputCharsetConverter = false;
if (hasPreprocessor()) {
- llvm::TextEncodingConverter *Converter =
- getPreprocessor().getTextEncoding().getConverter(CA_FromInputEncoding);
- UseInputCharsetConverter = (Converter != nullptr);
+ const std::string &InputEncoding = getLangOpts().InputEncoding;
+ if (!InputEncoding.empty()) {
+ // Store the input encoding name in SourceManager
+ getSourceManager().setInputEncodingName(InputEncoding);
+
+ // Add the converter to SourceManager's cache
+ auto ConverterOrErr = getSourceManager().getOrCreateConverter(InputEncoding, "UTF-8");
+ if (ConverterOrErr) {
+ UseInputCharsetConverter = true;
+ }
+ // If converter creation failed, UseInputCharsetConverter stays false
+ // and the error will be reported when createFileID tries to use it
+ }
}
return InitializeSourceManager(Input, getDiagnostics(),
>From d8b3642c7a811b8e740317ebe00160b27e98612b Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 12:30:21 -0400
Subject: [PATCH 15/29] fix bob code
---
clang/lib/Basic/SourceManager.cpp | 9 +++++----
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index b37bd96e95f3b..8d9a84d77dd65 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -673,17 +673,18 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
// it) or the conversion (or lack thereof) should be the same as that used
// previously.
auto [CacheConverter, CacheUsedByFileID] = IR.FileIDConverterInfo;
+ llvm::TextEncodingConverter *ConverterPtr = Converter ? *Converter : nullptr;
if (CacheUsedByFileID)
- assert(CacheConverter == Converter);
+ assert(CacheConverter == ConverterPtr);
else
- assert(!Converter || IR.IsBufferInvalid || !IR.getBufferIfLoaded());
+ assert(!ConverterPtr || IR.IsBufferInvalid || !IR.getBufferIfLoaded());
#endif
- IR.FileIDConverterInfo.setPointerAndInt(Converter, true);
+ IR.FileIDConverterInfo.setPointerAndInt(Converter ? *Converter : nullptr, true);
// If this is a named pipe, immediately load the buffer to ensure subsequent
// calls to ContentCache::getSize() are accurate.
// Do the same if character-encoding conversion was requested.
- if (IR.ContentsEntry->isNamedPipe() || Converter)
+ if (IR.ContentsEntry->isNamedPipe() || (Converter && *Converter))
(void)IR.getBufferOrNone(Diag, getFileManager(), SourceLocation());
return createFileIDImpl(IR, SourceFile.getName(), IncludePos, FileCharacter,
>From fe092a518e80fc3eb03566973570b335b2e5849a Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 12:47:25 -0400
Subject: [PATCH 16/29] Missing bracket
---
clang/lib/Basic/SourceManager.cpp | 1 +
1 file changed, 1 insertion(+)
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 8d9a84d77dd65..444e0125afe67 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -465,6 +465,7 @@ ContentCache &SourceManager::createMemBufferContentCache(
MemBufferInfos.push_back(Entry);
Entry->setBuffer(std::move(Buffer));
return *Entry;
+}
llvm::ErrorOr<llvm::TextEncodingConverter *>
SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding,
>From 6220e33efb06a339745b7a1a1fb1ea8a9f9257c0 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 13:00:55 -0400
Subject: [PATCH 17/29] bob
---
clang/include/clang/Basic/SourceManager.h | 21 +++-------------
.../include/clang/Frontend/CompilerInstance.h | 2 +-
clang/lib/Basic/SourceManager.cpp | 16 ++++++------
clang/lib/Frontend/CompilerInstance.cpp | 25 +++++++------------
.../lib/Frontend/VerifyDiagnosticConsumer.cpp | 7 ++++--
clang/lib/Lex/ModuleMap.cpp | 4 ++-
clang/lib/Lex/PPDirectives.cpp | 5 ++--
clang/lib/Lex/Preprocessor.cpp | 4 ++-
clang/lib/Serialization/ASTReader.cpp | 4 +--
9 files changed, 37 insertions(+), 51 deletions(-)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index dbe6b35fd4ecc..54873ddca3bb4 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -847,9 +847,6 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// we can add a cc1-level option to do so.
SmallVector<std::pair<std::string, FullSourceLoc>, 2> StoredModuleBuildStack;
- /// Name of the input encoding for -finput-charset conversion.
- std::string InputEncodingName;
-
/// Cache of all text encoding converters used by this SourceManager.
/// This includes both the input charset converter and file tag converters.
/// Maps from "source_encoding:target_encoding" to the converter.
@@ -876,16 +873,6 @@ class SourceManager : public RefCountedBase<SourceManager> {
FileManager &getFileManager() const { return FileMgr; }
- /// Set the input encoding name for -finput-charset conversion.
- void setInputEncodingName(llvm::StringRef Name) {
- InputEncodingName = Name.str();
- }
-
- /// Get the input encoding name for -finput-charset conversion.
- llvm::StringRef getInputEncodingName() const {
- return InputEncodingName;
- }
-
/// Get or create a text encoding converter from the cache.
/// This method manages all converters (input charset and file tag converters)
/// in a single cache owned by SourceManager.
@@ -955,13 +942,13 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// Create a new FileID that represents the specified file
/// being \#included from the specified IncludePosition.
- /// \param Converter Optional converter to use. If nullptr and
- /// UseInputCharsetConverter is true, will use the SourceManager's
- /// input charset converter for non-tagged files.
+ /// \param InputEncodingName The input encoding name to use for conversion.
+ /// If not empty and the file has no tag, will look up the converter from
+ /// the cache using this encoding name.
FileID createFileID(FileEntryRef SourceFile, SourceLocation IncludePos,
SrcMgr::CharacteristicKind FileCharacter,
int LoadedID = 0,
- bool UseInputCharsetConverter = false,
+ llvm::StringRef InputEncodingName = "",
SourceLocation::UIntTy LoadedOffset = 0);
/// Create a new FileID that represents the specified memory buffer.
diff --git a/clang/include/clang/Frontend/CompilerInstance.h b/clang/include/clang/Frontend/CompilerInstance.h
index 347d5807c4fc6..34d4a81a4d1e5 100644
--- a/clang/include/clang/Frontend/CompilerInstance.h
+++ b/clang/include/clang/Frontend/CompilerInstance.h
@@ -867,7 +867,7 @@ class CompilerInstance : public ModuleLoader {
DiagnosticsEngine &Diags,
FileManager &FileMgr,
SourceManager &SourceMgr,
- bool UseInputCharsetConverter = false);
+ llvm::StringRef InputEncodingName = "");
/// @}
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 444e0125afe67..a37ef701be35d 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -634,7 +634,7 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
SourceLocation IncludePos,
SrcMgr::CharacteristicKind FileCharacter,
int LoadedID,
- bool UseInputCharsetConverter,
+ llvm::StringRef InputEncodingName,
SourceLocation::UIntTy LoadedOffset) {
SrcMgr::ContentCache &IR = getOrCreateContentCache(SourceFile,
isSystem(FileCharacter));
@@ -655,16 +655,14 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
<< SourceFile.getName() << Converter.getError().message();
return FileID();
}
- } else if (UseInputCharsetConverter) {
+ } else if (!InputEncodingName.empty()) {
// No file tag but -finput-charset conversion is desired.
// Get the converter from the cache using the input encoding name.
- if (!InputEncodingName.empty()) {
- Converter = getOrCreateConverter(InputEncodingName, "UTF-8");
- if (!Converter) {
- Diag.Report(SourceLocation(), diag::err_cannot_open_file)
- << SourceFile.getName() << Converter.getError().message();
- return FileID();
- }
+ Converter = getOrCreateConverter(InputEncodingName, "UTF-8");
+ if (!Converter) {
+ Diag.Report(SourceLocation(), diag::err_cannot_open_file)
+ << SourceFile.getName() << Converter.getError().message();
+ return FileID();
}
}
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index 0994da1263786..f5e50d59f2156 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -912,34 +912,27 @@ CompilerInstance::createOutputFileImpl(StringRef OutputPath, bool Binary,
// Initialization Utilities
bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){
- // Check if we have an input charset converter
- bool UseInputCharsetConverter = false;
+ // Get the input encoding and add converter to cache if specified
+ std::string InputEncoding;
if (hasPreprocessor()) {
- const std::string &InputEncoding = getLangOpts().InputEncoding;
+ InputEncoding = getLangOpts().InputEncoding;
if (!InputEncoding.empty()) {
- // Store the input encoding name in SourceManager
- getSourceManager().setInputEncodingName(InputEncoding);
-
// Add the converter to SourceManager's cache
- auto ConverterOrErr = getSourceManager().getOrCreateConverter(InputEncoding, "UTF-8");
- if (ConverterOrErr) {
- UseInputCharsetConverter = true;
- }
- // If converter creation failed, UseInputCharsetConverter stays false
- // and the error will be reported when createFileID tries to use it
+ (void)getSourceManager().getOrCreateConverter(InputEncoding, "UTF-8");
+ // If converter creation fails, the error will be reported when createFileID tries to use it
}
}
return InitializeSourceManager(Input, getDiagnostics(),
getFileManager(), getSourceManager(),
- UseInputCharsetConverter);
+ InputEncoding);
}
// static
bool CompilerInstance::InitializeSourceManager(
const FrontendInputFile &Input,
DiagnosticsEngine &Diags, FileManager &FileMgr, SourceManager &SourceMgr,
- bool UseInputCharsetConverter) {
+ llvm::StringRef InputEncodingName) {
SrcMgr::CharacteristicKind Kind =
Input.getKind().getFormat() == InputKind::ModuleMap
? Input.isSystem() ? SrcMgr::C_System_ModuleMap
@@ -968,11 +961,11 @@ bool CompilerInstance::InitializeSourceManager(
return false;
}
- // Use the UseInputCharsetConverter parameter passed from the caller
+ // Pass the input encoding name for charset conversion if specified
SourceMgr.setMainFileID(
SourceMgr.createFileID(*FileOrErr, SourceLocation(), Kind,
/*LoadedID=*/0,
- UseInputCharsetConverter));
+ InputEncodingName));
assert(SourceMgr.getMainFileID().isValid() &&
"Couldn't establish MainFileID!");
diff --git a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
index 851a2d4b1cdca..4343c6e293cf7 100644
--- a/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
+++ b/clang/lib/Frontend/VerifyDiagnosticConsumer.cpp
@@ -612,10 +612,13 @@ static bool ParseDirective(StringRef S, ExpectedData *ED, SourceManager &SM,
FileID FID = SM.translateFile(*File);
// Use input charset converter if available, and file tag converters
// are handled by SourceManager's cache.
- if (FID.isInvalid())
+ if (FID.isInvalid()) {
+ // Get input encoding from LangOptions for charset conversion
+ llvm::StringRef InputEncoding = PP->getLangOpts().InputEncoding;
FID = SM.createFileID(*File, Pos, SrcMgr::C_User,
/*LoadedID=*/0,
- /*UseInputCharsetConverter=*/true);
+ InputEncoding);
+ }
if (PH.Next(Line) && Line > 0)
ExpectedLoc = SM.translateLineCol(FID, Line, 1);
diff --git a/clang/lib/Lex/ModuleMap.cpp b/clang/lib/Lex/ModuleMap.cpp
index 11c1f6e76bdb5..a8a7a0e45cb3b 100644
--- a/clang/lib/Lex/ModuleMap.cpp
+++ b/clang/lib/Lex/ModuleMap.cpp
@@ -1475,9 +1475,11 @@ bool ModuleMap::parseModuleMapFile(FileEntryRef File, bool IsSystem,
IsSystem ? SrcMgr::C_System_ModuleMap : SrcMgr::C_User_ModuleMap;
// Module map files are textual "source files". Use input charset converter
// if available, and file tag converters are handled by SourceManager's cache.
+ // Get input encoding from LangOptions for charset conversion
+ llvm::StringRef InputEncoding = LangOpts.InputEncoding;
LocalFID = SourceMgr.createFileID(File, ExternModuleLoc, FileCharacter,
/*LoadedID=*/0,
- /*UseInputCharsetConverter=*/true);
+ InputEncoding);
}
ID = LocalFID;
}
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 8cee4d5db57d1..07a5c0456401a 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -2798,10 +2798,11 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport(
IncludePos = SourceMgr.getExpansionRange(IncludePos).getEnd();
// Use the SourceManager's input charset converter for non-tagged files
- // by passing UseInputCharsetConverter=true
+ // by passing the input encoding name
+ llvm::StringRef InputEncoding = getLangOpts().InputEncoding;
FileID FID = SourceMgr.createFileID(*File, IncludePos, FileCharacter,
/*LoadedID=*/0,
- /*UseInputCharsetConverter=*/true);
+ InputEncoding);
if (!FID.isValid()) {
TheModuleLoader.HadFatalFailure = true;
return ImportAction::Failure;
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 6db4d704c88c7..374fd460f343c 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -651,10 +651,12 @@ void Preprocessor::EnterMainSourceFile() {
}
// Use input charset converter if available, and file tag converters
// are handled by SourceManager's cache.
+ // Get input encoding from LangOptions for charset conversion
+ llvm::StringRef InputEncoding = LangOpts.InputEncoding;
setPCHThroughHeaderFileID(SourceMgr.createFileID(
*File, SourceLocation(), SrcMgr::C_User,
/*LoadedID=*/0,
- /*UseInputCharsetConverter=*/true));
+ InputEncoding));
}
// Skip tokens from the Predefines and if needed the main file.
diff --git a/clang/lib/Serialization/ASTReader.cpp b/clang/lib/Serialization/ASTReader.cpp
index e1287dda4ea55..e5bed260d86f6 100644
--- a/clang/lib/Serialization/ASTReader.cpp
+++ b/clang/lib/Serialization/ASTReader.cpp
@@ -2005,8 +2005,8 @@ bool ASTReader::ReadSLocEntry(int ID) {
// Note: If conversion was originally necessary, OverriddenBuffer should be
// true and the associated handling will trigger.
FileID FID = SourceMgr.createFileID(*File, IncludeLoc, FileCharacter,
- ID, /*UseInputCharsetConverter=*/false,
- BaseOffset + Record[0]);
+ ID, /*InputEncodingName=*/"",
+ BaseOffset + Record[0]);
SrcMgr::FileInfo &FileInfo = SourceMgr.getSLocEntry(FID).getFile();
FileInfo.NumCreatedFIDs = Record[5];
if (Record[3])
>From 96328fbb64789cb0422f01bb9ac3b3cc99f93876 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 13:11:42 -0400
Subject: [PATCH 18/29] update createFileID
---
clang/include/clang/Basic/SourceManager.h | 2 +-
clang/lib/Basic/FileManager.cpp | 2 ++
clang/lib/Basic/SourceManager.cpp | 26 +++++++++++++++++------
clang/lib/Frontend/CompilerInstance.cpp | 11 +++++++++-
4 files changed, 32 insertions(+), 9 deletions(-)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 54873ddca3bb4..c44271c4344f0 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -973,7 +973,7 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// new FileID for the \p SourceFile.
FileID getOrCreateFileID(FileEntryRef SourceFile,
SrcMgr::CharacteristicKind FileCharacter,
- bool UseInputCharsetConverter = false);
+ llvm::StringRef InputEncodingName = "");
/// Creates an expansion SLocEntry for the substitution of an argument into a
/// function-like macro's body. Returns the start of the expansion.
diff --git a/clang/lib/Basic/FileManager.cpp b/clang/lib/Basic/FileManager.cpp
index 717a692661588..7cd293923d606 100644
--- a/clang/lib/Basic/FileManager.cpp
+++ b/clang/lib/Basic/FileManager.cpp
@@ -555,6 +555,8 @@ FileManager::getBufferForFile(FileEntryRef FE, bool isVolatile,
}
// Open the file with the requested mode.
+ llvm::errs() << "DEBUG: Opening file " << Filename << " with IsText=" << IsText
+ << (IsText ? " (autoconversion enabled)" : " (autoconversion DISABLED)") << "\n";
return getBufferForFileImpl(Filename, FileSize, isVolatile,
RequiresNullTerminator, IsText);
}
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index a37ef701be35d..f444bca97eaa1 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -151,6 +151,7 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
// Convert source from the input charset to UTF-8 if necessary.
llvm::TextEncodingConverter *Converter = FileIDConverterInfo.getPointer();
if (Converter) {
+ llvm::errs() << "DEBUG: Converting file " << ContentsEntry->getName() << " using converter\n";
StringRef OriginalBuf = Buffer->getBuffer();
llvm::SmallString<0> UTF8Buf;
UTF8Buf.reserve(OriginalBuf.size() + 1);
@@ -171,13 +172,17 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
// TODO: Consider adjusting the message to omit the "interpreting as
// UTF-8" recovery description if the warning has been upgraded to an
// error.
+ llvm::errs() << "DEBUG: Conversion failed for " << ContentsEntry->getName()
+ << ": " << EC.message() << " - falling back to UTF-8\n";
Diag.Report(Loc, diag::warn_charset_conversion_failed)
<< ContentsEntry->getName() << EC.message();
} else {
+ llvm::errs() << "DEBUG: Successfully converted " << ContentsEntry->getName()
+ << " from " << OriginalBuf.size() << " to " << UTF8Buf.size() << " bytes\n";
// TODO: Reclaim memory if the buffer size exceeds the content.
auto NewBuf = std::make_unique<llvm::SmallVectorMemoryBuffer>(
std::move(UTF8Buf), Buffer->getBufferIdentifier());
- Buffer = std::move(NewBuf);
+ Buffer = std::move(NewBuf);
}
}
@@ -649,21 +654,29 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
}
if (!Ccsid->empty()) {
// File has a tag, use the converter from SourceManager's cache
+ llvm::errs() << "DEBUG: File " << SourceFile.getName() << " has tag encoding: " << *Ccsid << "\n";
Converter = getOrCreateConverter(*Ccsid, "UTF-8");
if (!Converter) {
+ llvm::errs() << "DEBUG: Failed to get converter for file tag: " << Converter.getError().message() << "\n";
Diag.Report(SourceLocation(), diag::err_cannot_open_file)
<< SourceFile.getName() << Converter.getError().message();
return FileID();
}
+ llvm::errs() << "DEBUG: Using file tag converter for " << SourceFile.getName() << "\n";
} else if (!InputEncodingName.empty()) {
// No file tag but -finput-charset conversion is desired.
// Get the converter from the cache using the input encoding name.
+ llvm::errs() << "DEBUG: File " << SourceFile.getName() << " has no tag, using input charset: " << InputEncodingName << "\n";
Converter = getOrCreateConverter(InputEncodingName, "UTF-8");
if (!Converter) {
+ llvm::errs() << "DEBUG: Failed to get input charset converter: " << Converter.getError().message() << "\n";
Diag.Report(SourceLocation(), diag::err_cannot_open_file)
<< SourceFile.getName() << Converter.getError().message();
return FileID();
}
+ llvm::errs() << "DEBUG: Using input charset converter for " << SourceFile.getName() << "\n";
+ } else {
+ llvm::errs() << "DEBUG: File " << SourceFile.getName() << " has no tag and no input charset specified - no conversion\n";
}
#ifndef NDEBUG
@@ -722,13 +735,13 @@ FileID SourceManager::createFileID(const llvm::MemoryBufferRef &Buffer,
FileID
SourceManager::getOrCreateFileID(FileEntryRef SourceFile,
SrcMgr::CharacteristicKind FileCharacter,
- bool UseInputCharsetConverter) {
+ llvm::StringRef InputEncodingName) {
FileID ID = translateFile(SourceFile);
- return ID.isValid() ? ID
+ return ID.isValid()
+ ? ID
: createFileID(SourceFile, SourceLocation(),
- FileCharacter, UseInputCharsetConverter);
+ FileCharacter, /*LoadedID=*/0, InputEncodingName);
}
-
/// createFileID - Create a new FileID for the specified ContentCache and
/// include position. This works regardless of whether the ContentCache
/// corresponds to a file or some other input source.
@@ -2480,8 +2493,7 @@ SourceManagerForFile::SourceManagerForFile(StringRef FileName,
std::make_unique<DiagnosticsEngine>(DiagnosticIDs::create(), *DiagOpts);
SourceMgr = std::make_unique<SourceManager>(*Diagnostics, *FileMgr);
FileEntryRef FE = llvm::cantFail(FileMgr->getFileRef(FileName));
- FileID ID = SourceMgr->createFileID(
- FE, SourceLocation(), clang::SrcMgr::C_User, /*UseInputCharsetConverter=*/false);
+ FileID ID = SourceMgr->getOrCreateFileID(FE, clang::SrcMgr::C_User);
assert(ID.isValid());
SourceMgr->setMainFileID(ID);
}
diff --git a/clang/lib/Frontend/CompilerInstance.cpp b/clang/lib/Frontend/CompilerInstance.cpp
index f5e50d59f2156..e4902797dcbb9 100644
--- a/clang/lib/Frontend/CompilerInstance.cpp
+++ b/clang/lib/Frontend/CompilerInstance.cpp
@@ -917,9 +917,18 @@ bool CompilerInstance::InitializeSourceManager(const FrontendInputFile &Input){
if (hasPreprocessor()) {
InputEncoding = getLangOpts().InputEncoding;
if (!InputEncoding.empty()) {
+ llvm::errs() << "DEBUG: Setting input charset converter: " << InputEncoding << " -> UTF-8\n";
// Add the converter to SourceManager's cache
- (void)getSourceManager().getOrCreateConverter(InputEncoding, "UTF-8");
+ auto ConverterOrErr = getSourceManager().getOrCreateConverter(InputEncoding, "UTF-8");
+ if (ConverterOrErr) {
+ llvm::errs() << "DEBUG: Input charset converter successfully added to cache\n";
+ } else {
+ llvm::errs() << "DEBUG: Failed to create input charset converter: "
+ << ConverterOrErr.getError().message() << "\n";
+ }
// If converter creation fails, the error will be reported when createFileID tries to use it
+ } else {
+ llvm::errs() << "DEBUG: No input charset converter specified\n";
}
}
>From 54500b2fe0a8f460767a28dae373ad06bcb8d314 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 13:22:20 -0400
Subject: [PATCH 19/29] remove Lex/TextEncoding
---
clang/include/clang/Lex/Preprocessor.h | 4 ----
clang/include/clang/Lex/TextEncoding.h | 25 -------------------------
clang/lib/Lex/CMakeLists.txt | 1 -
clang/lib/Lex/TextEncoding.cpp | 20 --------------------
4 files changed, 50 deletions(-)
delete mode 100644 clang/include/clang/Lex/TextEncoding.h
delete mode 100644 clang/lib/Lex/TextEncoding.cpp
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 28a14eb76f8ef..8b684e85eb1c1 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -30,7 +30,6 @@
#include "clang/Lex/ModuleMap.h"
#include "clang/Lex/PPCallbacks.h"
#include "clang/Lex/PPEmbedParameters.h"
-#include "clang/Lex/TextEncoding.h"
#include "clang/Lex/Token.h"
#include "clang/Lex/TokenLexer.h"
#include "clang/Support/Compiler.h"
@@ -199,7 +198,6 @@ class Preprocessor {
std::unique_ptr<ScratchBuffer> ScratchBuf;
HeaderSearch &HeaderInfo;
ModuleLoader &TheModuleLoader;
- TextEncoding TE;
/// External source of macros.
ExternalPreprocessorSource *ExternalSource;
@@ -1267,8 +1265,6 @@ class Preprocessor {
Builtin::Context &getBuiltinInfo() { return *BuiltinInfo; }
llvm::BumpPtrAllocator &getPreprocessorAllocator() { return BP; }
- TextEncoding &getTextEncoding() { return TE; }
-
void setExternalSource(ExternalPreprocessorSource *Source) {
ExternalSource = Source;
}
diff --git a/clang/include/clang/Lex/TextEncoding.h b/clang/include/clang/Lex/TextEncoding.h
deleted file mode 100644
index 3e7653580e994..0000000000000
--- a/clang/include/clang/Lex/TextEncoding.h
+++ /dev/null
@@ -1,25 +0,0 @@
-//===-- clang/Lex/TextEncoding.h - Text Conversion Config -*- C++ -*-===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#ifndef LLVM_CLANG_LEX_TEXTENCODING_H
-#define LLVM_CLANG_LEX_TEXTENCODING_H
-
-#include "clang/Basic/LangOptions.h"
-#include "llvm/ADT/StringRef.h"
-#include "llvm/Support/TextEncoding.h"
-
-enum ConversionAction { CA_NoConversion, CA_FromInputEncoding };
-
-class TextEncoding {
-std::unique_ptr<llvm::TextEncodingConverter> FromInputEncodingConverter;
-
-public:
- llvm::TextEncodingConverter *getConverter(ConversionAction Action) const;
-};
-
-#endif
diff --git a/clang/lib/Lex/CMakeLists.txt b/clang/lib/Lex/CMakeLists.txt
index 7b0be7249cd99..f61737cd68021 100644
--- a/clang/lib/Lex/CMakeLists.txt
+++ b/clang/lib/Lex/CMakeLists.txt
@@ -29,7 +29,6 @@ add_clang_library(clangLex
Preprocessor.cpp
PreprocessorLexer.cpp
ScratchBuffer.cpp
- TextEncoding.cpp
TokenConcatenation.cpp
TokenLexer.cpp
diff --git a/clang/lib/Lex/TextEncoding.cpp b/clang/lib/Lex/TextEncoding.cpp
deleted file mode 100644
index 7a0fea3798277..0000000000000
--- a/clang/lib/Lex/TextEncoding.cpp
+++ /dev/null
@@ -1,20 +0,0 @@
-//===--- TextEncoding.cpp -------------------------------------------------===//
-//
-// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
-// See https://llvm.org/LICENSE.txt for license information.
-// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Lex/TextEncoding.h"
-#include "clang/Basic/DiagnosticDriver.h"
-
-llvm::TextEncodingConverter *
-TextEncoding::getConverter(ConversionAction Action) const {
- switch (Action) {
- case CA_FromInputEncoding:
- return FromInputEncodingConverter.get();
- default:
- return nullptr;
- }
-}
>From 404e6f889933c5399a30576ede8492d0b3f9eae9 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 13:58:12 -0400
Subject: [PATCH 20/29] Disable autoconversion if converter exists
---
clang/lib/Basic/SourceManager.cpp | 13 +++++++++++--
1 file changed, 11 insertions(+), 2 deletions(-)
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index f444bca97eaa1..475e542f60b91 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -120,8 +120,17 @@ ContentCache::getBufferOrNone(DiagnosticsEngine &Diag, FileManager &FM,
// Start with the assumption that the buffer is invalid to simplify early
// return paths.
IsBufferInvalid = true;
-
- auto BufferOrError = FM.getBufferForFile(*ContentsEntry, IsFileVolatile);
+
+ // If a converter is set, open the file in binary mode to get raw bytes
+ // and avoid platform-specific auto-conversion (e.g., EBCDIC->ASCII on z/OS,
+ // CRLF->LF on Windows). The explicit converter will handle all transformations.
+ bool NeedsExplicitConversion = FileIDConverterInfo.getPointer() != nullptr;
+ bool IsText = !NeedsExplicitConversion;
+
+ auto BufferOrError = FM.getBufferForFile(*ContentsEntry, IsFileVolatile,
+ /*RequiresNullTerminator=*/true,
+ /*MaybeLimit=*/std::nullopt,
+ IsText);
// If we were unable to open the file, then we are in an inconsistent
// situation where the content cache referenced a file which no longer
>From 910366bcd5530ac3ade98b37d5c7b13eed85ecb3 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 14:11:52 -0400
Subject: [PATCH 21/29] Normalize filetags
---
llvm/lib/Support/TextEncoding.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Support/TextEncoding.cpp b/llvm/lib/Support/TextEncoding.cpp
index 7bcca72d937d2..448ecb84ca48b 100644
--- a/llvm/lib/Support/TextEncoding.cpp
+++ b/llvm/lib/Support/TextEncoding.cpp
@@ -55,9 +55,9 @@ static void normalizeCharSetName(StringRef CSName,
std::optional<TextEncoding> TextEncodingConverter::getKnownEncoding(StringRef Name) {
SmallString<16> Normalized;
normalizeCharSetName(Name, Normalized);
- if (Normalized.equals("utf8"))
+ if (Normalized.equals("utf8") || Normalized.equals("1208"))
return TextEncoding::UTF8;
- if (Normalized.equals("ibm1047"))
+ if (Normalized.equals("ibm1047") || Normalized.equals("1047"))
return TextEncoding::IBM1047;
return std::nullopt;
}
>From 3f752dd9f7047bfcbb8a90b4d4fa649a692c8296 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 15:10:50 -0400
Subject: [PATCH 22/29] Remove mutex from ConverterCache
---
clang/include/clang/Basic/SourceManager.h | 5 -----
clang/lib/Basic/SourceManager.cpp | 13 +------------
2 files changed, 1 insertion(+), 17 deletions(-)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index c44271c4344f0..162a74b5191c4 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -50,7 +50,6 @@
#include "llvm/Support/Allocator.h"
#include "llvm/Support/Compiler.h"
#include "llvm/Support/MemoryBuffer.h"
-#include "llvm/Support/RWMutex.h"
#include "llvm/Support/TextEncoding.h"
#include <cassert>
#include <cstddef>
@@ -851,10 +850,6 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// This includes both the input charset converter and file tag converters.
/// Maps from "source_encoding:target_encoding" to the converter.
llvm::StringMap<std::unique_ptr<llvm::TextEncodingConverter>> ConverterCache;
-
- /// Read-write mutex to protect ConverterCache for thread-safe access.
- /// Allows multiple concurrent readers while ensuring exclusive write access.
- mutable llvm::sys::RWMutex ConverterCacheMutex;
public:
SourceManager(DiagnosticsEngine &Diag, FileManager &FileMgr,
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 475e542f60b91..f8333ee621be6 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -507,18 +507,7 @@ SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding,
CacheKey += TargetEncoding;
}
- // First, try to find the converter with a read lock
- {
- llvm::sys::ScopedReader ReadLock(ConverterCacheMutex);
- auto It = ConverterCache.find(CacheKey);
- if (It != ConverterCache.end())
- return It->second.get();
- }
-
- // Converter not found, acquire write lock to create it
- llvm::sys::ScopedWriter WriteLock(ConverterCacheMutex);
-
- // Double-check that another thread didn't create it while we were waiting
+ // Check if converter already exists in cache
auto It = ConverterCache.find(CacheKey);
if (It != ConverterCache.end())
return It->second.get();
>From 7e0c113d3604b25d740ce9c543101d736ac767f1 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 15:14:33 -0400
Subject: [PATCH 23/29] Remove SourceEncoding as argument from
getOrCreateConverter
---
clang/include/clang/Basic/SourceManager.h | 5 ++---
clang/lib/Basic/SourceManager.cpp | 10 ++++++----
2 files changed, 8 insertions(+), 7 deletions(-)
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 162a74b5191c4..2c45daf782f4c 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -872,11 +872,10 @@ class SourceManager : public RefCountedBase<SourceManager> {
/// This method manages all converters (input charset and file tag converters)
/// in a single cache owned by SourceManager.
/// \param SourceEncoding the source character encoding name
- /// \param TargetEncoding the target character encoding name
/// \return pointer to the converter or an error code
+ /// The target encoding is always UTF-8.
llvm::ErrorOr<llvm::TextEncodingConverter *>
- getOrCreateConverter(llvm::StringRef SourceEncoding,
- llvm::StringRef TargetEncoding);
+ getOrCreateConverter(llvm::StringRef SourceEncoding);
/// Set true if the SourceManager should report the original file name
/// for contents of files that were overridden by other files. Defaults to
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index f8333ee621be6..bd04a740f1190 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -482,8 +482,10 @@ ContentCache &SourceManager::createMemBufferContentCache(
}
llvm::ErrorOr<llvm::TextEncodingConverter *>
-SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding,
- llvm::StringRef TargetEncoding) {
+SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) {
+ // Target encoding is always UTF-8
+ llvm::StringRef TargetEncoding = "UTF-8";
+
// Use getKnownEncoding to get normalized encoding names
std::optional<llvm::TextEncoding> SourceKnown =
llvm::TextEncodingConverter::getKnownEncoding(SourceEncoding);
@@ -653,7 +655,7 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
if (!Ccsid->empty()) {
// File has a tag, use the converter from SourceManager's cache
llvm::errs() << "DEBUG: File " << SourceFile.getName() << " has tag encoding: " << *Ccsid << "\n";
- Converter = getOrCreateConverter(*Ccsid, "UTF-8");
+ Converter = getOrCreateConverter(*Ccsid);
if (!Converter) {
llvm::errs() << "DEBUG: Failed to get converter for file tag: " << Converter.getError().message() << "\n";
Diag.Report(SourceLocation(), diag::err_cannot_open_file)
@@ -665,7 +667,7 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
// No file tag but -finput-charset conversion is desired.
// Get the converter from the cache using the input encoding name.
llvm::errs() << "DEBUG: File " << SourceFile.getName() << " has no tag, using input charset: " << InputEncodingName << "\n";
- Converter = getOrCreateConverter(InputEncodingName, "UTF-8");
+ Converter = getOrCreateConverter(InputEncodingName);
if (!Converter) {
llvm::errs() << "DEBUG: Failed to get input charset converter: " << Converter.getError().message() << "\n";
Diag.Report(SourceLocation(), diag::err_cannot_open_file)
>From 379a9064c93df54020d5c88498022d601e85cb4c Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 15:16:30 -0400
Subject: [PATCH 24/29] Only use SourceEncoding as key
---
clang/lib/Basic/SourceManager.cpp | 8 +-------
1 file changed, 1 insertion(+), 7 deletions(-)
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index bd04a740f1190..e8fe8955fdf2e 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -495,19 +495,13 @@ SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) {
if (SourceKnown == TargetKnown)
return nullptr;
- // Create a cache key - use enum values for known encodings, raw names otherwise
+ // Create a cache key using only source encoding
llvm::SmallString<64> CacheKey;
if (SourceKnown) {
CacheKey += llvm::Twine(static_cast<int>(*SourceKnown)).str();
} else {
CacheKey += SourceEncoding;
}
- CacheKey += ":";
- if (TargetKnown) {
- CacheKey += llvm::Twine(static_cast<int>(*TargetKnown)).str();
- } else {
- CacheKey += TargetEncoding;
- }
// Check if converter already exists in cache
auto It = ConverterCache.find(CacheKey);
>From f97ec41e53acf548869e166c8815f066585f0160 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 15:25:58 -0400
Subject: [PATCH 25/29] Tag names
---
llvm/include/llvm/Support/AutoConvert.h | 6 ++++++
1 file changed, 6 insertions(+)
diff --git a/llvm/include/llvm/Support/AutoConvert.h b/llvm/include/llvm/Support/AutoConvert.h
index 337befec1b352..37b56cdc7e185 100644
--- a/llvm/include/llvm/Support/AutoConvert.h
+++ b/llvm/include/llvm/Support/AutoConvert.h
@@ -116,6 +116,12 @@ getEncodingNameFromFileTag(const Twine &FileName, const int FD = -1) {
if (Tag == 0)
return std::string(); // Return empty string for no tag
+ if (Tag == 1208)
+ return std::string("utf-8");
+
+ if (Tag == 1047)
+ return std::string("ibm-1047");
+
char Buffer[16];
snprintf(Buffer, sizeof(Buffer), "%03d", Tag);
return std::string(Buffer);
>From a8313a1a4f1d219ba1fbf4f33e51820dc31aff5a Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 15:31:38 -0400
Subject: [PATCH 26/29] Update getOrCreateConverter
---
clang/lib/Basic/SourceManager.cpp | 4 +---
1 file changed, 1 insertion(+), 3 deletions(-)
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index e8fe8955fdf2e..750e0301d12c1 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -489,10 +489,8 @@ SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) {
// Use getKnownEncoding to get normalized encoding names
std::optional<llvm::TextEncoding> SourceKnown =
llvm::TextEncodingConverter::getKnownEncoding(SourceEncoding);
- std::optional<llvm::TextEncoding> TargetKnown =
- llvm::TextEncodingConverter::getKnownEncoding(TargetEncoding);
- if (SourceKnown == TargetKnown)
+ if (SourceKnown && *SourceKnown == llvm::TextEncoding::UTF8)
return nullptr;
// Create a cache key using only source encoding
>From b1bda2cb337730f8d50a56768bc95774ef34fd0e Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 15:34:12 -0400
Subject: [PATCH 27/29] remove targetencoding
---
clang/lib/Basic/SourceManager.cpp | 5 +----
1 file changed, 1 insertion(+), 4 deletions(-)
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 750e0301d12c1..5a242a7573273 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -483,9 +483,6 @@ ContentCache &SourceManager::createMemBufferContentCache(
llvm::ErrorOr<llvm::TextEncodingConverter *>
SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) {
- // Target encoding is always UTF-8
- llvm::StringRef TargetEncoding = "UTF-8";
-
// Use getKnownEncoding to get normalized encoding names
std::optional<llvm::TextEncoding> SourceKnown =
llvm::TextEncodingConverter::getKnownEncoding(SourceEncoding);
@@ -508,7 +505,7 @@ SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) {
// Create a new converter
llvm::ErrorOr<llvm::TextEncodingConverter> NewConverter =
- llvm::TextEncodingConverter::create(SourceEncoding, TargetEncoding);
+ llvm::TextEncodingConverter::create(SourceEncoding, "UTF-8");
if (!NewConverter)
return NewConverter.getError();
>From 7a838cac054efb5d9e7b68280b74f79e1e8d65c6 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 15:46:18 -0400
Subject: [PATCH 28/29] Use canonical encoding names
---
clang/lib/Basic/SourceManager.cpp | 11 ++++-------
llvm/include/llvm/Support/TextEncoding.h | 5 +++++
llvm/lib/Support/TextEncoding.cpp | 11 +++++++++++
3 files changed, 20 insertions(+), 7 deletions(-)
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index 5a242a7573273..da686f354e510 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -490,13 +490,10 @@ SourceManager::getOrCreateConverter(llvm::StringRef SourceEncoding) {
if (SourceKnown && *SourceKnown == llvm::TextEncoding::UTF8)
return nullptr;
- // Create a cache key using only source encoding
- llvm::SmallString<64> CacheKey;
- if (SourceKnown) {
- CacheKey += llvm::Twine(static_cast<int>(*SourceKnown)).str();
- } else {
- CacheKey += SourceEncoding;
- }
+ // Create a cache key using canonical encoding name
+ llvm::StringRef CacheKey = SourceKnown
+ ? llvm::TextEncodingConverter::getKnownEncodingName(*SourceKnown)
+ : SourceEncoding;
// Check if converter already exists in cache
auto It = ConverterCache.find(CacheKey);
diff --git a/llvm/include/llvm/Support/TextEncoding.h b/llvm/include/llvm/Support/TextEncoding.h
index b9c851b8c8a0a..e934654afce8f 100644
--- a/llvm/include/llvm/Support/TextEncoding.h
+++ b/llvm/include/llvm/Support/TextEncoding.h
@@ -113,6 +113,11 @@ class TextEncodingConverter {
/// \return the TextEncoding enum value if known, std::nullopt otherwise
LLVM_ABI static std::optional<TextEncoding> getKnownEncoding(StringRef Name);
+ /// Returns the canonical name for a known encoding.
+ /// \param[in] Encoding the TextEncoding enum value
+ /// \return the canonical name for the encoding (e.g., "UTF-8" or "IBM-1047")
+ LLVM_ABI static StringRef getKnownEncodingName(TextEncoding Encoding);
+
TextEncodingConverter(const TextEncodingConverter &) = delete;
TextEncodingConverter &operator=(const TextEncodingConverter &) = delete;
diff --git a/llvm/lib/Support/TextEncoding.cpp b/llvm/lib/Support/TextEncoding.cpp
index 448ecb84ca48b..34c04f56b740b 100644
--- a/llvm/lib/Support/TextEncoding.cpp
+++ b/llvm/lib/Support/TextEncoding.cpp
@@ -62,6 +62,17 @@ std::optional<TextEncoding> TextEncodingConverter::getKnownEncoding(StringRef Na
return std::nullopt;
}
+// Returns the canonical name for a known encoding.
+StringRef TextEncodingConverter::getKnownEncodingName(TextEncoding Encoding) {
+ switch (Encoding) {
+ case TextEncoding::UTF8:
+ return "UTF-8";
+ case TextEncoding::IBM1047:
+ return "IBM-1047";
+ }
+ llvm_unreachable("Invalid TextEncoding value");
+}
+
[[maybe_unused]] static void HandleOverflow(size_t &Capacity, char *&Output,
size_t &OutputLength,
SmallVectorImpl<char> &Result) {
>From 3443285b192baf38f164a4c8aaca17b268d71206 Mon Sep 17 00:00:00 2001
From: alisonzhang <alisonzhang at ibm.com>
Date: Mon, 22 Jun 2026 15:53:52 -0400
Subject: [PATCH 29/29] Remove debug
---
clang/lib/Basic/SourceManager.cpp | 8 --------
1 file changed, 8 deletions(-)
diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp
index da686f354e510..d3b80726dd593 100644
--- a/clang/lib/Basic/SourceManager.cpp
+++ b/clang/lib/Basic/SourceManager.cpp
@@ -640,29 +640,21 @@ FileID SourceManager::createFileID(FileEntryRef SourceFile,
}
if (!Ccsid->empty()) {
// File has a tag, use the converter from SourceManager's cache
- llvm::errs() << "DEBUG: File " << SourceFile.getName() << " has tag encoding: " << *Ccsid << "\n";
Converter = getOrCreateConverter(*Ccsid);
if (!Converter) {
- llvm::errs() << "DEBUG: Failed to get converter for file tag: " << Converter.getError().message() << "\n";
Diag.Report(SourceLocation(), diag::err_cannot_open_file)
<< SourceFile.getName() << Converter.getError().message();
return FileID();
}
- llvm::errs() << "DEBUG: Using file tag converter for " << SourceFile.getName() << "\n";
} else if (!InputEncodingName.empty()) {
// No file tag but -finput-charset conversion is desired.
// Get the converter from the cache using the input encoding name.
- llvm::errs() << "DEBUG: File " << SourceFile.getName() << " has no tag, using input charset: " << InputEncodingName << "\n";
Converter = getOrCreateConverter(InputEncodingName);
if (!Converter) {
- llvm::errs() << "DEBUG: Failed to get input charset converter: " << Converter.getError().message() << "\n";
Diag.Report(SourceLocation(), diag::err_cannot_open_file)
<< SourceFile.getName() << Converter.getError().message();
return FileID();
}
- llvm::errs() << "DEBUG: Using input charset converter for " << SourceFile.getName() << "\n";
- } else {
- llvm::errs() << "DEBUG: File " << SourceFile.getName() << " has no tag and no input charset specified - no conversion\n";
}
#ifndef NDEBUG
More information about the cfe-commits
mailing list