[llvm] 6b3956e - [llvm-reduce] Add support for LTO bitcode files
Matthew Voss via llvm-commits
llvm-commits at lists.llvm.org
Thu Jun 30 08:58:33 PDT 2022
Author: Matthew Voss
Date: 2022-06-30T08:58:24-07:00
New Revision: 6b3956e123db1a53e38f5506aa164be4d644a477
URL: https://github.com/llvm/llvm-project/commit/6b3956e123db1a53e38f5506aa164be4d644a477
DIFF: https://github.com/llvm/llvm-project/commit/6b3956e123db1a53e38f5506aa164be4d644a477.diff
LOG: [llvm-reduce] Add support for LTO bitcode files
Adds support for reading and writing LTO bitcode files.
- Emit a summary if the original bitcode file had a summary
- Use split LTO units if the original bitcode file used them.
Reviewed By: arsenm
Differential Revision: https://reviews.llvm.org/D127168
Added:
llvm/test/tools/llvm-reduce/invalid-bitcode-error.ll
llvm/test/tools/llvm-reduce/temporary-files-as-bitcode-split.ll
Modified:
llvm/test/tools/llvm-reduce/Inputs/llvm-dis-and-filecheck.py
llvm/test/tools/llvm-reduce/fail-file-open.test
llvm/tools/llvm-reduce/CMakeLists.txt
llvm/tools/llvm-reduce/ReducerWorkItem.cpp
llvm/tools/llvm-reduce/ReducerWorkItem.h
llvm/tools/llvm-reduce/TestRunner.cpp
llvm/tools/llvm-reduce/TestRunner.h
llvm/tools/llvm-reduce/deltas/Delta.cpp
llvm/tools/llvm-reduce/llvm-reduce.cpp
Removed:
################################################################################
diff --git a/llvm/test/tools/llvm-reduce/Inputs/llvm-dis-and-filecheck.py b/llvm/test/tools/llvm-reduce/Inputs/llvm-dis-and-filecheck.py
index 9fa1363462327..672a932515b50 100755
--- a/llvm/test/tools/llvm-reduce/Inputs/llvm-dis-and-filecheck.py
+++ b/llvm/test/tools/llvm-reduce/Inputs/llvm-dis-and-filecheck.py
@@ -13,6 +13,7 @@
import sys
+import os
import subprocess
llvm_dis = sys.argv[1]
@@ -20,10 +21,23 @@
filecheck_args = [filecheck, ]
filecheck_args.extend(sys.argv[3:-1])
bitcode_file = sys.argv[-1]
+ir_file = bitcode_file + ".ll"
-disassemble = subprocess.Popen([llvm_dis, "-o", "-", bitcode_file],
- stdout=subprocess.PIPE)
-check = subprocess.Popen(filecheck_args, stdin=disassemble.stdout)
-disassemble.stdout.close()
+disassemble = subprocess.Popen([llvm_dis, "-o", ir_file, bitcode_file])
+if os.path.exists(ir_file + ".0"):
+ ir_file = ir_file + ".0"
+
+disassemble.communicate()
+
+if disassemble.returncode != 0:
+ print("stderr:")
+ print(disassemble.stderr)
+ print("stdout:")
+ print(disassemble.stdout)
+ sys.exit(1)
+
+check=None
+with open(ir_file, "r") as ir:
+ check = subprocess.Popen(filecheck_args, stdin=ir, stdout=sys.stdout)
check.communicate()
sys.exit(check.returncode)
diff --git a/llvm/test/tools/llvm-reduce/fail-file-open.test b/llvm/test/tools/llvm-reduce/fail-file-open.test
index 20fe5e2022f09..7c649e1d39f81 100644
--- a/llvm/test/tools/llvm-reduce/fail-file-open.test
+++ b/llvm/test/tools/llvm-reduce/fail-file-open.test
@@ -2,4 +2,4 @@
This file will not be read. An invalid file path is fed to llvm-reduce.
-# CHECK: llvm-reduce{{.*}}: {{.*}}.NotAFileInTestingDir: error:
\ No newline at end of file
+# CHECK: llvm-reduce{{.*}}: error: {{.*}}.NotAFileInTestingDir:
diff --git a/llvm/test/tools/llvm-reduce/invalid-bitcode-error.ll b/llvm/test/tools/llvm-reduce/invalid-bitcode-error.ll
new file mode 100644
index 0000000000000..94276fdedec4c
--- /dev/null
+++ b/llvm/test/tools/llvm-reduce/invalid-bitcode-error.ll
@@ -0,0 +1,3 @@
+; RUN: not %python %p/Inputs/llvm-dis-and-filecheck.py llvm-dis FileCheck %s %s 2>&1 | FileCheck %s
+; CHECK: stderr
+; CHECK: stdout
diff --git a/llvm/test/tools/llvm-reduce/temporary-files-as-bitcode-split.ll b/llvm/test/tools/llvm-reduce/temporary-files-as-bitcode-split.ll
new file mode 100644
index 0000000000000..98cbc56739183
--- /dev/null
+++ b/llvm/test/tools/llvm-reduce/temporary-files-as-bitcode-split.ll
@@ -0,0 +1,27 @@
+; RUN: opt --thinlto-bc --thinlto-split-lto-unit %s -o %t0
+; RUN: llvm-reduce -write-tmp-files-as-bitcode --delta-passes=basic-blocks %t0 -o %t1 \
+; RUN: --test %python --test-arg %p/Inputs/llvm-dis-and-filecheck.py --test-arg llvm-dis --test-arg FileCheck --test-arg --check-prefixes=CHECK-ALL,CHECK-INTERESTINGNESS --test-arg %s
+; RUN: cat %t1* | FileCheck --check-prefixes=CHECK-ALL,CHECK-FINAL %s
+
+ at g = internal global i8 42, !type !0
+
+; CHECK-INTERESTINGNESS: @callee(
+; CHECK-FINAL: declare void @callee()
+define void @callee() {
+ ret void
+}
+
+; CHECK-ALL: define void @caller()
+define void @caller() {
+entry:
+; CHECK-ALL: call void @callee()
+; CHECK-ALL: ret void
+ call void @callee()
+ ret void
+}
+define i8* @f() {
+ ; CHECK-ALL: ret i8* @g.{{([0-9a-f]{32})}}
+ ret i8* @g
+}
+
+!0 = !{i32 0, !"typeid"}
diff --git a/llvm/tools/llvm-reduce/CMakeLists.txt b/llvm/tools/llvm-reduce/CMakeLists.txt
index 717e75b5aa060..b1911f9c407b2 100644
--- a/llvm/tools/llvm-reduce/CMakeLists.txt
+++ b/llvm/tools/llvm-reduce/CMakeLists.txt
@@ -14,6 +14,7 @@ set(LLVM_LINK_COMPONENTS
Support
Target
TransformUtils
+ IPO
)
add_llvm_tool(llvm-reduce
diff --git a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
index 7f05ea8e56263..891553497b9bb 100644
--- a/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
+++ b/llvm/tools/llvm-reduce/ReducerWorkItem.cpp
@@ -7,6 +7,7 @@
//===----------------------------------------------------------------------===//
#include "ReducerWorkItem.h"
+#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/CodeGen/CommandFlags.h"
#include "llvm/CodeGen/MIRParser/MIRParser.h"
#include "llvm/CodeGen/MIRPrinter.h"
@@ -17,11 +18,14 @@
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
#include "llvm/CodeGen/TargetInstrInfo.h"
+#include "llvm/IR/ModuleSummaryIndex.h"
#include "llvm/IR/Verifier.h"
#include "llvm/IRReader/IRReader.h"
#include "llvm/MC/TargetRegistry.h"
#include "llvm/Support/Host.h"
+#include "llvm/Support/MemoryBufferRef.h"
#include "llvm/Support/SourceMgr.h"
+#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/WithColor.h"
#include "llvm/Target/TargetMachine.h"
#include "llvm/Transforms/Utils/Cloning.h"
@@ -31,6 +35,8 @@ static cl::opt<std::string> TargetTriple("mtriple",
cl::desc("Set the target triple"),
cl::cat(LLVMReduceOptions));
+void readBitcode(ReducerWorkItem &M, MemoryBufferRef Data, LLVMContext &Ctx, const char *ToolName);
+
static void cloneFrameInfo(
MachineFrameInfo &DstMFI, const MachineFrameInfo &SrcMFI,
const DenseMap<MachineBasicBlock *, MachineBasicBlock *> &Src2DstMBB) {
@@ -352,6 +358,13 @@ static std::unique_ptr<MachineFunction> cloneMF(MachineFunction *SrcMF,
return DstMF;
}
+static void initializeTargetInfo() {
+ InitializeAllTargets();
+ InitializeAllTargetMCs();
+ InitializeAllAsmPrinters();
+ InitializeAllAsmParsers();
+}
+
std::unique_ptr<ReducerWorkItem>
parseReducerWorkItem(const char *ToolName, StringRef Filename,
LLVMContext &Ctxt, std::unique_ptr<TargetMachine> &TM,
@@ -361,6 +374,8 @@ parseReducerWorkItem(const char *ToolName, StringRef Filename,
auto MMM = std::make_unique<ReducerWorkItem>();
if (IsMIR) {
+ initializeTargetInfo();
+
auto FileOrErr = MemoryBuffer::getFileOrSTDIN(Filename, /*IsText=*/true);
if (std::error_code EC = FileOrErr.getError()) {
WithColor::error(errs(), ToolName) << EC.message() << '\n';
@@ -409,17 +424,31 @@ parseReducerWorkItem(const char *ToolName, StringRef Filename,
MMM->M = std::move(M);
} else {
SMDiagnostic Err;
- std::unique_ptr<Module> Result = parseIRFile(Filename, Err, Ctxt);
- if (!Result) {
- Err.print(ToolName, errs());
- return std::unique_ptr<ReducerWorkItem>();
+ ErrorOr<std::unique_ptr<MemoryBuffer>> MB = MemoryBuffer::getFileOrSTDIN(Filename);
+ if (std::error_code EC = MB.getError()) {
+ WithColor::error(errs(), ToolName) << Filename << ": " << EC.message() << "\n";
+ return nullptr;
+ }
+
+ if (!isBitcode((const unsigned char *)(*MB)->getBufferStart(),
+ (const unsigned char *)(*MB)->getBufferEnd())) {
+ std::unique_ptr<Module> Result = parseIRFile(Filename, Err, Ctxt);
+ if (!Result) {
+ Err.print(ToolName, errs());
+ return nullptr;
+ }
+ MMM->M = std::move(Result);
+ } else {
+ readBitcode(*MMM, MemoryBufferRef(**MB), Ctxt, ToolName);
+
+ if (MMM->LTOInfo->IsThinLTO && MMM->LTOInfo->EnableSplitLTOUnit)
+ initializeTargetInfo();
}
- MMM->M = std::move(Result);
}
if (verifyReducerWorkItem(*MMM, &errs())) {
WithColor::error(errs(), ToolName)
<< Filename << " - input module is broken!\n";
- return std::unique_ptr<ReducerWorkItem>();
+ return nullptr;
}
return MMM;
}
diff --git a/llvm/tools/llvm-reduce/ReducerWorkItem.h b/llvm/tools/llvm-reduce/ReducerWorkItem.h
index 89330d2b83e43..7dd578687f855 100644
--- a/llvm/tools/llvm-reduce/ReducerWorkItem.h
+++ b/llvm/tools/llvm-reduce/ReducerWorkItem.h
@@ -9,9 +9,11 @@
#ifndef LLVM_TOOLS_LLVM_REDUCE_REDUCERWORKITEM_H
#define LLVM_TOOLS_LLVM_REDUCE_REDUCERWORKITEM_H
+#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/ModuleSummaryIndex.h"
#include "llvm/Target/TargetMachine.h"
using namespace llvm;
@@ -19,6 +21,7 @@ using namespace llvm;
class ReducerWorkItem {
public:
std::shared_ptr<Module> M;
+ std::unique_ptr<BitcodeLTOInfo> LTOInfo;
std::unique_ptr<MachineModuleInfo> MMI;
bool isMIR() const { return MMI != nullptr; }
diff --git a/llvm/tools/llvm-reduce/TestRunner.cpp b/llvm/tools/llvm-reduce/TestRunner.cpp
index 228e54ac83adb..166222e0ed795 100644
--- a/llvm/tools/llvm-reduce/TestRunner.cpp
+++ b/llvm/tools/llvm-reduce/TestRunner.cpp
@@ -15,9 +15,9 @@ using namespace llvm;
TestRunner::TestRunner(StringRef TestName,
const std::vector<std::string> &TestArgs,
std::unique_ptr<ReducerWorkItem> Program,
- std::unique_ptr<TargetMachine> TM)
+ std::unique_ptr<TargetMachine> TM, const char *ToolName)
: TestName(TestName), TestArgs(TestArgs), Program(std::move(Program)),
- TM(std::move(TM)) {
+ TM(std::move(TM)), ToolName(ToolName) {
assert(this->Program && "Initialized with null program?");
}
diff --git a/llvm/tools/llvm-reduce/TestRunner.h b/llvm/tools/llvm-reduce/TestRunner.h
index 8c0b843a846b3..6d7dd7a51ceba 100644
--- a/llvm/tools/llvm-reduce/TestRunner.h
+++ b/llvm/tools/llvm-reduce/TestRunner.h
@@ -28,7 +28,7 @@ class TestRunner {
public:
TestRunner(StringRef TestName, const std::vector<std::string> &TestArgs,
std::unique_ptr<ReducerWorkItem> Program,
- std::unique_ptr<TargetMachine> TM);
+ std::unique_ptr<TargetMachine> TM, const char *ToolName);
/// Runs the interesting-ness test for the specified file
/// @returns 0 if test was successful, 1 if otherwise
@@ -41,8 +41,11 @@ class TestRunner {
const TargetMachine *getTargetMachine() const { return TM.get(); }
+ const char *getToolName() const { return ToolName; }
+
private:
StringRef TestName;
+ const char *ToolName;
const std::vector<std::string> &TestArgs;
std::unique_ptr<ReducerWorkItem> Program;
std::unique_ptr<TargetMachine> TM;
diff --git a/llvm/tools/llvm-reduce/deltas/Delta.cpp b/llvm/tools/llvm-reduce/deltas/Delta.cpp
index 94fa1d936653c..55b96d84c9c81 100644
--- a/llvm/tools/llvm-reduce/deltas/Delta.cpp
+++ b/llvm/tools/llvm-reduce/deltas/Delta.cpp
@@ -15,11 +15,16 @@
#include "Delta.h"
#include "ReducerWorkItem.h"
#include "llvm/ADT/STLExtras.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/Analysis/ModuleSummaryAnalysis.h"
#include "llvm/Bitcode/BitcodeReader.h"
#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/CodeGen/MachineFunction.h"
+#include "llvm/IR/Module.h"
#include "llvm/IR/Verifier.h"
+#include "llvm/MC/TargetRegistry.h"
#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/MemoryBufferRef.h"
#include "llvm/Support/ThreadPool.h"
#include "llvm/Support/ToolOutputFile.h"
#include <fstream>
@@ -56,6 +61,11 @@ unsigned NumJobs = 1;
void writeOutput(ReducerWorkItem &M, llvm::StringRef Message);
+void writeBitcode(ReducerWorkItem &M, raw_ostream &OutStream);
+
+void readBitcode(ReducerWorkItem &M, MemoryBufferRef Data, LLVMContext &Ctx,
+ const char *ToolName);
+
bool isReduced(ReducerWorkItem &M, TestRunner &Test,
SmallString<128> &CurrentFilepath) {
// Write ReducerWorkItem to tmp file
@@ -70,7 +80,7 @@ bool isReduced(ReducerWorkItem &M, TestRunner &Test,
if (TmpFilesAsBitcode) {
llvm::raw_fd_ostream OutStream(FD, true);
- WriteBitcodeToFile(M, OutStream);
+ writeBitcode(M, OutStream);
OutStream.close();
if (OutStream.has_error()) {
errs() << "Error emitting bitcode to file '" << CurrentFilepath << "'!\n";
@@ -192,14 +202,10 @@ SmallString<0> ProcessChunkFromSerializedBitcode(
std::vector<Chunk> &ChunksStillConsideredInteresting,
SmallString<0> &OriginalBC, std::atomic<bool> &AnyReduced) {
LLVMContext Ctx;
- Expected<std::unique_ptr<Module>> MOrErr = parseBitcodeFile(
- MemoryBufferRef(StringRef(OriginalBC.data(), OriginalBC.size()),
- "<llvm-reduce tmp module>"),
- Ctx);
- if (!MOrErr)
- report_fatal_error("Failed to read bitcode");
auto CloneMMM = std::make_unique<ReducerWorkItem>();
- CloneMMM->M = std::move(MOrErr.get());
+ auto Data = MemoryBufferRef(StringRef(OriginalBC.data(), OriginalBC.size()),
+ "<bc file>");
+ readBitcode(*CloneMMM, Data, Ctx, Test.getToolName());
SmallString<0> Result;
if (std::unique_ptr<ReducerWorkItem> ChunkResult =
@@ -207,7 +213,7 @@ SmallString<0> ProcessChunkFromSerializedBitcode(
Test, ExtractChunksFromModule, UninterestingChunks,
ChunksStillConsideredInteresting)) {
raw_svector_ostream BCOS(Result);
- WriteBitcodeToFile(*ChunkResult->M, BCOS);
+ writeBitcode(*ChunkResult, BCOS);
// Communicate that the task reduced a chunk.
AnyReduced = true;
}
@@ -284,7 +290,7 @@ void llvm::runDeltaPass(TestRunner &Test,
SmallString<0> OriginalBC;
if (NumJobs > 1) {
raw_svector_ostream BCOS(OriginalBC);
- WriteBitcodeToFile(*Test.getProgram().M, BCOS);
+ writeBitcode(Test.getProgram(), BCOS);
}
std::deque<std::shared_future<SmallString<0>>> TaskQueue;
@@ -351,14 +357,11 @@ void llvm::runDeltaPass(TestRunner &Test,
continue;
}
- Expected<std::unique_ptr<Module>> MOrErr = parseBitcodeFile(
- MemoryBufferRef(StringRef(Res.data(), Res.size()),
- "<llvm-reduce tmp module>"),
- Test.getProgram().M->getContext());
- if (!MOrErr)
- report_fatal_error("Failed to read bitcode");
Result = std::make_unique<ReducerWorkItem>();
- Result->M = std::move(MOrErr.get());
+ auto Data = MemoryBufferRef(StringRef(Res.data(), Res.size()),
+ "<bc file>");
+ readBitcode(*Result, Data, Test.getProgram().M->getContext(),
+ Test.getToolName());
break;
}
// Forward I to the last chunk processed in parallel.
diff --git a/llvm/tools/llvm-reduce/llvm-reduce.cpp b/llvm/tools/llvm-reduce/llvm-reduce.cpp
index 3cf2bbcb95871..4d338f4eb282b 100644
--- a/llvm/tools/llvm-reduce/llvm-reduce.cpp
+++ b/llvm/tools/llvm-reduce/llvm-reduce.cpp
@@ -17,10 +17,15 @@
#include "DeltaManager.h"
#include "ReducerWorkItem.h"
#include "TestRunner.h"
+#include "llvm/Analysis/ProfileSummaryInfo.h"
+#include "llvm/Analysis/ModuleSummaryAnalysis.h"
#include "llvm/ADT/SmallString.h"
+#include "llvm/Bitcode/BitcodeReader.h"
+#include "llvm/Bitcode/BitcodeWriter.h"
#include "llvm/CodeGen/CommandFlags.h"
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
+#include "llvm/IR/LegacyPassManager.h"
#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/Verifier.h"
#include "llvm/IRReader/IRReader.h"
@@ -32,6 +37,7 @@
#include "llvm/Support/TargetSelect.h"
#include "llvm/Support/WithColor.h"
#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/IPO.h"
#include <system_error>
#include <vector>
@@ -94,13 +100,6 @@ static cl::opt<int>
static codegen::RegisterCodeGenFlags CGF;
-static void initializeTargetInfo() {
- InitializeAllTargets();
- InitializeAllTargetMCs();
- InitializeAllAsmPrinters();
- InitializeAllAsmParsers();
-}
-
void writeOutput(ReducerWorkItem &M, StringRef Message) {
if (ReplaceInput) // In-place
OutputFilename = InputFilename.c_str();
@@ -116,6 +115,39 @@ void writeOutput(ReducerWorkItem &M, StringRef Message) {
errs() << Message << OutputFilename << "\n";
}
+void writeBitcode(ReducerWorkItem &M, llvm::raw_ostream &OutStream) {
+ if (M.LTOInfo && M.LTOInfo->IsThinLTO && M.LTOInfo->EnableSplitLTOUnit) {
+ legacy::PassManager PM;
+ PM.add(llvm::createWriteThinLTOBitcodePass(OutStream));
+ PM.run(*(M.M));
+ } else {
+ std::unique_ptr<ModuleSummaryIndex> Index;
+ if (M.LTOInfo && M.LTOInfo->HasSummary) {
+ ProfileSummaryInfo PSI(M);
+ Index = std::make_unique<ModuleSummaryIndex>(
+ buildModuleSummaryIndex(M, nullptr, &PSI));
+ }
+ WriteBitcodeToFile(M, OutStream, Index.get());
+ }
+}
+
+void readBitcode(ReducerWorkItem &M, MemoryBufferRef Data, LLVMContext &Ctx, const char *ToolName) {
+ Expected<BitcodeFileContents> IF = llvm::getBitcodeFileContents(Data);
+ if (!IF) {
+ WithColor::error(errs(), ToolName) << IF.takeError();
+ exit(1);
+ }
+ BitcodeModule BM = IF->Mods[0];
+ Expected<BitcodeLTOInfo> LI = BM.getLTOInfo();
+ Expected<std::unique_ptr<Module>> MOrErr = BM.parseModule(Ctx);
+ if (!LI || !MOrErr) {
+ WithColor::error(errs(), ToolName) << IF.takeError();
+ exit(1);
+ }
+ M.LTOInfo = std::make_unique<BitcodeLTOInfo>(*LI);
+ M.M = std::move(MOrErr.get());
+}
+
int main(int Argc, char **Argv) {
InitLLVM X(Argc, Argv);
@@ -135,9 +167,6 @@ int main(int Argc, char **Argv) {
return 0;
}
- if (ReduceModeMIR)
- initializeTargetInfo();
-
LLVMContext Context;
std::unique_ptr<TargetMachine> TM;
@@ -149,7 +178,7 @@ int main(int Argc, char **Argv) {
// Initialize test environment
TestRunner Tester(TestFilename, TestArguments, std::move(OriginalProgram),
- std::move(TM));
+ std::move(TM), Argv[0]);
// Try to reduce code
runDeltaPasses(Tester, MaxPassIterations);
More information about the llvm-commits
mailing list