[llvm] [lipo] Support creating Universal 64 bit Mach-O files. (PR #67737)
Daniel RodrÃguez Troitiño via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 28 17:03:22 PDT 2023
https://github.com/drodriguez updated https://github.com/llvm/llvm-project/pull/67737
>From 2f5a932c9bcf48cbc631592ddf0b5d2b68e84a9c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Rodri=CC=81guez=20Troitin=CC=83o?=
<danielrodriguez at meta.com>
Date: Thu, 28 Sep 2023 13:35:17 -0700
Subject: [PATCH 1/2] [lipo] Support creating Universal 64 bit Mach-O files.
Xcode `lipo` seems to support a non-document `-fat64` option that
creates Universal Mach-O archives using 64 bit versions of the
`fat_arch` header, which allows offsets larger than 32 bits to be
specified.
Modify `llvm-lipo` to support the same flag, and use the value of the
flag to use either 32 bits or 64 bits Mach-O headers.
The Mach-O universal writer allows specifying a new option to write
these 64 bits headers. The default is still using 32 bits.
`dsymutil` implemented support for a similar flag in
https://reviews.llvm.org/D146879.
---
.../llvm/Object/MachOUniversalWriter.h | 6 +-
llvm/lib/Object/MachOUniversalWriter.cpp | 79 +++++++++++++------
llvm/test/tools/llvm-lipo/create-fat64.test | 11 +++
llvm/tools/llvm-lipo/LipoOpts.td | 3 +
llvm/tools/llvm-lipo/llvm-lipo.cpp | 15 ++--
5 files changed, 85 insertions(+), 29 deletions(-)
create mode 100644 llvm/test/tools/llvm-lipo/create-fat64.test
diff --git a/llvm/include/llvm/Object/MachOUniversalWriter.h b/llvm/include/llvm/Object/MachOUniversalWriter.h
index 4004f25f3fb7ecc..3270bc12960b048 100644
--- a/llvm/include/llvm/Object/MachOUniversalWriter.h
+++ b/llvm/include/llvm/Object/MachOUniversalWriter.h
@@ -97,9 +97,11 @@ class Slice {
}
};
-Error writeUniversalBinary(ArrayRef<Slice> Slices, StringRef OutputFileName);
+Error writeUniversalBinary(ArrayRef<Slice> Slices, StringRef OutputFileName,
+ bool UseFat64 = false);
-Error writeUniversalBinaryToStream(ArrayRef<Slice> Slices, raw_ostream &Out);
+Error writeUniversalBinaryToStream(ArrayRef<Slice> Slices, raw_ostream &Out,
+ bool UseFat64 = false);
} // end namespace object
diff --git a/llvm/lib/Object/MachOUniversalWriter.cpp b/llvm/lib/Object/MachOUniversalWriter.cpp
index 909a10b2c072a2e..aaab0dbac98caab 100644
--- a/llvm/lib/Object/MachOUniversalWriter.cpp
+++ b/llvm/lib/Object/MachOUniversalWriter.cpp
@@ -240,25 +240,48 @@ Expected<Slice> Slice::create(const IRObjectFile &IRO, uint32_t Align) {
return Slice{IRO, CPUType, CPUSubType, std::move(ArchName), Align};
}
-static Expected<SmallVector<MachO::fat_arch, 2>>
+template <typename FatArchTy> struct FatArchTraits {
+ static const uint64_t OffsetLimit;
+ static const std::string StructName;
+ static const uint8_t BitCount;
+};
+
+template <> struct FatArchTraits<MachO::fat_arch> {
+ static const uint64_t OffsetLimit = UINT32_MAX;
+ static const std::string StructName;
+ static const uint8_t BitCount = 32;
+};
+const std::string FatArchTraits<MachO::fat_arch>::StructName = "fat_arch";
+
+template <> struct FatArchTraits<MachO::fat_arch_64> {
+ static const uint64_t OffsetLimit = UINT64_MAX;
+ static const std::string StructName;
+ static const uint8_t BitCount = 64;
+};
+const std::string FatArchTraits<MachO::fat_arch_64>::StructName = "fat_arch_64";
+
+template <typename FatArchTy>
+static Expected<SmallVector<FatArchTy, 2>>
buildFatArchList(ArrayRef<Slice> Slices) {
- SmallVector<MachO::fat_arch, 2> FatArchList;
+ SmallVector<FatArchTy, 2> FatArchList;
uint64_t Offset =
- sizeof(MachO::fat_header) + Slices.size() * sizeof(MachO::fat_arch);
+ sizeof(MachO::fat_header) + Slices.size() * sizeof(FatArchTy);
for (const auto &S : Slices) {
Offset = alignTo(Offset, 1ull << S.getP2Alignment());
- if (Offset > UINT32_MAX)
+ if (Offset > FatArchTraits<FatArchTy>::OffsetLimit)
return createStringError(
std::errc::invalid_argument,
- ("fat file too large to be created because the offset "
- "field in struct fat_arch is only 32-bits and the offset " +
+ ("fat file too large to be created because the offset field in the "
+ "struct " +
+ Twine(FatArchTraits<FatArchTy>::StructName) + " is only " +
+ Twine(FatArchTraits<FatArchTy>::BitCount) + "-bits and the offset " +
Twine(Offset) + " for " + S.getBinary()->getFileName() +
" for architecture " + S.getArchString() + "exceeds that.")
.str()
.c_str());
- MachO::fat_arch FatArch;
+ FatArchTy FatArch;
FatArch.cputype = S.getCPUType();
FatArch.cpusubtype = S.getCPUSubType();
FatArch.offset = Offset;
@@ -270,17 +293,15 @@ buildFatArchList(ArrayRef<Slice> Slices) {
return FatArchList;
}
-Error object::writeUniversalBinaryToStream(ArrayRef<Slice> Slices,
- raw_ostream &Out) {
- MachO::fat_header FatHeader;
- FatHeader.magic = MachO::FAT_MAGIC;
- FatHeader.nfat_arch = Slices.size();
-
- Expected<SmallVector<MachO::fat_arch, 2>> FatArchListOrErr =
- buildFatArchList(Slices);
+template <typename FatArchTy>
+static Error writeUniversalArchsToStream(MachO::fat_header FatHeader,
+ ArrayRef<Slice> Slices,
+ raw_ostream &Out) {
+ Expected<SmallVector<FatArchTy, 2>> FatArchListOrErr =
+ buildFatArchList<FatArchTy>(Slices);
if (!FatArchListOrErr)
return FatArchListOrErr.takeError();
- SmallVector<MachO::fat_arch, 2> FatArchList = *FatArchListOrErr;
+ SmallVector<FatArchTy, 2> FatArchList = *FatArchListOrErr;
if (sys::IsLittleEndianHost)
MachO::swapStruct(FatHeader);
@@ -288,17 +309,17 @@ Error object::writeUniversalBinaryToStream(ArrayRef<Slice> Slices,
sizeof(MachO::fat_header));
if (sys::IsLittleEndianHost)
- for (MachO::fat_arch &FA : FatArchList)
+ for (FatArchTy &FA : FatArchList)
MachO::swapStruct(FA);
Out.write(reinterpret_cast<const char *>(FatArchList.data()),
- sizeof(MachO::fat_arch) * FatArchList.size());
+ sizeof(FatArchTy) * FatArchList.size());
if (sys::IsLittleEndianHost)
- for (MachO::fat_arch &FA : FatArchList)
+ for (FatArchTy &FA : FatArchList)
MachO::swapStruct(FA);
size_t Offset =
- sizeof(MachO::fat_header) + sizeof(MachO::fat_arch) * FatArchList.size();
+ sizeof(MachO::fat_header) + sizeof(FatArchTy) * FatArchList.size();
for (size_t Index = 0, Size = Slices.size(); Index < Size; ++Index) {
MemoryBufferRef BufferRef = Slices[Index].getBinary()->getMemoryBufferRef();
assert((Offset <= FatArchList[Index].offset) && "Incorrect slice offset");
@@ -311,8 +332,22 @@ Error object::writeUniversalBinaryToStream(ArrayRef<Slice> Slices,
return Error::success();
}
+Error object::writeUniversalBinaryToStream(ArrayRef<Slice> Slices,
+ raw_ostream &Out, bool UseFat64) {
+ MachO::fat_header FatHeader;
+ FatHeader.magic = UseFat64 ? MachO::FAT_MAGIC_64 : MachO::FAT_MAGIC;
+ FatHeader.nfat_arch = Slices.size();
+
+ if (UseFat64) {
+ return writeUniversalArchsToStream<MachO::fat_arch_64>(FatHeader, Slices,
+ Out);
+ } else {
+ return writeUniversalArchsToStream<MachO::fat_arch>(FatHeader, Slices, Out);
+ }
+}
+
Error object::writeUniversalBinary(ArrayRef<Slice> Slices,
- StringRef OutputFileName) {
+ StringRef OutputFileName, bool UseFat64) {
const bool IsExecutable = any_of(Slices, [](Slice S) {
return sys::fs::can_execute(S.getBinary()->getFileName());
});
@@ -324,7 +359,7 @@ Error object::writeUniversalBinary(ArrayRef<Slice> Slices,
if (!Temp)
return Temp.takeError();
raw_fd_ostream Out(Temp->FD, false);
- if (Error E = writeUniversalBinaryToStream(Slices, Out)) {
+ if (Error E = writeUniversalBinaryToStream(Slices, Out, UseFat64)) {
if (Error DiscardError = Temp->discard())
return joinErrors(std::move(E), std::move(DiscardError));
return E;
diff --git a/llvm/test/tools/llvm-lipo/create-fat64.test b/llvm/test/tools/llvm-lipo/create-fat64.test
new file mode 100644
index 000000000000000..1c420899ed18a7d
--- /dev/null
+++ b/llvm/test/tools/llvm-lipo/create-fat64.test
@@ -0,0 +1,11 @@
+# RUN: yaml2obj %p/Inputs/i386-slice.yaml -o %t-i386.o
+# RUN: yaml2obj %p/Inputs/x86_64-slice.yaml -o %t-x86_64.o
+
+# RUN: llvm-lipo %t-i386.o %t-x86_64.o -create -output %t-universal-32.o
+# RUN: llvm-objdump -m --universal-headers %t-universal-32.o | FileCheck %s -check-prefixes=FAT32
+
+# RUN: llvm-lipo %t-i386.o %t-x86_64.o -create -fat64 -output %t-universal-64.o
+# RUN: llvm-objdump -m --universal-headers %t-universal-64.o | FileCheck %s -check-prefixes=FAT64
+
+FAT32: fat_magic FAT_MAGIC
+FAT64: fat_magic FAT_MAGIC_64
diff --git a/llvm/tools/llvm-lipo/LipoOpts.td b/llvm/tools/llvm-lipo/LipoOpts.td
index 866353573cba309..32bdca4a5da1607 100644
--- a/llvm/tools/llvm-lipo/LipoOpts.td
+++ b/llvm/tools/llvm-lipo/LipoOpts.td
@@ -58,3 +58,6 @@ def replace
def output : Option<["-", "--"], "output", KIND_SEPARATE>,
HelpText<"Create output file with specified name">;
def o : JoinedOrSeparate<["-"], "o">, Alias<output>;
+
+def fat64 : Option<["-", "--"], "fat64", KIND_FLAG>,
+ HelpText<"Use 64 bits Universal Mach-O format">;
diff --git a/llvm/tools/llvm-lipo/llvm-lipo.cpp b/llvm/tools/llvm-lipo/llvm-lipo.cpp
index b1f202877b15e8d..88d77faa035fc9b 100644
--- a/llvm/tools/llvm-lipo/llvm-lipo.cpp
+++ b/llvm/tools/llvm-lipo/llvm-lipo.cpp
@@ -116,6 +116,7 @@ struct Config {
std::string ArchType;
std::string OutputFile;
LipoAction ActionToPerform;
+ bool UseFat64;
};
static Slice createSliceFromArchive(LLVMContext &LLVMCtx, const Archive &A) {
@@ -223,6 +224,8 @@ static Config parseLipoOptions(ArrayRef<const char *> ArgsArr) {
Twine(AlignmentValue));
}
+ C.UseFat64 = InputArgs.hasArg(LIPO_fat64);
+
SmallVector<opt::Arg *, 1> ActionArgs(InputArgs.filtered(LIPO_action_group));
if (ActionArgs.empty())
reportError("at least one action should be specified");
@@ -596,9 +599,11 @@ buildSlices(LLVMContext &LLVMCtx, ArrayRef<OwningBinary<Binary>> InputBinaries,
return Slices;
}
-[[noreturn]] static void createUniversalBinary(
- LLVMContext &LLVMCtx, ArrayRef<OwningBinary<Binary>> InputBinaries,
- const StringMap<const uint32_t> &Alignments, StringRef OutputFileName) {
+[[noreturn]] static void
+createUniversalBinary(LLVMContext &LLVMCtx,
+ ArrayRef<OwningBinary<Binary>> InputBinaries,
+ const StringMap<const uint32_t> &Alignments,
+ StringRef OutputFileName, bool UseFat64) {
assert(InputBinaries.size() >= 1 && "Incorrect number of input binaries");
assert(!OutputFileName.empty() && "Create expects a single output file");
@@ -609,7 +614,7 @@ buildSlices(LLVMContext &LLVMCtx, ArrayRef<OwningBinary<Binary>> InputBinaries,
checkUnusedAlignments(Slices, Alignments);
llvm::stable_sort(Slices);
- if (Error E = writeUniversalBinary(Slices, OutputFileName))
+ if (Error E = writeUniversalBinary(Slices, OutputFileName, UseFat64))
reportError(std::move(E));
exit(EXIT_SUCCESS);
@@ -748,7 +753,7 @@ int llvm_lipo_main(int argc, char **argv, const llvm::ToolContext &) {
break;
case LipoAction::CreateUniversal:
createUniversalBinary(LLVMCtx, InputBinaries, C.SegmentAlignments,
- C.OutputFile);
+ C.OutputFile, C.UseFat64);
break;
case LipoAction::ReplaceArch:
replaceSlices(LLVMCtx, InputBinaries, C.SegmentAlignments, C.OutputFile,
>From c997ec329a28c5d144a3005aec0092ec686bb6a2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Rodri=CC=81guez=20Troitin=CC=83o?=
<danielrodriguez at meta.com>
Date: Thu, 28 Sep 2023 17:02:51 -0700
Subject: [PATCH 2/2] fixup! [lipo] Support creating Universal 64 bit Mach-O
files.
Use enum for header types instead of plain boolean
---
.../llvm/Object/MachOUniversalWriter.h | 9 ++++++---
llvm/lib/Object/MachOUniversalWriter.cpp | 20 +++++++++++++------
llvm/tools/llvm-lipo/llvm-lipo.cpp | 9 +++++----
3 files changed, 25 insertions(+), 13 deletions(-)
diff --git a/llvm/include/llvm/Object/MachOUniversalWriter.h b/llvm/include/llvm/Object/MachOUniversalWriter.h
index 3270bc12960b048..f910ca0cf968def 100644
--- a/llvm/include/llvm/Object/MachOUniversalWriter.h
+++ b/llvm/include/llvm/Object/MachOUniversalWriter.h
@@ -97,11 +97,14 @@ class Slice {
}
};
+enum class FatHeaderType { FatHeader, Fat64Header };
+
Error writeUniversalBinary(ArrayRef<Slice> Slices, StringRef OutputFileName,
- bool UseFat64 = false);
+ FatHeaderType FatHeader = FatHeaderType::FatHeader);
-Error writeUniversalBinaryToStream(ArrayRef<Slice> Slices, raw_ostream &Out,
- bool UseFat64 = false);
+Error writeUniversalBinaryToStream(
+ ArrayRef<Slice> Slices, raw_ostream &Out,
+ FatHeaderType FatHeader = FatHeaderType::FatHeader);
} // end namespace object
diff --git a/llvm/lib/Object/MachOUniversalWriter.cpp b/llvm/lib/Object/MachOUniversalWriter.cpp
index aaab0dbac98caab..b4dff5f096af94a 100644
--- a/llvm/lib/Object/MachOUniversalWriter.cpp
+++ b/llvm/lib/Object/MachOUniversalWriter.cpp
@@ -333,21 +333,29 @@ static Error writeUniversalArchsToStream(MachO::fat_header FatHeader,
}
Error object::writeUniversalBinaryToStream(ArrayRef<Slice> Slices,
- raw_ostream &Out, bool UseFat64) {
+ raw_ostream &Out,
+ FatHeaderType HeaderType) {
MachO::fat_header FatHeader;
- FatHeader.magic = UseFat64 ? MachO::FAT_MAGIC_64 : MachO::FAT_MAGIC;
FatHeader.nfat_arch = Slices.size();
- if (UseFat64) {
+ switch (HeaderType) {
+ case FatHeaderType::Fat64Header:
+ FatHeader.magic = MachO::FAT_MAGIC_64;
return writeUniversalArchsToStream<MachO::fat_arch_64>(FatHeader, Slices,
Out);
- } else {
+ break;
+ case FatHeaderType::FatHeader:
+ FatHeader.magic = MachO::FAT_MAGIC;
return writeUniversalArchsToStream<MachO::fat_arch>(FatHeader, Slices, Out);
+ break;
+ default:
+ llvm_unreachable("Invalid fat header type");
}
}
Error object::writeUniversalBinary(ArrayRef<Slice> Slices,
- StringRef OutputFileName, bool UseFat64) {
+ StringRef OutputFileName,
+ FatHeaderType HeaderType) {
const bool IsExecutable = any_of(Slices, [](Slice S) {
return sys::fs::can_execute(S.getBinary()->getFileName());
});
@@ -359,7 +367,7 @@ Error object::writeUniversalBinary(ArrayRef<Slice> Slices,
if (!Temp)
return Temp.takeError();
raw_fd_ostream Out(Temp->FD, false);
- if (Error E = writeUniversalBinaryToStream(Slices, Out, UseFat64)) {
+ if (Error E = writeUniversalBinaryToStream(Slices, Out, HeaderType)) {
if (Error DiscardError = Temp->discard())
return joinErrors(std::move(E), std::move(DiscardError));
return E;
diff --git a/llvm/tools/llvm-lipo/llvm-lipo.cpp b/llvm/tools/llvm-lipo/llvm-lipo.cpp
index 88d77faa035fc9b..7e3275773b8add8 100644
--- a/llvm/tools/llvm-lipo/llvm-lipo.cpp
+++ b/llvm/tools/llvm-lipo/llvm-lipo.cpp
@@ -603,7 +603,7 @@ buildSlices(LLVMContext &LLVMCtx, ArrayRef<OwningBinary<Binary>> InputBinaries,
createUniversalBinary(LLVMContext &LLVMCtx,
ArrayRef<OwningBinary<Binary>> InputBinaries,
const StringMap<const uint32_t> &Alignments,
- StringRef OutputFileName, bool UseFat64) {
+ StringRef OutputFileName, FatHeaderType HeaderType) {
assert(InputBinaries.size() >= 1 && "Incorrect number of input binaries");
assert(!OutputFileName.empty() && "Create expects a single output file");
@@ -614,7 +614,7 @@ createUniversalBinary(LLVMContext &LLVMCtx,
checkUnusedAlignments(Slices, Alignments);
llvm::stable_sort(Slices);
- if (Error E = writeUniversalBinary(Slices, OutputFileName, UseFat64))
+ if (Error E = writeUniversalBinary(Slices, OutputFileName, HeaderType))
reportError(std::move(E));
exit(EXIT_SUCCESS);
@@ -752,8 +752,9 @@ int llvm_lipo_main(int argc, char **argv, const llvm::ToolContext &) {
C.OutputFile);
break;
case LipoAction::CreateUniversal:
- createUniversalBinary(LLVMCtx, InputBinaries, C.SegmentAlignments,
- C.OutputFile, C.UseFat64);
+ createUniversalBinary(
+ LLVMCtx, InputBinaries, C.SegmentAlignments, C.OutputFile,
+ C.UseFat64 ? FatHeaderType::Fat64Header : FatHeaderType::FatHeader);
break;
case LipoAction::ReplaceArch:
replaceSlices(LLVMCtx, InputBinaries, C.SegmentAlignments, C.OutputFile,
More information about the llvm-commits
mailing list