[clang] f7ce532 - [clang-offload-bundler] Add unbundling of archives containing bundled object files into device specific archives
Saiyedul Islam via cfe-commits
cfe-commits at lists.llvm.org
Wed Jun 30 05:26:20 PDT 2021
Author: Saiyedul Islam
Date: 2021-06-30T17:55:50+05:30
New Revision: f7ce532d622dc26eddd25f87faec0ff35dc0c2e9
URL: https://github.com/llvm/llvm-project/commit/f7ce532d622dc26eddd25f87faec0ff35dc0c2e9
DIFF: https://github.com/llvm/llvm-project/commit/f7ce532d622dc26eddd25f87faec0ff35dc0c2e9.diff
LOG: [clang-offload-bundler] Add unbundling of archives containing bundled object files into device specific archives
This patch adds unbundling support of an archive file. It takes an
archive file along with a set of offload targets as input.
Output is a device specific archive for each given offload target.
Input archive contains bundled code objects bundled using
clang-offload-bundler. Each generated device specific archive contains
a set of device code object files which are named as
<Parent Bundle Name>-<CodeObject-GPUArch>.
Entries in input archive can be of any binary type which is
supported by clang-offload-bundler, like *.bc. Output archives will
contain files in same type.
Example Usuage:
clang-offload-bundler --unbundle --inputs=lib-generic.a -type=a
-targets=openmp-amdgcn-amdhsa--gfx906,openmp-amdgcn-amdhsa--gfx908
-outputs=devicelib-gfx906.a,deviceLib-gfx908.a
Reviewed By: jdoerfert, yaxunl
Differential Revision: https://reviews.llvm.org/D93525
Added:
Modified:
clang/docs/ClangOffloadBundler.rst
clang/lib/Driver/ToolChains/Clang.cpp
clang/test/Driver/clang-offload-bundler.c
clang/test/Driver/hip-rdc-device-only.hip
clang/test/Driver/hip-toolchain-rdc-separate.hip
clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
Removed:
################################################################################
diff --git a/clang/docs/ClangOffloadBundler.rst b/clang/docs/ClangOffloadBundler.rst
index 68c5116b235f4..c92d8a94cfb54 100644
--- a/clang/docs/ClangOffloadBundler.rst
+++ b/clang/docs/ClangOffloadBundler.rst
@@ -121,7 +121,15 @@ Where:
============= ==============================================================
**target-triple**
- The target triple of the code object.
+ The target triple of the code object:
+
+.. code::
+
+ <Architecture>-<Vendor>-<OS>-<Environment>
+
+It is required to have all four components present, if target-id is present.
+Components are hyphen separated. If a component is not specified then the
+empty string must be used in its place.
**target-id**
The canonical target ID of the code object. Present only if the target
diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp
index c265e1c4e53cb..00939eae42998 100644
--- a/clang/lib/Driver/ToolChains/Clang.cpp
+++ b/clang/lib/Driver/ToolChains/Clang.cpp
@@ -7629,10 +7629,16 @@ void OffloadBundler::ConstructJob(Compilation &C, const JobAction &JA,
});
}
Triples += Action::GetOffloadKindName(CurKind);
- Triples += '-';
- Triples += CurTC->getTriple().normalize();
- if (CurKind == Action::OFK_HIP && CurDep->getOffloadingArch()) {
- Triples += '-';
+ Triples += "-";
+ std::string NormalizedTriple = CurTC->getTriple().normalize();
+ Triples += NormalizedTriple;
+
+ if (CurDep->getOffloadingArch() != nullptr) {
+ // If OffloadArch is present it can only appear as the 6th hypen
+ // sepearated field of Bundle Entry ID. So, pad required number of
+ // hyphens in Triple.
+ for (int i = 4 - StringRef(NormalizedTriple).count("-"); i > 0; i--)
+ Triples += "-";
Triples += CurDep->getOffloadingArch();
}
}
@@ -7702,11 +7708,17 @@ void OffloadBundler::ConstructJobMultipleOutputs(
auto &Dep = DepInfo[I];
Triples += Action::GetOffloadKindName(Dep.DependentOffloadKind);
- Triples += '-';
- Triples += Dep.DependentToolChain->getTriple().normalize();
- if (Dep.DependentOffloadKind == Action::OFK_HIP &&
- !Dep.DependentBoundArch.empty()) {
- Triples += '-';
+ Triples += "-";
+ std::string NormalizedTriple =
+ Dep.DependentToolChain->getTriple().normalize();
+ Triples += NormalizedTriple;
+
+ if (!Dep.DependentBoundArch.empty()) {
+ // If OffloadArch is present it can only appear as the 6th hypen
+ // sepearated field of Bundle Entry ID. So, pad required number of
+ // hyphens in Triple.
+ for (int i = 4 - StringRef(NormalizedTriple).count("-"); i > 0; i--)
+ Triples += "-";
Triples += Dep.DependentBoundArch;
}
}
diff --git a/clang/test/Driver/clang-offload-bundler.c b/clang/test/Driver/clang-offload-bundler.c
index faa6c5161a8f9..e1afa19570ec3 100644
--- a/clang/test/Driver/clang-offload-bundler.c
+++ b/clang/test/Driver/clang-offload-bundler.c
@@ -46,6 +46,7 @@
// CK-HELP: {{.*}}bc {{.*}}- llvm-bc
// CK-HELP: {{.*}}s {{.*}}- assembler
// CK-HELP: {{.*}}o {{.*}}- object
+// CK-HELP: {{.*}}a {{.*}}- archive of objects
// CK-HELP: {{.*}}gch {{.*}}- precompiled-header
// CK-HELP: {{.*}}ast {{.*}}- clang AST file
// CK-HELP: {{.*}}-unbundle {{.*}}- Unbundle bundled file into several output files.
@@ -103,6 +104,9 @@
// RUN: not clang-offload-bundler -type=i -targets=host-%itanium_abi_triple,host-%itanium_abi_triple,openmp-x86_64-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR9B
// CK-ERR9B: error: Duplicate targets are not allowed
+// RUN: not clang-offload-bundler -type=a -targets=hxst-powerpcxxle-ibm-linux-gnu,openxp-pxxerpc64le-ibm-linux-gnu,xpenmp-x86_xx-pc-linux-gnu -inputs=%t.i,%t.tgt1,%t.tgt2 -outputs=%t.bundle.i 2>&1 | FileCheck %s --check-prefix CK-ERR10A
+// CK-ERR10A: error: Archive files are only supported for unbundling
+
//
// Check text bundle. This is a readable format, so we check for the format we expect to find.
//
@@ -313,30 +317,30 @@
//
// Check error due to missing bundles
//
-// RUN: clang-offload-bundler -type=bc -targets=host-%itanium_abi_triple,hip-amdgcn-amd-amdhsa-gfx900 -inputs=%t.bc,%t.tgt1 -outputs=%t.hip.bundle.bc
+// RUN: clang-offload-bundler -type=bc -targets=host-%itanium_abi_triple,hip-amdgcn-amd-amdhsa--gfx900 -inputs=%t.bc,%t.tgt1 -outputs=%t.hip.bundle.bc
// RUN: not clang-offload-bundler -type=bc -inputs=%t.hip.bundle.bc -outputs=%t.tmp.bc -unbundle \
-// RUN: -targets=hip-amdgcn-amd-amdhsa-gfx906 \
+// RUN: -targets=hip-amdgcn-amd-amdhsa--gfx906 \
// RUN: 2>&1 | FileCheck -check-prefix=MISS1 %s
// RUN: not clang-offload-bundler -type=bc -inputs=%t.hip.bundle.bc -outputs=%t.tmp.bc,%t.tmp2.bc -unbundle \
-// RUN: -targets=hip-amdgcn-amd-amdhsa-gfx906,hip-amdgcn-amd-amdhsa-gfx900 \
+// RUN: -targets=hip-amdgcn-amd-amdhsa--gfx906,hip-amdgcn-amd-amdhsa--gfx900 \
// RUN: 2>&1 | FileCheck -check-prefix=MISS1 %s
-// MISS1: error: Can't find bundles for hip-amdgcn-amd-amdhsa-gfx906
+// MISS1: error: Can't find bundles for hip-amdgcn-amd-amdhsa--gfx906
// RUN: not clang-offload-bundler -type=bc -inputs=%t.hip.bundle.bc -outputs=%t.tmp.bc,%t.tmp2.bc -unbundle \
-// RUN: -targets=hip-amdgcn-amd-amdhsa-gfx906,hip-amdgcn-amd-amdhsa-gfx803 \
+// RUN: -targets=hip-amdgcn-amd-amdhsa--gfx906,hip-amdgcn-amd-amdhsa--gfx803 \
// RUN: 2>&1 | FileCheck -check-prefix=MISS2 %s
-// MISS2: error: Can't find bundles for hip-amdgcn-amd-amdhsa-gfx803 and hip-amdgcn-amd-amdhsa-gfx906
+// MISS2: error: Can't find bundles for hip-amdgcn-amd-amdhsa--gfx803 and hip-amdgcn-amd-amdhsa--gfx906
// RUN: not clang-offload-bundler -type=bc -inputs=%t.hip.bundle.bc -outputs=%t.tmp.bc,%t.tmp2.bc,%t.tmp3.bc -unbundle \
-// RUN: -targets=hip-amdgcn-amd-amdhsa-gfx906,hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx1010 \
+// RUN: -targets=hip-amdgcn-amd-amdhsa--gfx906,hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx1010 \
// RUN: 2>&1 | FileCheck -check-prefix=MISS3 %s
-// MISS3: error: Can't find bundles for hip-amdgcn-amd-amdhsa-gfx1010, hip-amdgcn-amd-amdhsa-gfx803, and hip-amdgcn-amd-amdhsa-gfx906
+// MISS3: error: Can't find bundles for hip-amdgcn-amd-amdhsa--gfx1010, hip-amdgcn-amd-amdhsa--gfx803, and hip-amdgcn-amd-amdhsa--gfx906
//
// Check error due to duplicate targets
//
-// RUN: not clang-offload-bundler -type=bc -targets=host-%itanium_abi_triple,hip-amdgcn-amd-amdhsa-gfx900,hip-amdgcn-amd-amdhsa-gfx900 \
+// RUN: not clang-offload-bundler -type=bc -targets=host-%itanium_abi_triple,hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx900 \
// RUN: -inputs=%t.bc,%t.tgt1,%t.tgt1 -outputs=%t.hip.bundle.bc 2>&1 | FileCheck -check-prefix=DUP %s
// RUN: not clang-offload-bundler -type=bc -inputs=%t.hip.bundle.bc -outputs=%t.tmp.bc,%t.tmp2.bc -unbundle \
-// RUN: -targets=hip-amdgcn-amd-amdhsa-gfx906,hip-amdgcn-amd-amdhsa-gfx906 \
+// RUN: -targets=hip-amdgcn-amd-amdhsa--gfx906,hip-amdgcn-amd-amdhsa--gfx906 \
// RUN: 2>&1 | FileCheck -check-prefix=DUP %s
// DUP: error: Duplicate targets are not allowed
//
@@ -364,17 +368,29 @@
//
// Check bundling without host target is allowed for HIP.
//
-// RUN: clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa-gfx900,hip-amdgcn-amd-amdhsa-gfx906 \
+// RUN: clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \
// RUN: -inputs=%t.tgt1,%t.tgt2 -outputs=%t.hip.bundle.bc
// RUN: clang-offload-bundler -type=bc -list -inputs=%t.hip.bundle.bc | FileCheck -check-prefix=NOHOST %s
-// RUN: clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa-gfx900,hip-amdgcn-amd-amdhsa-gfx906 \
+// RUN: clang-offload-bundler -type=bc -targets=hip-amdgcn-amd-amdhsa--gfx900,hip-amdgcn-amd-amdhsa--gfx906 \
// RUN: -outputs=%t.res.tgt1,%t.res.tgt2 -inputs=%t.hip.bundle.bc -unbundle
// RUN:
diff %t.tgt1 %t.res.tgt1
// RUN:
diff %t.tgt2 %t.res.tgt2
//
// NOHOST-NOT: host-
-// NOHOST-DAG: hip-amdgcn-amd-amdhsa-gfx900
-// NOHOST-DAG: hip-amdgcn-amd-amdhsa-gfx906
+// NOHOST-DAG: hip-amdgcn-amd-amdhsa--gfx900
+// NOHOST-DAG: hip-amdgcn-amd-amdhsa--gfx906
+// Check archive unbundling
+//
+// Create few code object bundles and archive them to create an input archive
+// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-amdgcn-amd-amdhsa--gfx906,openmp-amdgcn-amd-amdhsa--gfx908 -inputs=%t.o,%t.tgt1,%t.tgt2 -outputs=%t.simple.bundle
+// RUN: clang-offload-bundler -type=o -targets=host-%itanium_abi_triple,openmp-amdgcn-amd-amdhsa--gfx903 -inputs=%t.o,%t.tgt1 -outputs=%t.simple1.bundle
+// RUN: llvm-ar cr %t.input-archive.a %t.simple.bundle %t.simple1.bundle
+
+// RUN: clang-offload-bundler -unbundle -type=a -targets=openmp-amdgcn-amd-amdhsa--gfx906,openmp-amdgcn-amd-amdhsa--gfx908 -inputs=%t.input-archive.a -outputs=%t-archive-gfx906-simple.a,%t-archive-gfx908-simple.a
+// RUN: llvm-ar t %t-archive-gfx906-simple.a | FileCheck %s -check-prefix=GFX906
+// GFX906: simple-openmp-amdgcn-amd-amdhsa--gfx906
+// RUN: llvm-ar t %t-archive-gfx908-simple.a | FileCheck %s -check-prefix=GFX908
+// GFX908-NOT: {{gfx906}}
// Some code so that we can create a binary out of this file.
int A = 0;
diff --git a/clang/test/Driver/hip-rdc-device-only.hip b/clang/test/Driver/hip-rdc-device-only.hip
index ca8d54ea633e2..a95f636d777c1 100644
--- a/clang/test/Driver/hip-rdc-device-only.hip
+++ b/clang/test/Driver/hip-rdc-device-only.hip
@@ -82,7 +82,7 @@
// COMMON-SAME: {{.*}} {{".*a.cu"}}
// COMMON: "{{.*}}clang-offload-bundler" "-type={{(bc|ll)}}"
-// COMMON-SAME: "-targets=hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// COMMON-SAME: "-targets=hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
// COMMON-SAME: "-outputs=a-hip-amdgcn-amd-amdhsa.{{(bc|ll)}}"
// COMMON: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
@@ -112,7 +112,7 @@
// COMMON-SAME: {{.*}} {{".*b.hip"}}
// COMMON: "{{.*}}clang-offload-bundler" "-type={{(bc|ll)}}"
-// COMMON-SAME: "-targets=hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// COMMON-SAME: "-targets=hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
// COMMON-SAME: "-outputs=b-hip-amdgcn-amd-amdhsa.{{(bc|ll)}}"
// SAVETEMP: [[CLANG:".*clang.*"]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
@@ -142,7 +142,7 @@
// SAVETEMP-SAME: {{.*}} "-o" {{"a.*.ll"}} "-x" "ir" [[A_GFX900_TMP_BC]]
// SAVETEMP: "{{.*}}clang-offload-bundler" "-type=ll"
-// SAVETEMP-SAME: "-targets=hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// SAVETEMP-SAME: "-targets=hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
// SAVETEMP-SAME: "-outputs=a-hip-amdgcn-amd-amdhsa.ll"
// SAVETEMP: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa" "-aux-triple" "x86_64-unknown-linux-gnu"
@@ -172,7 +172,7 @@
// SAVETEMP-SAME: {{.*}} "-o" {{"b.*.ll"}} "-x" "ir" [[B_GFX900_TMP_BC]]
// SAVETEMP: "{{.*}}clang-offload-bundler" "-type=ll"
-// SAVETEMP-SAME: "-targets=hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// SAVETEMP-SAME: "-targets=hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
// SAVETEMP-SAME: "-outputs=b-hip-amdgcn-amd-amdhsa.ll"
// FAIL: error: cannot specify -o when generating multiple output files
diff --git a/clang/test/Driver/hip-toolchain-rdc-separate.hip b/clang/test/Driver/hip-toolchain-rdc-separate.hip
index 698ee14e74dc9..cdddbcc8fd216 100644
--- a/clang/test/Driver/hip-toolchain-rdc-separate.hip
+++ b/clang/test/Driver/hip-toolchain-rdc-separate.hip
@@ -44,7 +44,7 @@
// CHECK-SAME: {{.*}} [[A_SRC]]
// CHECK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
-// CHECK-SAME: "-targets=hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900,host-x86_64-unknown-linux-gnu"
+// CHECK-SAME: "-targets=hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900,host-x86_64-unknown-linux-gnu"
// CHECK-SAME: "-outputs=[[A_O:.*a.o]]" "-inputs=[[A_BC1]],[[A_BC2]],[[A_OBJ_HOST]]"
// CHECK: [[CLANG]] "-cc1" "-triple" "amdgcn-amd-amdhsa"
@@ -79,7 +79,7 @@
// CHECK-SAME: {{.*}} [[B_SRC]]
// CHECK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
-// CHECK-SAME: "-targets=hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900,host-x86_64-unknown-linux-gnu"
+// CHECK-SAME: "-targets=hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900,host-x86_64-unknown-linux-gnu"
// CHECK-SAME: "-outputs=[[B_O:.*b.o]]" "-inputs=[[B_BC1]],[[B_BC2]],[[B_OBJ_HOST]]"
// RUN: touch %T/a.o
@@ -91,22 +91,22 @@
// RUN: 2>&1 | FileCheck -check-prefix=LINK %s
// LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
-// LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
// LINK-SAME: "-inputs=[[A_O:.*a.o]]" "-outputs=[[A_OBJ_HOST:.*o]],{{.*o}},{{.*o}}"
// LINK: "-unbundle" "-allow-missing-bundles"
// LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
-// LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
// LINK-SAME: "-inputs=[[B_O:.*b.o]]" "-outputs=[[B_OBJ_HOST:.*o]],{{.*o}},{{.*o}}"
// LINK: "-unbundle" "-allow-missing-bundles"
// LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
-// LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
// LINK-SAME: "-inputs=[[A_O]]" "-outputs={{.*o}},[[A_BC1:.*o]],[[A_BC2:.*o]]"
// LINK: "-unbundle" "-allow-missing-bundles"
// LINK: [[BUNDLER:".*clang-offload-bundler"]] "-type=o"
-// LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa-gfx803,hip-amdgcn-amd-amdhsa-gfx900"
+// LINK-SAME: "-targets=host-x86_64-unknown-linux-gnu,hip-amdgcn-amd-amdhsa--gfx803,hip-amdgcn-amd-amdhsa--gfx900"
// LINK-SAME: "-inputs=[[B_O]]" "-outputs={{.*o}},[[B_BC1:.*o]],[[B_BC2:.*o]]"
// LINK: "-unbundle" "-allow-missing-bundles"
diff --git a/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp b/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
index a049ae725c89a..cb4f50ee1ed46 100644
--- a/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
+++ b/clang/tools/clang-offload-bundler/ClangOffloadBundler.cpp
@@ -22,14 +22,18 @@
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/StringSwitch.h"
#include "llvm/ADT/Triple.h"
+#include "llvm/Object/Archive.h"
+#include "llvm/Object/ArchiveWriter.h"
#include "llvm/Object/Binary.h"
#include "llvm/Object/ObjectFile.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
#include "llvm/Support/Errc.h"
#include "llvm/Support/Error.h"
#include "llvm/Support/ErrorOr.h"
#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Host.h"
#include "llvm/Support/MemoryBuffer.h"
#include "llvm/Support/Path.h"
#include "llvm/Support/Program.h"
@@ -82,6 +86,7 @@ static cl::opt<std::string>
" bc - llvm-bc\n"
" s - assembler\n"
" o - object\n"
+ " a - archive of objects\n"
" gch - precompiled-header\n"
" ast - clang AST file"),
cl::cat(ClangOffloadBundlerCategory));
@@ -123,20 +128,49 @@ static bool AllowNoHost = false;
/// Path to the current binary.
static std::string BundlerExecutable;
-/// Obtain the offload kind and real machine triple out of the target
-/// information specified by the user.
-static void getOffloadKindAndTriple(StringRef Target, StringRef &OffloadKind,
- StringRef &Triple) {
- auto KindTriplePair = Target.split('-');
- OffloadKind = KindTriplePair.first;
- Triple = KindTriplePair.second;
-}
-static bool hasHostKind(StringRef Target) {
+/// Obtain the offload kind, real machine triple, and an optional GPUArch
+/// out of the target information specified by the user.
+/// Bundle Entry ID (or, Offload Target String) has following components:
+/// * Offload Kind - Host, OpenMP, or HIP
+/// * Triple - Standard LLVM Triple
+/// * GPUArch (Optional) - Processor name, like gfx906 or sm_30
+/// In presence of Proc, the Triple should contain separator "-" for all
+/// standard four components, even if they are empty.
+struct OffloadTargetInfo {
StringRef OffloadKind;
- StringRef Triple;
- getOffloadKindAndTriple(Target, OffloadKind, Triple);
- return OffloadKind == "host";
-}
+ llvm::Triple Triple;
+ StringRef GPUArch;
+
+ OffloadTargetInfo(const StringRef Target) {
+ SmallVector<StringRef, 6> Components;
+ Target.split(Components, '-', 5);
+ Components.resize(6);
+ this->OffloadKind = Components[0];
+ this->Triple = llvm::Triple(Components[1], Components[2], Components[3],
+ Components[4]);
+ this->GPUArch = Components[5];
+ }
+
+ bool hasHostKind() const { return this->OffloadKind == "host"; }
+
+ bool isOffloadKindValid() const {
+ return OffloadKind == "host" || OffloadKind == "openmp" ||
+ OffloadKind == "hip" || OffloadKind == "hipv4";
+ }
+
+ bool isTripleValid() const {
+ return !Triple.str().empty() && Triple.getArch() != Triple::UnknownArch;
+ }
+
+ bool operator==(const OffloadTargetInfo &Target) const {
+ return OffloadKind == Target.OffloadKind &&
+ Triple.isCompatibleWith(Target.Triple) && GPUArch == Target.GPUArch;
+ }
+
+ std::string str() {
+ return Twine(OffloadKind + "-" + Triple.str() + "-" + GPUArch).str();
+ }
+};
/// Generic file handler interface.
class FileHandler {
@@ -163,7 +197,7 @@ class FileHandler {
virtual Error ReadBundleEnd(MemoryBuffer &Input) = 0;
/// Read the current bundle and write the result into the stream \a OS.
- virtual Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) = 0;
+ virtual Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) = 0;
/// Write the header of the bundled file to \a OS based on the information
/// gathered from \a Inputs.
@@ -378,7 +412,7 @@ class BinaryFileHandler final : public FileHandler {
return Error::success();
}
- Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
+ Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
assert(CurBundleInfo != BundlesInfo.end() && "Invalid reader info!");
StringRef FC = Input.getBuffer();
OS.write(FC.data() + CurBundleInfo->second.Offset,
@@ -541,7 +575,7 @@ class ObjectFileHandler final : public FileHandler {
Error ReadBundleEnd(MemoryBuffer &Input) final { return Error::success(); }
- Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
+ Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
Expected<StringRef> ContentOrErr = CurrentSection->getContents();
if (!ContentOrErr)
return ContentOrErr.takeError();
@@ -717,7 +751,7 @@ class TextFileHandler final : public FileHandler {
return Error::success();
}
- Error ReadBundle(raw_fd_ostream &OS, MemoryBuffer &Input) final {
+ Error ReadBundle(raw_ostream &OS, MemoryBuffer &Input) final {
StringRef FC = Input.getBuffer();
size_t BundleStart = ReadChars;
@@ -812,6 +846,8 @@ CreateFileHandler(MemoryBuffer &FirstInput) {
return std::make_unique<TextFileHandler>(/*Comment=*/"#");
if (FilesType == "o")
return CreateObjectFileHandler(FirstInput);
+ if (FilesType == "a")
+ return CreateObjectFileHandler(FirstInput);
if (FilesType == "gch")
return std::make_unique<BinaryFileHandler>();
if (FilesType == "ast")
@@ -956,7 +992,8 @@ static Error UnbundleFiles() {
Worklist.erase(Output);
// Record if we found the host bundle.
- if (hasHostKind(CurTriple))
+ auto OffloadInfo = OffloadTargetInfo(CurTriple);
+ if (OffloadInfo.hasHostKind())
FoundHostBundle = true;
}
@@ -989,7 +1026,8 @@ static Error UnbundleFiles() {
return createFileError(E.second, EC);
// If this entry has a host kind, copy the input file to the output file.
- if (hasHostKind(E.first()))
+ auto OffloadInfo = OffloadTargetInfo(E.getKey());
+ if (OffloadInfo.hasHostKind())
OutputFile.write(Input.getBufferStart(), Input.getBufferSize());
}
return Error::success();
@@ -1012,6 +1050,241 @@ static Error UnbundleFiles() {
return Error::success();
}
+static Archive::Kind getDefaultArchiveKindForHost() {
+ return Triple(sys::getDefaultTargetTriple()).isOSDarwin() ? Archive::K_DARWIN
+ : Archive::K_GNU;
+}
+
+/// @brief Checks if a code object \p CodeObjectInfo is compatible with a given
+/// target \p TargetInfo.
+/// @link https://clang.llvm.org/docs/ClangOffloadBundler.html#bundle-entry-id
+bool isCodeObjectCompatible(OffloadTargetInfo &CodeObjectInfo,
+ OffloadTargetInfo &TargetInfo) {
+
+ // Compatible in case of exact match.
+ if (CodeObjectInfo == TargetInfo) {
+ DEBUG_WITH_TYPE(
+ "CodeObjectCompatibility",
+ dbgs() << "Compatible: Exact match: " << CodeObjectInfo.str() << "\n");
+ return true;
+ }
+
+ // Incompatible if Kinds or Triples mismatch.
+ if (CodeObjectInfo.OffloadKind != TargetInfo.OffloadKind ||
+ !CodeObjectInfo.Triple.isCompatibleWith(TargetInfo.Triple)) {
+ DEBUG_WITH_TYPE(
+ "CodeObjectCompatibility",
+ dbgs() << "Incompatible: Kind/Triple mismatch \t[CodeObject: "
+ << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
+ << "]\n");
+ return false;
+ }
+
+ // Incompatible if GPUArch mismatch.
+ if (CodeObjectInfo.GPUArch != TargetInfo.GPUArch) {
+ DEBUG_WITH_TYPE("CodeObjectCompatibility",
+ dbgs() << "Incompatible: GPU Arch mismatch \t[CodeObject: "
+ << CodeObjectInfo.str()
+ << "]\t:\t[Target: " << TargetInfo.str() << "]\n");
+ return false;
+ }
+
+ DEBUG_WITH_TYPE(
+ "CodeObjectCompatibility",
+ dbgs() << "Compatible: Code Objects are compatible \t[CodeObject: "
+ << CodeObjectInfo.str() << "]\t:\t[Target: " << TargetInfo.str()
+ << "]\n");
+ return true;
+}
+
+/// @brief Computes a list of targets among all given targets which are
+/// compatible with this code object
+/// @param [in] Code Object \p CodeObject
+/// @param [out] List of all compatible targets \p CompatibleTargets among all
+/// given targets
+/// @return false, if no compatible target is found.
+static bool
+getCompatibleOffloadTargets(OffloadTargetInfo &CodeObjectInfo,
+ SmallVectorImpl<StringRef> &CompatibleTargets) {
+ if (!CompatibleTargets.empty()) {
+ DEBUG_WITH_TYPE("CodeObjectCompatibility",
+ dbgs() << "CompatibleTargets list should be empty\n");
+ return false;
+ }
+ for (auto &Target : TargetNames) {
+ auto TargetInfo = OffloadTargetInfo(Target);
+ if (isCodeObjectCompatible(CodeObjectInfo, TargetInfo))
+ CompatibleTargets.push_back(Target);
+ }
+ return !CompatibleTargets.empty();
+}
+
+/// UnbundleArchive takes an archive file (".a") as input containing bundled
+/// code object files, and a list of offload targets (not host), and extracts
+/// the code objects into a new archive file for each offload target. Each
+/// resulting archive file contains all code object files corresponding to that
+/// particular offload target. The created archive file does not
+/// contain an index of the symbols and code object files are named as
+/// <<Parent Bundle Name>-<CodeObject's GPUArch>>, with ':' replaced with '_'.
+static Error UnbundleArchive() {
+ std::vector<std::unique_ptr<MemoryBuffer>> ArchiveBuffers;
+
+ /// Map of target names with list of object files that will form the device
+ /// specific archive for that target
+ StringMap<std::vector<NewArchiveMember>> OutputArchivesMap;
+
+ // Map of target names and output archive filenames
+ StringMap<StringRef> TargetOutputFileNameMap;
+
+ auto Output = OutputFileNames.begin();
+ for (auto &Target : TargetNames) {
+ TargetOutputFileNameMap[Target] = *Output;
+ ++Output;
+ }
+
+ StringRef IFName = InputFileNames.front();
+ ErrorOr<std::unique_ptr<MemoryBuffer>> BufOrErr =
+ MemoryBuffer::getFileOrSTDIN(IFName, -1, false);
+ if (std::error_code EC = BufOrErr.getError())
+ return createFileError(InputFileNames.front(), EC);
+
+ ArchiveBuffers.push_back(std::move(*BufOrErr));
+ Expected<std::unique_ptr<llvm::object::Archive>> LibOrErr =
+ Archive::create(ArchiveBuffers.back()->getMemBufferRef());
+ if (!LibOrErr)
+ return LibOrErr.takeError();
+
+ auto Archive = std::move(*LibOrErr);
+
+ Error ArchiveErr = Error::success();
+ auto ChildEnd = Archive->child_end();
+
+ /// Iterate over all bundled code object files in the input archive.
+ for (auto ArchiveIter = Archive->child_begin(ArchiveErr);
+ ArchiveIter != ChildEnd; ++ArchiveIter) {
+ if (ArchiveErr)
+ return ArchiveErr;
+ auto ArchiveChildNameOrErr = (*ArchiveIter).getName();
+ if (!ArchiveChildNameOrErr)
+ return ArchiveChildNameOrErr.takeError();
+
+ StringRef BundledObjectFile = sys::path::filename(*ArchiveChildNameOrErr);
+
+ auto CodeObjectBufferRefOrErr = (*ArchiveIter).getMemoryBufferRef();
+ if (!CodeObjectBufferRefOrErr)
+ return CodeObjectBufferRefOrErr.takeError();
+
+ auto CodeObjectBuffer =
+ MemoryBuffer::getMemBuffer(*CodeObjectBufferRefOrErr, false);
+
+ Expected<std::unique_ptr<FileHandler>> FileHandlerOrErr =
+ CreateFileHandler(*CodeObjectBuffer);
+ if (!FileHandlerOrErr)
+ return FileHandlerOrErr.takeError();
+
+ std::unique_ptr<FileHandler> &FileHandler = *FileHandlerOrErr;
+ assert(FileHandler &&
+ "FileHandle creation failed for file in the archive!");
+
+ if (Error ReadErr = FileHandler.get()->ReadHeader(*CodeObjectBuffer))
+ return ReadErr;
+
+ Expected<Optional<StringRef>> CurBundleIDOrErr =
+ FileHandler->ReadBundleStart(*CodeObjectBuffer);
+ if (!CurBundleIDOrErr)
+ return CurBundleIDOrErr.takeError();
+
+ Optional<StringRef> OptionalCurBundleID = *CurBundleIDOrErr;
+ // No device code in this child, skip.
+ if (!OptionalCurBundleID.hasValue())
+ continue;
+ StringRef CodeObject = *OptionalCurBundleID;
+
+ // Process all bundle entries (CodeObjects) found in this child of input
+ // archive.
+ while (!CodeObject.empty()) {
+ SmallVector<StringRef> CompatibleTargets;
+ auto CodeObjectInfo = OffloadTargetInfo(CodeObject);
+ if (CodeObjectInfo.hasHostKind()) {
+ // Do nothing, we don't extract host code yet.
+ } else if (getCompatibleOffloadTargets(CodeObjectInfo,
+ CompatibleTargets)) {
+ std::string BundleData;
+ raw_string_ostream DataStream(BundleData);
+ if (Error Err =
+ FileHandler.get()->ReadBundle(DataStream, *CodeObjectBuffer))
+ return Err;
+
+ for (auto &CompatibleTarget : CompatibleTargets) {
+ SmallString<128> BundledObjectFileName;
+ BundledObjectFileName.assign(BundledObjectFile);
+ auto OutputBundleName =
+ Twine(llvm::sys::path::stem(BundledObjectFileName) + "-" +
+ CodeObject)
+ .str();
+ // Replace ':' in optional target feature list with '_' to ensure
+ // cross-platform validity.
+ std::replace(OutputBundleName.begin(), OutputBundleName.end(), ':',
+ '_');
+
+ std::unique_ptr<MemoryBuffer> MemBuf = MemoryBuffer::getMemBufferCopy(
+ DataStream.str(), OutputBundleName);
+ ArchiveBuffers.push_back(std::move(MemBuf));
+ llvm::MemoryBufferRef MemBufRef =
+ MemoryBufferRef(*(ArchiveBuffers.back()));
+
+ // For inserting <CompatibleTarget, list<CodeObject>> entry in
+ // OutputArchivesMap.
+ if (OutputArchivesMap.find(CompatibleTarget) ==
+ OutputArchivesMap.end()) {
+
+ std::vector<NewArchiveMember> ArchiveMembers;
+ ArchiveMembers.push_back(NewArchiveMember(MemBufRef));
+ OutputArchivesMap.insert_or_assign(CompatibleTarget,
+ std::move(ArchiveMembers));
+ } else {
+ OutputArchivesMap[CompatibleTarget].push_back(
+ NewArchiveMember(MemBufRef));
+ }
+ }
+ }
+
+ if (Error Err = FileHandler.get()->ReadBundleEnd(*CodeObjectBuffer))
+ return Err;
+
+ Expected<Optional<StringRef>> NextTripleOrErr =
+ FileHandler->ReadBundleStart(*CodeObjectBuffer);
+ if (!NextTripleOrErr)
+ return NextTripleOrErr.takeError();
+
+ CodeObject = ((*NextTripleOrErr).hasValue()) ? **NextTripleOrErr : "";
+ } // End of processing of all bundle entries of this child of input archive.
+ } // End of while over children of input archive.
+
+ assert(!ArchiveErr && "Error occured while reading archive!");
+
+ /// Write out an archive for each target
+ for (auto &Target : TargetNames) {
+ StringRef FileName = TargetOutputFileNameMap[Target];
+ StringMapIterator<std::vector<llvm::NewArchiveMember>> CurArchiveMembers =
+ OutputArchivesMap.find(Target);
+ if (CurArchiveMembers != OutputArchivesMap.end()) {
+ if (Error WriteErr = writeArchive(FileName, CurArchiveMembers->getValue(),
+ true, getDefaultArchiveKindForHost(),
+ true, false, nullptr))
+ return WriteErr;
+ } else if (!AllowMissingBundles) {
+ std::string ErrMsg =
+ Twine("no compatible code object found for the target '" + Target +
+ "' in heterogenous archive library: " + IFName)
+ .str();
+ return createStringError(inconvertibleErrorCode(), ErrMsg);
+ }
+ }
+
+ return Error::success();
+}
+
static void PrintVersion(raw_ostream &OS) {
OS << clang::getClangToolFullVersion("clang-offload-bundler") << '\n';
}
@@ -1096,6 +1369,11 @@ int main(int argc, const char **argv) {
"match in unbundling mode"));
}
} else {
+ if (FilesType == "a") {
+ reportError(createStringError(errc::invalid_argument,
+ "Archive files are only supported "
+ "for unbundling"));
+ }
if (OutputFileNames.size() != 1) {
reportError(createStringError(
errc::invalid_argument,
@@ -1121,40 +1399,28 @@ int main(int argc, const char **argv) {
}
ParsedTargets.insert(Target);
- StringRef Kind;
- StringRef Triple;
- getOffloadKindAndTriple(Target, Kind, Triple);
-
- bool KindIsValid = !Kind.empty();
- KindIsValid = KindIsValid && StringSwitch<bool>(Kind)
- .Case("host", true)
- .Case("openmp", true)
- .Case("hip", true)
- .Case("hipv4", true)
- .Default(false);
-
- bool TripleIsValid = !Triple.empty();
- llvm::Triple T(Triple);
- TripleIsValid &= T.getArch() != Triple::UnknownArch;
+ auto OffloadInfo = OffloadTargetInfo(Target);
+ bool KindIsValid = OffloadInfo.isOffloadKindValid();
+ bool TripleIsValid = OffloadInfo.isTripleValid();
if (!KindIsValid || !TripleIsValid) {
SmallVector<char, 128u> Buf;
raw_svector_ostream Msg(Buf);
Msg << "invalid target '" << Target << "'";
if (!KindIsValid)
- Msg << ", unknown offloading kind '" << Kind << "'";
+ Msg << ", unknown offloading kind '" << OffloadInfo.OffloadKind << "'";
if (!TripleIsValid)
- Msg << ", unknown target triple '" << Triple << "'";
+ Msg << ", unknown target triple '" << OffloadInfo.Triple.str() << "'";
reportError(createStringError(errc::invalid_argument, Msg.str()));
}
- if (KindIsValid && Kind == "host") {
+ if (KindIsValid && OffloadInfo.hasHostKind()) {
++HostTargetNum;
// Save the index of the input that refers to the host.
HostInputIndex = Index;
}
- if (Kind != "hip" && Kind != "hipv4")
+ if (OffloadInfo.OffloadKind != "hip" && OffloadInfo.OffloadKind != "hipv4")
HIPOnly = false;
++Index;
@@ -1174,6 +1440,14 @@ int main(int argc, const char **argv) {
Twine(HostTargetNum)));
}
- doWork([]() { return Unbundle ? UnbundleFiles() : BundleFiles(); });
+ doWork([]() {
+ if (Unbundle) {
+ if (FilesType == "a")
+ return UnbundleArchive();
+ else
+ return UnbundleFiles();
+ } else
+ return BundleFiles();
+ });
return 0;
}
More information about the cfe-commits
mailing list