[llvm] 2556ba4 - [ObjectYAML] Add support for DXContainer HASH

Chris Bieneman via llvm-commits llvm-commits at lists.llvm.org
Thu Oct 27 10:28:55 PDT 2022


Author: Chris Bieneman
Date: 2022-10-27T12:28:45-05:00
New Revision: 2556ba4a52ac1bca0f2866d70f6ccba776c2b578

URL: https://github.com/llvm/llvm-project/commit/2556ba4a52ac1bca0f2866d70f6ccba776c2b578
DIFF: https://github.com/llvm/llvm-project/commit/2556ba4a52ac1bca0f2866d70f6ccba776c2b578.diff

LOG: [ObjectYAML] Add support for DXContainer HASH

DXContainer files contain a part that has an MD5 of the generated
shader. This adds support to the ObjectYAML tooling to expand the hash
part data and hash iteself in preparation for adding hashing support to
DirectX code generation.

Reviewed By: python3kgae

Differential Revision: https://reviews.llvm.org/D136632

Added: 
    llvm/test/tools/obj2yaml/DXContainer/ShaderHash.yaml

Modified: 
    llvm/include/llvm/BinaryFormat/DXContainer.h
    llvm/include/llvm/BinaryFormat/DXContainerConstants.def
    llvm/include/llvm/Object/DXContainer.h
    llvm/include/llvm/ObjectYAML/DXContainerYAML.h
    llvm/lib/BinaryFormat/DXContainer.cpp
    llvm/lib/Object/DXContainer.cpp
    llvm/lib/ObjectYAML/DXContainerEmitter.cpp
    llvm/lib/ObjectYAML/DXContainerYAML.cpp
    llvm/test/tools/obj2yaml/DXContainer/ExplicitSizeAndOffsets.yaml
    llvm/test/tools/obj2yaml/DXContainer/OmitSizeAndOffsets.yaml
    llvm/test/tools/obj2yaml/DXContainer/ShaderFlags.yaml
    llvm/test/tools/obj2yaml/DXContainer/ShaderFlagsEmpty.yaml
    llvm/tools/obj2yaml/dxcontainer2yaml.cpp

Removed: 
    


################################################################################
diff  --git a/llvm/include/llvm/BinaryFormat/DXContainer.h b/llvm/include/llvm/BinaryFormat/DXContainer.h
index ad921d62ae435..44b77b11fdd38 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainer.h
+++ b/llvm/include/llvm/BinaryFormat/DXContainer.h
@@ -50,6 +50,8 @@ struct ShaderHash {
   uint32_t Flags; // dxbc::HashFlags
   uint8_t Digest[16];
 
+  bool isPopulated();
+
   void swapBytes() { sys::swapByteOrder(Flags); }
 };
 

diff  --git a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
index 43afb47fbdb64..7907bfcc31ea4 100644
--- a/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
+++ b/llvm/include/llvm/BinaryFormat/DXContainerConstants.def
@@ -2,6 +2,7 @@
 #ifdef CONTAINER_PART
 CONTAINER_PART(DXIL)
 CONTAINER_PART(SFI0)
+CONTAINER_PART(HASH)
 
 #undef CONTAINER_PART
 #endif 

diff  --git a/llvm/include/llvm/Object/DXContainer.h b/llvm/include/llvm/Object/DXContainer.h
index 91f46881563e8..5f92b30a17e10 100644
--- a/llvm/include/llvm/Object/DXContainer.h
+++ b/llvm/include/llvm/Object/DXContainer.h
@@ -36,11 +36,13 @@ class DXContainer {
   SmallVector<uint32_t, 4> PartOffsets;
   Optional<DXILData> DXIL;
   Optional<uint64_t> ShaderFlags;
+  Optional<dxbc::ShaderHash> Hash;
 
   Error parseHeader();
   Error parsePartOffsets();
   Error parseDXILHeader(uint32_t Offset);
   Error parseShaderFlags(uint32_t Offset);
+  Error parseHash(uint32_t Offset);
   friend class PartIterator;
 
 public:
@@ -120,6 +122,8 @@ class DXContainer {
   Optional<DXILData> getDXIL() const { return DXIL; }
 
   Optional<uint64_t> getShaderFlags() const { return ShaderFlags; }
+
+  Optional<dxbc::ShaderHash> getShaderHash() const { return Hash; }
 };
 
 } // namespace object

diff  --git a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
index cd966fc9d48cf..49602aa5eda16 100644
--- a/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
+++ b/llvm/include/llvm/ObjectYAML/DXContainerYAML.h
@@ -16,6 +16,7 @@
 #define LLVM_OBJECTYAML_DXCONTAINERYAML_H
 
 #include "llvm/ADT/StringRef.h"
+#include "llvm/BinaryFormat/DXContainer.h"
 #include "llvm/ObjectYAML/YAML.h"
 #include "llvm/Support/YAMLTraits.h"
 #include <cstdint>
@@ -61,6 +62,14 @@ struct ShaderFlags {
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 };
 
+struct ShaderHash {
+  ShaderHash() = default;
+  ShaderHash(const dxbc::ShaderHash &Data);
+
+  bool IncludesSource;
+  std::vector<llvm::yaml::Hex8> Digest;
+};
+
 struct Part {
   Part() = default;
   Part(std::string N, uint32_t S) : Name(N), Size(S) {}
@@ -68,6 +77,7 @@ struct Part {
   uint32_t Size;
   Optional<DXILProgram> Program;
   Optional<ShaderFlags> Flags;
+  Optional<ShaderHash> Hash;
 };
 
 struct Object {
@@ -101,6 +111,10 @@ template <> struct MappingTraits<DXContainerYAML::ShaderFlags> {
   static void mapping(IO &IO, DXContainerYAML::ShaderFlags &Flags);
 };
 
+template <> struct MappingTraits<DXContainerYAML::ShaderHash> {
+  static void mapping(IO &IO, DXContainerYAML::ShaderHash &Hash);
+};
+
 template <> struct MappingTraits<DXContainerYAML::Part> {
   static void mapping(IO &IO, DXContainerYAML::Part &Version);
 };

diff  --git a/llvm/lib/BinaryFormat/DXContainer.cpp b/llvm/lib/BinaryFormat/DXContainer.cpp
index 6704b7b4cd20c..60a89c66d28c8 100644
--- a/llvm/lib/BinaryFormat/DXContainer.cpp
+++ b/llvm/lib/BinaryFormat/DXContainer.cpp
@@ -15,7 +15,7 @@
 #include "llvm/ADT/StringSwitch.h"
 
 using namespace llvm;
-using namespace llvm;
+using namespace llvm::dxbc;
 
 dxbc::PartType dxbc::parsePartType(StringRef S) {
 #define CONTAINER_PART(PartName) .Case(#PartName, PartType::PartName)
@@ -23,3 +23,8 @@ dxbc::PartType dxbc::parsePartType(StringRef S) {
 #include "llvm/BinaryFormat/DXContainerConstants.def"
       .Default(dxbc::PartType::Unknown);
 }
+
+bool ShaderHash::isPopulated() {
+  static uint8_t Zeros[16] = {0};
+  return Flags > 0 || 0 != memcmp(&Digest, &Zeros, 16);
+}

diff  --git a/llvm/lib/Object/DXContainer.cpp b/llvm/lib/Object/DXContainer.cpp
index d9667db706c9f..a83cf70b31741 100644
--- a/llvm/lib/Object/DXContainer.cpp
+++ b/llvm/lib/Object/DXContainer.cpp
@@ -80,6 +80,17 @@ Error DXContainer::parseShaderFlags(uint32_t Offset) {
   return Error::success();
 }
 
+Error DXContainer::parseHash(uint32_t Offset) {
+  if (Hash)
+    return parseFailed("More than one HASH part is present in the file");
+  const char *Current = Data.getBuffer().data() + Offset;
+  dxbc::ShaderHash ReadHash;
+  if (Error Err = readStruct(Data.getBuffer(), Current, ReadHash))
+    return Err;
+  Hash = ReadHash;
+  return Error::success();
+}
+
 Error DXContainer::parsePartOffsets() {
   const char *Current = Data.getBuffer().data() + sizeof(dxbc::Header);
   for (uint32_t Part = 0; Part < Header.PartCount; ++Part) {
@@ -107,6 +118,10 @@ Error DXContainer::parsePartOffsets() {
       if (Error Err = parseShaderFlags(PartOffset + sizeof(dxbc::PartHeader)))
         return Err;
       break;
+    case dxbc::PartType::HASH:
+      if (Error Err = parseHash(PartOffset + sizeof(dxbc::PartHeader)))
+        return Err;
+      break;
     case dxbc::PartType::Unknown:
       break;
     }

diff  --git a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
index 32b4f0bd71a05..1dc5111959b4b 100644
--- a/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
+++ b/llvm/lib/ObjectYAML/DXContainerEmitter.cpp
@@ -125,7 +125,8 @@ void DXContainerWriter::writeParts(raw_ostream &OS) {
     dxbc::PartType PT = dxbc::parsePartType(P.Name);
 
     uint64_t DataStart = OS.tell();
-    if (PT == dxbc::PartType::DXIL) {
+    switch (PT) {
+    case dxbc::PartType::DXIL: {
       if (!P.Program)
         continue;
       dxbc::ProgramHeader Header;
@@ -167,7 +168,9 @@ void DXContainerWriter::writeParts(raw_ostream &OS) {
         OS.write(reinterpret_cast<char *>(P.Program->DXIL->data()),
                  P.Program->DXIL->size());
       }
-    } else if (PT == dxbc::PartType::SFI0) {
+      break;
+    }
+    case dxbc::PartType::SFI0: {
       // If we don't have any flags we can continue here and the data will be
       // zeroed out.
       if (!P.Flags.has_value())
@@ -176,6 +179,22 @@ void DXContainerWriter::writeParts(raw_ostream &OS) {
       if (sys::IsBigEndianHost)
         sys::swapByteOrder(Flags);
       OS.write(reinterpret_cast<char *>(&Flags), sizeof(uint64_t));
+      break;
+    }
+    case dxbc::PartType::HASH: {
+      if (!P.Hash.has_value())
+        continue;
+      dxbc::ShaderHash Hash = {0, {0}};
+      if (P.Hash->IncludesSource)
+        Hash.Flags |= static_cast<uint32_t>(dxbc::HashFlags::IncludesSource);
+      memcpy(&Hash.Digest[0], &P.Hash->Digest[0], 16);
+      if (sys::IsBigEndianHost)
+        Hash.swapBytes();
+      OS.write(reinterpret_cast<char *>(&Hash), sizeof(dxbc::ShaderHash));
+      break;
+    }
+    case dxbc::PartType::Unknown:
+      break; // Skip any handling for unrecognized parts.
     }
     uint64_t BytesWritten = OS.tell() - DataStart;
     RollingOffset += BytesWritten;

diff  --git a/llvm/lib/ObjectYAML/DXContainerYAML.cpp b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
index d9cc7f5f6bd18..1d1dd42d93f18 100644
--- a/llvm/lib/ObjectYAML/DXContainerYAML.cpp
+++ b/llvm/lib/ObjectYAML/DXContainerYAML.cpp
@@ -36,6 +36,13 @@ uint64_t DXContainerYAML::ShaderFlags::getEncodedFlags() {
   return Flag;
 }
 
+DXContainerYAML::ShaderHash::ShaderHash(const dxbc::ShaderHash &Data)
+    : IncludesSource((Data.Flags & static_cast<uint32_t>(
+                                       dxbc::HashFlags::IncludesSource)) != 0),
+      Digest(16, 0) {
+  memcpy(Digest.data(), &Data.Digest[0], 16);
+}
+
 namespace yaml {
 
 void MappingTraits<DXContainerYAML::VersionTuple>::mapping(
@@ -71,12 +78,19 @@ void MappingTraits<DXContainerYAML::ShaderFlags>::mapping(
 #include "llvm/BinaryFormat/DXContainerConstants.def"
 }
 
+void MappingTraits<DXContainerYAML::ShaderHash>::mapping(
+    IO &IO, DXContainerYAML::ShaderHash &Hash) {
+  IO.mapRequired("IncludesSource", Hash.IncludesSource);
+  IO.mapRequired("Digest", Hash.Digest);
+}
+
 void MappingTraits<DXContainerYAML::Part>::mapping(IO &IO,
                                                    DXContainerYAML::Part &P) {
   IO.mapRequired("Name", P.Name);
   IO.mapRequired("Size", P.Size);
   IO.mapOptional("Program", P.Program);
   IO.mapOptional("Flags", P.Flags);
+  IO.mapOptional("Hash", P.Hash);
 }
 
 void MappingTraits<DXContainerYAML::Object>::mapping(

diff  --git a/llvm/test/tools/obj2yaml/DXContainer/ExplicitSizeAndOffsets.yaml b/llvm/test/tools/obj2yaml/DXContainer/ExplicitSizeAndOffsets.yaml
index f807aeffc5dd5..040aeb014fd87 100644
--- a/llvm/test/tools/obj2yaml/DXContainer/ExplicitSizeAndOffsets.yaml
+++ b/llvm/test/tools/obj2yaml/DXContainer/ExplicitSizeAndOffsets.yaml
@@ -7,9 +7,9 @@ Header:
   Version:
     Major:           1
     Minor:           0
-  FileSize:        172
+  FileSize:        184
   PartCount:       7
-  PartOffsets:     [ 60, 76, 92, 108, 124, 140, 156 ]
+  PartOffsets:     [ 60, 76, 92, 108, 124, 140, 168 ]
 Parts:
   - Name:            SFI0
     Size:            8
@@ -22,7 +22,7 @@ Parts:
   - Name:            STAT
     Size:            8
   - Name:            HASH
-    Size:            8
+    Size:            20
   - Name:            CXIL
     Size:            8
 ...
@@ -34,9 +34,9 @@ Parts:
 # CHECK-NEXT:   Version:
 # CHECK-NEXT:     Major:           1
 # CHECK-NEXT:     Minor:           0
-# CHECK-NEXT:   FileSize:        172
+# CHECK-NEXT:   FileSize:        184
 # CHECK-NEXT:   PartCount:       7
-# CHECK-NEXT:   PartOffsets:     [ 60, 76, 92, 108, 124, 140, 156 ]
+# CHECK-NEXT:   PartOffsets:     [ 60, 76, 92, 108, 124, 140, 168 ]
 # CHECK-NEXT: Parts:
 # CHECK-NEXT:   - Name:            SFI0
 # CHECK-NEXT:     Size:            8
@@ -49,7 +49,7 @@ Parts:
 # CHECK-NEXT:   - Name:            STAT
 # CHECK-NEXT:     Size:            8
 # CHECK-NEXT:   - Name:            HASH
-# CHECK-NEXT:     Size:            8
+# CHECK-NEXT:     Size:            20
 # CHECK-NEXT:   - Name:            CXIL
 # CHECK-NEXT:     Size:            8
 # CHECK-NEXT: ...

diff  --git a/llvm/test/tools/obj2yaml/DXContainer/OmitSizeAndOffsets.yaml b/llvm/test/tools/obj2yaml/DXContainer/OmitSizeAndOffsets.yaml
index 3930afab0789f..62efd413533c1 100644
--- a/llvm/test/tools/obj2yaml/DXContainer/OmitSizeAndOffsets.yaml
+++ b/llvm/test/tools/obj2yaml/DXContainer/OmitSizeAndOffsets.yaml
@@ -20,7 +20,7 @@ Parts:
   - Name:            STAT
     Size:            8
   - Name:            HASH
-    Size:            8
+    Size:            20
   - Name:            CXIL
     Size:            8
 ...
@@ -32,9 +32,9 @@ Parts:
 # CHECK-NEXT:   Version:
 # CHECK-NEXT:     Major:           1
 # CHECK-NEXT:     Minor:           0
-# CHECK-NEXT:   FileSize:        172
+# CHECK-NEXT:   FileSize:        184
 # CHECK-NEXT:   PartCount:       7
-# CHECK-NEXT:   PartOffsets:     [ 60, 76, 92, 108, 124, 140, 156 ]
+# CHECK-NEXT:   PartOffsets:     [ 60, 76, 92, 108, 124, 140, 168 ]
 # CHECK-NEXT: Parts:
 # CHECK-NEXT:   - Name:            SFI0
 # CHECK-NEXT:     Size:            8
@@ -47,7 +47,7 @@ Parts:
 # CHECK-NEXT:   - Name:            STAT
 # CHECK-NEXT:     Size:            8
 # CHECK-NEXT:   - Name:            HASH
-# CHECK-NEXT:     Size:            8
+# CHECK-NEXT:     Size:            20
 # CHECK-NEXT:   - Name:            CXIL
 # CHECK-NEXT:     Size:            8
 # CHECK-NEXT: ...

diff  --git a/llvm/test/tools/obj2yaml/DXContainer/ShaderFlags.yaml b/llvm/test/tools/obj2yaml/DXContainer/ShaderFlags.yaml
index 93e884ca3473e..2a5a3ff9cd2a3 100644
--- a/llvm/test/tools/obj2yaml/DXContainer/ShaderFlags.yaml
+++ b/llvm/test/tools/obj2yaml/DXContainer/ShaderFlags.yaml
@@ -54,7 +54,7 @@ Parts:
   - Name:            STAT
     Size:            8
   - Name:            HASH
-    Size:            8
+    Size:            20
   - Name:            CXIL
     Size:            8
 ...

diff  --git a/llvm/test/tools/obj2yaml/DXContainer/ShaderFlagsEmpty.yaml b/llvm/test/tools/obj2yaml/DXContainer/ShaderFlagsEmpty.yaml
index 1bbc5d2ac30af..ef73eb829157b 100644
--- a/llvm/test/tools/obj2yaml/DXContainer/ShaderFlagsEmpty.yaml
+++ b/llvm/test/tools/obj2yaml/DXContainer/ShaderFlagsEmpty.yaml
@@ -53,7 +53,7 @@ Parts:
   - Name:            STAT
     Size:            8
   - Name:            HASH
-    Size:            8
+    Size:            20
   - Name:            CXIL
     Size:            8
 ...

diff  --git a/llvm/test/tools/obj2yaml/DXContainer/ShaderHash.yaml b/llvm/test/tools/obj2yaml/DXContainer/ShaderHash.yaml
new file mode 100644
index 0000000000000..5809434de424d
--- /dev/null
+++ b/llvm/test/tools/obj2yaml/DXContainer/ShaderHash.yaml
@@ -0,0 +1,33 @@
+# RUN: yaml2obj %s | obj2yaml | FileCheck %s 
+
+--- !dxcontainer
+Header:
+  Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+                     0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+  Version:
+    Major:           1
+    Minor:           0
+  PartCount:       1
+Parts:
+  - Name:            HASH
+    Size:            20
+    Hash:
+      IncludesSource:  true
+      Digest:          [ 0x00, 0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 
+                         0x10, 0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80 ]
+...
+
+# CHECK: --- !dxcontainer
+# CHECK-NEXT: Header:
+# CHECK-NEXT:   Hash:            [ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 
+# CHECK-NEXT:                      0x0, 0x0, 0x0, 0x0, 0x0, 0x0 ]
+# CHECK:        FileSize:        64
+# CHECK-NEXT:   PartCount:       1
+# CHECK-NEXT:   PartOffsets:     [ 36 ]
+# CHECK:        - Name:            HASH
+# CHECK-NEXT:     Size:            20
+# CHECK-NEXT:     Hash:
+# CHECK-NEXT:       IncludesSource:  true
+# CHECK-NEXT:       Digest:          [ 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x10,
+# CHECK-NEXT:                          0x20, 0x30, 0x40, 0x50, 0x60, 0x70, 0x80 ]
+# CHECK-NEXT: ...

diff  --git a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
index d8036babb727b..61b6a29025284 100644
--- a/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
+++ b/llvm/tools/obj2yaml/dxcontainer2yaml.cpp
@@ -67,6 +67,12 @@ dumpDXContainer(MemoryBufferRef Source) {
         NewPart.Flags = DXContainerYAML::ShaderFlags(*Flags);
       break;
     }
+    case dxbc::PartType::HASH: {
+      Optional<dxbc::ShaderHash> Hash = Container.getShaderHash();
+      if (Hash && Hash->isPopulated())
+        NewPart.Hash = DXContainerYAML::ShaderHash(*Hash);
+      break;
+    }
     case dxbc::PartType::Unknown:
       break;
     }


        


More information about the llvm-commits mailing list