[llvm] [LLVM-Reduce] - Distinct Metadata Reduction (PR #104624)
Robert Barinov via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 16 11:18:51 PDT 2024
https://github.com/rbintel updated https://github.com/llvm/llvm-project/pull/104624
>From eba4fae1b1e1a7727443f3822a823c7293246da4 Mon Sep 17 00:00:00 2001
From: "Barinov, Robert" <robert.barinov at intel.com>
Date: Fri, 16 Aug 2024 19:50:06 +0200
Subject: [PATCH 1/2] - Add a distinct metadata reduction pass, which traverses
the whole unnamed metadata tree and applies reduction where possible.
Previous version could do this only partially, either removing named
metadata, metadata attached to instructions or debug information.
- Add the --aggressive-md flag, which allows the reducer to reduce
named metadata operands more aggressively.
- Add tests for the new functionality.
---
.../Inputs/reduce-distinct-metadata.py | 19 +++
.../llvm-reduce/Inputs/remove-metadata.py | 0
.../llvm-reduce/reduce-distinct-metadata.ll | 28 ++++
.../llvm-reduce/reduce-named-metadata.ll | 3 -
.../test/tools/llvm-reduce/remove-metadata.ll | 7 +-
llvm/tools/llvm-reduce/CMakeLists.txt | 1 +
llvm/tools/llvm-reduce/DeltaManager.cpp | 4 +-
.../deltas/ReduceDistinctMetadata.cpp | 147 ++++++++++++++++++
.../deltas/ReduceDistinctMetadata.h | 23 +++
.../llvm-reduce/deltas/ReduceMetadata.cpp | 33 ++--
10 files changed, 248 insertions(+), 17 deletions(-)
create mode 100644 llvm/test/tools/llvm-reduce/Inputs/reduce-distinct-metadata.py
mode change 100755 => 100644 llvm/test/tools/llvm-reduce/Inputs/remove-metadata.py
create mode 100644 llvm/test/tools/llvm-reduce/reduce-distinct-metadata.ll
create mode 100644 llvm/tools/llvm-reduce/deltas/ReduceDistinctMetadata.cpp
create mode 100644 llvm/tools/llvm-reduce/deltas/ReduceDistinctMetadata.h
diff --git a/llvm/test/tools/llvm-reduce/Inputs/reduce-distinct-metadata.py b/llvm/test/tools/llvm-reduce/Inputs/reduce-distinct-metadata.py
new file mode 100644
index 00000000000000..605b4f825c3e69
--- /dev/null
+++ b/llvm/test/tools/llvm-reduce/Inputs/reduce-distinct-metadata.py
@@ -0,0 +1,19 @@
+# Helper script for distinct metadata reduction test
+
+import sys
+import re
+
+input = open(sys.argv[1], "r").read().splitlines()
+
+depth_map = {"0": 1, "1": 3, "2": 3, "3": 2, "4": 1}
+
+
+for i in range(len(depth_map)):
+ counter = 0
+ for line in input:
+ if re.match(rf".*interesting_{i}.*", line) != None:
+ counter += 1
+ if counter != depth_map[str(i)]:
+ sys.exit(1)
+
+sys.exit(0)
diff --git a/llvm/test/tools/llvm-reduce/Inputs/remove-metadata.py b/llvm/test/tools/llvm-reduce/Inputs/remove-metadata.py
old mode 100755
new mode 100644
diff --git a/llvm/test/tools/llvm-reduce/reduce-distinct-metadata.ll b/llvm/test/tools/llvm-reduce/reduce-distinct-metadata.ll
new file mode 100644
index 00000000000000..2ad0d8ea3e522e
--- /dev/null
+++ b/llvm/test/tools/llvm-reduce/reduce-distinct-metadata.ll
@@ -0,0 +1,28 @@
+; Test that every boring node is removed and all interesting distinct nodes remain after aggressive distinct metadata reduction.
+
+; RUN: llvm-reduce --aggressive-md --test %python --test-arg %p/Inputs/reduce-distinct-metadata.py %s -o %t
+; RUN: FileCheck %s < %t
+
+; CHECK-NOT: {{.*}}boring{{.*}}
+
+define void @main() {
+ ret void
+}
+
+!named.metadata = !{!0, !2}
+!llvm.test.other.metadata = !{}
+
+!0 = distinct !{!"interesting_0", !1, !3, !4, !10, !11}
+!1 = distinct !{!"interesting_1", !5, !7, !"something"}
+!2 = distinct !{!"boring_0", !3, !4, i32 5}
+!3 = distinct !{!"interesting_1", !3, !4}
+!4 = distinct !{!"interesting_1", !6, i2 1}
+!5 = distinct !{!"interesting_2", !8}
+!6 = distinct !{!"interesting_2", !10}
+!7 = distinct !{!"interesting_2", !12}
+!8 = distinct !{!"interesting_3", !10, !9}
+!9 = distinct !{!"interesting_3", !11, !13}
+!10 = distinct !{!"boring_1", i32 50}
+!11 = distinct !{!"boring_1", i32 2}
+!12 = distinct !{!"boring_3", i2 1}
+!13 = distinct !{!"interesting_4"}
diff --git a/llvm/test/tools/llvm-reduce/reduce-named-metadata.ll b/llvm/test/tools/llvm-reduce/reduce-named-metadata.ll
index 88b60560507b4b..9cc8ad4f645113 100644
--- a/llvm/test/tools/llvm-reduce/reduce-named-metadata.ll
+++ b/llvm/test/tools/llvm-reduce/reduce-named-metadata.ll
@@ -23,7 +23,6 @@
; RESULT: !opencl.used.extensions = !{![[OCL_EXTENSION:[0-9]+]]}
; RESULT: !opencl.used.optional.core.features = !{![[OCL_OPTIONAL_CORE_FEATURE:[0-9]+]]}
; RESULT: !opencl.compiler.options = !{![[OCL_COMPILER_OPTION:[0-9]+]]}
-; RESULT: !some.unknown.named = !{![[UNKNOWN_0:[0-9]+]], ![[UNKNOWN_1:[0-9]+]]}
; RESULT: ![[LLVM_IDENT]] = !{!"some llvm version 0"}
@@ -32,8 +31,6 @@
; RESULT: ![[OCL_EXTENSION]] = !{!"some ocl extension 1"}
; RESULT: ![[OCL_OPTIONAL_CORE_FEATURE]] = !{!"some ocl optional core feature 1"}
; RESULT: ![[OCL_COMPILER_OPTION]] = !{!"some ocl compiler option 1"}
-; RESULT: ![[UNKNOWN_0]] = !{!"some unknown option 0"}
-; RESULT: ![[UNKNOWN_1]] = !{!"some unknown option 1"}
!llvm.ident = !{!0, !1, !0}
!opencl.spir.version = !{!2, !3}
diff --git a/llvm/test/tools/llvm-reduce/remove-metadata.ll b/llvm/test/tools/llvm-reduce/remove-metadata.ll
index 51a50ca20a9854..d035e7532aa7fd 100644
--- a/llvm/test/tools/llvm-reduce/remove-metadata.ll
+++ b/llvm/test/tools/llvm-reduce/remove-metadata.ll
@@ -1,8 +1,8 @@
; Test that llvm-reduce can remove uninteresting metadata from an IR file.
; The Metadata pass erases named & unnamed metadata nodes.
;
-; RUN: llvm-reduce --test %python --test-arg %p/Inputs/remove-metadata.py %s -o %t
-; RUN: cat %t | FileCheck -implicit-check-not=! %s
+; RUN: llvm-reduce --aggressive-md --test %python --test-arg %p/Inputs/remove-metadata.py %s -o %t
+; RUN: FileCheck -implicit-check-not=! %s < %t
@global = global i32 0, !dbg !0
@@ -11,9 +11,8 @@ define void @main() !dbg !0 {
}
!uninteresting = !{!0}
-; CHECK: !interesting = !{!0}
+; CHECK: !interesting = !{}
!interesting = !{!1}
!0 = !{!"uninteresting"}
-; CHECK: !0 = !{!"interesting"}
!1 = !{!"interesting"}
diff --git a/llvm/tools/llvm-reduce/CMakeLists.txt b/llvm/tools/llvm-reduce/CMakeLists.txt
index a4c605fcd2443c..b8ad6f71b41e59 100644
--- a/llvm/tools/llvm-reduce/CMakeLists.txt
+++ b/llvm/tools/llvm-reduce/CMakeLists.txt
@@ -31,6 +31,7 @@ add_llvm_tool(llvm-reduce
deltas/ReduceAttributes.cpp
deltas/ReduceBasicBlocks.cpp
deltas/ReduceDIMetadata.cpp
+ deltas/ReduceDistinctMetadata.cpp
deltas/ReduceDbgRecords.cpp
deltas/ReduceFunctionBodies.cpp
deltas/ReduceFunctions.cpp
diff --git a/llvm/tools/llvm-reduce/DeltaManager.cpp b/llvm/tools/llvm-reduce/DeltaManager.cpp
index 67fbc2fdc7ad43..624b5306bc71be 100644
--- a/llvm/tools/llvm-reduce/DeltaManager.cpp
+++ b/llvm/tools/llvm-reduce/DeltaManager.cpp
@@ -21,6 +21,7 @@
#include "deltas/ReduceBasicBlocks.h"
#include "deltas/ReduceDIMetadata.h"
#include "deltas/ReduceDbgRecords.h"
+#include "deltas/ReduceDistinctMetadata.h"
#include "deltas/ReduceFunctionBodies.h"
#include "deltas/ReduceFunctions.h"
#include "deltas/ReduceGlobalObjects.h"
@@ -93,6 +94,7 @@ static cl::list<std::string>
DELTA_PASS("global-variables", reduceGlobalsDeltaPass) \
DELTA_PASS("di-metadata", reduceDIMetadataDeltaPass) \
DELTA_PASS("dbg-records", reduceDbgRecordDeltaPass) \
+ DELTA_PASS("distinct-metadata", reduceDistinctMetadataDeltaPass) \
DELTA_PASS("metadata", reduceMetadataDeltaPass) \
DELTA_PASS("named-metadata", reduceNamedMetadataDeltaPass) \
DELTA_PASS("arguments", reduceArgumentsDeltaPass) \
@@ -112,7 +114,7 @@ static cl::list<std::string>
DELTA_PASS("atomic-ordering", reduceAtomicOrderingDeltaPass) \
DELTA_PASS("syncscopes", reduceAtomicSyncScopesDeltaPass) \
DELTA_PASS("instruction-flags", reduceInstructionFlagsDeltaPass) \
-} while (false)
+ } while (false)
#define DELTA_PASSES_MIR \
do { \
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceDistinctMetadata.cpp b/llvm/tools/llvm-reduce/deltas/ReduceDistinctMetadata.cpp
new file mode 100644
index 00000000000000..6563b210b0a10e
--- /dev/null
+++ b/llvm/tools/llvm-reduce/deltas/ReduceDistinctMetadata.cpp
@@ -0,0 +1,147 @@
+//===- ReduceDistinctMetadata.cpp - Specialized Delta Pass ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===------------------------------------------------------------------------------===//
+//
+// This file implements two functions used by the Generic Delta Debugging
+// Algorithm, which are used to reduce unnamed distinct metadata nodes.
+//
+//===------------------------------------------------------------------------------===//
+
+#include "ReduceDistinctMetadata.h"
+#include "Delta.h"
+#include "llvm/ADT/Sequence.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/IR/InstIterator.h"
+#include <algorithm>
+#include <queue>
+
+using namespace llvm;
+
+// Traverse the graph breadth-first and try to remove unnamed metadata nodes
+void reduceNodes(MDNode *Root,
+ SetVector<std::pair<unsigned int, MDNode *>> &NodesToDelete,
+ MDNode *TemporaryNode, Oracle &O, Module &Program) {
+ std::queue<MDNode *> NodesToTraverse{};
+ // Keep track of visited nodes not to get into loops
+ SetVector<MDNode *> VisitedNodes{};
+ NodesToTraverse.push(Root);
+
+ while (!NodesToTraverse.empty()) {
+ MDNode *CurrentNode = NodesToTraverse.front();
+ NodesToTraverse.pop();
+
+ // Mark the nodes for removal
+ for (unsigned int I = 0; I < CurrentNode->getNumOperands(); ++I) {
+ if (MDNode *Operand =
+ dyn_cast<MDNode>(CurrentNode->getOperand(I).get())) {
+ // Check whether node has been visited
+ if (!VisitedNodes.contains(Operand)) {
+ NodesToTraverse.push(Operand);
+ VisitedNodes.insert(Operand);
+ }
+ // Delete the node only if it is distinct
+ if (Operand->isDistinct())
+ // Add to removal list
+ NodesToDelete.insert(std::make_pair(I, CurrentNode));
+ }
+ }
+
+ // Remove the nodes
+ for (auto [PositionToReplace, Node] : NodesToDelete) {
+ if (!O.shouldKeep())
+ Node->replaceOperandWith(PositionToReplace, TemporaryNode);
+ }
+ NodesToDelete.clear();
+ }
+}
+
+// After reducing metadata, we need to remove references to the temporary node,
+// this is also done with BFS
+void cleanUpTemporaries(NamedMDNode &NamedNode, MDTuple *TemporaryTuple,
+ Module &Program) {
+ std::queue<MDTuple *> NodesToTraverse{};
+ SetVector<MDTuple *> VisitedNodes{};
+
+ // Push all first level operands of the named node to the queue
+ for (auto I = NamedNode.op_begin(); I != NamedNode.op_end(); ++I) {
+ // If the node hasn't been traversed yet, add it to the queue of nodes to
+ // traverse.
+ if (MDTuple *TupleI = dyn_cast<MDTuple>((*I))) {
+ if (!VisitedNodes.contains(TupleI)) {
+ NodesToTraverse.push(TupleI);
+ VisitedNodes.insert(TupleI);
+ }
+ }
+ }
+
+ while (!NodesToTraverse.empty()) {
+ MDTuple *CurrentTuple = NodesToTraverse.front();
+ NodesToTraverse.pop();
+
+ // Shift all of the interesting elements to the left, pop remaining
+ // afterwards
+ if (CurrentTuple
+ ->isDistinct()) { // Do resizing and cleaning operations only if
+ // the node is distinct, as resizing is not
+ // supported for unique nodes and is redundant.
+ unsigned int NotToRemove = 0;
+ for (unsigned int I = 0; I < CurrentTuple->getNumOperands(); ++I) {
+ Metadata *Operand = CurrentTuple->getOperand(I).get();
+ // If current operand is not the temporary node, move it to the front
+ // and increase notToRemove so that it will be saved
+ if (Operand != TemporaryTuple) {
+ Metadata *TemporaryMetadata =
+ CurrentTuple->getOperand(NotToRemove).get();
+ CurrentTuple->replaceOperandWith(NotToRemove, Operand);
+ CurrentTuple->replaceOperandWith(I, TemporaryMetadata);
+ ++NotToRemove;
+ }
+ }
+
+ // Remove all the uninteresting elements
+ unsigned int OriginalOperands = CurrentTuple->getNumOperands();
+ for (unsigned int I = 0; I < OriginalOperands - NotToRemove; ++I)
+ CurrentTuple->pop_back();
+ }
+
+ // Push the remaining nodes into the queue
+ for (unsigned int I = 0; I < CurrentTuple->getNumOperands(); ++I) {
+ MDTuple *Operand = dyn_cast<MDTuple>(CurrentTuple->getOperand(I).get());
+ if (Operand && !VisitedNodes.contains(Operand)) {
+ NodesToTraverse.push(Operand);
+ // If the node hasn't been traversed yet, add it to the queue of nodes
+ // to traverse.
+ VisitedNodes.insert(Operand);
+ }
+ }
+ }
+}
+
+static void extractDistinctMetadataFromModule(Oracle &O,
+ ReducerWorkItem &WorkItem) {
+ Module &Program = WorkItem.getModule();
+ MDTuple *TemporaryTuple = MDTuple::getDistinct(
+ Program.getContext(), SmallVector<Metadata *, 1>{llvm::MDString::get(
+ Program.getContext(), "temporary_tuple")});
+ SetVector<std::pair<unsigned int, MDNode *>> NodesToDelete{};
+ for (NamedMDNode &NamedNode :
+ Program.named_metadata()) { // Iterate over the named nodes
+ for (unsigned int I = 0; I < NamedNode.getNumOperands();
+ ++I) { // Iterate over first level unnamed nodes..
+ if (MDTuple *Operand = dyn_cast<MDTuple>(NamedNode.getOperand(I)))
+ reduceNodes(Operand, NodesToDelete, TemporaryTuple, O, Program);
+ }
+ }
+ for (NamedMDNode &NamedNode : Program.named_metadata())
+ cleanUpTemporaries(NamedNode, TemporaryTuple, Program);
+}
+
+void llvm::reduceDistinctMetadataDeltaPass(TestRunner &Test) {
+ runDeltaPass(Test, extractDistinctMetadataFromModule,
+ "Reducing Distinct Metadata");
+}
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceDistinctMetadata.h b/llvm/tools/llvm-reduce/deltas/ReduceDistinctMetadata.h
new file mode 100644
index 00000000000000..d02e8e6107b756
--- /dev/null
+++ b/llvm/tools/llvm-reduce/deltas/ReduceDistinctMetadata.h
@@ -0,0 +1,23 @@
+//===- ReduceDistinctMetadata.h - Specialized Delta Pass ------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------------===//
+//
+// This file implements two functions used by the Generic Delta Debugging
+// Algorithm, which are used to reduce Metadata nodes.
+//
+//===----------------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVM_REDUCE_DELTAS_REDUCEDISTINCTMETADATA_H
+#define LLVM_TOOLS_LLVM_REDUCE_DELTAS_REDUCEDISTINCTMETADATA_H
+
+#include "TestRunner.h"
+
+namespace llvm {
+void reduceDistinctMetadataDeltaPass(TestRunner &Test);
+} // namespace llvm
+
+#endif
diff --git a/llvm/tools/llvm-reduce/deltas/ReduceMetadata.cpp b/llvm/tools/llvm-reduce/deltas/ReduceMetadata.cpp
index 30bf612fe9694f..ca5459616fdca5 100644
--- a/llvm/tools/llvm-reduce/deltas/ReduceMetadata.cpp
+++ b/llvm/tools/llvm-reduce/deltas/ReduceMetadata.cpp
@@ -20,6 +20,13 @@
using namespace llvm;
+extern cl::OptionCategory LLVMReduceOptions;
+
+static cl::opt<bool> AggressiveMetadataReduction(
+ "aggressive-md",
+ cl::desc("Reduce named metadata without taking its type into account"),
+ cl::cat(LLVMReduceOptions));
+
static bool shouldKeepDebugIntrinsicMetadata(Instruction &I, MDNode &MD) {
return isa<DILocation>(MD) && isa<DbgInfoIntrinsic>(I);
}
@@ -44,24 +51,32 @@ static constexpr StringLiteral ListNamedMetadata[] = {
static void reduceNamedMetadataOperands(Oracle &O, ReducerWorkItem &WorkItem) {
Module &M = WorkItem.getModule();
- for (StringRef MDName : ListNamedMetadata) {
- NamedMDNode *NamedNode = M.getNamedMetadata(MDName);
- if (!NamedNode)
- continue;
+ for (NamedMDNode &I : M.named_metadata()) {
+ // If we don't want to reduce mindlessly, check if our node is part of
+ // ListNamedMetadata before reducing it
+ if (!AggressiveMetadataReduction) {
+ bool Found = false;
+ for (StringRef MDName : ListNamedMetadata) {
+ if (I.getName() == MDName)
+ Found = true;
+ }
+ if (!Found)
+ continue;
+ }
bool MadeChange = false;
- SmallVector<MDNode *, 16> KeptOperands;
- for (auto I : seq<unsigned>(0, NamedNode->getNumOperands())) {
+ SmallVector<MDNode *> KeptOperands;
+ for (auto J : seq<unsigned>(0, I.getNumOperands())) {
if (O.shouldKeep())
- KeptOperands.push_back(NamedNode->getOperand(I));
+ KeptOperands.push_back(I.getOperand(J));
else
MadeChange = true;
}
if (MadeChange) {
- NamedNode->clearOperands();
+ I.clearOperands();
for (MDNode *KeptOperand : KeptOperands)
- NamedNode->addOperand(KeptOperand);
+ I.addOperand(KeptOperand);
}
}
}
>From a9bd5bd503620fc9aae3038a2038dc171afa5d6f Mon Sep 17 00:00:00 2001
From: Robert Barinov <165884206+rbintel at users.noreply.github.com>
Date: Fri, 16 Aug 2024 20:18:42 +0200
Subject: [PATCH 2/2] Revert changes made on this test
---
llvm/test/tools/llvm-reduce/reduce-named-metadata.ll | 3 +++
1 file changed, 3 insertions(+)
diff --git a/llvm/test/tools/llvm-reduce/reduce-named-metadata.ll b/llvm/test/tools/llvm-reduce/reduce-named-metadata.ll
index 9cc8ad4f645113..88b60560507b4b 100644
--- a/llvm/test/tools/llvm-reduce/reduce-named-metadata.ll
+++ b/llvm/test/tools/llvm-reduce/reduce-named-metadata.ll
@@ -23,6 +23,7 @@
; RESULT: !opencl.used.extensions = !{![[OCL_EXTENSION:[0-9]+]]}
; RESULT: !opencl.used.optional.core.features = !{![[OCL_OPTIONAL_CORE_FEATURE:[0-9]+]]}
; RESULT: !opencl.compiler.options = !{![[OCL_COMPILER_OPTION:[0-9]+]]}
+; RESULT: !some.unknown.named = !{![[UNKNOWN_0:[0-9]+]], ![[UNKNOWN_1:[0-9]+]]}
; RESULT: ![[LLVM_IDENT]] = !{!"some llvm version 0"}
@@ -31,6 +32,8 @@
; RESULT: ![[OCL_EXTENSION]] = !{!"some ocl extension 1"}
; RESULT: ![[OCL_OPTIONAL_CORE_FEATURE]] = !{!"some ocl optional core feature 1"}
; RESULT: ![[OCL_COMPILER_OPTION]] = !{!"some ocl compiler option 1"}
+; RESULT: ![[UNKNOWN_0]] = !{!"some unknown option 0"}
+; RESULT: ![[UNKNOWN_1]] = !{!"some unknown option 1"}
!llvm.ident = !{!0, !1, !0}
!opencl.spir.version = !{!2, !3}
More information about the llvm-commits
mailing list