[llvm-branch-commits] [llvm] [Analysis] Consolidate metadata utilities (NFC) (PR #243671)
Yonah Goldberg via llvm-branch-commits
llvm-branch-commits at lists.llvm.org
Tue Nov 17 17:02:32 PST 2026
https://github.com/YonahGoldberg created https://github.com/llvm/llvm-project/pull/243671
None
>From 76cb1d8d02c40815a21be6cf2dcda163baacaee7 Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 00:56:46 +0000
Subject: [PATCH 1/2] [Analysis] Consolidate metadata utilities (NFC)
---
llvm/include/llvm/Analysis/MetadataUtils.h | 88 ++++
llvm/include/llvm/Analysis/VectorUtils.h | 28 -
llvm/include/llvm/Transforms/Utils/Local.h | 33 +-
llvm/lib/Analysis/CMakeLists.txt | 1 +
llvm/lib/Analysis/MetadataUtils.cpp | 493 ++++++++++++++++++
llvm/lib/Analysis/VectorUtils.cpp | 163 +-----
llvm/lib/Transforms/Utils/Local.cpp | 306 +----------
.../Vectorize/LoadStoreVectorizer.cpp | 1 +
.../Transforms/Vectorize/SLPVectorizer.cpp | 1 +
llvm/lib/Transforms/Vectorize/VPlan.h | 1 +
10 files changed, 588 insertions(+), 527 deletions(-)
create mode 100644 llvm/include/llvm/Analysis/MetadataUtils.h
create mode 100644 llvm/lib/Analysis/MetadataUtils.cpp
diff --git a/llvm/include/llvm/Analysis/MetadataUtils.h b/llvm/include/llvm/Analysis/MetadataUtils.h
new file mode 100644
index 00000000000000..a7afe9514e0a8e
--- /dev/null
+++ b/llvm/include/llvm/Analysis/MetadataUtils.h
@@ -0,0 +1,88 @@
+//===- MetadataUtils.h - Metadata utilities ---------------------*- C++ -*-===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef LLVM_ANALYSIS_METADATAUTILS_H
+#define LLVM_ANALYSIS_METADATAUTILS_H
+
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallVector.h"
+#include "llvm/Support/Compiler.h"
+#include <cstdint>
+#include <optional>
+#include <utility>
+
+namespace llvm {
+class DataLayout;
+class LoadInst;
+class Instruction;
+class MDNode;
+class Value;
+
+/// Combine the metadata of two instructions so that K can replace J. This
+/// specifically handles the case of CSE-like transformations. Some
+/// metadata can only be kept if K dominates J. For this to be correct,
+/// K cannot be hoisted.
+///
+/// Unknown metadata is removed.
+LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
+ bool DoesKMove);
+
+/// Combine metadata of two instructions, where instruction J is a memory
+/// access that has been merged into K. This will intersect alias-analysis
+/// metadata, while preserving other known metadata.
+LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
+
+/// Copy the metadata from the source instruction to the destination (the
+/// replacement for the source instruction).
+LLVM_ABI void copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source);
+
+/// Copy a nonnull metadata node to a new load instruction.
+///
+/// This handles mapping it to range metadata if the new load is an integer
+/// load instead of a pointer load.
+LLVM_ABI void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
+ LoadInst &NewLI);
+
+/// Copy a range metadata node to a new load instruction.
+///
+/// This handles mapping it to nonnull metadata if the new load is a pointer
+/// load instead of an integer load and the range doesn't cover null.
+LLVM_ABI void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
+ MDNode *N, LoadInst &NewLI);
+
+/// Compute the union of two access-group lists.
+///
+/// If the list contains just one access group, it is returned directly. If the
+/// list is empty, returns nullptr.
+LLVM_ABI MDNode *uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2);
+
+/// Compute the access-group list of access groups that @p Inst1 and @p Inst2
+/// are both in. If either instruction does not access memory at all, it is
+/// considered to be in every list.
+///
+/// If the list contains just one access group, it is returned directly. If the
+/// list is empty, returns nullptr.
+LLVM_ABI MDNode *intersectAccessGroups(const Instruction *Inst1,
+ const Instruction *Inst2);
+
+/// Add metadata from \p Inst to \p Metadata, if it can be preserved after
+/// vectorization. It can be preserved after vectorization if the kind is one of
+/// [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,
+/// MD_access_group, MD_mmra].
+LLVM_ABI void getMetadataToPropagate(
+ Instruction *Inst,
+ SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata);
+
+/// Propagate metadata that can be preserved after vectorization from \p VL to
+/// \p I, retaining only information valid for all sources and the destination.
+/// Return \p I.
+LLVM_ABI Instruction *propagateMetadata(Instruction *I, ArrayRef<Value *> VL);
+
+} // namespace llvm
+
+#endif // LLVM_ANALYSIS_METADATAUTILS_H
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 3b0642b1ffbffe..8106b6de07b269 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -388,34 +388,6 @@ LLVM_ABI MapVector<Instruction *, uint64_t>
computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB,
const TargetTransformInfo *TTI = nullptr);
-/// Compute the union of two access-group lists.
-///
-/// If the list contains just one access group, it is returned directly. If the
-/// list is empty, returns nullptr.
-LLVM_ABI MDNode *uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2);
-
-/// Compute the access-group list of access groups that @p Inst1 and @p Inst2
-/// are both in. If either instruction does not access memory at all, it is
-/// considered to be in every list.
-///
-/// If the list contains just one access group, it is returned directly. If the
-/// list is empty, returns nullptr.
-LLVM_ABI MDNode *intersectAccessGroups(const Instruction *Inst1,
- const Instruction *Inst2);
-
-/// Add metadata from \p Inst to \p Metadata, if it can be preserved after
-/// vectorization. It can be preserved after vectorization if the kind is one of
-/// [MD_tbaa, MD_alias_scope, MD_noalias, MD_fpmath, MD_nontemporal,
-/// MD_access_group, MD_mmra].
-LLVM_ABI void getMetadataToPropagate(
- Instruction *Inst,
- SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata);
-
-/// Propagate metadata that can be preserved after vectorization from \p VL to
-/// \p I, retaining only information valid for all sources and the destination.
-/// Return \p I.
-LLVM_ABI Instruction *propagateMetadata(Instruction *I, ArrayRef<Value *> VL);
-
/// Create a mask that filters the members of an interleave group where there
/// are gaps.
///
diff --git a/llvm/include/llvm/Transforms/Utils/Local.h b/llvm/include/llvm/Transforms/Utils/Local.h
index 98b8ddd992282e..4991332881debd 100644
--- a/llvm/include/llvm/Transforms/Utils/Local.h
+++ b/llvm/include/llvm/Transforms/Utils/Local.h
@@ -15,6 +15,7 @@
#define LLVM_TRANSFORMS_UTILS_LOCAL_H
#include "llvm/ADT/ArrayRef.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/IR/Dominators.h"
#include "llvm/Support/CommandLine.h"
#include "llvm/Support/Compiler.h"
@@ -413,24 +414,6 @@ LLVM_ABI bool removeUnreachableBlocks(Function &F,
MemorySSAUpdater *MSSAU = nullptr,
bool FoldInstsToUnreachable = true);
-/// Combine the metadata of two instructions so that K can replace J. This
-/// specifically handles the case of CSE-like transformations. Some
-/// metadata can only be kept if K dominates J. For this to be correct,
-/// K cannot be hoisted.
-///
-/// Unknown metadata is removed.
-LLVM_ABI void combineMetadataForCSE(Instruction *K, const Instruction *J,
- bool DoesKMove);
-
-/// Combine metadata of two instructions, where instruction J is a memory
-/// access that has been merged into K. This will intersect alias-analysis
-/// metadata, while preserving other known metadata.
-LLVM_ABI void combineAAMetadata(Instruction *K, const Instruction *J);
-
-/// Copy the metadata from the source instruction to the destination (the
-/// replacement for the source instruction).
-LLVM_ABI void copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source);
-
/// Patch the replacement so that it is not more restrictive than the value
/// being replaced. Unless DoesKMove is set, it assumes that the replacement
/// does not get moved from its original position.
@@ -486,20 +469,6 @@ LLVM_ABI unsigned replaceDominatedUsesWithIf(
LLVM_ABI bool callsGCLeafFunction(const CallBase *Call,
const TargetLibraryInfo &TLI);
-/// Copy a nonnull metadata node to a new load instruction.
-///
-/// This handles mapping it to range metadata if the new load is an integer
-/// load instead of a pointer load.
-LLVM_ABI void copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
- LoadInst &NewLI);
-
-/// Copy a range metadata node to a new load instruction.
-///
-/// This handles mapping it to nonnull metadata if the new load is a pointer
-/// load instead of an integer load and the range doesn't cover null.
-LLVM_ABI void copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
- MDNode *N, LoadInst &NewLI);
-
/// Remove the debug intrinsic instructions for the given instruction.
LLVM_ABI void dropDebugUsers(Instruction &I);
diff --git a/llvm/lib/Analysis/CMakeLists.txt b/llvm/lib/Analysis/CMakeLists.txt
index 9c89cb4887d71f..301f517588a796 100644
--- a/llvm/lib/Analysis/CMakeLists.txt
+++ b/llvm/lib/Analysis/CMakeLists.txt
@@ -139,6 +139,7 @@ add_llvm_component_library(LLVMAnalysis
MemoryProfileInfo.cpp
MemorySSA.cpp
MemorySSAUpdater.cpp
+ MetadataUtils.cpp
ModelUnderTrainingRunner.cpp
ModuleDebugInfoPrinter.cpp
ModuleSummaryAnalysis.cpp
diff --git a/llvm/lib/Analysis/MetadataUtils.cpp b/llvm/lib/Analysis/MetadataUtils.cpp
new file mode 100644
index 00000000000000..a4d63aa51b8120
--- /dev/null
+++ b/llvm/lib/Analysis/MetadataUtils.cpp
@@ -0,0 +1,493 @@
+//===- MetadataUtils.cpp - Metadata utilities --------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Analysis/MetadataUtils.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SetVector.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/IR/ConstantRange.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/MDBuilder.h"
+#include "llvm/IR/MemCacheHint.h"
+#include "llvm/IR/MemoryModelRelaxationAnnotations.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/Operator.h"
+#include "llvm/IR/ProfDataUtils.h"
+
+using namespace llvm;
+
+/// Add all access groups in @p AccGroups to @p List.
+template <typename ListT>
+static void addToAccessGroupList(ListT &List, MDNode *AccGroups) {
+ // Interpret an access group as a list containing itself.
+ if (AccGroups->getNumOperands() == 0) {
+ assert(isValidAsAccessGroup(AccGroups) && "Node must be an access group");
+ List.insert(AccGroups);
+ return;
+ }
+
+ for (const auto &AccGroupListOp : AccGroups->operands()) {
+ auto *Item = cast<MDNode>(AccGroupListOp.get());
+ assert(isValidAsAccessGroup(Item) && "List item must be an access group");
+ List.insert(Item);
+ }
+}
+
+MDNode *llvm::uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2) {
+ if (!AccGroups1)
+ return AccGroups2;
+ if (!AccGroups2)
+ return AccGroups1;
+ if (AccGroups1 == AccGroups2)
+ return AccGroups1;
+
+ SmallSetVector<Metadata *, 4> Union;
+ addToAccessGroupList(Union, AccGroups1);
+ addToAccessGroupList(Union, AccGroups2);
+
+ if (Union.size() == 0)
+ return nullptr;
+ if (Union.size() == 1)
+ return cast<MDNode>(Union.front());
+
+ LLVMContext &Ctx = AccGroups1->getContext();
+ return MDNode::get(Ctx, Union.getArrayRef());
+}
+
+MDNode *llvm::intersectAccessGroups(const Instruction *Inst1,
+ const Instruction *Inst2) {
+ bool MayAccessMem1 = Inst1->mayReadOrWriteMemory();
+ bool MayAccessMem2 = Inst2->mayReadOrWriteMemory();
+
+ if (!MayAccessMem1 && !MayAccessMem2)
+ return nullptr;
+ if (!MayAccessMem1)
+ return Inst2->getMetadata(LLVMContext::MD_access_group);
+ if (!MayAccessMem2)
+ return Inst1->getMetadata(LLVMContext::MD_access_group);
+
+ MDNode *MD1 = Inst1->getMetadata(LLVMContext::MD_access_group);
+ MDNode *MD2 = Inst2->getMetadata(LLVMContext::MD_access_group);
+ if (!MD1 || !MD2)
+ return nullptr;
+ if (MD1 == MD2)
+ return MD1;
+
+ // Use set for scalable 'contains' check.
+ SmallPtrSet<Metadata *, 4> AccGroupSet2;
+ addToAccessGroupList(AccGroupSet2, MD2);
+
+ SmallVector<Metadata *, 4> Intersection;
+ if (MD1->getNumOperands() == 0) {
+ assert(isValidAsAccessGroup(MD1) && "Node must be an access group");
+ if (AccGroupSet2.count(MD1))
+ Intersection.push_back(MD1);
+ } else {
+ for (const MDOperand &Node : MD1->operands()) {
+ auto *Item = cast<MDNode>(Node.get());
+ assert(isValidAsAccessGroup(Item) && "List item must be an access group");
+ if (AccGroupSet2.count(Item))
+ Intersection.push_back(Item);
+ }
+ }
+
+ if (Intersection.size() == 0)
+ return nullptr;
+ if (Intersection.size() == 1)
+ return cast<MDNode>(Intersection.front());
+
+ LLVMContext &Ctx = Inst1->getContext();
+ return MDNode::get(Ctx, Intersection);
+}
+
+/// Add metadata from \p Inst to \p Metadata, if it can be preserved after
+/// vectorization.
+void llvm::getMetadataToPropagate(
+ Instruction *Inst,
+ SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata) {
+ Inst->getAllMetadataOtherThanDebugLoc(Metadata);
+ static const unsigned SupportedIDs[] = {
+ LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
+ LLVMContext::MD_noalias, LLVMContext::MD_fpmath,
+ LLVMContext::MD_nontemporal, LLVMContext::MD_invariant_load,
+ LLVMContext::MD_access_group, LLVMContext::MD_mmra};
+
+ // Remove any unsupported metadata kinds from Metadata.
+ for (unsigned Idx = 0; Idx != Metadata.size();) {
+ if (is_contained(SupportedIDs, Metadata[Idx].first)) {
+ ++Idx;
+ } else {
+ // Swap element to end and remove it.
+ std::swap(Metadata[Idx], Metadata.back());
+ Metadata.pop_back();
+ }
+ }
+}
+
+/// \returns \p I after propagating metadata from \p VL.
+Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
+ if (VL.empty())
+ return Inst;
+
+ // Keep cache hints out of getMetadataToPropagate(): VPlan changes operand
+ // layouts without remapping cache-hint indices.
+ propagateMemCacheHint(*Inst, VL);
+
+ SmallVector<std::pair<unsigned, MDNode *>> Metadata;
+ getMetadataToPropagate(cast<Instruction>(VL[0]), Metadata);
+
+ for (auto &[Kind, MD] : Metadata) {
+ // Skip MMRA metadata if the instruction cannot have it.
+ if (Kind == LLVMContext::MD_mmra && !canInstructionHaveMMRAs(*Inst))
+ continue;
+
+ for (int J = 1, E = VL.size(); MD && J != E; ++J) {
+ const Instruction *IJ = cast<Instruction>(VL[J]);
+ MDNode *IMD = IJ->getMetadata(Kind);
+
+ switch (Kind) {
+ case LLVMContext::MD_mmra: {
+ MD = MMRAMetadata::combine(Inst->getContext(), MD, IMD);
+ break;
+ }
+ case LLVMContext::MD_tbaa:
+ MD = MDNode::getMostGenericTBAA(MD, IMD);
+ break;
+ case LLVMContext::MD_alias_scope:
+ MD = MDNode::getMostGenericAliasScope(MD, IMD);
+ break;
+ case LLVMContext::MD_fpmath:
+ MD = MDNode::getMostGenericFPMath(MD, IMD);
+ break;
+ case LLVMContext::MD_noalias:
+ case LLVMContext::MD_nontemporal:
+ case LLVMContext::MD_invariant_load:
+ MD = MDNode::intersect(MD, IMD);
+ break;
+ case LLVMContext::MD_access_group:
+ MD = intersectAccessGroups(Inst, IJ);
+ break;
+ default:
+ llvm_unreachable("unhandled metadata");
+ }
+ }
+
+ Inst->setMetadata(Kind, MD);
+ }
+
+ return Inst;
+}
+
+/// If AAOnly is set, only intersect alias analysis metadata and preserve other
+/// known metadata. Unknown metadata is always dropped.
+static void combineMetadata(Instruction *K, const Instruction *J,
+ bool DoesKMove, bool AAOnly = false) {
+ SmallVector<std::pair<unsigned, MDNode *>, 4> Metadata;
+ K->getAllMetadataOtherThanDebugLoc(Metadata);
+ for (const auto &MD : Metadata) {
+ unsigned Kind = MD.first;
+ MDNode *JMD = J->getMetadata(Kind);
+ MDNode *KMD = MD.second;
+
+ // TODO: Assert that this switch is exhaustive for fixed MD kinds.
+ switch (Kind) {
+ default:
+ K->setMetadata(Kind, nullptr); // Remove unknown metadata
+ break;
+ case LLVMContext::MD_dbg:
+ llvm_unreachable("getAllMetadataOtherThanDebugLoc returned a MD_dbg");
+ case LLVMContext::MD_DIAssignID:
+ if (!AAOnly)
+ K->mergeDIAssignID(J);
+ break;
+ case LLVMContext::MD_tbaa:
+ if (DoesKMove)
+ K->setMetadata(Kind, MDNode::getMostGenericTBAA(JMD, KMD));
+ break;
+ case LLVMContext::MD_alias_scope:
+ if (DoesKMove)
+ K->setMetadata(Kind, MDNode::getMostGenericAliasScope(JMD, KMD));
+ break;
+ case LLVMContext::MD_noalias:
+ case LLVMContext::MD_mem_parallel_loop_access:
+ if (DoesKMove)
+ K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
+ break;
+ case LLVMContext::MD_access_group:
+ if (DoesKMove)
+ K->setMetadata(LLVMContext::MD_access_group,
+ intersectAccessGroups(K, J));
+ break;
+ case LLVMContext::MD_range:
+ if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+ K->setMetadata(Kind, MDNode::getMostGenericRange(JMD, KMD));
+ break;
+ case LLVMContext::MD_nofpclass:
+ if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+ K->setMetadata(Kind, MDNode::getMostGenericNoFPClass(JMD, KMD));
+ break;
+ case LLVMContext::MD_fpmath:
+ if (!AAOnly)
+ K->setMetadata(Kind, MDNode::getMostGenericFPMath(JMD, KMD));
+ break;
+ case LLVMContext::MD_invariant_load:
+ case LLVMContext::MD_invariant_group:
+ // If K moves, only keep the invariant metadata if it is present on
+ // both instructions; otherwise the invariant would be asserted on a
+ // path (J's) that never promised it. If K does not move, K stays on
+ // its original path, so its existing metadata remains valid.
+ if (DoesKMove)
+ K->setMetadata(Kind, JMD);
+ break;
+ case LLVMContext::MD_nonnull:
+ if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+ K->setMetadata(Kind, JMD);
+ break;
+ // Keep empty cases for prof, mmra, memprof, and callsite to prevent them
+ // from being removed as unknown metadata. The actual merging is handled
+ // separately below.
+ case LLVMContext::MD_prof:
+ case LLVMContext::MD_mmra:
+ case LLVMContext::MD_memprof:
+ case LLVMContext::MD_callsite:
+ break;
+ case LLVMContext::MD_callee_type:
+ if (!AAOnly) {
+ K->setMetadata(LLVMContext::MD_callee_type,
+ MDNode::getMergedCalleeTypeMetadata(KMD, JMD));
+ }
+ break;
+ case LLVMContext::MD_callees:
+ // If K moves, it replaces J on J's path and must allow J's callees as
+ // well. If K does not move, its callees remain valid.
+ if (!AAOnly && DoesKMove)
+ K->setMetadata(Kind, MDNode::getMergedCalleesMetadata(KMD, JMD));
+ break;
+ case LLVMContext::MD_align:
+ if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
+ K->setMetadata(
+ Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
+ break;
+ case LLVMContext::MD_dereferenceable:
+ case LLVMContext::MD_dereferenceable_or_null:
+ if (!AAOnly && DoesKMove)
+ K->setMetadata(
+ Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
+ break;
+ case LLVMContext::MD_preserve_access_index:
+ // Preserve !preserve.access.index in K.
+ break;
+ case LLVMContext::MD_noundef:
+ // If K does move, keep noundef if it is present in both instructions.
+ if (!AAOnly && DoesKMove)
+ K->setMetadata(Kind, JMD);
+ break;
+ case LLVMContext::MD_nontemporal:
+ // Preserve !nontemporal if it is present on both instructions.
+ if (!AAOnly)
+ K->setMetadata(Kind, JMD);
+ break;
+ case LLVMContext::MD_mem_cache_hint:
+ // Preserve !mem.cache_hint only if it is present and equivalent on both
+ // instructions.
+ if (!AAOnly && KMD != JMD)
+ K->setMetadata(Kind, nullptr);
+ break;
+ case LLVMContext::MD_noalias_addrspace:
+ if (DoesKMove)
+ K->setMetadata(Kind, MDNode::getMostGenericNoaliasAddrspace(JMD, KMD));
+ break;
+ case LLVMContext::MD_nosanitize:
+ // Preserve !nosanitize if both K and J have it.
+ K->setMetadata(Kind, JMD);
+ break;
+ case LLVMContext::MD_captures:
+ K->setMetadata(
+ Kind, MDNode::fromCaptureComponents(
+ K->getContext(), MDNode::toCaptureComponents(JMD) |
+ MDNode::toCaptureComponents(KMD)));
+ break;
+ case LLVMContext::MD_alloc_token:
+ if (!AAOnly && KMD != JMD)
+ K->setMetadata(Kind, MDNode::getMergedAllocTokenMetadata(KMD, JMD));
+ break;
+ }
+ }
+
+ // Merge MMRAs.
+ // This is handled separately because we also want to handle cases where K
+ // doesn't have tags but J does.
+ auto JMMRA = J->getMetadata(LLVMContext::MD_mmra);
+ auto KMMRA = K->getMetadata(LLVMContext::MD_mmra);
+ if (JMMRA || KMMRA) {
+ K->setMetadata(LLVMContext::MD_mmra,
+ MMRAMetadata::combine(K->getContext(), JMMRA, KMMRA));
+ }
+
+ // Merge memprof metadata.
+ // Handle separately to support cases where only one instruction has the
+ // metadata.
+ auto *JMemProf = J->getMetadata(LLVMContext::MD_memprof);
+ auto *KMemProf = K->getMetadata(LLVMContext::MD_memprof);
+ if (!AAOnly && (JMemProf || KMemProf)) {
+ K->setMetadata(LLVMContext::MD_memprof,
+ MDNode::getMergedMemProfMetadata(KMemProf, JMemProf));
+ }
+
+ // Merge callsite metadata.
+ // Handle separately to support cases where only one instruction has the
+ // metadata.
+ auto *JCallSite = J->getMetadata(LLVMContext::MD_callsite);
+ auto *KCallSite = K->getMetadata(LLVMContext::MD_callsite);
+ if (!AAOnly && (JCallSite || KCallSite)) {
+ K->setMetadata(LLVMContext::MD_callsite,
+ MDNode::getMergedCallsiteMetadata(KCallSite, JCallSite));
+ }
+
+ // Merge prof metadata.
+ // Handle separately to support cases where only one instruction has the
+ // metadata.
+ auto *JProf = J->getMetadata(LLVMContext::MD_prof);
+ auto *KProf = K->getMetadata(LLVMContext::MD_prof);
+ if (!AAOnly && (JProf || KProf)) {
+ // We do not need to merge metadata if we have select instructions and the
+ // one we're keeping (K) does not move (explicitly merging would drop the
+ // information in this case). Otherwise, explicitly handle the merging.
+ if (DoesKMove || (!isa<SelectInst>(J) || !isa<SelectInst>(K)) ||
+ ProfcheckDisableMetadataFixes)
+ K->setMetadata(LLVMContext::MD_prof,
+ MDNode::getMergedProfMetadata(KProf, JProf, K, J));
+ }
+}
+
+void llvm::combineMetadataForCSE(Instruction *K, const Instruction *J,
+ bool DoesKMove) {
+ combineMetadata(K, J, DoesKMove);
+}
+
+void llvm::combineAAMetadata(Instruction *K, const Instruction *J) {
+ combineMetadata(K, J, /*DoesKMove=*/true, /*AAOnly=*/true);
+}
+
+void llvm::copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source) {
+ SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
+ Source.getAllMetadata(MD);
+ MDBuilder MDB(Dest.getContext());
+ Type *NewType = Dest.getType();
+ const DataLayout &DL = Source.getDataLayout();
+ for (const auto &MDPair : MD) {
+ unsigned ID = MDPair.first;
+ MDNode *N = MDPair.second;
+ // Note, essentially every kind of metadata should be preserved here! This
+ // routine is supposed to clone a load instruction changing *only its type*.
+ // The only metadata it makes sense to drop is metadata which is invalidated
+ // when the pointer type changes. This should essentially never be the case
+ // in LLVM, but we explicitly switch over only known metadata to be
+ // conservatively correct. If you are adding metadata to LLVM which pertains
+ // to loads, you almost certainly want to add it here.
+ switch (ID) {
+ case LLVMContext::MD_dbg:
+ case LLVMContext::MD_tbaa:
+ case LLVMContext::MD_prof:
+ case LLVMContext::MD_fpmath:
+ case LLVMContext::MD_tbaa_struct:
+ case LLVMContext::MD_invariant_load:
+ case LLVMContext::MD_alias_scope:
+ case LLVMContext::MD_noalias:
+ case LLVMContext::MD_nontemporal:
+ case LLVMContext::MD_mem_cache_hint:
+ case LLVMContext::MD_mem_parallel_loop_access:
+ case LLVMContext::MD_access_group:
+ case LLVMContext::MD_noundef:
+ case LLVMContext::MD_noalias_addrspace:
+ case LLVMContext::MD_invariant_group:
+ case LLVMContext::MD_mmra:
+ // All of these directly apply.
+ Dest.setMetadata(ID, N);
+ break;
+
+ case LLVMContext::MD_nonnull:
+ copyNonnullMetadata(Source, N, Dest);
+ break;
+
+ case LLVMContext::MD_align:
+ case LLVMContext::MD_dereferenceable:
+ case LLVMContext::MD_dereferenceable_or_null:
+ // These only directly apply if the new type is also a pointer.
+ if (NewType->isPointerTy())
+ Dest.setMetadata(ID, N);
+ break;
+
+ case LLVMContext::MD_range:
+ copyRangeMetadata(DL, Source, N, Dest);
+ break;
+
+ case LLVMContext::MD_nofpclass:
+ // This only applies if the floating-point type interpretation. This
+ // should handle degenerate cases like casting between a scalar and single
+ // element vector.
+ if (NewType->getScalarType() == Source.getType()->getScalarType())
+ Dest.setMetadata(ID, N);
+ break;
+ }
+ }
+}
+
+void llvm::copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
+ LoadInst &NewLI) {
+ auto *NewTy = NewLI.getType();
+
+ // This only directly applies if the new type is also a pointer.
+ if (NewTy->isPointerTy()) {
+ NewLI.setMetadata(LLVMContext::MD_nonnull, N);
+ return;
+ }
+
+ // The only other translation we can do is to integral loads with !range
+ // metadata.
+ if (!NewTy->isIntegerTy())
+ return;
+
+ MDBuilder MDB(NewLI.getContext());
+ const Value *Ptr = OldLI.getPointerOperand();
+ auto *ITy = cast<IntegerType>(NewTy);
+ auto *NullInt = ConstantExpr::getPtrToInt(
+ ConstantPointerNull::get(cast<PointerType>(Ptr->getType())), ITy);
+ auto *NonNullInt = ConstantExpr::getAdd(NullInt, ConstantInt::get(ITy, 1));
+ NewLI.setMetadata(LLVMContext::MD_range,
+ MDB.createRange(NonNullInt, NullInt));
+}
+
+void llvm::copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
+ MDNode *N, LoadInst &NewLI) {
+ auto *NewTy = NewLI.getType();
+ // Simply copy the metadata if the type did not change.
+ if (NewTy == OldLI.getType()) {
+ NewLI.setMetadata(LLVMContext::MD_range, N);
+ return;
+ }
+
+ // Give up unless it is converted to a pointer where there is a single very
+ // valuable mapping we can do reliably.
+ // FIXME: It would be nice to propagate this in more ways, but the type
+ // conversions make it hard.
+ if (!NewTy->isPointerTy())
+ return;
+
+ unsigned BitWidth = DL.getPointerTypeSizeInBits(NewTy);
+ if (BitWidth == OldLI.getType()->getScalarSizeInBits() &&
+ !getConstantRangeFromMetadata(*N).contains(APInt(BitWidth, 0))) {
+ MDNode *NN = MDNode::get(OldLI.getContext(), {});
+ NewLI.setMetadata(LLVMContext::MD_nonnull, NN);
+ }
+}
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index b5ccb8574b6026..fe5fc300cf3e18 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -10,6 +10,7 @@
//
//===----------------------------------------------------------------------===//
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/VectorUtils.h"
#include "AnalysisOptions.h"
#include "llvm/ADT/EquivalenceClasses.h"
@@ -1030,168 +1031,6 @@ llvm::computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB,
return MinBWs;
}
-/// Add all access groups in @p AccGroups to @p List.
-template <typename ListT>
-static void addToAccessGroupList(ListT &List, MDNode *AccGroups) {
- // Interpret an access group as a list containing itself.
- if (AccGroups->getNumOperands() == 0) {
- assert(isValidAsAccessGroup(AccGroups) && "Node must be an access group");
- List.insert(AccGroups);
- return;
- }
-
- for (const auto &AccGroupListOp : AccGroups->operands()) {
- auto *Item = cast<MDNode>(AccGroupListOp.get());
- assert(isValidAsAccessGroup(Item) && "List item must be an access group");
- List.insert(Item);
- }
-}
-
-MDNode *llvm::uniteAccessGroups(MDNode *AccGroups1, MDNode *AccGroups2) {
- if (!AccGroups1)
- return AccGroups2;
- if (!AccGroups2)
- return AccGroups1;
- if (AccGroups1 == AccGroups2)
- return AccGroups1;
-
- SmallSetVector<Metadata *, 4> Union;
- addToAccessGroupList(Union, AccGroups1);
- addToAccessGroupList(Union, AccGroups2);
-
- if (Union.size() == 0)
- return nullptr;
- if (Union.size() == 1)
- return cast<MDNode>(Union.front());
-
- LLVMContext &Ctx = AccGroups1->getContext();
- return MDNode::get(Ctx, Union.getArrayRef());
-}
-
-MDNode *llvm::intersectAccessGroups(const Instruction *Inst1,
- const Instruction *Inst2) {
- bool MayAccessMem1 = Inst1->mayReadOrWriteMemory();
- bool MayAccessMem2 = Inst2->mayReadOrWriteMemory();
-
- if (!MayAccessMem1 && !MayAccessMem2)
- return nullptr;
- if (!MayAccessMem1)
- return Inst2->getMetadata(LLVMContext::MD_access_group);
- if (!MayAccessMem2)
- return Inst1->getMetadata(LLVMContext::MD_access_group);
-
- MDNode *MD1 = Inst1->getMetadata(LLVMContext::MD_access_group);
- MDNode *MD2 = Inst2->getMetadata(LLVMContext::MD_access_group);
- if (!MD1 || !MD2)
- return nullptr;
- if (MD1 == MD2)
- return MD1;
-
- // Use set for scalable 'contains' check.
- SmallPtrSet<Metadata *, 4> AccGroupSet2;
- addToAccessGroupList(AccGroupSet2, MD2);
-
- SmallVector<Metadata *, 4> Intersection;
- if (MD1->getNumOperands() == 0) {
- assert(isValidAsAccessGroup(MD1) && "Node must be an access group");
- if (AccGroupSet2.count(MD1))
- Intersection.push_back(MD1);
- } else {
- for (const MDOperand &Node : MD1->operands()) {
- auto *Item = cast<MDNode>(Node.get());
- assert(isValidAsAccessGroup(Item) && "List item must be an access group");
- if (AccGroupSet2.count(Item))
- Intersection.push_back(Item);
- }
- }
-
- if (Intersection.size() == 0)
- return nullptr;
- if (Intersection.size() == 1)
- return cast<MDNode>(Intersection.front());
-
- LLVMContext &Ctx = Inst1->getContext();
- return MDNode::get(Ctx, Intersection);
-}
-
-/// Add metadata from \p Inst to \p Metadata, if it can be preserved after
-/// vectorization.
-void llvm::getMetadataToPropagate(
- Instruction *Inst,
- SmallVectorImpl<std::pair<unsigned, MDNode *>> &Metadata) {
- Inst->getAllMetadataOtherThanDebugLoc(Metadata);
- static const unsigned SupportedIDs[] = {
- LLVMContext::MD_tbaa, LLVMContext::MD_alias_scope,
- LLVMContext::MD_noalias, LLVMContext::MD_fpmath,
- LLVMContext::MD_nontemporal, LLVMContext::MD_invariant_load,
- LLVMContext::MD_access_group, LLVMContext::MD_mmra};
-
- // Remove any unsupported metadata kinds from Metadata.
- for (unsigned Idx = 0; Idx != Metadata.size();) {
- if (is_contained(SupportedIDs, Metadata[Idx].first)) {
- ++Idx;
- } else {
- // Swap element to end and remove it.
- std::swap(Metadata[Idx], Metadata.back());
- Metadata.pop_back();
- }
- }
-}
-
-/// \returns \p I after propagating metadata from \p VL.
-Instruction *llvm::propagateMetadata(Instruction *Inst, ArrayRef<Value *> VL) {
- if (VL.empty())
- return Inst;
-
- // Keep cache hints out of getMetadataToPropagate(): VPlan changes operand
- // layouts without remapping cache-hint indices.
- propagateMemCacheHint(*Inst, VL);
-
- SmallVector<std::pair<unsigned, MDNode *>> Metadata;
- getMetadataToPropagate(cast<Instruction>(VL[0]), Metadata);
-
- for (auto &[Kind, MD] : Metadata) {
- // Skip MMRA metadata if the instruction cannot have it.
- if (Kind == LLVMContext::MD_mmra && !canInstructionHaveMMRAs(*Inst))
- continue;
-
- for (int J = 1, E = VL.size(); MD && J != E; ++J) {
- const Instruction *IJ = cast<Instruction>(VL[J]);
- MDNode *IMD = IJ->getMetadata(Kind);
-
- switch (Kind) {
- case LLVMContext::MD_mmra: {
- MD = MMRAMetadata::combine(Inst->getContext(), MD, IMD);
- break;
- }
- case LLVMContext::MD_tbaa:
- MD = MDNode::getMostGenericTBAA(MD, IMD);
- break;
- case LLVMContext::MD_alias_scope:
- MD = MDNode::getMostGenericAliasScope(MD, IMD);
- break;
- case LLVMContext::MD_fpmath:
- MD = MDNode::getMostGenericFPMath(MD, IMD);
- break;
- case LLVMContext::MD_noalias:
- case LLVMContext::MD_nontemporal:
- case LLVMContext::MD_invariant_load:
- MD = MDNode::intersect(MD, IMD);
- break;
- case LLVMContext::MD_access_group:
- MD = intersectAccessGroups(Inst, IJ);
- break;
- default:
- llvm_unreachable("unhandled metadata");
- }
- }
-
- Inst->setMetadata(Kind, MD);
- }
-
- return Inst;
-}
-
Constant *
llvm::createBitMaskForGaps(IRBuilderBase &Builder, unsigned VF,
const InterleaveGroup<Instruction> &Group) {
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index 8b6d53483dca04..ac2223d5c169bc 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -11,6 +11,7 @@
//
//===----------------------------------------------------------------------===//
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/DenseMap.h"
@@ -2990,262 +2991,6 @@ bool llvm::removeUnreachableBlocks(Function &F, DomTreeUpdater *DTU,
return Changed;
}
-/// If AAOnly is set, only intersect alias analysis metadata and preserve other
-/// known metadata. Unknown metadata is always dropped.
-static void combineMetadata(Instruction *K, const Instruction *J,
- bool DoesKMove, bool AAOnly = false) {
- SmallVector<std::pair<unsigned, MDNode *>, 4> Metadata;
- K->getAllMetadataOtherThanDebugLoc(Metadata);
- for (const auto &MD : Metadata) {
- unsigned Kind = MD.first;
- MDNode *JMD = J->getMetadata(Kind);
- MDNode *KMD = MD.second;
-
- // TODO: Assert that this switch is exhaustive for fixed MD kinds.
- switch (Kind) {
- default:
- K->setMetadata(Kind, nullptr); // Remove unknown metadata
- break;
- case LLVMContext::MD_dbg:
- llvm_unreachable("getAllMetadataOtherThanDebugLoc returned a MD_dbg");
- case LLVMContext::MD_DIAssignID:
- if (!AAOnly)
- K->mergeDIAssignID(J);
- break;
- case LLVMContext::MD_tbaa:
- if (DoesKMove)
- K->setMetadata(Kind, MDNode::getMostGenericTBAA(JMD, KMD));
- break;
- case LLVMContext::MD_alias_scope:
- if (DoesKMove)
- K->setMetadata(Kind, MDNode::getMostGenericAliasScope(JMD, KMD));
- break;
- case LLVMContext::MD_noalias:
- case LLVMContext::MD_mem_parallel_loop_access:
- if (DoesKMove)
- K->setMetadata(Kind, MDNode::intersect(JMD, KMD));
- break;
- case LLVMContext::MD_access_group:
- if (DoesKMove)
- K->setMetadata(LLVMContext::MD_access_group,
- intersectAccessGroups(K, J));
- break;
- case LLVMContext::MD_range:
- if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
- K->setMetadata(Kind, MDNode::getMostGenericRange(JMD, KMD));
- break;
- case LLVMContext::MD_nofpclass:
- if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
- K->setMetadata(Kind, MDNode::getMostGenericNoFPClass(JMD, KMD));
- break;
- case LLVMContext::MD_fpmath:
- if (!AAOnly)
- K->setMetadata(Kind, MDNode::getMostGenericFPMath(JMD, KMD));
- break;
- case LLVMContext::MD_invariant_load:
- case LLVMContext::MD_invariant_group:
- // If K moves, only keep the invariant metadata if it is present on
- // both instructions; otherwise the invariant would be asserted on a
- // path (J's) that never promised it. If K does not move, K stays on
- // its original path, so its existing metadata remains valid.
- if (DoesKMove)
- K->setMetadata(Kind, JMD);
- break;
- case LLVMContext::MD_nonnull:
- if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
- K->setMetadata(Kind, JMD);
- break;
- // Keep empty cases for prof, mmra, memprof, and callsite to prevent them
- // from being removed as unknown metadata. The actual merging is handled
- // separately below.
- case LLVMContext::MD_prof:
- case LLVMContext::MD_mmra:
- case LLVMContext::MD_memprof:
- case LLVMContext::MD_callsite:
- break;
- case LLVMContext::MD_callee_type:
- if (!AAOnly) {
- K->setMetadata(LLVMContext::MD_callee_type,
- MDNode::getMergedCalleeTypeMetadata(KMD, JMD));
- }
- break;
- case LLVMContext::MD_callees:
- // If K moves, it replaces J on J's path and must allow J's callees as
- // well. If K does not move, its callees remain valid.
- if (!AAOnly && DoesKMove)
- K->setMetadata(Kind, MDNode::getMergedCalleesMetadata(KMD, JMD));
- break;
- case LLVMContext::MD_align:
- if (!AAOnly && (DoesKMove || !K->hasMetadata(LLVMContext::MD_noundef)))
- K->setMetadata(
- Kind, MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
- break;
- case LLVMContext::MD_dereferenceable:
- case LLVMContext::MD_dereferenceable_or_null:
- if (!AAOnly && DoesKMove)
- K->setMetadata(Kind,
- MDNode::getMostGenericAlignmentOrDereferenceable(JMD, KMD));
- break;
- case LLVMContext::MD_preserve_access_index:
- // Preserve !preserve.access.index in K.
- break;
- case LLVMContext::MD_noundef:
- // If K does move, keep noundef if it is present in both instructions.
- if (!AAOnly && DoesKMove)
- K->setMetadata(Kind, JMD);
- break;
- case LLVMContext::MD_nontemporal:
- // Preserve !nontemporal if it is present on both instructions.
- if (!AAOnly)
- K->setMetadata(Kind, JMD);
- break;
- case LLVMContext::MD_mem_cache_hint:
- // Preserve !mem.cache_hint only if it is present and equivalent on both
- // instructions.
- if (!AAOnly && KMD != JMD)
- K->setMetadata(Kind, nullptr);
- break;
- case LLVMContext::MD_noalias_addrspace:
- if (DoesKMove)
- K->setMetadata(Kind,
- MDNode::getMostGenericNoaliasAddrspace(JMD, KMD));
- break;
- case LLVMContext::MD_nosanitize:
- // Preserve !nosanitize if both K and J have it.
- K->setMetadata(Kind, JMD);
- break;
- case LLVMContext::MD_captures:
- K->setMetadata(
- Kind, MDNode::fromCaptureComponents(
- K->getContext(), MDNode::toCaptureComponents(JMD) |
- MDNode::toCaptureComponents(KMD)));
- break;
- case LLVMContext::MD_alloc_token:
- if (!AAOnly && KMD != JMD)
- K->setMetadata(Kind, MDNode::getMergedAllocTokenMetadata(KMD, JMD));
- break;
- }
- }
-
- // Merge MMRAs.
- // This is handled separately because we also want to handle cases where K
- // doesn't have tags but J does.
- auto JMMRA = J->getMetadata(LLVMContext::MD_mmra);
- auto KMMRA = K->getMetadata(LLVMContext::MD_mmra);
- if (JMMRA || KMMRA) {
- K->setMetadata(LLVMContext::MD_mmra,
- MMRAMetadata::combine(K->getContext(), JMMRA, KMMRA));
- }
-
- // Merge memprof metadata.
- // Handle separately to support cases where only one instruction has the
- // metadata.
- auto *JMemProf = J->getMetadata(LLVMContext::MD_memprof);
- auto *KMemProf = K->getMetadata(LLVMContext::MD_memprof);
- if (!AAOnly && (JMemProf || KMemProf)) {
- K->setMetadata(LLVMContext::MD_memprof,
- MDNode::getMergedMemProfMetadata(KMemProf, JMemProf));
- }
-
- // Merge callsite metadata.
- // Handle separately to support cases where only one instruction has the
- // metadata.
- auto *JCallSite = J->getMetadata(LLVMContext::MD_callsite);
- auto *KCallSite = K->getMetadata(LLVMContext::MD_callsite);
- if (!AAOnly && (JCallSite || KCallSite)) {
- K->setMetadata(LLVMContext::MD_callsite,
- MDNode::getMergedCallsiteMetadata(KCallSite, JCallSite));
- }
-
- // Merge prof metadata.
- // Handle separately to support cases where only one instruction has the
- // metadata.
- auto *JProf = J->getMetadata(LLVMContext::MD_prof);
- auto *KProf = K->getMetadata(LLVMContext::MD_prof);
- if (!AAOnly && (JProf || KProf)) {
- // We do not need to merge metadata if we have select instructions and the
- // one we're keeping (K) does not move (explicitly merging would drop the
- // information in this case). Otherwise, explicitly handle the merging.
- if (DoesKMove || (!isa<SelectInst>(J) || !isa<SelectInst>(K)) ||
- ProfcheckDisableMetadataFixes)
- K->setMetadata(LLVMContext::MD_prof,
- MDNode::getMergedProfMetadata(KProf, JProf, K, J));
- }
-}
-
-void llvm::combineMetadataForCSE(Instruction *K, const Instruction *J,
- bool DoesKMove) {
- combineMetadata(K, J, DoesKMove);
-}
-
-void llvm::combineAAMetadata(Instruction *K, const Instruction *J) {
- combineMetadata(K, J, /*DoesKMove=*/true, /*AAOnly=*/true);
-}
-
-void llvm::copyMetadataForLoad(LoadInst &Dest, const LoadInst &Source) {
- SmallVector<std::pair<unsigned, MDNode *>, 8> MD;
- Source.getAllMetadata(MD);
- MDBuilder MDB(Dest.getContext());
- Type *NewType = Dest.getType();
- const DataLayout &DL = Source.getDataLayout();
- for (const auto &MDPair : MD) {
- unsigned ID = MDPair.first;
- MDNode *N = MDPair.second;
- // Note, essentially every kind of metadata should be preserved here! This
- // routine is supposed to clone a load instruction changing *only its type*.
- // The only metadata it makes sense to drop is metadata which is invalidated
- // when the pointer type changes. This should essentially never be the case
- // in LLVM, but we explicitly switch over only known metadata to be
- // conservatively correct. If you are adding metadata to LLVM which pertains
- // to loads, you almost certainly want to add it here.
- switch (ID) {
- case LLVMContext::MD_dbg:
- case LLVMContext::MD_tbaa:
- case LLVMContext::MD_prof:
- case LLVMContext::MD_fpmath:
- case LLVMContext::MD_tbaa_struct:
- case LLVMContext::MD_invariant_load:
- case LLVMContext::MD_alias_scope:
- case LLVMContext::MD_noalias:
- case LLVMContext::MD_nontemporal:
- case LLVMContext::MD_mem_cache_hint:
- case LLVMContext::MD_mem_parallel_loop_access:
- case LLVMContext::MD_access_group:
- case LLVMContext::MD_noundef:
- case LLVMContext::MD_noalias_addrspace:
- case LLVMContext::MD_invariant_group:
- case LLVMContext::MD_mmra:
- // All of these directly apply.
- Dest.setMetadata(ID, N);
- break;
-
- case LLVMContext::MD_nonnull:
- copyNonnullMetadata(Source, N, Dest);
- break;
-
- case LLVMContext::MD_align:
- case LLVMContext::MD_dereferenceable:
- case LLVMContext::MD_dereferenceable_or_null:
- // These only directly apply if the new type is also a pointer.
- if (NewType->isPointerTy())
- Dest.setMetadata(ID, N);
- break;
-
- case LLVMContext::MD_range:
- copyRangeMetadata(DL, Source, N, Dest);
- break;
-
- case LLVMContext::MD_nofpclass:
- // This only applies if the floating-point type interpretation. This
- // should handle degenerate cases like casting between a scalar and single
- // element vector.
- if (NewType->getScalarType() == Source.getType()->getScalarType())
- Dest.setMetadata(ID, N);
- break;
- }
- }
-}
-
void llvm::patchReplacementInstruction(Instruction *I, Value *Repl,
bool DoesKMove) {
auto *ReplInst = dyn_cast<Instruction>(Repl);
@@ -3398,55 +3143,6 @@ bool llvm::callsGCLeafFunction(const CallBase *Call,
return TLI.has(TLI.getLibFunc(*Call));
}
-void llvm::copyNonnullMetadata(const LoadInst &OldLI, MDNode *N,
- LoadInst &NewLI) {
- auto *NewTy = NewLI.getType();
-
- // This only directly applies if the new type is also a pointer.
- if (NewTy->isPointerTy()) {
- NewLI.setMetadata(LLVMContext::MD_nonnull, N);
- return;
- }
-
- // The only other translation we can do is to integral loads with !range
- // metadata.
- if (!NewTy->isIntegerTy())
- return;
-
- MDBuilder MDB(NewLI.getContext());
- const Value *Ptr = OldLI.getPointerOperand();
- auto *ITy = cast<IntegerType>(NewTy);
- auto *NullInt = ConstantExpr::getPtrToInt(
- ConstantPointerNull::get(cast<PointerType>(Ptr->getType())), ITy);
- auto *NonNullInt = ConstantExpr::getAdd(NullInt, ConstantInt::get(ITy, 1));
- NewLI.setMetadata(LLVMContext::MD_range,
- MDB.createRange(NonNullInt, NullInt));
-}
-
-void llvm::copyRangeMetadata(const DataLayout &DL, const LoadInst &OldLI,
- MDNode *N, LoadInst &NewLI) {
- auto *NewTy = NewLI.getType();
- // Simply copy the metadata if the type did not change.
- if (NewTy == OldLI.getType()) {
- NewLI.setMetadata(LLVMContext::MD_range, N);
- return;
- }
-
- // Give up unless it is converted to a pointer where there is a single very
- // valuable mapping we can do reliably.
- // FIXME: It would be nice to propagate this in more ways, but the type
- // conversions make it hard.
- if (!NewTy->isPointerTy())
- return;
-
- unsigned BitWidth = DL.getPointerTypeSizeInBits(NewTy);
- if (BitWidth == OldLI.getType()->getScalarSizeInBits() &&
- !getConstantRangeFromMetadata(*N).contains(APInt(BitWidth, 0))) {
- MDNode *NN = MDNode::get(OldLI.getContext(), {});
- NewLI.setMetadata(LLVMContext::MD_nonnull, NN);
- }
-}
-
void llvm::dropDebugUsers(Instruction &I) {
SmallVector<DbgVariableRecord *, 1> DPUsers;
findDbgUsers(&I, DPUsers);
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
index f301b931d675aa..dff031f71773f6 100644
--- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp
@@ -72,6 +72,7 @@
#include "llvm/Analysis/AliasAnalysis.h"
#include "llvm/Analysis/AssumptionCache.h"
#include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/TargetTransformInfo.h"
#include "llvm/Analysis/ValueTracking.h"
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 50992fc3f89f44..bce9eed740fd69 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -50,6 +50,7 @@
#include "llvm/Analysis/Loads.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
diff --git a/llvm/lib/Transforms/Vectorize/VPlan.h b/llvm/lib/Transforms/Vectorize/VPlan.h
index 5d89141cde6f2e..4a185cbcf2f0c3 100644
--- a/llvm/lib/Transforms/Vectorize/VPlan.h
+++ b/llvm/lib/Transforms/Vectorize/VPlan.h
@@ -35,6 +35,7 @@
#include "llvm/ADT/ilist_node.h"
#include "llvm/Analysis/IVDescriptors.h"
#include "llvm/Analysis/MemoryLocation.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/VectorUtils.h"
#include "llvm/IR/Attributes.h"
#include "llvm/IR/DebugLoc.h"
>From 9a4e295cb5ab3d49bfdd3292c954093c75c34d3d Mon Sep 17 00:00:00 2001
From: Yonah Goldberg <ygoldberg at nvidia.com>
Date: Wed, 18 Nov 2026 00:57:44 +0000
Subject: [PATCH 2/2] [Analysis] Preserve metadata declarations through
existing headers
---
llvm/include/llvm/Analysis/VectorUtils.h | 1 +
llvm/lib/Analysis/VectorUtils.cpp | 2 +-
llvm/lib/Transforms/Utils/Local.cpp | 2 +-
3 files changed, 3 insertions(+), 2 deletions(-)
diff --git a/llvm/include/llvm/Analysis/VectorUtils.h b/llvm/include/llvm/Analysis/VectorUtils.h
index 8106b6de07b269..6a48b007a7f4f3 100644
--- a/llvm/include/llvm/Analysis/VectorUtils.h
+++ b/llvm/include/llvm/Analysis/VectorUtils.h
@@ -18,6 +18,7 @@
#include "llvm/ADT/Sequence.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/Analysis/LoopAccessAnalysis.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/IR/Module.h"
#include "llvm/IR/VFABIDemangler.h"
#include "llvm/IR/VectorTypeUtils.h"
diff --git a/llvm/lib/Analysis/VectorUtils.cpp b/llvm/lib/Analysis/VectorUtils.cpp
index fe5fc300cf3e18..8b479de7037b9e 100644
--- a/llvm/lib/Analysis/VectorUtils.cpp
+++ b/llvm/lib/Analysis/VectorUtils.cpp
@@ -10,7 +10,6 @@
//
//===----------------------------------------------------------------------===//
-#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/VectorUtils.h"
#include "AnalysisOptions.h"
#include "llvm/ADT/EquivalenceClasses.h"
@@ -19,6 +18,7 @@
#include "llvm/Analysis/DemandedBits.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/LoopIterator.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/ScalarEvolution.h"
#include "llvm/Analysis/ScalarEvolutionExpressions.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
diff --git a/llvm/lib/Transforms/Utils/Local.cpp b/llvm/lib/Transforms/Utils/Local.cpp
index ac2223d5c169bc..16d7586564c816 100644
--- a/llvm/lib/Transforms/Utils/Local.cpp
+++ b/llvm/lib/Transforms/Utils/Local.cpp
@@ -11,7 +11,6 @@
//
//===----------------------------------------------------------------------===//
-#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Transforms/Utils/Local.h"
#include "llvm/ADT/APInt.h"
#include "llvm/ADT/DenseMap.h"
@@ -27,6 +26,7 @@
#include "llvm/Analysis/DomTreeUpdater.h"
#include "llvm/Analysis/InstructionSimplify.h"
#include "llvm/Analysis/MemoryBuiltins.h"
+#include "llvm/Analysis/MetadataUtils.h"
#include "llvm/Analysis/MemorySSAUpdater.h"
#include "llvm/Analysis/TargetLibraryInfo.h"
#include "llvm/Analysis/ValueTracking.h"
More information about the llvm-branch-commits
mailing list