+MetadataAsValue::MetadataAsValue(Type *Ty, Metadata *MD)
+ : Value(Ty, MetadataAsValueVal), MD(MD) {
+ track();
+}
+
+MetadataAsValue::~MetadataAsValue() {
+ getType()->getContext().pImpl->MetadataAsValues.erase(MD);
+ untrack();
+}
+
+/// \brief Canonicalize metadata arguments to intrinsics.
+///
+/// To support bitcode upgrades (and assembly semantic sugar) for \a
+/// MetadataAsValue, we need to canonicalize certain metadata.
+///
+/// - nullptr is replaced by an empty MDNode.
+/// - An MDNode with a single null operand is replaced by an empty MDNode.
+/// - An MDNode whose only operand is a \a ConstantAsMetadata gets skipped.
+///
+/// This maintains readability of bitcode from when metadata was a type of
+/// value, and these bridges were unnecessary.
+static Metadata *canonicalizeMetadataForValue(LLVMContext &Context,
+ Metadata *MD) {
+ if (!MD)
+ // !{}
+ return MDNode::get(Context, None);
+
+ // Return early if this isn't a single-operand MDNode.
+ auto *N = dyn_cast<MDNode>(MD);
+ if (!N || N->getNumOperands() != 1)
+ return MD;
+
+ if (!N->getOperand(0))
+ // !{}
+ return MDNode::get(Context, None);
+
+ if (auto *C = dyn_cast<ConstantAsMetadata>(N->getOperand(0)))
+ // Look through the MDNode.
+ return C;
+
+ return MD;
+}
+
+MetadataAsValue *MetadataAsValue::get(LLVMContext &Context, Metadata *MD) {
+ MD = canonicalizeMetadataForValue(Context, MD);
+ auto *&Entry = Context.pImpl->MetadataAsValues[MD];
+ if (!Entry)
+ Entry = new MetadataAsValue(Type::getMetadataTy(Context), MD);
+ return Entry;
+}
+
+MetadataAsValue *MetadataAsValue::getIfExists(LLVMContext &Context,
+ Metadata *MD) {
+ MD = canonicalizeMetadataForValue(Context, MD);
+ auto &Store = Context.pImpl->MetadataAsValues;
+ auto I = Store.find(MD);
+ return I == Store.end() ? nullptr : I->second;
+}
+
+void MetadataAsValue::handleChangedMetadata(Metadata *MD) {
+ LLVMContext &Context = getContext();
+ MD = canonicalizeMetadataForValue(Context, MD);
+ auto &Store = Context.pImpl->MetadataAsValues;
+
+ // Stop tracking the old metadata.
+ Store.erase(this->MD);
+ untrack();
+ this->MD = nullptr;
+
+ // Start tracking MD, or RAUW if necessary.
+ auto *&Entry = Store[MD];
+ if (Entry) {
+ replaceAllUsesWith(Entry);
+ delete this;
+ return;
+ }
+
+ this->MD = MD;
+ track();
+ Entry = this;
+}
+
+void MetadataAsValue::track() {
+ if (MD)
+ MetadataTracking::track(&MD, *MD, *this);
+}
+
+void MetadataAsValue::untrack() {
+ if (MD)
+ MetadataTracking::untrack(MD);
+}
+
+void ReplaceableMetadataImpl::addRef(void *Ref, OwnerTy Owner) {
+ bool WasInserted =
+ UseMap.insert(std::make_pair(Ref, std::make_pair(Owner, NextIndex)))
+ .second;
+ (void)WasInserted;
+ assert(WasInserted && "Expected to add a reference");
+
+ ++NextIndex;
+ assert(NextIndex != 0 && "Unexpected overflow");
+}
+
+void ReplaceableMetadataImpl::dropRef(void *Ref) {
+ bool WasErased = UseMap.erase(Ref);
+ (void)WasErased;
+ assert(WasErased && "Expected to drop a reference");
+}
+
+void ReplaceableMetadataImpl::moveRef(void *Ref, void *New,
+ const Metadata &MD) {
+ auto I = UseMap.find(Ref);
+ assert(I != UseMap.end() && "Expected to move a reference");
+ auto OwnerAndIndex = I->second;
+ UseMap.erase(I);
+ bool WasInserted = UseMap.insert(std::make_pair(New, OwnerAndIndex)).second;
+ (void)WasInserted;
+ assert(WasInserted && "Expected to add a reference");
+
+ // Check that the references are direct if there's no owner.
+ (void)MD;
+ assert((OwnerAndIndex.first || *static_cast<Metadata **>(Ref) == &MD) &&
+ "Reference without owner must be direct");
+ assert((OwnerAndIndex.first || *static_cast<Metadata **>(New) == &MD) &&
+ "Reference without owner must be direct");
+}
+
+void ReplaceableMetadataImpl::replaceAllUsesWith(Metadata *MD) {
+ assert(!(MD && isa<MDNodeFwdDecl>(MD)) && "Expected non-temp node");
+
+ if (UseMap.empty())
+ return;
+
+ // Copy out uses since UseMap will get touched below.
+ typedef std::pair<void *, std::pair<OwnerTy, uint64_t>> UseTy;
+ SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
+ std::sort(Uses.begin(), Uses.end(), [](const UseTy &L, const UseTy &R) {
+ return L.second.second < R.second.second;
+ });
+ for (const auto &Pair : Uses) {
+ OwnerTy Owner = Pair.second.first;
+ if (!Owner) {
+ // Update unowned tracking references directly.
+ Metadata *&Ref = *static_cast<Metadata **>(Pair.first);
+ Ref = MD;
+ if (MD)
+ MetadataTracking::track(Ref);
+ UseMap.erase(Pair.first);
+ continue;
+ }
+
+ // Check for MetadataAsValue.
+ if (Owner.is<MetadataAsValue *>()) {
+ Owner.get<MetadataAsValue *>()->handleChangedMetadata(MD);
+ continue;
+ }
+
+ // There's a Metadata owner -- dispatch.
+ Metadata *OwnerMD = Owner.get<Metadata *>();
+ switch (OwnerMD->getMetadataID()) {
+#define HANDLE_METADATA_LEAF(CLASS) \
+ case Metadata::CLASS##Kind: \
+ cast<CLASS>(OwnerMD)->handleChangedOperand(Pair.first, MD); \
+ continue;
+#include "llvm/IR/Metadata.def"
+ default:
+ llvm_unreachable("Invalid metadata subclass");
+ }
+ }
+ assert(UseMap.empty() && "Expected all uses to be replaced");
+}
+
+void ReplaceableMetadataImpl::resolveAllUses(bool ResolveUsers) {
+ if (UseMap.empty())
+ return;
+
+ if (!ResolveUsers) {
+ UseMap.clear();
+ return;
+ }
+
+ // Copy out uses since UseMap could get touched below.
+ typedef std::pair<void *, std::pair<OwnerTy, uint64_t>> UseTy;
+ SmallVector<UseTy, 8> Uses(UseMap.begin(), UseMap.end());
+ std::sort(Uses.begin(), Uses.end(), [](const UseTy &L, const UseTy &R) {
+ return L.second.second < R.second.second;
+ });
+ UseMap.clear();
+ for (const auto &Pair : Uses) {
+ auto Owner = Pair.second.first;
+ if (!Owner)
+ continue;
+ if (Owner.is<MetadataAsValue *>())
+ continue;
+
+ // Resolve GenericMDNodes that point at this.
+ auto *OwnerMD = dyn_cast<GenericMDNode>(Owner.get<Metadata *>());
+ if (!OwnerMD)
+ continue;
+ if (OwnerMD->isResolved())
+ continue;
+ if (!--OwnerMD->SubclassData32)
+ OwnerMD->resolve();
+ }
+}
+
+static Function *getLocalFunction(Value *V) {
+ assert(V && "Expected value");
+ if (auto *A = dyn_cast<Argument>(V))
+ return A->getParent();
+ if (BasicBlock *BB = cast<Instruction>(V)->getParent())
+ return BB->getParent();
+ return nullptr;
+}
+
+ValueAsMetadata *ValueAsMetadata::get(Value *V) {
+ assert(V && "Unexpected null Value");
+
+ auto &Context = V->getContext();
+ auto *&Entry = Context.pImpl->ValuesAsMetadata[V];
+ if (!Entry) {
+ assert((isa<Constant>(V) || isa<Argument>(V) || isa<Instruction>(V)) &&
+ "Expected constant or function-local value");
+ assert(!V->NameAndIsUsedByMD.getInt() &&
+ "Expected this to be the only metadata use");
+ V->NameAndIsUsedByMD.setInt(true);
+ if (auto *C = dyn_cast<Constant>(V))
+ Entry = new ConstantAsMetadata(C);
+ else
+ Entry = new LocalAsMetadata(V);
+ }
+
+ return Entry;
+}
+
+ValueAsMetadata *ValueAsMetadata::getIfExists(Value *V) {
+ assert(V && "Unexpected null Value");
+ return V->getContext().pImpl->ValuesAsMetadata.lookup(V);
+}
+
+void ValueAsMetadata::handleDeletion(Value *V) {
+ assert(V && "Expected valid value");
+
+ auto &Store = V->getType()->getContext().pImpl->ValuesAsMetadata;
+ auto I = Store.find(V);
+ if (I == Store.end())
+ return;
+
+ // Remove old entry from the map.
+ ValueAsMetadata *MD = I->second;
+ assert(MD && "Expected valid metadata");
+ assert(MD->getValue() == V && "Expected valid mapping");
+ Store.erase(I);
+
+ // Delete the metadata.
+ MD->replaceAllUsesWith(nullptr);
+ delete MD;
+}
+
+void ValueAsMetadata::handleRAUW(Value *From, Value *To) {
+ assert(From && "Expected valid value");
+ assert(To && "Expected valid value");
+ assert(From != To && "Expected changed value");
+ assert(From->getType() == To->getType() && "Unexpected type change");
+
+ LLVMContext &Context = From->getType()->getContext();
+ auto &Store = Context.pImpl->ValuesAsMetadata;
+ auto I = Store.find(From);
+ if (I == Store.end()) {
+ assert(!From->NameAndIsUsedByMD.getInt() &&
+ "Expected From not to be used by metadata");
+ return;
+ }
+
+ // Remove old entry from the map.
+ assert(From->NameAndIsUsedByMD.getInt() &&
+ "Expected From to be used by metadata");
+ From->NameAndIsUsedByMD.setInt(false);
+ ValueAsMetadata *MD = I->second;
+ assert(MD && "Expected valid metadata");
+ assert(MD->getValue() == From && "Expected valid mapping");
+ Store.erase(I);
+
+ if (isa<LocalAsMetadata>(MD)) {
+ if (auto *C = dyn_cast<Constant>(To)) {
+ // Local became a constant.
+ MD->replaceAllUsesWith(ConstantAsMetadata::get(C));
+ delete MD;
+ return;
+ }
+ if (getLocalFunction(From) && getLocalFunction(To) &&
+ getLocalFunction(From) != getLocalFunction(To)) {
+ // Function changed.
+ MD->replaceAllUsesWith(nullptr);
+ delete MD;
+ return;
+ }
+ } else if (!isa<Constant>(To)) {
+ // Changed to function-local value.
+ MD->replaceAllUsesWith(nullptr);
+ delete MD;
+ return;
+ }
+
+ auto *&Entry = Store[To];
+ if (Entry) {
+ // The target already exists.
+ MD->replaceAllUsesWith(Entry);
+ delete MD;
+ return;
+ }
+
+ // Update MD in place (and update the map entry).
+ assert(!To->NameAndIsUsedByMD.getInt() &&
+ "Expected this to be the only metadata use");
+ To->NameAndIsUsedByMD.setInt(true);
+ MD->V = To;
+ Entry = MD;
+}