std::string getPGOFuncName(StringRef RawFuncName,
GlobalValue::LinkageTypes Linkage,
- StringRef FileName) {
+ StringRef FileName,
+ uint64_t Version LLVM_ATTRIBUTE_UNUSED) {
// Function names may be prefixed with a binary '1' to indicate
// that the backend should not modify the symbols due to any platform
return FuncName;
}
-std::string getPGOFuncName(const Function &F) {
- return getPGOFuncName(F.getName(), F.getLinkage(), F.getParent()->getName());
+std::string getPGOFuncName(const Function &F, uint64_t Version) {
+ return getPGOFuncName(F.getName(), F.getLinkage(), F.getParent()->getName(),
+ Version);
+}
+
+StringRef getFuncNameWithoutPrefix(StringRef PGOFuncName, StringRef FileName) {
+ if (FileName.empty())
+ return PGOFuncName;
+ // Drop the file name including ':'. See also getPGOFuncName.
+ if (PGOFuncName.startswith(FileName))
+ PGOFuncName = PGOFuncName.drop_front(FileName.size() + 1);
+ return PGOFuncName;
+}
+
+// \p FuncName is the string used as profile lookup key for the function. A
+// symbol is created to hold the name. Return the legalized symbol name.
+static std::string getPGOFuncNameVarName(StringRef FuncName,
+ GlobalValue::LinkageTypes Linkage) {
+ std::string VarName = getInstrProfNameVarPrefix();
+ VarName += FuncName;
+
+ if (!GlobalValue::isLocalLinkage(Linkage))
+ return VarName;
+
+ // Now fix up illegal chars in local VarName that may upset the assembler.
+ const char *InvalidChars = "-:<>\"'";
+ size_t found = VarName.find_first_of(InvalidChars);
+ while (found != std::string::npos) {
+ VarName[found] = '_';
+ found = VarName.find_first_of(InvalidChars, found + 1);
+ }
+ return VarName;
}
GlobalVariable *createPGOFuncNameVar(Module &M,
auto *Value = ConstantDataArray::getString(M.getContext(), FuncName, false);
auto FuncNameVar =
new GlobalVariable(M, Value->getType(), true, Linkage, Value,
- Twine(getInstrProfNameVarPrefix()) + FuncName);
+ getPGOFuncNameVarName(FuncName, Linkage));
// Hide the symbol so that we correctly get a copy for each executable.
if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage()))
return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), FuncName);
}
-/// Return the total size in bytes of the on-disk value profile data
-/// given the data stored in Record.
-uint32_t getValueProfDataSize(ValueProfRecordClosure *Closure) {
- uint32_t Kind;
- uint32_t TotalSize = sizeof(ValueProfData);
- const void *Record = Closure->Record;
- uint32_t NumValueKinds = Closure->GetNumValueKinds(Record);
- if (NumValueKinds == 0)
- return TotalSize;
-
- for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
- uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind);
- if (!NumValueSites)
+instrprof_error
+InstrProfValueSiteRecord::mergeValueData(InstrProfValueSiteRecord &Input,
+ uint64_t Weight) {
+ this->sortByTargetValues();
+ Input.sortByTargetValues();
+ auto I = ValueData.begin();
+ auto IE = ValueData.end();
+ instrprof_error Result = instrprof_error::success;
+ for (auto J = Input.ValueData.begin(), JE = Input.ValueData.end(); J != JE;
+ ++J) {
+ while (I != IE && I->Value < J->Value)
+ ++I;
+ if (I != IE && I->Value == J->Value) {
+ uint64_t JCount = J->Count;
+ bool Overflowed;
+ if (Weight > 1) {
+ JCount = SaturatingMultiply(JCount, Weight, &Overflowed);
+ if (Overflowed)
+ Result = instrprof_error::counter_overflow;
+ }
+ I->Count = SaturatingAdd(I->Count, JCount, &Overflowed);
+ if (Overflowed)
+ Result = instrprof_error::counter_overflow;
+ ++I;
continue;
- TotalSize += getValueProfRecordSize(NumValueSites,
- Closure->GetNumValueData(Record, Kind));
+ }
+ ValueData.insert(I, *J);
+ }
+ return Result;
+}
+
+// Merge Value Profile data from Src record to this record for ValueKind.
+// Scale merged value counts by \p Weight.
+instrprof_error InstrProfRecord::mergeValueProfData(uint32_t ValueKind,
+ InstrProfRecord &Src,
+ uint64_t Weight) {
+ uint32_t ThisNumValueSites = getNumValueSites(ValueKind);
+ uint32_t OtherNumValueSites = Src.getNumValueSites(ValueKind);
+ if (ThisNumValueSites != OtherNumValueSites)
+ return instrprof_error::value_site_count_mismatch;
+ std::vector<InstrProfValueSiteRecord> &ThisSiteRecords =
+ getValueSitesForKind(ValueKind);
+ std::vector<InstrProfValueSiteRecord> &OtherSiteRecords =
+ Src.getValueSitesForKind(ValueKind);
+ instrprof_error Result = instrprof_error::success;
+ for (uint32_t I = 0; I < ThisNumValueSites; I++)
+ MergeResult(Result,
+ ThisSiteRecords[I].mergeValueData(OtherSiteRecords[I], Weight));
+ return Result;
+}
+
+instrprof_error InstrProfRecord::merge(InstrProfRecord &Other,
+ uint64_t Weight) {
+ // If the number of counters doesn't match we either have bad data
+ // or a hash collision.
+ if (Counts.size() != Other.Counts.size())
+ return instrprof_error::count_mismatch;
+
+ instrprof_error Result = instrprof_error::success;
+
+ for (size_t I = 0, E = Other.Counts.size(); I < E; ++I) {
+ bool Overflowed;
+ uint64_t OtherCount = Other.Counts[I];
+ if (Weight > 1) {
+ OtherCount = SaturatingMultiply(OtherCount, Weight, &Overflowed);
+ if (Overflowed)
+ Result = instrprof_error::counter_overflow;
+ }
+ Counts[I] = SaturatingAdd(Counts[I], OtherCount, &Overflowed);
+ if (Overflowed)
+ Result = instrprof_error::counter_overflow;
+ }
+
+ for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind)
+ MergeResult(Result, mergeValueProfData(Kind, Other, Weight));
+
+ return Result;
+}
+// Map indirect call target name hash to name string.
+uint64_t InstrProfRecord::remapValue(uint64_t Value, uint32_t ValueKind,
+ ValueMapType *HashKeys) {
+ if (!HashKeys)
+ return Value;
+ switch (ValueKind) {
+ case IPVK_IndirectCallTarget: {
+ auto Result =
+ std::lower_bound(HashKeys->begin(), HashKeys->end(), Value,
+ [](const std::pair<uint64_t, const char *> &LHS,
+ uint64_t RHS) { return LHS.first < RHS; });
+ if (Result != HashKeys->end())
+ Value = (uint64_t)Result->second;
+ break;
}
- return TotalSize;
-}
-
-// Extract data from \c Closure and serialize into \c This instance.
-void serializeValueProfRecordFrom(ValueProfRecord *This,
- ValueProfRecordClosure *Closure,
- uint32_t ValueKind, uint32_t NumValueSites) {
- uint32_t S;
- const void *Record = Closure->Record;
- This->Kind = ValueKind;
- This->NumValueSites = NumValueSites;
- InstrProfValueData *DstVD = getValueProfRecordValueData(This);
-
- for (S = 0; S < NumValueSites; S++) {
- uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S);
- This->SiteCountArray[S] = ND;
- Closure->GetValueForSite(Record, DstVD, ValueKind, S,
- Closure->RemapValueData);
- DstVD += ND;
}
+ return Value;
}
-ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure) {
- uint32_t TotalSize = getValueProfDataSize(Closure);
+void InstrProfRecord::updateStrings(InstrProfStringTable *StrTab) {
+ if (!StrTab)
+ return;
- ValueProfData *VPD = Closure->AllocValueProfData(TotalSize);
+ Name = StrTab->insertString(Name);
+ for (auto &VSite : IndirectCallSites)
+ for (auto &VData : VSite.ValueData)
+ VData.Value = (uint64_t)StrTab->insertString((const char *)VData.Value);
+}
- VPD->TotalSize = TotalSize;
- VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record);
- ValueProfRecord *VR = getFirstValueProfRecord(VPD);
- for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
- uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind);
- if (!NumValueSites)
- continue;
- serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites);
- VR = getValueProfRecordNext(VR);
+void InstrProfRecord::addValueData(uint32_t ValueKind, uint32_t Site,
+ InstrProfValueData *VData, uint32_t N,
+ ValueMapType *HashKeys) {
+ for (uint32_t I = 0; I < N; I++) {
+ VData[I].Value = remapValue(VData[I].Value, ValueKind, HashKeys);
}
- return VPD;
+ std::vector<InstrProfValueSiteRecord> &ValueSites =
+ getValueSitesForKind(ValueKind);
+ if (N == 0)
+ ValueSites.push_back(InstrProfValueSiteRecord());
+ else
+ ValueSites.emplace_back(VData, VData + N);
}
-// C wrappers of InstrProfRecord member functions used in Closure.
-// These C wrappers are used as adaptors so that C++ code can be
-// invoked as callbacks.
+#define INSTR_PROF_COMMON_API_IMPL
+#include "llvm/ProfileData/InstrProfData.inc"
+
+/*!
+ * \brief ValueProfRecordClosure Interface implementation for InstrProfRecord
+ * class. These C wrappers are used as adaptors so that C++ code can be
+ * invoked as callbacks.
+ */
uint32_t getNumValueKindsInstrProf(const void *Record) {
return reinterpret_cast<const InstrProfRecord *>(Record)->getNumValueKinds();
}
uint64_t stringToHash(uint32_t ValueKind, uint64_t Value) {
switch (ValueKind) {
case IPVK_IndirectCallTarget:
- return IndexedInstrProf::ComputeHash(IndexedInstrProf::HashType,
- (const char *)Value);
+ return IndexedInstrProf::ComputeHash((const char *)Value);
break;
default:
llvm_unreachable("value kind not handled !");
}
ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) {
- return (ValueProfData *)(new (::operator new(TotalSizeInBytes))
- ValueProfData());
+ ValueProfData *VD =
+ (ValueProfData *)(new (::operator new(TotalSizeInBytes)) ValueProfData());
+ memset(VD, 0, TotalSizeInBytes);
+ return VD;
}
static ValueProfRecordClosure InstrProfRecordClosure = {
getNumValueDataForSiteInstrProf,
stringToHash,
getValueForSiteInstrProf,
- allocValueProfDataInstrProf
-};
+ allocValueProfDataInstrProf};
// Wrapper implementation using the closure mechanism.
uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
InstrProfRecordClosure.Record = &Record;
std::unique_ptr<ValueProfData> VPD(
- serializeValueProfDataFrom(&InstrProfRecordClosure));
+ serializeValueProfDataFrom(&InstrProfRecordClosure, nullptr));
return VPD;
}
ValueData += ValueDataCount;
}
}
+
// For writing/serializing, Old is the host endianness, and New is
// byte order intended on disk. For Reading/deserialization, Old
// is the on-disk source endianness, and New is the host endianness.
ValueProfData());
}
+instrprof_error ValueProfData::checkIntegrity() {
+ if (NumValueKinds > IPVK_Last + 1)
+ return instrprof_error::malformed;
+ // Total size needs to be mulltiple of quadword size.
+ if (TotalSize % sizeof(uint64_t))
+ return instrprof_error::malformed;
+
+ ValueProfRecord *VR = getFirstValueProfRecord(this);
+ for (uint32_t K = 0; K < this->NumValueKinds; K++) {
+ if (VR->Kind > IPVK_Last)
+ return instrprof_error::malformed;
+ VR = getValueProfRecordNext(VR);
+ if ((char *)VR - (char *)this > (ptrdiff_t)TotalSize)
+ return instrprof_error::malformed;
+ }
+ return instrprof_error::success;
+}
+
ErrorOr<std::unique_ptr<ValueProfData>>
ValueProfData::getValueProfData(const unsigned char *D,
const unsigned char *const BufferEnd,
const unsigned char *Header = D;
uint32_t TotalSize = swapToHostOrder<uint32_t>(Header, Endianness);
- uint32_t NumValueKinds = swapToHostOrder<uint32_t>(Header, Endianness);
-
if (D + TotalSize > BufferEnd)
return instrprof_error::too_large;
- if (NumValueKinds > IPVK_Last + 1)
- return instrprof_error::malformed;
- // Total size needs to be mulltiple of quadword size.
- if (TotalSize % sizeof(uint64_t))
- return instrprof_error::malformed;
std::unique_ptr<ValueProfData> VPD = allocValueProfData(TotalSize);
-
memcpy(VPD.get(), D, TotalSize);
// Byte swap.
VPD->swapBytesToHost(Endianness);
- // Data integrity check:
- ValueProfRecord *VR = getFirstValueProfRecord(VPD.get());
- for (uint32_t K = 0; K < VPD->NumValueKinds; K++) {
- if (VR->Kind > IPVK_Last)
- return instrprof_error::malformed;
- VR = getValueProfRecordNext(VR);
- if ((char *)VR - (char *)VPD.get() > (ptrdiff_t)TotalSize)
- return instrprof_error::malformed;
- }
+ instrprof_error EC = VPD->checkIntegrity();
+ if (EC != instrprof_error::success)
+ return EC;
return std::move(VPD);
}
projects / oota-llvm.git / blobdiff