X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FProfileData%2FInstrProf.cpp;h=8a3a61ff67935429d66d6b411580d6da2d78d416;hb=f722f529e7f3147ee286fb1d0baa2bb4e2313967;hp=9652ba88483363c4d532a1582c82c0341381f898;hpb=c96f87a89298932b0f4f3187a4270f558c48898a;p=oota-llvm.git diff --git a/lib/ProfileData/InstrProf.cpp b/lib/ProfileData/InstrProf.cpp index 9652ba88483..8a3a61ff679 100644 --- a/lib/ProfileData/InstrProf.cpp +++ b/lib/ProfileData/InstrProf.cpp @@ -12,6 +12,10 @@ // //===----------------------------------------------------------------------===// +#include "llvm/IR/Constants.h" +#include "llvm/IR/Function.h" +#include "llvm/IR/Module.h" +#include "llvm/IR/GlobalVariable.h" #include "llvm/ProfileData/InstrProf.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/ManagedStatic.h" @@ -28,30 +32,32 @@ class InstrProfErrorCategoryType : public std::error_category { return "Success"; case instrprof_error::eof: return "End of File"; + case instrprof_error::unrecognized_format: + return "Unrecognized instrumentation profile encoding format"; case instrprof_error::bad_magic: - return "Invalid profile data (bad magic)"; + return "Invalid instrumentation profile data (bad magic)"; case instrprof_error::bad_header: - return "Invalid profile data (file header is corrupt)"; + return "Invalid instrumentation profile data (file header is corrupt)"; case instrprof_error::unsupported_version: - return "Unsupported profiling format version"; + return "Unsupported instrumentation profile format version"; case instrprof_error::unsupported_hash_type: - return "Unsupported profiling hash"; + return "Unsupported instrumentation profile hash type"; case instrprof_error::too_large: return "Too much profile data"; case instrprof_error::truncated: return "Truncated profile data"; case instrprof_error::malformed: - return "Malformed profile data"; + return "Malformed instrumentation profile data"; case instrprof_error::unknown_function: return "No profile data available for function"; case instrprof_error::hash_mismatch: - return "Function hash mismatch"; + return "Function control flow change detected (hash mismatch)"; case instrprof_error::count_mismatch: - return "Function count mismatch"; + return "Function basic block count change detected (counter mismatch)"; case instrprof_error::counter_overflow: return "Counter overflow"; case instrprof_error::value_site_count_mismatch: - return "Function's value site counts mismatch"; + return "Function value site count change detected (counter mismatch)"; } llvm_unreachable("A value of instrprof_error has no message."); } @@ -63,3 +69,330 @@ static ManagedStatic ErrorCategory; const std::error_category &llvm::instrprof_category() { return *ErrorCategory; } + +namespace llvm { + +std::string getPGOFuncName(StringRef RawFuncName, + GlobalValue::LinkageTypes Linkage, + StringRef FileName) { + + // Function names may be prefixed with a binary '1' to indicate + // that the backend should not modify the symbols due to any platform + // naming convention. Do not include that '1' in the PGO profile name. + if (RawFuncName[0] == '\1') + RawFuncName = RawFuncName.substr(1); + + std::string FuncName = RawFuncName; + if (llvm::GlobalValue::isLocalLinkage(Linkage)) { + // For local symbols, prepend the main file name to distinguish them. + // Do not include the full path in the file name since there's no guarantee + // that it will stay the same, e.g., if the files are checked out from + // version control in different locations. + if (FileName.empty()) + FuncName = FuncName.insert(0, ":"); + else + FuncName = FuncName.insert(0, FileName.str() + ":"); + } + return FuncName; +} + +std::string getPGOFuncName(const Function &F) { + return getPGOFuncName(F.getName(), F.getLinkage(), F.getParent()->getName()); +} + +GlobalVariable *createPGOFuncNameVar(Module &M, + GlobalValue::LinkageTypes Linkage, + StringRef FuncName) { + + // We generally want to match the function's linkage, but available_externally + // and extern_weak both have the wrong semantics, and anything that doesn't + // need to link across compilation units doesn't need to be visible at all. + if (Linkage == GlobalValue::ExternalWeakLinkage) + Linkage = GlobalValue::LinkOnceAnyLinkage; + else if (Linkage == GlobalValue::AvailableExternallyLinkage) + Linkage = GlobalValue::LinkOnceODRLinkage; + else if (Linkage == GlobalValue::InternalLinkage || + Linkage == GlobalValue::ExternalLinkage) + Linkage = GlobalValue::PrivateLinkage; + + auto *Value = ConstantDataArray::getString(M.getContext(), FuncName, false); + auto FuncNameVar = + new GlobalVariable(M, Value->getType(), true, Linkage, Value, + Twine(getInstrProfNameVarPrefix()) + FuncName); + + // Hide the symbol so that we correctly get a copy for each executable. + if (!GlobalValue::isLocalLinkage(FuncNameVar->getLinkage())) + FuncNameVar->setVisibility(GlobalValue::HiddenVisibility); + + return FuncNameVar; +} + +GlobalVariable *createPGOFuncNameVar(Function &F, StringRef FuncName) { + return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), FuncName); +} + +/// Return the total size in bytes of the on-disk value profile data +/// given the data stored in Record. +uint32_t getValueProfDataSize(ValueProfRecordClosure *Closure) { + uint32_t Kind; + uint32_t TotalSize = sizeof(ValueProfData); + const void *Record = Closure->Record; + uint32_t NumValueKinds = Closure->GetNumValueKinds(Record); + if (NumValueKinds == 0) + return TotalSize; + + for (Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { + uint32_t NumValueSites = Closure->GetNumValueSites(Record, Kind); + if (!NumValueSites) + continue; + TotalSize += getValueProfRecordSize(NumValueSites, + Closure->GetNumValueData(Record, Kind)); + } + return TotalSize; +} + +// Extract data from \c Closure and serialize into \c This instance. +void serializeValueProfRecordFrom(ValueProfRecord *This, + ValueProfRecordClosure *Closure, + uint32_t ValueKind, uint32_t NumValueSites) { + uint32_t S; + const void *Record = Closure->Record; + This->Kind = ValueKind; + This->NumValueSites = NumValueSites; + InstrProfValueData *DstVD = getValueProfRecordValueData(This); + + for (S = 0; S < NumValueSites; S++) { + uint32_t ND = Closure->GetNumValueDataForSite(Record, ValueKind, S); + This->SiteCountArray[S] = ND; + Closure->GetValueForSite(Record, DstVD, ValueKind, S, + Closure->RemapValueData); + DstVD += ND; + } +} + +ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure) { + uint32_t TotalSize = getValueProfDataSize(Closure); + + ValueProfData *VPD = Closure->AllocValueProfData(TotalSize); + + VPD->TotalSize = TotalSize; + VPD->NumValueKinds = Closure->GetNumValueKinds(Closure->Record); + ValueProfRecord *VR = getFirstValueProfRecord(VPD); + for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; Kind++) { + uint32_t NumValueSites = Closure->GetNumValueSites(Closure->Record, Kind); + if (!NumValueSites) + continue; + serializeValueProfRecordFrom(VR, Closure, Kind, NumValueSites); + VR = getValueProfRecordNext(VR); + } + return VPD; +} + +// C wrappers of InstrProfRecord member functions used in Closure. +// These C wrappers are used as adaptors so that C++ code can be +// invoked as callbacks. +uint32_t getNumValueKindsInstrProf(const void *Record) { + return reinterpret_cast(Record)->getNumValueKinds(); +} + +uint32_t getNumValueSitesInstrProf(const void *Record, uint32_t VKind) { + return reinterpret_cast(Record) + ->getNumValueSites(VKind); +} + +uint32_t getNumValueDataInstrProf(const void *Record, uint32_t VKind) { + return reinterpret_cast(Record) + ->getNumValueData(VKind); +} + +uint32_t getNumValueDataForSiteInstrProf(const void *R, uint32_t VK, + uint32_t S) { + return reinterpret_cast(R) + ->getNumValueDataForSite(VK, S); +} + +void getValueForSiteInstrProf(const void *R, InstrProfValueData *Dst, + uint32_t K, uint32_t S, + uint64_t (*Mapper)(uint32_t, uint64_t)) { + return reinterpret_cast(R) + ->getValueForSite(Dst, K, S, Mapper); +} + +uint64_t stringToHash(uint32_t ValueKind, uint64_t Value) { + switch (ValueKind) { + case IPVK_IndirectCallTarget: + return IndexedInstrProf::ComputeHash(IndexedInstrProf::HashType, + (const char *)Value); + break; + default: + llvm_unreachable("value kind not handled !"); + } + return Value; +} + +ValueProfData *allocValueProfDataInstrProf(size_t TotalSizeInBytes) { + return (ValueProfData *)(new (::operator new(TotalSizeInBytes)) + ValueProfData()); +} + +static ValueProfRecordClosure InstrProfRecordClosure = { + 0, + getNumValueKindsInstrProf, + getNumValueSitesInstrProf, + getNumValueDataInstrProf, + getNumValueDataForSiteInstrProf, + stringToHash, + getValueForSiteInstrProf, + allocValueProfDataInstrProf +}; + +// Wrapper implementation using the closure mechanism. +uint32_t ValueProfData::getSize(const InstrProfRecord &Record) { + InstrProfRecordClosure.Record = &Record; + return getValueProfDataSize(&InstrProfRecordClosure); +} + +// Wrapper implementation using the closure mechanism. +std::unique_ptr +ValueProfData::serializeFrom(const InstrProfRecord &Record) { + InstrProfRecordClosure.Record = &Record; + + std::unique_ptr VPD( + serializeValueProfDataFrom(&InstrProfRecordClosure)); + return VPD; +} + +void ValueProfRecord::deserializeTo(InstrProfRecord &Record, + InstrProfRecord::ValueMapType *VMap) { + Record.reserveSites(Kind, NumValueSites); + + InstrProfValueData *ValueData = getValueProfRecordValueData(this); + for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) { + uint8_t ValueDataCount = this->SiteCountArray[VSite]; + Record.addValueData(Kind, VSite, ValueData, ValueDataCount, VMap); + ValueData += ValueDataCount; + } +} +// For writing/serializing, Old is the host endianness, and New is +// byte order intended on disk. For Reading/deserialization, Old +// is the on-disk source endianness, and New is the host endianness. +void ValueProfRecord::swapBytes(support::endianness Old, + support::endianness New) { + using namespace support; + if (Old == New) + return; + + if (getHostEndianness() != Old) { + sys::swapByteOrder(NumValueSites); + sys::swapByteOrder(Kind); + } + uint32_t ND = getValueProfRecordNumValueData(this); + InstrProfValueData *VD = getValueProfRecordValueData(this); + + // No need to swap byte array: SiteCountArrray. + for (uint32_t I = 0; I < ND; I++) { + sys::swapByteOrder(VD[I].Value); + sys::swapByteOrder(VD[I].Count); + } + if (getHostEndianness() == Old) { + sys::swapByteOrder(NumValueSites); + sys::swapByteOrder(Kind); + } +} + +void ValueProfData::deserializeTo(InstrProfRecord &Record, + InstrProfRecord::ValueMapType *VMap) { + if (NumValueKinds == 0) + return; + + ValueProfRecord *VR = getFirstValueProfRecord(this); + for (uint32_t K = 0; K < NumValueKinds; K++) { + VR->deserializeTo(Record, VMap); + VR = getValueProfRecordNext(VR); + } +} + +template +static T swapToHostOrder(const unsigned char *&D, support::endianness Orig) { + using namespace support; + if (Orig == little) + return endian::readNext(D); + else + return endian::readNext(D); +} + +static std::unique_ptr allocValueProfData(uint32_t TotalSize) { + return std::unique_ptr(new (::operator new(TotalSize)) + ValueProfData()); +} + +ErrorOr> +ValueProfData::getValueProfData(const unsigned char *D, + const unsigned char *const BufferEnd, + support::endianness Endianness) { + using namespace support; + if (D + sizeof(ValueProfData) > BufferEnd) + return instrprof_error::truncated; + + const unsigned char *Header = D; + uint32_t TotalSize = swapToHostOrder(Header, Endianness); + uint32_t NumValueKinds = swapToHostOrder(Header, Endianness); + + if (D + TotalSize > BufferEnd) + return instrprof_error::too_large; + if (NumValueKinds > IPVK_Last + 1) + return instrprof_error::malformed; + // Total size needs to be mulltiple of quadword size. + if (TotalSize % sizeof(uint64_t)) + return instrprof_error::malformed; + + std::unique_ptr VPD = allocValueProfData(TotalSize); + + memcpy(VPD.get(), D, TotalSize); + // Byte swap. + VPD->swapBytesToHost(Endianness); + + // Data integrity check: + ValueProfRecord *VR = getFirstValueProfRecord(VPD.get()); + for (uint32_t K = 0; K < VPD->NumValueKinds; K++) { + if (VR->Kind > IPVK_Last) + return instrprof_error::malformed; + VR = getValueProfRecordNext(VR); + if ((char *)VR - (char *)VPD.get() > (ptrdiff_t)TotalSize) + return instrprof_error::malformed; + } + + return std::move(VPD); +} + +void ValueProfData::swapBytesToHost(support::endianness Endianness) { + using namespace support; + if (Endianness == getHostEndianness()) + return; + + sys::swapByteOrder(TotalSize); + sys::swapByteOrder(NumValueKinds); + + ValueProfRecord *VR = getFirstValueProfRecord(this); + for (uint32_t K = 0; K < NumValueKinds; K++) { + VR->swapBytes(Endianness, getHostEndianness()); + VR = getValueProfRecordNext(VR); + } +} + +void ValueProfData::swapBytesFromHost(support::endianness Endianness) { + using namespace support; + if (Endianness == getHostEndianness()) + return; + + ValueProfRecord *VR = getFirstValueProfRecord(this); + for (uint32_t K = 0; K < NumValueKinds; K++) { + ValueProfRecord *NVR = getValueProfRecordNext(VR); + VR->swapBytes(getHostEndianness(), Endianness); + VR = NVR; + } + sys::swapByteOrder(TotalSize); + sys::swapByteOrder(NumValueKinds); +} + +}