X-Git-Url: http://plrg.eecs.uci.edu/git/?a=blobdiff_plain;f=lib%2FProfileData%2FInstrProfReader.cpp;h=fe52e1312ef467d6565d7e4de9f6f07539aaa5f1;hb=931217997cd3c9696284e73cace5b3a761147b36;hp=8ad50615a25080440cf74e84975ecdf921a47f1d;hpb=f747d124daab9f732b6693f7b45a1abf213282b7;p=oota-llvm.git diff --git a/lib/ProfileData/InstrProfReader.cpp b/lib/ProfileData/InstrProfReader.cpp index 8ad50615a25..fe52e1312ef 100644 --- a/lib/ProfileData/InstrProfReader.cpp +++ b/lib/ProfileData/InstrProfReader.cpp @@ -13,7 +13,6 @@ //===----------------------------------------------------------------------===// #include "llvm/ProfileData/InstrProfReader.h" -#include "InstrProfIndexed.h" #include "llvm/ADT/STLExtras.h" #include @@ -55,8 +54,10 @@ InstrProfReader::create(std::unique_ptr Buffer) { Result.reset(new RawInstrProfReader64(std::move(Buffer))); else if (RawInstrProfReader32::hasFormat(*Buffer)) Result.reset(new RawInstrProfReader32(std::move(Buffer))); - else + else if (TextInstrProfReader::hasFormat(*Buffer)) Result.reset(new TextInstrProfReader(std::move(Buffer))); + else + return instrprof_error::unrecognized_format; // Initialize the reader and return the result. if (std::error_code EC = initializeReader(*Result)) @@ -98,6 +99,15 @@ void InstrProfIterator::Increment() { *this = InstrProfIterator(); } +bool TextInstrProfReader::hasFormat(const MemoryBuffer &Buffer) { + // Verify that this really looks like plain ASCII text by checking a + // 'reasonable' number of characters (up to profile magic size). + size_t count = std::min(Buffer.getBufferSize(), sizeof(uint64_t)); + StringRef buffer = Buffer.getBufferStart(); + return count == 0 || std::all_of(buffer.begin(), buffer.begin() + count, + [](char c) { return ::isprint(c) || ::isspace(c); }); +} + std::error_code TextInstrProfReader::readNextRecord(InstrProfRecord &Record) { // Skip empty lines and comments. while (!Line.is_at_end() && (Line->empty() || Line->startswith("#"))) @@ -139,54 +149,25 @@ std::error_code TextInstrProfReader::readNextRecord(InstrProfRecord &Record) { return success(); } -template -static uint64_t getRawMagic(); - -template <> -uint64_t getRawMagic() { - return - uint64_t(255) << 56 | - uint64_t('l') << 48 | - uint64_t('p') << 40 | - uint64_t('r') << 32 | - uint64_t('o') << 24 | - uint64_t('f') << 16 | - uint64_t('r') << 8 | - uint64_t(129); -} - -template <> -uint64_t getRawMagic() { - return - uint64_t(255) << 56 | - uint64_t('l') << 48 | - uint64_t('p') << 40 | - uint64_t('r') << 32 | - uint64_t('o') << 24 | - uint64_t('f') << 16 | - uint64_t('R') << 8 | - uint64_t(129); -} - template bool RawInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) { if (DataBuffer.getBufferSize() < sizeof(uint64_t)) return false; uint64_t Magic = *reinterpret_cast(DataBuffer.getBufferStart()); - return getRawMagic() == Magic || - sys::getSwappedBytes(getRawMagic()) == Magic; + return RawInstrProf::getMagic() == Magic || + sys::getSwappedBytes(RawInstrProf::getMagic()) == Magic; } template std::error_code RawInstrProfReader::readHeader() { if (!hasFormat(*DataBuffer)) return error(instrprof_error::bad_magic); - if (DataBuffer->getBufferSize() < sizeof(RawHeader)) + if (DataBuffer->getBufferSize() < sizeof(RawInstrProf::Header)) return error(instrprof_error::bad_header); - auto *Header = - reinterpret_cast(DataBuffer->getBufferStart()); - ShouldSwapBytes = Header->Magic != getRawMagic(); + auto *Header = reinterpret_cast( + DataBuffer->getBufferStart()); + ShouldSwapBytes = Header->Magic != RawInstrProf::getMagic(); return readHeader(*Header); } @@ -202,80 +183,107 @@ RawInstrProfReader::readNextHeader(const char *CurrentPos) { return instrprof_error::eof; // If there isn't enough space for another header, this is probably just // garbage at the end of the file. - if (CurrentPos + sizeof(RawHeader) > End) + if (CurrentPos + sizeof(RawInstrProf::Header) > End) return instrprof_error::malformed; // The writer ensures each profile is padded to start at an aligned address. if (reinterpret_cast(CurrentPos) % alignOf()) return instrprof_error::malformed; // The magic should have the same byte order as in the previous header. uint64_t Magic = *reinterpret_cast(CurrentPos); - if (Magic != swap(getRawMagic())) + if (Magic != swap(RawInstrProf::getMagic())) return instrprof_error::bad_magic; // There's another profile to read, so we need to process the header. - auto *Header = reinterpret_cast(CurrentPos); + auto *Header = reinterpret_cast(CurrentPos); return readHeader(*Header); } -static uint64_t getRawVersion() { - return 1; -} - template -std::error_code -RawInstrProfReader::readHeader(const RawHeader &Header) { - if (swap(Header.Version) != getRawVersion()) +std::error_code RawInstrProfReader::readHeader( + const RawInstrProf::Header &Header) { + if (swap(Header.Version) != RawInstrProf::Version) return error(instrprof_error::unsupported_version); CountersDelta = swap(Header.CountersDelta); NamesDelta = swap(Header.NamesDelta); + ValueDataDelta = swap(Header.ValueDataDelta); auto DataSize = swap(Header.DataSize); auto CountersSize = swap(Header.CountersSize); auto NamesSize = swap(Header.NamesSize); + auto ValueDataSize = swap(Header.ValueDataSize); + ValueKindLast = swap(Header.ValueKindLast); - ptrdiff_t DataOffset = sizeof(RawHeader); - ptrdiff_t CountersOffset = DataOffset + sizeof(ProfileData) * DataSize; + auto DataSizeInBytes = DataSize * sizeof(RawInstrProf::ProfileData); + auto PaddingSize = getNumPaddingBytes(NamesSize); + + ptrdiff_t DataOffset = sizeof(RawInstrProf::Header); + ptrdiff_t CountersOffset = DataOffset + DataSizeInBytes; ptrdiff_t NamesOffset = CountersOffset + sizeof(uint64_t) * CountersSize; - size_t ProfileSize = NamesOffset + sizeof(char) * NamesSize; + ptrdiff_t ValueDataOffset = NamesOffset + NamesSize + PaddingSize; + size_t ProfileSize = ValueDataOffset + ValueDataSize; auto *Start = reinterpret_cast(&Header); if (Start + ProfileSize > DataBuffer->getBufferEnd()) return error(instrprof_error::bad_header); - Data = reinterpret_cast(Start + DataOffset); + Data = reinterpret_cast *>( + Start + DataOffset); DataEnd = Data + DataSize; CountersStart = reinterpret_cast(Start + CountersOffset); NamesStart = Start + NamesOffset; + ValueDataStart = reinterpret_cast(Start + ValueDataOffset); ProfileEnd = Start + ProfileSize; + FunctionPtrToNameMap.clear(); + for (const RawInstrProf::ProfileData *I = Data; I != DataEnd; ++I) { + const IntPtrT FPtr = swap(I->FunctionPointer); + if (!FPtr) + continue; + StringRef FunctionName(getName(I->NamePtr), swap(I->NameSize)); + const char* NameEntryPtr = StringTable.insertString(FunctionName); + FunctionPtrToNameMap.push_back(std::pair + (FPtr, NameEntryPtr)); + } + std::sort(FunctionPtrToNameMap.begin(), FunctionPtrToNameMap.end(), less_first()); + FunctionPtrToNameMap.erase(std::unique(FunctionPtrToNameMap.begin(), + FunctionPtrToNameMap.end()), + FunctionPtrToNameMap.end()); return success(); } template -std::error_code -RawInstrProfReader::readNextRecord(InstrProfRecord &Record) { - if (Data == DataEnd) - if (std::error_code EC = readNextHeader(ProfileEnd)) - return EC; +std::error_code RawInstrProfReader::readName(InstrProfRecord &Record) { + Record.Name = StringRef(getName(Data->NamePtr), swap(Data->NameSize)); + if (Record.Name.data() < NamesStart || + Record.Name.data() + Record.Name.size() > + reinterpret_cast(ValueDataStart)) + return error(instrprof_error::malformed); + return success(); +} - // Get the raw data. - StringRef RawName(getName(Data->NamePtr), swap(Data->NameSize)); +template +std::error_code RawInstrProfReader::readFuncHash( + InstrProfRecord &Record) { + Record.Hash = swap(Data->FuncHash); + return success(); +} + +template +std::error_code RawInstrProfReader::readRawCounts( + InstrProfRecord &Record) { uint32_t NumCounters = swap(Data->NumCounters); + IntPtrT CounterPtr = Data->CounterPtr; if (NumCounters == 0) return error(instrprof_error::malformed); - auto RawCounts = makeArrayRef(getCounter(Data->CounterPtr), NumCounters); - // Check bounds. + auto RawCounts = makeArrayRef(getCounter(CounterPtr), NumCounters); auto *NamesStartAsCounter = reinterpret_cast(NamesStart); - if (RawName.data() < NamesStart || - RawName.data() + RawName.size() > DataBuffer->getBufferEnd() || - RawCounts.data() < CountersStart || + + // Check bounds. + if (RawCounts.data() < CountersStart || RawCounts.data() + RawCounts.size() > NamesStartAsCounter) return error(instrprof_error::malformed); - // Store the data in Record, byte-swapping as necessary. - Record.Hash = swap(Data->FuncHash); - Record.Name = RawName; if (ShouldSwapBytes) { Record.Counts.clear(); Record.Counts.reserve(RawCounts.size()); @@ -284,8 +292,80 @@ RawInstrProfReader::readNextRecord(InstrProfRecord &Record) { } else Record.Counts = RawCounts; + return success(); +} + +template +std::error_code RawInstrProfReader::readValueData( + InstrProfRecord &Record) { + + Record.clearValueData(); + if (!Data->Values || (ValueDataDelta == 0)) + return success(); + + // Read value data. + uint64_t NumVSites = 0; + for (uint32_t Kind = IPVK_First; Kind <= ValueKindLast; ++Kind) + NumVSites += swap(Data->NumValueSites[Kind]); + NumVSites += getNumPaddingBytes(NumVSites); + + auto VDataCounts = makeArrayRef(getValueDataCounts(Data->Values), NumVSites); + // Check bounds. + if (VDataCounts.data() < ValueDataStart || + VDataCounts.data() + VDataCounts.size() > + reinterpret_cast(ProfileEnd)) + return error(instrprof_error::malformed); + + const InstrProfValueData *VDataPtr = + getValueData(swap(Data->Values) + NumVSites); + for (uint32_t Kind = IPVK_First; Kind <= ValueKindLast; ++Kind) { + NumVSites = swap(Data->NumValueSites[Kind]); + Record.reserveSites(Kind, NumVSites); + for (uint32_t VSite = 0; VSite < NumVSites; ++VSite) { + + uint32_t VDataCount = VDataCounts[VSite]; + if ((const char *)(VDataPtr + VDataCount) > ProfileEnd) + return error(instrprof_error::malformed); + + std::vector CurrentValues; + CurrentValues.reserve(VDataCount); + for (uint32_t VIndex = 0; VIndex < VDataCount; ++VIndex) { + uint64_t TargetValue = swap(VDataPtr->Value); + uint64_t Count = swap(VDataPtr->Count); + CurrentValues.push_back({TargetValue, Count}); + ++VDataPtr; + } + Record.addValueData(Kind, VSite, CurrentValues.data(), + VDataCount, &FunctionPtrToNameMap); + } + } + return success(); +} + +template +std::error_code RawInstrProfReader::readNextRecord( + InstrProfRecord &Record) { + if (atEnd()) + if (std::error_code EC = readNextHeader(ProfileEnd)) + return EC; + + // Read name ad set it in Record. + if (std::error_code EC = readName(Record)) + return EC; + + // Read FuncHash and set it in Record. + if (std::error_code EC = readFuncHash(Record)) + return EC; + + // Read raw counts and set Record. + if (std::error_code EC = readRawCounts(Record)) + return EC; + + // Read value data and set Record. + if (std::error_code EC = readValueData(Record)) return EC; + // Iterate. - ++Data; + advanceData(); return success(); } @@ -304,56 +384,15 @@ typedef InstrProfLookupTrait::offset_type offset_type; bool InstrProfLookupTrait::ReadValueProfilingData( const unsigned char *&D, const unsigned char *const End) { + ErrorOr> VDataPtrOrErr = + ValueProfData::getValueProfData(D, End, ValueProfDataEndianness); - using namespace support; - // Read number of value kinds with value sites. - if (D + sizeof(uint64_t) > End) + if (VDataPtrOrErr.getError()) return false; - uint64_t ValueKindCount = endian::readNext(D); - - for (uint32_t Kind = 0; Kind < ValueKindCount; ++Kind) { - // Read value kind and number of value sites for kind. - if (D + 2 * sizeof(uint64_t) > End) - return false; - uint64_t ValueKind = endian::readNext(D); - uint64_t ValueSiteCount = endian::readNext(D); + VDataPtrOrErr.get()->deserializeTo(DataBuffer.back(), &HashKeys); + D += VDataPtrOrErr.get()->TotalSize; - std::vector &ValueSites = - DataBuffer.back().getValueSitesForKind(ValueKind); - ValueSites.reserve(ValueSiteCount); - for (uint64_t VSite = 0; VSite < ValueSiteCount; ++VSite) { - // Read number of value data pairs at value site. - if (D + sizeof(uint64_t) > End) - return false; - uint64_t ValueDataCount = - endian::readNext(D); - - // Check if there are as many ValueDataPairs as ValueDataCount in memory. - if (D + (ValueDataCount << 1) * sizeof(uint64_t) > End) - return false; - - InstrProfValueSiteRecord VSiteRecord; - for (uint64_t VCount = 0; VCount < ValueDataCount; ++VCount) { - uint64_t Value = endian::readNext(D); - uint64_t NumTaken = endian::readNext(D); - switch (ValueKind) { - case IPVK_IndirectCallTarget: { - auto Result = - std::lower_bound(HashKeys.begin(), HashKeys.end(), Value, - [](const std::pair &LHS, - uint64_t RHS) { return LHS.first < RHS; }); - assert(Result != HashKeys.end() && - "Hash does not match any known keys\n"); - Value = (uint64_t)Result->second; - break; - } - } - VSiteRecord.ValueData.push_back(std::make_pair(Value, NumTaken)); - } - ValueSites.push_back(std::move(VSiteRecord)); - } - } return true; } @@ -391,7 +430,7 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, for (uint64_t J = 0; J < CountsSize; ++J) CounterBuffer.push_back(endian::readNext(D)); - DataBuffer.push_back(InstrProfRecord(K, Hash, std::move(CounterBuffer))); + DataBuffer.emplace_back(K, Hash, std::move(CounterBuffer)); // Read value profiling data. if (FormatVersion > 2 && !ReadValueProfilingData(D, End)) { @@ -402,6 +441,52 @@ data_type InstrProfLookupTrait::ReadData(StringRef K, const unsigned char *D, return DataBuffer; } +std::error_code +InstrProfReaderIndex::getRecords(StringRef FuncName, + ArrayRef &Data) { + auto Iter = Index->find(FuncName); + if (Iter == Index->end()) + return instrprof_error::unknown_function; + + Data = (*Iter); + if (Data.empty()) + return instrprof_error::malformed; + + return instrprof_error::success; +} + +std::error_code InstrProfReaderIndex::getRecords( + ArrayRef &Data) { + if (atEnd()) return instrprof_error::eof; + + Data = *RecordIterator; + + if (Data.empty()) return instrprof_error::malformed; + + return instrprof_error::success; +} + +void InstrProfReaderIndex::Init(const unsigned char *Buckets, + const unsigned char *const Payload, + const unsigned char *const Base, + IndexedInstrProf::HashT HashType, + uint64_t Version) { + FormatVersion = Version; + Index.reset(IndexType::Create(Buckets, Payload, Base, + InstrProfLookupTrait(HashType, Version))); + // Form the map of hash values to const char* keys in profiling data. + std::vector> HashKeys; + for (auto Key : Index->keys()) { + const char *KeyTableRef = StringTable.insertString(Key); + HashKeys.push_back(std::make_pair(ComputeHash(HashType, Key), KeyTableRef)); + } + std::sort(HashKeys.begin(), HashKeys.end(), less_first()); + HashKeys.erase(std::unique(HashKeys.begin(), HashKeys.end()), HashKeys.end()); + // Set the hash key map for the InstrLookupTrait + Index->getInfoObj().setHashKeys(std::move(HashKeys)); + RecordIterator = Index->data_begin(); +} + bool IndexedInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) { if (DataBuffer.getBufferSize() < 8) return false; @@ -421,82 +506,78 @@ std::error_code IndexedInstrProfReader::readHeader() { using namespace support; + auto *Header = reinterpret_cast(Cur); + Cur += sizeof(IndexedInstrProf::Header); + // Check the magic number. - uint64_t Magic = endian::readNext(Cur); + uint64_t Magic = endian::byte_swap(Header->Magic); if (Magic != IndexedInstrProf::Magic) return error(instrprof_error::bad_magic); // Read the version. - FormatVersion = endian::readNext(Cur); + uint64_t FormatVersion = endian::byte_swap(Header->Version); if (FormatVersion > IndexedInstrProf::Version) return error(instrprof_error::unsupported_version); // Read the maximal function count. - MaxFunctionCount = endian::readNext(Cur); + MaxFunctionCount = + endian::byte_swap(Header->MaxFunctionCount); // Read the hash type and start offset. IndexedInstrProf::HashT HashType = static_cast( - endian::readNext(Cur)); + endian::byte_swap(Header->HashType)); if (HashType > IndexedInstrProf::HashT::Last) return error(instrprof_error::unsupported_hash_type); - uint64_t HashOffset = endian::readNext(Cur); - // The rest of the file is an on disk hash table. - Index.reset(InstrProfReaderIndex::Create( - Start + HashOffset, Cur, Start, - InstrProfLookupTrait(HashType, FormatVersion))); + uint64_t HashOffset = endian::byte_swap(Header->HashOffset); - // Form the map of hash values to const char* keys in profiling data. - std::vector> HashKeys; - for (auto Key : Index->keys()) { - const char *KeyTableRef = StringTable.insertString(Key); - HashKeys.push_back(std::make_pair(ComputeHash(HashType, Key), KeyTableRef)); - } - std::sort(HashKeys.begin(), HashKeys.end(), less_first()); - HashKeys.erase(std::unique(HashKeys.begin(), HashKeys.end()), HashKeys.end()); - // Set the hash key map for the InstrLookupTrait - Index->getInfoObj().setHashKeys(std::move(HashKeys)); - // Set up our iterator for readNextRecord. - RecordIterator = Index->data_begin(); + // The rest of the file is an on disk hash table. + Index.Init(Start + HashOffset, Cur, Start, HashType, FormatVersion); return success(); } -std::error_code IndexedInstrProfReader::getFunctionCounts( - StringRef FuncName, uint64_t FuncHash, std::vector &Counts) { - auto Iter = Index->find(FuncName); - if (Iter == Index->end()) - return error(instrprof_error::unknown_function); - +ErrorOr +IndexedInstrProfReader::getInstrProfRecord(StringRef FuncName, + uint64_t FuncHash) { + ArrayRef Data; + std::error_code EC = Index.getRecords(FuncName, Data); + if (EC != instrprof_error::success) + return EC; // Found it. Look for counters with the right hash. - ArrayRef Data = (*Iter); - if (Data.empty()) - return error(instrprof_error::malformed); - for (unsigned I = 0, E = Data.size(); I < E; ++I) { // Check for a match and fill the vector if there is one. if (Data[I].Hash == FuncHash) { - Counts = Data[I].Counts; - return success(); + return std::move(Data[I]); } } return error(instrprof_error::hash_mismatch); } std::error_code -IndexedInstrProfReader::readNextRecord(InstrProfRecord &Record) { - // Are we out of records? - if (RecordIterator == Index->data_end()) - return error(instrprof_error::eof); +IndexedInstrProfReader::getFunctionCounts(StringRef FuncName, uint64_t FuncHash, + std::vector &Counts) { + ErrorOr Record = getInstrProfRecord(FuncName, FuncHash); + if (std::error_code EC = Record.getError()) + return EC; - if ((*RecordIterator).empty()) - return error(instrprof_error::malformed); + Counts = Record.get().Counts; + return success(); +} +std::error_code IndexedInstrProfReader::readNextRecord( + InstrProfRecord &Record) { static unsigned RecordIndex = 0; - ArrayRef Data = (*RecordIterator); + + ArrayRef Data; + + std::error_code EC = Index.getRecords(Data); + if (EC != instrprof_error::success) + return error(EC); + Record = Data[RecordIndex++]; if (RecordIndex >= Data.size()) { - ++RecordIterator; + Index.advanceToNextKey(); RecordIndex = 0; } return success();