From: Xinliang David Li Date: Wed, 25 Nov 2015 23:31:18 +0000 (+0000) Subject: [PGO] Implement ValueProfiling Closure interfaces for runtime value profile data X-Git-Url: http://plrg.eecs.uci.edu/git/?p=oota-llvm.git;a=commitdiff_plain;h=931217997cd3c9696284e73cace5b3a761147b36;hp=5062813b46e69c0ee735e6fefaa05f2c466e524f [PGO] Implement ValueProfiling Closure interfaces for runtime value profile data This is one of the many steps to commonize value profiling support between profile runtime and compiler/llvm tools. After this change, profiler runtime now can share the same C APIs to do VP serialization/deseriazation with LLVM host tools (and produces value data in identical format between indexed and raw profile). It is not yet enabled in profiler runtime yet. Also added a unit test case to test runtime profile data serialization/deserialization interfaces implemented using common closure code. git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@254110 91177308-0d34-0410-b5e6-96231b3b80d8 --- diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h index 3c1a4d08277..778fe16c2e6 100644 --- a/include/llvm/ProfileData/InstrProf.h +++ b/include/llvm/ProfileData/InstrProf.h @@ -553,7 +553,52 @@ typedef struct ValueProfRecordClosure { ValueProfData *(*AllocValueProfData)(size_t TotalSizeInBytes); } ValueProfRecordClosure; -/// Return the \c ValueProfRecord header size including the padding bytes. +/* A wrapper struct that represents value profile runtime data. + * Like InstrProfRecord class which is used by profiling host tools, + * ValueProfRuntimeRecord also implements the abstract intefaces defined in + * ValueProfRecordClosure so that the runtime data can be serialized using + * shared C implementation. In this structure, NumValueSites and Nodes + * members are the primary fields while other fields hold the derived + * information for fast implementation of closure interfaces. + */ +typedef struct ValueProfRuntimeRecord { + /* Number of sites for each value profile kind. */ + uint16_t *NumValueSites; + /* An array of linked-list headers. The size of of the array is the + * total number of value profile sites : sum(NumValueSites[*])). Each + * linked-list stores the values profiled for a value profile site. */ + ValueProfNode **Nodes; + + /* Total number of value profile kinds which have at least one + * value profile sites. */ + uint32_t NumValueKinds; + /* An array recording the number of values tracked at each site. + * The size of the array is TotalNumValueSites. + */ + uint8_t *SiteCountArray[IPVK_Last + 1]; + ValueProfNode **NodesKind[IPVK_Last + 1]; +} ValueProfRuntimeRecord; + +/* Initialize the record for runtime value profile data. */ +void initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, + uint16_t *NumValueSites, + ValueProfNode **Nodes); + +/* Release memory allocated for the runtime record. */ +void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord); + +/* Return the size of ValueProfData structure that can be used to store + the value profile data collected at runtime. */ +uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record); + +/* Return a ValueProfData instance that stores the data collected at runtime. */ +ValueProfData * +serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record); + + +/*! \brief Return the \c ValueProfRecord header size including the + * padding bytes. + */ inline uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) + sizeof(uint8_t) * NumValueSites; @@ -562,21 +607,24 @@ inline uint32_t getValueProfRecordHeaderSize(uint32_t NumValueSites) { return Size; } -/// Return the total size of the value profile record including the -/// header and the value data. +/*! \brief Return the total size of the value profile record including the + * header and the value data. + */ inline uint32_t getValueProfRecordSize(uint32_t NumValueSites, uint32_t NumValueData) { return getValueProfRecordHeaderSize(NumValueSites) + sizeof(InstrProfValueData) * NumValueData; } -/// Return the pointer to the start of value data array. +/*! \brief Return the pointer to the start of value data array. + */ inline InstrProfValueData *getValueProfRecordValueData(ValueProfRecord *This) { return (InstrProfValueData *)((char *)This + getValueProfRecordHeaderSize( This->NumValueSites)); } -/// Return the total number of value data for \c This record. +/*! \brief Return the total number of value data for \c This record. + */ inline uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { uint32_t NumValueData = 0; uint32_t I; @@ -585,7 +633,8 @@ inline uint32_t getValueProfRecordNumValueData(ValueProfRecord *This) { return NumValueData; } -/// Use this method to advance to the next \c This \c ValueProfRecord. +/* \brief Use this method to advance to the next \c This \c ValueProfRecord. + */ inline ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { uint32_t NumValueData = getValueProfRecordNumValueData(This); return (ValueProfRecord *)((char *)This + @@ -593,11 +642,14 @@ inline ValueProfRecord *getValueProfRecordNext(ValueProfRecord *This) { NumValueData)); } -/// Return the first \c ValueProfRecord instance. +/*! \brief Return the first \c ValueProfRecord instance. + */ inline ValueProfRecord *getFirstValueProfRecord(ValueProfData *This) { return (ValueProfRecord *)((char *)This + sizeof(ValueProfData)); } + + namespace IndexedInstrProf { enum class HashT : uint32_t { diff --git a/lib/ProfileData/InstrProf.cpp b/lib/ProfileData/InstrProf.cpp index 8a3a61ff679..86cd76c9a20 100644 --- a/lib/ProfileData/InstrProf.cpp +++ b/lib/ProfileData/InstrProf.cpp @@ -188,9 +188,10 @@ ValueProfData *serializeValueProfDataFrom(ValueProfRecordClosure *Closure) { return VPD; } -// C wrappers of InstrProfRecord member functions used in Closure. -// These C wrappers are used as adaptors so that C++ code can be -// invoked as callbacks. +/*! \brief ValueProfRecordClosure Interface implementation for InstrProfRecord + * class. These C wrappers are used as adaptors so that C++ code can be + * invoked as callbacks. + */ uint32_t getNumValueKindsInstrProf(const void *Record) { return reinterpret_cast(Record)->getNumValueKinds(); } @@ -262,6 +263,120 @@ ValueProfData::serializeFrom(const InstrProfRecord &Record) { return VPD; } +/* The value profiler runtime library stores the value profile data + * for a given function in NumValueSites and Nodes. This is the + * method to initialize the RuntimeRecord with the runtime data to + * pre-compute the information needed to efficiently implement + * ValueProfRecordClosure's callback interfaces. + */ +void initializeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord, + uint16_t *NumValueSites, + ValueProfNode **Nodes) { + unsigned I, J, S = 0, NumValueKinds = 0; + RuntimeRecord->NumValueSites = NumValueSites; + RuntimeRecord->Nodes = Nodes; + for (I = 0; I <= IPVK_Last; I++) { + uint16_t N = NumValueSites[I]; + if (!N) { + RuntimeRecord->SiteCountArray[I] = 0; + continue; + } + NumValueKinds++; + RuntimeRecord->SiteCountArray[I] = (uint8_t *)calloc(N, 1); + RuntimeRecord->NodesKind[I] = &RuntimeRecord->Nodes[S]; + for (J = 0; J < N; J++) { + uint8_t C = 0; + ValueProfNode *Site = RuntimeRecord->Nodes[S + J]; + while (Site) { + C++; + Site = Site->Next; + } + if (C > UCHAR_MAX) + C = UCHAR_MAX; + RuntimeRecord->SiteCountArray[I][J] = C; + } + S += N; + } + RuntimeRecord->NumValueKinds = NumValueKinds; +} + +void finalizeValueProfRuntimeRecord(ValueProfRuntimeRecord *RuntimeRecord) { + unsigned I; + for (I = 0; I <= IPVK_Last; I++) { + if (RuntimeRecord->SiteCountArray[I]) + free(RuntimeRecord->SiteCountArray[I]); + } +} + +/* ValueProfRecordClosure Interface implementation for + * ValueProfDataRuntimeRecord. */ +uint32_t getNumValueKindsRT(const void *R) { + return ((const ValueProfRuntimeRecord *)R)->NumValueKinds; +} + +uint32_t getNumValueSitesRT(const void *R, uint32_t VK) { + return ((const ValueProfRuntimeRecord *)R)->NumValueSites[VK]; +} + +uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK, uint32_t S) { + const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; + return Record->SiteCountArray[VK][S]; +} + +uint32_t getNumValueDataRT(const void *R, uint32_t VK) { + unsigned I, S = 0; + const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; + if (Record->SiteCountArray[VK] == 0) + return 0; + for (I = 0; I < Record->NumValueSites[VK]; I++) + S += Record->SiteCountArray[VK][I]; + return S; +} + +void getValueForSiteRT(const void *R, InstrProfValueData *Dst, uint32_t VK, + uint32_t S, uint64_t (*Mapper)(uint32_t, uint64_t)) { + unsigned I, N = 0; + const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R; + N = getNumValueDataForSiteRT(R, VK, S); + ValueProfNode *VNode = Record->NodesKind[VK][S]; + for (I = 0; I < N; I++) { + Dst[I] = VNode->VData; + VNode = VNode->Next; + } +} + +ValueProfData *allocValueProfDataRT(size_t TotalSizeInBytes) { + return (ValueProfData *)calloc(TotalSizeInBytes, 1); +} + +static ValueProfRecordClosure RTRecordClosure = {0, + getNumValueKindsRT, + getNumValueSitesRT, + getNumValueDataRT, + getNumValueDataForSiteRT, + 0, + getValueForSiteRT, + allocValueProfDataRT}; + +/* Return the size of ValueProfData structure to store data + * recorded in the runtime record. + */ +uint32_t getValueProfDataSizeRT(const ValueProfRuntimeRecord *Record) { + RTRecordClosure.Record = Record; + return getValueProfDataSize(&RTRecordClosure); +} + +/* Return a ValueProfData instance that stores the data collected + from runtime. */ +ValueProfData * +serializeValueProfDataFromRT(const ValueProfRuntimeRecord *Record) { + RTRecordClosure.Record = Record; + return serializeValueProfDataFrom(&RTRecordClosure); +} + + + + void ValueProfRecord::deserializeTo(InstrProfRecord &Record, InstrProfRecord::ValueMapType *VMap) { Record.reserveSites(Kind, NumValueSites); @@ -273,6 +388,7 @@ void ValueProfRecord::deserializeTo(InstrProfRecord &Record, ValueData += ValueDataCount; } } + // For writing/serializing, Old is the host endianness, and New is // byte order intended on disk. For Reading/deserialization, Old // is the on-disk source endianness, and New is the host endianness. diff --git a/unittests/ProfileData/InstrProfTest.cpp b/unittests/ProfileData/InstrProfTest.cpp index 6891931e4d2..051bf391b95 100644 --- a/unittests/ProfileData/InstrProfTest.cpp +++ b/unittests/ProfileData/InstrProfTest.cpp @@ -382,6 +382,98 @@ TEST_F(InstrProfTest, get_icall_data_merge1_saturation) { ASSERT_EQ(Max, VD[0].Count); } +// Synthesize runtime value profile data. +ValueProfNode Site1Values[5] = {{{uint64_t("callee1"), 400}, &Site1Values[1]}, + {{uint64_t("callee2"), 1000}, &Site1Values[2]}, + {{uint64_t("callee3"), 500}, &Site1Values[3]}, + {{uint64_t("callee4"), 300}, &Site1Values[4]}, + {{uint64_t("callee5"), 100}, 0}}; + +ValueProfNode Site2Values[4] = {{{uint64_t("callee5"), 800}, &Site2Values[1]}, + {{uint64_t("callee3"), 1000}, &Site2Values[2]}, + {{uint64_t("callee2"), 2500}, &Site2Values[3]}, + {{uint64_t("callee1"), 1300}, 0}}; + +ValueProfNode Site3Values[3] = {{{uint64_t("callee6"), 800}, &Site3Values[1]}, + {{uint64_t("callee3"), 1000}, &Site3Values[2]}, + {{uint64_t("callee4"), 5500}, 0}}; + +ValueProfNode Site4Values[2] = {{{uint64_t("callee2"), 1800}, &Site4Values[1]}, + {{uint64_t("callee3"), 2000}, 0}}; + +static ValueProfNode *ValueProfNodes[5] = {&Site1Values[0], &Site2Values[0], + &Site3Values[0], &Site4Values[0], 0}; +static uint16_t NumValueSites[IPVK_Last + 1] = {5}; +TEST_F(InstrProfTest, runtime_value_prof_data_read_write) { + ValueProfRuntimeRecord RTRecord; + initializeValueProfRuntimeRecord(&RTRecord, &NumValueSites[0], + &ValueProfNodes[0]); + + ValueProfData *VPData = serializeValueProfDataFromRT(&RTRecord); + + InstrProfRecord Record("caller", 0x1234, {1ULL << 31, 2}); + + VPData->deserializeTo(Record, 0); + + // Now read data from Record and sanity check the data + ASSERT_EQ(5U, Record.getNumValueSites(IPVK_IndirectCallTarget)); + ASSERT_EQ(5U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 0)); + ASSERT_EQ(4U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 1)); + ASSERT_EQ(3U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 2)); + ASSERT_EQ(2U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 3)); + ASSERT_EQ(0U, Record.getNumValueDataForSite(IPVK_IndirectCallTarget, 4)); + + auto Cmp = [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) { + return VD1.Count > VD2.Count; + }; + std::unique_ptr VD_0( + Record.getValueForSite(IPVK_IndirectCallTarget, 0)); + std::sort(&VD_0[0], &VD_0[5], Cmp); + ASSERT_EQ(StringRef((const char *)VD_0[0].Value, 7), StringRef("callee2")); + ASSERT_EQ(1000U, VD_0[0].Count); + ASSERT_EQ(StringRef((const char *)VD_0[1].Value, 7), StringRef("callee3")); + ASSERT_EQ(500U, VD_0[1].Count); + ASSERT_EQ(StringRef((const char *)VD_0[2].Value, 7), StringRef("callee1")); + ASSERT_EQ(400U, VD_0[2].Count); + ASSERT_EQ(StringRef((const char *)VD_0[3].Value, 7), StringRef("callee4")); + ASSERT_EQ(300U, VD_0[3].Count); + ASSERT_EQ(StringRef((const char *)VD_0[4].Value, 7), StringRef("callee5")); + ASSERT_EQ(100U, VD_0[4].Count); + + std::unique_ptr VD_1( + Record.getValueForSite(IPVK_IndirectCallTarget, 1)); + std::sort(&VD_1[0], &VD_1[4], Cmp); + ASSERT_EQ(StringRef((const char *)VD_1[0].Value, 7), StringRef("callee2")); + ASSERT_EQ(2500U, VD_1[0].Count); + ASSERT_EQ(StringRef((const char *)VD_1[1].Value, 7), StringRef("callee1")); + ASSERT_EQ(1300U, VD_1[1].Count); + ASSERT_EQ(StringRef((const char *)VD_1[2].Value, 7), StringRef("callee3")); + ASSERT_EQ(1000U, VD_1[2].Count); + ASSERT_EQ(StringRef((const char *)VD_1[3].Value, 7), StringRef("callee5")); + ASSERT_EQ(800U, VD_1[3].Count); + + std::unique_ptr VD_2( + Record.getValueForSite(IPVK_IndirectCallTarget, 2)); + std::sort(&VD_2[0], &VD_2[3], Cmp); + ASSERT_EQ(StringRef((const char *)VD_2[0].Value, 7), StringRef("callee4")); + ASSERT_EQ(5500U, VD_2[0].Count); + ASSERT_EQ(StringRef((const char *)VD_2[1].Value, 7), StringRef("callee3")); + ASSERT_EQ(1000U, VD_2[1].Count); + ASSERT_EQ(StringRef((const char *)VD_2[2].Value, 7), StringRef("callee6")); + ASSERT_EQ(800U, VD_2[2].Count); + + std::unique_ptr VD_3( + Record.getValueForSite(IPVK_IndirectCallTarget, 3)); + std::sort(&VD_3[0], &VD_3[2], Cmp); + ASSERT_EQ(StringRef((const char *)VD_3[0].Value, 7), StringRef("callee3")); + ASSERT_EQ(2000U, VD_3[0].Count); + ASSERT_EQ(StringRef((const char *)VD_3[1].Value, 7), StringRef("callee2")); + ASSERT_EQ(1800U, VD_3[1].Count); + + finalizeValueProfRuntimeRecord(&RTRecord); + free(VPData); +} + TEST_F(InstrProfTest, get_max_function_count) { InstrProfRecord Record1("foo", 0x1234, {1ULL << 31, 2}); InstrProfRecord Record2("bar", 0, {1ULL << 63});