[PGO] Make indexed value profile data more compact

- Make indexed value profile data more compact by peeling out
  the per-site value count field into its own smaller sized array.
- Introduced formal data structure definitions to specify value
  profile data layout in indexed format. Previously the layout
  of the data is only assumed in the client code (scattered in
  three different places : size computation, EmitData, and ReadData
- The new data structure  serves as a central place for layout documentation.
- Add interfaces to force BE output for value profile data (testing purpose)
- Add byte swap unit tests

Differential Revision: http://reviews.llvm.org/D14401

git-svn-id: https://llvm.org/svn/llvm-project/llvm/trunk@252563 91177308-0d34-0410-b5e6-96231b3b80d8

diff --git a/include/llvm/ProfileData/InstrProf.h b/include/llvm/ProfileData/InstrProf.h
index 0afbc38c37428936272d2a3fecc031fc75ef8ade..607f29de8a23f8523397146de9f3ac3c9c21637b 100644 (file)
--- a/include/llvm/ProfileData/InstrProf.h
+++ b/include/llvm/ProfileData/InstrProf.h
@@ -21,6 +21,7 @@
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/ErrorHandling.h"
 #include "llvm/IR/GlobalValue.h"
 #include "llvm/Support/Endian.h"
 #include "llvm/Support/ErrorHandling.h"

+#include "llvm/Support/ErrorOr.h"

 #include "llvm/Support/MD5.h"
 #include <cstdint>
 #include <list>
 #include "llvm/Support/MD5.h"
 #include <cstdint>
 #include <list>


@@ -318,6 +319,16 @@ uint32_t InstrProfRecord::getNumValueKinds() const {
   return NumValueKinds;
 }
 
   return NumValueKinds;
 }
 

+uint32_t InstrProfRecord::getNumValueData(uint32_t ValueKind) const {
+  uint32_t N = 0;
+  const std::vector<InstrProfValueSiteRecord> &SiteRecords =
+      getValueSitesForKind(ValueKind);
+  for (auto &SR : SiteRecords) {
+    N += SR.ValueData.size();
+  }
+  return N;
+}
+

 uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const {
   return getValueSitesForKind(ValueKind).size();
 }
 uint32_t InstrProfRecord::getNumValueSites(uint32_t ValueKind) const {
   return getValueSitesForKind(ValueKind).size();
 }


@@ -419,6 +430,8 @@ const uint64_t Magic = 0x8169666f72706cff;  // "\xfflprofi\x81"
 const uint64_t Version = 3;
 const HashT HashType = HashT::MD5;
 
 const uint64_t Version = 3;
 const HashT HashType = HashT::MD5;
 

+// This structure defines the file header of the LLVM profile
+// data file in indexed-format.

 struct Header {
   uint64_t Magic;
   uint64_t Version;
 struct Header {
   uint64_t Magic;
   uint64_t Version;


@@ -427,6 +440,105 @@ struct Header {
   uint64_t HashOffset;
 };
 
   uint64_t HashOffset;
 };
 

+inline support::endianness getHostEndianness() {
+  return sys::IsLittleEndianHost ? support::little : support::big;
+}
+
+/// This is the header of the data structure that defines the on-disk
+/// layout of the value profile data of a particular kind for one function.
+struct ValueProfRecord {
+  // The kind of the value profile record.
+  uint32_t Kind;
+  // The number of value profile sites. It is guaranteed to be non-zero;
+  // otherwise the record for this kind won't be emitted.
+  uint32_t NumValueSites;
+  // The first element of the array that stores the number of profiled
+  // values for each value site. The size of the array is NumValueSites.
+  // Since NumValueSites is greater than zero, there is at least one
+  // element in the array.
+  uint8_t SiteCountArray[1];
+
+  // The fake declaration is for documentation purpose only.
+  // Align the start of next field to be on 8 byte boundaries.
+  // uint8_t Padding[X];
+
+  // The array of value profile data. The size of the array is the sum
+  // of all elements in SiteCountArray[].
+  // InstrProfValueData ValueData[];
+
+  /// Return the \c ValueProfRecord header size including the padding bytes.
+  static uint32_t getHeaderSize(uint32_t NumValueSites);
+  /// Return the total size of the value profile record including the
+  /// header and the value data.
+  static uint32_t getSize(uint32_t NumValueSites, uint32_t NumValueData);
+  /// Return the total size of the value profile record including the
+  /// header and the value data.
+  uint32_t getSize() const { return getSize(NumValueSites, getNumValueData()); }
+  /// Use this method to advance to the next \c ValueProfRecord.
+  ValueProfRecord *getNext();
+  /// Return the pointer to the first value profile data.
+  InstrProfValueData *getValueData();
+  /// Return the number of value sites.
+  uint32_t getNumValueSites() const { return NumValueSites; }
+  /// Return the number of value data.
+  uint32_t getNumValueData() const;
+  /// Read data from this record and save it to Record.
+  void deserializeTo(InstrProfRecord &Record,
+                     InstrProfRecord::ValueMapType *VMap);
+  /// Extract data from \c Record and serialize into this instance.
+  void serializeFrom(const InstrProfRecord &Record, uint32_t ValueKind,
+                     uint32_t NumValueSites);
+  /// In-place byte swap:
+  /// Do byte swap for this instance. \c Old is the original order before
+  /// the swap, and \c New is the New byte order.
+  void swapBytes(support::endianness Old, support::endianness New);
+};
+
+/// Per-function header/control data structure for value profiling
+/// data in indexed format.
+struct ValueProfData {
+  // Total size in bytes including this field. It must be a multiple
+  // of sizeof(uint64_t).
+  uint32_t TotalSize;
+  // The number of value profile kinds that has value profile data.
+  // In this implementation, a value profile kind is considered to
+  // have profile data if the number of value profile sites for the
+  // kind is not zero. More aggressively, the implemnetation can
+  // choose to check the actual data value: if none of the value sites
+  // has any profiled values, the kind can be skipped.
+  uint32_t NumValueKinds;
+
+  // Following are a sequence of variable length records. The prefix/header
+  // of each record is defined by ValueProfRecord type. The number of
+  // records is NumValueKinds.
+  // ValueProfRecord Record_1;
+  // ValueProfRecord Record_N;
+
+  /// Return the total size in bytes of the on-disk value profile data
+  /// given the data stored in Record.
+  static uint32_t getSize(const InstrProfRecord &Record);
+  /// Return a pointer to \c ValueProfData instance ready to be streamed.
+  static std::unique_ptr<ValueProfData>
+  serializeFrom(const InstrProfRecord &Record);
+  /// Return a pointer to \c ValueProfileData instance ready to be read.
+  /// All data in the instance are properly byte swapped. The input
+  /// data is assumed to be in little endian order.
+  static ErrorOr<std::unique_ptr<ValueProfData>>
+  getValueProfData(const unsigned char *D, const unsigned char *const BufferEnd,
+                   support::endianness SrcDataEndianness);
+  /// Swap byte order from \c Endianness order to host byte order.
+  void swapBytesToHost(support::endianness Endianness);
+  /// Swap byte order from host byte order to \c Endianness order.
+  void swapBytesFromHost(support::endianness Endianness);
+  /// Return the total size of \c ValueProfileData.
+  uint32_t getSize() const { return TotalSize; }
+  /// Read data from this data and save it to \c Record.
+  void deserializeTo(InstrProfRecord &Record,
+                     InstrProfRecord::ValueMapType *VMap);
+  /// Return the first \c ValueProfRecord instance.
+  ValueProfRecord *getFirstValueProfRecord();
+};
+

 }  // end namespace IndexedInstrProf
 
 namespace RawInstrProf {
 }  // end namespace IndexedInstrProf
 
 namespace RawInstrProf {


@@ -458,20 +570,20 @@ inline uint64_t getMagic<uint32_t>() {
          uint64_t('R') << 8 | uint64_t(129);
 }
 
          uint64_t('R') << 8 | uint64_t(129);
 }
 

+// Per-function profile data header/control structure.

 // The definition should match the structure defined in
 // compiler-rt/lib/profile/InstrProfiling.h.
 // It should also match the synthesized type in
 // Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters.
 // The definition should match the structure defined in
 // compiler-rt/lib/profile/InstrProfiling.h.
 // It should also match the synthesized type in
 // Transforms/Instrumentation/InstrProfiling.cpp:getOrCreateRegionCounters.

-

 template <class IntPtrT> struct ProfileData {
   #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
   #include "llvm/ProfileData/InstrProfData.inc"
 };
 
 template <class IntPtrT> struct ProfileData {
   #define INSTR_PROF_DATA(Type, LLVMType, Name, Init) Type Name;
   #include "llvm/ProfileData/InstrProfData.inc"
 };
 

+// File header structure of the LLVM profile data in raw format.

 // The definition should match the header referenced in
 // compiler-rt/lib/profile/InstrProfilingFile.c  and
 // InstrProfilingBuffer.c.
 // The definition should match the header referenced in
 // compiler-rt/lib/profile/InstrProfilingFile.c  and
 // InstrProfilingBuffer.c.

-

 struct Header {
   const uint64_t Magic;
   const uint64_t Version;
 struct Header {
   const uint64_t Magic;
   const uint64_t Version;


@@ -486,6 +598,14 @@ struct Header {
 
 namespace coverage {
 
 
 namespace coverage {
 

+// Profile coverage map has the following layout:
+// [CoverageMapFileHeader]
+// [ArrayStart]
+//  [CovMapFunctionRecord]
+//  [CovMapFunctionRecord]
+//  ...
+// [ArrayEnd]
+// [Encoded Region Mapping Data]

 LLVM_PACKED_START
 template <class IntPtrT> struct CovMapFunctionRecord {
   #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Type Name;
 LLVM_PACKED_START
 template <class IntPtrT> struct CovMapFunctionRecord {
   #define COVMAP_FUNC_RECORD(Type, LLVMType, Name, Init) Type Name;

diff --git a/include/llvm/ProfileData/InstrProfReader.h b/include/llvm/ProfileData/InstrProfReader.h
index 69e899d806ea4005fe44a15987046913e36a30bf..d0f5a57b1ae80e89c2100fd05e7d1e809c5628d3 100644 (file)
--- a/include/llvm/ProfileData/InstrProfReader.h
+++ b/include/llvm/ProfileData/InstrProfReader.h
@@ -186,11 +186,16 @@ class InstrProfLookupTrait {
   std::vector<InstrProfRecord> DataBuffer;
   IndexedInstrProf::HashT HashType;
   unsigned FormatVersion;
   std::vector<InstrProfRecord> DataBuffer;
   IndexedInstrProf::HashT HashType;
   unsigned FormatVersion;

+  // Endianness of the input value profile data.
+  // It should be LE by default, but can be changed
+  // for testing purpose.
+  support::endianness ValueProfDataEndianness;

   std::vector<std::pair<uint64_t, const char *>> HashKeys;
 
 public:
   InstrProfLookupTrait(IndexedInstrProf::HashT HashType, unsigned FormatVersion)
   std::vector<std::pair<uint64_t, const char *>> HashKeys;
 
 public:
   InstrProfLookupTrait(IndexedInstrProf::HashT HashType, unsigned FormatVersion)

-      : HashType(HashType), FormatVersion(FormatVersion) {}
+      : HashType(HashType), FormatVersion(FormatVersion),
+        ValueProfDataEndianness(support::little) {}

 
   typedef ArrayRef<InstrProfRecord> data_type;
 
 
   typedef ArrayRef<InstrProfRecord> data_type;
 


@@ -223,6 +228,11 @@ public:
   bool ReadValueProfilingData(const unsigned char *&D,
                               const unsigned char *const End);
   data_type ReadData(StringRef K, const unsigned char *D, offset_type N);
   bool ReadValueProfilingData(const unsigned char *&D,
                               const unsigned char *const End);
   data_type ReadData(StringRef K, const unsigned char *D, offset_type N);

+
+  // Used for testing purpose only.
+  void setValueProfDataEndianness(support::endianness Endianness) {
+    ValueProfDataEndianness = Endianness;
+  }

 };
 
 class InstrProfReaderIndex {
 };
 
 class InstrProfReaderIndex {


@@ -251,6 +261,10 @@ class InstrProfReaderIndex {
 
   void advanceToNextKey() { RecordIterator++; }
   bool atEnd() const { return RecordIterator == Index->data_end(); }
 
   void advanceToNextKey() { RecordIterator++; }
   bool atEnd() const { return RecordIterator == Index->data_end(); }

+  // Used for testing purpose only.
+  void setValueProfDataEndianness(support::endianness Endianness) {
+    Index->getInfoObj().setValueProfDataEndianness(Endianness);
+  }

 };
 
 /// Reader for the indexed binary instrprof format.
 };
 
 /// Reader for the indexed binary instrprof format.


@@ -295,6 +309,11 @@ private:
 
   static ErrorOr<std::unique_ptr<IndexedInstrProfReader>>
   create(std::unique_ptr<MemoryBuffer> Buffer);
 
   static ErrorOr<std::unique_ptr<IndexedInstrProfReader>>
   create(std::unique_ptr<MemoryBuffer> Buffer);

+
+  // Used for testing purpose only.
+  void setValueProfDataEndianness(support::endianness Endianness) {
+    Index.setValueProfDataEndianness(Endianness);
+  }

 };
 
 } // end namespace llvm
 };
 
 } // end namespace llvm

diff --git a/include/llvm/ProfileData/InstrProfWriter.h b/include/llvm/ProfileData/InstrProfWriter.h
index da0d956e8a9871896855ebede64b6fe3f1dd8aad..5b122d69977eaa15afa6163208432b0265657485 100644 (file)
--- a/include/llvm/ProfileData/InstrProfWriter.h
+++ b/include/llvm/ProfileData/InstrProfWriter.h
@@ -46,6 +46,9 @@ public:
   /// Write the profile, returning the raw data. For testing.
   std::unique_ptr<MemoryBuffer> writeBuffer();
 
   /// Write the profile, returning the raw data. For testing.
   std::unique_ptr<MemoryBuffer> writeBuffer();
 

+  // Internal interface for testing purpose only.
+  void setValueProfDataEndianness(support::endianness Endianness);
+

 private:
   std::pair<uint64_t, uint64_t> writeImpl(raw_ostream &OS);
 };
 private:
   std::pair<uint64_t, uint64_t> writeImpl(raw_ostream &OS);
 };

diff --git a/lib/ProfileData/InstrProf.cpp b/lib/ProfileData/InstrProf.cpp
index 377d77197d98237e28aec9affdec5841bcf45572..22c04fbf58e15cac7be7ec2895ce5b2714bef358 100644 (file)
--- a/lib/ProfileData/InstrProf.cpp
+++ b/lib/ProfileData/InstrProf.cpp
@@ -128,4 +128,232 @@ GlobalVariable *createPGOFuncNameVar(Module &M,
 GlobalVariable *createPGOFuncNameVar(Function &F, StringRef FuncName) {
   return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), FuncName);
 }
 GlobalVariable *createPGOFuncNameVar(Function &F, StringRef FuncName) {
   return createPGOFuncNameVar(*F.getParent(), F.getLinkage(), FuncName);
 }

+
+namespace IndexedInstrProf {
+
+uint32_t ValueProfRecord::getHeaderSize(uint32_t NumValueSites) {
+  uint32_t Size = offsetof(ValueProfRecord, SiteCountArray) +
+                  sizeof(uint8_t) * NumValueSites;
+  // Round the size to multiple of 8 bytes.
+  Size = (Size + 7) & ~7;
+  return Size;
+}
+
+uint32_t ValueProfRecord::getSize(uint32_t NumValueSites,
+                                  uint32_t NumValueData) {
+  return getHeaderSize(NumValueSites) +
+         sizeof(InstrProfValueData) * NumValueData;
+}
+
+void ValueProfRecord::deserializeTo(InstrProfRecord &Record,
+                                    InstrProfRecord::ValueMapType *VMap) {
+  Record.reserveSites(Kind, NumValueSites);
+
+  InstrProfValueData *ValueData = this->getValueData();
+  for (uint64_t VSite = 0; VSite < NumValueSites; ++VSite) {
+    uint8_t ValueDataCount = this->SiteCountArray[VSite];
+    Record.addValueData(Kind, VSite, ValueData, ValueDataCount, VMap);
+    ValueData += ValueDataCount;
+  }
+}
+
+void ValueProfRecord::serializeFrom(const InstrProfRecord &Record,
+                                    uint32_t ValueKind,
+                                    uint32_t NumValueSites) {
+  Kind = ValueKind;
+  this->NumValueSites = NumValueSites;
+  InstrProfValueData *DstVD = getValueData();
+  for (uint32_t S = 0; S < NumValueSites; S++) {
+    uint32_t ND = Record.getNumValueDataForSite(ValueKind, S);
+    SiteCountArray[S] = ND;
+    std::unique_ptr<InstrProfValueData[]> SrcVD =
+        Record.getValueForSite(ValueKind, S);
+    for (uint32_t I = 0; I < ND; I++) {
+      DstVD[I] = SrcVD[I];
+      switch (ValueKind) {
+      case IPVK_IndirectCallTarget:
+        DstVD[I].Value = ComputeHash(HashType, (const char *)DstVD[I].Value);
+        break;
+      default:
+        llvm_unreachable("value kind not handled !");
+      }
+    }
+    DstVD += ND;
+  }
+}
+
+template <class T> static T swapToHostOrder(T v, support::endianness Orig) {
+  if (Orig == getHostEndianness())
+    return v;
+  sys::swapByteOrder<T>(v);
+  return v;
+}
+
+// For writing/serializing,  Old is the host endianness, and  New is
+// byte order intended on disk. For Reading/deserialization, Old
+// is the on-disk source endianness, and New is the host endianness.
+void ValueProfRecord::swapBytes(support::endianness Old,
+                                support::endianness New) {
+  using namespace support;
+  if (Old == New)
+    return;
+
+  if (getHostEndianness() != Old) {
+    sys::swapByteOrder<uint32_t>(NumValueSites);
+    sys::swapByteOrder<uint32_t>(Kind);
+  }
+  uint32_t ND = getNumValueData();
+  InstrProfValueData *VD = getValueData();
+
+  // No need to swap byte array: SiteCountArrray.
+  for (uint32_t I = 0; I < ND; I++) {
+    sys::swapByteOrder<uint64_t>(VD[I].Value);
+    sys::swapByteOrder<uint64_t>(VD[I].Count);
+  }
+  if (getHostEndianness() == Old) {
+    sys::swapByteOrder<uint32_t>(NumValueSites);
+    sys::swapByteOrder<uint32_t>(Kind);
+  }
+}
+
+uint32_t ValueProfData::getSize(const InstrProfRecord &Record) {
+  uint32_t TotalSize = sizeof(ValueProfData);
+  uint32_t NumValueKinds = Record.getNumValueKinds();
+  if (NumValueKinds == 0)
+    return TotalSize;
+
+  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
+    uint32_t NumValueSites = Record.getNumValueSites(Kind);
+    if (!NumValueSites)
+      continue;
+    TotalSize +=
+        ValueProfRecord::getSize(NumValueSites, Record.getNumValueData(Kind));
+  }
+  return TotalSize;
+}
+
+void ValueProfData::deserializeTo(InstrProfRecord &Record,
+                                  InstrProfRecord::ValueMapType *VMap) {
+  if (NumValueKinds == 0)
+    return;
+
+  ValueProfRecord *VR = getFirstValueProfRecord();
+  for (uint32_t K = 0; K < NumValueKinds; K++) {
+    VR->deserializeTo(Record, VMap);
+    VR = VR->getNext();
+  }
+}
+
+std::unique_ptr<ValueProfData>
+ValueProfData::serializeFrom(const InstrProfRecord &Record) {
+  uint32_t TotalSize = getSize(Record);
+  std::unique_ptr<ValueProfData> VPD(
+      reinterpret_cast<ValueProfData *>(new char[TotalSize]));
+
+  VPD->TotalSize = TotalSize;
+  VPD->NumValueKinds = Record.getNumValueKinds();
+  ValueProfRecord *VR = VPD->getFirstValueProfRecord();
+  for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; Kind++) {
+    uint32_t NumValueSites = Record.getNumValueSites(Kind);
+    if (!NumValueSites)
+      continue;
+    VR->serializeFrom(Record, Kind, NumValueSites);
+    VR = VR->getNext();
+  }
+  return VPD;
+}
+
+ErrorOr<std::unique_ptr<ValueProfData>>
+ValueProfData::getValueProfData(const unsigned char *D,
+                                const unsigned char *const BufferEnd,
+                                support::endianness Endianness) {
+  using namespace support;
+  if (D + sizeof(ValueProfData) > BufferEnd)
+    return instrprof_error::truncated;
+
+  uint32_t TotalSize = swapToHostOrder<uint32_t>(
+      reinterpret_cast<const uint32_t *>(D)[0], Endianness);
+  uint32_t NumValueKinds = swapToHostOrder<uint32_t>(
+      reinterpret_cast<const uint32_t *>(D)[1], Endianness);
+
+  if (D + TotalSize > BufferEnd)
+    return instrprof_error::too_large;
+  if (NumValueKinds > IPVK_Last + 1)
+    return instrprof_error::malformed;
+  // Total size needs to be mulltiple of quadword size.
+  if (TotalSize % sizeof(uint64_t))
+    return instrprof_error::malformed;
+
+  std::unique_ptr<ValueProfData> VPD(
+      reinterpret_cast<ValueProfData *>(new char[TotalSize]));
+  memcpy(VPD.get(), D, TotalSize);
+  // Byte swap.
+  VPD->swapBytesToHost(Endianness);
+
+  // Data integrety check:
+  ValueProfRecord *VR = VPD->getFirstValueProfRecord();
+  for (uint32_t K = 0; K < VPD->NumValueKinds; K++) {
+    if (VR->Kind > IPVK_Last)
+      return instrprof_error::malformed;
+    VR = VR->getNext();
+    if ((char *)VR - (char *)VPD.get() > TotalSize)
+      return instrprof_error::malformed;
+  }
+
+  D += TotalSize;
+  return std::move(VPD);
+}
+
+void ValueProfData::swapBytesToHost(support::endianness Endianness) {
+  using namespace support;
+  if (Endianness == getHostEndianness())
+    return;
+
+  sys::swapByteOrder<uint32_t>(TotalSize);
+  sys::swapByteOrder<uint32_t>(NumValueKinds);
+
+  ValueProfRecord *VR = getFirstValueProfRecord();
+  for (uint32_t K = 0; K < NumValueKinds; K++) {
+    VR->swapBytes(Endianness, getHostEndianness());
+    VR = VR->getNext();
+  }
+}
+
+void ValueProfData::swapBytesFromHost(support::endianness Endianness) {
+  using namespace support;
+  if (Endianness == getHostEndianness())
+    return;
+
+  ValueProfRecord *VR = getFirstValueProfRecord();
+  for (uint32_t K = 0; K < NumValueKinds; K++) {
+    ValueProfRecord *NVR = VR->getNext();
+    VR->swapBytes(getHostEndianness(), Endianness);
+    VR = NVR;
+  }
+  sys::swapByteOrder<uint32_t>(TotalSize);
+  sys::swapByteOrder<uint32_t>(NumValueKinds);
+}
+
+ValueProfRecord *ValueProfData::getFirstValueProfRecord() {
+  return reinterpret_cast<ValueProfRecord *>((char *)this +
+                                             sizeof(ValueProfData));
+}
+
+uint32_t ValueProfRecord::getNumValueData() const {
+  uint32_t NumValueData = 0;
+  for (uint32_t I = 0; I < NumValueSites; I++)
+    NumValueData += SiteCountArray[I];
+  return NumValueData;
+}
+
+ValueProfRecord *ValueProfRecord::getNext() {
+  return reinterpret_cast<ValueProfRecord *>((char *)this + getSize());
+}
+
+InstrProfValueData *ValueProfRecord::getValueData() {
+  return reinterpret_cast<InstrProfValueData *>((char *)this +
+                                                getHeaderSize(NumValueSites));
+}
+
+} // End of IndexedInstrProf namespace.

 }
 }

diff --git a/lib/ProfileData/InstrProfReader.cpp b/lib/ProfileData/InstrProfReader.cpp
index 66fd961c84f1ccfb90bbc50ab852f81f39ca8c35..b968e6beafcaeea6cb77eafa4d45858558f095d9 100644 (file)
--- a/lib/ProfileData/InstrProfReader.cpp
+++ b/lib/ProfileData/InstrProfReader.cpp
@@ -298,51 +298,16 @@ typedef InstrProfLookupTrait::offset_type offset_type;
 
 bool InstrProfLookupTrait::ReadValueProfilingData(
     const unsigned char *&D, const unsigned char *const End) {
 
 bool InstrProfLookupTrait::ReadValueProfilingData(
     const unsigned char *&D, const unsigned char *const End) {

+  ErrorOr<std::unique_ptr<IndexedInstrProf::ValueProfData>> VDataPtrOrErr =
+      IndexedInstrProf::ValueProfData::getValueProfData(
+          D, End, ValueProfDataEndianness);

 
 

-  using namespace support;
-  // Read number of value kinds with value sites.
-  if (D + sizeof(uint64_t) > End)
+  if (VDataPtrOrErr.getError())

     return false;
     return false;

-  uint64_t ValueKindCount = endian::readNext<uint64_t, little, unaligned>(D);
-
-  InstrProfRecord &ProfRecord = DataBuffer.back();
-  for (uint32_t Kind = 0; Kind < ValueKindCount; ++Kind) {
-
-    // Read value kind and number of value sites for kind.
-    if (D + 2 * sizeof(uint64_t) > End)
-      return false;
-
-    uint64_t ValueKind = endian::readNext<uint64_t, little, unaligned>(D);
-    uint64_t ValueSiteCount = endian::readNext<uint64_t, little, unaligned>(D);

 
 

-    ProfRecord.reserveSites(ValueKind, ValueSiteCount);
+  VDataPtrOrErr.get()->deserializeTo(DataBuffer.back(), &HashKeys);
+  D += VDataPtrOrErr.get()->TotalSize;

 
 

-    for (uint64_t VSite = 0; VSite < ValueSiteCount; ++VSite) {
-      // Read number of value data pairs at value site.
-      if (D + sizeof(uint64_t) > End)
-        return false;
-
-      uint64_t ValueDataCount =
-          endian::readNext<uint64_t, little, unaligned>(D);
-
-      // Check if there are as many ValueDataPairs as ValueDataCount in memory.
-      if (D + (ValueDataCount << 1) * sizeof(uint64_t) > End)
-        return false;
-
-      std::unique_ptr<InstrProfValueData[]> VDataPtr(
-          ValueDataCount == 0 ? nullptr
-                              : new InstrProfValueData[ValueDataCount]);
-
-      for (uint64_t VCount = 0; VCount < ValueDataCount; ++VCount) {
-        VDataPtr[VCount].Value =
-            endian::readNext<uint64_t, little, unaligned>(D);
-        VDataPtr[VCount].Count =
-            endian::readNext<uint64_t, little, unaligned>(D);
-      }
-      ProfRecord.addValueData(ValueKind, VSite, VDataPtr.get(), ValueDataCount,
-                              &HashKeys);
-    }
-  }

   return true;
 }
 
   return true;
 }
 

diff --git a/lib/ProfileData/InstrProfWriter.cpp b/lib/ProfileData/InstrProfWriter.cpp
index b073d8df918bd76934308606a821706a2ec71072..b6725df52780250393570ea42a744386f3fef45b 100644 (file)
--- a/lib/ProfileData/InstrProfWriter.cpp
+++ b/lib/ProfileData/InstrProfWriter.cpp
@@ -20,6 +20,8 @@
 using namespace llvm;
 
 namespace {
 using namespace llvm;
 
 namespace {

+static support::endianness ValueProfDataEndianness = support::little;
+

 class InstrProfRecordTrait {
 public:
   typedef StringRef key_type;
 class InstrProfRecordTrait {
 public:
   typedef StringRef key_type;


@@ -51,20 +53,7 @@ public:
       M += ProfRecord.Counts.size() * sizeof(uint64_t);
 
       // Value data
       M += ProfRecord.Counts.size() * sizeof(uint64_t);
 
       // Value data

-      M += sizeof(uint64_t); // Number of value kinds with value sites.
-      for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
-        uint32_t NumValueSites = ProfRecord.getNumValueSites(Kind);
-        if (NumValueSites == 0)
-          continue;
-        M += sizeof(uint64_t); // Value kind
-        M += sizeof(uint64_t); // The number of value sites for given value kind
-        for (uint32_t I = 0; I < NumValueSites; I++) {
-          M += sizeof(uint64_t); // Number of value data pairs at a value site
-          uint64_t NumValueDataForSite =
-              ProfRecord.getNumValueDataForSite(Kind, I);
-          M += 2 * sizeof(uint64_t) * NumValueDataForSite; // Value data pairs
-        }
-      }
+      M += IndexedInstrProf::ValueProfData::getSize(ProfileData.second);

     }
     LE.write<offset_type>(M);
 
     }
     LE.write<offset_type>(M);
 


@@ -87,36 +76,12 @@ public:
       for (uint64_t I : ProfRecord.Counts)
         LE.write<uint64_t>(I);
 
       for (uint64_t I : ProfRecord.Counts)
         LE.write<uint64_t>(I);
 

-      // Compute the number of value kinds with value sites.
-      uint64_t NumValueKinds = ProfRecord.getNumValueKinds();
-      LE.write<uint64_t>(NumValueKinds);
-

       // Write value data
       // Write value data

-      for (uint32_t Kind = IPVK_First; Kind <= IPVK_Last; ++Kind) {
-        uint32_t NumValueSites = ProfRecord.getNumValueSites(Kind);
-        if (NumValueSites == 0)
-          continue;
-        LE.write<uint64_t>(Kind); // Write value kind
-        // Write number of value sites for current value kind
-        LE.write<uint64_t>(NumValueSites);
-
-        for (uint32_t I = 0; I < NumValueSites; I++) {
-          // Write number of value data pairs at this value site
-          uint64_t NumValueDataForSite =
-              ProfRecord.getNumValueDataForSite(Kind, I);
-          LE.write<uint64_t>(NumValueDataForSite);
-          std::unique_ptr<InstrProfValueData[]> VD =
-              ProfRecord.getValueForSite(Kind, I);
-
-          for (uint32_t V = 0; V < NumValueDataForSite; V++) {
-            if (Kind == IPVK_IndirectCallTarget)
-              LE.write<uint64_t>(ComputeHash((const char *)VD[V].Value));
-            else
-              LE.write<uint64_t>(VD[V].Value);
-            LE.write<uint64_t>(VD[V].Count);
-          }
-        }
-      }
+      std::unique_ptr<IndexedInstrProf::ValueProfData> VDataPtr =
+          IndexedInstrProf::ValueProfData::serializeFrom(ProfileData.second);
+      uint32_t S = VDataPtr->getSize();
+      VDataPtr->swapBytesFromHost(ValueProfDataEndianness);
+      Out.write((const char *)VDataPtr.get(), S);

     }
   }
 };
     }
   }
 };


@@ -148,6 +113,12 @@ static std::error_code combineInstrProfRecords(InstrProfRecord &Dest,
   return instrprof_error::success;
 }
 
   return instrprof_error::success;
 }
 

+// Internal interface for testing purpose only.
+void InstrProfWriter::setValueProfDataEndianness(
+    support::endianness Endianness) {
+  ValueProfDataEndianness = Endianness;
+}
+

 void InstrProfWriter::updateStringTableReferences(InstrProfRecord &I) {
   I.updateStrings(&StringTable);
 }
 void InstrProfWriter::updateStringTableReferences(InstrProfRecord &I) {
   I.updateStrings(&StringTable);
 }

diff --git a/unittests/ProfileData/InstrProfTest.cpp b/unittests/ProfileData/InstrProfTest.cpp
index 64c50bf3225714d85868d9c02614210e0897aba1..a9257fe316e0c8c37d333650e92315081f504546 100644 (file)
--- a/unittests/ProfileData/InstrProfTest.cpp
+++ b/unittests/ProfileData/InstrProfTest.cpp
@@ -133,7 +133,54 @@ TEST_F(InstrProfTest, get_icall_data_read_write) {
                               {(uint64_t) "callee2", 2},
                               {(uint64_t) "callee3", 3}};
   Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
                               {(uint64_t) "callee2", 2},
                               {(uint64_t) "callee3", 3}};
   Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);

-  // No valeu profile data at the second site.
+  // No value profile data at the second site.
+  Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
+  InstrProfValueData VD2[] = {{(uint64_t) "callee1", 1},
+                              {(uint64_t) "callee2", 2}};
+  Record1.addValueData(IPVK_IndirectCallTarget, 2, VD2, 2, nullptr);
+  InstrProfValueData VD3[] = {{(uint64_t) "callee1", 1}};
+  Record1.addValueData(IPVK_IndirectCallTarget, 3, VD3, 1, nullptr);
+
+  Writer.addRecord(std::move(Record1));
+  Writer.addRecord(std::move(Record2));
+  Writer.addRecord(std::move(Record3));
+  Writer.addRecord(std::move(Record4));
+  auto Profile = Writer.writeBuffer();
+  readProfile(std::move(Profile));
+
+  ErrorOr<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
+  ASSERT_TRUE(NoError(R.getError()));
+  ASSERT_EQ(4U, R.get().getNumValueSites(IPVK_IndirectCallTarget));
+  ASSERT_EQ(3U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 0));
+  ASSERT_EQ(0U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 1));
+  ASSERT_EQ(2U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 2));
+  ASSERT_EQ(1U, R.get().getNumValueDataForSite(IPVK_IndirectCallTarget, 3));
+
+  std::unique_ptr<InstrProfValueData[]> VD =
+      R.get().getValueForSite(IPVK_IndirectCallTarget, 0);
+  // Now sort the target acording to frequency.
+  std::sort(&VD[0], &VD[3],
+            [](const InstrProfValueData &VD1, const InstrProfValueData &VD2) {
+              return VD1.Count > VD2.Count;
+            });
+  ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
+  ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
+  ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
+}
+
+TEST_F(InstrProfTest, get_icall_data_read_write_big_endian) {
+  InstrProfRecord Record1("caller", 0x1234, {1, 2});
+  InstrProfRecord Record2("callee1", 0x1235, {3, 4});
+  InstrProfRecord Record3("callee2", 0x1235, {3, 4});
+  InstrProfRecord Record4("callee3", 0x1235, {3, 4});
+
+  // 4 value sites.
+  Record1.reserveSites(IPVK_IndirectCallTarget, 4);
+  InstrProfValueData VD0[] = {{(uint64_t) "callee1", 1},
+                              {(uint64_t) "callee2", 2},
+                              {(uint64_t) "callee3", 3}};
+  Record1.addValueData(IPVK_IndirectCallTarget, 0, VD0, 3, nullptr);
+  // No value profile data at the second site.

   Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
   InstrProfValueData VD2[] = {{(uint64_t) "callee1", 1},
                               {(uint64_t) "callee2", 2}};
   Record1.addValueData(IPVK_IndirectCallTarget, 1, nullptr, 0, nullptr);
   InstrProfValueData VD2[] = {{(uint64_t) "callee1", 1},
                               {(uint64_t) "callee2", 2}};


@@ -145,9 +192,16 @@ TEST_F(InstrProfTest, get_icall_data_read_write) {
   Writer.addRecord(std::move(Record2));
   Writer.addRecord(std::move(Record3));
   Writer.addRecord(std::move(Record4));
   Writer.addRecord(std::move(Record2));
   Writer.addRecord(std::move(Record3));
   Writer.addRecord(std::move(Record4));

+
+  // Set big endian output.
+  Writer.setValueProfDataEndianness(support::big);
+

   auto Profile = Writer.writeBuffer();
   readProfile(std::move(Profile));
 
   auto Profile = Writer.writeBuffer();
   readProfile(std::move(Profile));
 

+  // Set big endian input.
+  Reader->setValueProfDataEndianness(support::big);
+

   ErrorOr<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
   ASSERT_TRUE(NoError(R.getError()));
   ASSERT_EQ(4U, R.get().getNumValueSites(IPVK_IndirectCallTarget));
   ErrorOr<InstrProfRecord> R = Reader->getInstrProfRecord("caller", 0x1234);
   ASSERT_TRUE(NoError(R.getError()));
   ASSERT_EQ(4U, R.get().getNumValueSites(IPVK_IndirectCallTarget));


@@ -166,6 +220,9 @@ TEST_F(InstrProfTest, get_icall_data_read_write) {
   ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
   ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
   ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));
   ASSERT_EQ(StringRef((const char *)VD[0].Value, 7), StringRef("callee3"));
   ASSERT_EQ(StringRef((const char *)VD[1].Value, 7), StringRef("callee2"));
   ASSERT_EQ(StringRef((const char *)VD[2].Value, 7), StringRef("callee1"));

+
+  // Restore little endian default:
+  Writer.setValueProfDataEndianness(support::little);

 }
 
 TEST_F(InstrProfTest, get_icall_data_merge1) {
 }
 
 TEST_F(InstrProfTest, get_icall_data_merge1) {