#include "InstrProfiling.h"
#include "InstrProfilingInternal.h"
+#include "InstrProfilingUtil.h" /* For PS4 getenv shim. */
#include <limits.h>
#include <stdio.h>
#include <stdlib.h>
#define INSTR_PROF_COMMON_API_IMPL
#include "InstrProfData.inc"
-#define PROF_OOM(Msg) PROF_ERR(Msg ":%s\n", "Out of memory");
-#define PROF_OOM_RETURN(Msg) \
- { \
- PROF_OOM(Msg) \
- free(ValueDataArray); \
- return NULL; \
- }
+static int hasStaticCounters = 1;
+static int OutOfNodesWarnings = 0;
+static int hasNonDefaultValsPerSite = 0;
+#define INSTR_PROF_MAX_VP_WARNS 10
+#define INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE 8
+#define INSTR_PROF_VNODE_POOL_SIZE 1024
+
+#ifndef _MSC_VER
+/* A shared static pool in addition to the vnodes statically
+ * allocated by the compiler. */
+COMPILER_RT_VISIBILITY ValueProfNode
+ lprofValueProfNodes[INSTR_PROF_VNODE_POOL_SIZE] COMPILER_RT_SECTION(
+ COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME_STR);
+#endif
-#if COMPILER_RT_HAS_ATOMICS != 1
-COMPILER_RT_VISIBILITY
-uint32_t BoolCmpXchg(void **Ptr, void *OldV, void *NewV) {
- void *R = *Ptr;
- if (R == OldV) {
- *Ptr = NewV;
- return 1;
+COMPILER_RT_VISIBILITY uint32_t VPMaxNumValsPerSite =
+ INSTR_PROF_DEFAULT_NUM_VAL_PER_SITE;
+
+COMPILER_RT_VISIBILITY void lprofSetupValueProfiler() {
+ const char *Str = 0;
+ Str = getenv("LLVM_VP_MAX_NUM_VALS_PER_SITE");
+ if (Str && Str[0]) {
+ VPMaxNumValsPerSite = atoi(Str);
+ hasNonDefaultValsPerSite = 1;
}
- return 0;
+ if (VPMaxNumValsPerSite > INSTR_PROF_MAX_NUM_VAL_PER_SITE)
+ VPMaxNumValsPerSite = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
+}
+
+COMPILER_RT_VISIBILITY void lprofSetMaxValsPerSite(uint32_t MaxVals) {
+ VPMaxNumValsPerSite = MaxVals;
+ hasNonDefaultValsPerSite = 1;
}
-#endif
/* This method is only used in value profiler mock testing. */
COMPILER_RT_VISIBILITY void
static int allocateValueProfileCounters(__llvm_profile_data *Data) {
uint64_t NumVSites = 0;
uint32_t VKI;
+
+ /* This function will never be called when value site array is allocated
+ statically at compile time. */
+ hasStaticCounters = 0;
+ /* When dynamic allocation is enabled, allow tracking the max number of
+ * values allowd. */
+ if (!hasNonDefaultValsPerSite)
+ VPMaxNumValsPerSite = INSTR_PROF_MAX_NUM_VAL_PER_SITE;
+
for (VKI = IPVK_First; VKI <= IPVK_Last; ++VKI)
NumVSites += Data->NumValueSites[VKI];
return 1;
}
+static ValueProfNode *allocateOneNode(__llvm_profile_data *Data, uint32_t Index,
+ uint64_t Value) {
+ ValueProfNode *Node;
+
+ if (!hasStaticCounters)
+ return (ValueProfNode *)calloc(1, sizeof(ValueProfNode));
+
+ /* Early check to avoid value wrapping around. */
+ if (CurrentVNode + 1 > EndVNode) {
+ if (OutOfNodesWarnings++ < INSTR_PROF_MAX_VP_WARNS) {
+ PROF_WARN("Unable to track new values: %s. "
+ " Consider using option -mllvm -vp-counters-per-site=<n> to "
+ "allocate more"
+ " value profile counters at compile time. \n",
+ "Running out of static counters");
+ }
+ return 0;
+ }
+ Node = COMPILER_RT_PTR_FETCH_ADD(ValueProfNode, CurrentVNode, 1);
+ /* Due to section padding, EndVNode point to a byte which is one pass
+ * an incomplete VNode, so we need to skip the last incomplete node. */
+ if (Node + 1 > EndVNode)
+ return 0;
+
+ return Node;
+}
+
COMPILER_RT_VISIBILITY void
__llvm_profile_instrument_target(uint64_t TargetValue, void *Data,
uint32_t CounterIndex) {
-
__llvm_profile_data *PData = (__llvm_profile_data *)Data;
if (!PData)
return;
ValueProfNode **ValueCounters = (ValueProfNode **)PData->Values;
ValueProfNode *PrevVNode = NULL;
- ValueProfNode *CurrentVNode = ValueCounters[CounterIndex];
+ ValueProfNode *MinCountVNode = NULL;
+ ValueProfNode *CurVNode = ValueCounters[CounterIndex];
+ uint64_t MinCount = UINT64_MAX;
uint8_t VDataCount = 0;
- while (CurrentVNode) {
- if (TargetValue == CurrentVNode->VData.Value) {
- CurrentVNode->VData.Count++;
+ while (CurVNode) {
+ if (TargetValue == CurVNode->Value) {
+ CurVNode->Count++;
return;
}
- PrevVNode = CurrentVNode;
- CurrentVNode = CurrentVNode->Next;
+ if (CurVNode->Count < MinCount) {
+ MinCount = CurVNode->Count;
+ MinCountVNode = CurVNode;
+ }
+ PrevVNode = CurVNode;
+ CurVNode = CurVNode->Next;
++VDataCount;
}
- if (VDataCount >= INSTR_PROF_MAX_NUM_VAL_PER_SITE)
+ if (VDataCount >= VPMaxNumValsPerSite) {
+ /* Bump down the min count node's count. If it reaches 0,
+ * evict it. This eviction/replacement policy makes hot
+ * targets more sticky while cold targets less so. In other
+ * words, it makes it less likely for the hot targets to be
+ * prematurally evicted during warmup/establishment period,
+ * when their counts are still low. In a special case when
+ * the number of values tracked is reduced to only one, this
+ * policy will guarantee that the dominating target with >50%
+ * total count will survive in the end. Note that this scheme
+ * allows the runtime to track the min count node in an adaptive
+ * manner. It can correct previous mistakes and eventually
+ * lock on a cold target that is alread in stable state.
+ *
+ * In very rare cases, this replacement scheme may still lead
+ * to target loss. For instance, out of \c N value slots, \c N-1
+ * slots are occupied by luke warm targets during the warmup
+ * period and the remaining one slot is competed by two or more
+ * very hot targets. If those hot targets occur in an interleaved
+ * way, none of them will survive (gain enough weight to throw out
+ * other established entries) due to the ping-pong effect.
+ * To handle this situation, user can choose to increase the max
+ * number of tracked values per value site. Alternatively, a more
+ * expensive eviction mechanism can be implemented. It requires
+ * the runtime to track the total number of evictions per-site.
+ * When the total number of evictions reaches certain threshold,
+ * the runtime can wipe out more than one lowest count entries
+ * to give space for hot targets.
+ */
+ if (!(--MinCountVNode->Count)) {
+ CurVNode = MinCountVNode;
+ CurVNode->Value = TargetValue;
+ CurVNode->Count++;
+ }
return;
+ }
- CurrentVNode = (ValueProfNode *)calloc(1, sizeof(ValueProfNode));
- if (!CurrentVNode)
+ CurVNode = allocateOneNode(PData, CounterIndex, TargetValue);
+ if (!CurVNode)
return;
-
- CurrentVNode->VData.Value = TargetValue;
- CurrentVNode->VData.Count++;
+ CurVNode->Value = TargetValue;
+ CurVNode->Count++;
uint32_t Success = 0;
if (!ValueCounters[CounterIndex])
Success =
- COMPILER_RT_BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurrentVNode);
+ COMPILER_RT_BOOL_CMPXCHG(&ValueCounters[CounterIndex], 0, CurVNode);
else if (PrevVNode && !PrevVNode->Next)
- Success = COMPILER_RT_BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurrentVNode);
+ Success = COMPILER_RT_BOOL_CMPXCHG(&(PrevVNode->Next), 0, CurVNode);
- if (!Success) {
- free(CurrentVNode);
+ if (!Success && !hasStaticCounters) {
+ free(CurVNode);
return;
}
}
-COMPILER_RT_VISIBILITY ValueProfData **
-__llvm_profile_gather_value_data(uint64_t *ValueDataSize) {
- size_t S = 0;
- __llvm_profile_data *I;
- ValueProfData **ValueDataArray;
-
- const __llvm_profile_data *DataEnd = __llvm_profile_end_data();
- const __llvm_profile_data *DataBegin = __llvm_profile_begin_data();
-
- if (!ValueDataSize)
- return NULL;
-
- ValueDataArray =
- (ValueProfData **)calloc(DataEnd - DataBegin, sizeof(void *));
- if (!ValueDataArray)
- PROF_OOM_RETURN("Failed to write value profile data ");
-
- /*
- * Compute the total Size of the buffer to hold ValueProfData
- * structures for functions with value profile data.
- */
- for (I = (__llvm_profile_data *)DataBegin; I != DataEnd; ++I) {
- ValueProfRuntimeRecord R;
- if (initializeValueProfRuntimeRecord(&R, I->NumValueSites, I->Values))
- PROF_OOM_RETURN("Failed to write value profile data ");
-
- /* Compute the size of ValueProfData from this runtime record. */
- if (getNumValueKindsRT(&R) != 0) {
- ValueProfData *VD = NULL;
- uint32_t VS = getValueProfDataSizeRT(&R);
- VD = (ValueProfData *)calloc(VS, sizeof(uint8_t));
- if (!VD)
- PROF_OOM_RETURN("Failed to write value profile data ");
- serializeValueProfDataFromRT(&R, VD);
- ValueDataArray[I - DataBegin] = VD;
- S += VS;
+/*
+ * A wrapper struct that represents value profile runtime data.
+ * Like InstrProfRecord class which is used by profiling host tools,
+ * ValueProfRuntimeRecord also implements the abstract intefaces defined in
+ * ValueProfRecordClosure so that the runtime data can be serialized using
+ * shared C implementation.
+ */
+typedef struct ValueProfRuntimeRecord {
+ const __llvm_profile_data *Data;
+ ValueProfNode **NodesKind[IPVK_Last + 1];
+ uint8_t **SiteCountArray;
+} ValueProfRuntimeRecord;
+
+/* ValueProfRecordClosure Interface implementation. */
+
+static uint32_t getNumValueSitesRT(const void *R, uint32_t VK) {
+ return ((const ValueProfRuntimeRecord *)R)->Data->NumValueSites[VK];
+}
+
+static uint32_t getNumValueDataRT(const void *R, uint32_t VK) {
+ uint32_t S = 0, I;
+ const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
+ if (Record->SiteCountArray[VK] == INSTR_PROF_NULLPTR)
+ return 0;
+ for (I = 0; I < Record->Data->NumValueSites[VK]; I++)
+ S += Record->SiteCountArray[VK][I];
+ return S;
+}
+
+static uint32_t getNumValueDataForSiteRT(const void *R, uint32_t VK,
+ uint32_t S) {
+ const ValueProfRuntimeRecord *Record = (const ValueProfRuntimeRecord *)R;
+ return Record->SiteCountArray[VK][S];
+}
+
+static ValueProfRuntimeRecord RTRecord;
+static ValueProfRecordClosure RTRecordClosure = {
+ &RTRecord, INSTR_PROF_NULLPTR, /* GetNumValueKinds */
+ getNumValueSitesRT, getNumValueDataRT, getNumValueDataForSiteRT,
+ INSTR_PROF_NULLPTR, /* RemapValueData */
+ INSTR_PROF_NULLPTR, /* GetValueForSite, */
+ INSTR_PROF_NULLPTR /* AllocValueProfData */
+};
+
+static uint32_t
+initializeValueProfRuntimeRecord(const __llvm_profile_data *Data,
+ uint8_t *SiteCountArray[]) {
+ unsigned I, J, S = 0, NumValueKinds = 0;
+ ValueProfNode **Nodes = (ValueProfNode **)Data->Values;
+ RTRecord.Data = Data;
+ RTRecord.SiteCountArray = SiteCountArray;
+ for (I = 0; I <= IPVK_Last; I++) {
+ uint16_t N = Data->NumValueSites[I];
+ if (!N)
+ continue;
+
+ NumValueKinds++;
+
+ RTRecord.NodesKind[I] = Nodes ? &Nodes[S] : INSTR_PROF_NULLPTR;
+ for (J = 0; J < N; J++) {
+ /* Compute value count for each site. */
+ uint32_t C = 0;
+ ValueProfNode *Site =
+ Nodes ? RTRecord.NodesKind[I][J] : INSTR_PROF_NULLPTR;
+ while (Site) {
+ C++;
+ Site = Site->Next;
+ }
+ if (C > UCHAR_MAX)
+ C = UCHAR_MAX;
+ RTRecord.SiteCountArray[I][J] = C;
}
- finalizeValueProfRuntimeRecord(&R);
+ S += N;
}
+ return NumValueKinds;
+}
- if (!S) {
- free(ValueDataArray);
- ValueDataArray = NULL;
+static ValueProfNode *getNextNValueData(uint32_t VK, uint32_t Site,
+ InstrProfValueData *Dst,
+ ValueProfNode *StartNode, uint32_t N) {
+ unsigned I;
+ ValueProfNode *VNode = StartNode ? StartNode : RTRecord.NodesKind[VK][Site];
+ for (I = 0; I < N; I++) {
+ Dst[I].Value = VNode->Value;
+ Dst[I].Count = VNode->Count;
+ VNode = VNode->Next;
}
+ return VNode;
+}
+
+static uint32_t getValueProfDataSizeWrapper(void) {
+ return getValueProfDataSize(&RTRecordClosure);
+}
+
+static uint32_t getNumValueDataForSiteWrapper(uint32_t VK, uint32_t S) {
+ return getNumValueDataForSiteRT(&RTRecord, VK, S);
+}
+
+static VPDataReaderType TheVPDataReader = {
+ initializeValueProfRuntimeRecord, getValueProfRecordHeaderSize,
+ getFirstValueProfRecord, getNumValueDataForSiteWrapper,
+ getValueProfDataSizeWrapper, getNextNValueData};
- *ValueDataSize = S;
- return ValueDataArray;
+COMPILER_RT_VISIBILITY VPDataReaderType *lprofGetVPDataReader() {
+ return &TheVPDataReader;
}
projects / rustc.git / blobdiff