internal float Accrete(HistogramHeatPoint point, AggregationMethod aggregatioMethod, float percentile = 0f)
{
switch (aggregatioMethod)
{
case AggregationMethod.Increment:
return(point.histogram.Count);
case AggregationMethod.Cumulative:
return(point.histogram.Sum(x => (float)Convert.ToDecimal(x)));
case AggregationMethod.Min:
point.histogram.Sort();
return(point.histogram[0]);
case AggregationMethod.Max:
point.histogram.Sort();
return(point.histogram[point.histogram.Count - 1]);
case AggregationMethod.First:
return(point.first);
case AggregationMethod.Last:
return(point.last);
case AggregationMethod.Average:
return(point.histogram.Sum(x => (float)Convert.ToDecimal(x)) / point.histogram.Count);
case AggregationMethod.Percentile:
point.histogram.Sort();
int percentileIdx = (int)((percentile / 100f) * point.histogram.Count);
int finalIdx = (percentileIdx >= point.histogram.Count - 1) ? point.histogram.Count - 1 : percentileIdx;
return(point.histogram[finalIdx]);
}
return(0);
}
internal void LoadStream(Dictionary <Tuplish, List <HistogramHeatPoint> > histograms,
Dictionary <string, int> headers,
string path,
DateTime startDate, DateTime endDate,
List <string> aggregateOn,
Dictionary <string, float> smoothOn,
List <string> groupOn,
string remapDensityToField)
{
bool doRemap = !string.IsNullOrEmpty(remapDensityToField);
if (doRemap)
{
aggregateOn.Add(remapDensityToField);
}
if (!System.IO.File.Exists(path))
{
Debug.LogWarningFormat("File {0} not found.", path);
return;
}
string tsv = IonicGZip.DecompressFile(path);
string[] rows = tsv.Split('\n');
m_ReportRows += rows.Length;
// Define indices
int nameIndex = headers["name"];
int submitTimeIndex = headers["submit_time"];
int paramsIndex = headers["custom_params"];
int userIdIndex = headers["userid"];
int sessionIdIndex = headers["sessionid"];
int platformIndex = headers["platform"];
int isDebugIndex = headers["debug_device"];
for (int a = 0; a < rows.Length; a++)
{
List <string> rowData = new List <string>(rows[a].Split('\t'));
if (rowData.Count < 6)
{
// Re-enable this log if you want to see empty lines
//Debug.Log ("No data in line...skipping");
continue;
}
string userId = rowData[userIdIndex];
string sessionId = rowData[sessionIdIndex];
string eventName = rowData[nameIndex];
string paramsData = rowData[paramsIndex];
double unixTimeStamp = double.Parse(rowData[submitTimeIndex]);
DateTime rowDate = DateTimeUtils.s_Epoch.AddMilliseconds(unixTimeStamp);
string platform = rowData[platformIndex];
bool isDebug = bool.Parse(rowData[isDebugIndex]);
// Pass on rows outside any date trimming
if (rowDate < startDate || rowDate > endDate)
{
continue;
}
Dictionary <string, object> datum = MiniJSON.Json.Deserialize(paramsData) as Dictionary <string, object>;
// If no x/y, this isn't a Heatmap Event. Pass.
if (!datum.ContainsKey("x") || !datum.ContainsKey("y"))
{
// Re-enable this log line if you want to be see events that aren't valid for heatmapping
//Debug.Log ("Unable to find x/y in: " + datum.ToString () + ". Skipping...");
continue;
}
// Passed all checks. Consider as legal point
m_ReportLegalPoints++;
// Construct both the list of elements that signify a unique item...
var pointTupleList = new List <object> {
eventName
};
// ...and a point to contain the data
HistogramHeatPoint point = new HistogramHeatPoint();
foreach (var ag in aggregateOn)
{
float floatValue = 0f;
object arbitraryValue = 0f;
// Special cases for userIDs, sessionIDs, platform, and debug, which aren't in the JSON
if (ag == "userID")
{
arbitraryValue = userId;
}
else if (ag == "sessionID")
{
arbitraryValue = sessionId;
}
else if (ag == "platform")
{
arbitraryValue = platform;
}
else if (ag == "debug")
{
arbitraryValue = isDebug;
}
else if (datum.ContainsKey(ag))
{
// parse and divide all in smoothing list
float.TryParse((string)datum[ag], out floatValue);
if (smoothOn.ContainsKey(ag))
{
floatValue = Divide(floatValue, smoothOn[ag]);
}
else
{
floatValue = 0;
}
arbitraryValue = floatValue;
}
pointTupleList.Add(arbitraryValue);
// Add values to the point
if (pointProperties.Contains(ag))
{
point[ag] = floatValue;
}
}
// Turn the pointTupleList into a key
var pointTuple = new Tuplish(pointTupleList.ToArray());
float remapValue = 1f;
if (doRemap && datum.ContainsKey(remapDensityToField))
{
float.TryParse((string)datum[remapDensityToField], out remapValue);
}
if (m_PointDict.ContainsKey(pointTuple))
{
// Use existing point if it exists...
point = m_PointDict[pointTuple];
point.histogram.Add(remapValue);
if (rowDate < point.firstDate)
{
point.first = remapValue;
point.firstDate = rowDate;
}
else if (rowDate > point.lastDate)
{
point.last = remapValue;
point.lastDate = rowDate;
}
}
else
{
// ...or else use the one we've been constructing
point.histogram.Add(remapValue);
point.first = remapValue;
point.last = remapValue;
point.firstDate = rowDate;
point.lastDate = rowDate;
// CREATE GROUPING LIST
var groupTupleList = new List <object>();
foreach (var field in groupOn)
{
// Special case for eventName
if (field == "eventName")
{
groupTupleList.Add(eventName);
}
// Special cases for... userID
else if (field == "userID")
{
groupTupleList.Add("user: "******"sessionID")
{
groupTupleList.Add("session: " + sessionId);
}
// ... debug ...
else if (field == "debug")
{
groupTupleList.Add("debug: " + isDebug);
}
// ... platform
else if (field == "platform")
{
groupTupleList.Add("platform: " + platform);
}
// Everything else just added to key
else if (datum.ContainsKey(field))
{
groupTupleList.Add(field + ": " + datum[field]);
}
}
var groupTuple = new Tuplish(groupTupleList.ToArray());
// Create the event list if the key doesn't exist
if (!histograms.ContainsKey(groupTuple))
{
histograms.Add(groupTuple, new List <HistogramHeatPoint>());
}
// FINALLY, ADD THE POINT TO THE CORRECT GROUP...
histograms[groupTuple].Add(point);
// ...AND THE POINT DICT
m_PointDict[pointTuple] = point;
}
}
}
