internal float Accrete(HistogramHeatPoint point, AggregationMethod aggregatioMethod, float percentile = 0f) { switch (aggregatioMethod) { case AggregationMethod.Increment: return(point.histogram.Count); case AggregationMethod.Cumulative: return(point.histogram.Sum(x => (float)Convert.ToDecimal(x))); case AggregationMethod.Min: point.histogram.Sort(); return(point.histogram[0]); case AggregationMethod.Max: point.histogram.Sort(); return(point.histogram[point.histogram.Count - 1]); case AggregationMethod.First: return(point.first); case AggregationMethod.Last: return(point.last); case AggregationMethod.Average: return(point.histogram.Sum(x => (float)Convert.ToDecimal(x)) / point.histogram.Count); case AggregationMethod.Percentile: point.histogram.Sort(); int percentileIdx = (int)((percentile / 100f) * point.histogram.Count); int finalIdx = (percentileIdx >= point.histogram.Count - 1) ? point.histogram.Count - 1 : percentileIdx; return(point.histogram[finalIdx]); } return(0); }
internal void LoadStream(Dictionary <Tuplish, List <HistogramHeatPoint> > histograms, Dictionary <string, int> headers, string path, DateTime startDate, DateTime endDate, List <string> aggregateOn, Dictionary <string, float> smoothOn, List <string> groupOn, string remapDensityToField) { bool doRemap = !string.IsNullOrEmpty(remapDensityToField); if (doRemap) { aggregateOn.Add(remapDensityToField); } if (!System.IO.File.Exists(path)) { Debug.LogWarningFormat("File {0} not found.", path); return; } string tsv = IonicGZip.DecompressFile(path); string[] rows = tsv.Split('\n'); m_ReportRows += rows.Length; // Define indices int nameIndex = headers["name"]; int submitTimeIndex = headers["submit_time"]; int paramsIndex = headers["custom_params"]; int userIdIndex = headers["userid"]; int sessionIdIndex = headers["sessionid"]; int platformIndex = headers["platform"]; int isDebugIndex = headers["debug_device"]; for (int a = 0; a < rows.Length; a++) { List <string> rowData = new List <string>(rows[a].Split('\t')); if (rowData.Count < 6) { // Re-enable this log if you want to see empty lines //Debug.Log ("No data in line...skipping"); continue; } string userId = rowData[userIdIndex]; string sessionId = rowData[sessionIdIndex]; string eventName = rowData[nameIndex]; string paramsData = rowData[paramsIndex]; double unixTimeStamp = double.Parse(rowData[submitTimeIndex]); DateTime rowDate = DateTimeUtils.s_Epoch.AddMilliseconds(unixTimeStamp); string platform = rowData[platformIndex]; bool isDebug = bool.Parse(rowData[isDebugIndex]); // Pass on rows outside any date trimming if (rowDate < startDate || rowDate > endDate) { continue; } Dictionary <string, object> datum = MiniJSON.Json.Deserialize(paramsData) as Dictionary <string, object>; // If no x/y, this isn't a Heatmap Event. Pass. if (!datum.ContainsKey("x") || !datum.ContainsKey("y")) { // Re-enable this log line if you want to be see events that aren't valid for heatmapping //Debug.Log ("Unable to find x/y in: " + datum.ToString () + ". Skipping..."); continue; } // Passed all checks. Consider as legal point m_ReportLegalPoints++; // Construct both the list of elements that signify a unique item... var pointTupleList = new List <object> { eventName }; // ...and a point to contain the data HistogramHeatPoint point = new HistogramHeatPoint(); foreach (var ag in aggregateOn) { float floatValue = 0f; object arbitraryValue = 0f; // Special cases for userIDs, sessionIDs, platform, and debug, which aren't in the JSON if (ag == "userID") { arbitraryValue = userId; } else if (ag == "sessionID") { arbitraryValue = sessionId; } else if (ag == "platform") { arbitraryValue = platform; } else if (ag == "debug") { arbitraryValue = isDebug; } else if (datum.ContainsKey(ag)) { // parse and divide all in smoothing list float.TryParse((string)datum[ag], out floatValue); if (smoothOn.ContainsKey(ag)) { floatValue = Divide(floatValue, smoothOn[ag]); } else { floatValue = 0; } arbitraryValue = floatValue; } pointTupleList.Add(arbitraryValue); // Add values to the point if (pointProperties.Contains(ag)) { point[ag] = floatValue; } } // Turn the pointTupleList into a key var pointTuple = new Tuplish(pointTupleList.ToArray()); float remapValue = 1f; if (doRemap && datum.ContainsKey(remapDensityToField)) { float.TryParse((string)datum[remapDensityToField], out remapValue); } if (m_PointDict.ContainsKey(pointTuple)) { // Use existing point if it exists... point = m_PointDict[pointTuple]; point.histogram.Add(remapValue); if (rowDate < point.firstDate) { point.first = remapValue; point.firstDate = rowDate; } else if (rowDate > point.lastDate) { point.last = remapValue; point.lastDate = rowDate; } } else { // ...or else use the one we've been constructing point.histogram.Add(remapValue); point.first = remapValue; point.last = remapValue; point.firstDate = rowDate; point.lastDate = rowDate; // CREATE GROUPING LIST var groupTupleList = new List <object>(); foreach (var field in groupOn) { // Special case for eventName if (field == "eventName") { groupTupleList.Add(eventName); } // Special cases for... userID else if (field == "userID") { groupTupleList.Add("user: "******"sessionID") { groupTupleList.Add("session: " + sessionId); } // ... debug ... else if (field == "debug") { groupTupleList.Add("debug: " + isDebug); } // ... platform else if (field == "platform") { groupTupleList.Add("platform: " + platform); } // Everything else just added to key else if (datum.ContainsKey(field)) { groupTupleList.Add(field + ": " + datum[field]); } } var groupTuple = new Tuplish(groupTupleList.ToArray()); // Create the event list if the key doesn't exist if (!histograms.ContainsKey(groupTuple)) { histograms.Add(groupTuple, new List <HistogramHeatPoint>()); } // FINALLY, ADD THE POINT TO THE CORRECT GROUP... histograms[groupTuple].Add(point); // ...AND THE POINT DICT m_PointDict[pointTuple] = point; } } }