protected void LoadStream(Dictionary<string, List<Dictionary<string, float>>> outputData, string path, DateTime startDate, DateTime endDate, float space, float time, float angle, bool disaggregateTime, bool disaggregateAngle, List<string> events, string outputFileName) { var reader = new StreamReader(path); using (reader) { string tsv = reader.ReadToEnd(); string[] rows = tsv.Split('\n'); m_ReportRows += rows.Length; for (int a = 0; a < rows.Length; a++) { string[] rowData = rows[a].Split('\t'); if (string.IsNullOrEmpty(rowData[0]) || string.IsNullOrEmpty(rowData[2]) || string.IsNullOrEmpty(rowData[3])) { // Re-enable this log if you want to see empty lines //Debug.Log ("Empty Line...skipping"); continue; } DateTime rowDate = DateTime.Parse(rowData[0]); // Pass on rows outside any date trimming if (rowDate < startDate || rowDate > endDate) { continue; } string eventName = rowData[2]; Dictionary<string, object> datum = MiniJSON.Json.Deserialize(rowData[3]) as Dictionary<string, object>; // If we're filtering events, pass if not in list if (events.Count > 0 && events.IndexOf(eventName) == -1) { continue; } // If no x/y, this isn't a Heatmap Event. Pass. if (!datum.ContainsKey("x") || !datum.ContainsKey("y")) { // Re-enable this log line if you want to be see events that aren't valid for heatmapping //Debug.Log ("Unable to find x/y in: " + datum.ToString () + ". Skipping..."); continue; } // Passed all checks. Consider as legal point m_ReportLegalPoints++; float x = float.Parse((string)datum["x"]); float y = float.Parse((string)datum["y"]); // z is optional float z = datum.ContainsKey("z") ? float.Parse((string)datum["z"]) : 0; // Round x = Divide(x, space); y = Divide(y, space); z = Divide(z, space); // t is optional and always 0 if we're not disaggregating float t = !datum.ContainsKey("t") || !disaggregateTime ? 0 : float.Parse((string)datum["t"]); t = Divide(t, time); // rotation values are optional and always 0 if we're not disaggragating float rx = !datum.ContainsKey("rx") || !disaggregateAngle ? 0 : float.Parse((string)datum["rx"]); rx = Divide(rx, angle); float ry = !datum.ContainsKey("ry") || !disaggregateAngle ? 0 : float.Parse((string)datum["ry"]); ry = Divide(ry, angle); float rz = !datum.ContainsKey("rz") || !disaggregateAngle ? 0 : float.Parse((string)datum["rz"]); rz = Divide(rz, angle); // Tuple-like key to determine if this point is unique, or needs to be merged with another var tuple = new Tuplish(new object[]{ eventName, x, y, z, t, rx, ry, rz }); Dictionary<string, float> point; if (m_PointDict.ContainsKey(tuple)) { // Use existing point if it exists point = m_PointDict[tuple]; point["d"] = point["d"] + 1; } else { // Create point if it doesn't exist point = new Dictionary<string, float>(); point["x"] = x; point["y"] = y; point["z"] = z; point["t"] = t; point["rx"] = rx; point["ry"] = ry; point["rz"] = rz; point["d"] = 1; m_PointDict[tuple] = point; // Create the event list if it doesn't exist if (!outputData.ContainsKey(eventName)) { outputData.Add(eventName, new List<Dictionary<string, float>>()); } // Add the new point to the list outputData[eventName].Add(point); } } } }
protected void LoadStream(Dictionary <string, List <Dictionary <string, float> > > outputData, string path, DateTime startDate, DateTime endDate, float space, float time, float angle, bool disaggregateTime, bool disaggregateAngle, List <string> events, string outputFileName) { var reader = new StreamReader(path); using (reader) { string tsv = reader.ReadToEnd(); string[] rows = tsv.Split('\n'); m_ReportRows += rows.Length; for (int a = 0; a < rows.Length; a++) { string[] rowData = rows[a].Split('\t'); if (string.IsNullOrEmpty(rowData[0]) || string.IsNullOrEmpty(rowData[2]) || string.IsNullOrEmpty(rowData[3])) { // Re-enable this log if you want to see empty lines //if (UnityEngine.UI.Windows.Constants.LOGS_ENABLED == true) UnityEngine.Debug.Log ("Empty Line...skipping"); continue; } DateTime rowDate = DateTime.Parse(rowData[0]); // Pass on rows outside any date trimming if (rowDate < startDate || rowDate > endDate) { continue; } string eventName = rowData[2]; Dictionary <string, object> datum = MiniJSON.Json.Deserialize(rowData[3]) as Dictionary <string, object>; // If we're filtering events, pass if not in list if (events.Count > 0 && events.IndexOf(eventName) == -1) { continue; } // If no x/y, this isn't a Heatmap Event. Pass. if (!datum.ContainsKey("x") || !datum.ContainsKey("y")) { // Re-enable this log line if you want to be see events that aren't valid for heatmapping //if (UnityEngine.UI.Windows.Constants.LOGS_ENABLED == true) UnityEngine.Debug.Log ("Unable to find x/y in: " + datum.ToString () + ". Skipping..."); continue; } // Passed all checks. Consider as legal point m_ReportLegalPoints++; float x = float.Parse((string)datum["x"]); float y = float.Parse((string)datum["y"]); // z is optional float z = datum.ContainsKey("z") ? float.Parse((string)datum["z"]) : 0; // Round x = Divide(x, space); y = Divide(y, space); z = Divide(z, space); // t is optional and always 0 if we're not disaggregating float t = !datum.ContainsKey("t") || !disaggregateTime ? 0 : float.Parse((string)datum["t"]); t = Divide(t, time); // rotation values are optional and always 0 if we're not disaggragating float rx = !datum.ContainsKey("rx") || !disaggregateAngle ? 0 : float.Parse((string)datum["rx"]); rx = Divide(rx, angle); float ry = !datum.ContainsKey("ry") || !disaggregateAngle ? 0 : float.Parse((string)datum["ry"]); ry = Divide(ry, angle); float rz = !datum.ContainsKey("rz") || !disaggregateAngle ? 0 : float.Parse((string)datum["rz"]); rz = Divide(rz, angle); // Tuple-like key to determine if this point is unique, or needs to be merged with another var tuple = new Tuplish(new object[] { eventName, x, y, z, t, rx, ry, rz }); Dictionary <string, float> point; if (m_PointDict.ContainsKey(tuple)) { // Use existing point if it exists point = m_PointDict[tuple]; point["d"] = point["d"] + 1; } else { // Create point if it doesn't exist point = new Dictionary <string, float>(); point["x"] = x; point["y"] = y; point["z"] = z; point["t"] = t; point["rx"] = rx; point["ry"] = ry; point["rz"] = rz; point["d"] = 1; m_PointDict[tuple] = point; // Create the event list if it doesn't exist if (!outputData.ContainsKey(eventName)) { outputData.Add(eventName, new List <Dictionary <string, float> >()); } // Add the new point to the list outputData[eventName].Add(point); } } } }
internal void LoadStream(Dictionary <Tuplish, List <HistogramHeatPoint> > histograms, Dictionary <string, int> headers, string path, DateTime startDate, DateTime endDate, List <string> aggregateOn, Dictionary <string, float> smoothOn, List <string> groupOn, string remapDensityToField) { bool doRemap = !string.IsNullOrEmpty(remapDensityToField); if (doRemap) { aggregateOn.Add(remapDensityToField); } if (!System.IO.File.Exists(path)) { Debug.LogWarningFormat("File {0} not found.", path); return; } string tsv = IonicGZip.DecompressFile(path); string[] rows = tsv.Split('\n'); m_ReportRows += rows.Length; // Define indices int nameIndex = headers["name"]; int submitTimeIndex = headers["submit_time"]; int paramsIndex = headers["custom_params"]; int userIdIndex = headers["userid"]; int sessionIdIndex = headers["sessionid"]; int platformIndex = headers["platform"]; int isDebugIndex = headers["debug_device"]; for (int a = 0; a < rows.Length; a++) { List <string> rowData = new List <string>(rows[a].Split('\t')); if (rowData.Count < 6) { // Re-enable this log if you want to see empty lines //Debug.Log ("No data in line...skipping"); continue; } string userId = rowData[userIdIndex]; string sessionId = rowData[sessionIdIndex]; string eventName = rowData[nameIndex]; string paramsData = rowData[paramsIndex]; double unixTimeStamp = double.Parse(rowData[submitTimeIndex]); DateTime rowDate = DateTimeUtils.s_Epoch.AddMilliseconds(unixTimeStamp); string platform = rowData[platformIndex]; bool isDebug = bool.Parse(rowData[isDebugIndex]); // Pass on rows outside any date trimming if (rowDate < startDate || rowDate > endDate) { continue; } Dictionary <string, object> datum = MiniJSON.Json.Deserialize(paramsData) as Dictionary <string, object>; // If no x/y, this isn't a Heatmap Event. Pass. if (!datum.ContainsKey("x") || !datum.ContainsKey("y")) { // Re-enable this log line if you want to be see events that aren't valid for heatmapping //Debug.Log ("Unable to find x/y in: " + datum.ToString () + ". Skipping..."); continue; } // Passed all checks. Consider as legal point m_ReportLegalPoints++; // Construct both the list of elements that signify a unique item... var pointTupleList = new List <object> { eventName }; // ...and a point to contain the data HistogramHeatPoint point = new HistogramHeatPoint(); foreach (var ag in aggregateOn) { float floatValue = 0f; object arbitraryValue = 0f; // Special cases for userIDs, sessionIDs, platform, and debug, which aren't in the JSON if (ag == "userID") { arbitraryValue = userId; } else if (ag == "sessionID") { arbitraryValue = sessionId; } else if (ag == "platform") { arbitraryValue = platform; } else if (ag == "debug") { arbitraryValue = isDebug; } else if (datum.ContainsKey(ag)) { // parse and divide all in smoothing list float.TryParse((string)datum[ag], out floatValue); if (smoothOn.ContainsKey(ag)) { floatValue = Divide(floatValue, smoothOn[ag]); } else { floatValue = 0; } arbitraryValue = floatValue; } pointTupleList.Add(arbitraryValue); // Add values to the point if (pointProperties.Contains(ag)) { point[ag] = floatValue; } } // Turn the pointTupleList into a key var pointTuple = new Tuplish(pointTupleList.ToArray()); float remapValue = 1f; if (doRemap && datum.ContainsKey(remapDensityToField)) { float.TryParse((string)datum[remapDensityToField], out remapValue); } if (m_PointDict.ContainsKey(pointTuple)) { // Use existing point if it exists... point = m_PointDict[pointTuple]; point.histogram.Add(remapValue); if (rowDate < point.firstDate) { point.first = remapValue; point.firstDate = rowDate; } else if (rowDate > point.lastDate) { point.last = remapValue; point.lastDate = rowDate; } } else { // ...or else use the one we've been constructing point.histogram.Add(remapValue); point.first = remapValue; point.last = remapValue; point.firstDate = rowDate; point.lastDate = rowDate; // CREATE GROUPING LIST var groupTupleList = new List <object>(); foreach (var field in groupOn) { // Special case for eventName if (field == "eventName") { groupTupleList.Add(eventName); } // Special cases for... userID else if (field == "userID") { groupTupleList.Add("user: "******"sessionID") { groupTupleList.Add("session: " + sessionId); } // ... debug ... else if (field == "debug") { groupTupleList.Add("debug: " + isDebug); } // ... platform else if (field == "platform") { groupTupleList.Add("platform: " + platform); } // Everything else just added to key else if (datum.ContainsKey(field)) { groupTupleList.Add(field + ": " + datum[field]); } } var groupTuple = new Tuplish(groupTupleList.ToArray()); // Create the event list if the key doesn't exist if (!histograms.ContainsKey(groupTuple)) { histograms.Add(groupTuple, new List <HistogramHeatPoint>()); } // FINALLY, ADD THE POINT TO THE CORRECT GROUP... histograms[groupTuple].Add(point); // ...AND THE POINT DICT m_PointDict[pointTuple] = point; } } }