protected void LoadStream(Dictionary<string, List<Dictionary<string, float>>> outputData,
string path,
DateTime startDate, DateTime endDate,
float space, float time, float angle,
bool disaggregateTime, bool disaggregateAngle,
List<string> events, string outputFileName)
{
var reader = new StreamReader(path);
using (reader)
{
string tsv = reader.ReadToEnd();
string[] rows = tsv.Split('\n');
m_ReportRows += rows.Length;
for (int a = 0; a < rows.Length; a++)
{
string[] rowData = rows[a].Split('\t');
if (string.IsNullOrEmpty(rowData[0]) || string.IsNullOrEmpty(rowData[2]) || string.IsNullOrEmpty(rowData[3]))
{
// Re-enable this log if you want to see empty lines
//Debug.Log ("Empty Line...skipping");
continue;
}
DateTime rowDate = DateTime.Parse(rowData[0]);
// Pass on rows outside any date trimming
if (rowDate < startDate || rowDate > endDate)
{
continue;
}
string eventName = rowData[2];
Dictionary<string, object> datum = MiniJSON.Json.Deserialize(rowData[3]) as Dictionary<string, object>;
// If we're filtering events, pass if not in list
if (events.Count > 0 && events.IndexOf(eventName) == -1)
{
continue;
}
// If no x/y, this isn't a Heatmap Event. Pass.
if (!datum.ContainsKey("x") || !datum.ContainsKey("y"))
{
// Re-enable this log line if you want to be see events that aren't valid for heatmapping
//Debug.Log ("Unable to find x/y in: " + datum.ToString () + ". Skipping...");
continue;
}
// Passed all checks. Consider as legal point
m_ReportLegalPoints++;
float x = float.Parse((string)datum["x"]);
float y = float.Parse((string)datum["y"]);
// z is optional
float z = datum.ContainsKey("z") ? float.Parse((string)datum["z"]) : 0;
// Round
x = Divide(x, space);
y = Divide(y, space);
z = Divide(z, space);
// t is optional and always 0 if we're not disaggregating
float t = !datum.ContainsKey("t") || !disaggregateTime ? 0 : float.Parse((string)datum["t"]);
t = Divide(t, time);
// rotation values are optional and always 0 if we're not disaggragating
float rx = !datum.ContainsKey("rx") || !disaggregateAngle ? 0 : float.Parse((string)datum["rx"]);
rx = Divide(rx, angle);
float ry = !datum.ContainsKey("ry") || !disaggregateAngle ? 0 : float.Parse((string)datum["ry"]);
ry = Divide(ry, angle);
float rz = !datum.ContainsKey("rz") || !disaggregateAngle ? 0 : float.Parse((string)datum["rz"]);
rz = Divide(rz, angle);
// Tuple-like key to determine if this point is unique, or needs to be merged with another
var tuple = new Tuplish(new object[]{ eventName, x, y, z, t, rx, ry, rz });
Dictionary<string, float> point;
if (m_PointDict.ContainsKey(tuple))
{
// Use existing point if it exists
point = m_PointDict[tuple];
point["d"] = point["d"] + 1;
}
else
{
// Create point if it doesn't exist
point = new Dictionary<string, float>();
point["x"] = x;
point["y"] = y;
point["z"] = z;
point["t"] = t;
point["rx"] = rx;
point["ry"] = ry;
point["rz"] = rz;
point["d"] = 1;
m_PointDict[tuple] = point;
// Create the event list if it doesn't exist
if (!outputData.ContainsKey(eventName))
{
outputData.Add(eventName, new List<Dictionary<string, float>>());
}
// Add the new point to the list
outputData[eventName].Add(point);
}
}
}
}
protected void LoadStream(Dictionary <string, List <Dictionary <string, float> > > outputData,
string path,
DateTime startDate, DateTime endDate,
float space, float time, float angle,
bool disaggregateTime, bool disaggregateAngle,
List <string> events, string outputFileName)
{
var reader = new StreamReader(path);
using (reader)
{
string tsv = reader.ReadToEnd();
string[] rows = tsv.Split('\n');
m_ReportRows += rows.Length;
for (int a = 0; a < rows.Length; a++)
{
string[] rowData = rows[a].Split('\t');
if (string.IsNullOrEmpty(rowData[0]) || string.IsNullOrEmpty(rowData[2]) || string.IsNullOrEmpty(rowData[3]))
{
// Re-enable this log if you want to see empty lines
//if (UnityEngine.UI.Windows.Constants.LOGS_ENABLED == true) UnityEngine.Debug.Log ("Empty Line...skipping");
continue;
}
DateTime rowDate = DateTime.Parse(rowData[0]);
// Pass on rows outside any date trimming
if (rowDate < startDate || rowDate > endDate)
{
continue;
}
string eventName = rowData[2];
Dictionary <string, object> datum = MiniJSON.Json.Deserialize(rowData[3]) as Dictionary <string, object>;
// If we're filtering events, pass if not in list
if (events.Count > 0 && events.IndexOf(eventName) == -1)
{
continue;
}
// If no x/y, this isn't a Heatmap Event. Pass.
if (!datum.ContainsKey("x") || !datum.ContainsKey("y"))
{
// Re-enable this log line if you want to be see events that aren't valid for heatmapping
//if (UnityEngine.UI.Windows.Constants.LOGS_ENABLED == true) UnityEngine.Debug.Log ("Unable to find x/y in: " + datum.ToString () + ". Skipping...");
continue;
}
// Passed all checks. Consider as legal point
m_ReportLegalPoints++;
float x = float.Parse((string)datum["x"]);
float y = float.Parse((string)datum["y"]);
// z is optional
float z = datum.ContainsKey("z") ? float.Parse((string)datum["z"]) : 0;
// Round
x = Divide(x, space);
y = Divide(y, space);
z = Divide(z, space);
// t is optional and always 0 if we're not disaggregating
float t = !datum.ContainsKey("t") || !disaggregateTime ? 0 : float.Parse((string)datum["t"]);
t = Divide(t, time);
// rotation values are optional and always 0 if we're not disaggragating
float rx = !datum.ContainsKey("rx") || !disaggregateAngle ? 0 : float.Parse((string)datum["rx"]);
rx = Divide(rx, angle);
float ry = !datum.ContainsKey("ry") || !disaggregateAngle ? 0 : float.Parse((string)datum["ry"]);
ry = Divide(ry, angle);
float rz = !datum.ContainsKey("rz") || !disaggregateAngle ? 0 : float.Parse((string)datum["rz"]);
rz = Divide(rz, angle);
// Tuple-like key to determine if this point is unique, or needs to be merged with another
var tuple = new Tuplish(new object[] { eventName, x, y, z, t, rx, ry, rz });
Dictionary <string, float> point;
if (m_PointDict.ContainsKey(tuple))
{
// Use existing point if it exists
point = m_PointDict[tuple];
point["d"] = point["d"] + 1;
}
else
{
// Create point if it doesn't exist
point = new Dictionary <string, float>();
point["x"] = x;
point["y"] = y;
point["z"] = z;
point["t"] = t;
point["rx"] = rx;
point["ry"] = ry;
point["rz"] = rz;
point["d"] = 1;
m_PointDict[tuple] = point;
// Create the event list if it doesn't exist
if (!outputData.ContainsKey(eventName))
{
outputData.Add(eventName, new List <Dictionary <string, float> >());
}
// Add the new point to the list
outputData[eventName].Add(point);
}
}
}
}
internal void LoadStream(Dictionary <Tuplish, List <HistogramHeatPoint> > histograms,
Dictionary <string, int> headers,
string path,
DateTime startDate, DateTime endDate,
List <string> aggregateOn,
Dictionary <string, float> smoothOn,
List <string> groupOn,
string remapDensityToField)
{
bool doRemap = !string.IsNullOrEmpty(remapDensityToField);
if (doRemap)
{
aggregateOn.Add(remapDensityToField);
}
if (!System.IO.File.Exists(path))
{
Debug.LogWarningFormat("File {0} not found.", path);
return;
}
string tsv = IonicGZip.DecompressFile(path);
string[] rows = tsv.Split('\n');
m_ReportRows += rows.Length;
// Define indices
int nameIndex = headers["name"];
int submitTimeIndex = headers["submit_time"];
int paramsIndex = headers["custom_params"];
int userIdIndex = headers["userid"];
int sessionIdIndex = headers["sessionid"];
int platformIndex = headers["platform"];
int isDebugIndex = headers["debug_device"];
for (int a = 0; a < rows.Length; a++)
{
List <string> rowData = new List <string>(rows[a].Split('\t'));
if (rowData.Count < 6)
{
// Re-enable this log if you want to see empty lines
//Debug.Log ("No data in line...skipping");
continue;
}
string userId = rowData[userIdIndex];
string sessionId = rowData[sessionIdIndex];
string eventName = rowData[nameIndex];
string paramsData = rowData[paramsIndex];
double unixTimeStamp = double.Parse(rowData[submitTimeIndex]);
DateTime rowDate = DateTimeUtils.s_Epoch.AddMilliseconds(unixTimeStamp);
string platform = rowData[platformIndex];
bool isDebug = bool.Parse(rowData[isDebugIndex]);
// Pass on rows outside any date trimming
if (rowDate < startDate || rowDate > endDate)
{
continue;
}
Dictionary <string, object> datum = MiniJSON.Json.Deserialize(paramsData) as Dictionary <string, object>;
// If no x/y, this isn't a Heatmap Event. Pass.
if (!datum.ContainsKey("x") || !datum.ContainsKey("y"))
{
// Re-enable this log line if you want to be see events that aren't valid for heatmapping
//Debug.Log ("Unable to find x/y in: " + datum.ToString () + ". Skipping...");
continue;
}
// Passed all checks. Consider as legal point
m_ReportLegalPoints++;
// Construct both the list of elements that signify a unique item...
var pointTupleList = new List <object> {
eventName
};
// ...and a point to contain the data
HistogramHeatPoint point = new HistogramHeatPoint();
foreach (var ag in aggregateOn)
{
float floatValue = 0f;
object arbitraryValue = 0f;
// Special cases for userIDs, sessionIDs, platform, and debug, which aren't in the JSON
if (ag == "userID")
{
arbitraryValue = userId;
}
else if (ag == "sessionID")
{
arbitraryValue = sessionId;
}
else if (ag == "platform")
{
arbitraryValue = platform;
}
else if (ag == "debug")
{
arbitraryValue = isDebug;
}
else if (datum.ContainsKey(ag))
{
// parse and divide all in smoothing list
float.TryParse((string)datum[ag], out floatValue);
if (smoothOn.ContainsKey(ag))
{
floatValue = Divide(floatValue, smoothOn[ag]);
}
else
{
floatValue = 0;
}
arbitraryValue = floatValue;
}
pointTupleList.Add(arbitraryValue);
// Add values to the point
if (pointProperties.Contains(ag))
{
point[ag] = floatValue;
}
}
// Turn the pointTupleList into a key
var pointTuple = new Tuplish(pointTupleList.ToArray());
float remapValue = 1f;
if (doRemap && datum.ContainsKey(remapDensityToField))
{
float.TryParse((string)datum[remapDensityToField], out remapValue);
}
if (m_PointDict.ContainsKey(pointTuple))
{
// Use existing point if it exists...
point = m_PointDict[pointTuple];
point.histogram.Add(remapValue);
if (rowDate < point.firstDate)
{
point.first = remapValue;
point.firstDate = rowDate;
}
else if (rowDate > point.lastDate)
{
point.last = remapValue;
point.lastDate = rowDate;
}
}
else
{
// ...or else use the one we've been constructing
point.histogram.Add(remapValue);
point.first = remapValue;
point.last = remapValue;
point.firstDate = rowDate;
point.lastDate = rowDate;
// CREATE GROUPING LIST
var groupTupleList = new List <object>();
foreach (var field in groupOn)
{
// Special case for eventName
if (field == "eventName")
{
groupTupleList.Add(eventName);
}
// Special cases for... userID
else if (field == "userID")
{
groupTupleList.Add("user: "******"sessionID")
{
groupTupleList.Add("session: " + sessionId);
}
// ... debug ...
else if (field == "debug")
{
groupTupleList.Add("debug: " + isDebug);
}
// ... platform
else if (field == "platform")
{
groupTupleList.Add("platform: " + platform);
}
// Everything else just added to key
else if (datum.ContainsKey(field))
{
groupTupleList.Add(field + ": " + datum[field]);
}
}
var groupTuple = new Tuplish(groupTupleList.ToArray());
// Create the event list if the key doesn't exist
if (!histograms.ContainsKey(groupTuple))
{
histograms.Add(groupTuple, new List <HistogramHeatPoint>());
}
// FINALLY, ADD THE POINT TO THE CORRECT GROUP...
histograms[groupTuple].Add(point);
// ...AND THE POINT DICT
m_PointDict[pointTuple] = point;
}
}
}
