int SaveFile(string data, string fileName, bool compress)
{
string savePath = System.IO.Path.Combine(GetSavePath(), "RawDataFolder");
// Create the save path if necessary
if (!System.IO.Directory.Exists(savePath))
{
System.IO.Directory.CreateDirectory(savePath);
}
string outputFileName = fileName;
string path = System.IO.Path.Combine(savePath, outputFileName);
int size = 0;
if (compress)
{
IonicGZip.CompressAndSave(path, data);
}
else
{
using (System.IO.StreamWriter file =
new System.IO.StreamWriter(path))
{
file.Write(data);
}
}
System.IO.FileInfo fileInfo = new System.IO.FileInfo(path);
size = (int)fileInfo.Length;
return(size);
}
internal Dictionary <string, int> GetHeaders()
{
var retv = new Dictionary <string, int>();
string path = System.IO.Path.Combine(m_DataPath, "RawDataFolder");
path = System.IO.Path.Combine(path, "custom_headers.gz");
string tsv = IonicGZip.DecompressFile(path);
tsv = tsv.Replace("\n", "");
List <string> rowData = new List <string>(tsv.Split('\t'));
for (var a = 0; a < headerKeys.Length; a++)
{
retv.Add(headerKeys[a], rowData.IndexOf(headerKeys[a]));
}
return(retv);
}
internal void LoadStream(Dictionary <string, int> headers,
List <string> inputFiles,
DateTime startDate, DateTime endDate)
{
bool eventNamesUpdated = false;
bool eventDataUpdated = false;
bool eventSeparatorsUpdated = false;
separatorsToEvents = new Dictionary <string, object>();
heatpointsToEvents = new Dictionary <string, object>();
foreach (string path in inputFiles)
{
if (!System.IO.File.Exists(path))
{
//Debug.LogWarningFormat("File {0} not found.", path);
return;
}
string tsv = IonicGZip.DecompressFile(path);
string[] rows = tsv.Split('\n');
// Define indices
int nameIndex = headers["name"];
int submitTimeIndex = headers["submit_time"];
int paramsIndex = headers["custom_params"];
int userIdIndex = headers["userid"];
int sessionIdIndex = headers["sessionid"];
int platformIndex = headers["platform"];
int isDebugIndex = headers["debug_device"];
for (int a = 0; a < rows.Length; a++)
{
List <string> rowData = new List <string>(rows[a].Split('\t'));
if (rowData.Count < 6)
{
// Re-enable this log if you want to see empty lines
////Debug.Log ("No data in line...skipping");
continue;
}
string userId = rowData[userIdIndex];
string sessionId = rowData[sessionIdIndex];
string eventName = rowData[nameIndex];
string paramsData = rowData[paramsIndex];
double unixTimeStamp = double.Parse(rowData[submitTimeIndex]);
DateTime rowDate = DateTimeUtils.s_Epoch.AddMilliseconds(unixTimeStamp);
string platform = rowData[platformIndex];
// Pass on rows outside any date trimming
if (rowDate < startDate || rowDate > endDate)
{
//Debug.Log("passing on a row");
continue;
}
Dictionary <string, object> datum = MiniJSON.Json.Deserialize(paramsData) as Dictionary <string, object>;
// If no x/y, this isn't a Heatmap Event. Pass.
if (datum == null || (!datum.ContainsKey("x") || !datum.ContainsKey("y")))
{
// Re-enable this log line if you want to be see events that aren't valid for heatmapping
//Debug.Log ("Unable to find x/y in: Skipping...");
continue;
}
if (TryUpdateEventNames(eventName))
{
eventNamesUpdated = true;
}
var heatmapSeparator = CreateSeparatorForRow(headers, rowData);
var sepDict = (separatorsToEvents[eventName] as Dictionary <string, List <HeatmapSeparators> >);
if (!sepDict.ContainsKey(sessionId))
{
sepDict.Add(sessionId, new List <HeatmapSeparators>());
eventSeparatorsUpdated = true;
}
if (!sepDict[sessionId].Contains(heatmapSeparator))
{
sepDict[sessionId].Add(heatmapSeparator);
eventSeparatorsUpdated = true;
}
var heatpointDict = (heatpointsToEvents[eventName] as Dictionary <string, List <HeatPoint> >);
if (!heatpointDict.ContainsKey(sessionId))
{
heatpointDict.Add(sessionId, new List <HeatPoint>());
eventDataUpdated = true;
}
eventDataUpdated = true;
//make a heatpoint
HeatPoint heatpoint = new HeatPoint();
float x = 0, y = 0, z = 0, t = 0, rx = 0, ry = 0, rz = 0;
foreach (KeyValuePair <string, object> pointKV in datum)
{
switch (pointKV.Key)
{
case "x":
x = (float)Convert.ToDouble(pointKV.Value);
break;
case "y":
y = (float)Convert.ToDouble(pointKV.Value);
break;
case "z":
z = (float)Convert.ToDouble(pointKV.Value);
break;
case "t":
t = (float)Convert.ToDouble(pointKV.Value);
break;
case "rx":
rx = (float)Convert.ToDouble(pointKV.Value);
break;
case "ry":
ry = (float)Convert.ToDouble(pointKV.Value);
break;
case "rz":
rz = (float)Convert.ToDouble(pointKV.Value);
break;
}
}
heatpoint.density = 1;
heatpoint.position = new Vector3(x, y, z);
heatpoint.rotation = new Vector3(rx, ry, rz);
heatpoint.time = t;
if (heatpointDict[sessionId].Contains(heatpoint))
{
var indexOfHeatpoint = heatpointDict[sessionId].IndexOf(heatpoint);
var temp = heatpointDict[sessionId].ElementAt(indexOfHeatpoint);
heatpointDict[sessionId].RemoveAt(indexOfHeatpoint);
temp.density += heatpoint.density;
heatpointDict[sessionId].Insert(indexOfHeatpoint, temp);
eventDataUpdated = true;
}
else
{
heatpointDict[sessionId].Add(heatpoint);
eventDataUpdated = true;
}
}
}
if (eventNamesUpdated)
{
if (UpdateEventNames != null)
{
UpdateEventNames(separatorsToEvents.Keys.ToList());
}
}
if (eventSeparatorsUpdated)
{
if (UpdateSeparators != null)
{
UpdateSeparators(separatorsToEvents);
}
}
if (eventDataUpdated)
{
if (UpdateEventData != null)
{
UpdateEventData(heatpointsToEvents);
}
}
}
internal void LoadStream(Dictionary <Tuplish, List <HistogramHeatPoint> > histograms,
Dictionary <string, int> headers,
string path,
DateTime startDate, DateTime endDate,
List <string> aggregateOn,
Dictionary <string, float> smoothOn,
List <string> groupOn,
string remapDensityToField)
{
bool doRemap = !string.IsNullOrEmpty(remapDensityToField);
if (doRemap)
{
aggregateOn.Add(remapDensityToField);
}
if (!System.IO.File.Exists(path))
{
Debug.LogWarningFormat("File {0} not found.", path);
return;
}
string tsv = IonicGZip.DecompressFile(path);
string[] rows = tsv.Split('\n');
m_ReportRows += rows.Length;
// Define indices
int nameIndex = headers["name"];
int submitTimeIndex = headers["submit_time"];
int paramsIndex = headers["custom_params"];
int userIdIndex = headers["userid"];
int sessionIdIndex = headers["sessionid"];
int platformIndex = headers["platform"];
int isDebugIndex = headers["debug_device"];
for (int a = 0; a < rows.Length; a++)
{
List <string> rowData = new List <string>(rows[a].Split('\t'));
if (rowData.Count < 6)
{
// Re-enable this log if you want to see empty lines
//Debug.Log ("No data in line...skipping");
continue;
}
string userId = rowData[userIdIndex];
string sessionId = rowData[sessionIdIndex];
string eventName = rowData[nameIndex];
string paramsData = rowData[paramsIndex];
double unixTimeStamp = double.Parse(rowData[submitTimeIndex]);
DateTime rowDate = DateTimeUtils.s_Epoch.AddMilliseconds(unixTimeStamp);
string platform = rowData[platformIndex];
bool isDebug = bool.Parse(rowData[isDebugIndex]);
// Pass on rows outside any date trimming
if (rowDate < startDate || rowDate > endDate)
{
continue;
}
Dictionary <string, object> datum = MiniJSON.Json.Deserialize(paramsData) as Dictionary <string, object>;
// If no x/y, this isn't a Heatmap Event. Pass.
if (!datum.ContainsKey("x") || !datum.ContainsKey("y"))
{
// Re-enable this log line if you want to be see events that aren't valid for heatmapping
//Debug.Log ("Unable to find x/y in: " + datum.ToString () + ". Skipping...");
continue;
}
// Passed all checks. Consider as legal point
m_ReportLegalPoints++;
// Construct both the list of elements that signify a unique item...
var pointTupleList = new List <object> {
eventName
};
// ...and a point to contain the data
HistogramHeatPoint point = new HistogramHeatPoint();
foreach (var ag in aggregateOn)
{
float floatValue = 0f;
object arbitraryValue = 0f;
// Special cases for userIDs, sessionIDs, platform, and debug, which aren't in the JSON
if (ag == "userID")
{
arbitraryValue = userId;
}
else if (ag == "sessionID")
{
arbitraryValue = sessionId;
}
else if (ag == "platform")
{
arbitraryValue = platform;
}
else if (ag == "debug")
{
arbitraryValue = isDebug;
}
else if (datum.ContainsKey(ag))
{
// parse and divide all in smoothing list
float.TryParse((string)datum[ag], out floatValue);
if (smoothOn.ContainsKey(ag))
{
floatValue = Divide(floatValue, smoothOn[ag]);
}
else
{
floatValue = 0;
}
arbitraryValue = floatValue;
}
pointTupleList.Add(arbitraryValue);
// Add values to the point
if (pointProperties.Contains(ag))
{
point[ag] = floatValue;
}
}
// Turn the pointTupleList into a key
var pointTuple = new Tuplish(pointTupleList.ToArray());
float remapValue = 1f;
if (doRemap && datum.ContainsKey(remapDensityToField))
{
float.TryParse((string)datum[remapDensityToField], out remapValue);
}
if (m_PointDict.ContainsKey(pointTuple))
{
// Use existing point if it exists...
point = m_PointDict[pointTuple];
point.histogram.Add(remapValue);
if (rowDate < point.firstDate)
{
point.first = remapValue;
point.firstDate = rowDate;
}
else if (rowDate > point.lastDate)
{
point.last = remapValue;
point.lastDate = rowDate;
}
}
else
{
// ...or else use the one we've been constructing
point.histogram.Add(remapValue);
point.first = remapValue;
point.last = remapValue;
point.firstDate = rowDate;
point.lastDate = rowDate;
// CREATE GROUPING LIST
var groupTupleList = new List <object>();
foreach (var field in groupOn)
{
// Special case for eventName
if (field == "eventName")
{
groupTupleList.Add(eventName);
}
// Special cases for... userID
else if (field == "userID")
{
groupTupleList.Add("user: "******"sessionID")
{
groupTupleList.Add("session: " + sessionId);
}
// ... debug ...
else if (field == "debug")
{
groupTupleList.Add("debug: " + isDebug);
}
// ... platform
else if (field == "platform")
{
groupTupleList.Add("platform: " + platform);
}
// Everything else just added to key
else if (datum.ContainsKey(field))
{
groupTupleList.Add(field + ": " + datum[field]);
}
}
var groupTuple = new Tuplish(groupTupleList.ToArray());
// Create the event list if the key doesn't exist
if (!histograms.ContainsKey(groupTuple))
{
histograms.Add(groupTuple, new List <HistogramHeatPoint>());
}
// FINALLY, ADD THE POINT TO THE CORRECT GROUP...
histograms[groupTuple].Add(point);
// ...AND THE POINT DICT
m_PointDict[pointTuple] = point;
}
}
}
