int SaveFile(string data, string fileName, bool compress)
        {
            string savePath = System.IO.Path.Combine(GetSavePath(), "RawDataFolder");

            // Create the save path if necessary
            if (!System.IO.Directory.Exists(savePath))
            {
                System.IO.Directory.CreateDirectory(savePath);
            }
            string outputFileName = fileName;
            string path           = System.IO.Path.Combine(savePath, outputFileName);
            int    size           = 0;

            if (compress)
            {
                IonicGZip.CompressAndSave(path, data);
            }
            else
            {
                using (System.IO.StreamWriter file =
                           new System.IO.StreamWriter(path))
                {
                    file.Write(data);
                }
            }
            System.IO.FileInfo fileInfo = new System.IO.FileInfo(path);
            size = (int)fileInfo.Length;
            return(size);
        }
        internal Dictionary <string, int> GetHeaders()
        {
            var    retv = new Dictionary <string, int>();
            string path = System.IO.Path.Combine(m_DataPath, "RawDataFolder");

            path = System.IO.Path.Combine(path, "custom_headers.gz");
            string tsv = IonicGZip.DecompressFile(path);

            tsv = tsv.Replace("\n", "");
            List <string> rowData = new List <string>(tsv.Split('\t'));

            for (var a = 0; a < headerKeys.Length; a++)
            {
                retv.Add(headerKeys[a], rowData.IndexOf(headerKeys[a]));
            }
            return(retv);
        }
        internal void LoadStream(Dictionary <string, int> headers,
                                 List <string> inputFiles,
                                 DateTime startDate, DateTime endDate)
        {
            bool eventNamesUpdated      = false;
            bool eventDataUpdated       = false;
            bool eventSeparatorsUpdated = false;

            separatorsToEvents = new Dictionary <string, object>();
            heatpointsToEvents = new Dictionary <string, object>();

            foreach (string path in inputFiles)
            {
                if (!System.IO.File.Exists(path))
                {
                    //Debug.LogWarningFormat("File {0} not found.", path);
                    return;
                }

                string tsv = IonicGZip.DecompressFile(path);

                string[] rows = tsv.Split('\n');

                // Define indices
                int nameIndex = headers["name"];

                int submitTimeIndex = headers["submit_time"];
                int paramsIndex     = headers["custom_params"];
                int userIdIndex     = headers["userid"];
                int sessionIdIndex  = headers["sessionid"];
                int platformIndex   = headers["platform"];
                int isDebugIndex    = headers["debug_device"];

                for (int a = 0; a < rows.Length; a++)
                {
                    List <string> rowData = new List <string>(rows[a].Split('\t'));
                    if (rowData.Count < 6)
                    {
                        // Re-enable this log if you want to see empty lines
                        ////Debug.Log ("No data in line...skipping");
                        continue;
                    }



                    string   userId        = rowData[userIdIndex];
                    string   sessionId     = rowData[sessionIdIndex];
                    string   eventName     = rowData[nameIndex];
                    string   paramsData    = rowData[paramsIndex];
                    double   unixTimeStamp = double.Parse(rowData[submitTimeIndex]);
                    DateTime rowDate       = DateTimeUtils.s_Epoch.AddMilliseconds(unixTimeStamp);


                    string platform = rowData[platformIndex];

                    // Pass on rows outside any date trimming
                    if (rowDate < startDate || rowDate > endDate)
                    {
                        //Debug.Log("passing on a row");
                        continue;
                    }

                    Dictionary <string, object> datum = MiniJSON.Json.Deserialize(paramsData) as Dictionary <string, object>;
                    // If no x/y, this isn't a Heatmap Event. Pass.
                    if (datum == null || (!datum.ContainsKey("x") || !datum.ContainsKey("y")))
                    {
                        // Re-enable this log line if you want to be see events that aren't valid for heatmapping
                        //Debug.Log ("Unable to find x/y in: Skipping...");
                        continue;
                    }

                    if (TryUpdateEventNames(eventName))
                    {
                        eventNamesUpdated = true;
                    }

                    var heatmapSeparator = CreateSeparatorForRow(headers, rowData);
                    var sepDict          = (separatorsToEvents[eventName] as Dictionary <string, List <HeatmapSeparators> >);
                    if (!sepDict.ContainsKey(sessionId))
                    {
                        sepDict.Add(sessionId, new List <HeatmapSeparators>());
                        eventSeparatorsUpdated = true;
                    }

                    if (!sepDict[sessionId].Contains(heatmapSeparator))
                    {
                        sepDict[sessionId].Add(heatmapSeparator);
                        eventSeparatorsUpdated = true;
                    }

                    var heatpointDict = (heatpointsToEvents[eventName] as Dictionary <string, List <HeatPoint> >);
                    if (!heatpointDict.ContainsKey(sessionId))
                    {
                        heatpointDict.Add(sessionId, new List <HeatPoint>());
                        eventDataUpdated = true;
                    }
                    eventDataUpdated = true;

                    //make a heatpoint
                    HeatPoint heatpoint = new HeatPoint();

                    float x = 0, y = 0, z = 0, t = 0, rx = 0, ry = 0, rz = 0;
                    foreach (KeyValuePair <string, object> pointKV in datum)
                    {
                        switch (pointKV.Key)
                        {
                        case "x":
                            x = (float)Convert.ToDouble(pointKV.Value);
                            break;

                        case "y":
                            y = (float)Convert.ToDouble(pointKV.Value);
                            break;

                        case "z":
                            z = (float)Convert.ToDouble(pointKV.Value);
                            break;

                        case "t":
                            t = (float)Convert.ToDouble(pointKV.Value);
                            break;

                        case "rx":
                            rx = (float)Convert.ToDouble(pointKV.Value);
                            break;

                        case "ry":
                            ry = (float)Convert.ToDouble(pointKV.Value);
                            break;

                        case "rz":
                            rz = (float)Convert.ToDouble(pointKV.Value);
                            break;
                        }
                    }
                    heatpoint.density  = 1;
                    heatpoint.position = new Vector3(x, y, z);
                    heatpoint.rotation = new Vector3(rx, ry, rz);
                    heatpoint.time     = t;
                    if (heatpointDict[sessionId].Contains(heatpoint))
                    {
                        var indexOfHeatpoint = heatpointDict[sessionId].IndexOf(heatpoint);
                        var temp             = heatpointDict[sessionId].ElementAt(indexOfHeatpoint);
                        heatpointDict[sessionId].RemoveAt(indexOfHeatpoint);
                        temp.density += heatpoint.density;
                        heatpointDict[sessionId].Insert(indexOfHeatpoint, temp);
                        eventDataUpdated = true;
                    }
                    else
                    {
                        heatpointDict[sessionId].Add(heatpoint);
                        eventDataUpdated = true;
                    }
                }
            }

            if (eventNamesUpdated)
            {
                if (UpdateEventNames != null)
                {
                    UpdateEventNames(separatorsToEvents.Keys.ToList());
                }
            }
            if (eventSeparatorsUpdated)
            {
                if (UpdateSeparators != null)
                {
                    UpdateSeparators(separatorsToEvents);
                }
            }
            if (eventDataUpdated)
            {
                if (UpdateEventData != null)
                {
                    UpdateEventData(heatpointsToEvents);
                }
            }
        }
Пример #4
0
        internal void LoadStream(Dictionary <Tuplish, List <HistogramHeatPoint> > histograms,
                                 Dictionary <string, int> headers,
                                 string path,
                                 DateTime startDate, DateTime endDate,
                                 List <string> aggregateOn,
                                 Dictionary <string, float> smoothOn,
                                 List <string> groupOn,
                                 string remapDensityToField)
        {
            bool doRemap = !string.IsNullOrEmpty(remapDensityToField);

            if (doRemap)
            {
                aggregateOn.Add(remapDensityToField);
            }

            if (!System.IO.File.Exists(path))
            {
                Debug.LogWarningFormat("File {0} not found.", path);
                return;
            }

            string tsv = IonicGZip.DecompressFile(path);

            string[] rows = tsv.Split('\n');
            m_ReportRows += rows.Length;

            // Define indices
            int nameIndex       = headers["name"];
            int submitTimeIndex = headers["submit_time"];
            int paramsIndex     = headers["custom_params"];
            int userIdIndex     = headers["userid"];
            int sessionIdIndex  = headers["sessionid"];
            int platformIndex   = headers["platform"];
            int isDebugIndex    = headers["debug_device"];

            for (int a = 0; a < rows.Length; a++)
            {
                List <string> rowData = new List <string>(rows[a].Split('\t'));
                if (rowData.Count < 6)
                {
                    // Re-enable this log if you want to see empty lines
                    //Debug.Log ("No data in line...skipping");
                    continue;
                }

                string   userId        = rowData[userIdIndex];
                string   sessionId     = rowData[sessionIdIndex];
                string   eventName     = rowData[nameIndex];
                string   paramsData    = rowData[paramsIndex];
                double   unixTimeStamp = double.Parse(rowData[submitTimeIndex]);
                DateTime rowDate       = DateTimeUtils.s_Epoch.AddMilliseconds(unixTimeStamp);


                string platform = rowData[platformIndex];
                bool   isDebug  = bool.Parse(rowData[isDebugIndex]);

                // Pass on rows outside any date trimming
                if (rowDate < startDate || rowDate > endDate)
                {
                    continue;
                }

                Dictionary <string, object> datum = MiniJSON.Json.Deserialize(paramsData) as Dictionary <string, object>;
                // If no x/y, this isn't a Heatmap Event. Pass.
                if (!datum.ContainsKey("x") || !datum.ContainsKey("y"))
                {
                    // Re-enable this log line if you want to be see events that aren't valid for heatmapping
                    //Debug.Log ("Unable to find x/y in: " + datum.ToString () + ". Skipping...");
                    continue;
                }

                // Passed all checks. Consider as legal point
                m_ReportLegalPoints++;

                // Construct both the list of elements that signify a unique item...
                var pointTupleList = new List <object> {
                    eventName
                };
                // ...and a point to contain the data
                HistogramHeatPoint point = new HistogramHeatPoint();
                foreach (var ag in aggregateOn)
                {
                    float  floatValue     = 0f;
                    object arbitraryValue = 0f;
                    // Special cases for userIDs, sessionIDs, platform, and debug, which aren't in the JSON
                    if (ag == "userID")
                    {
                        arbitraryValue = userId;
                    }
                    else if (ag == "sessionID")
                    {
                        arbitraryValue = sessionId;
                    }
                    else if (ag == "platform")
                    {
                        arbitraryValue = platform;
                    }
                    else if (ag == "debug")
                    {
                        arbitraryValue = isDebug;
                    }
                    else if (datum.ContainsKey(ag))
                    {
                        // parse and divide all in smoothing list
                        float.TryParse((string)datum[ag], out floatValue);
                        if (smoothOn.ContainsKey(ag))
                        {
                            floatValue = Divide(floatValue, smoothOn[ag]);
                        }
                        else
                        {
                            floatValue = 0;
                        }
                        arbitraryValue = floatValue;
                    }

                    pointTupleList.Add(arbitraryValue);
                    // Add values to the point
                    if (pointProperties.Contains(ag))
                    {
                        point[ag] = floatValue;
                    }
                }
                // Turn the pointTupleList into a key
                var pointTuple = new Tuplish(pointTupleList.ToArray());

                float remapValue = 1f;
                if (doRemap && datum.ContainsKey(remapDensityToField))
                {
                    float.TryParse((string)datum[remapDensityToField], out remapValue);
                }

                if (m_PointDict.ContainsKey(pointTuple))
                {
                    // Use existing point if it exists...
                    point = m_PointDict[pointTuple];
                    point.histogram.Add(remapValue);

                    if (rowDate < point.firstDate)
                    {
                        point.first     = remapValue;
                        point.firstDate = rowDate;
                    }
                    else if (rowDate > point.lastDate)
                    {
                        point.last     = remapValue;
                        point.lastDate = rowDate;
                    }
                }
                else
                {
                    // ...or else use the one we've been constructing
                    point.histogram.Add(remapValue);
                    point.first     = remapValue;
                    point.last      = remapValue;
                    point.firstDate = rowDate;
                    point.lastDate  = rowDate;

                    // CREATE GROUPING LIST
                    var groupTupleList = new List <object>();
                    foreach (var field in groupOn)
                    {
                        // Special case for eventName
                        if (field == "eventName")
                        {
                            groupTupleList.Add(eventName);
                        }
                        // Special cases for... userID
                        else if (field == "userID")
                        {
                            groupTupleList.Add("user: "******"sessionID")
                        {
                            groupTupleList.Add("session: " + sessionId);
                        }
                        // ... debug ...
                        else if (field == "debug")
                        {
                            groupTupleList.Add("debug: " + isDebug);
                        }
                        // ... platform
                        else if (field == "platform")
                        {
                            groupTupleList.Add("platform: " + platform);
                        }
                        // Everything else just added to key
                        else if (datum.ContainsKey(field))
                        {
                            groupTupleList.Add(field + ": " + datum[field]);
                        }
                    }
                    var groupTuple = new Tuplish(groupTupleList.ToArray());

                    // Create the event list if the key doesn't exist
                    if (!histograms.ContainsKey(groupTuple))
                    {
                        histograms.Add(groupTuple, new List <HistogramHeatPoint>());
                    }

                    // FINALLY, ADD THE POINT TO THE CORRECT GROUP...
                    histograms[groupTuple].Add(point);
                    // ...AND THE POINT DICT
                    m_PointDict[pointTuple] = point;
                }
            }
        }