//private List<Closure> downloadCurrentINCIXMLFromWeb(double pollingOrRequestFrequency)//download current INCI XML data from NextBus with processing for INCITime
        //{
        //    List<Closure> downloadedINCIXML = new List<Closure>();
        //    Uri uriToDownload = new Uri("http://www1.toronto.ca/transportation/roadrestrictions/RoadRestrictions.xml");

        //    repeatWebDownload:
        //    try
        //    {
        //        using (WebClient client = new WebClient())
        //        {
        //            //download string and write to file
        //            string xml = client.DownloadString(uriToDownload);
        //            string xmlString = Convert.ToString(xml);
        //            using (StringReader reader = new StringReader(xmlString))
        //            {
        //                XmlSerializer serializer = new XmlSerializer(typeof(Closures));
        //                string headerString = reader.ReadLine();
        //                string currentVehString = reader.ReadToEnd();
        //                MemoryStream memStream = new MemoryStream(Encoding.UTF8.GetBytes(currentVehString));
        //                Closures currentXMLINCIPoints = (Closures)serializer.Deserialize(memStream);
        //                downloadedINCIXML.AddRange(currentXMLINCIPoints.Closure);
        //            }//end using reader
        //        }//end using cilent
        //    }
        //    catch (Exception e)
        //    {
        //        //sleep to aviod too many requests
        //        Thread.Sleep(Convert.ToInt32(1000 * pollingOrRequestFrequency / 2));//check delay at "half of" polling frequency (minute), until it is time for download
        //        goto repeatWebDownload;
        //    }

        //    return downloadedINCIXML;
        //}

        /// <summary>
        ///
        /// </summary>
        /// <param name="RouteTags"></param>
        /// <param name="startTime"></param>
        /// <param name="endTime"></param>
        /// <returns></returns>
        public List <SSIncidentDataTable> RetriveINCIData(DateTime startTime, DateTime endTime, List <string> RouteTags = null)
        {
            List <SSIncidentDataTable> finalPackageOfINCIDataAllRoutes = new List <SSIncidentDataTable>();
            List <SSIncidentDataTable> allDataInTheINCIDataFile        = new List <SSIncidentDataTable>();

            //For incident and weather data, saves are daily
            startTime = startTime.Date;                                                                           //beginning of first day
            endTime   = endTime.Date.AddDays(1).AddSeconds(-1);                                                   //end of second day

            DateTime lastIntermediateTime = startTime;                                                            //intermediate time 59:59 (min:sec) from start
            DateTime intermediateTime     = startTime.Date.AddHours(startTime.Hour).AddSeconds(fileSaveFreq - 1); //intermediate time 59:59 (min:sec) from start

            while (intermediateTime.DateTimeToEpochTime() <= endTime.DateTimeToEpochTime())
            {
                string currentfilename = string.Format(xmlINCIFilenameFormat, lastIntermediateTime.ToUniversalTime().DateTimeNamingFormat(), intermediateTime.ToUniversalTime().DateTimeNamingFormat());
                allDataInTheINCIDataFile.AddRange(convertRawINCIToSSFormat(SSUtil.DeSerializeXMLObject <List <Closure> >(xmlINCIFolderPath + currentfilename)));
                //increment to read next file
                foreach (SSIncidentDataTable aINCIPoint in allDataInTheINCIDataFile)
                {
                    //identify any filtering needed - none to start with
                    finalPackageOfINCIDataAllRoutes.Add(aINCIPoint);
                }
                lastIntermediateTime = lastIntermediateTime.AddSeconds(fileSaveFreq);
                intermediateTime     = intermediateTime.AddSeconds(fileSaveFreq);
            }
            allDataInTheINCIDataFile.Clear();        //clear Temp list
            return(finalPackageOfINCIDataAllRoutes); //return result list
        }
Example #2
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="RouteTags"></param>
        /// <param name="startTime"></param>
        /// <param name="endTime"></param>
        /// <returns></returns>
        public List <SSVehGPSDataTable> RetriveGPSData(List <string> RouteTags, DateTime startTime, DateTime endTime)
        {
            List <SSVehGPSDataTable> finalPackageOfGPSDataAllRoutes = new List <SSVehGPSDataTable>();
            //List<SSVehGPSDataTable> allDataInTheGPSDataFile = new List<SSVehGPSDataTable>();
            Dictionary <string, List <SSVehGPSDataTable> > allDataInGPSDataFile = new Dictionary <string, List <SSVehGPSDataTable> >();
            List <string> gpsDataFilenames = new List <string>();                                                 //in order

            DateTime lastIntermediateTime = startTime;                                                            //intermediate time 59:59 (min:sec) from start
            DateTime intermediateTime     = startTime.Date.AddHours(startTime.Hour).AddSeconds(fileSaveFreq - 1); //intermediate time 59:59 (min:sec) from start

            //initiate filenames
            while (intermediateTime.DateTimeToEpochTime() <= endTime.DateTimeToEpochTime())
            {
                string currentfilename = string.Format(xmlGPSFilenameFormat, lastIntermediateTime.ToUniversalTime().DateTimeNamingFormat(), intermediateTime.ToUniversalTime().DateTimeNamingFormat());
                gpsDataFilenames.Add(currentfilename);
                //increment to read next file
                lastIntermediateTime = lastIntermediateTime.AddSeconds(fileSaveFreq);
                intermediateTime     = intermediateTime.AddSeconds(fileSaveFreq);
            }

            //load all files in parallel
            ParallelOptions pOptions = new ParallelOptions();

            pOptions.MaxDegreeOfParallelism = Environment.ProcessorCount;
            Parallel.For(0, gpsDataFilenames.Count, pOptions, i =>
            {
                string currentfilename = gpsDataFilenames[i];
                allDataInGPSDataFile.Add(currentfilename, (convertRawGPSToSSFormat(SSUtil.DeSerializeXMLObject <List <Vehicle> >(xmlGPSFolderPath + currentfilename))));
            });
            //add each file, then each gps point, in order
            foreach (string currentfilename in gpsDataFilenames)
            {
                //filter by routeTag
                foreach (SSVehGPSDataTable aGPSPoint in allDataInGPSDataFile[currentfilename])
                {
                    string aGPSPointRouteTag = aGPSPoint.TripCode == "NULL" ? "" : aGPSPoint.TripCode.Split('_')[0];
                    if (RouteTags.Contains(aGPSPointRouteTag))
                    {
                        finalPackageOfGPSDataAllRoutes.Add(aGPSPoint);
                    }
                }
                //List<SSVehGPSDataTable> filteredData = allDataInGPSDataFile[currentfilename].Where(v => RouteTags.Contains(v.TripCode.Split('_')[0])).ToList();
                //finalPackageOfGPSDataAllRoutes.AddRange(filteredData);
            }

            allDataInGPSDataFile.Clear();
            gpsDataFilenames.Clear();
            return(finalPackageOfGPSDataAllRoutes);//return result list
        }
        public int startLiveDownloadTask(DateTime pollingStart, DateTime pollingEnd, double pollingFrequency)
        {
            //fileSaveFreq must be 1 day, otherwise, needs to modify nextSaveTime
            int downloadState = 0;

            //NOTE: polling freq: 60s < bk freq: 120s < save freq: 3600s * 24 = 1 day
            downloadedINCIXMLData_UNSAVED = new ConcurrentDictionary <string, Closure>(); //INCITime & VehID as tuple index
            DateTime nextDownloadTime;                                                    // = DateTime.Now.ToLocalTime().AddSeconds(pollingFrequency);
            DateTime nextBackupTime;                                                      // = DateTime.Now.ToLocalTime().AddSeconds(bkSaveFreq);
            //DateTime nextSaveTime = DateTime.Now.ToLocalTime().AddSeconds(fileSaveFreq);
            DateTime nextSaveTime;                                                        // = pollingStart.Date.AddSeconds(pollingStart.Hour * 3600 - 1 + fileSaveFreq);//save on the last second of the next hour
            long     lastLogUpdatedSave = 0;
            string   bkfilename         = "current-RRXML-bk.xml";

            //load last saved backup, if it is from within the last 5 minutes
            if ((DateTime.Now.DateTimeToEpochTime() - File.GetLastWriteTime(bkfilename).DateTimeToEpochTime()) < (5 * 60))
            {
                List <Closure> bkINCIPoints = new List <Closure>();
                bkINCIPoints = SSUtil.DeSerializeXMLObject <List <Closure> >(xmlINCIFolderPath + bkfilename);
                foreach (Closure aINCIPoint in bkINCIPoints)
                {
                    downloadedINCIXMLData_UNSAVED.AddOrUpdate(aINCIPoint.Id, aINCIPoint, (k, v) => aINCIPoint);//replaces existing if it exists
                }
            }

            if (pollingStart.DateTimeToEpochTime() >= DateTime.Now.ToLocalTime().DateTimeToEpochTime())//future download - put thread to sleep until start - proper state
            {
                //return pollingStart.DateTimeToEpochTime() - downloadStartTime.DateTimeToEpochTime();
                Thread.Sleep(Convert.ToInt32(1000 * (pollingStart.DateTimeToEpochTime() - DateTime.Now.ToLocalTime().DateTimeToEpochTime() - 1)));
                nextDownloadTime = pollingStart.AddSeconds(0);
                nextBackupTime   = pollingStart.AddSeconds(0);
                nextSaveTime     = DateTime.Now.ToLocalTime().Date.AddSeconds(-1 + fileSaveFreq);                                                                                                       //asume save frequencies are in the scale of days, save at midnighht
            }
            else if ((pollingStart.DateTimeToEpochTime() <= DateTime.Now.ToLocalTime().DateTimeToEpochTime()) && (pollingEnd.DateTimeToEpochTime() > DateTime.Now.ToLocalTime().DateTimeToEpochTime())) //some missing downloads - start period is before download can take place
            {
                //download at next hour, catch whatever data can be retrived.
                //Thread.Sleep(Convert.ToInt32(1000 * (3600 - (DateTime.Now.ToLocalTime().Minute * 60 + DateTime.Now.ToLocalTime().Second) - 1)));
                downloadState    = -1;
                nextDownloadTime = DateTime.Now.ToLocalTime().AddSeconds(0);                      //no delay for download
                nextBackupTime   = DateTime.Now.ToLocalTime().AddSeconds(0);                      //no delay for backup
                nextSaveTime     = DateTime.Now.ToLocalTime().Date.AddSeconds(-1 + fileSaveFreq); //asume save frequencies are in the scale of days, save at midnighht
            }
            else //if (pollingEnd.DateTimeToEpochTime() <= DateTime.Now.ToLocalTime().DateTimeToEpochTime())//nothing to download - end period is before download can take place
            {
                nextDownloadTime = new DateTime();
                nextBackupTime   = new DateTime();
                nextSaveTime     = new DateTime();
                downloadState    = -2;
                return(downloadState);
            }

            //initial save message
            if (lastLogUpdatedSave != nextSaveTime.DateTimeToEpochTime())
            {
                lastLogUpdatedSave = nextSaveTime.DateTimeToEpochTime();
                LogUpdateEventArgs args = new LogUpdateEventArgs();
                args.logMessage = String.Format("Rd Closures Upcoming Save: {0}.", nextSaveTime.DateTimeISO8601Format());
                OnLogUpdate(args);
            }

            //download started after a proper wait - proper state, or start immediate if some data can be downloaded (state: -1)
            while (pollingEnd.DateTimeToEpochTime() >= DateTime.Now.ToLocalTime().DateTimeToEpochTime())
            {
                List <long> nextOpTime = new List <long>();
                nextOpTime.Add(nextDownloadTime.DateTimeToEpochTime());
                nextOpTime.Add(nextBackupTime.DateTimeToEpochTime());
                nextOpTime.Add(nextSaveTime.DateTimeToEpochTime());
                nextOpTime.Sort();
                if ((nextOpTime[0] > DateTime.Now.ToLocalTime().DateTimeToEpochTime()))
                {
                    if (lastLogUpdatedSave != nextSaveTime.DateTimeToEpochTime())
                    {
                        lastLogUpdatedSave = nextSaveTime.DateTimeToEpochTime();
                        LogUpdateEventArgs args = new LogUpdateEventArgs();
                        args.logMessage = String.Format("Rd Closures Upcoming Save: {0}.", nextSaveTime.DateTimeISO8601Format());
                        OnLogUpdate(args);
                    }

                    long sleepTime = nextOpTime[0] - DateTime.Now.ToLocalTime().DateTimeToEpochTime();
                    Thread.Sleep(Convert.ToInt32(1000 * sleepTime));//brief sleep in between operations
                }
                //this condition structure prioritize download to mem, then backup, then save
                if (nextDownloadTime.DateTimeToEpochTime() <= DateTime.Now.ToLocalTime().DateTimeToEpochTime())
                {
                    nextDownloadTime = nextDownloadTime.AddSeconds(pollingFrequency);
                    downloadCurrentINCIXMLFromWeb(pollingFrequency);
                    //List < Closure > newINCIPoints = downloadCurrentINCIXMLFromWeb(pollingFrequency);
                    //foreach (Closure aINCIPoint in downloadedINCIXML)
                    //{
                    //    downloadedINCIXMLData_UNSAVED.AddOrUpdate(aINCIPoint.Id, aINCIPoint, (k, v) => aINCIPoint);//replaces existing if it exists
                    //}
                }
                else if (nextBackupTime.DateTimeToEpochTime() <= DateTime.Now.ToLocalTime().DateTimeToEpochTime())
                {
                    nextBackupTime = nextBackupTime.AddSeconds(bkSaveFreq);
                    SSUtil.SerializeXMLObject(downloadedINCIXMLData_UNSAVED.Values.ToList(), xmlINCIFolderPath + bkfilename);
                }
                else if (nextSaveTime.DateTimeToEpochTime() <= DateTime.Now.ToLocalTime().DateTimeToEpochTime())
                {
                    if (downloadedINCIXMLData_UNSAVED.Values.Count > 0)//avoid saving empty file
                    {
                        string newfilename = string.Format(xmlINCIFilenameFormat, nextSaveTime.AddSeconds(-fileSaveFreq + 1).ToUniversalTime().DateTimeNamingFormat(), nextSaveTime.ToUniversalTime().DateTimeNamingFormat());
                        SSUtil.SerializeXMLObject(downloadedINCIXMLData_UNSAVED.Values.ToList(), xmlINCIFolderPath + newfilename);
                        downloadedINCIXMLData_UNSAVED.Clear();//clear mem
                    }
                    nextSaveTime = nextSaveTime.AddSeconds(fileSaveFreq);
                }
            }
            return(downloadState);
            //downloadState = -2;//no download has taken place, date out of range
            //downloadState = -1;//download completed with some error - some missing data
            //downloadState = 0;//download completed with no error
        }
        public ConcurrentDictionary <int, SSINTNXSignalDataTable> getCombinedIntxnData()
        {
            ConcurrentDictionary <int, SSINTNXSignalDataTable> combinedTable = new ConcurrentDictionary <int, SSINTNXSignalDataTable>();

            //try
            //{
            //read all Intxn data sheets from xml
            //traffic_signals.xml
            if (trafficSignalsFilePath != null)
            {
                Traffic_signals trafficSignalData = SSUtil.DeSerializeXMLObject <Traffic_signals>(trafficSignalsFilePath);
                foreach (RecordTrafficSignals records in trafficSignalData.Record)
                {
                    int pxID = Convert.ToInt32(records.Px);
                    SSINTNXSignalDataTable commonData = combinedTable.ContainsKey(pxID) ? combinedTable[pxID] : new SSINTNXSignalDataTable();
                    commonData.pxID            = pxID;
                    commonData.mainStreet      = records.Main;
                    commonData.midblockStreet  = records.Mid_block;
                    commonData.sideStreet1     = records.Side1;
                    commonData.sideStreet2     = records.Side2;
                    commonData.Latitude        = Convert.ToDouble(records.Lat);
                    commonData.Longitude       = Convert.ToDouble(records.Long);
                    commonData.signal_system   = records.Signal_system;
                    commonData.mode_of_control = records.Mode_of_control;
                    commonData.isNotFXT        = records.Mode_of_control.Contains("FXT") ? false : true;
                    commonData.transit_preempt = records.Transit_preempt.Contains("1") ? true : false;
                    commonData.fire_preempt    = records.Fire_preempt.Contains("1") ? true : false;
                    commonData.Rail_preempt    = records.Rail_preempt.Contains("1") ? true : false;
                    int noOfApproaches = records.No_of_signalized_approaches.Length == 1 ? Convert.ToInt32(records.No_of_signalized_approaches) : 0;
                    commonData.vehVol = -1; //no volume default
                    commonData.pedVol = -1; //no volume default
                    commonData.no_of_signalized_approaches = noOfApproaches;
                    commonData.isSignalizedIntxn           = true;
                    commonData.isPedCross       = false;
                    commonData.isFlasingBeacons = false;
                    combinedTable.AddOrUpdate(pxID, commonData, (k, v) => v);
                }
            }
            //pedestrian_crossovers.xml
            if (pedCrossoverFilePath != null)
            {
                Pedestrian_crossovers pedCrossData = SSUtil.DeSerializeXMLObject <Pedestrian_crossovers>(pedCrossoverFilePath);
                foreach (RecordPedCross records in pedCrossData.Record)
                {
                    int pxID = Convert.ToInt32(records.Px);
                    SSINTNXSignalDataTable commonData;
                    if (combinedTable.ContainsKey(pxID))
                    {
                        commonData = combinedTable[pxID];
                    }
                    else
                    {
                        commonData                = new SSINTNXSignalDataTable();
                        commonData.pxID           = pxID;
                        commonData.mainStreet     = records.Main;
                        commonData.midblockStreet = records.Midblock;
                        commonData.sideStreet1    = records.Side1;
                        commonData.sideStreet2    = records.Side2;
                        commonData.Latitude       = Convert.ToDouble(records.Lat);
                        commonData.Longitude      = Convert.ToDouble(records.Long);
                        commonData.no_of_signalized_approaches = 2;
                        commonData.vehVol = -1; //no volume default
                        commonData.pedVol = -1; //no volume default
                    }
                    commonData.isSignalizedIntxn = false;
                    commonData.isPedCross        = true;
                    commonData.isFlasingBeacons  = false;
                    combinedTable.AddOrUpdate(pxID, commonData, (k, v) => v);
                }
            }
            //flashing_beacons.xml
            if (flashingBeaconFilePath != null)
            {
                Flashing_beacons flashingBeaconData = SSUtil.DeSerializeXMLObject <Flashing_beacons>(flashingBeaconFilePath);
                foreach (RecordFlashingBeacons records in flashingBeaconData.Record)
                {
                    int pxID = Convert.ToInt32(records.Px);
                    SSINTNXSignalDataTable commonData;
                    if (combinedTable.ContainsKey(pxID))
                    {
                        commonData = combinedTable[pxID];
                    }
                    else
                    {
                        commonData                = new SSINTNXSignalDataTable();
                        commonData.pxID           = pxID;
                        commonData.mainStreet     = records.Main;
                        commonData.midblockStreet = records.Midblock;
                        commonData.sideStreet1    = records.Side1;
                        commonData.sideStreet2    = records.Side2;
                        commonData.Latitude       = Convert.ToDouble(records.Lat);
                        commonData.Longitude      = Convert.ToDouble(records.Long);
                        commonData.no_of_signalized_approaches = 2;
                        commonData.vehVol = -1; //no volume default
                        commonData.pedVol = -1; //no volume default
                    }
                    commonData.isSignalizedIntxn = false;
                    commonData.isPedCross        = false;
                    commonData.isFlasingBeacons  = true;
                    combinedTable.AddOrUpdate(pxID, commonData, (k, v) => v);
                }
            }
            //traffic_vols.xml
            if (trafficVolFilePath != null)
            {
                Traffic_vols trafficVolData = SSUtil.DeSerializeXMLObject <Traffic_vols>(trafficVolFilePath);
                foreach (RecordTrafficVols records in trafficVolData.Record)
                {
                    int pxID = Convert.ToInt32(records.Px);
                    SSINTNXSignalDataTable commonData;
                    if (combinedTable.ContainsKey(pxID))
                    {
                        commonData = combinedTable[pxID];
                    }
                    else
                    {
                        commonData                = new SSINTNXSignalDataTable();
                        commonData.pxID           = pxID;
                        commonData.mainStreet     = records.Main;
                        commonData.midblockStreet = records.Midblock;
                        commonData.sideStreet1    = records.Side1;
                        commonData.sideStreet2    = records.Side2;
                        commonData.Latitude       = Convert.ToDouble(records.Lat);
                        commonData.Longitude      = Convert.ToDouble(records.Long);

                        commonData.signal_system   = "unknownFromVolFile";
                        commonData.mode_of_control = "unknownFromVolFile";
                        commonData.isNotFXT        = false;
                        commonData.transit_preempt = false;
                        commonData.fire_preempt    = false;
                        commonData.Rail_preempt    = false;
                        int noOfApproaches = (commonData.midblockStreet.Length > 0) ? 2 : ((commonData.sideStreet2.Contains("PRIVATE ACCESS")) ? 3 : 4); //estimates
                        commonData.no_of_signalized_approaches = noOfApproaches;
                        commonData.isSignalizedIntxn           = true;                                                                                   //assumes only signalized intersection has volume measurements
                        commonData.isPedCross       = false;
                        commonData.isFlasingBeacons = false;
                    }
                    commonData.countDate = DateTime.ParseExact(records.CountDate, "M-d-yyyy", System.Globalization.CultureInfo.InvariantCulture);
                    string vehVol = Regex.Replace(records.VehVol8Hr, @"\s+", "");
                    commonData.vehVol = Convert.ToInt32(vehVol);
                    string pedVol = Regex.Replace(records.PedVol8Hr, @"\s+", "");
                    commonData.pedVol = Convert.ToInt32(pedVol);
                    combinedTable.AddOrUpdate(pxID, commonData, (k, v) => v);
                }
            }
            //}
            //catch (Exception e)
            //{
            //    return null;
            //}
            return(combinedTable);
        }