//private List<Closure> downloadCurrentINCIXMLFromWeb(double pollingOrRequestFrequency)//download current INCI XML data from NextBus with processing for INCITime
//{
// List<Closure> downloadedINCIXML = new List<Closure>();
// Uri uriToDownload = new Uri("http://www1.toronto.ca/transportation/roadrestrictions/RoadRestrictions.xml");
// repeatWebDownload:
// try
// {
// using (WebClient client = new WebClient())
// {
// //download string and write to file
// string xml = client.DownloadString(uriToDownload);
// string xmlString = Convert.ToString(xml);
// using (StringReader reader = new StringReader(xmlString))
// {
// XmlSerializer serializer = new XmlSerializer(typeof(Closures));
// string headerString = reader.ReadLine();
// string currentVehString = reader.ReadToEnd();
// MemoryStream memStream = new MemoryStream(Encoding.UTF8.GetBytes(currentVehString));
// Closures currentXMLINCIPoints = (Closures)serializer.Deserialize(memStream);
// downloadedINCIXML.AddRange(currentXMLINCIPoints.Closure);
// }//end using reader
// }//end using cilent
// }
// catch (Exception e)
// {
// //sleep to aviod too many requests
// Thread.Sleep(Convert.ToInt32(1000 * pollingOrRequestFrequency / 2));//check delay at "half of" polling frequency (minute), until it is time for download
// goto repeatWebDownload;
// }
// return downloadedINCIXML;
//}
/// <summary>
///
/// </summary>
/// <param name="RouteTags"></param>
/// <param name="startTime"></param>
/// <param name="endTime"></param>
/// <returns></returns>
public List <SSIncidentDataTable> RetriveINCIData(DateTime startTime, DateTime endTime, List <string> RouteTags = null)
{
List <SSIncidentDataTable> finalPackageOfINCIDataAllRoutes = new List <SSIncidentDataTable>();
List <SSIncidentDataTable> allDataInTheINCIDataFile = new List <SSIncidentDataTable>();
//For incident and weather data, saves are daily
startTime = startTime.Date; //beginning of first day
endTime = endTime.Date.AddDays(1).AddSeconds(-1); //end of second day
DateTime lastIntermediateTime = startTime; //intermediate time 59:59 (min:sec) from start
DateTime intermediateTime = startTime.Date.AddHours(startTime.Hour).AddSeconds(fileSaveFreq - 1); //intermediate time 59:59 (min:sec) from start
while (intermediateTime.DateTimeToEpochTime() <= endTime.DateTimeToEpochTime())
{
string currentfilename = string.Format(xmlINCIFilenameFormat, lastIntermediateTime.ToUniversalTime().DateTimeNamingFormat(), intermediateTime.ToUniversalTime().DateTimeNamingFormat());
allDataInTheINCIDataFile.AddRange(convertRawINCIToSSFormat(SSUtil.DeSerializeXMLObject <List <Closure> >(xmlINCIFolderPath + currentfilename)));
//increment to read next file
foreach (SSIncidentDataTable aINCIPoint in allDataInTheINCIDataFile)
{
//identify any filtering needed - none to start with
finalPackageOfINCIDataAllRoutes.Add(aINCIPoint);
}
lastIntermediateTime = lastIntermediateTime.AddSeconds(fileSaveFreq);
intermediateTime = intermediateTime.AddSeconds(fileSaveFreq);
}
allDataInTheINCIDataFile.Clear(); //clear Temp list
return(finalPackageOfINCIDataAllRoutes); //return result list
}
/// <summary>
///
/// </summary>
/// <param name="RouteTags"></param>
/// <param name="startTime"></param>
/// <param name="endTime"></param>
/// <returns></returns>
public List <SSVehGPSDataTable> RetriveGPSData(List <string> RouteTags, DateTime startTime, DateTime endTime)
{
List <SSVehGPSDataTable> finalPackageOfGPSDataAllRoutes = new List <SSVehGPSDataTable>();
//List<SSVehGPSDataTable> allDataInTheGPSDataFile = new List<SSVehGPSDataTable>();
Dictionary <string, List <SSVehGPSDataTable> > allDataInGPSDataFile = new Dictionary <string, List <SSVehGPSDataTable> >();
List <string> gpsDataFilenames = new List <string>(); //in order
DateTime lastIntermediateTime = startTime; //intermediate time 59:59 (min:sec) from start
DateTime intermediateTime = startTime.Date.AddHours(startTime.Hour).AddSeconds(fileSaveFreq - 1); //intermediate time 59:59 (min:sec) from start
//initiate filenames
while (intermediateTime.DateTimeToEpochTime() <= endTime.DateTimeToEpochTime())
{
string currentfilename = string.Format(xmlGPSFilenameFormat, lastIntermediateTime.ToUniversalTime().DateTimeNamingFormat(), intermediateTime.ToUniversalTime().DateTimeNamingFormat());
gpsDataFilenames.Add(currentfilename);
//increment to read next file
lastIntermediateTime = lastIntermediateTime.AddSeconds(fileSaveFreq);
intermediateTime = intermediateTime.AddSeconds(fileSaveFreq);
}
//load all files in parallel
ParallelOptions pOptions = new ParallelOptions();
pOptions.MaxDegreeOfParallelism = Environment.ProcessorCount;
Parallel.For(0, gpsDataFilenames.Count, pOptions, i =>
{
string currentfilename = gpsDataFilenames[i];
allDataInGPSDataFile.Add(currentfilename, (convertRawGPSToSSFormat(SSUtil.DeSerializeXMLObject <List <Vehicle> >(xmlGPSFolderPath + currentfilename))));
});
//add each file, then each gps point, in order
foreach (string currentfilename in gpsDataFilenames)
{
//filter by routeTag
foreach (SSVehGPSDataTable aGPSPoint in allDataInGPSDataFile[currentfilename])
{
string aGPSPointRouteTag = aGPSPoint.TripCode == "NULL" ? "" : aGPSPoint.TripCode.Split('_')[0];
if (RouteTags.Contains(aGPSPointRouteTag))
{
finalPackageOfGPSDataAllRoutes.Add(aGPSPoint);
}
}
//List<SSVehGPSDataTable> filteredData = allDataInGPSDataFile[currentfilename].Where(v => RouteTags.Contains(v.TripCode.Split('_')[0])).ToList();
//finalPackageOfGPSDataAllRoutes.AddRange(filteredData);
}
allDataInGPSDataFile.Clear();
gpsDataFilenames.Clear();
return(finalPackageOfGPSDataAllRoutes);//return result list
}
public int startLiveDownloadTask(DateTime pollingStart, DateTime pollingEnd, double pollingFrequency)
{
//fileSaveFreq must be 1 day, otherwise, needs to modify nextSaveTime
int downloadState = 0;
//NOTE: polling freq: 60s < bk freq: 120s < save freq: 3600s * 24 = 1 day
downloadedINCIXMLData_UNSAVED = new ConcurrentDictionary <string, Closure>(); //INCITime & VehID as tuple index
DateTime nextDownloadTime; // = DateTime.Now.ToLocalTime().AddSeconds(pollingFrequency);
DateTime nextBackupTime; // = DateTime.Now.ToLocalTime().AddSeconds(bkSaveFreq);
//DateTime nextSaveTime = DateTime.Now.ToLocalTime().AddSeconds(fileSaveFreq);
DateTime nextSaveTime; // = pollingStart.Date.AddSeconds(pollingStart.Hour * 3600 - 1 + fileSaveFreq);//save on the last second of the next hour
long lastLogUpdatedSave = 0;
string bkfilename = "current-RRXML-bk.xml";
//load last saved backup, if it is from within the last 5 minutes
if ((DateTime.Now.DateTimeToEpochTime() - File.GetLastWriteTime(bkfilename).DateTimeToEpochTime()) < (5 * 60))
{
List <Closure> bkINCIPoints = new List <Closure>();
bkINCIPoints = SSUtil.DeSerializeXMLObject <List <Closure> >(xmlINCIFolderPath + bkfilename);
foreach (Closure aINCIPoint in bkINCIPoints)
{
downloadedINCIXMLData_UNSAVED.AddOrUpdate(aINCIPoint.Id, aINCIPoint, (k, v) => aINCIPoint);//replaces existing if it exists
}
}
if (pollingStart.DateTimeToEpochTime() >= DateTime.Now.ToLocalTime().DateTimeToEpochTime())//future download - put thread to sleep until start - proper state
{
//return pollingStart.DateTimeToEpochTime() - downloadStartTime.DateTimeToEpochTime();
Thread.Sleep(Convert.ToInt32(1000 * (pollingStart.DateTimeToEpochTime() - DateTime.Now.ToLocalTime().DateTimeToEpochTime() - 1)));
nextDownloadTime = pollingStart.AddSeconds(0);
nextBackupTime = pollingStart.AddSeconds(0);
nextSaveTime = DateTime.Now.ToLocalTime().Date.AddSeconds(-1 + fileSaveFreq); //asume save frequencies are in the scale of days, save at midnighht
}
else if ((pollingStart.DateTimeToEpochTime() <= DateTime.Now.ToLocalTime().DateTimeToEpochTime()) && (pollingEnd.DateTimeToEpochTime() > DateTime.Now.ToLocalTime().DateTimeToEpochTime())) //some missing downloads - start period is before download can take place
{
//download at next hour, catch whatever data can be retrived.
//Thread.Sleep(Convert.ToInt32(1000 * (3600 - (DateTime.Now.ToLocalTime().Minute * 60 + DateTime.Now.ToLocalTime().Second) - 1)));
downloadState = -1;
nextDownloadTime = DateTime.Now.ToLocalTime().AddSeconds(0); //no delay for download
nextBackupTime = DateTime.Now.ToLocalTime().AddSeconds(0); //no delay for backup
nextSaveTime = DateTime.Now.ToLocalTime().Date.AddSeconds(-1 + fileSaveFreq); //asume save frequencies are in the scale of days, save at midnighht
}
else //if (pollingEnd.DateTimeToEpochTime() <= DateTime.Now.ToLocalTime().DateTimeToEpochTime())//nothing to download - end period is before download can take place
{
nextDownloadTime = new DateTime();
nextBackupTime = new DateTime();
nextSaveTime = new DateTime();
downloadState = -2;
return(downloadState);
}
//initial save message
if (lastLogUpdatedSave != nextSaveTime.DateTimeToEpochTime())
{
lastLogUpdatedSave = nextSaveTime.DateTimeToEpochTime();
LogUpdateEventArgs args = new LogUpdateEventArgs();
args.logMessage = String.Format("Rd Closures Upcoming Save: {0}.", nextSaveTime.DateTimeISO8601Format());
OnLogUpdate(args);
}
//download started after a proper wait - proper state, or start immediate if some data can be downloaded (state: -1)
while (pollingEnd.DateTimeToEpochTime() >= DateTime.Now.ToLocalTime().DateTimeToEpochTime())
{
List <long> nextOpTime = new List <long>();
nextOpTime.Add(nextDownloadTime.DateTimeToEpochTime());
nextOpTime.Add(nextBackupTime.DateTimeToEpochTime());
nextOpTime.Add(nextSaveTime.DateTimeToEpochTime());
nextOpTime.Sort();
if ((nextOpTime[0] > DateTime.Now.ToLocalTime().DateTimeToEpochTime()))
{
if (lastLogUpdatedSave != nextSaveTime.DateTimeToEpochTime())
{
lastLogUpdatedSave = nextSaveTime.DateTimeToEpochTime();
LogUpdateEventArgs args = new LogUpdateEventArgs();
args.logMessage = String.Format("Rd Closures Upcoming Save: {0}.", nextSaveTime.DateTimeISO8601Format());
OnLogUpdate(args);
}
long sleepTime = nextOpTime[0] - DateTime.Now.ToLocalTime().DateTimeToEpochTime();
Thread.Sleep(Convert.ToInt32(1000 * sleepTime));//brief sleep in between operations
}
//this condition structure prioritize download to mem, then backup, then save
if (nextDownloadTime.DateTimeToEpochTime() <= DateTime.Now.ToLocalTime().DateTimeToEpochTime())
{
nextDownloadTime = nextDownloadTime.AddSeconds(pollingFrequency);
downloadCurrentINCIXMLFromWeb(pollingFrequency);
//List < Closure > newINCIPoints = downloadCurrentINCIXMLFromWeb(pollingFrequency);
//foreach (Closure aINCIPoint in downloadedINCIXML)
//{
// downloadedINCIXMLData_UNSAVED.AddOrUpdate(aINCIPoint.Id, aINCIPoint, (k, v) => aINCIPoint);//replaces existing if it exists
//}
}
else if (nextBackupTime.DateTimeToEpochTime() <= DateTime.Now.ToLocalTime().DateTimeToEpochTime())
{
nextBackupTime = nextBackupTime.AddSeconds(bkSaveFreq);
SSUtil.SerializeXMLObject(downloadedINCIXMLData_UNSAVED.Values.ToList(), xmlINCIFolderPath + bkfilename);
}
else if (nextSaveTime.DateTimeToEpochTime() <= DateTime.Now.ToLocalTime().DateTimeToEpochTime())
{
if (downloadedINCIXMLData_UNSAVED.Values.Count > 0)//avoid saving empty file
{
string newfilename = string.Format(xmlINCIFilenameFormat, nextSaveTime.AddSeconds(-fileSaveFreq + 1).ToUniversalTime().DateTimeNamingFormat(), nextSaveTime.ToUniversalTime().DateTimeNamingFormat());
SSUtil.SerializeXMLObject(downloadedINCIXMLData_UNSAVED.Values.ToList(), xmlINCIFolderPath + newfilename);
downloadedINCIXMLData_UNSAVED.Clear();//clear mem
}
nextSaveTime = nextSaveTime.AddSeconds(fileSaveFreq);
}
}
return(downloadState);
//downloadState = -2;//no download has taken place, date out of range
//downloadState = -1;//download completed with some error - some missing data
//downloadState = 0;//download completed with no error
}
public ConcurrentDictionary <int, SSINTNXSignalDataTable> getCombinedIntxnData()
{
ConcurrentDictionary <int, SSINTNXSignalDataTable> combinedTable = new ConcurrentDictionary <int, SSINTNXSignalDataTable>();
//try
//{
//read all Intxn data sheets from xml
//traffic_signals.xml
if (trafficSignalsFilePath != null)
{
Traffic_signals trafficSignalData = SSUtil.DeSerializeXMLObject <Traffic_signals>(trafficSignalsFilePath);
foreach (RecordTrafficSignals records in trafficSignalData.Record)
{
int pxID = Convert.ToInt32(records.Px);
SSINTNXSignalDataTable commonData = combinedTable.ContainsKey(pxID) ? combinedTable[pxID] : new SSINTNXSignalDataTable();
commonData.pxID = pxID;
commonData.mainStreet = records.Main;
commonData.midblockStreet = records.Mid_block;
commonData.sideStreet1 = records.Side1;
commonData.sideStreet2 = records.Side2;
commonData.Latitude = Convert.ToDouble(records.Lat);
commonData.Longitude = Convert.ToDouble(records.Long);
commonData.signal_system = records.Signal_system;
commonData.mode_of_control = records.Mode_of_control;
commonData.isNotFXT = records.Mode_of_control.Contains("FXT") ? false : true;
commonData.transit_preempt = records.Transit_preempt.Contains("1") ? true : false;
commonData.fire_preempt = records.Fire_preempt.Contains("1") ? true : false;
commonData.Rail_preempt = records.Rail_preempt.Contains("1") ? true : false;
int noOfApproaches = records.No_of_signalized_approaches.Length == 1 ? Convert.ToInt32(records.No_of_signalized_approaches) : 0;
commonData.vehVol = -1; //no volume default
commonData.pedVol = -1; //no volume default
commonData.no_of_signalized_approaches = noOfApproaches;
commonData.isSignalizedIntxn = true;
commonData.isPedCross = false;
commonData.isFlasingBeacons = false;
combinedTable.AddOrUpdate(pxID, commonData, (k, v) => v);
}
}
//pedestrian_crossovers.xml
if (pedCrossoverFilePath != null)
{
Pedestrian_crossovers pedCrossData = SSUtil.DeSerializeXMLObject <Pedestrian_crossovers>(pedCrossoverFilePath);
foreach (RecordPedCross records in pedCrossData.Record)
{
int pxID = Convert.ToInt32(records.Px);
SSINTNXSignalDataTable commonData;
if (combinedTable.ContainsKey(pxID))
{
commonData = combinedTable[pxID];
}
else
{
commonData = new SSINTNXSignalDataTable();
commonData.pxID = pxID;
commonData.mainStreet = records.Main;
commonData.midblockStreet = records.Midblock;
commonData.sideStreet1 = records.Side1;
commonData.sideStreet2 = records.Side2;
commonData.Latitude = Convert.ToDouble(records.Lat);
commonData.Longitude = Convert.ToDouble(records.Long);
commonData.no_of_signalized_approaches = 2;
commonData.vehVol = -1; //no volume default
commonData.pedVol = -1; //no volume default
}
commonData.isSignalizedIntxn = false;
commonData.isPedCross = true;
commonData.isFlasingBeacons = false;
combinedTable.AddOrUpdate(pxID, commonData, (k, v) => v);
}
}
//flashing_beacons.xml
if (flashingBeaconFilePath != null)
{
Flashing_beacons flashingBeaconData = SSUtil.DeSerializeXMLObject <Flashing_beacons>(flashingBeaconFilePath);
foreach (RecordFlashingBeacons records in flashingBeaconData.Record)
{
int pxID = Convert.ToInt32(records.Px);
SSINTNXSignalDataTable commonData;
if (combinedTable.ContainsKey(pxID))
{
commonData = combinedTable[pxID];
}
else
{
commonData = new SSINTNXSignalDataTable();
commonData.pxID = pxID;
commonData.mainStreet = records.Main;
commonData.midblockStreet = records.Midblock;
commonData.sideStreet1 = records.Side1;
commonData.sideStreet2 = records.Side2;
commonData.Latitude = Convert.ToDouble(records.Lat);
commonData.Longitude = Convert.ToDouble(records.Long);
commonData.no_of_signalized_approaches = 2;
commonData.vehVol = -1; //no volume default
commonData.pedVol = -1; //no volume default
}
commonData.isSignalizedIntxn = false;
commonData.isPedCross = false;
commonData.isFlasingBeacons = true;
combinedTable.AddOrUpdate(pxID, commonData, (k, v) => v);
}
}
//traffic_vols.xml
if (trafficVolFilePath != null)
{
Traffic_vols trafficVolData = SSUtil.DeSerializeXMLObject <Traffic_vols>(trafficVolFilePath);
foreach (RecordTrafficVols records in trafficVolData.Record)
{
int pxID = Convert.ToInt32(records.Px);
SSINTNXSignalDataTable commonData;
if (combinedTable.ContainsKey(pxID))
{
commonData = combinedTable[pxID];
}
else
{
commonData = new SSINTNXSignalDataTable();
commonData.pxID = pxID;
commonData.mainStreet = records.Main;
commonData.midblockStreet = records.Midblock;
commonData.sideStreet1 = records.Side1;
commonData.sideStreet2 = records.Side2;
commonData.Latitude = Convert.ToDouble(records.Lat);
commonData.Longitude = Convert.ToDouble(records.Long);
commonData.signal_system = "unknownFromVolFile";
commonData.mode_of_control = "unknownFromVolFile";
commonData.isNotFXT = false;
commonData.transit_preempt = false;
commonData.fire_preempt = false;
commonData.Rail_preempt = false;
int noOfApproaches = (commonData.midblockStreet.Length > 0) ? 2 : ((commonData.sideStreet2.Contains("PRIVATE ACCESS")) ? 3 : 4); //estimates
commonData.no_of_signalized_approaches = noOfApproaches;
commonData.isSignalizedIntxn = true; //assumes only signalized intersection has volume measurements
commonData.isPedCross = false;
commonData.isFlasingBeacons = false;
}
commonData.countDate = DateTime.ParseExact(records.CountDate, "M-d-yyyy", System.Globalization.CultureInfo.InvariantCulture);
string vehVol = Regex.Replace(records.VehVol8Hr, @"\s+", "");
commonData.vehVol = Convert.ToInt32(vehVol);
string pedVol = Regex.Replace(records.PedVol8Hr, @"\s+", "");
commonData.pedVol = Convert.ToInt32(pedVol);
combinedTable.AddOrUpdate(pxID, commonData, (k, v) => v);
}
}
//}
//catch (Exception e)
//{
// return null;
//}
return(combinedTable);
}