private static SurveyEntry CleanPersonRecord(Network network, SurveyEntry personRecord)
{
var trips = personRecord?.Trips;
if (trips != null)
{
// remove short trips
for (int i = 0; i < trips.Count; i++)
{
if (trips[i].TripEndTime - trips[i].TripStartTime < 5.0f)
{
trips.RemoveAt(i);
i--;
}
}
// remove trips that are too fast
for (int i = 0; i < trips.Count; i++)
{
var distance = network.ComputeDistance(trips[i].Origin, trips[i].Destination);
var deltaTime = trips[i].TripEndTime - trips[i].TripStartTime;
// remove trips that are faster than 120km/h
if (distance / deltaTime > 200.0f)
{
trips.RemoveAt(i);
i--;
}
}
}
return(personRecord);
}
public static IEnumerable <SurveyEntry> EnumerateSurvey(string filePath)
{
using (var reader = new CsvReader(filePath))
{
// burn header
reader.LoadLine();
long currentPerson = 0;
bool validPerson = false;
var seperators = new char[] { ',', '\t' };
var toReturn = new SurveyEntry();
while (reader.LoadLine(out var columns))
{
if (columns > 19)
{
TripEntry trip;
// the last two digits are the ID for the trip
reader.Get(out long personNumber, 19);
personNumber = personNumber / 100;
if (personNumber != currentPerson)
{
if (validPerson)
{
yield return(toReturn);
toReturn = new SurveyEntry();
}
else
{
toReturn.Clear();
}
currentPerson = personNumber;
validPerson = true;
}
if (!ConvertFloatStringToInt(reader, 24, out trip.Origin) ||
!ConvertFloatStringToInt(reader, 25, out trip.Destination) ||
!ConvertMode(reader, 9, out trip.Mode) || !ConvertTime(reader, 4, out trip.TripStartTime) || !ConvertTime(reader, 5, out trip.TripEndTime) ||
!ConvertFloatStringToInt(reader, 22, out var originInZone) ||
!ConvertFloatStringToInt(reader, 23, out var destinationInZone) ||
originInZone < 1f ||
destinationInZone < 1f ||
trip.Origin < 0 ||
trip.Destination < 0)
{
validPerson = false;
}
else
{
toReturn.Add(trip);
}
}
}
if (validPerson)
{
yield return(toReturn);
}
}
}
private static void WriteTransitSurveydata(Network[] networks, Dictionary <int, DensityData> densityData)
{
var rand = new Random(123546);
using (StreamWriter writer = new StreamWriter("SyntheticCellTraces-Transit.csv"))
using (StreamWriter writer2 = new StreamWriter("SyntheticCellTracesTest-Transit.csv"))
{
// Write the headers for both streams
// WriteHeaders(writer, false); WriteHeaders(writer2, false);
StringBuilder builder = new StringBuilder();
StringBuilder builder2 = new StringBuilder();
foreach (var personRecord in TransitData.StreamTransitRiderDays(@"G:\TMG\Research\Montevideo\NewTransitData\2018.12.04\od_may_2018_vero.csv", networks[0], TransitData.LoadStopToStop(
@"G:\TMG\Research\Montevideo\BusNetwork\BusStopsEmmeIDmapping.csv")))
{
// Convert into a survey entry
SurveyEntry entry = new SurveyEntry();
for (int i = 0; i < personRecord.Count; i++)
{
entry.Add(new TripEntry()
{
Origin = GetStopCentroid(networks, personRecord[i].OriginNode, personRecord[i].StartTimeOfDay, rand),
Destination = GetStopCentroid(networks, personRecord[i].DestinationNode, personRecord[i].StartTimeOfDay, rand),
Mode = 1,
TripStartTime = (float)personRecord[i].StartTimeOfDay,
TripEndTime = (float)personRecord[i].EndTimeOfDay
});
}
RecordsFor(builder, builder2, networks, entry, false, densityData);
// 80% of the records will be stored in the training set
if (rand.NextDouble() < 0.8)
{
writer.Write(builder);
}
else
{
writer2.Write(builder);
}
builder.Clear();
builder2.Clear();
}
}
}
public static IEnumerable <SurveyEntry> EnumerateCellTraces(Network am, string stepFilePath, Dictionary <int, DensityData> densityData)
{
Random r = new Random();
using (var reader = new CsvReader(stepFilePath))
{
// burn header
reader.LoadLine();
string previousPersonID = null;
string previousDate = null;
bool validPerson = false;
var toReturn = new SurveyEntry();
while (reader.LoadLine(out var columns))
{
if (columns >= 7)
{
reader.Get(out string stepType, 7);
reader.Get(out string date, 0);
if (stepType == "trip")
{
reader.Get(out string personID, 1);
// if we are starting a new person
if (previousPersonID != personID || previousDate != date)
{
if (validPerson)
{
yield return(toReturn);
validPerson = true;
toReturn = new SurveyEntry();
}
else
{
validPerson = true;
toReturn.Trips.Clear();
}
previousDate = date;
previousPersonID = personID;
}
// only sore the data if they are valid
if (validPerson)
{
reader.Get(out int origin, 3);
reader.Get(out int destination, 4);
reader.Get(out float tripStart, 5);
reader.Get(out float tripEnd, 6);
if (!am.PickCentroidInZone(ref origin, r) ||
!am.PickCentroidInZone(ref destination, r))
{
validPerson = false;
continue;
}
toReturn.Add(new TripEntry()
{
Mode = 0,
TripStartTime = tripStart,
TripEndTime = tripEnd,
Origin = origin,
Destination = destination
});
}
}
}
}
if (validPerson)
{
yield return(toReturn);
}
}
}
static bool RecordsFor(StringBuilder writer, StringBuilder mainFeatures, Network[] network, SurveyEntry entry, bool applyODInsteadOfMode,
Dictionary <int, DensityData> densityData)
{
/*
* Step 1) Find nextTrip
* Step 2) Assign Zeros until trip start
* Step 3) Step through trip depending on made and set delta distance when the zone changes
*/
var trips = entry.Trips;
int timeStep = 0;
const int minutesPerTimeStep = 5;
// km/minute
const float walkSpeed = 4.0f / 60.0f;
const int numberOfSegmentsInADay = (60 * 24) / minutesPerTimeStep;
// return back the empty buffer
if (entry.Trips.Count <= 0)
{
return(false);
}
int currentZone;
void EmitNothing()
{
mainFeatures.Append(",0.0");
timeStep++;
}
void emitDistance(float distance)
{
mainFeatures.Append(',');
mainFeatures.Append(distance);
timeStep++;
}
for (int i = 0; i < trips.Count; i++)
{
var startTime = (int)Math.Round(trips[i].TripStartTime, 0) / minutesPerTimeStep;
// Step 2
for (; timeStep < startTime && timeStep < numberOfSegmentsInADay;)
{
// TODO: Change this to emit more than nothing
EmitNothing();
}
// check to see if the day is over
if (timeStep >= numberOfSegmentsInADay)
{
break;
}
if (trips[i].Origin != trips[i].Destination)
{
var timeOfDay = GetTimeOfDay(i);
// Step 3
switch (trips[i].Mode)
{
// auto
case 0:
{
var path = network[timeOfDay].GetFastestPath(trips[i].Origin, trips[i].Destination);
if (path == null)
{
throw new InvalidOperationException($"Unable to find a path between {trips[i].Origin} and {trips[i].Destination}!");
}
// go through the path and find the points in time following the path
var accTime = 0.0f;
var distance = 0.0f;
var changeZone = false;
currentZone = network[timeOfDay].GetZone(trips[i].Origin);
for (int j = 0; j < path.Count && timeStep < numberOfSegmentsInADay; j++)
{
var travelTime = network[timeOfDay].GetTime(path[j].origin, path[j].destination);
var linkDistance = network[timeOfDay].GetDistance(path[j].origin, path[j].destination);
var destinationZone = network[timeOfDay].GetZone(path[j].destination);
changeZone = changeZone | (currentZone != destinationZone);
currentZone = destinationZone;
accTime += travelTime;
distance += linkDistance;
if (accTime >= minutesPerTimeStep)
{
accTime -= minutesPerTimeStep;
// if we have changed zones record how far we have travelled
if (changeZone)
{
emitDistance(distance);
distance = 0.0f;
changeZone = false;
}
else
{
EmitNothing();
}
}
}
// clean up the remainder
if (changeZone && timeStep < numberOfSegmentsInADay)
{
emitDistance(distance);
}
}
break;
// transit
case 1:
{
var path = network[timeOfDay].GetPathThroughTransit(trips[i].Origin, trips[i].Destination);
if (path == null)
{
Console.WriteLine($"Unable to find a path between {trips[i].Origin} and {trips[i].Destination}!");
writer.Clear();
return(false);
}
// go through the path and find the points in time following the path
var accTime = 0.0f;
var distance = 0.0f;
var changeZone = false;
currentZone = network[timeOfDay].GetZone(trips[i].Origin);
// the first path segment gives us the origin
var currentNode = path[0].node;
for (int j = 1; j < path.Count && timeStep < numberOfSegmentsInADay; j++)
{
// in-vehicle segment
if (path[j].line != "-")
{
var subPath = network[timeOfDay].GetTransitTravelOnRouteSegments(path[j].line, currentNode, path[j].node);
if (subPath == null)
{
Console.WriteLine("No path from " + currentNode + " to " + path[j].node + " on transit line " + path[j].line);
System.Environment.Exit(0);
}
for (int k = 0; k < subPath.Count && timeStep < numberOfSegmentsInADay; k++)
{
var(destinationNode, time) = subPath[k];
var destinationZone = network[timeOfDay].GetZone(destinationNode);
changeZone = changeZone | (currentZone != destinationZone);
currentZone = destinationZone;
accTime += time;
// use the full network for distances
distance += network[timeOfDay].GetDistance(currentNode, destinationNode);
if (accTime >= minutesPerTimeStep)
{
accTime -= minutesPerTimeStep;
if (changeZone)
{
emitDistance(distance);
distance = 0.0f;
changeZone = false;
}
else
{
EmitNothing();
}
}
currentNode = destinationNode;
}
}
// aux transit
else
{
// make sure that the next node actually exists
while (j < path.Count && !network[timeOfDay].HasNode(path[j].node))
{
j++;
}
var nextNode = j < path.Count ? path[j].node : entry.Trips[i].Destination;
var d = network[timeOfDay].GetDistance(currentNode, nextNode);
var destinationZone = network[timeOfDay].GetZone(nextNode);
changeZone = changeZone | (currentZone != destinationZone);
currentZone = destinationZone;
distance += d;
accTime += d / walkSpeed;
if (accTime >= minutesPerTimeStep)
{
accTime -= minutesPerTimeStep;
if (changeZone)
{
emitDistance(distance);
distance = 0.0f;
changeZone = false;
}
else
{
EmitNothing();
}
}
currentZone = destinationZone;
}
currentNode = j < path.Count ? path[j].node : -1;
}
// clean up the remainder
if (changeZone && timeStep < numberOfSegmentsInADay)
{
emitDistance(distance);
}
}
break;
// active transportation
case 2:
// walk along the road?
{
var path = network[timeOfDay].GetFastestPath(trips[i].Origin, trips[i].Destination);
if (path == null)
{
throw new InvalidOperationException($"Unable to find a path between {trips[i].Origin} and {trips[i].Destination}!");
}
// go through the path and find the points in time following the path
var accTime = 0.0f;
var distance = 0.0f;
var changeZone = false;
currentZone = network[timeOfDay].GetZone(trips[i].Origin);
for (int j = 0; j < path.Count && timeStep < numberOfSegmentsInADay; j++)
{
var linkDistance = network[timeOfDay].GetDistance(path[j].origin, path[j].destination);
var destinationZone = network[timeOfDay].GetZone(path[j].destination);
changeZone = changeZone | (currentZone != destinationZone);
currentZone = destinationZone;
accTime += linkDistance / walkSpeed;
distance += linkDistance;
if (accTime >= minutesPerTimeStep)
{
accTime -= minutesPerTimeStep;
// if we have changed zones record how far we have travelled
if (changeZone)
{
emitDistance(distance);
distance = 0.0f;
changeZone = false;
}
else
{
EmitNothing();
}
}
}
// clean up the remainder
if (changeZone && timeStep < numberOfSegmentsInADay)
{
emitDistance(distance);
}
}
break;
default:
throw new NotImplementedException($"UNKNOWN MODE {trips[i].Mode}");
}
}
// intrazonal
else
{
// just continue on to the next trip
}
}
if (timeStep > numberOfSegmentsInADay)
{
throw new InvalidOperationException("There were more time steps emitted than there were time steps in the day!");
}
// Finish writing the data for the rest of the day
for (; timeStep < numberOfSegmentsInADay;)
{
EmitNothing();
}
for (int i = 0; i < trips.Count; i++)
{
// make sure the trip starts during the day
if (trips[i].Origin != trips[i].Destination && trips[i].TripStartTime < 24 * 60)
{
if (applyODInsteadOfMode)
{
writer.Append(trips[i].Origin);
writer.Append(',');
writer.Append(trips[i].Destination);
writer.Append(',');
writer.Append(trips[i].TripStartTime);
}
else
{
writer.Append(trips[i].Mode);
}
writer.Append(mainFeatures);
// write out the trip specific data (1 if the activity is occurring)
for (int j = 0; j < numberOfSegmentsInADay; j++)
{
writer.Append(trips[i].TripStartTime <= j * minutesPerTimeStep && j * minutesPerTimeStep < trips[i].TripEndTime ? ",1.0" : ",0.0");
}
// Write out the density variables for origin then destination (population,employment,household)
if (densityData.TryGetValue(network[0].GetZone(entry.Trips[i].Origin), out var originDensity))
{
writer.Append(',');
writer.Append(originDensity.PopulationDensity);
writer.Append(',');
writer.Append(originDensity.EmploymentDensity);
writer.Append(',');
writer.Append(originDensity.HouseholdDensity);
}
else
{
writer.Append(",0.0,0.0,0.0");
}
if (densityData.TryGetValue(network[0].GetZone(entry.Trips[i].Destination), out var densityDensity))
{
writer.Append(',');
writer.Append(densityDensity.PopulationDensity);
writer.Append(',');
writer.Append(densityDensity.EmploymentDensity);
writer.Append(',');
writer.Append(densityDensity.HouseholdDensity);
}
else
{
writer.Append(",0.0,0.0,0.0");
}
// write the total distance of the trip
writer.Append(',');
writer.Append(TripDistance(network[0], entry.Trips[i]));
writer.AppendLine();
}
}
return(true);
}