private static SurveyEntry CleanPersonRecord(Network network, SurveyEntry personRecord) { var trips = personRecord?.Trips; if (trips != null) { // remove short trips for (int i = 0; i < trips.Count; i++) { if (trips[i].TripEndTime - trips[i].TripStartTime < 5.0f) { trips.RemoveAt(i); i--; } } // remove trips that are too fast for (int i = 0; i < trips.Count; i++) { var distance = network.ComputeDistance(trips[i].Origin, trips[i].Destination); var deltaTime = trips[i].TripEndTime - trips[i].TripStartTime; // remove trips that are faster than 120km/h if (distance / deltaTime > 200.0f) { trips.RemoveAt(i); i--; } } } return(personRecord); }
public static IEnumerable <SurveyEntry> EnumerateSurvey(string filePath) { using (var reader = new CsvReader(filePath)) { // burn header reader.LoadLine(); long currentPerson = 0; bool validPerson = false; var seperators = new char[] { ',', '\t' }; var toReturn = new SurveyEntry(); while (reader.LoadLine(out var columns)) { if (columns > 19) { TripEntry trip; // the last two digits are the ID for the trip reader.Get(out long personNumber, 19); personNumber = personNumber / 100; if (personNumber != currentPerson) { if (validPerson) { yield return(toReturn); toReturn = new SurveyEntry(); } else { toReturn.Clear(); } currentPerson = personNumber; validPerson = true; } if (!ConvertFloatStringToInt(reader, 24, out trip.Origin) || !ConvertFloatStringToInt(reader, 25, out trip.Destination) || !ConvertMode(reader, 9, out trip.Mode) || !ConvertTime(reader, 4, out trip.TripStartTime) || !ConvertTime(reader, 5, out trip.TripEndTime) || !ConvertFloatStringToInt(reader, 22, out var originInZone) || !ConvertFloatStringToInt(reader, 23, out var destinationInZone) || originInZone < 1f || destinationInZone < 1f || trip.Origin < 0 || trip.Destination < 0) { validPerson = false; } else { toReturn.Add(trip); } } } if (validPerson) { yield return(toReturn); } } }
private static void WriteTransitSurveydata(Network[] networks, Dictionary <int, DensityData> densityData) { var rand = new Random(123546); using (StreamWriter writer = new StreamWriter("SyntheticCellTraces-Transit.csv")) using (StreamWriter writer2 = new StreamWriter("SyntheticCellTracesTest-Transit.csv")) { // Write the headers for both streams // WriteHeaders(writer, false); WriteHeaders(writer2, false); StringBuilder builder = new StringBuilder(); StringBuilder builder2 = new StringBuilder(); foreach (var personRecord in TransitData.StreamTransitRiderDays(@"G:\TMG\Research\Montevideo\NewTransitData\2018.12.04\od_may_2018_vero.csv", networks[0], TransitData.LoadStopToStop( @"G:\TMG\Research\Montevideo\BusNetwork\BusStopsEmmeIDmapping.csv"))) { // Convert into a survey entry SurveyEntry entry = new SurveyEntry(); for (int i = 0; i < personRecord.Count; i++) { entry.Add(new TripEntry() { Origin = GetStopCentroid(networks, personRecord[i].OriginNode, personRecord[i].StartTimeOfDay, rand), Destination = GetStopCentroid(networks, personRecord[i].DestinationNode, personRecord[i].StartTimeOfDay, rand), Mode = 1, TripStartTime = (float)personRecord[i].StartTimeOfDay, TripEndTime = (float)personRecord[i].EndTimeOfDay }); } RecordsFor(builder, builder2, networks, entry, false, densityData); // 80% of the records will be stored in the training set if (rand.NextDouble() < 0.8) { writer.Write(builder); } else { writer2.Write(builder); } builder.Clear(); builder2.Clear(); } } }
public static IEnumerable <SurveyEntry> EnumerateCellTraces(Network am, string stepFilePath, Dictionary <int, DensityData> densityData) { Random r = new Random(); using (var reader = new CsvReader(stepFilePath)) { // burn header reader.LoadLine(); string previousPersonID = null; string previousDate = null; bool validPerson = false; var toReturn = new SurveyEntry(); while (reader.LoadLine(out var columns)) { if (columns >= 7) { reader.Get(out string stepType, 7); reader.Get(out string date, 0); if (stepType == "trip") { reader.Get(out string personID, 1); // if we are starting a new person if (previousPersonID != personID || previousDate != date) { if (validPerson) { yield return(toReturn); validPerson = true; toReturn = new SurveyEntry(); } else { validPerson = true; toReturn.Trips.Clear(); } previousDate = date; previousPersonID = personID; } // only sore the data if they are valid if (validPerson) { reader.Get(out int origin, 3); reader.Get(out int destination, 4); reader.Get(out float tripStart, 5); reader.Get(out float tripEnd, 6); if (!am.PickCentroidInZone(ref origin, r) || !am.PickCentroidInZone(ref destination, r)) { validPerson = false; continue; } toReturn.Add(new TripEntry() { Mode = 0, TripStartTime = tripStart, TripEndTime = tripEnd, Origin = origin, Destination = destination }); } } } } if (validPerson) { yield return(toReturn); } } }
static bool RecordsFor(StringBuilder writer, StringBuilder mainFeatures, Network[] network, SurveyEntry entry, bool applyODInsteadOfMode, Dictionary <int, DensityData> densityData) { /* * Step 1) Find nextTrip * Step 2) Assign Zeros until trip start * Step 3) Step through trip depending on made and set delta distance when the zone changes */ var trips = entry.Trips; int timeStep = 0; const int minutesPerTimeStep = 5; // km/minute const float walkSpeed = 4.0f / 60.0f; const int numberOfSegmentsInADay = (60 * 24) / minutesPerTimeStep; // return back the empty buffer if (entry.Trips.Count <= 0) { return(false); } int currentZone; void EmitNothing() { mainFeatures.Append(",0.0"); timeStep++; } void emitDistance(float distance) { mainFeatures.Append(','); mainFeatures.Append(distance); timeStep++; } for (int i = 0; i < trips.Count; i++) { var startTime = (int)Math.Round(trips[i].TripStartTime, 0) / minutesPerTimeStep; // Step 2 for (; timeStep < startTime && timeStep < numberOfSegmentsInADay;) { // TODO: Change this to emit more than nothing EmitNothing(); } // check to see if the day is over if (timeStep >= numberOfSegmentsInADay) { break; } if (trips[i].Origin != trips[i].Destination) { var timeOfDay = GetTimeOfDay(i); // Step 3 switch (trips[i].Mode) { // auto case 0: { var path = network[timeOfDay].GetFastestPath(trips[i].Origin, trips[i].Destination); if (path == null) { throw new InvalidOperationException($"Unable to find a path between {trips[i].Origin} and {trips[i].Destination}!"); } // go through the path and find the points in time following the path var accTime = 0.0f; var distance = 0.0f; var changeZone = false; currentZone = network[timeOfDay].GetZone(trips[i].Origin); for (int j = 0; j < path.Count && timeStep < numberOfSegmentsInADay; j++) { var travelTime = network[timeOfDay].GetTime(path[j].origin, path[j].destination); var linkDistance = network[timeOfDay].GetDistance(path[j].origin, path[j].destination); var destinationZone = network[timeOfDay].GetZone(path[j].destination); changeZone = changeZone | (currentZone != destinationZone); currentZone = destinationZone; accTime += travelTime; distance += linkDistance; if (accTime >= minutesPerTimeStep) { accTime -= minutesPerTimeStep; // if we have changed zones record how far we have travelled if (changeZone) { emitDistance(distance); distance = 0.0f; changeZone = false; } else { EmitNothing(); } } } // clean up the remainder if (changeZone && timeStep < numberOfSegmentsInADay) { emitDistance(distance); } } break; // transit case 1: { var path = network[timeOfDay].GetPathThroughTransit(trips[i].Origin, trips[i].Destination); if (path == null) { Console.WriteLine($"Unable to find a path between {trips[i].Origin} and {trips[i].Destination}!"); writer.Clear(); return(false); } // go through the path and find the points in time following the path var accTime = 0.0f; var distance = 0.0f; var changeZone = false; currentZone = network[timeOfDay].GetZone(trips[i].Origin); // the first path segment gives us the origin var currentNode = path[0].node; for (int j = 1; j < path.Count && timeStep < numberOfSegmentsInADay; j++) { // in-vehicle segment if (path[j].line != "-") { var subPath = network[timeOfDay].GetTransitTravelOnRouteSegments(path[j].line, currentNode, path[j].node); if (subPath == null) { Console.WriteLine("No path from " + currentNode + " to " + path[j].node + " on transit line " + path[j].line); System.Environment.Exit(0); } for (int k = 0; k < subPath.Count && timeStep < numberOfSegmentsInADay; k++) { var(destinationNode, time) = subPath[k]; var destinationZone = network[timeOfDay].GetZone(destinationNode); changeZone = changeZone | (currentZone != destinationZone); currentZone = destinationZone; accTime += time; // use the full network for distances distance += network[timeOfDay].GetDistance(currentNode, destinationNode); if (accTime >= minutesPerTimeStep) { accTime -= minutesPerTimeStep; if (changeZone) { emitDistance(distance); distance = 0.0f; changeZone = false; } else { EmitNothing(); } } currentNode = destinationNode; } } // aux transit else { // make sure that the next node actually exists while (j < path.Count && !network[timeOfDay].HasNode(path[j].node)) { j++; } var nextNode = j < path.Count ? path[j].node : entry.Trips[i].Destination; var d = network[timeOfDay].GetDistance(currentNode, nextNode); var destinationZone = network[timeOfDay].GetZone(nextNode); changeZone = changeZone | (currentZone != destinationZone); currentZone = destinationZone; distance += d; accTime += d / walkSpeed; if (accTime >= minutesPerTimeStep) { accTime -= minutesPerTimeStep; if (changeZone) { emitDistance(distance); distance = 0.0f; changeZone = false; } else { EmitNothing(); } } currentZone = destinationZone; } currentNode = j < path.Count ? path[j].node : -1; } // clean up the remainder if (changeZone && timeStep < numberOfSegmentsInADay) { emitDistance(distance); } } break; // active transportation case 2: // walk along the road? { var path = network[timeOfDay].GetFastestPath(trips[i].Origin, trips[i].Destination); if (path == null) { throw new InvalidOperationException($"Unable to find a path between {trips[i].Origin} and {trips[i].Destination}!"); } // go through the path and find the points in time following the path var accTime = 0.0f; var distance = 0.0f; var changeZone = false; currentZone = network[timeOfDay].GetZone(trips[i].Origin); for (int j = 0; j < path.Count && timeStep < numberOfSegmentsInADay; j++) { var linkDistance = network[timeOfDay].GetDistance(path[j].origin, path[j].destination); var destinationZone = network[timeOfDay].GetZone(path[j].destination); changeZone = changeZone | (currentZone != destinationZone); currentZone = destinationZone; accTime += linkDistance / walkSpeed; distance += linkDistance; if (accTime >= minutesPerTimeStep) { accTime -= minutesPerTimeStep; // if we have changed zones record how far we have travelled if (changeZone) { emitDistance(distance); distance = 0.0f; changeZone = false; } else { EmitNothing(); } } } // clean up the remainder if (changeZone && timeStep < numberOfSegmentsInADay) { emitDistance(distance); } } break; default: throw new NotImplementedException($"UNKNOWN MODE {trips[i].Mode}"); } } // intrazonal else { // just continue on to the next trip } } if (timeStep > numberOfSegmentsInADay) { throw new InvalidOperationException("There were more time steps emitted than there were time steps in the day!"); } // Finish writing the data for the rest of the day for (; timeStep < numberOfSegmentsInADay;) { EmitNothing(); } for (int i = 0; i < trips.Count; i++) { // make sure the trip starts during the day if (trips[i].Origin != trips[i].Destination && trips[i].TripStartTime < 24 * 60) { if (applyODInsteadOfMode) { writer.Append(trips[i].Origin); writer.Append(','); writer.Append(trips[i].Destination); writer.Append(','); writer.Append(trips[i].TripStartTime); } else { writer.Append(trips[i].Mode); } writer.Append(mainFeatures); // write out the trip specific data (1 if the activity is occurring) for (int j = 0; j < numberOfSegmentsInADay; j++) { writer.Append(trips[i].TripStartTime <= j * minutesPerTimeStep && j * minutesPerTimeStep < trips[i].TripEndTime ? ",1.0" : ",0.0"); } // Write out the density variables for origin then destination (population,employment,household) if (densityData.TryGetValue(network[0].GetZone(entry.Trips[i].Origin), out var originDensity)) { writer.Append(','); writer.Append(originDensity.PopulationDensity); writer.Append(','); writer.Append(originDensity.EmploymentDensity); writer.Append(','); writer.Append(originDensity.HouseholdDensity); } else { writer.Append(",0.0,0.0,0.0"); } if (densityData.TryGetValue(network[0].GetZone(entry.Trips[i].Destination), out var densityDensity)) { writer.Append(','); writer.Append(densityDensity.PopulationDensity); writer.Append(','); writer.Append(densityDensity.EmploymentDensity); writer.Append(','); writer.Append(densityDensity.HouseholdDensity); } else { writer.Append(",0.0,0.0,0.0"); } // write the total distance of the trip writer.Append(','); writer.Append(TripDistance(network[0], entry.Trips[i])); writer.AppendLine(); } } return(true); }