private async Task Start() { var settings = Settings(); var connectionPolicy = new ConnectionPolicy { ConnectionMode = ConnectionMode.Direct, ConnectionProtocol = Protocol.Tcp, MaxConnectionLimit = 100 }; this.client = new DocumentClient(new Uri(settings.CosmosEndpointUri), settings.CosmosPrimaryKey, connectionPolicy); var gtfsDataFolder = await DownloadGtfsData(settings); await CreateCollectionIfNotExists(settings); // Read the downloaded. None of the data is processed here, it's // just loaded into objects to be processed later. var rawData = new TransitFeedData().Read(gtfsDataFolder); // Convert the transit feed from raw data objects to the structure // to be uploaded to Cosmos. var processedData = new ProcessedTransitFeedData().Process(rawData); await WriteDataToCosmos(settings.CosmosDatabaseName, settings.CosmosCollectionName, processedData); }
/// <summary> /// Processes the TransitFeedData into a list of Trips and Stops to be /// uploaded to Cosmos DB. /// </summary> /// <remarks> /// The GTFS data is in the following structure. /// /// +--------+ /// | Routes | /// +--------+ /// ^ +------+ /// | | | /// +--------+ +------------+ +-------+ | /// | Trips |<--| Stop Times |-->| Stops |<-+ /// +--------+ +------------+ +-------+ /// | /// +-----------------+ /// | | /// v v /// +----------+ +----------------+ /// | Calendar | | Calendar Dates | /// +----------+ +----------------+ /// /// To most easily add references to the data as it processed, its /// processed in the following order: /// - Routes /// - Calendar /// - Calendar Dates /// - Trips /// - Stops /// - Stop Times /// </remarks> public ProcessedTransitFeedData Process(TransitFeedData data) { // Routes foreach (var transitFeedRoute in data.Routes) { var id = transitFeedRoute.Key; var routeData = transitFeedRoute.Value; var route = new Route { Id = id, ShortName = routeData.ShortName, LongName = routeData.LongName, Type = routeData.Type, Color = routeData.Color }; _routes.Add(id, route); } // Calendar foreach (var transitFeedCalendar in data.Calendar) { var serviceId = transitFeedCalendar.Key; var calendarData = transitFeedCalendar.Value; var calendar = new ServiceCalendar { Monday = calendarData.Monday, Tuesday = calendarData.Tuesday, Wednesday = calendarData.Wednesday, Thursday = calendarData.Thursday, Friday = calendarData.Friday, Saturday = calendarData.Saturday, Sunday = calendarData.Sunday, StartDate = calendarData.StartDate, EndDate = calendarData.EndDate }; _calendars.Add(serviceId, calendar); } // Calendar Exception foreach (var transitFeedCalendarException in data.CalendarExceptions) { var serviceId = transitFeedCalendarException.ServiceId; var calendarException = new ServiceCalendarException { Date = transitFeedCalendarException.Date, ExceptionType = transitFeedCalendarException.ExceptionType }; if (!_calendarExceptions.ContainsKey(serviceId)) { _calendarExceptions.Add(serviceId, new List <ServiceCalendarException>()); } _calendarExceptions[serviceId].Add(calendarException); } // Trips foreach (var transitFeedTrip in data.Trips) { var id = transitFeedTrip.Key; var tripData = transitFeedTrip.Value; var trip = new Trip { Id = id, Headsign = tripData.Headsign, Route = _routes[tripData.RouteId], Calendar = _calendars.GetValueOrDefault(tripData.ServiceId), CalendarExceptions = _calendarExceptions.GetValueOrDefault(tripData.ServiceId) }; _trips.Add(id, trip); Trips.Add(trip); } // Stops foreach (var transitFeedStop in data.Stops) { var id = transitFeedStop.Key; var stopData = transitFeedStop.Value; var stop = new Stop { Id = id, Code = stopData.Code, Name = stopData.Name, Platform = stopData.Platform, Location = new StopLocation { Latitude = stopData.Latitude, Longitude = stopData.Longitude } }; _stops.Add(id, stop); } // Put a reference to the parent stop in each stop, and build out a // list of all of the top level stops. foreach (var stop in _stops) { var parentId = data.Stops[stop.Value.Id].ParentId; if (parentId is object) { // If this has a parent, add references between the parent // and the child. var parent = _stops[parentId]; stop.Value.Parent = parent; if (parent.Children is null) { parent.Children = new HashSet <Stop>(); } parent.Children.Add(stop.Value); } else { // If this does not have a parent, add it to the output // public list of stops Stops.Add(stop.Value); } } // Stop Times foreach (var transitFeedStopTime in data.StopTimes) { var stopId = transitFeedStopTime.StopId; var stop = _stops[stopId]; var stopTopLevel = HighestParentStop(stop); var stopTopLevelId = stopTopLevel.Id; var stopTime = new StopTime { StopTopLevel = stopTopLevelId, Stop = stopId, ArrivalTime = transitFeedStopTime.ArrivalTime, DepartureTime = transitFeedStopTime.DepartureTime, Sequence = transitFeedStopTime.Sequence, PickupType = transitFeedStopTime.PickupType, DropOffType = transitFeedStopTime.DropOffType }; // StopTimes only relate to a stop and a time. Add a reference // for this StopTime to both of those. No list is maintained. var trip = _trips[transitFeedStopTime.TripId]; trip.StopTimes.Add(stopTime); AddRouteTypeToStopServicedByList(stop, trip.Route.Type); } return(this); }