private GtfsFeed GetFeed(GtfsFeedParserVisitor parser) { GtfsFeed feed = new GtfsFeed(); feed.Accept(parser); return(parser.Feed); }
public static GtfsFeed ReadGtfs(this Stream stream, GtfsFileOptions options = GtfsFileOptions.All) { GtfsFeed feed = null; stream.ReadGtfsAsync(options).ContinueWith(g => feed = g.Result).Wait(); return(feed); }
/// <summary> /// Facade for parsing GTFS feed /// </summary> public GtfsFeed Parse(string feedPath) { GtfsFeedParserVisitor parser = new GtfsFeedParserVisitor(feedPath); GtfsFeed feed = new GtfsFeed(); feed.Accept(parser); return parser.Feed; }
public RequiredFieldVisitorTest(ITestOutputHelper output, VisitorPathFixture visitorPathFixture) { _output = output; _visitorPathFixture = visitorPathFixture; _sut = new RequiredFieldVisitor(); _parsedFeedGood = GetParsedFeed(_visitorPathFixture.GoodFeedPath); _parsedFeedNonExisting = GetParsedFeed(_visitorPathFixture.NonExistingFeedPath); }
private GtfsFeed GetParsedFeed(string feedPath) { GtfsFeed feed = new GtfsFeed(); GtfsFeedParserVisitor parseVisitor = new GtfsFeedParserVisitor(feedPath); feed.Accept(parseVisitor); return(parseVisitor.Feed); }
public void ReadITFromWeb() { var zipUrl = MethodInfo.GetCurrentMethod().GetTestProperty("url"); GtfsFeed gtfs = null; ReadGtfsFromWeb(zipUrl).ContinueWith(g => gtfs = g.Result).Wait(); Assert.IsTrue(gtfs.Shapes.features.Length > 0, "Shapes should not be empty."); SerializeGtfs(gtfs, "intercity-transit.json"); }
/// <summary> /// Facade for parsing GTFS feed /// </summary> public GtfsFeed Parse(string feedPath) { GtfsFeedParserVisitor parser = new GtfsFeedParserVisitor(feedPath); GtfsFeed feed = new GtfsFeed(); feed.Accept(parser); return(parser.Feed); }
public void EmptyFeedShouldBeInvalid() { RequiredFileVisitor sut = new RequiredFileVisitor(); GtfsFeed emptyFeed = new GtfsFeed(); emptyFeed.Accept(sut); Assert.False(sut.IsValid); }
public GtfsFeedValidatorTest(ITestOutputHelper output, VisitorPathFixture visitorPathFixture) { _output = output; _visitorPathFixture = visitorPathFixture; _parsedFeedGood = new GtfsFeedParser().Parse(_visitorPathFixture.GoodFeedPath); _parsedFeedBad = new GtfsFeedParser().Parse(_visitorPathFixture.BadFeedPath); _parsedFeedNonExisting = new GtfsFeedParser().Parse(_visitorPathFixture.NonExistingFeedPath); _sut = new GtfsFeedValidator(); }
private async Task <GtfsFeed> ReadGtfsFromWeb(string url = null) { GtfsFeed gtfs = null; using (var httpClient = new HttpClient()) using (var stream = await httpClient.GetStreamAsync(url)) { stream.ReadGtfsAsync().ContinueWith(t => gtfs = t.Result).Wait(); } RunTestsOnGtfs(gtfs); return(gtfs); }
public void ReadSampleGtfs() { var zipPath = MethodInfo.GetCurrentMethod().GetTestProperty("gtfsFile"); Assert.IsTrue(File.Exists(zipPath), "File not found: {0}", Path.GetFullPath(zipPath)); GtfsFeed gtfs = null; using (FileStream stream = File.Open(zipPath, FileMode.Open, FileAccess.Read)) { stream.ReadGtfsAsync().ContinueWith(t => gtfs = t.Result).Wait(); } RunTestsOnGtfs(gtfs); }
public void ParserReturnParsedGtfsFeedWithAtLeastOneRecordInEachProperty() { GtfsFeed feed = new GtfsFeed(); feed.Accept(_sut); _output.WriteLine("_sut.Feed.Count: {0}", _sut.Feed.Agencies.Count); Assert.NotEmpty(_sut.Feed.Agencies); _output.WriteLine("_sut.Feed.Count: {0}", _sut.Feed.Calendars.Count); Assert.NotEmpty(_sut.Feed.Calendars); _output.WriteLine("_sut.Feed.CalendarDates.Count: {0}", _sut.Feed.CalendarDates.Count); Assert.NotEmpty(_sut.Feed.CalendarDates); _output.WriteLine("_sut.Feed.FareAttributes.Count: {0}", _sut.Feed.FareAttributes.Count); Assert.NotEmpty(_sut.Feed.FareAttributes); _output.WriteLine("_sut.Feed.FareRules.Count: {0}", _sut.Feed.FareRules.Count); Assert.NotEmpty(_sut.Feed.FareRules); _output.WriteLine("_sut.Feed.FeedInfos.Count: {0}", _sut.Feed.FeedInfos.Count); Assert.NotEmpty(_sut.Feed.FeedInfos); _output.WriteLine("_sut.Feed.Frequencies.Count: {0}", _sut.Feed.Frequencies.Count); Assert.NotEmpty(_sut.Feed.Frequencies); _output.WriteLine("_sut.Feed.Routes.Count: {0}", _sut.Feed.Routes.Count); Assert.NotEmpty(_sut.Feed.Routes); _output.WriteLine("_sut.Feed.Shapes.Count: {0}", _sut.Feed.Shapes.Count); Assert.NotEmpty(_sut.Feed.Shapes); _output.WriteLine("_sut.Feed.Stops.Count: {0}", _sut.Feed.Stops.Count); Assert.NotEmpty(_sut.Feed.Stops); _output.WriteLine("_sut.Feed.StopTimes.Count: {0}", _sut.Feed.StopTimes.Count); Assert.NotEmpty(_sut.Feed.StopTimes); _output.WriteLine("_sut.Feed.Transfers.Count: {0}", _sut.Feed.Transfers.Count); Assert.NotEmpty(_sut.Feed.Transfers); _output.WriteLine("_sut.Feed.Trips.Count: {0}", _sut.Feed.Trips.Count); Assert.NotEmpty(_sut.Feed.Trips); }
private void SerializeGtfs(GtfsFeed gtfs, string filename) { var serializer = new JsonSerializer() { Formatting = Formatting.Indented, NullValueHandling = NullValueHandling.Ignore }; filename = Path.Combine(TestContext.ResultsDirectory, filename); Task.Run(() => { using (var writer = new StreamWriter(filename)) { serializer.Serialize(writer, gtfs); } }).Wait(); TestContext.AddResultFile(filename); }
public void ParserDoesntReturnEmptyGtfsFeedProperties() { GtfsFeed feed = _sut.Parse(FEED_PATH); Assert.NotNull(feed.Agencies); Assert.NotNull(feed.Calendars); Assert.NotNull(feed.CalendarDates); Assert.NotNull(feed.FareAttributes); Assert.NotNull(feed.FareRules); Assert.NotNull(feed.FeedInfos); Assert.NotNull(feed.Frequencies); Assert.NotNull(feed.Routes); Assert.NotNull(feed.Shapes); Assert.NotNull(feed.Stops); Assert.NotNull(feed.StopTimes); Assert.NotNull(feed.Transfers); Assert.NotNull(feed.Trips); }
public async Task SerializeAsync(Stream zipStream, GtfsFeed feed) { await Task.Yield(); using (var archive = new ZipArchive(zipStream, ZipArchiveMode.Create, true)) { using (var entryStream = archive.CreateEntry("agency.txt").Open()) { _csvSerializer.SerializeAgencies(entryStream, feed.Agencies); } using (var entryStream = archive.CreateEntry("stops.txt").Open()) { _csvSerializer.SerializeStops(entryStream, feed.Stops); } using (var entryStream = archive.CreateEntry("routes.txt").Open()) { _csvSerializer.SerializeRoutes(entryStream, feed.Routes); } using (var entryStream = archive.CreateEntry("trips.txt").Open()) { _csvSerializer.SerializeTrips(entryStream, feed.Trips); } using (var entryStream = archive.CreateEntry("stop_times.txt").Open()) { _csvSerializer.SerializeStopTimes(entryStream, feed.StopTimes); } using (var entryStream = archive.CreateEntry("calendar.txt").Open()) { _csvSerializer.SerializeCalendar(entryStream, feed.Calendar); } using (var entryStream = archive.CreateEntry("shapes.txt").Open()) { _csvSerializer.SerializeShapes(entryStream, feed.Shapes); } } }
public void ParserDoesntReturnEmptyGtfsFeedProperties() { GtfsFeed feed = new GtfsFeed(); feed.Accept(_sut); feed = _sut.Feed; Assert.NotNull(feed.Agencies); Assert.NotNull(feed.Calendars); Assert.NotNull(feed.CalendarDates); Assert.NotNull(feed.FareAttributes); Assert.NotNull(feed.FareRules); Assert.NotNull(feed.FeedInfos); Assert.NotNull(feed.Frequencies); Assert.NotNull(feed.Routes); Assert.NotNull(feed.Shapes); Assert.NotNull(feed.Stops); Assert.NotNull(feed.StopTimes); Assert.NotNull(feed.Transfers); Assert.NotNull(feed.Trips); }
/// <summary> /// A common set of tests to run on a GTFS feed. /// </summary> /// <param name="gtfs"></param> private static void RunTestsOnGtfs(GtfsFeed gtfs) { // Test for the different required tables. Assert.IsNotNull(gtfs, "The GTFS object cannot be null"); Assert.IsNotNull(gtfs.Agency, "The agency list cannot be null."); Assert.IsNotNull(gtfs.Stops, "The stops list cannot be null."); Assert.IsNotNull(gtfs.Routes, "The routes list cannot be null."); Assert.IsNotNull(gtfs.Trips, "The trips list cannot be null."); Assert.IsNotNull(gtfs.StopTimes, "The stop_times list cannot be null."); Assert.IsNotNull(gtfs.Calendar, "The calendar list cannot be null."); // Test agency foreach (var agency in gtfs.Agency) { Assert.IsNotNull(agency.agency_name, "agency.agency_name cannot be null."); Assert.IsNotNull(agency.agency_url, "agency.agency_url cannot be null."); Assert.IsNotNull(agency.agency_timezone, "agency.agency_timezone cannot be null."); } foreach (var stop in gtfs.Stops.features) { Assert.IsNotNull(stop.id, "stop.stop_id cannot be null."); Assert.IsNotNull(stop.properties.stop_name, "stop.stop_name cannot be null."); // Test to see if the lat is a "valid WGS 84 latitude". Assert.IsTrue(stop.geometry.coordinates[1].IsValidLatitude()); Assert.IsTrue(stop.geometry.coordinates[0].IsValidLongitude()); } foreach (var route in gtfs.Routes) { Assert.IsNotNull(route.route_id, "route.route_id cannot be null."); Assert.IsNotNull(route.route_short_name, "route.route_short_name cannot be null."); Assert.IsNotNull(route.route_long_name, "route.route_long_name cannot be null."); } CollectionAssert.AllItemsAreUnique(gtfs.Trips.Select(g => g.trip_id).ToArray()); foreach (var trip in gtfs.Trips) { Assert.IsNotNull(trip.route_id, "trip.route_id cannot be null."); Assert.IsNotNull(trip.service_id, "trip.service_id cannot be null."); Assert.IsNotNull(trip.trip_id, "trip.trip_id cannot be null."); } foreach (var stopTime in gtfs.StopTimes) { Assert.IsNotNull(stopTime.trip_id, "stopTime.trip_id cannot be null."); Assert.IsNotNull(stopTime.stop_id, "stopTime.stop_id cannot be null."); Assert.IsNotNull(stopTime.stop_sequence, "stopTime.stop_sequence cannot be null."); } CollectionAssert.AllItemsAreUnique(gtfs.Calendar.Select(g => g.service_id).ToArray()); foreach (var item in gtfs.Calendar) { Assert.IsNotNull(item.service_id, "item.service_id cannot be null."); Assert.IsNotNull(item.start_date, "item.start_date cannot be null."); Assert.IsNotNull(item.end_date, "item.end_date cannot be null."); } if (gtfs.CalendarDates != null) { CollectionAssert.AllItemsAreUnique(gtfs.CalendarDates.Select(g => string.Concat(g.service_id, g.date)).ToArray(), "Each (service_id, date) pair can only appear once in calendar_dates.txt."); foreach (var item in gtfs.CalendarDates) { Assert.IsNotNull(item.service_id, "item.service_id cannot be null."); Assert.IsNotNull(item.date, "item.date cannot be null."); } } if (gtfs.FareAttributes != null) { CollectionAssert.AllItemsAreUnique(gtfs.FareAttributes.Select(g => g.fare_id).ToArray()); foreach (var item in gtfs.FareAttributes) { Assert.IsNotNull(item.fare_id, "item.fare_id cannot be null."); Assert.IsTrue(item.price >= 0); Assert.IsTrue(!item.transfers.HasValue || (item.transfers.Value.IsInRange(0, 2)), "Number of transfers must be between 0 and 2 if provided at all."); } } if (gtfs.FareRules != null) { CollectionAssert.AllItemsAreNotNull(gtfs.FareRules.Select(g => g.fare_id).ToArray(), "FareRule.fare_id cannot be null."); } if (gtfs.Shapes != null) { Assert.IsTrue(gtfs.Shapes.features != null && gtfs.Shapes.features.Length > 0, "Shapes property is not null but does not contain any features."); foreach (var item in gtfs.Shapes.features) { Assert.IsNotNull(item.id, "shape_id cannot be null."); Assert.IsTrue(((ShapeFeature)item).geometry.coordinates.HasValidLongitudesAndLatitudes()); } } }
/// <summary> /// Reads a GTFS zip archive stream and converts it into a <see cref="GtfsFeed"/>. /// </summary> /// <param name="stream">A zip archive <see cref="Stream"/> containing General Transit Feet Specification data.</param> /// <returns>A <see cref="GtfsFeed"/> representation of the contents of <paramref name="stream"/>.</returns> public static GtfsFeed ReadGtfs(this Stream stream) { GtfsFeed feed = null; if (stream.CanSeek) { var zip = new ZipFile(stream); feed = new GtfsFeed { Agency = zip.ParseCsv <Agency>("agency.txt", true), Stops = zip.ParseCsv <Stop>("stops.txt", true), Routes = zip.ParseCsv <Route>("routes.txt", true), Trips = zip.ParseCsv <Trip>("trips.txt", true), StopTimes = zip.ParseCsv <StopTime>("stop_times.txt", true), Calendar = zip.ParseCsv <Calendar>("calendar.txt", true), CalendarDates = zip.ParseCsv <CalendarDate>("calendar_dates.txt"), FareAttributes = zip.ParseCsv <FareAttribute>("fare_attributes.txt"), FareRules = zip.ParseCsv <FareRule>("fare_rules.txt"), Shapes = zip.ParseCsv <Shape>("shapes.txt"), Frequencies = zip.ParseCsv <Frequency>("frequencies.txt"), Transfers = zip.ParseCsv <Transfer>("transfers.txt"), }; var feedInfo = zip.ParseCsv <FeedInfo>("feed_info.txt"); feed.FeedInfo = feedInfo != null?feedInfo.FirstOrDefault() : null; } else { var txtRe = new Regex(@"\.txt$", RegexOptions.IgnoreCase); feed = new GtfsFeed(); // See https://github.com/icsharpcode/SharpZipLib/wiki/Zip-Samples#wiki-anchorUnpackZIS var zipInputStream = new ZipInputStream(stream); var zipEntry = zipInputStream.GetNextEntry(); while (zipEntry != null) { if (zipEntry.IsFile && txtRe.IsMatch(zipEntry.Name)) { byte[] buffer = new byte[4096]; using (var memStream = new MemoryStream()) { StreamUtils.Copy(zipInputStream, memStream, buffer); memStream.Position = 0; switch (zipEntry.Name) { case "agency.txt": feed.Agency = memStream.ParseCsv <Agency>(); break; case "stops.txt": feed.Stops = memStream.ParseCsv <Stop>(); break; case "routes.txt": feed.Routes = memStream.ParseCsv <Route>(); break; case "trips.txt": feed.Trips = memStream.ParseCsv <Trip>(); break; case "stop_times.txt": feed.StopTimes = memStream.ParseCsv <StopTime>(); break; case "calendar.txt": feed.Calendar = memStream.ParseCsv <Calendar>(); break; case "calendar_dates.txt": feed.CalendarDates = memStream.ParseCsv <CalendarDate>(); break; case "fare_attributes.txt": feed.FareAttributes = memStream.ParseCsv <FareAttribute>(); break; case "fare_rules.txt": feed.FareRules = memStream.ParseCsv <FareRule>(); break; case "shapes.txt": feed.Shapes = memStream.ParseCsv <Shape>(); break; case "frequencies.txt": feed.Frequencies = memStream.ParseCsv <Frequency>(); break; case "transfers.txt": feed.Transfers = memStream.ParseCsv <Transfer>(); break; default: break; } } } zipEntry = zipInputStream.GetNextEntry(); } } return(feed); }
/// <summary> /// Retrieves GTFS data for a specified agency. /// Data will be retrieved from an in-memory list if available and up-to-date. /// Otherwise the GTFS data will be requested from the GTFS Data Exchange website /// and stored for future use. /// </summary> /// <param name="agencyId">The GTFS-Data-Exchange agency identifier.</param> /// <param name="lastModified">If an "If-Modified-Since" header is present, that value can be used here. Otherwise omit.</param> /// <param name="etags">If the HTTP request contains "If-None-Match", it can be passed in here. Otherwise it can be omitted.</param> /// <returns>Returns a <see cref="FeedRequestResponse"/>.</returns> /// <exception cref="ArgumentException">Thrown if <paramref name="agencyId"/> is <see langword="null"/> or consists only of whitespace.</exception> /// <exception cref="AgencyQueryException">Thrown if the query to gtfs-data-exchange for information fails.</exception> public FeedRequestResponse GetGtfs(string agencyId, DateTimeOffset?lastModified = default(DateTimeOffset?), IEnumerable <EntityTagHeaderValue> etags = null) { if (string.IsNullOrWhiteSpace(agencyId)) { throw new ArgumentException("The agencyId was not provided."); } // Get the record with the matching agency ID. var feedRecord = _feedList.FirstOrDefault(r => string.Compare(r.AgencyId, agencyId, true) == 0); // Check to see if there are matching Etags... if (feedRecord != null && feedRecord.Etag != null && etags != null) { var match = etags.FirstOrDefault(et => et == feedRecord.Etag); if (match != null) { return(new FeedRequestResponse { NotModified = true }); } } HttpClient client = null; GtfsFeed gtfs = feedRecord != null ? feedRecord.GtfsData : null; EntityTagHeaderValue outEtag = null; AgencyResponse agencyResponse = null; FeedRequestResponse output = null; try { // Check for an existing record... // Check online to see if there is a newer feed available. // This will be skipped if there is no matching GTFS feed stored for the specified agency. const string urlFmt = "http://www.gtfs-data-exchange.com/api/agency?agency={0}"; Uri uri = new Uri(string.Format(urlFmt, agencyId)); client = new HttpClient(); client.DefaultRequestHeaders.IfModifiedSince = lastModified; if (etags != null) { foreach (var etag in etags) { client.DefaultRequestHeaders.IfNoneMatch.Add(etag); } } client.GetAsync(uri).ContinueWith((t) => { outEtag = t.Result.Headers.ETag; if (t.Result.StatusCode == HttpStatusCode.NotModified) { output = new FeedRequestResponse { NotModified = true }; } else if (t.Result.StatusCode == HttpStatusCode.OK) { t.Result.Content.ReadAsStreamAsync().ContinueWith(streamTask => { using (var streamReader = new StreamReader(streamTask.Result)) using (var jsonReader = new JsonTextReader(streamReader)) { var serializer = JsonSerializer.Create(); agencyResponse = serializer.Deserialize <AgencyResponse>(jsonReader); } }).Wait(); } }).Wait(); // If the request for GTFS info returned a "Not Modified" response, return now. if (output == null) { if (agencyResponse.status_code != 200) { throw new AgencyQueryException(agencyResponse); } if (lastModified.HasValue && lastModified.Value >= agencyResponse.data.agency.date_last_updated.FromJSDateToDateTimeOffset()) { if (feedRecord == null) { feedRecord = new FeedRecord { DateLastUpdated = lastModified.Value, Etag = outEtag }; } } else { if (feedRecord == null || (agencyResponse.data.agency.date_last_updated.FromJSDateToDateTimeOffset() > feedRecord.DateLastUpdated)) { // Get the GTFS file... Uri zipUri = new Uri(String.Join("/", agencyResponse.data.agency.dataexchange_url.TrimEnd('/'), "latest.zip")); // TODO: make the request and parse the GTFS... if (client == null) { client = new HttpClient(); } else { client.DefaultRequestHeaders.Clear(); } client.GetStreamAsync(zipUri).ContinueWith(t => { t.Result.ReadGtfsAsync().ContinueWith(gtfsTask => { gtfs = gtfsTask.Result; // Delete the existing feedRecord. if (feedRecord != null) { _feedList.Remove(feedRecord); } feedRecord = new FeedRecord { GtfsData = gtfs, AgencyId = agencyId, DateLastUpdated = agencyResponse.data.agency.date_last_updated.FromJSDateToDateTimeOffset(), Etag = outEtag }; // Add the new GTFS feed data to the in-memory collection. _feedList.Add(feedRecord); }).Wait(); }).Wait(); } } } } finally { if (client != null) { client.Dispose(); } } return(new FeedRequestResponse { FeedRecord = feedRecord }); }
public GtfsFeedParserVisitor(Stream dataStream) { _zipData = new ZipArchive(dataStream); Feed = new GtfsFeed(); }
public GtfsFeedParserVisitor(string feedPath) { _feedPath = feedPath; Feed = new GtfsFeed(); }