public static byte[] Serialize(string geojson) { var reader = new GeoJsonReader(); var fc = reader.Read <FeatureCollection>(geojson); var bytes = FeatureCollectionConversions.Serialize(fc, GeometryType.Unknown); return(bytes); }
public void TigerRoadsTest() { var geojson = File.ReadAllText("../../../../../../test/data/tiger_roads.geojson"); var reader = new GeoJsonReader(); var fcExpected = reader.Read <FeatureCollection>(geojson); var bytes = FeatureCollectionConversions.Serialize(fcExpected, GeometryType.LineString); var fcActual = FeatureCollectionConversions.Deserialize(bytes); Assert.AreEqual(fcExpected.Count, fcActual.Count); }
public FeatureCollectionConversionsBenchmark() { var point = GeometryRoundtripTests.MakeFeature("POINT (1.2 -2.1)"); var polygon = GeometryRoundtripTests.MakeFeature("POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))"); var polygonZ = GeometryRoundtripTests.MakeFeature("POLYGON Z((30 10 1, 40 40 2, 20 40 3, 10 20 4, 30 10 5))"); var attributes = new Dictionary <string, object>() { ["test1"] = 1, ["test2"] = 1.1, ["test3"] = "test", ["test4"] = true, ["test5"] = "teståöä2", ["test6"] = false, }; var pointWithAttributes = GeometryRoundtripTests.MakeFeature("POINT (1.2 -2.1)", attributes); pointFixture = new GeometryFixture() { fc = GeometryRoundtripTests.MakeFeatureCollection(point), geometryType = GeometryConversions.ToGeometryType(point.Geometry), dimensions = GeometryRoundtripTests.GetDimensions(point.Geometry), }; pointFixture.flatgeobuf = FeatureCollectionConversions.Serialize(pointFixture.fc, pointFixture.geometryType, pointFixture.dimensions); polygonFixture = new GeometryFixture() { fc = GeometryRoundtripTests.MakeFeatureCollection(polygon), geometryType = GeometryConversions.ToGeometryType(polygon.Geometry), dimensions = GeometryRoundtripTests.GetDimensions(polygon.Geometry), }; polygonFixture.flatgeobuf = FeatureCollectionConversions.Serialize(polygonFixture.fc, polygonFixture.geometryType, polygonFixture.dimensions); polygonZFixture = new GeometryFixture() { fc = GeometryRoundtripTests.MakeFeatureCollection(polygonZ), geometryType = GeometryConversions.ToGeometryType(polygonZ.Geometry), dimensions = GeometryRoundtripTests.GetDimensions(polygonZ.Geometry), }; polygonZFixture.flatgeobuf = FeatureCollectionConversions.Serialize(polygonZFixture.fc, polygonZFixture.geometryType, polygonZFixture.dimensions); pointWithAttributesFixture = new GeometryFixture() { fc = GeometryRoundtripTests.MakeFeatureCollection(pointWithAttributes), geometryType = GeometryConversions.ToGeometryType(pointWithAttributes.Geometry), dimensions = GeometryRoundtripTests.GetDimensions(pointWithAttributes.Geometry), }; pointWithAttributesFixture.flatgeobuf = FeatureCollectionConversions.Serialize(pointWithAttributesFixture.fc, pointWithAttributesFixture.geometryType, pointWithAttributesFixture.dimensions); }
public void SerializePointWithAttributes() { FeatureCollectionConversions.Serialize(pointWithAttributesFixture.fc, pointWithAttributesFixture.geometryType, pointWithAttributesFixture.dimensions); }
public void SerializePolygonZ() { FeatureCollectionConversions.Serialize(polygonZFixture.fc, polygonZFixture.geometryType, polygonZFixture.dimensions); }
private async Task RunAsync(CancellationToken stoppingToken) { try { IEnumerable <Feature> GetFeatures() { var maxTrackCount = _configuration.GetValueOrDefault("MaxFeatures", int.MaxValue); var tracks = _db.Tracks.Where(x => x.Id < maxTrackCount) .Where(x => x.UserId != null).OrderBy(x => x.Id); var trackCount = 0; foreach (var track in tracks) { trackCount++; if (stoppingToken.IsCancellationRequested) { break; } if (track.GpxFile == null) { continue; } _logger.LogInformation( $"Track {trackCount + 1}/{maxTrackCount} {track.Id}: {track.GpxFile?.Length} bytes"); // try compressed. string?xml = null; try { using (var memoryStream = new MemoryStream(track.GpxFile)) using (var gzipStream1 = new GZipStream(memoryStream, CompressionMode.Decompress)) using (var gzipStream2 = new GZipStream(gzipStream1, CompressionMode.Decompress)) using (var streamReader = new StreamReader(gzipStream2)) { xml = streamReader.ReadToEnd(); } } catch (Exception e) { } // try uncompressed. if (string.IsNullOrWhiteSpace(xml)) { try { using (var memoryStream = new MemoryStream(track.GpxFile)) using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) using (var streamReader = new StreamReader(gzipStream)) { xml = streamReader.ReadToEnd(); } } catch (Exception e) { } } // read gpx. Feature[]? trackFeatures = null; try { if (!string.IsNullOrWhiteSpace(xml)) { using (var reader = new StringReader(xml)) using (var xmlReader = XmlReader.Create(reader)) { (_, trackFeatures, _) = GpxReader.ReadFeatures(xmlReader, new GpxReaderSettings() { DefaultCreatorIfMissing = "OSM" }, GeometryFactory.Default); } } } catch (Exception e) { _logger.LogError(e, "Unhandled exception while parsing GPX"); } if (trackFeatures != null) { foreach (var tf in trackFeatures) { if (stoppingToken.IsCancellationRequested) { break; } if (tf.Geometry is Point) { continue; } if (tf.Geometry is Polygon) { continue; } if (tf.Geometry is MultiLineString mls) { foreach (var g in mls) { if (g is LineString ls) { if (ls.Count >= 2) { yield return(new Feature(ls, new AttributesTable() { { "track_id", track.Id } })); } } } } else if (tf.Geometry is LineString ls) { if (ls.Count >= 2) { yield return(new Feature(tf.Geometry, new AttributesTable() { { "track_id", track.Id } })); } } } } else { _logger.LogWarning("Failed to parse track: {trackId} - {trackFileName}", track.Id, track.GpxFileName); } } } var outputFile = this._configuration.GetValueOrDefault("OutputFile", "osm-gpx.fbg"); _logger.LogDebug("Building output file: {outputFile}", outputFile); using var outputFileStream = File.Open(outputFile, FileMode.Create); FeatureCollectionConversions.Serialize(outputFileStream, GetFeatures(), FlatGeobuf.GeometryType.LineString); _logger.LogInformation("Built output file: {outputFile}", outputFile); } catch (Exception e) { _logger.LogError(e, $"Unhandled exception exporting GPX tracks"); } }
private async Task RunAsync(CancellationToken stoppingToken) { try { using var scope = _serviceProvider.CreateScope(); var db = scope.ServiceProvider .GetRequiredService <OsmDbContext>(); IEnumerable <Feature> GetFeatures() { var maxTrackCount = _configuration.GetValueOrDefault("MaxFeatures", int.MaxValue); var tracks = db.Tracks.Where(x => x.Id < maxTrackCount) .Where(x => x.UserId != null).OrderBy(x => x.Id); var trackCount = 0; foreach (var track in tracks) { trackCount++; if (stoppingToken.IsCancellationRequested) { break; } if (track.GpxFile == null) { continue; } _logger.LogInformation( "Track {trackCount}/{maxTrackCount} {trackId}: {trackLength} bytes", trackCount + 1, maxTrackCount, track.Id, track.GpxFile?.Length); // build attributes var attributes = new AttributesTable() { { "bdp_id", track.Id }, { "track_id", track.OsmTrackId }, { "user_id", track.UserId ?? -1 }, { "file_name", track.GpxFileName ?? string.Empty }, { "timestamp", (track.OsmTimeStamp ?? DateTime.UnixEpoch).ToUnixTime() }, { "tags", string.Join(',', track.Tags ?? Array.Empty <string>()) } }; // try compressed. string?xml = null; try { using (var memoryStream = new MemoryStream(track.GpxFile)) using (var gzipStream1 = new GZipStream(memoryStream, CompressionMode.Decompress)) using (var gzipStream2 = new GZipStream(gzipStream1, CompressionMode.Decompress)) using (var streamReader = new StreamReader(gzipStream2)) { xml = streamReader.ReadToEnd(); } } catch (Exception e) { } // try uncompressed. if (string.IsNullOrWhiteSpace(xml)) { try { using (var memoryStream = new MemoryStream(track.GpxFile)) using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) using (var streamReader = new StreamReader(gzipStream)) { xml = streamReader.ReadToEnd(); } } catch (Exception e) { } } // read gpx. Feature[]? trackFeatures = null; try { if (!string.IsNullOrWhiteSpace(xml)) { using (var reader = new StringReader(xml)) using (var xmlReader = XmlReader.Create(reader)) { (_, trackFeatures, _) = GpxReader.ReadFeatures(xmlReader, new GpxReaderSettings() { DefaultCreatorIfMissing = "OSM" }, GeometryFactory.Default); } } } catch (Exception e) { _logger.LogError(e, "Unhandled exception while parsing GPX"); } if (trackFeatures != null) { foreach (var tf in trackFeatures) { if (stoppingToken.IsCancellationRequested) { break; } if (tf.Geometry is Point) { continue; } if (tf.Geometry is Polygon) { continue; } if (tf.Geometry is MultiLineString mls) { foreach (var g in mls) { if (g is LineString ls) { if (ls.Count >= 2) { yield return(new Feature(ls, attributes)); } } } } else if (tf.Geometry is LineString ls) { if (ls.Count >= 2) { yield return(new Feature(tf.Geometry, attributes)); } } } } else { _logger.LogWarning("Failed to parse track: {trackId} - {trackFileName}", track.Id, track.GpxFileName); } } } var outputFileInfo = new FileInfo( this._configuration.GetValueOrDefault("OutputFile", "osm-gpx.fbg")); var tempOutputFile = Path.Combine(outputFileInfo.Directory.FullName, ".osm-gpx.fbg.part"); _logger.LogDebug("Building output file: {tempOutputFile}", tempOutputFile); await using var outputFileStream = File.Open(tempOutputFile, FileMode.Create); FeatureCollectionConversions.Serialize(outputFileStream, GetFeatures(), FlatGeobuf.GeometryType.LineString, columns: new [] { new ColumnMeta() { Name = "bdp_id", Type = ColumnType.Long }, new ColumnMeta() { Name = "track_id", Type = ColumnType.Long }, new ColumnMeta() { Name = "user_id", Type = ColumnType.Int }, new ColumnMeta() { Name = "file_name", Type = ColumnType.String }, new ColumnMeta() { Name = "timestamp", Type = ColumnType.Long }, new ColumnMeta() { Name = "tags", Type = ColumnType.String } }.ToList()); if (outputFileInfo.Exists) { outputFileInfo.Delete(); } File.Move(tempOutputFile, outputFileInfo.FullName); _logger.LogDebug("Moved output file: {tempOutputFile}->{outputFile}", tempOutputFile, outputFileInfo.FullName); _logger.LogInformation("Built output file: {outputFile}", outputFileInfo.FullName); } catch (Exception e) { _logger.LogError(e, "Unhandled exception exporting GPX tracks"); } }