public static byte[] Serialize(string geojson)
{
var reader = new GeoJsonReader();
var fc = reader.Read <FeatureCollection>(geojson);
var bytes = FeatureCollectionConversions.Serialize(fc, GeometryType.Unknown);
return(bytes);
}
public void TigerRoadsTest()
{
var geojson = File.ReadAllText("../../../../../../test/data/tiger_roads.geojson");
var reader = new GeoJsonReader();
var fcExpected = reader.Read <FeatureCollection>(geojson);
var bytes = FeatureCollectionConversions.Serialize(fcExpected, GeometryType.LineString);
var fcActual = FeatureCollectionConversions.Deserialize(bytes);
Assert.AreEqual(fcExpected.Count, fcActual.Count);
}
public FeatureCollectionConversionsBenchmark()
{
var point = GeometryRoundtripTests.MakeFeature("POINT (1.2 -2.1)");
var polygon = GeometryRoundtripTests.MakeFeature("POLYGON ((30 10, 40 40, 20 40, 10 20, 30 10))");
var polygonZ = GeometryRoundtripTests.MakeFeature("POLYGON Z((30 10 1, 40 40 2, 20 40 3, 10 20 4, 30 10 5))");
var attributes = new Dictionary <string, object>()
{
["test1"] = 1,
["test2"] = 1.1,
["test3"] = "test",
["test4"] = true,
["test5"] = "teståöä2",
["test6"] = false,
};
var pointWithAttributes = GeometryRoundtripTests.MakeFeature("POINT (1.2 -2.1)", attributes);
pointFixture = new GeometryFixture()
{
fc = GeometryRoundtripTests.MakeFeatureCollection(point),
geometryType = GeometryConversions.ToGeometryType(point.Geometry),
dimensions = GeometryRoundtripTests.GetDimensions(point.Geometry),
};
pointFixture.flatgeobuf = FeatureCollectionConversions.Serialize(pointFixture.fc, pointFixture.geometryType, pointFixture.dimensions);
polygonFixture = new GeometryFixture()
{
fc = GeometryRoundtripTests.MakeFeatureCollection(polygon),
geometryType = GeometryConversions.ToGeometryType(polygon.Geometry),
dimensions = GeometryRoundtripTests.GetDimensions(polygon.Geometry),
};
polygonFixture.flatgeobuf = FeatureCollectionConversions.Serialize(polygonFixture.fc, polygonFixture.geometryType, polygonFixture.dimensions);
polygonZFixture = new GeometryFixture()
{
fc = GeometryRoundtripTests.MakeFeatureCollection(polygonZ),
geometryType = GeometryConversions.ToGeometryType(polygonZ.Geometry),
dimensions = GeometryRoundtripTests.GetDimensions(polygonZ.Geometry),
};
polygonZFixture.flatgeobuf = FeatureCollectionConversions.Serialize(polygonZFixture.fc, polygonZFixture.geometryType, polygonZFixture.dimensions);
pointWithAttributesFixture = new GeometryFixture()
{
fc = GeometryRoundtripTests.MakeFeatureCollection(pointWithAttributes),
geometryType = GeometryConversions.ToGeometryType(pointWithAttributes.Geometry),
dimensions = GeometryRoundtripTests.GetDimensions(pointWithAttributes.Geometry),
};
pointWithAttributesFixture.flatgeobuf = FeatureCollectionConversions.Serialize(pointWithAttributesFixture.fc, pointWithAttributesFixture.geometryType, pointWithAttributesFixture.dimensions);
}
public void SerializePointWithAttributes()
{
FeatureCollectionConversions.Serialize(pointWithAttributesFixture.fc, pointWithAttributesFixture.geometryType, pointWithAttributesFixture.dimensions);
}
public void SerializePolygonZ()
{
FeatureCollectionConversions.Serialize(polygonZFixture.fc, polygonZFixture.geometryType, polygonZFixture.dimensions);
}
private async Task RunAsync(CancellationToken stoppingToken)
{
try
{
IEnumerable <Feature> GetFeatures()
{
var maxTrackCount = _configuration.GetValueOrDefault("MaxFeatures", int.MaxValue);
var tracks = _db.Tracks.Where(x => x.Id < maxTrackCount)
.Where(x => x.UserId != null).OrderBy(x => x.Id);
var trackCount = 0;
foreach (var track in tracks)
{
trackCount++;
if (stoppingToken.IsCancellationRequested)
{
break;
}
if (track.GpxFile == null)
{
continue;
}
_logger.LogInformation(
$"Track {trackCount + 1}/{maxTrackCount} {track.Id}: {track.GpxFile?.Length} bytes");
// try compressed.
string?xml = null;
try {
using (var memoryStream = new MemoryStream(track.GpxFile))
using (var gzipStream1 = new GZipStream(memoryStream, CompressionMode.Decompress))
using (var gzipStream2 = new GZipStream(gzipStream1, CompressionMode.Decompress))
using (var streamReader = new StreamReader(gzipStream2)) {
xml = streamReader.ReadToEnd();
}
}
catch (Exception e) {
}
// try uncompressed.
if (string.IsNullOrWhiteSpace(xml))
{
try {
using (var memoryStream = new MemoryStream(track.GpxFile))
using (var gzipStream =
new GZipStream(memoryStream, CompressionMode.Decompress))
using (var streamReader = new StreamReader(gzipStream)) {
xml = streamReader.ReadToEnd();
}
}
catch (Exception e) {
}
}
// read gpx.
Feature[]? trackFeatures = null;
try {
if (!string.IsNullOrWhiteSpace(xml))
{
using (var reader = new StringReader(xml))
using (var xmlReader = XmlReader.Create(reader)) {
(_, trackFeatures, _) =
GpxReader.ReadFeatures(xmlReader, new GpxReaderSettings()
{
DefaultCreatorIfMissing = "OSM"
}, GeometryFactory.Default);
}
}
}
catch (Exception e) {
_logger.LogError(e, "Unhandled exception while parsing GPX");
}
if (trackFeatures != null)
{
foreach (var tf in trackFeatures)
{
if (stoppingToken.IsCancellationRequested)
{
break;
}
if (tf.Geometry is Point)
{
continue;
}
if (tf.Geometry is Polygon)
{
continue;
}
if (tf.Geometry is MultiLineString mls)
{
foreach (var g in mls)
{
if (g is LineString ls)
{
if (ls.Count >= 2)
{
yield return(new Feature(ls, new AttributesTable()
{
{ "track_id", track.Id }
}));
}
}
}
}
else if (tf.Geometry is LineString ls)
{
if (ls.Count >= 2)
{
yield return(new Feature(tf.Geometry, new AttributesTable()
{
{ "track_id", track.Id }
}));
}
}
}
}
else
{
_logger.LogWarning("Failed to parse track: {trackId} - {trackFileName}", track.Id,
track.GpxFileName);
}
}
}
var outputFile = this._configuration.GetValueOrDefault("OutputFile", "osm-gpx.fbg");
_logger.LogDebug("Building output file: {outputFile}", outputFile);
using var outputFileStream = File.Open(outputFile, FileMode.Create);
FeatureCollectionConversions.Serialize(outputFileStream, GetFeatures(), FlatGeobuf.GeometryType.LineString);
_logger.LogInformation("Built output file: {outputFile}", outputFile);
}
catch (Exception e)
{
_logger.LogError(e, $"Unhandled exception exporting GPX tracks");
}
}
private async Task RunAsync(CancellationToken stoppingToken)
{
try
{
using var scope = _serviceProvider.CreateScope();
var db = scope.ServiceProvider
.GetRequiredService <OsmDbContext>();
IEnumerable <Feature> GetFeatures()
{
var maxTrackCount = _configuration.GetValueOrDefault("MaxFeatures", int.MaxValue);
var tracks = db.Tracks.Where(x => x.Id < maxTrackCount)
.Where(x => x.UserId != null).OrderBy(x => x.Id);
var trackCount = 0;
foreach (var track in tracks)
{
trackCount++;
if (stoppingToken.IsCancellationRequested)
{
break;
}
if (track.GpxFile == null)
{
continue;
}
_logger.LogInformation(
"Track {trackCount}/{maxTrackCount} {trackId}: {trackLength} bytes",
trackCount + 1, maxTrackCount, track.Id, track.GpxFile?.Length);
// build attributes
var attributes = new AttributesTable()
{
{ "bdp_id", track.Id },
{ "track_id", track.OsmTrackId },
{ "user_id", track.UserId ?? -1 },
{ "file_name", track.GpxFileName ?? string.Empty },
{ "timestamp", (track.OsmTimeStamp ?? DateTime.UnixEpoch).ToUnixTime() },
{ "tags", string.Join(',', track.Tags ?? Array.Empty <string>()) }
};
// try compressed.
string?xml = null;
try {
using (var memoryStream = new MemoryStream(track.GpxFile))
using (var gzipStream1 = new GZipStream(memoryStream, CompressionMode.Decompress))
using (var gzipStream2 = new GZipStream(gzipStream1, CompressionMode.Decompress))
using (var streamReader = new StreamReader(gzipStream2)) {
xml = streamReader.ReadToEnd();
}
}
catch (Exception e) {
}
// try uncompressed.
if (string.IsNullOrWhiteSpace(xml))
{
try {
using (var memoryStream = new MemoryStream(track.GpxFile))
using (var gzipStream =
new GZipStream(memoryStream, CompressionMode.Decompress))
using (var streamReader = new StreamReader(gzipStream)) {
xml = streamReader.ReadToEnd();
}
}
catch (Exception e) {
}
}
// read gpx.
Feature[]? trackFeatures = null;
try {
if (!string.IsNullOrWhiteSpace(xml))
{
using (var reader = new StringReader(xml))
using (var xmlReader = XmlReader.Create(reader)) {
(_, trackFeatures, _) =
GpxReader.ReadFeatures(xmlReader, new GpxReaderSettings()
{
DefaultCreatorIfMissing = "OSM"
}, GeometryFactory.Default);
}
}
}
catch (Exception e) {
_logger.LogError(e, "Unhandled exception while parsing GPX");
}
if (trackFeatures != null)
{
foreach (var tf in trackFeatures)
{
if (stoppingToken.IsCancellationRequested)
{
break;
}
if (tf.Geometry is Point)
{
continue;
}
if (tf.Geometry is Polygon)
{
continue;
}
if (tf.Geometry is MultiLineString mls)
{
foreach (var g in mls)
{
if (g is LineString ls)
{
if (ls.Count >= 2)
{
yield return(new Feature(ls, attributes));
}
}
}
}
else if (tf.Geometry is LineString ls)
{
if (ls.Count >= 2)
{
yield return(new Feature(tf.Geometry, attributes));
}
}
}
}
else
{
_logger.LogWarning("Failed to parse track: {trackId} - {trackFileName}", track.Id,
track.GpxFileName);
}
}
}
var outputFileInfo = new FileInfo(
this._configuration.GetValueOrDefault("OutputFile", "osm-gpx.fbg"));
var tempOutputFile = Path.Combine(outputFileInfo.Directory.FullName, ".osm-gpx.fbg.part");
_logger.LogDebug("Building output file: {tempOutputFile}", tempOutputFile);
await using var outputFileStream = File.Open(tempOutputFile, FileMode.Create);
FeatureCollectionConversions.Serialize(outputFileStream, GetFeatures(), FlatGeobuf.GeometryType.LineString,
columns: new []
{
new ColumnMeta()
{
Name = "bdp_id",
Type = ColumnType.Long
},
new ColumnMeta()
{
Name = "track_id",
Type = ColumnType.Long
},
new ColumnMeta()
{
Name = "user_id",
Type = ColumnType.Int
},
new ColumnMeta()
{
Name = "file_name",
Type = ColumnType.String
},
new ColumnMeta()
{
Name = "timestamp",
Type = ColumnType.Long
},
new ColumnMeta()
{
Name = "tags",
Type = ColumnType.String
}
}.ToList());
if (outputFileInfo.Exists)
{
outputFileInfo.Delete();
}
File.Move(tempOutputFile, outputFileInfo.FullName);
_logger.LogDebug("Moved output file: {tempOutputFile}->{outputFile}",
tempOutputFile, outputFileInfo.FullName);
_logger.LogInformation("Built output file: {outputFile}", outputFileInfo.FullName);
}
catch (Exception e)
{
_logger.LogError(e, "Unhandled exception exporting GPX tracks");
}
}
