private async Task RunAsync(CancellationToken stoppingToken) { try { IEnumerable <Feature> GetFeatures() { var maxTrackCount = _configuration.GetValueOrDefault("MaxFeatures", int.MaxValue); var tracks = _db.Tracks.Where(x => x.Id < maxTrackCount) .Where(x => x.UserId != null).OrderBy(x => x.Id); var trackCount = 0; foreach (var track in tracks) { trackCount++; if (stoppingToken.IsCancellationRequested) { break; } if (track.GpxFile == null) { continue; } _logger.LogInformation( $"Track {trackCount + 1}/{maxTrackCount} {track.Id}: {track.GpxFile?.Length} bytes"); // try compressed. string?xml = null; try { using (var memoryStream = new MemoryStream(track.GpxFile)) using (var gzipStream1 = new GZipStream(memoryStream, CompressionMode.Decompress)) using (var gzipStream2 = new GZipStream(gzipStream1, CompressionMode.Decompress)) using (var streamReader = new StreamReader(gzipStream2)) { xml = streamReader.ReadToEnd(); } } catch (Exception e) { } // try uncompressed. if (string.IsNullOrWhiteSpace(xml)) { try { using (var memoryStream = new MemoryStream(track.GpxFile)) using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) using (var streamReader = new StreamReader(gzipStream)) { xml = streamReader.ReadToEnd(); } } catch (Exception e) { } } // read gpx. Feature[]? trackFeatures = null; try { if (!string.IsNullOrWhiteSpace(xml)) { using (var reader = new StringReader(xml)) using (var xmlReader = XmlReader.Create(reader)) { (_, trackFeatures, _) = GpxReader.ReadFeatures(xmlReader, new GpxReaderSettings() { DefaultCreatorIfMissing = "OSM" }, GeometryFactory.Default); } } } catch (Exception e) { _logger.LogError(e, "Unhandled exception while parsing GPX"); } if (trackFeatures != null) { foreach (var tf in trackFeatures) { if (stoppingToken.IsCancellationRequested) { break; } if (tf.Geometry is Point) { continue; } if (tf.Geometry is Polygon) { continue; } if (tf.Geometry is MultiLineString mls) { foreach (var g in mls) { if (g is LineString ls) { if (ls.Count >= 2) { yield return(new Feature(ls, new AttributesTable() { { "track_id", track.Id } })); } } } } else if (tf.Geometry is LineString ls) { if (ls.Count >= 2) { yield return(new Feature(tf.Geometry, new AttributesTable() { { "track_id", track.Id } })); } } } } else { _logger.LogWarning("Failed to parse track: {trackId} - {trackFileName}", track.Id, track.GpxFileName); } } } var outputFile = this._configuration.GetValueOrDefault("OutputFile", "osm-gpx.fbg"); _logger.LogDebug("Building output file: {outputFile}", outputFile); using var outputFileStream = File.Open(outputFile, FileMode.Create); FeatureCollectionConversions.Serialize(outputFileStream, GetFeatures(), FlatGeobuf.GeometryType.LineString); _logger.LogInformation("Built output file: {outputFile}", outputFile); } catch (Exception e) { _logger.LogError(e, $"Unhandled exception exporting GPX tracks"); } }
private async Task RunAsync(CancellationToken stoppingToken) { try { using var scope = _serviceProvider.CreateScope(); var db = scope.ServiceProvider .GetRequiredService <OsmDbContext>(); IEnumerable <Feature> GetFeatures() { var maxTrackCount = _configuration.GetValueOrDefault("MaxFeatures", int.MaxValue); var tracks = db.Tracks.Where(x => x.Id < maxTrackCount) .Where(x => x.UserId != null).OrderBy(x => x.Id); var trackCount = 0; foreach (var track in tracks) { trackCount++; if (stoppingToken.IsCancellationRequested) { break; } if (track.GpxFile == null) { continue; } _logger.LogInformation( "Track {trackCount}/{maxTrackCount} {trackId}: {trackLength} bytes", trackCount + 1, maxTrackCount, track.Id, track.GpxFile?.Length); // build attributes var attributes = new AttributesTable() { { "bdp_id", track.Id }, { "track_id", track.OsmTrackId }, { "user_id", track.UserId ?? -1 }, { "file_name", track.GpxFileName ?? string.Empty }, { "timestamp", (track.OsmTimeStamp ?? DateTime.UnixEpoch).ToUnixTime() }, { "tags", string.Join(',', track.Tags ?? Array.Empty <string>()) } }; // try compressed. string?xml = null; try { using (var memoryStream = new MemoryStream(track.GpxFile)) using (var gzipStream1 = new GZipStream(memoryStream, CompressionMode.Decompress)) using (var gzipStream2 = new GZipStream(gzipStream1, CompressionMode.Decompress)) using (var streamReader = new StreamReader(gzipStream2)) { xml = streamReader.ReadToEnd(); } } catch (Exception e) { } // try uncompressed. if (string.IsNullOrWhiteSpace(xml)) { try { using (var memoryStream = new MemoryStream(track.GpxFile)) using (var gzipStream = new GZipStream(memoryStream, CompressionMode.Decompress)) using (var streamReader = new StreamReader(gzipStream)) { xml = streamReader.ReadToEnd(); } } catch (Exception e) { } } // read gpx. Feature[]? trackFeatures = null; try { if (!string.IsNullOrWhiteSpace(xml)) { using (var reader = new StringReader(xml)) using (var xmlReader = XmlReader.Create(reader)) { (_, trackFeatures, _) = GpxReader.ReadFeatures(xmlReader, new GpxReaderSettings() { DefaultCreatorIfMissing = "OSM" }, GeometryFactory.Default); } } } catch (Exception e) { _logger.LogError(e, "Unhandled exception while parsing GPX"); } if (trackFeatures != null) { foreach (var tf in trackFeatures) { if (stoppingToken.IsCancellationRequested) { break; } if (tf.Geometry is Point) { continue; } if (tf.Geometry is Polygon) { continue; } if (tf.Geometry is MultiLineString mls) { foreach (var g in mls) { if (g is LineString ls) { if (ls.Count >= 2) { yield return(new Feature(ls, attributes)); } } } } else if (tf.Geometry is LineString ls) { if (ls.Count >= 2) { yield return(new Feature(tf.Geometry, attributes)); } } } } else { _logger.LogWarning("Failed to parse track: {trackId} - {trackFileName}", track.Id, track.GpxFileName); } } } var outputFileInfo = new FileInfo( this._configuration.GetValueOrDefault("OutputFile", "osm-gpx.fbg")); var tempOutputFile = Path.Combine(outputFileInfo.Directory.FullName, ".osm-gpx.fbg.part"); _logger.LogDebug("Building output file: {tempOutputFile}", tempOutputFile); await using var outputFileStream = File.Open(tempOutputFile, FileMode.Create); FeatureCollectionConversions.Serialize(outputFileStream, GetFeatures(), FlatGeobuf.GeometryType.LineString, columns: new [] { new ColumnMeta() { Name = "bdp_id", Type = ColumnType.Long }, new ColumnMeta() { Name = "track_id", Type = ColumnType.Long }, new ColumnMeta() { Name = "user_id", Type = ColumnType.Int }, new ColumnMeta() { Name = "file_name", Type = ColumnType.String }, new ColumnMeta() { Name = "timestamp", Type = ColumnType.Long }, new ColumnMeta() { Name = "tags", Type = ColumnType.String } }.ToList()); if (outputFileInfo.Exists) { outputFileInfo.Delete(); } File.Move(tempOutputFile, outputFileInfo.FullName); _logger.LogDebug("Moved output file: {tempOutputFile}->{outputFile}", tempOutputFile, outputFileInfo.FullName); _logger.LogInformation("Built output file: {outputFile}", outputFileInfo.FullName); } catch (Exception e) { _logger.LogError(e, "Unhandled exception exporting GPX tracks"); } }