/// <summary> /// Move to the next object. /// </summary> public override bool MoveNext(bool ignoreNodes, bool ignoreWays, bool ignoreRelations) { if (_mergedSource == null) { var mergedSource = new OsmStreamFilterMerge(); mergedSource.RegisterSource(this.Source); mergedSource.RegisterSource(_creations); _mergedSource = mergedSource; } OsmGeo modified; while (_mergedSource.MoveNext(ignoreNodes, ignoreWays, ignoreRelations)) { // returns the next out of the merged source of the creations and the source. _current = _mergedSource.Current(); // check for deletions or modifications. var key = new OsmGeoKey(_current); if (_deletions.Contains(key)) { // move next. _current = null; continue; } else if (_modifications.TryGetValue(key, out modified)) { _current = modified; } return(true); } return(false); }
public static Dictionary <long, Tuple <double, double> > FindNodes(IEnumerable <long> nodeIds, OsmStreamSource source) { Queue <long> idQueue = new Queue <long>(); foreach (long id in nodeIds.OrderBy(n => n)) { idQueue.Enqueue(id); } var nodes = new Dictionary <long, Tuple <double, double> >(); while (idQueue.Count > 0 && source.MoveNextNode()) { if (source.Current().Id == idQueue.Peek()) { nodes.Add(idQueue.Dequeue(), new Tuple <double, double>((source.Current() as Node).Latitude.Value, (source.Current() as Node).Longitude.Value)); } } return(nodes); }
private void transferData(OsmStreamSource src, OsmStreamTarget sqlTarget) { src.Initialize(); sqlTarget.Initialize(); while (src.MoveNext(false, false, false) && !_shouldStop) { OsmGeo sourceObject = src.Current(); if (!typeCount.ContainsKey(sourceObject.Type)) { typeCount.Add(sourceObject.Type, 0); } typeCount[sourceObject.Type] = 1 + typeCount[sourceObject.Type]; if (sourceObject is Node) { sqlTarget.AddNode(sourceObject as Node); } else if (sourceObject is Way) { sqlTarget.AddWay(sourceObject as Way); } else if (sourceObject is Relation) { sqlTarget.AddRelation(sourceObject as Relation); } } try { sqlTarget.Flush(); sqlTarget.Close(); } catch (Exception ex) { Console.Error.WriteLine(ex); } }
private bool ConcurrentMoveNext(OsmStreamSource source, out OsmGeo current, bool ignore_nodes = false, bool igonre_ways = false, bool ignore_relations = false) { bool available = false; current = null; lock (_source_lock) { if (_cancel_pull) { return false; } available = source.MoveNext(); if (available) { current = source.Current(); _pull_progress++; } } return available; }
/// <summary> /// Returns the current object. /// </summary> /// <returns></returns> public override OsmGeo Current() { OsmGeo current = _reader.Current(); switch (current.Type) { case OsmGeoType.Node: _node++; if ((_node % 10000) == 0) { #if !WINDOWS_PHONE Process p = Process.GetCurrentProcess(); long stop = DateTime.Now.Ticks; float seconds = ((float)(stop - _start)) / (float)TimeSpan.TicksPerSecond; if ((_node % 1000000) == 0) { GC.Collect(); } OsmSharp.Logging.Log.TraceEvent("OsmSharp.Osm.Data.Streams.Filters.OsmStreamFilterProgress", TraceEventType.Information, "Node[{0}]: {1}nodes/s @ {2}MB", _node, (int)((double)_node / seconds), p.PrivateMemorySize64 / 1024 / 1024); #endif #if WINDOWS_PHONE long stop = DateTime.Now.Ticks; float seconds = ((float)(stop - _start)) / (float)TimeSpan.TicksPerSecond; if ((_node % 1000000) == 0) { GC.Collect(); } OsmSharp.Logging.Log.TraceEvent("OsmSharp.Osm.Data.Streams.Filters.OsmStreamFilterProgress", TraceEventType.Information, "Node[{0}]: {1}nodes/s", _node, (int)((double)_node / seconds)); #endif } break; case OsmGeoType.Relation: _relation++; if ((_relation % 1000) == 0) { #if !WINDOWS_PHONE Process p = Process.GetCurrentProcess(); long stop = DateTime.Now.Ticks; float seconds = ((float)(stop - _start)) / (float)TimeSpan.TicksPerSecond; if ((_relation % 10000) == 0) { GC.Collect(); } OsmSharp.Logging.Log.TraceEvent("OsmSharp.Osm.Data.Streams.Filters.OsmStreamFilterProgress", TraceEventType.Information, "Relation[{0}]: {1}relations/s @ {2}MB", _relation, (int)((double)_relation / seconds), p.PrivateMemorySize64 / 1024 / 1024); #endif #if WINDOWS_PHONE long stop = DateTime.Now.Ticks; float seconds = ((float)(stop - _start)) / (float)TimeSpan.TicksPerSecond; if ((_relation % 10000) == 0) { GC.Collect(); } OsmSharp.Logging.Log.TraceEvent("OsmSharp.Osm.Data.Streams.Filters.OsmStreamFilterProgress", TraceEventType.Information, "Relation[{0}]: {1}relations/s", _relation, (int)((double)_relation / seconds)); #endif } break; case OsmGeoType.Way: _way++; if ((_way % 10000) == 0) { #if !WINDOWS_PHONE Process p = Process.GetCurrentProcess(); long stop = DateTime.Now.Ticks; float seconds = ((float)(stop - _start)) / (float)TimeSpan.TicksPerSecond; if ((_way % 100000) == 0) { GC.Collect(); } OsmSharp.Logging.Log.TraceEvent("OsmSharp.Osm.Data.Streams.Filters.OsmStreamFilterProgress", TraceEventType.Information, "Way[{0}]: {1}ways/s @ {2}MB", _way, (int)((double)_way / seconds), p.PrivateMemorySize64 / 1024 / 1024); #endif #if WINDOWS_PHONE long stop = DateTime.Now.Ticks; float seconds = ((float)(stop - _start)) / (float)TimeSpan.TicksPerSecond; if ((_way % 100000) == 0) { GC.Collect(); } OsmSharp.Logging.Log.TraceEvent("OsmSharp.Osm.Data.Streams.Filters.OsmStreamFilterProgress", TraceEventType.Information, "Way[{0}]: {1}ways/s", _way, (int)((double)_way / seconds)); #endif } break; } return(current); }
/// <summary> /// Returns the current object. /// </summary> /// <returns></returns> public override OsmGeo Current() { OsmGeo current = _reader.Current(); // keep the start ticks. long ticksStart = DateTime.Now.Ticks; if (!_lastType.HasValue) { // has a last type. _lastTypeStart = DateTime.Now.Ticks; _lastType = current.Type; } if (_lastType != current.Type) { // the last type has changed. long lastTicks = ticksStart - _lastTypeStart; switch (_lastType) { case OsmGeoType.Node: _nodeTicks = _nodeTicks + lastTicks; break; case OsmGeoType.Way: _wayTicks = _wayTicks + lastTicks; break; case OsmGeoType.Relation: _relationTicks = _relationTicks + lastTicks; break; } // start new ticks. _lastTypeStart = DateTime.Now.Ticks; _lastType = current.Type; } switch (current.Type) { case OsmGeoType.Node: _node++; if ((_node % 10000) == 0) { TimeSpan nodeSpan = new TimeSpan(_nodeTicks + (ticksStart - _lastTypeStart)); double nodePerSecond = System.Math.Round((double)_node / nodeSpan.TotalSeconds, 2); #if !WINDOWS_PHONE Process p = Process.GetCurrentProcess(); OsmSharp.Logging.Log.TraceEvent("OsmSharp.Osm.Data.Streams.Filters.OsmStreamFilterProgress", TraceEventType.Information, "Node[{0}]: {1}nodes/s @ {2}MB", _node, nodePerSecond, p.PrivateMemorySize64 / 1024 / 1024); #endif #if WINDOWS_PHONE OsmSharp.Logging.Log.TraceEvent("OsmSharp.Osm.Data.Streams.Filters.OsmStreamFilterProgress", TraceEventType.Information, "Node[{0}]: {1}nodes/s", _node, nodePerSecond); #endif } break; case OsmGeoType.Relation: _relation++; if ((_relation % 1000) == 0) { TimeSpan relationSpan = new TimeSpan(_relationTicks + (ticksStart - _lastTypeStart)); double relationPerSecond = System.Math.Round((double)_relation / relationSpan.TotalSeconds, 2); #if !WINDOWS_PHONE Process p = Process.GetCurrentProcess(); OsmSharp.Logging.Log.TraceEvent("OsmSharp.Osm.Data.Streams.Filters.OsmStreamFilterProgress", TraceEventType.Information, "Relation[{0}]: {1}relations/s @ {2}MB", _relation, relationPerSecond, p.PrivateMemorySize64 / 1024 / 1024); #endif #if WINDOWS_PHONE OsmSharp.Logging.Log.TraceEvent("OsmSharp.Osm.Data.Streams.Filters.OsmStreamFilterProgress", TraceEventType.Information, "Relation[{0}]: {1}relations/s @ {2}MB", _relation, relationPerSecond); #endif } break; case OsmGeoType.Way: _way++; if ((_way % 10000) == 0) { TimeSpan waySpan = new TimeSpan(_wayTicks + (ticksStart - _lastTypeStart)); double wayPerSecond = System.Math.Round((double)_way / waySpan.TotalSeconds, 2); #if !WINDOWS_PHONE Process p = Process.GetCurrentProcess(); OsmSharp.Logging.Log.TraceEvent("OsmSharp.Osm.Data.Streams.Filters.OsmStreamFilterProgress", TraceEventType.Information, "Way[{0}]: {1}ways/s @ {2}MB", _way, wayPerSecond, p.PrivateMemorySize64 / 1024 / 1024); #endif #if WINDOWS_PHONE OsmSharp.Logging.Log.TraceEvent("OsmSharp.Osm.Data.Streams.Filters.OsmStreamFilterProgress", TraceEventType.Information, "Way[{0}]: {1}ways/s @ {2}MB", _way, wayPerSecond); #endif } break; } return(current); }
/// <summary> /// Moves to the next object. /// </summary> /// <returns></returns> public override bool MoveNext() { if (!_cachingDone) { // first seek & cache. this.Seek(); _cachingDone = true; } while (_simpleSource.MoveNext()) { // there is data. OsmGeo currentSimple = _simpleSource.Current(); switch (currentSimple.Type) { case OsmGeoType.Node: // create complete version. _current = currentSimple as Node; if (_current != null && _current.Tags == null) { // make sure nodes have a default tag collection that is empty not null. _current.Tags = new OsmSharp.Collections.Tags.TagsCollection(); } if (_nodesToInclude.Contains(currentSimple.Id.Value)) { // node needs to be cached. _dataCache.AddNode(currentSimple as Node); _nodesToInclude.Remove(currentSimple.Id.Value); } break; case OsmGeoType.Way: // create complete way. _current = CompleteWay.CreateFrom(currentSimple as Way, _dataCache); if (_waysToInclude.Contains(currentSimple.Id.Value)) { // keep the way because it is needed later on. _dataCache.AddWay(currentSimple as Way); _waysToInclude.Remove(currentSimple.Id.Value); } else { // only report that the nodes have been used when the way can be let-go. Way way = currentSimple as Way; if (way.Nodes != null) { // report usage. way.Nodes.ForEach(x => this.ReportNodeUsage(x)); } } break; case OsmGeoType.Relation: // create complate relation. _current = CompleteRelation.CreateFrom(currentSimple as Relation, _dataCache); if (!_relationsToInclude.Contains(currentSimple.Id.Value)) { // only report relation usage when the relation can be let go. Relation relation = currentSimple as Relation; if (relation.Members != null) { foreach (RelationMember member in relation.Members) { switch (member.MemberType.Value) { case OsmGeoType.Node: this.ReportNodeUsage(member.MemberId.Value); break; case OsmGeoType.Way: this.ReportWayUsage(member.MemberId.Value); break; case OsmGeoType.Relation: this.ReportRelationUsage(member.MemberId.Value); break; } } } } break; } if (_current != null) { // only return complete objects. return(true); } } return(false); }
/// <summary> /// Processes the nodes in the given stream until the first on-node object is reached. /// </summary> /// <param name="source">The source stream.</param> /// <param name="path">The based path of the db.</param> /// <param name="maxZoom">The maximum zoom.</param> /// <param name="tile">The tile being split.</param> /// <param name="nonEmptyTiles">The subtiles that have data in them.</param> /// <param name="hasNext">A flag indicating if there is still more data.</param> /// <param name="compressed">A flag to allow compression of target files.</param> /// <returns>The indexed node id's with a masked zoom.</returns> public static Index Process(OsmStreamSource source, string path, uint maxZoom, Tile tile, out List <Tile> nonEmptyTiles, out bool hasNext, bool compressed = false) { try { // build the set of possible subtiles. var subtiles = new Dictionary <ulong, Stream>(); foreach (var subTile in tile.GetSubtilesAt(tile.Zoom + 2)) { subtiles.Add(subTile.LocalId, null); } // go over all nodes. var nodeIndex = new Index(); hasNext = false; while (source.MoveNext()) { var current = source.Current(); if (current.Type != OsmGeoType.Node) { hasNext = true; break; } // calculate tile. var n = (current as Node); var nodeTile = Tiles.Tile.WorldToTileIndex(n.Latitude.Value, n.Longitude.Value, tile.Zoom + 2); // is tile a subtile. if (!subtiles.TryGetValue(nodeTile.LocalId, out var stream)) { continue; } // initialize stream if needed. if (stream == null) { stream = DatabaseCommon.CreateTile(path, OsmGeoType.Node, nodeTile, compressed); subtiles[nodeTile.LocalId] = stream; } // write node. stream.Append(n); // add node to index. nodeIndex.Add(n.Id.Value, nodeTile.BuildMask2()); } // flush/dispose all subtile streams. // keep all non-empty tiles. nonEmptyTiles = new List <Tile>(); foreach (var subtile in subtiles) { if (subtile.Value == null) { continue; } subtile.Value.Flush(); subtile.Value.Dispose(); if (tile.Zoom + 2 < maxZoom) { nonEmptyTiles.Add(Tile.FromLocalId(tile.Zoom + 2, subtile.Key)); } } return(nodeIndex); } catch (Exception ex) { Console.WriteLine($"Unhandled exception in {nameof(NodeProcessor)}.{nameof(Process)}: {ex}"); throw; } }
/// <summary> /// Move to the next object. /// </summary> public override bool MoveNext(bool ignoreNodes, bool ignoreWays, bool ignoreRelations) { if (_source1 != null && _source2 == null) { return(_source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations)); } if (_source1 == null && _source2 != null) { return(_source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations)); } OsmGeo source1Current = null; OsmGeo source2Current = null; // get currents or move to first. if (_source1Status == null) { _source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } if (_source1Status.Value) { source1Current = _source1.Current(); } if (_source2Status == null) { _source2Status = _source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } if (_source2Status.Value) { source2Current = _source2.Current(); } // compare currents and select next. OsmGeo newCurrent; if (source1Current == null && source2Current == null) { return(false); } else if (source1Current == null) { newCurrent = source2Current; _source2Status = _source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } else if (source2Current == null) { newCurrent = source1Current; _source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } else { var comp = source1Current.CompareByIdAndType(source2Current); if (comp == 0) { // oeps, conflict here! if (_resolutionType == ConflictResolutionType.None) { // no conflict resolution return both. newCurrent = source1Current; _source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } else if (_resolutionType == ConflictResolutionType.FirstStream) { // return only the object from the first stream. newCurrent = source1Current; _source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); _source2Status = _source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } else { throw new NotImplementedException(string.Format("Conflict resolution {0} not implemented.", _resolutionType)); } } else if (comp < 0) { // return from first stream. newCurrent = source1Current; _source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } else { // return from second stream. newCurrent = source2Current; _source2Status = _source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } } // make sure the result is sorted. if (_current != null && _current.CompareByIdAndType(newCurrent) > 0) { throw new Exceptions.StreamNotSortedException(); } _current = newCurrent; return(true); }
/// <summary> /// Gets the current object. /// </summary> /// <returns></returns> public override OsmGeo Current() { return(_source.Current()); }
/// <summary> /// Moves to the next object. /// </summary> /// <returns></returns> public override bool MoveNext() { if (!_cachingDone) { // first seek & cache. this.Seek(); this.CacheRelations(); _cachingDone = true; } while (_simpleSource.MoveNext()) { // there is data. OsmGeo currentSimple = _simpleSource.Current(); if (currentSimple.Id == 198214128 || currentSimple.Id == 1014892489) { System.Diagnostics.Debug.WriteLine(""); } switch (currentSimple.Type) { case OsmGeoType.Node: // create complete version. _current = CompleteNode.CreateFrom(currentSimple as Node); if (_nodesToInclude.Contains(currentSimple.Id.Value)) { // node needs to be cached. _dataCache.AddNode(currentSimple as Node); _nodesToInclude.Remove(currentSimple.Id.Value); } break; case OsmGeoType.Way: // create complete way. _current = CompleteWay.CreateFrom(currentSimple as Way, _dataCache); if (_waysToInclude.Contains(currentSimple.Id.Value)) { // keep the way because it is needed later on. _dataCache.AddWay(currentSimple as Way); _waysToInclude.Remove(currentSimple.Id.Value); } else { // only report that the nodes have been used when the way can be let-go. Way way = currentSimple as Way; if (way.Nodes != null) { // report usage. way.Nodes.ForEach(x => this.ReportNodeUsage(x)); } } break; case OsmGeoType.Relation: // create complate relation. _current = CompleteRelation.CreateFrom(currentSimple as Relation, _dataCache); if (!_relationsToInclude.Contains(currentSimple.Id.Value)) { // only report relation usage when the relation can be let go. Relation relation = currentSimple as Relation; if (relation.Members != null) { foreach (RelationMember member in relation.Members) { switch (member.MemberType.Value) { case OsmGeoType.Node: this.ReportNodeUsage(member.MemberId.Value); break; case OsmGeoType.Way: this.ReportWayUsage(member.MemberId.Value); break; case OsmGeoType.Relation: this.ReportRelationUsage(member.MemberId.Value); break; } } } } break; } if (_current != null) { // only return complete objects. return(true); } } return(false); }
/// <summary> /// Processes the relations in the given stream. Assumed the current stream position already contains a relation. /// </summary> /// <param name="source">The source stream.</param> /// <param name="path">The based path of the db.</param> /// <param name="maxZoom">The maximum zoom.</param> /// <param name="tile">The tile being split.</param> /// <param name="nodeIndex">The node index.</param> /// <param name="wayIndex">The way index.</param> /// <param name="compressed">A flag to allow compression of target files.</param> /// <returns>The indexed node id's with a masked zoom.</returns> public static Index Process(OsmStreamSource source, string path, uint maxZoom, Tile tile, Index nodeIndex, Index wayIndex, bool compressed = false) { try { // split relations. var subtiles = new Dictionary <ulong, Stream>(); foreach (var subTile in tile.GetSubtilesAt(tile.Zoom + 2)) { subtiles.Add(subTile.LocalId, null); } // build the relations index. var relationIndex = new Index(); do { var current = source.Current(); if (current.Type != OsmGeoType.Relation) { break; } // calculate tile. var r = (current as Relation); if (r?.Members == null) { continue; } int?mask = 0; foreach (var member in r.Members) { if (member == null) { Log.Warning($"Member of relation {r.Id} is null!"); continue; } switch (member.Type) { case OsmGeoType.Node: if (nodeIndex != null && nodeIndex.TryGetMask(member.Id, out var nodeMask)) { mask |= nodeMask; } break; case OsmGeoType.Way: if (wayIndex != null && wayIndex.TryGetMask(member.Id, out var wayMask)) { mask |= wayMask; } break; case OsmGeoType.Relation: //mask = null; break; default: throw new ArgumentOutOfRangeException(); } if (mask == null) { // fail fast. break; } } if (mask == null) { // could not determine mask. continue; } // add relation to output(s). foreach (var relationTile in tile.SubTilesForMask2(mask.Value)) { // is tile a subtile. if (!subtiles.TryGetValue(relationTile.LocalId, out var stream)) { continue; } // initialize stream if needed. if (stream == null) { stream = DatabaseCommon.CreateTile(path, OsmGeoType.Relation, relationTile, compressed); subtiles[relationTile.LocalId] = stream; } // write way. stream.Append(r); } // add way to index. relationIndex.Add(r.Id.Value, mask.Value); } while (source.MoveNext()); // flush/dispose all subtile streams. foreach (var subtile in subtiles) { if (subtile.Value == null) { continue; } subtile.Value.Flush(); subtile.Value.Dispose(); } return(relationIndex); } catch (Exception ex) { Console.WriteLine($"Unhandled exception in {nameof(RelationProcessor)}.{nameof(Process)}: {ex}"); throw; } }
/// <summary> /// Processes the ways in the given stream until the first on-way object is reached. Assumed the current stream position already contains a way. /// </summary> /// <param name="source">The source stream.</param> /// <param name="path">The based path of the db.</param> /// <param name="maxZoom">The maximum zoom.</param> /// <param name="tile">The tile being split.</param> /// <param name="nodeIndex">The node index.</param> /// <param name="compressed">A flag to allow compression of target files.</param> /// <returns>The indexed node id's with a masked zoom.</returns> public static Index Process(OsmStreamSource source, string path, uint maxZoom, Tile tile, Index nodeIndex, bool compressed = false) { try { // split ways. var subtiles = new Dictionary <ulong, Stream>(); foreach (var subTile in tile.GetSubtilesAt(tile.Zoom + 2)) { subtiles.Add(subTile.LocalId, null); } // build the ways index. var wayIndex = new Index(); do { var current = source.Current(); if (current.Type != OsmGeoType.Way) { break; } // calculate tile. var w = (current as Way); if (w.Nodes == null) { continue; } var mask = 0; foreach (var node in w.Nodes) { if (nodeIndex.TryGetMask(node, out var nodeMask)) { mask |= nodeMask; } } // add way to output(s). foreach (var wayTile in tile.SubTilesForMask2(mask)) { // is tile a subtile. if (!subtiles.TryGetValue(wayTile.LocalId, out var stream)) { continue; } // initialize stream if needed. if (stream == null) { stream = DatabaseCommon.CreateTile(path, OsmGeoType.Way, wayTile, compressed); subtiles[wayTile.LocalId] = stream; } // write way. stream.Append(w); } // add way to index. wayIndex.Add(w.Id.Value, mask); } while (source.MoveNext()); // flush/dispose all subtile streams. foreach (var subtile in subtiles) { if (subtile.Value == null) { continue; } subtile.Value.Flush(); subtile.Value.Dispose(); } return(wayIndex); } catch (Exception ex) { Console.WriteLine($"Unhandled exception in {nameof(WayProcessor)}.{nameof(Process)}: {ex}"); throw; } }