/// <summary> /// Move to the next object. /// </summary> public override bool MoveNext(bool ignoreNodes, bool ignoreWays, bool ignoreRelations) { if (_mergedSource == null) { var mergedSource = new OsmStreamFilterMerge(); mergedSource.RegisterSource(this.Source); mergedSource.RegisterSource(_creations); _mergedSource = mergedSource; } OsmGeo modified; while (_mergedSource.MoveNext(ignoreNodes, ignoreWays, ignoreRelations)) { // returns the next out of the merged source of the creations and the source. _current = _mergedSource.Current(); // check for deletions or modifications. var key = new OsmGeoKey(_current); if (_deletions.Contains(key)) { // move next. _current = null; continue; } else if (_modifications.TryGetValue(key, out modified)) { _current = modified; } return(true); } return(false); }
/// <summary> /// Moves to the next object. /// </summary> public override bool MoveNext(bool ignoreNodes, bool ignoreWays, bool ignoreRelations) { if (_source == null) { if (_overpassData.Length == 0) { // download data. using (var overpassStream = OverpassDownload.ToStream(_query)) { overpassStream.CopyTo(_overpassData); _overpassData.Seek(0, SeekOrigin.Begin); } } var xmlSource = new XmlOsmStreamSource(_overpassData); var xmlSourceList = new List <OsmGeo>(xmlSource); xmlSourceList.Sort((x, y) => { if (x.Type == y.Type) { return(x.Id.Value.CompareTo(y.Id.Value)); } if (x.Type == OsmSharp.OsmGeoType.Node) { return(-1); } else if (x.Type == OsmSharp.OsmGeoType.Way) { if (y.Type == OsmSharp.OsmGeoType.Node) { return(1); } return(-1); } return(1); }); _source = new OsmSharp.Streams.OsmEnumerableStreamSource( xmlSourceList); } return(_source.MoveNext(ignoreNodes, ignoreWays, ignoreRelations)); }
private void transferData(OsmStreamSource src, OsmStreamTarget sqlTarget) { src.Initialize(); sqlTarget.Initialize(); while (src.MoveNext(false, false, false) && !_shouldStop) { OsmGeo sourceObject = src.Current(); if (!typeCount.ContainsKey(sourceObject.Type)) { typeCount.Add(sourceObject.Type, 0); } typeCount[sourceObject.Type] = 1 + typeCount[sourceObject.Type]; if (sourceObject is Node) { sqlTarget.AddNode(sourceObject as Node); } else if (sourceObject is Way) { sqlTarget.AddWay(sourceObject as Way); } else if (sourceObject is Relation) { sqlTarget.AddRelation(sourceObject as Relation); } } try { sqlTarget.Flush(); sqlTarget.Close(); } catch (Exception ex) { Console.Error.WriteLine(ex); } }
private bool ConcurrentMoveNext(OsmStreamSource source, out OsmGeo current, bool ignore_nodes = false, bool igonre_ways = false, bool ignore_relations = false) { bool available = false; current = null; lock (_source_lock) { if (_cancel_pull) { return false; } available = source.MoveNext(); if (available) { current = source.Current(); _pull_progress++; } } return available; }
/// <summary> /// Move to the next object. /// </summary> /// <returns></returns> public override bool MoveNext() { return(_reader.MoveNext()); }
/// <summary> /// Processes the nodes in the given stream until the first on-node object is reached. /// </summary> /// <param name="source">The source stream.</param> /// <param name="path">The based path of the db.</param> /// <param name="maxZoom">The maximum zoom.</param> /// <param name="tile">The tile being split.</param> /// <param name="nonEmptyTiles">The subtiles that have data in them.</param> /// <param name="hasNext">A flag indicating if there is still more data.</param> /// <param name="compressed">A flag to allow compression of target files.</param> /// <returns>The indexed node id's with a masked zoom.</returns> public static Index Process(OsmStreamSource source, string path, uint maxZoom, Tile tile, out List <Tile> nonEmptyTiles, out bool hasNext, bool compressed = false) { try { // build the set of possible subtiles. var subtiles = new Dictionary <ulong, Stream>(); foreach (var subTile in tile.GetSubtilesAt(tile.Zoom + 2)) { subtiles.Add(subTile.LocalId, null); } // go over all nodes. var nodeIndex = new Index(); hasNext = false; while (source.MoveNext()) { var current = source.Current(); if (current.Type != OsmGeoType.Node) { hasNext = true; break; } // calculate tile. var n = (current as Node); var nodeTile = Tiles.Tile.WorldToTileIndex(n.Latitude.Value, n.Longitude.Value, tile.Zoom + 2); // is tile a subtile. if (!subtiles.TryGetValue(nodeTile.LocalId, out var stream)) { continue; } // initialize stream if needed. if (stream == null) { stream = DatabaseCommon.CreateTile(path, OsmGeoType.Node, nodeTile, compressed); subtiles[nodeTile.LocalId] = stream; } // write node. stream.Append(n); // add node to index. nodeIndex.Add(n.Id.Value, nodeTile.BuildMask2()); } // flush/dispose all subtile streams. // keep all non-empty tiles. nonEmptyTiles = new List <Tile>(); foreach (var subtile in subtiles) { if (subtile.Value == null) { continue; } subtile.Value.Flush(); subtile.Value.Dispose(); if (tile.Zoom + 2 < maxZoom) { nonEmptyTiles.Add(Tile.FromLocalId(tile.Zoom + 2, subtile.Key)); } } return(nodeIndex); } catch (Exception ex) { Console.WriteLine($"Unhandled exception in {nameof(NodeProcessor)}.{nameof(Process)}: {ex}"); throw; } }
/// <summary> /// Moves to the next object. /// </summary> /// <returns></returns> public override bool MoveNext() { if (!_cachingDone) { // first seek & cache. this.Seek(); _cachingDone = true; } while (_simpleSource.MoveNext()) { // there is data. OsmGeo currentSimple = _simpleSource.Current(); switch (currentSimple.Type) { case OsmGeoType.Node: // create complete version. _current = currentSimple as Node; if (_current != null && _current.Tags == null) { // make sure nodes have a default tag collection that is empty not null. _current.Tags = new OsmSharp.Collections.Tags.TagsCollection(); } if (_nodesToInclude.Contains(currentSimple.Id.Value)) { // node needs to be cached. _dataCache.AddNode(currentSimple as Node); _nodesToInclude.Remove(currentSimple.Id.Value); } break; case OsmGeoType.Way: // create complete way. _current = CompleteWay.CreateFrom(currentSimple as Way, _dataCache); if (_waysToInclude.Contains(currentSimple.Id.Value)) { // keep the way because it is needed later on. _dataCache.AddWay(currentSimple as Way); _waysToInclude.Remove(currentSimple.Id.Value); } else { // only report that the nodes have been used when the way can be let-go. Way way = currentSimple as Way; if (way.Nodes != null) { // report usage. way.Nodes.ForEach(x => this.ReportNodeUsage(x)); } } break; case OsmGeoType.Relation: // create complate relation. _current = CompleteRelation.CreateFrom(currentSimple as Relation, _dataCache); if (!_relationsToInclude.Contains(currentSimple.Id.Value)) { // only report relation usage when the relation can be let go. Relation relation = currentSimple as Relation; if (relation.Members != null) { foreach (RelationMember member in relation.Members) { switch (member.MemberType.Value) { case OsmGeoType.Node: this.ReportNodeUsage(member.MemberId.Value); break; case OsmGeoType.Way: this.ReportWayUsage(member.MemberId.Value); break; case OsmGeoType.Relation: this.ReportRelationUsage(member.MemberId.Value); break; } } } } break; } if (_current != null) { // only return complete objects. return(true); } } return(false); }
/// <summary> /// Move to the next object. /// </summary> public override bool MoveNext(bool ignoreNodes, bool ignoreWays, bool ignoreRelations) { if (_source1 != null && _source2 == null) { return(_source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations)); } if (_source1 == null && _source2 != null) { return(_source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations)); } OsmGeo source1Current = null; OsmGeo source2Current = null; // get currents or move to first. if (_source1Status == null) { _source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } if (_source1Status.Value) { source1Current = _source1.Current(); } if (_source2Status == null) { _source2Status = _source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } if (_source2Status.Value) { source2Current = _source2.Current(); } // compare currents and select next. OsmGeo newCurrent; if (source1Current == null && source2Current == null) { return(false); } else if (source1Current == null) { newCurrent = source2Current; _source2Status = _source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } else if (source2Current == null) { newCurrent = source1Current; _source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } else { var comp = source1Current.CompareByIdAndType(source2Current); if (comp == 0) { // oeps, conflict here! if (_resolutionType == ConflictResolutionType.None) { // no conflict resolution return both. newCurrent = source1Current; _source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } else if (_resolutionType == ConflictResolutionType.FirstStream) { // return only the object from the first stream. newCurrent = source1Current; _source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); _source2Status = _source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } else { throw new NotImplementedException(string.Format("Conflict resolution {0} not implemented.", _resolutionType)); } } else if (comp < 0) { // return from first stream. newCurrent = source1Current; _source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } else { // return from second stream. newCurrent = source2Current; _source2Status = _source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations); } } // make sure the result is sorted. if (_current != null && _current.CompareByIdAndType(newCurrent) > 0) { throw new Exceptions.StreamNotSortedException(); } _current = newCurrent; return(true); }
/// <summary> /// Moves to the next object. /// </summary> /// <returns></returns> public override bool MoveNext() { if (!_cachingDone) { // first seek & cache. this.Seek(); this.CacheRelations(); _cachingDone = true; } while (_simpleSource.MoveNext()) { // there is data. OsmGeo currentSimple = _simpleSource.Current(); if (currentSimple.Id == 198214128 || currentSimple.Id == 1014892489) { System.Diagnostics.Debug.WriteLine(""); } switch (currentSimple.Type) { case OsmGeoType.Node: // create complete version. _current = CompleteNode.CreateFrom(currentSimple as Node); if (_nodesToInclude.Contains(currentSimple.Id.Value)) { // node needs to be cached. _dataCache.AddNode(currentSimple as Node); _nodesToInclude.Remove(currentSimple.Id.Value); } break; case OsmGeoType.Way: // create complete way. _current = CompleteWay.CreateFrom(currentSimple as Way, _dataCache); if (_waysToInclude.Contains(currentSimple.Id.Value)) { // keep the way because it is needed later on. _dataCache.AddWay(currentSimple as Way); _waysToInclude.Remove(currentSimple.Id.Value); } else { // only report that the nodes have been used when the way can be let-go. Way way = currentSimple as Way; if (way.Nodes != null) { // report usage. way.Nodes.ForEach(x => this.ReportNodeUsage(x)); } } break; case OsmGeoType.Relation: // create complate relation. _current = CompleteRelation.CreateFrom(currentSimple as Relation, _dataCache); if (!_relationsToInclude.Contains(currentSimple.Id.Value)) { // only report relation usage when the relation can be let go. Relation relation = currentSimple as Relation; if (relation.Members != null) { foreach (RelationMember member in relation.Members) { switch (member.MemberType.Value) { case OsmGeoType.Node: this.ReportNodeUsage(member.MemberId.Value); break; case OsmGeoType.Way: this.ReportWayUsage(member.MemberId.Value); break; case OsmGeoType.Relation: this.ReportRelationUsage(member.MemberId.Value); break; } } } } break; } if (_current != null) { // only return complete objects. return(true); } } return(false); }
/// <summary> /// Processes the relations in the given stream. Assumed the current stream position already contains a relation. /// </summary> /// <param name="source">The source stream.</param> /// <param name="path">The based path of the db.</param> /// <param name="maxZoom">The maximum zoom.</param> /// <param name="tile">The tile being split.</param> /// <param name="nodeIndex">The node index.</param> /// <param name="wayIndex">The way index.</param> /// <param name="compressed">A flag to allow compression of target files.</param> /// <returns>The indexed node id's with a masked zoom.</returns> public static Index Process(OsmStreamSource source, string path, uint maxZoom, Tile tile, Index nodeIndex, Index wayIndex, bool compressed = false) { try { // split relations. var subtiles = new Dictionary <ulong, Stream>(); foreach (var subTile in tile.GetSubtilesAt(tile.Zoom + 2)) { subtiles.Add(subTile.LocalId, null); } // build the relations index. var relationIndex = new Index(); do { var current = source.Current(); if (current.Type != OsmGeoType.Relation) { break; } // calculate tile. var r = (current as Relation); if (r?.Members == null) { continue; } int?mask = 0; foreach (var member in r.Members) { if (member == null) { Log.Warning($"Member of relation {r.Id} is null!"); continue; } switch (member.Type) { case OsmGeoType.Node: if (nodeIndex != null && nodeIndex.TryGetMask(member.Id, out var nodeMask)) { mask |= nodeMask; } break; case OsmGeoType.Way: if (wayIndex != null && wayIndex.TryGetMask(member.Id, out var wayMask)) { mask |= wayMask; } break; case OsmGeoType.Relation: //mask = null; break; default: throw new ArgumentOutOfRangeException(); } if (mask == null) { // fail fast. break; } } if (mask == null) { // could not determine mask. continue; } // add relation to output(s). foreach (var relationTile in tile.SubTilesForMask2(mask.Value)) { // is tile a subtile. if (!subtiles.TryGetValue(relationTile.LocalId, out var stream)) { continue; } // initialize stream if needed. if (stream == null) { stream = DatabaseCommon.CreateTile(path, OsmGeoType.Relation, relationTile, compressed); subtiles[relationTile.LocalId] = stream; } // write way. stream.Append(r); } // add way to index. relationIndex.Add(r.Id.Value, mask.Value); } while (source.MoveNext()); // flush/dispose all subtile streams. foreach (var subtile in subtiles) { if (subtile.Value == null) { continue; } subtile.Value.Flush(); subtile.Value.Dispose(); } return(relationIndex); } catch (Exception ex) { Console.WriteLine($"Unhandled exception in {nameof(RelationProcessor)}.{nameof(Process)}: {ex}"); throw; } }
/// <summary> /// Processes the ways in the given stream until the first on-way object is reached. Assumed the current stream position already contains a way. /// </summary> /// <param name="source">The source stream.</param> /// <param name="path">The based path of the db.</param> /// <param name="maxZoom">The maximum zoom.</param> /// <param name="tile">The tile being split.</param> /// <param name="nodeIndex">The node index.</param> /// <param name="compressed">A flag to allow compression of target files.</param> /// <returns>The indexed node id's with a masked zoom.</returns> public static Index Process(OsmStreamSource source, string path, uint maxZoom, Tile tile, Index nodeIndex, bool compressed = false) { try { // split ways. var subtiles = new Dictionary <ulong, Stream>(); foreach (var subTile in tile.GetSubtilesAt(tile.Zoom + 2)) { subtiles.Add(subTile.LocalId, null); } // build the ways index. var wayIndex = new Index(); do { var current = source.Current(); if (current.Type != OsmGeoType.Way) { break; } // calculate tile. var w = (current as Way); if (w.Nodes == null) { continue; } var mask = 0; foreach (var node in w.Nodes) { if (nodeIndex.TryGetMask(node, out var nodeMask)) { mask |= nodeMask; } } // add way to output(s). foreach (var wayTile in tile.SubTilesForMask2(mask)) { // is tile a subtile. if (!subtiles.TryGetValue(wayTile.LocalId, out var stream)) { continue; } // initialize stream if needed. if (stream == null) { stream = DatabaseCommon.CreateTile(path, OsmGeoType.Way, wayTile, compressed); subtiles[wayTile.LocalId] = stream; } // write way. stream.Append(w); } // add way to index. wayIndex.Add(w.Id.Value, mask); } while (source.MoveNext()); // flush/dispose all subtile streams. foreach (var subtile in subtiles) { if (subtile.Value == null) { continue; } subtile.Value.Flush(); subtile.Value.Dispose(); } return(wayIndex); } catch (Exception ex) { Console.WriteLine($"Unhandled exception in {nameof(WayProcessor)}.{nameof(Process)}: {ex}"); throw; } }