/// <summary>
/// Move to the next object.
/// </summary>
public override bool MoveNext(bool ignoreNodes, bool ignoreWays, bool ignoreRelations)
{
if (_mergedSource == null)
{
var mergedSource = new OsmStreamFilterMerge();
mergedSource.RegisterSource(this.Source);
mergedSource.RegisterSource(_creations);
_mergedSource = mergedSource;
}
OsmGeo modified;
while (_mergedSource.MoveNext(ignoreNodes, ignoreWays, ignoreRelations))
{
// returns the next out of the merged source of the creations and the source.
_current = _mergedSource.Current();
// check for deletions or modifications.
var key = new OsmGeoKey(_current);
if (_deletions.Contains(key))
{
// move next.
_current = null;
continue;
}
else if (_modifications.TryGetValue(key, out modified))
{
_current = modified;
}
return(true);
}
return(false);
}
/// <summary>
/// Moves to the next object.
/// </summary>
public override bool MoveNext(bool ignoreNodes, bool ignoreWays, bool ignoreRelations)
{
if (_source == null)
{
if (_overpassData.Length == 0)
{ // download data.
using (var overpassStream = OverpassDownload.ToStream(_query))
{
overpassStream.CopyTo(_overpassData);
_overpassData.Seek(0, SeekOrigin.Begin);
}
}
var xmlSource = new XmlOsmStreamSource(_overpassData);
var xmlSourceList = new List <OsmGeo>(xmlSource);
xmlSourceList.Sort((x, y) =>
{
if (x.Type == y.Type)
{
return(x.Id.Value.CompareTo(y.Id.Value));
}
if (x.Type == OsmSharp.OsmGeoType.Node)
{
return(-1);
}
else if (x.Type == OsmSharp.OsmGeoType.Way)
{
if (y.Type == OsmSharp.OsmGeoType.Node)
{
return(1);
}
return(-1);
}
return(1);
});
_source = new OsmSharp.Streams.OsmEnumerableStreamSource(
xmlSourceList);
}
return(_source.MoveNext(ignoreNodes, ignoreWays, ignoreRelations));
}
private void transferData(OsmStreamSource src, OsmStreamTarget sqlTarget)
{
src.Initialize();
sqlTarget.Initialize();
while (src.MoveNext(false, false, false) && !_shouldStop)
{
OsmGeo sourceObject = src.Current();
if (!typeCount.ContainsKey(sourceObject.Type))
{
typeCount.Add(sourceObject.Type, 0);
}
typeCount[sourceObject.Type] = 1 + typeCount[sourceObject.Type];
if (sourceObject is Node)
{
sqlTarget.AddNode(sourceObject as Node);
}
else if (sourceObject is Way)
{
sqlTarget.AddWay(sourceObject as Way);
}
else if (sourceObject is Relation)
{
sqlTarget.AddRelation(sourceObject as Relation);
}
}
try
{
sqlTarget.Flush();
sqlTarget.Close();
}
catch (Exception ex)
{
Console.Error.WriteLine(ex);
}
}
private bool ConcurrentMoveNext(OsmStreamSource source, out OsmGeo current,
bool ignore_nodes = false, bool igonre_ways = false,
bool ignore_relations = false)
{
bool available = false;
current = null;
lock (_source_lock)
{
if (_cancel_pull)
{
return false;
}
available = source.MoveNext();
if (available)
{
current = source.Current();
_pull_progress++;
}
}
return available;
}
/// <summary>
/// Move to the next object.
/// </summary>
/// <returns></returns>
public override bool MoveNext()
{
return(_reader.MoveNext());
}
/// <summary>
/// Processes the nodes in the given stream until the first on-node object is reached.
/// </summary>
/// <param name="source">The source stream.</param>
/// <param name="path">The based path of the db.</param>
/// <param name="maxZoom">The maximum zoom.</param>
/// <param name="tile">The tile being split.</param>
/// <param name="nonEmptyTiles">The subtiles that have data in them.</param>
/// <param name="hasNext">A flag indicating if there is still more data.</param>
/// <param name="compressed">A flag to allow compression of target files.</param>
/// <returns>The indexed node id's with a masked zoom.</returns>
public static Index Process(OsmStreamSource source, string path, uint maxZoom, Tile tile,
out List <Tile> nonEmptyTiles, out bool hasNext, bool compressed = false)
{
try
{
// build the set of possible subtiles.
var subtiles = new Dictionary <ulong, Stream>();
foreach (var subTile in tile.GetSubtilesAt(tile.Zoom + 2))
{
subtiles.Add(subTile.LocalId, null);
}
// go over all nodes.
var nodeIndex = new Index();
hasNext = false;
while (source.MoveNext())
{
var current = source.Current();
if (current.Type != OsmGeoType.Node)
{
hasNext = true;
break;
}
// calculate tile.
var n = (current as Node);
var nodeTile = Tiles.Tile.WorldToTileIndex(n.Latitude.Value, n.Longitude.Value, tile.Zoom + 2);
// is tile a subtile.
if (!subtiles.TryGetValue(nodeTile.LocalId, out var stream))
{
continue;
}
// initialize stream if needed.
if (stream == null)
{
stream = DatabaseCommon.CreateTile(path, OsmGeoType.Node, nodeTile, compressed);
subtiles[nodeTile.LocalId] = stream;
}
// write node.
stream.Append(n);
// add node to index.
nodeIndex.Add(n.Id.Value, nodeTile.BuildMask2());
}
// flush/dispose all subtile streams.
// keep all non-empty tiles.
nonEmptyTiles = new List <Tile>();
foreach (var subtile in subtiles)
{
if (subtile.Value == null)
{
continue;
}
subtile.Value.Flush();
subtile.Value.Dispose();
if (tile.Zoom + 2 < maxZoom)
{
nonEmptyTiles.Add(Tile.FromLocalId(tile.Zoom + 2, subtile.Key));
}
}
return(nodeIndex);
}
catch (Exception ex)
{
Console.WriteLine($"Unhandled exception in {nameof(NodeProcessor)}.{nameof(Process)}: {ex}");
throw;
}
}
/// <summary>
/// Moves to the next object.
/// </summary>
/// <returns></returns>
public override bool MoveNext()
{
if (!_cachingDone)
{ // first seek & cache.
this.Seek();
_cachingDone = true;
}
while (_simpleSource.MoveNext())
{ // there is data.
OsmGeo currentSimple = _simpleSource.Current();
switch (currentSimple.Type)
{
case OsmGeoType.Node:
// create complete version.
_current = currentSimple as Node;
if (_current != null && _current.Tags == null)
{ // make sure nodes have a default tag collection that is empty not null.
_current.Tags = new OsmSharp.Collections.Tags.TagsCollection();
}
if (_nodesToInclude.Contains(currentSimple.Id.Value))
{ // node needs to be cached.
_dataCache.AddNode(currentSimple as Node);
_nodesToInclude.Remove(currentSimple.Id.Value);
}
break;
case OsmGeoType.Way:
// create complete way.
_current = CompleteWay.CreateFrom(currentSimple as Way, _dataCache);
if (_waysToInclude.Contains(currentSimple.Id.Value))
{ // keep the way because it is needed later on.
_dataCache.AddWay(currentSimple as Way);
_waysToInclude.Remove(currentSimple.Id.Value);
}
else
{ // only report that the nodes have been used when the way can be let-go.
Way way = currentSimple as Way;
if (way.Nodes != null)
{ // report usage.
way.Nodes.ForEach(x => this.ReportNodeUsage(x));
}
}
break;
case OsmGeoType.Relation:
// create complate relation.
_current = CompleteRelation.CreateFrom(currentSimple as Relation, _dataCache);
if (!_relationsToInclude.Contains(currentSimple.Id.Value))
{ // only report relation usage when the relation can be let go.
Relation relation = currentSimple as Relation;
if (relation.Members != null)
{
foreach (RelationMember member in relation.Members)
{
switch (member.MemberType.Value)
{
case OsmGeoType.Node:
this.ReportNodeUsage(member.MemberId.Value);
break;
case OsmGeoType.Way:
this.ReportWayUsage(member.MemberId.Value);
break;
case OsmGeoType.Relation:
this.ReportRelationUsage(member.MemberId.Value);
break;
}
}
}
}
break;
}
if (_current != null)
{ // only return complete objects.
return(true);
}
}
return(false);
}
/// <summary>
/// Move to the next object.
/// </summary>
public override bool MoveNext(bool ignoreNodes, bool ignoreWays, bool ignoreRelations)
{
if (_source1 != null && _source2 == null)
{
return(_source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations));
}
if (_source1 == null && _source2 != null)
{
return(_source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations));
}
OsmGeo source1Current = null;
OsmGeo source2Current = null;
// get currents or move to first.
if (_source1Status == null)
{
_source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations);
}
if (_source1Status.Value)
{
source1Current = _source1.Current();
}
if (_source2Status == null)
{
_source2Status = _source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations);
}
if (_source2Status.Value)
{
source2Current = _source2.Current();
}
// compare currents and select next.
OsmGeo newCurrent;
if (source1Current == null && source2Current == null)
{
return(false);
}
else if (source1Current == null)
{
newCurrent = source2Current;
_source2Status = _source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations);
}
else if (source2Current == null)
{
newCurrent = source1Current;
_source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations);
}
else
{
var comp = source1Current.CompareByIdAndType(source2Current);
if (comp == 0)
{ // oeps, conflict here!
if (_resolutionType == ConflictResolutionType.None)
{ // no conflict resolution return both.
newCurrent = source1Current;
_source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations);
}
else if (_resolutionType == ConflictResolutionType.FirstStream)
{ // return only the object from the first stream.
newCurrent = source1Current;
_source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations);
_source2Status = _source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations);
}
else
{
throw new NotImplementedException(string.Format("Conflict resolution {0} not implemented.", _resolutionType));
}
}
else if (comp < 0)
{ // return from first stream.
newCurrent = source1Current;
_source1Status = _source1.MoveNext(ignoreNodes, ignoreWays, ignoreRelations);
}
else
{ // return from second stream.
newCurrent = source2Current;
_source2Status = _source2.MoveNext(ignoreNodes, ignoreWays, ignoreRelations);
}
}
// make sure the result is sorted.
if (_current != null &&
_current.CompareByIdAndType(newCurrent) > 0)
{
throw new Exceptions.StreamNotSortedException();
}
_current = newCurrent;
return(true);
}
/// <summary>
/// Moves to the next object.
/// </summary>
/// <returns></returns>
public override bool MoveNext()
{
if (!_cachingDone)
{ // first seek & cache.
this.Seek();
this.CacheRelations();
_cachingDone = true;
}
while (_simpleSource.MoveNext())
{ // there is data.
OsmGeo currentSimple = _simpleSource.Current();
if (currentSimple.Id == 198214128 || currentSimple.Id == 1014892489)
{
System.Diagnostics.Debug.WriteLine("");
}
switch (currentSimple.Type)
{
case OsmGeoType.Node:
// create complete version.
_current = CompleteNode.CreateFrom(currentSimple as Node);
if (_nodesToInclude.Contains(currentSimple.Id.Value))
{ // node needs to be cached.
_dataCache.AddNode(currentSimple as Node);
_nodesToInclude.Remove(currentSimple.Id.Value);
}
break;
case OsmGeoType.Way:
// create complete way.
_current = CompleteWay.CreateFrom(currentSimple as Way, _dataCache);
if (_waysToInclude.Contains(currentSimple.Id.Value))
{ // keep the way because it is needed later on.
_dataCache.AddWay(currentSimple as Way);
_waysToInclude.Remove(currentSimple.Id.Value);
}
else
{ // only report that the nodes have been used when the way can be let-go.
Way way = currentSimple as Way;
if (way.Nodes != null)
{ // report usage.
way.Nodes.ForEach(x => this.ReportNodeUsage(x));
}
}
break;
case OsmGeoType.Relation:
// create complate relation.
_current = CompleteRelation.CreateFrom(currentSimple as Relation, _dataCache);
if (!_relationsToInclude.Contains(currentSimple.Id.Value))
{ // only report relation usage when the relation can be let go.
Relation relation = currentSimple as Relation;
if (relation.Members != null)
{
foreach (RelationMember member in relation.Members)
{
switch (member.MemberType.Value)
{
case OsmGeoType.Node:
this.ReportNodeUsage(member.MemberId.Value);
break;
case OsmGeoType.Way:
this.ReportWayUsage(member.MemberId.Value);
break;
case OsmGeoType.Relation:
this.ReportRelationUsage(member.MemberId.Value);
break;
}
}
}
}
break;
}
if (_current != null)
{ // only return complete objects.
return(true);
}
}
return(false);
}
/// <summary>
/// Processes the relations in the given stream. Assumed the current stream position already contains a relation.
/// </summary>
/// <param name="source">The source stream.</param>
/// <param name="path">The based path of the db.</param>
/// <param name="maxZoom">The maximum zoom.</param>
/// <param name="tile">The tile being split.</param>
/// <param name="nodeIndex">The node index.</param>
/// <param name="wayIndex">The way index.</param>
/// <param name="compressed">A flag to allow compression of target files.</param>
/// <returns>The indexed node id's with a masked zoom.</returns>
public static Index Process(OsmStreamSource source, string path, uint maxZoom, Tile tile,
Index nodeIndex, Index wayIndex, bool compressed = false)
{
try
{
// split relations.
var subtiles = new Dictionary <ulong, Stream>();
foreach (var subTile in tile.GetSubtilesAt(tile.Zoom + 2))
{
subtiles.Add(subTile.LocalId, null);
}
// build the relations index.
var relationIndex = new Index();
do
{
var current = source.Current();
if (current.Type != OsmGeoType.Relation)
{
break;
}
// calculate tile.
var r = (current as Relation);
if (r?.Members == null)
{
continue;
}
int?mask = 0;
foreach (var member in r.Members)
{
if (member == null)
{
Log.Warning($"Member of relation {r.Id} is null!");
continue;
}
switch (member.Type)
{
case OsmGeoType.Node:
if (nodeIndex != null &&
nodeIndex.TryGetMask(member.Id, out var nodeMask))
{
mask |= nodeMask;
}
break;
case OsmGeoType.Way:
if (wayIndex != null &&
wayIndex.TryGetMask(member.Id, out var wayMask))
{
mask |= wayMask;
}
break;
case OsmGeoType.Relation:
//mask = null;
break;
default:
throw new ArgumentOutOfRangeException();
}
if (mask == null)
{
// fail fast.
break;
}
}
if (mask == null)
{
// could not determine mask.
continue;
}
// add relation to output(s).
foreach (var relationTile in tile.SubTilesForMask2(mask.Value))
{
// is tile a subtile.
if (!subtiles.TryGetValue(relationTile.LocalId, out var stream))
{
continue;
}
// initialize stream if needed.
if (stream == null)
{
stream = DatabaseCommon.CreateTile(path, OsmGeoType.Relation, relationTile, compressed);
subtiles[relationTile.LocalId] = stream;
}
// write way.
stream.Append(r);
}
// add way to index.
relationIndex.Add(r.Id.Value, mask.Value);
} while (source.MoveNext());
// flush/dispose all subtile streams.
foreach (var subtile in subtiles)
{
if (subtile.Value == null)
{
continue;
}
subtile.Value.Flush();
subtile.Value.Dispose();
}
return(relationIndex);
}
catch (Exception ex)
{
Console.WriteLine($"Unhandled exception in {nameof(RelationProcessor)}.{nameof(Process)}: {ex}");
throw;
}
}
/// <summary>
/// Processes the ways in the given stream until the first on-way object is reached. Assumed the current stream position already contains a way.
/// </summary>
/// <param name="source">The source stream.</param>
/// <param name="path">The based path of the db.</param>
/// <param name="maxZoom">The maximum zoom.</param>
/// <param name="tile">The tile being split.</param>
/// <param name="nodeIndex">The node index.</param>
/// <param name="compressed">A flag to allow compression of target files.</param>
/// <returns>The indexed node id's with a masked zoom.</returns>
public static Index Process(OsmStreamSource source, string path, uint maxZoom, Tile tile,
Index nodeIndex, bool compressed = false)
{
try
{
// split ways.
var subtiles = new Dictionary <ulong, Stream>();
foreach (var subTile in tile.GetSubtilesAt(tile.Zoom + 2))
{
subtiles.Add(subTile.LocalId, null);
}
// build the ways index.
var wayIndex = new Index();
do
{
var current = source.Current();
if (current.Type != OsmGeoType.Way)
{
break;
}
// calculate tile.
var w = (current as Way);
if (w.Nodes == null)
{
continue;
}
var mask = 0;
foreach (var node in w.Nodes)
{
if (nodeIndex.TryGetMask(node, out var nodeMask))
{
mask |= nodeMask;
}
}
// add way to output(s).
foreach (var wayTile in tile.SubTilesForMask2(mask))
{
// is tile a subtile.
if (!subtiles.TryGetValue(wayTile.LocalId, out var stream))
{
continue;
}
// initialize stream if needed.
if (stream == null)
{
stream = DatabaseCommon.CreateTile(path, OsmGeoType.Way, wayTile, compressed);
subtiles[wayTile.LocalId] = stream;
}
// write way.
stream.Append(w);
}
// add way to index.
wayIndex.Add(w.Id.Value, mask);
} while (source.MoveNext());
// flush/dispose all subtile streams.
foreach (var subtile in subtiles)
{
if (subtile.Value == null)
{
continue;
}
subtile.Value.Flush();
subtile.Value.Dispose();
}
return(wayIndex);
}
catch (Exception ex)
{
Console.WriteLine($"Unhandled exception in {nameof(WayProcessor)}.{nameof(Process)}: {ex}");
throw;
}
}
