private void ProcessWays(PrimitiveAccessor accessor)
{
var ways = accessor.Ways.ToList();
watch.Stop();
if (ways.Any())
{
Console.Write($"Way! {totalWayNodes:#,###}. ");
nodesIndex.Flush();
waysBuf.AddRange(ways);
foreach (var id in ways.SelectMany(x => x.NodeIds))
{
totalWayNodes++;
wayBufNodeIds.Add(id);
}
if (totalWayNodes >= 10_000_000)
{
MergeAndFlushWays();
}
}
}
public void ProcessPrimitives(PrimitiveAccessor accessor, string data)
{
foreach (var node in accessor.Nodes)
{
var lat = (uint)(Helpers.CoordAsInt(node.Lat) + int.MaxValue) >> 18;
var lon = (uint)(Helpers.CoordAsInt(node.Lon) + int.MaxValue) >> 17;
map[lat, lon]++;
}
}
public void ProcessPrimitives(PrimitiveAccessor accessor, string data)
{
Console.Write($"Nodes: {totalCnt:#,###}. ");
foreach (var node in accessor.Nodes)
{
totalCnt++;
var sNode = new SNode
{
Id = node.Id,
Lat = Helpers.CoordAsInt(node.Lat),
Lon = Helpers.CoordAsInt(node.Lon)
};
index.Add(sNode);
}
}
private void ProcessNodes(PrimitiveAccessor accessor)
{
foreach (var node in accessor.Nodes)
{
watch.Stop();
totalNodesCount++;
var lat = Helpers.CoordAsInt(node.Lat);
var lon = Helpers.CoordAsInt(node.Lon);
var mNode = new MapNode {
Id = node.Id, Lat = lat, Lon = lon
};
nodesIndex.WriteNode(mNode);
}
}
public void ProcessPrimitives(PrimitiveAccessor accessor, string data)
{
if (data == null)
{
return;
}
if (accessor == null)
{
throw new ArgumentNullException(nameof(accessor));
}
watch.Start();
Console.Write($"Decode: {watch.Elapsed}. Nodes: {totalNodesCount}.\r");
ProcessNodes(accessor);
ProcessWays(accessor);
}
public void ProcessPrimitives(PrimitiveAccessor accessor, string data)
{
Console.Write($"Nodes: {totalNodesCount}.\r");
foreach (var node in accessor.Nodes)
{
totalNodesCount++;
var lat = Helpers.CoordAsInt(node.Lat);
var lon = Helpers.CoordAsInt(node.Lon);
var mNode = new MapNode {
Id = node.Id, Lat = lat, Lon = lon
};
AddToSort(mNode);
}
}
public void ProcessPrimitives(PrimitiveAccessor accessor, string data)
{
foreach (var node in accessor.Nodes)
{
if (node.Tags != null)
{
foreach (var tag in node.Tags)
{
AddTag(tag, RelationMemberTypes.Node, node.Id);
}
}
}
foreach (var way in accessor.Ways)
{
if (way.Tags != null)
{
foreach (var tag in way.Tags)
{
AddTag(tag, RelationMemberTypes.Way, way.Id);
}
}
}
foreach (var relation in accessor.Relations)
{
if (relation.Tags != null)
{
foreach (var tag in relation.Tags)
{
AddTag(tag, RelationMemberTypes.Relation, relation.Id);
}
}
}
Console.Write(
$"\r\nTag stats: {keys.Count:#,###}. Values: {values.Count:#,###}. Total: {valuesCount:#,###}. Size: {dataSize:#,###}. ");
Console.SetCursorPosition(0, 0);
if (dataSize > 1024 * 1024 * 1024) // 1_073_741_824
{
WriteTmpData();
}
}
public void ProcessPrimitives(PrimitiveAccessor accessor, string data)
{
var present = false;
foreach (var node in accessor.Nodes)
{
present = true;
if (node.Tags != null)
{
foreach (var tag in node.Tags)
{
AddTag(tag);
}
}
}
if (!present && reset == 0)
{
reset = 1;
WriteStats(".node-stats");
tagValuesStat.Clear();
}
present = false;
foreach (var way in accessor.Ways)
{
present = true;
if (way.Tags != null)
{
foreach (var tag in way.Tags)
{
AddTag(tag);
}
}
}
if (!present && reset == 1)
{
reset = 2;
WriteStats(".way-stats");
tagValuesStat.Clear();
}
foreach (var relation in accessor.Relations)
{
if (relation.Tags != null)
{
foreach (var tag in relation.Tags)
{
AddTag(tag);
}
}
}
Console.Write($" Count: ${tagValuesStat.Count:0,000}. \r");
if (watch.ElapsedMilliseconds > nextShow)
{
ShowTop();
nextShow = watch.ElapsedMilliseconds + 5000;
}
FilterStatIfNeeded();
}
public void ProcessPrimitives(PrimitiveAccessor accessor, string data)
{
if (accessor == null)
{
throw new ArgumentNullException(nameof(accessor));
}
var nodeIds = accessor.Relations
.SelectMany(x => x.Items)
.Where(i => i.MemberType == RelationMemberTypes.Node)
.Select(i => i.Id)
.Distinct()
.OrderBy(x => x);
var nodes = nodesIndex.ReadAllNodesById(nodeIds).ToDictionary(x => x.Id);
foreach (var relation in accessor.Relations)
{
var rect = new BoundingRect();
var itemInfos = new List <RelationItemInfo>();
foreach (var relationItem in relation.Items)
{
switch (relationItem.MemberType)
{
case RelationMemberTypes.Node:
if (!nodes.TryGetValue(relationItem.Id, out var node))
{
itemInfos.Add(new RelationItemInfo
{
Id = (ulong)relationItem.Id,
Type = relationItem.MemberType,
MidLat = int.MinValue,
MidLon = int.MinValue
});
}
else
{
rect.Extend(node.Lat, node.Lon);
itemInfos.Add(new RelationItemInfo
{
Id = (ulong)relationItem.Id,
Type = relationItem.MemberType,
MidLat = node.Lat,
MidLon = node.Lon
});
}
break;
case RelationMemberTypes.Way:
var way = waysData.FindWayInfo((ulong)relationItem.Id);
if (way == null)
{
itemInfos.Add(new RelationItemInfo
{
Id = (ulong)relationItem.Id,
Type = relationItem.MemberType,
MidLat = int.MinValue,
MidLon = int.MinValue
});
}
else
{
rect.Extend(way.Rect);
itemInfos.Add(new RelationItemInfo
{
Id = (ulong)relationItem.Id,
Type = relationItem.MemberType,
MidLat = int.MinValue,
MidLon = int.MinValue
});
}
break;
case RelationMemberTypes.Relation:
itemInfos.Add(new RelationItemInfo
{
Id = (ulong)relationItem.Id,
Type = relationItem.MemberType,
MidLat = 0,
MidLon = 0
});
// Relations will be assembled recursively.
break;
default:
throw new NotSupportedException("Unknown relation type.");
}
}
relationsFile.Add(relation, 0, itemInfos, rect); // ToDo: Implement type detection
}
}
public async Task Process()
{
var poolCount = 1; //Environment.ProcessorCount + 2;
var pool = new Semaphore(poolCount, poolCount);
var watch = Stopwatch.StartNew();
var waitWatch = new Stopwatch();
try
{
var parser = new PbfBlobParser(stream);
Blob blob;
// Skip to first way offset. ToDo: remove this.
parser.SkipBlob(processStartOffset);
while ((blob = await PbfBlobParser.ReadBlobAsync(parser)) != null)
{
if (blob.Header.Type != "OSMData")
{
continue;
}
Console.Write($"\rOffset: {blob.Header.StartPosition:#,###}, {waitWatch.Elapsed}/{watch.Elapsed}.");
var data = processor.BlobRead(blob);
var reader = PbfPrimitiveReader.Create(blob);
var accessor = new PrimitiveAccessor(reader);
accessor.StartRead();
waitWatch.Start();
pool.WaitOne();
waitWatch.Stop();
Task.Run(async() =>
{
try
{
processor.ProcessPrimitives(accessor, data);
}
catch (Exception e)
{
Console.WriteLine(e);
}
finally
{
pool.Release(1);
}
});
}
for (var i = 0; i < poolCount; i++)
{
pool.WaitOne();
}
processor.Finish();
}
catch (Exception e)
{
Console.WriteLine($"Error processing file. After reading {stream.Position} of {stream.Length} bytes.");
Console.WriteLine(e);
}
}
public void ProcessPrimitives(PrimitiveAccessor accessor, BlobIdsInfo info)
{
if (info != null)
{
var nodeCnt = 0;
var minId = long.MaxValue;
var maxId = long.MinValue;
var waysCnt = 0;
var relationCnt = 0;
var nodes = accessor.Nodes;
foreach (var node in nodes)
{
minId = Math.Min(minId, node.Id);
maxId = Math.Max(maxId, node.Id);
nodeCnt++;
var lon = (uint)Math.Round((node.Lon - 180) / 180 * uint.MaxValue);
var lat = (uint)Math.Round((node.Lat - 90) / 90 * uint.MaxValue);
//nodesIndexWriter.Write(node.Id, lon, lat);
}
if (!waysStarted && accessor.Ways.Any())
{
waysStarted = true;
Console.WriteLine($"\r\nWays offset: {info.StartPosition}.");
}
foreach (var way in accessor.Ways)
{
waysCnt++;
}
if (!relationsStarted && accessor.Relations.Any())
{
relationsStarted = true;
Console.WriteLine($"\r\nRelations offset: {info.StartPosition}.");
}
foreach (var relation in accessor.Relations)
{
relationCnt++;
totalRelsCount++;
var itemsCount = relation.Items.Count;
maxMembersCount = Math.Max(maxMembersCount, itemsCount);
totalMemberCount += (ulong)itemsCount;
if (itemsCount > 400)
{
itemsCount = 400;
}
if (itemsCount < memberCounts.Length)
{
memberCounts[itemsCount]++;
}
}
Console.WriteLine($"\r\nMax Count: {maxMembersCount:#,###}. Avg: {1.0 * totalMemberCount / totalRelsCount}. Total: {totalRelsCount:#,###}.");
for (int i = 1; i <= 400; i++)
{
Console.Write($"{i:000}: {memberCounts[i]:00000}, ");
if (i % 10 == 0)
{
Console.WriteLine();
}
}
Console.SetCursorPosition(0, 0);
info.MinNodeId = minId;
info.MaxNodeId = maxId;
info.NodesCount = nodeCnt;
info.WaysCount = waysCnt;
info.RelationsCount = relationCnt;
}
}
