public static async Task ProcessBlob(string blobHeaderType,
PbfPrimitiveReader pbfPrimitiveReader, BlobIdsInfo info)
{
var nodeCnt = 0;
var minId = long.MaxValue;
var maxId = long.MinValue;
var waysCnt = 0;
var relationCnt = 0;
if (blobHeaderType == "OSMHeader")
{
var header = await pbfPrimitiveReader.ReadHeader();
}
if (blobHeaderType == "OSMData")
{
var data = pbfPrimitiveReader.ReadData();
if (data != null || data.PrimitiveGroup != null)
{
var nodes = PrimitiveDecoder.DecodeDenseNodes(data);
foreach (var node in nodes)
{
minId = Math.Min(minId, node.Id);
maxId = Math.Max(maxId, node.Id);
nodeCnt++;
var lon = (uint)Math.Round((node.Lon - 180) / 180 * uint.MaxValue);
var lat = (uint)Math.Round((node.Lat - 90) / 90 * uint.MaxValue);
//nodesIndexWriter.Write(node.Id, lon, lat);
}
foreach (var way in data.PrimitiveGroup.Where(x => x.Ways != null).SelectMany(x => x.Ways))
{
waysCnt++;
}
foreach (var relation in data.PrimitiveGroup.Where(x => x.Relations != null)
.SelectMany(x => x.Relations))
{
relationCnt++;
}
}
}
info.MinNodeId = minId;
info.MaxNodeId = maxId;
info.NodesCount = nodeCnt;
info.WaysCount = waysCnt;
info.RelationsCount = relationCnt;
}
public static async Task Process(string fileName)
{
ulong maxBlobSize = 0;
long maxDataSize = 0;
var blobCnt = 0;
long nodeCnt = 0;
var poolCount = Environment.ProcessorCount + 2;
var pool = new Semaphore(poolCount, poolCount);
using (var stream = File.OpenRead(fileName))
{
try
{
var parser = new PbfBlobParser(stream);
// parser.SkipBlob(1789874958 - 4);
while (true)
{
var blobHeader = await parser.ReadBlobHeader();
if (blobHeader == null)
{
break;
}
blobCnt++;
if (blobCnt < 0)
{
parser.SkipBlob(blobHeader.DataSize);
Console.Write($" Skipped: {blobCnt}. \r");
continue;
}
var message = await parser.ReadBlobAsync(blobHeader);
maxBlobSize = Math.Max(maxBlobSize, blobHeader.DataSize);
maxDataSize = Math.Max(maxDataSize, message.RawSize);
var headerType = blobHeader.Type;
var reader = PbfPrimitiveReader.Create(message);
var startPosition = blobHeader.StartPosition;
var info = new BlobIdsInfo
{
StartPosition = startPosition
};
infos.Add(info);
pool.WaitOne();
Console.Write($" Nodes: {nodeCnt.ToString("#,##0", CultureInfo.CurrentUICulture)}.\r");
Task.Run(async() =>
{
try
{
await ProcessBlob(headerType, reader, info);
Interlocked.Add(ref nodeCnt, info.NodesCount);
}
catch (Exception e)
{
Console.WriteLine(e);
}
finally
{
pool.Release(1);
}
});
}
for (var i = 0; i < poolCount; i++)
{
pool.WaitOne();
}
BlobIdsInfo.WriteIdsIndex(fileName, infos);
}
catch (Exception e)
{
Console.WriteLine($"Error processing file. After reading {stream.Position} of {stream.Length} bytes.");
Console.WriteLine(e);
}
}
Console.WriteLine(
$"Statistic: max blob {maxBlobSize}, data: {maxDataSize}, total cnt: {blobCnt}, nodes: {nodeCnt}");
Console.ReadLine();
}
