public static async Task Process(string fileName, string indexFileName)
{
var index = BlobIdsInfo.ReadIdsIndex(indexFileName).ToList();
Console.WriteLine($"Index size: {index.Count}");
using (var stream = File.OpenRead(fileName))
{
foreach (var info in index.Where(x => x.WaysCount > 0).Take(5))
{
stream.Position = 0;
var parser = new PbfBlobParser(stream);
parser.SkipBlob((ulong)(info.StartPosition - 4));
var header = await parser.ReadBlobHeader();
var blob = await parser.ReadBlobAsync(header);
var primitiveReader = PbfPrimitiveReader.Create(blob);
var data = primitiveReader.ReadData();
var ways = PrimitiveDecoder.DecodeWays(data).ToList();
Console.WriteLine();
foreach (var way in ways)
{
var nodeIds = way.NodeIds.Distinct().OrderBy(x => x).ToList();
var blobsToRead = FindBlobIdsToRead(index, nodeIds);
Console.WriteLine($"Id: {way.Id}, Cnt: {blobsToRead.Count}. {string.Join(", ",blobsToRead)}");
}
}
}
Console.WriteLine("Done! Press any key....");
Console.ReadKey();
}
public async Task Process()
{
var poolCount = 1; //Environment.ProcessorCount + 2;
var pool = new Semaphore(poolCount, poolCount);
var watch = Stopwatch.StartNew();
var waitWatch = new Stopwatch();
try
{
var parser = new PbfBlobParser(stream);
Blob blob;
// Skip to first way offset. ToDo: remove this.
parser.SkipBlob(processStartOffset);
while ((blob = await PbfBlobParser.ReadBlobAsync(parser)) != null)
{
if (blob.Header.Type != "OSMData")
{
continue;
}
Console.Write($"\rOffset: {blob.Header.StartPosition:#,###}, {waitWatch.Elapsed}/{watch.Elapsed}.");
var data = processor.BlobRead(blob);
var reader = PbfPrimitiveReader.Create(blob);
var accessor = new PrimitiveAccessor(reader);
accessor.StartRead();
waitWatch.Start();
pool.WaitOne();
waitWatch.Stop();
Task.Run(async() =>
{
try
{
processor.ProcessPrimitives(accessor, data);
}
catch (Exception e)
{
Console.WriteLine(e);
}
finally
{
pool.Release(1);
}
});
}
for (var i = 0; i < poolCount; i++)
{
pool.WaitOne();
}
processor.Finish();
}
catch (Exception e)
{
Console.WriteLine($"Error processing file. After reading {stream.Position} of {stream.Length} bytes.");
Console.WriteLine(e);
}
}
