public static async Task Process(string fileName, string indexFileName) { var index = BlobIdsInfo.ReadIdsIndex(indexFileName).ToList(); Console.WriteLine($"Index size: {index.Count}"); using (var stream = File.OpenRead(fileName)) { foreach (var info in index.Where(x => x.WaysCount > 0).Take(5)) { stream.Position = 0; var parser = new PbfBlobParser(stream); parser.SkipBlob((ulong)(info.StartPosition - 4)); var header = await parser.ReadBlobHeader(); var blob = await parser.ReadBlobAsync(header); var primitiveReader = PbfPrimitiveReader.Create(blob); var data = primitiveReader.ReadData(); var ways = PrimitiveDecoder.DecodeWays(data).ToList(); Console.WriteLine(); foreach (var way in ways) { var nodeIds = way.NodeIds.Distinct().OrderBy(x => x).ToList(); var blobsToRead = FindBlobIdsToRead(index, nodeIds); Console.WriteLine($"Id: {way.Id}, Cnt: {blobsToRead.Count}. {string.Join(", ",blobsToRead)}"); } } } Console.WriteLine("Done! Press any key...."); Console.ReadKey(); }
public async Task Process() { var poolCount = 1; //Environment.ProcessorCount + 2; var pool = new Semaphore(poolCount, poolCount); var watch = Stopwatch.StartNew(); var waitWatch = new Stopwatch(); try { var parser = new PbfBlobParser(stream); Blob blob; // Skip to first way offset. ToDo: remove this. parser.SkipBlob(processStartOffset); while ((blob = await PbfBlobParser.ReadBlobAsync(parser)) != null) { if (blob.Header.Type != "OSMData") { continue; } Console.Write($"\rOffset: {blob.Header.StartPosition:#,###}, {waitWatch.Elapsed}/{watch.Elapsed}."); var data = processor.BlobRead(blob); var reader = PbfPrimitiveReader.Create(blob); var accessor = new PrimitiveAccessor(reader); accessor.StartRead(); waitWatch.Start(); pool.WaitOne(); waitWatch.Stop(); Task.Run(async() => { try { processor.ProcessPrimitives(accessor, data); } catch (Exception e) { Console.WriteLine(e); } finally { pool.Release(1); } }); } for (var i = 0; i < poolCount; i++) { pool.WaitOne(); } processor.Finish(); } catch (Exception e) { Console.WriteLine($"Error processing file. After reading {stream.Position} of {stream.Length} bytes."); Console.WriteLine(e); } }