public IObservable<ParkingArea> Extract(IMongoQuery query) { return Observable.Create<ParkingArea>(async observer => { var totalEntitiesToProcess = serverProvider.Database.OsmCompoundQueryCount(query); var operationCounter = new OperationCounter(log, string.Format("OSM Parking Area Extraction from {0}",query), totalEntitiesToProcess); var cursor = serverProvider.Database.OsmCompoundQuery(query); foreach (var entity in cursor) { var entityContext = new OsmEntityContext(serverProvider.Database, entity); var parkingArea = new ParkingArea() { Id = Guid.NewGuid(), Source = ParkingAreaSource.Maps, Confidence = configuration.Model.Confidence }; var isValid = true; foreach (var extractionRule in extractionRules) { var continueExtraction = await extractionRule.Extract(entityContext, parkingArea); if (!continueExtraction) { log.Info("Entity {0} was dropped from extraction by rule {1}", entity, extractionRule.GetType().Name.Replace("Extraction", string.Empty).Replace("Rule", string.Empty)); isValid = false; break; } } if (isValid) { observer.OnNext(parkingArea); } operationCounter.Increment(); } }); }
public IObservable<OsmEntity> ReadEntities(XmlReader reader,Stream underlyingStream,OperationCountDelegate progressDelegate) { var mapper = new OsmEntityMapper(); var streamLength = underlyingStream.Length; var operationCounter = new OperationCounter(progressDelegate, streamLength, 1000*1000*100); reader.MoveToContent(); return CreateElementSource(reader).Select<XElement, OsmEntity>(element => { var increment = underlyingStream.Position - operationCounter.Count; operationCounter.Increment(increment); if (element.Name == "node") { return mapper.ToNode(element); } if (element.Name == "way") { return mapper.ToWay(element); } if (element.Name == "relation") { return mapper.ToRelation(element); } return null; }).Where(e => e != null); }
public async Task<long> ImportFile(string filename) { var operationCounter = new OperationCounter(OperationCounter.DefaultActionNoPercent(log, string.Format("OSM entity import from {0}",filename)), callGranularity: 1000 * 100); var reader = new OsmEntityReader(); var completionSource = new TaskCompletionSource<int>(); reader.ReadEntities(filename, OperationCounter.DefaultAction(log, string.Format("OSM entity import from {0} (file progress)",filename))).Buffer(1024 * 10).Subscribe(batch => { var groups = batch.GroupBy(b => b.Type); foreach (var group in groups) { var collection = serverProvider.Database.GetOsmCollection(group.Key); collection.InsertBatch(group); } operationCounter.Increment(batch.Count); }, () => completionSource.SetResult(0)); await completionSource.Task; return operationCounter.Count; }