public IObservable<ParkingArea> Extract(IMongoQuery query)
{
return Observable.Create<ParkingArea>(async observer =>
{
var totalEntitiesToProcess = serverProvider.Database.OsmCompoundQueryCount(query);
var operationCounter = new OperationCounter(log, string.Format("OSM Parking Area Extraction from {0}",query), totalEntitiesToProcess);
var cursor = serverProvider.Database.OsmCompoundQuery(query);
foreach (var entity in cursor)
{
var entityContext = new OsmEntityContext(serverProvider.Database, entity);
var parkingArea = new ParkingArea() { Id = Guid.NewGuid(), Source = ParkingAreaSource.Maps, Confidence = configuration.Model.Confidence };
var isValid = true;
foreach (var extractionRule in extractionRules)
{
var continueExtraction = await extractionRule.Extract(entityContext, parkingArea);
if (!continueExtraction)
{
log.Info("Entity {0} was dropped from extraction by rule {1}", entity, extractionRule.GetType().Name.Replace("Extraction", string.Empty).Replace("Rule", string.Empty));
isValid = false;
break;
}
}
if (isValid)
{
observer.OnNext(parkingArea);
}
operationCounter.Increment();
}
});
}
public IObservable<OsmEntity> ReadEntities(XmlReader reader,Stream underlyingStream,OperationCountDelegate progressDelegate)
{
var mapper = new OsmEntityMapper();
var streamLength = underlyingStream.Length;
var operationCounter = new OperationCounter(progressDelegate, streamLength, 1000*1000*100);
reader.MoveToContent();
return CreateElementSource(reader).Select<XElement, OsmEntity>(element =>
{
var increment = underlyingStream.Position - operationCounter.Count;
operationCounter.Increment(increment);
if (element.Name == "node")
{
return mapper.ToNode(element);
}
if (element.Name == "way")
{
return mapper.ToWay(element);
}
if (element.Name == "relation")
{
return mapper.ToRelation(element);
}
return null;
}).Where(e => e != null);
}
public async Task<long> ImportFile(string filename)
{
var operationCounter = new OperationCounter(OperationCounter.DefaultActionNoPercent(log, string.Format("OSM entity import from {0}",filename)), callGranularity: 1000 * 100);
var reader = new OsmEntityReader();
var completionSource = new TaskCompletionSource<int>();
reader.ReadEntities(filename, OperationCounter.DefaultAction(log, string.Format("OSM entity import from {0} (file progress)",filename))).Buffer(1024 * 10).Subscribe(batch =>
{
var groups = batch.GroupBy(b => b.Type);
foreach (var group in groups)
{
var collection = serverProvider.Database.GetOsmCollection(group.Key);
collection.InsertBatch(group);
}
operationCounter.Increment(batch.Count);
}, () => completionSource.SetResult(0));
await completionSource.Task;
return operationCounter.Count;
}
