public static Task <IntegratedDocument> Wrap(object o)
{
var doc = new IntegratedDocument();
doc.Document = new Lazy <BsonDocument>(() => o.ToBsonDocument());
return(Task.FromResult(doc));
}
public IIntegratedDocument Clone()
{
var newDocument = new IntegratedDocument();
newDocument.Document = new Lazy <BsonDocument>(CloneDocument);
newDocument.Reserved = Reserved.Clone().ToBsonDocument();
newDocument.APIId = this.APIId;
newDocument.IntegrationId = this.IntegrationId;
return(newDocument);
}
/// <summary>
/// Runs the donut.
/// </summary>
/// <param name="donut"></param>
/// <param name="getFeatureGenerator"></param>
/// <returns></returns>
public async Task <IHarvesterResult> Run(TDonut donut, IFeatureGenerator <TData> getFeatureGenerator)
{
var integration = donut.Context.Integration;
//Create our destination block
var donutBlock = donut.CreateDataflowBlock(getFeatureGenerator);
var dataProcessingBlock = donutBlock.FlowBlock;
_featuresBlock = donutBlock.FeaturePropagator;
var insertCreator = new TransformBlock <FeaturesWrapper <TData>, BsonDocument>((x) =>
{
var rawFeatures = new BsonDocument();
var featuresDocument = new IntegratedDocument(rawFeatures);
//add some cleanup, or feature document definition, because right now the original document is used
//either clean it up or create a new one with just the features.
//if (doc.Document.Value.Contains("events")) doc.Document.Value.Remove("events");
//if (doc.Document.Value.Contains("browsing_statistics")) doc.Document.Value.Remove("browsing_statistics");
foreach (var featurePair in x.Features)
{
var name = featurePair.Key;
if (string.IsNullOrEmpty(name))
{
continue;
}
var featureval = featurePair.Value;
rawFeatures.Set(name, BsonValue.Create(featureval));
}
featuresDocument.IntegrationId = integration.Id;
featuresDocument.APIId = integration.APIKey.Id;
x.Features = null;
return(rawFeatures);
});
var insertBatcher = new MongoInsertBatch <BsonDocument>(_featuresCollection, 3000);
insertCreator.LinkTo(insertBatcher.BatchBlock, new DataflowLinkOptions {
PropagateCompletion = true
});
//Insert our features
_featuresBlock.LinkTo(insertCreator, new DataflowLinkOptions {
PropagateCompletion = true
});
//After all data is processed, extract the features
dataProcessingBlock.ContinueWith(() =>
{
var extractionTask = RunFeatureExtraction(donut);
Task.WaitAll(extractionTask);
});
_harvester.SetDestination(dataProcessingBlock);
var harvesterRun = await _harvester.Run();
//If we have to repeat it, handle this..
return(harvesterRun);
}