/// <summary> /// Processes data from the source to the target. /// </summary> public void Process() { _timer.Start(); var targetsToEnrich = GetTargetsToEnrich(); // load by source id foreach (var result in targetsToEnrich) { // gets all the enrichers to process against var unProcessedEnrichers = GetEnrichers(result.ToString()); //enricher.Enrich() var enrichmentController = new EnricherProcessor(); // enrich valid and invalid items enrichmentController .Enrich(result.Invalid, unProcessedEnrichers); // enrich valid and invalid items enrichmentController .Enrich(result.Output, unProcessedEnrichers); // save new output ProcessResults(result); } }
protected void WhenProcessingEnrichmentsAgainstTheSource() { _controller.Enrich(_flowBatch, new[] { _enrichmentTarget }, _enrichers); }
/// <summary> /// Process unriched data from a given targetDirectory file. /// </summary> public void Process( FlowBatch flowBatch, IEnumerable <IEnricher <TTarget> > enrichers, IEnrichmentTarget <TTarget> target) { foreach (var enricher in enrichers) { // load by targetDirectory id var logRepository = _logRepo.Get(_flow, enricher.SourceEntityType); if (logRepository == null) // todo replace with flow { logRepository = new EnricherLog(flowBatch.Flow, enricher.SourceEntityType); } // enrichers var unProcessedEnrichers = new List <IEnricher <TTarget> >(); // aggregate list of enrichers that haven't been processed for the target if (logRepository.GetHasBeenProcessed(enricher, target.AddressId)) { if (_logger.IsTraceEnabled) { _logger.Trace("Target has already been updated."); } continue; } // enrich valid and invalid items var enrichmentController = new EnricherProcessor(_logRepo); enrichmentController .Enrich(flowBatch, new[] { target }, unProcessedEnrichers); // get results to save var results = target.Get(); // output result var outResult = new FlowSnapShot <TTarget>( flowBatch, enricher.SourceEntityType, enricher.AddressId, new FlowEntity(typeof(TTarget)), target.AddressId) { Batch = flowBatch, TargetType = new FlowEntity(typeof(TTarget)), SourceType = enricher.SourceEntityType, SourceAddressId = enricher.AddressId, }; // process and save new enriched file ProcessSnapShot(outResult, results); // save new flow file // targetDirectory repository var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TTarget> >() { DataDir = this.DataDir.FullName }; // save resports resultRepo.Save(outResult); } }