protected void AndGivenAProcessResult() { this._processResult = new FlowSnapShot <Whole>() { SourceType = new FlowEntity(typeof(Part)), TargetType = new FlowEntity(typeof(Whole)), Batch = _flowBatch, Invalid = new List <Whole>(), Valid = new List <Whole>() // valid entities { new Whole() { Key = "Key1", FirstName = "A", LastName = "LastName" }, new Whole() { Key = "Key2", FirstName = "", LastName = "" }, } }; _wholeDir = $@"{_workingDirectory}\Whole"; _repo = new FlowSnapshotRepo <FlowSnapShot <Whole> >(_wholeDir); _repo.Save(_processResult); }
/// <summary> /// Process a file file within a given batch. /// </summary> /// <param name="flowFile">The address to read the primary source entities from.</param> /// <param name="flowBatch">The batch to process the file in.</param> /// <returns></returns> public virtual FlowSnapshot Process(FileInfo flowFile, FlowBatch flowBatch) { try { // read the incoming batch of data IEnumerable <TIn> incomingData = _reader.Read(flowFile.FullName); // create the processor to batch process it var processor = GetProcessor(); // save results to output directory if (!Directory.Exists(Config.OutDirectory)) { Directory.CreateDirectory(Config.OutDirectory); } // process incoming data into a snapshot result FlowSnapShot <TOut> result = processor.Process(incomingData, flowBatch); // save results var outputRepository = new FlowSnapshotRepo <FlowSnapShot <TOut> >(Config.OutDirectory); var targetAddressId = outputRepository.Save(result); return(result); } catch (Exception ex) { throw new Exception( $"Failed to process flow file {flowFile.Name} due to an unexpected error. Batch process is {flowBatch}.", ex); } }
protected void WhenSavingASnapShot() { var flowSnapShot = new FlowSnapShot <TargetType>(_batch); flowSnapShot.Valid = new List <TargetType>() { new TargetType() { Name = "Name", MaybeDate = DateTime.UtcNow.Date, MaybeInt = 2 }, new TargetType() { Name = "Name-2", MaybeDate = DateTime.UtcNow.Date, MaybeInt = 3 } }; var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TargetType> >(); resultRepo.Save(flowSnapShot); }
/// <summary> /// Process unriched data from a given targetDirectory file. /// </summary> public void Process( FlowBatch flowBatch, IEnumerable <IEnricher <TTarget> > enrichers, IEnrichmentTarget <TTarget> target) { foreach (var enricher in enrichers) { // load by targetDirectory id var logRepository = _logRepo.Get(_flow, enricher.SourceEntityType); if (logRepository == null) // todo replace with flow { logRepository = new EnricherLog(flowBatch.Flow, enricher.SourceEntityType); } // enrichers var unProcessedEnrichers = new List <IEnricher <TTarget> >(); // aggregate list of enrichers that haven't been processed for the target if (logRepository.GetHasBeenProcessed(enricher, target.AddressId)) { if (_logger.IsTraceEnabled) { _logger.Trace("Target has already been updated."); } continue; } // enrich valid and invalid items var enrichmentController = new EnricherProcessor(_logRepo); enrichmentController .Enrich(flowBatch, new[] { target }, unProcessedEnrichers); // get results to save var results = target.Get(); // output result var outResult = new FlowSnapShot <TTarget>( flowBatch, enricher.SourceEntityType, enricher.AddressId, new FlowEntity(typeof(TTarget)), target.AddressId) { Batch = flowBatch, TargetType = new FlowEntity(typeof(TTarget)), SourceType = enricher.SourceEntityType, SourceAddressId = enricher.AddressId, }; // process and save new enriched file ProcessSnapShot(outResult, results); // save new flow file // targetDirectory repository var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TTarget> >() { DataDir = this.DataDir.FullName }; // save resports resultRepo.Save(outResult); } }