protected void WhenProcessingTheEnrichment() { // get list of enrichers var enrichers = new List <Enricher <Part, Whole> >(); foreach (var file in Directory.GetFiles(_partDir)) { var enricherBuilder = new CsvEnricherBuilder <Part, Whole>() { FilePath = file }; enrichers.Add(enricherBuilder.Get()); } // get list of files to process var targetEnrichers = new List <EnrichmentTarget <WholeSource, Whole> >(); foreach (var file in new DirectoryInfo(_wholeDir).GetFiles()) { var repository = new FlowSnapshotRepo <FlowSnapShot <Whole> >(); targetEnrichers.Add(new EnrichmentTarget <WholeSource, Whole>(repository) { DataSource = file }); } // process results foreach (var target in targetEnrichers) { _subject.Process(_flowBatch, enrichers, target); } }
protected void AndGivenAProcessResult() { this._processResult = new FlowSnapShot <Whole>() { SourceType = new FlowEntity(typeof(Part)), TargetType = new FlowEntity(typeof(Whole)), Batch = _flowBatch, Invalid = new List <Whole>(), Valid = new List <Whole>() // valid entities { new Whole() { Key = "Key1", FirstName = "A", LastName = "LastName" }, new Whole() { Key = "Key2", FirstName = "", LastName = "" }, } }; _wholeDir = $@"{_workingDirectory}\Whole"; _repo = new FlowSnapshotRepo <FlowSnapShot <Whole> >(_wholeDir); _repo.Save(_processResult); }
/// <summary> /// Process a file file within a given batch. /// </summary> /// <param name="flowFile">The address to read the primary source entities from.</param> /// <param name="flowBatch">The batch to process the file in.</param> /// <returns></returns> public virtual FlowSnapshot Process(FileInfo flowFile, FlowBatch flowBatch) { try { // read the incoming batch of data IEnumerable <TIn> incomingData = _reader.Read(flowFile.FullName); // create the processor to batch process it var processor = GetProcessor(); // save results to output directory if (!Directory.Exists(Config.OutDirectory)) { Directory.CreateDirectory(Config.OutDirectory); } // process incoming data into a snapshot result FlowSnapShot <TOut> result = processor.Process(incomingData, flowBatch); // save results var outputRepository = new FlowSnapshotRepo <FlowSnapShot <TOut> >(Config.OutDirectory); var targetAddressId = outputRepository.Save(result); return(result); } catch (Exception ex) { throw new Exception( $"Failed to process flow file {flowFile.Name} due to an unexpected error. Batch process is {flowBatch}.", ex); } }
/// <summary> /// Gets the reader function. /// </summary> /// <returns></returns> private static IReader <TIn> GetReader() { return(new GenericResultReader <TIn>(flowFileSource => { var repoository = new FlowSnapshotRepo <FlowSnapShot <TIn> >(flowFileSource); var processResult = repoository.Get(flowFileSource); return processResult.Valid; })); }
protected void ThenShouldNotBeAbleToLoadNonExistingSnapShot() { var flowSnapShot = new FlowSnapShot <TargetType>(_batch); var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TargetType> >(); var loadedSnapShot = resultRepo.Get( new FlowEntity(typeof(TargetType)).EntityTypeId, "SampleFlow", 2); // batch does not exist Assert.Null(loadedSnapShot); }
protected void AndThenShouldBeAbleToRestoreProcessState() { var repo = new FlowSnapshotRepo <FlowSnapShot <Dto1> >(Environment.CurrentDirectory); var targetEntityType = new FlowEntity(typeof(Dto2)); var result = repo.Get(targetEntityType.EntityTypeId, _flow.Code, BatchId); Assert.NotNull(result); Assert.Equal(CsvItemsToCreate - 1, result.ProcessedCount); Assert.Empty(result.Errors); }
protected void ThenShouldBeAbleToLoadTheSnapShot() { var flowSnapShot = new FlowSnapShot <TargetType>(_batch); var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TargetType> >(); var loadedSnapShot = resultRepo.Get( new FlowEntity(typeof(TargetType)).EntityTypeId, "SampleFlow", 1); Assert.NotNull(loadedSnapShot); Assert.NotNull(loadedSnapShot.Valid.FirstOrDefault(m => m.Name == "Name")); Assert.NotNull(loadedSnapShot.Valid.FirstOrDefault(m => m.MaybeInt == 2)); Assert.True(loadedSnapShot.Valid.Last()?.MaybeInt == 3); Assert.NotNull(loadedSnapShot.TargetType); }
protected void WhenSavingASnapShot() { var flowSnapShot = new FlowSnapShot <TargetType>(_batch); flowSnapShot.Valid = new List <TargetType>() { new TargetType() { Name = "Name", MaybeDate = DateTime.UtcNow.Date, MaybeInt = 2 }, new TargetType() { Name = "Name-2", MaybeDate = DateTime.UtcNow.Date, MaybeInt = 3 } }; var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TargetType> >(); resultRepo.Save(flowSnapShot); }
protected void AndGivenAWiredUpContainer() { _cb = new ContainerBuilder(); _cb.RegisterGeneric(typeof(SpecProvider <>)) .As(typeof(IProvideSpecifications <>)) .As(typeof(SpecProvider <>)) .SingleInstance(); _cb.RegisterGeneric(typeof(Processor <,>)) .As(typeof(Processor <,>)); _cb.RegisterGeneric(typeof(ProcessorEngine <>)) .As(typeof(ProcessorEngine <>)); _cb.RegisterGeneric(typeof(ProcessorConfiguration <,>)) .As(typeof(ProcessorConfiguration <,>)); //specialist _cb.Register(m => { var s = new SpecProvider <Dto1>(); s.Add(a => { if (a.Source.Key == "Key-5") { return(new SpecResult("Arbitrary invalid reason.")); } return(SpecResult.Success); }, "Key", "Description"); s.MergeFrom(mq => mq.Contact, m.Resolve <SpecProvider <Contact> >()); return(s); }).AsSelf().AsImplementedInterfaces(); _cb.Register(m => { var s = new SpecProvider <Address>(); s.Add(a => { if (a.ContactId == 0) { return(new SpecResult("Contact Id has not been assigned.")); } return(SpecResult.Success); }, "Key", "Description"); return(s); }).AsSelf().AsImplementedInterfaces(); _cb.Register(m => { var s = new FlowSnapshotRepo <FlowSnapshot>(Environment.CurrentDirectory); return(s); }).AsSelf().AsImplementedInterfaces(); _container = _cb.Build(); }
protected void AndGivenAProcessorResultsRepository() { _repository = new FlowSnapshotRepo <FlowSnapshot>(Environment.CurrentDirectory); }
/// <summary> /// Creates a new instance. /// </summary> public EnrichmentTarget(FlowSnapshotRepo <FlowSnapShot <TTarget> > repository) { // targetDirectory repository this._resultRepo = repository ?? new FlowSnapshotRepo <FlowSnapShot <TTarget> >(); }
/// <summary> /// Process unriched data from a given targetDirectory file. /// </summary> public void Process( FlowBatch flowBatch, IEnumerable <IEnricher <TTarget> > enrichers, IEnrichmentTarget <TTarget> target) { foreach (var enricher in enrichers) { // load by targetDirectory id var logRepository = _logRepo.Get(_flow, enricher.SourceEntityType); if (logRepository == null) // todo replace with flow { logRepository = new EnricherLog(flowBatch.Flow, enricher.SourceEntityType); } // enrichers var unProcessedEnrichers = new List <IEnricher <TTarget> >(); // aggregate list of enrichers that haven't been processed for the target if (logRepository.GetHasBeenProcessed(enricher, target.AddressId)) { if (_logger.IsTraceEnabled) { _logger.Trace("Target has already been updated."); } continue; } // enrich valid and invalid items var enrichmentController = new EnricherProcessor(_logRepo); enrichmentController .Enrich(flowBatch, new[] { target }, unProcessedEnrichers); // get results to save var results = target.Get(); // output result var outResult = new FlowSnapShot <TTarget>( flowBatch, enricher.SourceEntityType, enricher.AddressId, new FlowEntity(typeof(TTarget)), target.AddressId) { Batch = flowBatch, TargetType = new FlowEntity(typeof(TTarget)), SourceType = enricher.SourceEntityType, SourceAddressId = enricher.AddressId, }; // process and save new enriched file ProcessSnapShot(outResult, results); // save new flow file // targetDirectory repository var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TTarget> >() { DataDir = this.DataDir.FullName }; // save resports resultRepo.Save(outResult); } }