Ejemplo n.º 1
0
        /// <summary>
        ///     Gets the process result.
        /// </summary>
        public FlowSnapShot <TTarget> GetProcessResult()
        {
            // load results to get the invalid items
            FlowSnapShot <TTarget> result = _resultRepo.Get(DataSource.FullName);

            return(result);
        }
Ejemplo n.º 2
0
        protected void AndGivenAProcessResult()
        {
            this._processResult = new FlowSnapShot <Whole>()
            {
                SourceType = new FlowEntity(typeof(Part)),
                TargetType = new FlowEntity(typeof(Whole)),
                Batch      = _flowBatch,
                Invalid    = new List <Whole>(),
                Valid      = new List <Whole>() // valid entities
                {
                    new Whole()
                    {
                        Key = "Key1", FirstName = "A", LastName = "LastName"
                    },
                    new Whole()
                    {
                        Key = "Key2", FirstName = "", LastName = ""
                    },
                }
            };

            _wholeDir = $@"{_workingDirectory}\Whole";

            _repo = new FlowSnapshotRepo <FlowSnapShot <Whole> >(_wholeDir);
            _repo.Save(_processResult);
        }
Ejemplo n.º 3
0
        protected void AndWhenChainingTheProcessedResultsToLastProcessor()
        {
            // chain 2
            var processorC = new Processor <Dto2, Address>(
                new ProcessorConfiguration <Dto2, Address>(
                    _specProviderFroAddress,
                    new SpecProvider <IEnumerable <Address> >()),
                new ProcessorEngine <Dto2>())
            {
                CreateOutput = dtoIn => dtoIn.Addresses,
                // save to database
                OnCommitting = processedItems =>
                {
                    // save addresses now to database and update fk reference obtained above
                    using (var db = new TestModel())
                    {
                        db.Addresses.AddRange(processedItems);

                        // save results
                        db.SaveChanges();
                    }
                }
            };

            //
            _resultC = processorC.Process(_resultB.Valid, _batchProcess);

            // save
            _repository.Save(_resultC);
        }
Ejemplo n.º 4
0
        /// <summary>
        ///     Process a file file within a given batch.
        /// </summary>
        /// <param name="flowFile">The address to read the primary source entities from.</param>
        /// <param name="flowBatch">The batch to process the file in.</param>
        /// <returns></returns>
        public virtual FlowSnapshot Process(FileInfo flowFile, FlowBatch flowBatch)
        {
            try
            {
                // read the incoming batch of data
                IEnumerable <TIn> incomingData = _reader.Read(flowFile.FullName);

                // create the processor to batch process it
                var processor = GetProcessor();

                // save results to output directory
                if (!Directory.Exists(Config.OutDirectory))
                {
                    Directory.CreateDirectory(Config.OutDirectory);
                }

                // process incoming data into a snapshot result
                FlowSnapShot <TOut> result = processor.Process(incomingData, flowBatch);

                // save results
                var outputRepository = new FlowSnapshotRepo <FlowSnapShot <TOut> >(Config.OutDirectory);

                var targetAddressId = outputRepository.Save(result);

                return(result);
            }
            catch (Exception ex)
            {
                throw new Exception(
                          $"Failed to process flow file {flowFile.Name} due to an unexpected error. Batch process is {flowBatch}.",
                          ex);
            }
        }
Ejemplo n.º 5
0
        protected void WhenProcessingADenormalizedFileUsingProcessingRules()
        {
            var processorA = new Processor <DenormalizedEntity, Dto1>(
                new ProcessorConfiguration <DenormalizedEntity, Dto1>(_specProvider,
                                                                      new SpecProvider <IEnumerable <Dto1> >()),
                new ProcessorEngine <DenormalizedEntity>())
            {
                BeginProcessingItem = (dtoIn, dtoOut) =>
                {
                    dtoOut.Source  = dtoIn;
                    dtoOut.Contact = new Contact
                    {
                        Id          = 0,
                        Name        = dtoIn.ContactName,
                        Description = dtoIn.ContactDescription
                    };
                }
            };

            _processorA = processorA;

            // process from csv
            var enumeration = new CsvProcessorReader <DenormalizedEntity>().Read(DataFileToCreate);

            _resultA = processorA.Process(enumeration, _batchProcess);

            _repository.Save(_resultA);

            // todo: save to data store
        }
        protected void ThenShouldNotBeAbleToLoadNonExistingSnapShot()
        {
            var flowSnapShot = new FlowSnapShot <TargetType>(_batch);
            var resultRepo   = new FlowSnapshotRepo <FlowSnapShot <TargetType> >();

            var loadedSnapShot = resultRepo.Get(
                new FlowEntity(typeof(TargetType)).EntityTypeId,
                "SampleFlow",
                2); // batch does not exist

            Assert.Null(loadedSnapShot);
        }
Ejemplo n.º 7
0
        private void ProcessSnapShot(FlowSnapShot <TTarget> outResult, IEnumerable <TTarget> results)
        {
            var valid   = new List <TTarget>();
            var inValid = new List <TTarget>();

            // output errors
            var processErrors = new List <ErrorEvent>();

            foreach (var entityOut in results)
            {
                // validate enity
                var errors = _entityRules.ToErrors(entityOut).ToList();

                if (!errors.Any())
                {
                    // no error - valid
                    valid.Add(entityOut);
                }
                else
                {
                    // get all errors and add to error collection
                    foreach (var error in errors)
                    {
                        processErrors.Add(new ErrorEvent {
                            Message = error.Message, Type = error.Type
                        });
                    }

                    inValid.Add(entityOut);
                }
            }

            // validate collection
            var collectionErrors = _entityCollection.ToErrors(valid);

            if (!collectionErrors.Any())
            {
                outResult.Invalid = inValid;
                outResult.Valid   = valid;
            }
            else
            {
                // add all to invalid list
                inValid.AddRange(valid);

                // result.Output = 0
                outResult.Invalid = inValid;
                outResult.Valid.Clear();
            }

            // reset process errors
            outResult.Errors = processErrors;
        }
Ejemplo n.º 8
0
        protected void AndWhenChainingTheProcessedResultsToAnotherProcessor()
        {
            // chain 2
            var processorB = new Processor <Dto1, Dto2>(
                new ProcessorConfiguration <Dto1, Dto2>(
                    new SpecProvider <Dto2>(),
                    new SpecProvider <IEnumerable <Dto2> >()),
                new ProcessorEngine <Dto1>())
            {
                BeginProcessingItem = (dtoIn, dtoOut) =>
                {
                    // map object and preserve incoming result
                    dtoOut.Source  = dtoIn.Source;
                    dtoOut.Contact = dtoIn.Contact;
                    // don't update FK references or database ids
                    dtoOut.Addresses = new[]
                    {
                        new Address
                        {
                            StreetName = dtoIn.Source.Address1
                        },
                        new Address
                        {
                            StreetName = dtoIn.Source.Address2
                        }
                    };
                },
                // save to database
                OnCommitting = processedItems =>
                {
                    // save contacts to database
                    using (var db = new TestModel())
                    {
                        // ReSharper disable once PossibleMultipleEnumeration
                        db.Contacts.AddRange(processedItems.Select(m => m.Contact));

                        // save results to update contact idss
                        db.SaveChanges();

                        // update mappings and fk references
                        // ReSharper disable once PossibleMultipleEnumeration
                        processedItems.Each(m => { m.Addresses.Each(n => { n.ContactId = m.Contact.Id; }); });
                    }
                }
            };

            _resultB = processorB.Process(_resultA.Valid, _batchProcess);

            _repository.Save(_resultB);
        }
        protected void ThenShouldBeAbleToLoadTheSnapShot()
        {
            var flowSnapShot = new FlowSnapShot <TargetType>(_batch);
            var resultRepo   = new FlowSnapshotRepo <FlowSnapShot <TargetType> >();

            var loadedSnapShot = resultRepo.Get(
                new FlowEntity(typeof(TargetType)).EntityTypeId,
                "SampleFlow",
                1);

            Assert.NotNull(loadedSnapShot);

            Assert.NotNull(loadedSnapShot.Valid.FirstOrDefault(m => m.Name == "Name"));
            Assert.NotNull(loadedSnapShot.Valid.FirstOrDefault(m => m.MaybeInt == 2));
            Assert.True(loadedSnapShot.Valid.Last()?.MaybeInt == 3);

            Assert.NotNull(loadedSnapShot.TargetType);
        }
        protected void WhenSavingASnapShot()
        {
            var flowSnapShot = new FlowSnapShot <TargetType>(_batch);

            flowSnapShot.Valid = new List <TargetType>()
            {
                new TargetType()
                {
                    Name = "Name", MaybeDate = DateTime.UtcNow.Date, MaybeInt = 2
                },
                new TargetType()
                {
                    Name = "Name-2", MaybeDate = DateTime.UtcNow.Date, MaybeInt = 3
                }
            };
            var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TargetType> >();

            resultRepo.Save(flowSnapShot);
        }
Ejemplo n.º 11
0
        protected void AndWhenChainingTheProcessedResultsToLastProcessor()
        {
            // chain 2
            var processorC = _container.Resolve <Processor <Dto2, Address> >();

            processorC.CreateOutput = dtoIn => dtoIn.Addresses;
            // save to database
            processorC.OnCommitting = processedItems =>
            {
                // save addresses now to database and update fk reference obtained above
                using (var db = new TestModel())
                {
                    db.Addresses.AddRange(processedItems);

                    // save results
                    db.SaveChanges();
                }
            };

            _resultC = processorC.Process(_resultB.Valid, _batchProcess);

            _repository.Save(_resultC);
        }
Ejemplo n.º 12
0
        protected void WhenProcessingADenormalizedFileUsingProcessingRules()
        {
            var processorA = _container.Resolve <Processor <DenormalizedEntity, Dto1> >();

            processorA.BeginProcessingItem = (dtoIn, dtoOut) =>
            {
                dtoOut.Source  = dtoIn;
                dtoOut.Contact = new Contact
                {
                    Id          = 0,
                    Name        = dtoIn.ContactName,
                    Description = dtoIn.ContactDescription
                };
            };

            _processorA = processorA;

            // process from csv
            var enumeration = new CsvProcessorReader <DenormalizedEntity>().Read(DataFileToCreate);

            _resultA = processorA.Process(enumeration, _batchProcess);

            _repository.Save(_resultA);
        }
Ejemplo n.º 13
0
        /// <summary>
        ///     Process unriched data from a given targetDirectory file.
        /// </summary>
        public void Process(
            FlowBatch flowBatch,
            IEnumerable <IEnricher <TTarget> > enrichers,
            IEnrichmentTarget <TTarget> target)
        {
            foreach (var enricher in enrichers)
            {
                // load by targetDirectory id
                var logRepository = _logRepo.Get(_flow, enricher.SourceEntityType);

                if (logRepository == null) // todo replace with flow
                {
                    logRepository = new EnricherLog(flowBatch.Flow, enricher.SourceEntityType);
                }

                // enrichers
                var unProcessedEnrichers = new List <IEnricher <TTarget> >();

                // aggregate list of enrichers that haven't been processed for the target
                if (logRepository.GetHasBeenProcessed(enricher, target.AddressId))
                {
                    if (_logger.IsTraceEnabled)
                    {
                        _logger.Trace("Target has already been updated.");
                    }

                    continue;
                }

                // enrich valid and invalid items
                var enrichmentController = new EnricherProcessor(_logRepo);
                enrichmentController
                .Enrich(flowBatch, new[] { target }, unProcessedEnrichers);

                // get results to save
                var results = target.Get();

                // output result
                var outResult = new FlowSnapShot <TTarget>(
                    flowBatch,
                    enricher.SourceEntityType,
                    enricher.AddressId,
                    new FlowEntity(typeof(TTarget)),
                    target.AddressId)
                {
                    Batch           = flowBatch,
                    TargetType      = new FlowEntity(typeof(TTarget)),
                    SourceType      = enricher.SourceEntityType,
                    SourceAddressId = enricher.AddressId,
                };

                // process and save new enriched file
                ProcessSnapShot(outResult, results);

                // save new flow file
                // targetDirectory repository
                var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TTarget> >()
                {
                    DataDir = this.DataDir.FullName
                };

                // save resports
                resultRepo.Save(outResult);
            }
        }