/// <summary>
/// Gets the process result.
/// </summary>
public FlowSnapShot <TTarget> GetProcessResult()
{
// load results to get the invalid items
FlowSnapShot <TTarget> result = _resultRepo.Get(DataSource.FullName);
return(result);
}
protected void AndGivenAProcessResult()
{
this._processResult = new FlowSnapShot <Whole>()
{
SourceType = new FlowEntity(typeof(Part)),
TargetType = new FlowEntity(typeof(Whole)),
Batch = _flowBatch,
Invalid = new List <Whole>(),
Valid = new List <Whole>() // valid entities
{
new Whole()
{
Key = "Key1", FirstName = "A", LastName = "LastName"
},
new Whole()
{
Key = "Key2", FirstName = "", LastName = ""
},
}
};
_wholeDir = $@"{_workingDirectory}\Whole";
_repo = new FlowSnapshotRepo <FlowSnapShot <Whole> >(_wholeDir);
_repo.Save(_processResult);
}
protected void AndWhenChainingTheProcessedResultsToLastProcessor()
{
// chain 2
var processorC = new Processor <Dto2, Address>(
new ProcessorConfiguration <Dto2, Address>(
_specProviderFroAddress,
new SpecProvider <IEnumerable <Address> >()),
new ProcessorEngine <Dto2>())
{
CreateOutput = dtoIn => dtoIn.Addresses,
// save to database
OnCommitting = processedItems =>
{
// save addresses now to database and update fk reference obtained above
using (var db = new TestModel())
{
db.Addresses.AddRange(processedItems);
// save results
db.SaveChanges();
}
}
};
//
_resultC = processorC.Process(_resultB.Valid, _batchProcess);
// save
_repository.Save(_resultC);
}
/// <summary>
/// Process a file file within a given batch.
/// </summary>
/// <param name="flowFile">The address to read the primary source entities from.</param>
/// <param name="flowBatch">The batch to process the file in.</param>
/// <returns></returns>
public virtual FlowSnapshot Process(FileInfo flowFile, FlowBatch flowBatch)
{
try
{
// read the incoming batch of data
IEnumerable <TIn> incomingData = _reader.Read(flowFile.FullName);
// create the processor to batch process it
var processor = GetProcessor();
// save results to output directory
if (!Directory.Exists(Config.OutDirectory))
{
Directory.CreateDirectory(Config.OutDirectory);
}
// process incoming data into a snapshot result
FlowSnapShot <TOut> result = processor.Process(incomingData, flowBatch);
// save results
var outputRepository = new FlowSnapshotRepo <FlowSnapShot <TOut> >(Config.OutDirectory);
var targetAddressId = outputRepository.Save(result);
return(result);
}
catch (Exception ex)
{
throw new Exception(
$"Failed to process flow file {flowFile.Name} due to an unexpected error. Batch process is {flowBatch}.",
ex);
}
}
protected void WhenProcessingADenormalizedFileUsingProcessingRules()
{
var processorA = new Processor <DenormalizedEntity, Dto1>(
new ProcessorConfiguration <DenormalizedEntity, Dto1>(_specProvider,
new SpecProvider <IEnumerable <Dto1> >()),
new ProcessorEngine <DenormalizedEntity>())
{
BeginProcessingItem = (dtoIn, dtoOut) =>
{
dtoOut.Source = dtoIn;
dtoOut.Contact = new Contact
{
Id = 0,
Name = dtoIn.ContactName,
Description = dtoIn.ContactDescription
};
}
};
_processorA = processorA;
// process from csv
var enumeration = new CsvProcessorReader <DenormalizedEntity>().Read(DataFileToCreate);
_resultA = processorA.Process(enumeration, _batchProcess);
_repository.Save(_resultA);
// todo: save to data store
}
protected void ThenShouldNotBeAbleToLoadNonExistingSnapShot()
{
var flowSnapShot = new FlowSnapShot <TargetType>(_batch);
var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TargetType> >();
var loadedSnapShot = resultRepo.Get(
new FlowEntity(typeof(TargetType)).EntityTypeId,
"SampleFlow",
2); // batch does not exist
Assert.Null(loadedSnapShot);
}
private void ProcessSnapShot(FlowSnapShot <TTarget> outResult, IEnumerable <TTarget> results)
{
var valid = new List <TTarget>();
var inValid = new List <TTarget>();
// output errors
var processErrors = new List <ErrorEvent>();
foreach (var entityOut in results)
{
// validate enity
var errors = _entityRules.ToErrors(entityOut).ToList();
if (!errors.Any())
{
// no error - valid
valid.Add(entityOut);
}
else
{
// get all errors and add to error collection
foreach (var error in errors)
{
processErrors.Add(new ErrorEvent {
Message = error.Message, Type = error.Type
});
}
inValid.Add(entityOut);
}
}
// validate collection
var collectionErrors = _entityCollection.ToErrors(valid);
if (!collectionErrors.Any())
{
outResult.Invalid = inValid;
outResult.Valid = valid;
}
else
{
// add all to invalid list
inValid.AddRange(valid);
// result.Output = 0
outResult.Invalid = inValid;
outResult.Valid.Clear();
}
// reset process errors
outResult.Errors = processErrors;
}
protected void AndWhenChainingTheProcessedResultsToAnotherProcessor()
{
// chain 2
var processorB = new Processor <Dto1, Dto2>(
new ProcessorConfiguration <Dto1, Dto2>(
new SpecProvider <Dto2>(),
new SpecProvider <IEnumerable <Dto2> >()),
new ProcessorEngine <Dto1>())
{
BeginProcessingItem = (dtoIn, dtoOut) =>
{
// map object and preserve incoming result
dtoOut.Source = dtoIn.Source;
dtoOut.Contact = dtoIn.Contact;
// don't update FK references or database ids
dtoOut.Addresses = new[]
{
new Address
{
StreetName = dtoIn.Source.Address1
},
new Address
{
StreetName = dtoIn.Source.Address2
}
};
},
// save to database
OnCommitting = processedItems =>
{
// save contacts to database
using (var db = new TestModel())
{
// ReSharper disable once PossibleMultipleEnumeration
db.Contacts.AddRange(processedItems.Select(m => m.Contact));
// save results to update contact idss
db.SaveChanges();
// update mappings and fk references
// ReSharper disable once PossibleMultipleEnumeration
processedItems.Each(m => { m.Addresses.Each(n => { n.ContactId = m.Contact.Id; }); });
}
}
};
_resultB = processorB.Process(_resultA.Valid, _batchProcess);
_repository.Save(_resultB);
}
protected void ThenShouldBeAbleToLoadTheSnapShot()
{
var flowSnapShot = new FlowSnapShot <TargetType>(_batch);
var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TargetType> >();
var loadedSnapShot = resultRepo.Get(
new FlowEntity(typeof(TargetType)).EntityTypeId,
"SampleFlow",
1);
Assert.NotNull(loadedSnapShot);
Assert.NotNull(loadedSnapShot.Valid.FirstOrDefault(m => m.Name == "Name"));
Assert.NotNull(loadedSnapShot.Valid.FirstOrDefault(m => m.MaybeInt == 2));
Assert.True(loadedSnapShot.Valid.Last()?.MaybeInt == 3);
Assert.NotNull(loadedSnapShot.TargetType);
}
protected void WhenSavingASnapShot()
{
var flowSnapShot = new FlowSnapShot <TargetType>(_batch);
flowSnapShot.Valid = new List <TargetType>()
{
new TargetType()
{
Name = "Name", MaybeDate = DateTime.UtcNow.Date, MaybeInt = 2
},
new TargetType()
{
Name = "Name-2", MaybeDate = DateTime.UtcNow.Date, MaybeInt = 3
}
};
var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TargetType> >();
resultRepo.Save(flowSnapShot);
}
protected void AndWhenChainingTheProcessedResultsToLastProcessor()
{
// chain 2
var processorC = _container.Resolve <Processor <Dto2, Address> >();
processorC.CreateOutput = dtoIn => dtoIn.Addresses;
// save to database
processorC.OnCommitting = processedItems =>
{
// save addresses now to database and update fk reference obtained above
using (var db = new TestModel())
{
db.Addresses.AddRange(processedItems);
// save results
db.SaveChanges();
}
};
_resultC = processorC.Process(_resultB.Valid, _batchProcess);
_repository.Save(_resultC);
}
protected void WhenProcessingADenormalizedFileUsingProcessingRules()
{
var processorA = _container.Resolve <Processor <DenormalizedEntity, Dto1> >();
processorA.BeginProcessingItem = (dtoIn, dtoOut) =>
{
dtoOut.Source = dtoIn;
dtoOut.Contact = new Contact
{
Id = 0,
Name = dtoIn.ContactName,
Description = dtoIn.ContactDescription
};
};
_processorA = processorA;
// process from csv
var enumeration = new CsvProcessorReader <DenormalizedEntity>().Read(DataFileToCreate);
_resultA = processorA.Process(enumeration, _batchProcess);
_repository.Save(_resultA);
}
/// <summary>
/// Process unriched data from a given targetDirectory file.
/// </summary>
public void Process(
FlowBatch flowBatch,
IEnumerable <IEnricher <TTarget> > enrichers,
IEnrichmentTarget <TTarget> target)
{
foreach (var enricher in enrichers)
{
// load by targetDirectory id
var logRepository = _logRepo.Get(_flow, enricher.SourceEntityType);
if (logRepository == null) // todo replace with flow
{
logRepository = new EnricherLog(flowBatch.Flow, enricher.SourceEntityType);
}
// enrichers
var unProcessedEnrichers = new List <IEnricher <TTarget> >();
// aggregate list of enrichers that haven't been processed for the target
if (logRepository.GetHasBeenProcessed(enricher, target.AddressId))
{
if (_logger.IsTraceEnabled)
{
_logger.Trace("Target has already been updated.");
}
continue;
}
// enrich valid and invalid items
var enrichmentController = new EnricherProcessor(_logRepo);
enrichmentController
.Enrich(flowBatch, new[] { target }, unProcessedEnrichers);
// get results to save
var results = target.Get();
// output result
var outResult = new FlowSnapShot <TTarget>(
flowBatch,
enricher.SourceEntityType,
enricher.AddressId,
new FlowEntity(typeof(TTarget)),
target.AddressId)
{
Batch = flowBatch,
TargetType = new FlowEntity(typeof(TTarget)),
SourceType = enricher.SourceEntityType,
SourceAddressId = enricher.AddressId,
};
// process and save new enriched file
ProcessSnapShot(outResult, results);
// save new flow file
// targetDirectory repository
var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TTarget> >()
{
DataDir = this.DataDir.FullName
};
// save resports
resultRepo.Save(outResult);
}
}