protected void AndGivenAProcessResult()
{
this._processResult = new FlowSnapShot <Whole>()
{
SourceType = new FlowEntity(typeof(Part)),
TargetType = new FlowEntity(typeof(Whole)),
Batch = _flowBatch,
Invalid = new List <Whole>(),
Valid = new List <Whole>() // valid entities
{
new Whole()
{
Key = "Key1", FirstName = "A", LastName = "LastName"
},
new Whole()
{
Key = "Key2", FirstName = "", LastName = ""
},
}
};
_wholeDir = $@"{_workingDirectory}\Whole";
_repo = new FlowSnapshotRepo <FlowSnapShot <Whole> >(_wholeDir);
_repo.Save(_processResult);
}
/// <summary>
/// Process a file file within a given batch.
/// </summary>
/// <param name="flowFile">The address to read the primary source entities from.</param>
/// <param name="flowBatch">The batch to process the file in.</param>
/// <returns></returns>
public virtual FlowSnapshot Process(FileInfo flowFile, FlowBatch flowBatch)
{
try
{
// read the incoming batch of data
IEnumerable <TIn> incomingData = _reader.Read(flowFile.FullName);
// create the processor to batch process it
var processor = GetProcessor();
// save results to output directory
if (!Directory.Exists(Config.OutDirectory))
{
Directory.CreateDirectory(Config.OutDirectory);
}
// process incoming data into a snapshot result
FlowSnapShot <TOut> result = processor.Process(incomingData, flowBatch);
// save results
var outputRepository = new FlowSnapshotRepo <FlowSnapShot <TOut> >(Config.OutDirectory);
var targetAddressId = outputRepository.Save(result);
return(result);
}
catch (Exception ex)
{
throw new Exception(
$"Failed to process flow file {flowFile.Name} due to an unexpected error. Batch process is {flowBatch}.",
ex);
}
}
protected void WhenSavingASnapShot()
{
var flowSnapShot = new FlowSnapShot <TargetType>(_batch);
flowSnapShot.Valid = new List <TargetType>()
{
new TargetType()
{
Name = "Name", MaybeDate = DateTime.UtcNow.Date, MaybeInt = 2
},
new TargetType()
{
Name = "Name-2", MaybeDate = DateTime.UtcNow.Date, MaybeInt = 3
}
};
var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TargetType> >();
resultRepo.Save(flowSnapShot);
}
/// <summary>
/// Process unriched data from a given targetDirectory file.
/// </summary>
public void Process(
FlowBatch flowBatch,
IEnumerable <IEnricher <TTarget> > enrichers,
IEnrichmentTarget <TTarget> target)
{
foreach (var enricher in enrichers)
{
// load by targetDirectory id
var logRepository = _logRepo.Get(_flow, enricher.SourceEntityType);
if (logRepository == null) // todo replace with flow
{
logRepository = new EnricherLog(flowBatch.Flow, enricher.SourceEntityType);
}
// enrichers
var unProcessedEnrichers = new List <IEnricher <TTarget> >();
// aggregate list of enrichers that haven't been processed for the target
if (logRepository.GetHasBeenProcessed(enricher, target.AddressId))
{
if (_logger.IsTraceEnabled)
{
_logger.Trace("Target has already been updated.");
}
continue;
}
// enrich valid and invalid items
var enrichmentController = new EnricherProcessor(_logRepo);
enrichmentController
.Enrich(flowBatch, new[] { target }, unProcessedEnrichers);
// get results to save
var results = target.Get();
// output result
var outResult = new FlowSnapShot <TTarget>(
flowBatch,
enricher.SourceEntityType,
enricher.AddressId,
new FlowEntity(typeof(TTarget)),
target.AddressId)
{
Batch = flowBatch,
TargetType = new FlowEntity(typeof(TTarget)),
SourceType = enricher.SourceEntityType,
SourceAddressId = enricher.AddressId,
};
// process and save new enriched file
ProcessSnapShot(outResult, results);
// save new flow file
// targetDirectory repository
var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TTarget> >()
{
DataDir = this.DataDir.FullName
};
// save resports
resultRepo.Save(outResult);
}
}