/// <summary>
/// Process a file file within a given batch.
/// </summary>
/// <param name="flowFile">The address to read the primary source entities from.</param>
/// <param name="flowBatch">The batch to process the file in.</param>
/// <returns></returns>
public virtual FlowSnapshot Process(FileInfo flowFile, FlowBatch flowBatch)
{
try
{
// read the incoming batch of data
IEnumerable <TIn> incomingData = _reader.Read(flowFile.FullName);
// create the processor to batch process it
var processor = GetProcessor();
// save results to output directory
if (!Directory.Exists(Config.OutDirectory))
{
Directory.CreateDirectory(Config.OutDirectory);
}
// process incoming data into a snapshot result
FlowSnapShot <TOut> result = processor.Process(incomingData, flowBatch);
// save results
var outputRepository = new FlowSnapshotRepo <FlowSnapShot <TOut> >(Config.OutDirectory);
var targetAddressId = outputRepository.Save(result);
return(result);
}
catch (Exception ex)
{
throw new Exception(
$"Failed to process flow file {flowFile.Name} due to an unexpected error. Batch process is {flowBatch}.",
ex);
}
}
protected void GivenANewFlowAndFlowBatch()
{
// clean up existing files
foreach (var file in Directory.GetFiles(Environment.CurrentDirectory, "*SampleFlow*.json"))
{
File.Delete(file);
}
_flowService = new FlowService(new FlowIdRepo());
_batch = _flowService.GetNewFlowBatch("SampleFlow");
}
protected void AndGivenANewFlow()
{
// cleanup any existing file
foreach (var file in Directory.GetFiles(Environment.CurrentDirectory, "*Test*.json"))
{
File.Delete(file);
}
this._flowService = new FlowService(new FlowIdRepo());
this._flow = this._flowService.CreateNew("Test");
this._flowBatch = this._flowService.GetNewFlowBatch("Test");
}
FlowBatch GetNewBatchProcess()
{
// load flow file
var flowFile = _flowFileRepo.Get(FlowId);
// save
_flowFileRepo.Save(flowFile);
// load batch process
var batchProcess = new FlowBatch()
{
FlowId = flowFile.FlowId,
BatchId = flowFile.BatchId
};
return(batchProcess);
}
protected void GivenAFlowAndABatchProcess()
{
this._flow = new FlowId("Test");
this._flowBatch = new FlowBatch(_flow, 1);
}
protected void AndGivenAbatchProcess()
{
_batchProcess = new FlowBatch(_flow, BatchId);
}
/// <summary>
/// Creates a new instance.
/// </summary>
public InMemoryEnrichmentTarget(IEnumerable <TTarget> source, FlowBatch batch, string uniqueAddressId)
{
this._uniqueAddressId = uniqueAddressId;
this._batch = batch;
this._source = source;
}
/// <summary>
/// Enriches the source data with data from parts.
/// </summary>
/// <typeparam name="TTarget">The whole data type to enrich.</typeparam>
/// <param name="targets">The data source for 'whole' objects that will be enriched..</param>
/// <param name="enrichers">The sources to enrich 'whole' target objects.</param>
/// <returns></returns>
public void Enrich <TTarget>(
FlowBatch flowBatch,
IEnumerable <IEnrichmentTarget <TTarget> > targets,
IEnumerable <IEnricher <TTarget> > enrichers)
{
if (_logger.IsDebugEnabled)
{
_logger.Debug("Enriching targets.");
}
// parallelize enrichment of the targets
foreach (var enricher in enrichers.AsParallel())
{
// enrichment log to record transactions
var db = new EnricherLog(flowBatch.Flow, enricher.SourceEntityType);
// list of items to create
// ReSharper disable once PossibleMultipleEnumeration
foreach (var target in targets)
{
// record event so that it is not duplicated
var logEntry = new EnrichmentLogEntry
{
DateCreated = DateTime.UtcNow,
Batch = flowBatch,
SourceAddressId = enricher.AddressId,
TargetAddressId = target.AddressId,
TargetEntity = FlowEntity.Create <TTarget>()
};
if (_logger.IsDebugEnabled)
{
_logger.Debug($"Enriching target with enricher {enricher}.");
}
foreach (var entity in target.Get())
{
try
{
// find the parts that can enrich the whole
var foundParts = enricher.Find(entity);
// ReSharper disable once SuspiciousTypeConversion.Global
foreach (var part in foundParts)
{
try
{
// enrich the whole from the part
enricher.Enrich(part, entity);
// increment entities enriched
logEntry.EntitiesEnriched++;
}
catch (Exception ex)
{
throw new ApplicationException(
$"Failed to enrich {target} from {part} due to an unexpected error.",
ex);
}
}
}
catch (ApplicationException aex)
{
db.Exceptions.Add(aex);
throw;
}
catch (Exception ex)
{
db.Exceptions.Add(ex);
throw new Exception(
$"Failed to enrich whole {target} from source {targets} due to an unexpected error.", ex);
}
}
// update transasction log
lock (this)
{
// the date/time completed
logEntry.Completed = DateTime.UtcNow;
// add log entry after completes
db.Logs.Add(logEntry);
}
}
this._logRepository.Save(db);
}
}
/// <summary>
/// Process unriched data from a given targetDirectory file.
/// </summary>
public void Process(
FlowBatch flowBatch,
IEnumerable <IEnricher <TTarget> > enrichers,
IEnrichmentTarget <TTarget> target)
{
foreach (var enricher in enrichers)
{
// load by targetDirectory id
var logRepository = _logRepo.Get(_flow, enricher.SourceEntityType);
if (logRepository == null) // todo replace with flow
{
logRepository = new EnricherLog(flowBatch.Flow, enricher.SourceEntityType);
}
// enrichers
var unProcessedEnrichers = new List <IEnricher <TTarget> >();
// aggregate list of enrichers that haven't been processed for the target
if (logRepository.GetHasBeenProcessed(enricher, target.AddressId))
{
if (_logger.IsTraceEnabled)
{
_logger.Trace("Target has already been updated.");
}
continue;
}
// enrich valid and invalid items
var enrichmentController = new EnricherProcessor(_logRepo);
enrichmentController
.Enrich(flowBatch, new[] { target }, unProcessedEnrichers);
// get results to save
var results = target.Get();
// output result
var outResult = new FlowSnapShot <TTarget>(
flowBatch,
enricher.SourceEntityType,
enricher.AddressId,
new FlowEntity(typeof(TTarget)),
target.AddressId)
{
Batch = flowBatch,
TargetType = new FlowEntity(typeof(TTarget)),
SourceType = enricher.SourceEntityType,
SourceAddressId = enricher.AddressId,
};
// process and save new enriched file
ProcessSnapShot(outResult, results);
// save new flow file
// targetDirectory repository
var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TTarget> >()
{
DataDir = this.DataDir.FullName
};
// save resports
resultRepo.Save(outResult);
}
}
protected void AndGivenAbatchProcess()
{
_batchProcess = new FlowBatch(new FutureState.Flow.FlowId(_flowCode), BatchId);
}