/// <summary>
/// Enriches the source data with data from parts.
/// </summary>
/// <typeparam name="TTarget">The whole data type to enrich.</typeparam>
/// <param name="targets">The data source for 'whole' objects that will be enriched..</param>
/// <param name="enrichers">The sources to enrich 'whole' target objects.</param>
/// <returns></returns>
public void Enrich <TTarget>(
FlowBatch flowBatch,
IEnumerable <IEnrichmentTarget <TTarget> > targets,
IEnumerable <IEnricher <TTarget> > enrichers)
{
if (_logger.IsDebugEnabled)
{
_logger.Debug("Enriching targets.");
}
// parallelize enrichment of the targets
foreach (var enricher in enrichers.AsParallel())
{
// enrichment log to record transactions
var db = new EnricherLog(flowBatch.Flow, enricher.SourceEntityType);
// list of items to create
// ReSharper disable once PossibleMultipleEnumeration
foreach (var target in targets)
{
// record event so that it is not duplicated
var logEntry = new EnrichmentLogEntry
{
DateCreated = DateTime.UtcNow,
Batch = flowBatch,
SourceAddressId = enricher.AddressId,
TargetAddressId = target.AddressId,
TargetEntity = FlowEntity.Create <TTarget>()
};
if (_logger.IsDebugEnabled)
{
_logger.Debug($"Enriching target with enricher {enricher}.");
}
foreach (var entity in target.Get())
{
try
{
// find the parts that can enrich the whole
var foundParts = enricher.Find(entity);
// ReSharper disable once SuspiciousTypeConversion.Global
foreach (var part in foundParts)
{
try
{
// enrich the whole from the part
enricher.Enrich(part, entity);
// increment entities enriched
logEntry.EntitiesEnriched++;
}
catch (Exception ex)
{
throw new ApplicationException(
$"Failed to enrich {target} from {part} due to an unexpected error.",
ex);
}
}
}
catch (ApplicationException aex)
{
db.Exceptions.Add(aex);
throw;
}
catch (Exception ex)
{
db.Exceptions.Add(ex);
throw new Exception(
$"Failed to enrich whole {target} from source {targets} due to an unexpected error.", ex);
}
}
// update transasction log
lock (this)
{
// the date/time completed
logEntry.Completed = DateTime.UtcNow;
// add log entry after completes
db.Logs.Add(logEntry);
}
}
this._logRepository.Save(db);
}
}
/// <summary>
/// Process unriched data from a given targetDirectory file.
/// </summary>
public void Process(
FlowBatch flowBatch,
IEnumerable <IEnricher <TTarget> > enrichers,
IEnrichmentTarget <TTarget> target)
{
foreach (var enricher in enrichers)
{
// load by targetDirectory id
var logRepository = _logRepo.Get(_flow, enricher.SourceEntityType);
if (logRepository == null) // todo replace with flow
{
logRepository = new EnricherLog(flowBatch.Flow, enricher.SourceEntityType);
}
// enrichers
var unProcessedEnrichers = new List <IEnricher <TTarget> >();
// aggregate list of enrichers that haven't been processed for the target
if (logRepository.GetHasBeenProcessed(enricher, target.AddressId))
{
if (_logger.IsTraceEnabled)
{
_logger.Trace("Target has already been updated.");
}
continue;
}
// enrich valid and invalid items
var enrichmentController = new EnricherProcessor(_logRepo);
enrichmentController
.Enrich(flowBatch, new[] { target }, unProcessedEnrichers);
// get results to save
var results = target.Get();
// output result
var outResult = new FlowSnapShot <TTarget>(
flowBatch,
enricher.SourceEntityType,
enricher.AddressId,
new FlowEntity(typeof(TTarget)),
target.AddressId)
{
Batch = flowBatch,
TargetType = new FlowEntity(typeof(TTarget)),
SourceType = enricher.SourceEntityType,
SourceAddressId = enricher.AddressId,
};
// process and save new enriched file
ProcessSnapShot(outResult, results);
// save new flow file
// targetDirectory repository
var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TTarget> >()
{
DataDir = this.DataDir.FullName
};
// save resports
resultRepo.Save(outResult);
}
}
