protected void WhenProcessingTheEnrichment()
{
// get list of enrichers
var enrichers = new List <Enricher <Part, Whole> >();
foreach (var file in Directory.GetFiles(_partDir))
{
var enricherBuilder = new CsvEnricherBuilder <Part, Whole>()
{
FilePath = file
};
enrichers.Add(enricherBuilder.Get());
}
// get list of files to process
var targetEnrichers = new List <EnrichmentTarget <WholeSource, Whole> >();
foreach (var file in new DirectoryInfo(_wholeDir).GetFiles())
{
var repository = new FlowSnapshotRepo <FlowSnapShot <Whole> >();
targetEnrichers.Add(new EnrichmentTarget <WholeSource, Whole>(repository)
{
DataSource = file
});
}
// process results
foreach (var target in targetEnrichers)
{
_subject.Process(_flowBatch, enrichers, target);
}
}
protected void AndGivenAProcessResult()
{
this._processResult = new FlowSnapShot <Whole>()
{
SourceType = new FlowEntity(typeof(Part)),
TargetType = new FlowEntity(typeof(Whole)),
Batch = _flowBatch,
Invalid = new List <Whole>(),
Valid = new List <Whole>() // valid entities
{
new Whole()
{
Key = "Key1", FirstName = "A", LastName = "LastName"
},
new Whole()
{
Key = "Key2", FirstName = "", LastName = ""
},
}
};
_wholeDir = $@"{_workingDirectory}\Whole";
_repo = new FlowSnapshotRepo <FlowSnapShot <Whole> >(_wholeDir);
_repo.Save(_processResult);
}
/// <summary>
/// Process a file file within a given batch.
/// </summary>
/// <param name="flowFile">The address to read the primary source entities from.</param>
/// <param name="flowBatch">The batch to process the file in.</param>
/// <returns></returns>
public virtual FlowSnapshot Process(FileInfo flowFile, FlowBatch flowBatch)
{
try
{
// read the incoming batch of data
IEnumerable <TIn> incomingData = _reader.Read(flowFile.FullName);
// create the processor to batch process it
var processor = GetProcessor();
// save results to output directory
if (!Directory.Exists(Config.OutDirectory))
{
Directory.CreateDirectory(Config.OutDirectory);
}
// process incoming data into a snapshot result
FlowSnapShot <TOut> result = processor.Process(incomingData, flowBatch);
// save results
var outputRepository = new FlowSnapshotRepo <FlowSnapShot <TOut> >(Config.OutDirectory);
var targetAddressId = outputRepository.Save(result);
return(result);
}
catch (Exception ex)
{
throw new Exception(
$"Failed to process flow file {flowFile.Name} due to an unexpected error. Batch process is {flowBatch}.",
ex);
}
}
/// <summary>
/// Gets the reader function.
/// </summary>
/// <returns></returns>
private static IReader <TIn> GetReader()
{
return(new GenericResultReader <TIn>(flowFileSource =>
{
var repoository = new FlowSnapshotRepo <FlowSnapShot <TIn> >(flowFileSource);
var processResult = repoository.Get(flowFileSource);
return processResult.Valid;
}));
}
protected void ThenShouldNotBeAbleToLoadNonExistingSnapShot()
{
var flowSnapShot = new FlowSnapShot <TargetType>(_batch);
var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TargetType> >();
var loadedSnapShot = resultRepo.Get(
new FlowEntity(typeof(TargetType)).EntityTypeId,
"SampleFlow",
2); // batch does not exist
Assert.Null(loadedSnapShot);
}
protected void AndThenShouldBeAbleToRestoreProcessState()
{
var repo =
new FlowSnapshotRepo <FlowSnapShot <Dto1> >(Environment.CurrentDirectory);
var targetEntityType = new FlowEntity(typeof(Dto2));
var result = repo.Get(targetEntityType.EntityTypeId, _flow.Code, BatchId);
Assert.NotNull(result);
Assert.Equal(CsvItemsToCreate - 1, result.ProcessedCount);
Assert.Empty(result.Errors);
}
protected void ThenShouldBeAbleToLoadTheSnapShot()
{
var flowSnapShot = new FlowSnapShot <TargetType>(_batch);
var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TargetType> >();
var loadedSnapShot = resultRepo.Get(
new FlowEntity(typeof(TargetType)).EntityTypeId,
"SampleFlow",
1);
Assert.NotNull(loadedSnapShot);
Assert.NotNull(loadedSnapShot.Valid.FirstOrDefault(m => m.Name == "Name"));
Assert.NotNull(loadedSnapShot.Valid.FirstOrDefault(m => m.MaybeInt == 2));
Assert.True(loadedSnapShot.Valid.Last()?.MaybeInt == 3);
Assert.NotNull(loadedSnapShot.TargetType);
}
protected void WhenSavingASnapShot()
{
var flowSnapShot = new FlowSnapShot <TargetType>(_batch);
flowSnapShot.Valid = new List <TargetType>()
{
new TargetType()
{
Name = "Name", MaybeDate = DateTime.UtcNow.Date, MaybeInt = 2
},
new TargetType()
{
Name = "Name-2", MaybeDate = DateTime.UtcNow.Date, MaybeInt = 3
}
};
var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TargetType> >();
resultRepo.Save(flowSnapShot);
}
protected void AndGivenAWiredUpContainer()
{
_cb = new ContainerBuilder();
_cb.RegisterGeneric(typeof(SpecProvider <>))
.As(typeof(IProvideSpecifications <>))
.As(typeof(SpecProvider <>))
.SingleInstance();
_cb.RegisterGeneric(typeof(Processor <,>))
.As(typeof(Processor <,>));
_cb.RegisterGeneric(typeof(ProcessorEngine <>))
.As(typeof(ProcessorEngine <>));
_cb.RegisterGeneric(typeof(ProcessorConfiguration <,>))
.As(typeof(ProcessorConfiguration <,>));
//specialist
_cb.Register(m =>
{
var s = new SpecProvider <Dto1>();
s.Add(a =>
{
if (a.Source.Key == "Key-5")
{
return(new SpecResult("Arbitrary invalid reason."));
}
return(SpecResult.Success);
}, "Key", "Description");
s.MergeFrom(mq => mq.Contact, m.Resolve <SpecProvider <Contact> >());
return(s);
}).AsSelf().AsImplementedInterfaces();
_cb.Register(m =>
{
var s = new SpecProvider <Address>();
s.Add(a =>
{
if (a.ContactId == 0)
{
return(new SpecResult("Contact Id has not been assigned."));
}
return(SpecResult.Success);
}, "Key", "Description");
return(s);
}).AsSelf().AsImplementedInterfaces();
_cb.Register(m =>
{
var s = new FlowSnapshotRepo <FlowSnapshot>(Environment.CurrentDirectory);
return(s);
}).AsSelf().AsImplementedInterfaces();
_container = _cb.Build();
}
protected void AndGivenAProcessorResultsRepository()
{
_repository = new FlowSnapshotRepo <FlowSnapshot>(Environment.CurrentDirectory);
}
/// <summary>
/// Creates a new instance.
/// </summary>
public EnrichmentTarget(FlowSnapshotRepo <FlowSnapShot <TTarget> > repository)
{
// targetDirectory repository
this._resultRepo = repository ?? new FlowSnapshotRepo <FlowSnapShot <TTarget> >();
}
/// <summary>
/// Process unriched data from a given targetDirectory file.
/// </summary>
public void Process(
FlowBatch flowBatch,
IEnumerable <IEnricher <TTarget> > enrichers,
IEnrichmentTarget <TTarget> target)
{
foreach (var enricher in enrichers)
{
// load by targetDirectory id
var logRepository = _logRepo.Get(_flow, enricher.SourceEntityType);
if (logRepository == null) // todo replace with flow
{
logRepository = new EnricherLog(flowBatch.Flow, enricher.SourceEntityType);
}
// enrichers
var unProcessedEnrichers = new List <IEnricher <TTarget> >();
// aggregate list of enrichers that haven't been processed for the target
if (logRepository.GetHasBeenProcessed(enricher, target.AddressId))
{
if (_logger.IsTraceEnabled)
{
_logger.Trace("Target has already been updated.");
}
continue;
}
// enrich valid and invalid items
var enrichmentController = new EnricherProcessor(_logRepo);
enrichmentController
.Enrich(flowBatch, new[] { target }, unProcessedEnrichers);
// get results to save
var results = target.Get();
// output result
var outResult = new FlowSnapShot <TTarget>(
flowBatch,
enricher.SourceEntityType,
enricher.AddressId,
new FlowEntity(typeof(TTarget)),
target.AddressId)
{
Batch = flowBatch,
TargetType = new FlowEntity(typeof(TTarget)),
SourceType = enricher.SourceEntityType,
SourceAddressId = enricher.AddressId,
};
// process and save new enriched file
ProcessSnapShot(outResult, results);
// save new flow file
// targetDirectory repository
var resultRepo = new FlowSnapshotRepo <FlowSnapShot <TTarget> >()
{
DataDir = this.DataDir.FullName
};
// save resports
resultRepo.Save(outResult);
}
}
