/// <returns></returns>
public DataFileLog Get(FlowEntity flowEntity)
{
Guard.ArgumentNotNull(flowEntity, nameof(flowEntity));
lock (_syncLock)
{
var fileName = $@"{DataDir}\flow-{flowEntity.EntityTypeId}-log.json";
if (!File.Exists(fileName))
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"File {fileName} could not be found.");
}
return(default(DataFileLog));
}
if (_logger.IsInfoEnabled)
{
_logger.Info($"Reading data source log from {fileName}.");
}
var body = File.ReadAllText(fileName);
var result = JsonConvert.DeserializeObject <DataFileLog>(body);
if (_logger.IsInfoEnabled)
{
_logger.Info($"Read data source log {fileName}.");
}
return(result);
}
}
/// <summary>
/// Registers an entity against the flow.
/// </summary>
/// <typeparam name="TEntityType">The entity type to create.</typeparam>
/// <param name="flowCode">The flowto save the registered entity type against.</param>
/// <returns></returns>
public FlowEntity RegisterEntity <TEntityType>(string flowCode)
{
var flow = Get(flowCode);
string entityTypeId = typeof(TEntityType).Name;
if (flow.Entities.Any(m => string.Equals(m.EntityTypeId, entityTypeId)))
{
throw new InvalidOperationException($"The entity type has already been registered.");
}
var entity = new FlowEntity()
{
AssemblyQualifiedTypeName = typeof(TEntityType).AssemblyQualifiedName,
DateAdded = DateTime.UtcNow,
EntityTypeId = entityTypeId
};
// add to collection
flow.Entities.Add(entity);
// update
_repo.Save(flow);
//
return(entity);
}
/// <summary>
/// Gets the enrichment log for a given target process type.
/// </summary>
public EnricherLog Get(FlowId flow, FlowEntity sourceEntityType)
{
Guard.ArgumentNotNull(flow, nameof(flow));
Guard.ArgumentNotNull(sourceEntityType, nameof(sourceEntityType));
// source id would be the name of a processor
var fileName =
$@"{DataDir}\{flow}-Enrichment-{sourceEntityType.EntityTypeId}.json";
if (!File.Exists(fileName))
{
if (_logger.IsInfoEnabled)
{
_logger.Info($"File {fileName} could not be found.");
}
return(null);
}
var content = File.ReadAllText(fileName);
var log = JsonConvert.DeserializeObject <EnricherLog>(
content,
new JsonSerializerSettings());
return(log);
}
/// <summary>
/// Creates a new instance.
/// </summary>
/// <param name="entityType">The entity encoded in the data sources.</param>
/// <param name="filePattern">The file pattern to use to scan for data source snapshot files.</param>
public DataFileLog(FlowEntity entityType, string filePattern) : this()
{
Guard.ArgumentNotNull(entityType, nameof(entityType));
Entries = new List <DataFileLogEntry>();
EntityType = entityType;
FileTypes = filePattern;
}
/// <summary>
/// Creates a new instance
/// </summary>
public FlowSnapShot(
FlowBatch flowBatch,
FlowEntity sourceType, string sourceAddressId,
FlowEntity targetType, string targetAddressId)
: base(flowBatch, sourceType, sourceAddressId, targetType, targetAddressId)
{
this.Valid = new List <TEntityOut>();
this.Invalid = new List <TEntityOut>();
}
/// <summary>
/// Creates a new instance.
/// </summary>
/// <param name="flow">The related flow.</param>
/// <param name="sourceEntityType">The source entity type id.</param>
public EnricherLog(FlowId flow, FlowEntity sourceEntityType)
{
Guard.ArgumentNotNull(flow, nameof(flow));
Guard.ArgumentNotNull(sourceEntityType, nameof(sourceEntityType));
Flow = flow;
EntityType = sourceEntityType;
Logs = new List <EnrichmentLogEntry>();
Exceptions = new List <Exception>();
}
/// <summary>
/// Creates a new instance.
/// </summary>
/// <param name="flowEntity">The flow entity encoded in the directory.</param>
/// <param name="sourceDirectories">The directories to poll for new files.</param>
/// <param name="repository">The repository to populate with new files.</param>
/// <param name="polltime">The number of seconds to check for new data files and refresh the target directory.</param>
public DataSourceProducerFromDirectory(DirectoryInfo[] sourceDirectories, FlowEntity flowEntity, DataSourceLogRepo repository, int polltime = 1)
{
Guard.ArgumentNotNull(flowEntity, nameof(flowEntity));
Guard.ArgumentNotNull(repository, nameof(repository));
Guard.ArgumentNotNull(sourceDirectories, nameof(sourceDirectories));
_sourceDirectories = sourceDirectories;
_flowEntity = flowEntity;
_repository = repository;
PollInterval = TimeSpan.FromSeconds(polltime);
_timer = new Timer(polltime);
_timer.Elapsed += _timer_Elapsed;
}
/// <summary>
/// Creates a new instance
/// </summary>
public FlowSnapshot(
FlowBatch flowBatch,
FlowEntity sourceType, string sourceAddressId,
FlowEntity targetType, string targetAddressId = null) : this()
{
Guard.ArgumentNotNull(flowBatch, nameof(flowBatch));
Guard.ArgumentNotNull(sourceType, nameof(sourceType));
Guard.ArgumentNotNull(targetType, nameof(targetType));
Batch = flowBatch;
SourceType = sourceType;
SourceAddressId = sourceAddressId;
TargetAddressId = targetAddressId;
TargetType = targetType;
}
/// <summary>
/// Registers and entity with a given flow.
/// </summary>
public void RegisterEntity(string flowCode, FlowEntity entity)
{
var flow = Get(flowCode);
if (flow.Entities.Any(m => string.Equals(m.EntityTypeId, entity.EntityTypeId)))
{
throw new InvalidOperationException($"The entity type has already been registered.");
}
flow.Entities.Add(entity);
// record data added
entity.DateAdded = DateTime.UtcNow;
// update
_repo.Save(flow);
}
/// <summary>
/// Gets the next flow file that has not been processed.
/// </summary>
public FileInfo GetNextFlowFile(FlowEntity entity, FlowId flow)
{
// this enumerate working folder
var dataFilesDirectory = FlowFileController.Config.InDirectory;
var flowFiles = new DirectoryInfo(dataFilesDirectory)
.GetFiles("*.*")
.OrderBy(m => m.CreationTimeUtc)
.ToList();
if (flowFiles.Any())
{
if (_logger.IsTraceEnabled)
{
_logger.Trace($"Found {flowFiles.Count} files under {dataFilesDirectory}.");
}
// determine if it the data source file has been processed
var flowTransactionLog = _dataFileTranRepo.Get(entity, flow.Code);
foreach (var flowFile in flowFiles)
{
// determine if the file was processed by the given processor
var processLogEntry = flowTransactionLog.Entries.FirstOrDefault(
m => string.Equals(
flowFile.FullName, m.AddressId, StringComparison.OrdinalIgnoreCase));
if (processLogEntry == null)
{
return(flowFile);
}
}
}
else
{
if (_logger.IsWarnEnabled)
{
_logger.Warn($"No files were discovered under {dataFilesDirectory}.");
}
}
return(null);
}
/// <summary>
/// Gets the total number of files/data sources that represent a given flow entity.
/// </summary>
/// <param name="flowEntity">The flow entity to deliver.</param>
/// <returns></returns>
public int Count(FlowEntity flowEntity)
{
return(GetEntries(flowEntity).Count());
}
/// <summary>
/// Gets whether a given data source file has been processed at any point of time.
/// </summary>
public bool Contains(FlowEntity flowEntity, string addressId)
{
var entries = GetEntries(flowEntity);
return(entries.Any(m => m.AddressId.Equals(addressId, StringComparison.OrdinalIgnoreCase)));
}
/// <summary>
/// Gets all the entries for a given flow entity.
/// </summary>
public IEnumerable <DataFileLogEntry> GetEntries(FlowEntity flowEntity)
{
return(Get(flowEntity).Entries);
}
