/// <returns></returns> public DataFileLog Get(FlowEntity flowEntity) { Guard.ArgumentNotNull(flowEntity, nameof(flowEntity)); lock (_syncLock) { var fileName = $@"{DataDir}\flow-{flowEntity.EntityTypeId}-log.json"; if (!File.Exists(fileName)) { if (_logger.IsInfoEnabled) { _logger.Info($"File {fileName} could not be found."); } return(default(DataFileLog)); } if (_logger.IsInfoEnabled) { _logger.Info($"Reading data source log from {fileName}."); } var body = File.ReadAllText(fileName); var result = JsonConvert.DeserializeObject <DataFileLog>(body); if (_logger.IsInfoEnabled) { _logger.Info($"Read data source log {fileName}."); } return(result); } }
/// <summary> /// Registers an entity against the flow. /// </summary> /// <typeparam name="TEntityType">The entity type to create.</typeparam> /// <param name="flowCode">The flowto save the registered entity type against.</param> /// <returns></returns> public FlowEntity RegisterEntity <TEntityType>(string flowCode) { var flow = Get(flowCode); string entityTypeId = typeof(TEntityType).Name; if (flow.Entities.Any(m => string.Equals(m.EntityTypeId, entityTypeId))) { throw new InvalidOperationException($"The entity type has already been registered."); } var entity = new FlowEntity() { AssemblyQualifiedTypeName = typeof(TEntityType).AssemblyQualifiedName, DateAdded = DateTime.UtcNow, EntityTypeId = entityTypeId }; // add to collection flow.Entities.Add(entity); // update _repo.Save(flow); // return(entity); }
/// <summary> /// Gets the enrichment log for a given target process type. /// </summary> public EnricherLog Get(FlowId flow, FlowEntity sourceEntityType) { Guard.ArgumentNotNull(flow, nameof(flow)); Guard.ArgumentNotNull(sourceEntityType, nameof(sourceEntityType)); // source id would be the name of a processor var fileName = $@"{DataDir}\{flow}-Enrichment-{sourceEntityType.EntityTypeId}.json"; if (!File.Exists(fileName)) { if (_logger.IsInfoEnabled) { _logger.Info($"File {fileName} could not be found."); } return(null); } var content = File.ReadAllText(fileName); var log = JsonConvert.DeserializeObject <EnricherLog>( content, new JsonSerializerSettings()); return(log); }
/// <summary> /// Creates a new instance. /// </summary> /// <param name="entityType">The entity encoded in the data sources.</param> /// <param name="filePattern">The file pattern to use to scan for data source snapshot files.</param> public DataFileLog(FlowEntity entityType, string filePattern) : this() { Guard.ArgumentNotNull(entityType, nameof(entityType)); Entries = new List <DataFileLogEntry>(); EntityType = entityType; FileTypes = filePattern; }
/// <summary> /// Creates a new instance /// </summary> public FlowSnapShot( FlowBatch flowBatch, FlowEntity sourceType, string sourceAddressId, FlowEntity targetType, string targetAddressId) : base(flowBatch, sourceType, sourceAddressId, targetType, targetAddressId) { this.Valid = new List <TEntityOut>(); this.Invalid = new List <TEntityOut>(); }
/// <summary> /// Creates a new instance. /// </summary> /// <param name="flow">The related flow.</param> /// <param name="sourceEntityType">The source entity type id.</param> public EnricherLog(FlowId flow, FlowEntity sourceEntityType) { Guard.ArgumentNotNull(flow, nameof(flow)); Guard.ArgumentNotNull(sourceEntityType, nameof(sourceEntityType)); Flow = flow; EntityType = sourceEntityType; Logs = new List <EnrichmentLogEntry>(); Exceptions = new List <Exception>(); }
/// <summary> /// Creates a new instance. /// </summary> /// <param name="flowEntity">The flow entity encoded in the directory.</param> /// <param name="sourceDirectories">The directories to poll for new files.</param> /// <param name="repository">The repository to populate with new files.</param> /// <param name="polltime">The number of seconds to check for new data files and refresh the target directory.</param> public DataSourceProducerFromDirectory(DirectoryInfo[] sourceDirectories, FlowEntity flowEntity, DataSourceLogRepo repository, int polltime = 1) { Guard.ArgumentNotNull(flowEntity, nameof(flowEntity)); Guard.ArgumentNotNull(repository, nameof(repository)); Guard.ArgumentNotNull(sourceDirectories, nameof(sourceDirectories)); _sourceDirectories = sourceDirectories; _flowEntity = flowEntity; _repository = repository; PollInterval = TimeSpan.FromSeconds(polltime); _timer = new Timer(polltime); _timer.Elapsed += _timer_Elapsed; }
/// <summary> /// Creates a new instance /// </summary> public FlowSnapshot( FlowBatch flowBatch, FlowEntity sourceType, string sourceAddressId, FlowEntity targetType, string targetAddressId = null) : this() { Guard.ArgumentNotNull(flowBatch, nameof(flowBatch)); Guard.ArgumentNotNull(sourceType, nameof(sourceType)); Guard.ArgumentNotNull(targetType, nameof(targetType)); Batch = flowBatch; SourceType = sourceType; SourceAddressId = sourceAddressId; TargetAddressId = targetAddressId; TargetType = targetType; }
/// <summary> /// Registers and entity with a given flow. /// </summary> public void RegisterEntity(string flowCode, FlowEntity entity) { var flow = Get(flowCode); if (flow.Entities.Any(m => string.Equals(m.EntityTypeId, entity.EntityTypeId))) { throw new InvalidOperationException($"The entity type has already been registered."); } flow.Entities.Add(entity); // record data added entity.DateAdded = DateTime.UtcNow; // update _repo.Save(flow); }
/// <summary> /// Gets the next flow file that has not been processed. /// </summary> public FileInfo GetNextFlowFile(FlowEntity entity, FlowId flow) { // this enumerate working folder var dataFilesDirectory = FlowFileController.Config.InDirectory; var flowFiles = new DirectoryInfo(dataFilesDirectory) .GetFiles("*.*") .OrderBy(m => m.CreationTimeUtc) .ToList(); if (flowFiles.Any()) { if (_logger.IsTraceEnabled) { _logger.Trace($"Found {flowFiles.Count} files under {dataFilesDirectory}."); } // determine if it the data source file has been processed var flowTransactionLog = _dataFileTranRepo.Get(entity, flow.Code); foreach (var flowFile in flowFiles) { // determine if the file was processed by the given processor var processLogEntry = flowTransactionLog.Entries.FirstOrDefault( m => string.Equals( flowFile.FullName, m.AddressId, StringComparison.OrdinalIgnoreCase)); if (processLogEntry == null) { return(flowFile); } } } else { if (_logger.IsWarnEnabled) { _logger.Warn($"No files were discovered under {dataFilesDirectory}."); } } return(null); }
/// <summary> /// Gets the total number of files/data sources that represent a given flow entity. /// </summary> /// <param name="flowEntity">The flow entity to deliver.</param> /// <returns></returns> public int Count(FlowEntity flowEntity) { return(GetEntries(flowEntity).Count()); }
/// <summary> /// Gets whether a given data source file has been processed at any point of time. /// </summary> public bool Contains(FlowEntity flowEntity, string addressId) { var entries = GetEntries(flowEntity); return(entries.Any(m => m.AddressId.Equals(addressId, StringComparison.OrdinalIgnoreCase))); }
/// <summary> /// Gets all the entries for a given flow entity. /// </summary> public IEnumerable <DataFileLogEntry> GetEntries(FlowEntity flowEntity) { return(Get(flowEntity).Entries); }