/// <summary>
/// Determines whether the specified file should be processed.
/// </summary>
/// <param name="filePath">The candidate file for processing.</param>
/// <returns>True if the file should be processed, false otherwise.</returns>
public virtual bool ShouldProcessFile(string filePath)
{
if (IsStatusFile(filePath))
{
return(false);
}
string statusFilePath = GetStatusFile(filePath);
if (!File.Exists(statusFilePath))
{
return(true);
}
StatusFileEntry statusEntry = null;
try
{
GetLastStatus(statusFilePath, out statusEntry);
}
catch (IOException)
{
// if we get an exception reading the status file, it's
// likely because someone started processing and has it locked
return(false);
}
return(statusEntry == null || (statusEntry.State != ProcessingState.Processed &&
statusEntry.ProcessCount < MaxProcessCount));
}
/// <summary>
/// Process the file indicated by the specified <see cref="FileSystemEventArgs"/>.
/// </summary>
/// <param name="eventArgs">The <see cref="FileSystemEventArgs"/> indicating the file to process.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use.</param>
/// <returns>
/// A <see cref="Task"/> that returns true if the file was processed successfully, false otherwise.
/// </returns>
public virtual async Task <bool> ProcessFileAsync(FileSystemEventArgs eventArgs, CancellationToken cancellationToken)
{
try
{
string filePath = eventArgs.FullPath;
using (StreamWriter statusWriter = AquireStatusFileLock(filePath, eventArgs.ChangeType))
{
if (statusWriter == null)
{
return(false);
}
// write an entry indicating the file is being processed
StatusFileEntry status = new StatusFileEntry
{
State = ProcessingState.Processing,
Timestamp = DateTime.UtcNow,
LastWrite = File.GetLastWriteTimeUtc(filePath),
ChangeType = eventArgs.ChangeType,
InstanceId = InstanceId
};
_serializer.Serialize(statusWriter, status);
statusWriter.WriteLine();
// invoke the job function
TriggeredFunctionData input = new TriggeredFunctionData
{
// TODO: set this properly
ParentId = null,
TriggerValue = eventArgs
};
FunctionResult result = await _executor.TryExecuteAsync(input, cancellationToken);
if (result.Succeeded)
{
// write a status entry indicating processing is complete
status.State = ProcessingState.Processed;
status.Timestamp = DateTime.UtcNow;
_serializer.Serialize(statusWriter, status);
statusWriter.WriteLine();
return(true);
}
else
{
// If the function failed, we leave the in progress status
// file as is (it will show "Processing"). The file will be
// reprocessed later on a clean-up pass.
statusWriter.Close();
cancellationToken.ThrowIfCancellationRequested();
return(false);
}
}
}
catch
{
return(false);
}
}
/// <summary>
/// Clean up any files that have been fully processed
/// </summary>
public virtual void CleanupProcessedFiles()
{
int filesDeleted = 0;
string[] statusFiles = Directory.GetFiles(_filePath, GetStatusFile("*"));
foreach (string statusFilePath in statusFiles)
{
try
{
// verify that the file has been fully processed
// if we're unable to get the last status or the file
// is not Processed, skip it
StatusFileEntry statusEntry = null;
if (!GetLastStatus(statusFilePath, out statusEntry) ||
statusEntry.State != ProcessingState.Processed)
{
continue;
}
// get all files starting with that file name. For example, for
// status file input.dat.status, this might return input.dat and
// input.dat.meta (if the file has other companion files)
string targetFileName = Path.GetFileNameWithoutExtension(statusFilePath);
string[] files = Directory.GetFiles(_filePath, targetFileName + "*");
// first delete the non status file(s)
foreach (string filePath in files)
{
if (IsStatusFile(filePath))
{
continue;
}
if (TryDelete(filePath))
{
filesDeleted++;
}
}
// then delete the status file
if (TryDelete(statusFilePath))
{
filesDeleted++;
}
}
catch
{
// ignore any delete failures
}
}
if (filesDeleted > 0)
{
_logger.LogDebug($"File Cleanup ({_filePath}): {filesDeleted} files deleted");
}
}
/// <summary>
/// Determines whether the specified file should be processed.
/// </summary>
/// <param name="filePath">The candidate file for processing.</param>
/// <returns>True if the file should be processed, false otherwise.</returns>
public virtual bool ShouldProcessFile(string filePath)
{
string statusFilePath = GetStatusFile(filePath);
if (!File.Exists(statusFilePath))
{
return(true);
}
StatusFileEntry statusEntry = GetLastStatus(statusFilePath);
return(statusEntry == null || statusEntry.State != ProcessingState.Processed);
}
internal StreamWriter AquireStatusFileLock(string filePath, WatcherChangeTypes changeType)
{
Stream stream = null;
try
{
// Attempt to create (or update) the companion status file and lock it. The status
// file is the mechanism for handling multi-instance concurrency.
string statusFilePath = GetStatusFile(filePath);
stream = File.Open(statusFilePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
// Once we've established the lock, we need to check to ensure that another instance
// hasn't already processed the file in the time between our getting the event and
// aquiring the lock.
StatusFileEntry statusEntry = GetLastStatus(stream);
if (statusEntry != null && statusEntry.State == ProcessingState.Processed)
{
// For file Create, we have no additional checks to perform. However for
// file Change, we need to also check the LastWrite value for the entry
// since there can be multiple Processed entries in the file over time.
if (changeType == WatcherChangeTypes.Created)
{
return(null);
}
else if (changeType == WatcherChangeTypes.Changed &&
File.GetLastWriteTimeUtc(filePath) == statusEntry.LastWrite)
{
return(null);
}
}
stream.Seek(0, SeekOrigin.End);
StreamWriter streamReader = new StreamWriter(stream);
streamReader.AutoFlush = true;
stream = null;
return(streamReader);
}
catch
{
return(null);
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
}
internal bool GetLastStatus(string statusFilePath, out StatusFileEntry statusEntry)
{
statusEntry = null;
if (!File.Exists(statusFilePath))
{
return(false);
}
using (Stream stream = File.OpenRead(statusFilePath))
{
statusEntry = GetLastStatus(stream);
}
return(statusEntry != null);
}
/// <summary>
/// Clean up any files that have been fully processed
/// </summary>
public virtual void CleanupProcessedFiles()
{
string[] statusFiles = Directory.GetFiles(_filePath, GetStatusFile("*"));
foreach (string statusFilePath in statusFiles)
{
try
{
// verify that the file has been fully processed
StatusFileEntry statusEntry = GetLastStatus(statusFilePath);
if (statusEntry.State != ProcessingState.Processed)
{
continue;
}
// get all files starting with that file name. For example, for
// status file input.dat.status, this might return input.dat and
// input.dat.meta (if the file has other companion files)
string targetFileName = Path.GetFileNameWithoutExtension(statusFilePath);
string[] files = Directory.GetFiles(_filePath, targetFileName + "*");
// first delete the non status file(s)
foreach (string filePath in files)
{
if (Path.GetExtension(filePath).TrimStart('.') == StatusFileExtension)
{
continue;
}
TryDelete(filePath);
}
// then delete the status file
TryDelete(statusFilePath);
}
catch
{
// ignore any delete failures
}
}
}
internal StatusFileEntry GetLastStatus(Stream statusFileStream)
{
StatusFileEntry statusEntry = null;
using (StreamReader reader = new StreamReader(statusFileStream, Encoding.UTF8, false, 1024, true))
{
string text = reader.ReadToEnd();
string[] fileLines = text.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
string lastLine = fileLines.LastOrDefault();
if (!string.IsNullOrEmpty(lastLine))
{
using (StringReader stringReader = new StringReader(lastLine))
{
statusEntry = (StatusFileEntry)_serializer.Deserialize(stringReader, typeof(StatusFileEntry));
}
}
}
statusFileStream.Seek(0, SeekOrigin.End);
return(statusEntry);
}
public void Cleanup_AutoDeleteOn_DeletesCompletedFiles()
{
FileTriggerAttribute attribute = new FileTriggerAttribute(attributeSubPath, "*.dat", autoDelete: true);
FileProcessor localProcessor = CreateTestProcessor(attribute);
// create a completed file set
string completedFile = WriteTestFile("dat");
string completedStatusFile = localProcessor.GetStatusFile(completedFile);
StatusFileEntry status = new StatusFileEntry
{
State = ProcessingState.Processing,
Timestamp = DateTime.UtcNow,
ChangeType = WatcherChangeTypes.Created,
InstanceId = "1"
};
StringWriter sw = new StringWriter();
_serializer.Serialize(sw, status);
sw.WriteLine();
status.State = ProcessingState.Processed;
status.Timestamp = status.Timestamp + TimeSpan.FromSeconds(15);
_serializer.Serialize(sw, status);
sw.WriteLine();
sw.Flush();
File.WriteAllText(completedStatusFile, sw.ToString());
// include an additional companion metadata file
string completedAdditionalFile = completedFile + ".metadata";
File.WriteAllText(completedAdditionalFile, "Data");
// write a file that SHOULDN'T be deleted
string dontDeleteFile = Path.ChangeExtension(completedFile, "json");
File.WriteAllText(dontDeleteFile, "Data");
// create an incomplete file set
string incompleteFile = WriteTestFile("dat");
string incompleteStatusFile = localProcessor.GetStatusFile(incompleteFile);
status = new StatusFileEntry
{
State = ProcessingState.Processing,
Timestamp = DateTime.UtcNow,
ChangeType = WatcherChangeTypes.Created,
InstanceId = "1"
};
sw = new StringWriter();
_serializer.Serialize(sw, status);
sw.WriteLine();
File.WriteAllText(incompleteStatusFile, sw.ToString());
localProcessor.Cleanup();
// expect the completed set to be deleted
Assert.False(File.Exists(completedFile));
Assert.False(File.Exists(completedAdditionalFile));
Assert.False(File.Exists(completedStatusFile));
Assert.True(File.Exists(dontDeleteFile));
// expect the incomplete set to remain
Assert.False(File.Exists(completedFile));
Assert.False(File.Exists(completedStatusFile));
}
/// <summary>
/// Process the file indicated by the specified <see cref="FileSystemEventArgs"/>.
/// </summary>
/// <param name="eventArgs">The <see cref="FileSystemEventArgs"/> indicating the file to process.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use.</param>
/// <returns>
/// A <see cref="Task"/> that returns true if the file was processed successfully, false otherwise.
/// </returns>
public virtual async Task<bool> ProcessFileAsync(FileSystemEventArgs eventArgs, CancellationToken cancellationToken)
{
try
{
string filePath = eventArgs.FullPath;
using (StreamWriter statusWriter = AquireStatusFileLock(filePath, eventArgs.ChangeType))
{
if (statusWriter == null)
{
return false;
}
// write an entry indicating the file is being processed
StatusFileEntry status = new StatusFileEntry
{
State = ProcessingState.Processing,
Timestamp = DateTime.UtcNow,
LastWrite = File.GetLastWriteTimeUtc(filePath),
ChangeType = eventArgs.ChangeType,
InstanceId = InstanceId
};
_serializer.Serialize(statusWriter, status);
statusWriter.WriteLine();
// invoke the job function
TriggeredFunctionData input = new TriggeredFunctionData
{
// TODO: set this properly
ParentId = null,
TriggerValue = eventArgs
};
FunctionResult result = await _executor.TryExecuteAsync(input, cancellationToken);
if (result.Succeeded)
{
// write a status entry indicating processing is complete
status.State = ProcessingState.Processed;
status.Timestamp = DateTime.UtcNow;
_serializer.Serialize(statusWriter, status);
statusWriter.WriteLine();
return true;
}
else
{
// If the function failed, we leave the in progress status
// file as is (it will show "Processing"). The file will be
// reprocessed later on a clean-up pass.
statusWriter.Close();
cancellationToken.ThrowIfCancellationRequested();
return false;
}
}
}
catch
{
return false;
}
}
/// <summary>
/// Process the file indicated by the specified <see cref="FileSystemEventArgs"/>.
/// </summary>
/// <param name="eventArgs">The <see cref="FileSystemEventArgs"/> indicating the file to process.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use.</param>
/// <returns>
/// A <see cref="Task"/> that returns true if the file was processed successfully, false otherwise.
/// </returns>
public virtual async Task <bool> ProcessFileAsync(FileSystemEventArgs eventArgs, CancellationToken cancellationToken)
{
try
{
StatusFileEntry status = null;
string filePath = eventArgs.FullPath;
using (StreamWriter statusWriter = AcquireStatusFileLock(filePath, eventArgs.ChangeType, out status))
{
if (statusWriter == null)
{
return(false);
}
// We've acquired the lock. The current status might be either Failed
// or Processing (if processing failed before we were unable to update
// the file status to Failed)
int processCount = 0;
if (status != null)
{
processCount = status.ProcessCount;
}
while (processCount++ < MaxProcessCount)
{
FunctionResult result = null;
if (result != null)
{
TimeSpan delay = GetRetryInterval(result, processCount);
await Task.Delay(delay);
}
// write an entry indicating the file is being processed
status = new StatusFileEntry
{
State = ProcessingState.Processing,
Timestamp = DateTime.Now,
LastWrite = File.GetLastWriteTimeUtc(filePath),
ChangeType = eventArgs.ChangeType,
InstanceId = InstanceId,
ProcessCount = processCount
};
_serializer.Serialize(statusWriter, status);
statusWriter.WriteLine();
// invoke the job function
TriggeredFunctionData input = new TriggeredFunctionData
{
TriggerValue = eventArgs
};
result = await _executor.TryExecuteAsync(input, cancellationToken);
// write a status entry indicating the state of processing
status.State = result.Succeeded ? ProcessingState.Processed : ProcessingState.Failed;
status.Timestamp = DateTime.Now;
_serializer.Serialize(statusWriter, status);
statusWriter.WriteLine();
if (result.Succeeded)
{
return(true);
}
}
return(false);
}
}
catch
{
return(false);
}
}
internal bool GetLastStatus(string statusFilePath, out StatusFileEntry statusEntry)
{
statusEntry = null;
if (!File.Exists(statusFilePath))
{
return false;
}
using (Stream stream = File.OpenRead(statusFilePath))
{
statusEntry = GetLastStatus(stream);
}
return statusEntry != null;
}
internal StreamWriter AcquireStatusFileLock(string filePath, WatcherChangeTypes changeType, out StatusFileEntry statusEntry)
{
Stream stream = null;
statusEntry = null;
try
{
// Attempt to create (or update) the companion status file and lock it. The status
// file is the mechanism for handling multi-instance concurrency.
string statusFilePath = GetStatusFile(filePath);
stream = File.Open(statusFilePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
// Once we've established the lock, we need to check to ensure that another instance
// hasn't already processed the file in the time between our getting the event and
// acquiring the lock.
statusEntry = GetLastStatus(stream);
if (statusEntry != null && statusEntry.State == ProcessingState.Processed)
{
// For file Create, we have no additional checks to perform. However for
// file Change, we need to also check the LastWrite value for the entry
// since there can be multiple Processed entries in the file over time.
if (changeType == WatcherChangeTypes.Created)
{
return null;
}
else if (changeType == WatcherChangeTypes.Changed &&
File.GetLastWriteTimeUtc(filePath) == statusEntry.LastWrite)
{
return null;
}
}
stream.Seek(0, SeekOrigin.End);
StreamWriter streamReader = new StreamWriter(stream);
streamReader.AutoFlush = true;
stream = null;
return streamReader;
}
catch
{
return null;
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
}
/// <summary>
/// Process the file indicated by the specified <see cref="FileSystemEventArgs"/>.
/// </summary>
/// <param name="eventArgs">The <see cref="FileSystemEventArgs"/> indicating the file to process.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use.</param>
/// <returns>
/// A <see cref="Task"/> that returns true if the file was processed successfully, false otherwise.
/// </returns>
public virtual async Task<bool> ProcessFileAsync(FileSystemEventArgs eventArgs, CancellationToken cancellationToken)
{
try
{
StatusFileEntry status = null;
string filePath = eventArgs.FullPath;
using (StreamWriter statusWriter = AcquireStatusFileLock(filePath, eventArgs.ChangeType, out status))
{
if (statusWriter == null)
{
return false;
}
// We've acquired the lock. The current status might be either Failed
// or Processing (if processing failed before we were unable to update
// the file status to Failed)
int processCount = 0;
if (status != null)
{
processCount = status.ProcessCount;
}
while (processCount++ < MaxProcessCount)
{
FunctionResult result = null;
if (result != null)
{
TimeSpan delay = GetRetryInterval(result, processCount);
await Task.Delay(delay);
}
// write an entry indicating the file is being processed
status = new StatusFileEntry
{
State = ProcessingState.Processing,
Timestamp = DateTime.Now,
LastWrite = File.GetLastWriteTimeUtc(filePath),
ChangeType = eventArgs.ChangeType,
InstanceId = InstanceId,
ProcessCount = processCount
};
_serializer.Serialize(statusWriter, status);
statusWriter.WriteLine();
// invoke the job function
TriggeredFunctionData input = new TriggeredFunctionData
{
TriggerValue = eventArgs
};
result = await _executor.TryExecuteAsync(input, cancellationToken);
// write a status entry indicating the state of processing
status.State = result.Succeeded ? ProcessingState.Processed : ProcessingState.Failed;
status.Timestamp = DateTime.Now;
_serializer.Serialize(statusWriter, status);
statusWriter.WriteLine();
if (result.Succeeded)
{
return true;
}
}
return false;
}
}
catch
{
return false;
}
}