public async Task ProcessFileAsync_JobFunctionSucceeds()
{
string testFile = WriteTestFile("dat");
FunctionResult result = new FunctionResult(true);
mockExecutor.Setup(p => p.TryExecuteAsync(It.IsAny <TriggeredFunctionData>(), It.IsAny <CancellationToken>())).ReturnsAsync(result);
FileSystemEventArgs eventArgs = new FileSystemEventArgs(WatcherChangeTypes.Created, combinedTestFilePath, Path.GetFileName(testFile));
await processor.ProcessFileAsync(eventArgs, CancellationToken.None);
Assert.Equal(2, Directory.GetFiles(combinedTestFilePath).Length);
string expectedStatusFile = processor.GetStatusFile(testFile);
Assert.True(File.Exists(testFile));
Assert.True(File.Exists(expectedStatusFile));
string[] statusLines = File.ReadAllLines(expectedStatusFile);
Assert.Equal(2, statusLines.Length);
StatusFileEntry entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(statusLines[0]), typeof(StatusFileEntry));
Assert.Equal(ProcessingState.Processing, entry.State);
Assert.Equal(WatcherChangeTypes.Created, entry.ChangeType);
Assert.Equal(processor.InstanceId, entry.InstanceId);
entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(statusLines[1]), typeof(StatusFileEntry));
Assert.Equal(ProcessingState.Processed, entry.State);
Assert.Equal(WatcherChangeTypes.Created, entry.ChangeType);
Assert.Equal(processor.InstanceId, entry.InstanceId);
}
/// <summary>
/// Determines whether the specified file should be processed.
/// </summary>
/// <param name="filePath">The candidate file for processing.</param>
/// <returns>True if the file should be processed, false otherwise.</returns>
public virtual bool ShouldProcessFile(string filePath)
{
if (IsStatusFile(filePath))
{
return(false);
}
string statusFilePath = GetStatusFile(filePath);
if (!File.Exists(statusFilePath))
{
return(true);
}
StatusFileEntry statusEntry = null;
try
{
GetLastStatus(statusFilePath, out statusEntry);
}
catch (IOException)
{
// if we get an exception reading the status file, it's
// likely because someone started processing and has it locked
return(false);
}
return(statusEntry == null || (statusEntry.State != ProcessingState.Processed &&
statusEntry.ProcessCount < MaxProcessCount));
}
private void ProcessFiles()
{
// scan for any files that require processing (either new unprocessed files,
// or files that have failed previous processing)
IEnumerable <string> unprocessedFiles = Directory.GetFiles(_watchPath, _attribute.Filter)
.Where(p => _processor.ShouldProcessFile(p)).ToArray();
foreach (string fileToProcess in unprocessedFiles)
{
WatcherChangeTypes changeType = WatcherChangeTypes.Created;
string statusFilePath = _processor.GetStatusFile(fileToProcess);
if (File.Exists(statusFilePath))
{
// if an in progress status file exists, we determine the ChangeType
// from the last entry (incomplete) in the file
StatusFileEntry statusEntry = _processor.GetLastStatus(statusFilePath);
if (statusEntry != null)
{
changeType = statusEntry.ChangeType;
}
}
string fileName = Path.GetFileName(fileToProcess);
FileSystemEventArgs args = new FileSystemEventArgs(changeType, _watchPath, fileName);
_workQueue.Post(args);
}
}
public async Task ProcessFileAsync_ChangeTypeChange_Success()
{
FileTriggerAttribute attribute = new FileTriggerAttribute(AttributeSubPath, "*.dat");
processor = CreateTestProcessor(attribute);
string testFile = WriteTestFile("dat");
FunctionResult result = new FunctionResult(true);
mockExecutor.Setup(p => p.TryExecuteAsync(It.IsAny <TriggeredFunctionData>(), It.IsAny <CancellationToken>())).ReturnsAsync(result);
// first process a Create event
string testFilePath = Path.GetDirectoryName(testFile);
string testFileName = Path.GetFileName(testFile);
FileSystemEventArgs eventArgs = new FileSystemEventArgs(WatcherChangeTypes.Created, testFilePath, testFileName);
bool fileProcessedSuccessfully = await processor.ProcessFileAsync(eventArgs, CancellationToken.None);
Assert.True(fileProcessedSuccessfully);
// Wait briefly so the changed time is slightly different than the created time
await Task.Delay(10);
// now process a Change event
File.WriteAllText(testFile, "update");
eventArgs = new FileSystemEventArgs(WatcherChangeTypes.Changed, testFilePath, testFileName);
fileProcessedSuccessfully = await processor.ProcessFileAsync(eventArgs, CancellationToken.None);
Assert.True(fileProcessedSuccessfully);
string expectedStatusFile = processor.GetStatusFile(testFile);
Assert.True(File.Exists(testFile));
Assert.True(File.Exists(expectedStatusFile));
string[] lines = File.ReadAllLines(expectedStatusFile);
Assert.Equal(4, lines.Length);
StatusFileEntry entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(lines[0]), typeof(StatusFileEntry));
Assert.Equal(ProcessingState.Processing, entry.State);
Assert.Equal(WatcherChangeTypes.Created, entry.ChangeType);
Assert.Equal(InstanceId.Substring(0, 20), entry.InstanceId);
entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(lines[1]), typeof(StatusFileEntry));
Assert.Equal(ProcessingState.Processed, entry.State);
Assert.Equal(WatcherChangeTypes.Created, entry.ChangeType);
Assert.Equal(InstanceId.Substring(0, 20), entry.InstanceId);
entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(lines[2]), typeof(StatusFileEntry));
Assert.Equal(ProcessingState.Processing, entry.State);
Assert.Equal(WatcherChangeTypes.Changed, entry.ChangeType);
Assert.Equal(InstanceId.Substring(0, 20), entry.InstanceId);
entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(lines[3]), typeof(StatusFileEntry));
Assert.Equal(ProcessingState.Processed, entry.State);
Assert.Equal(WatcherChangeTypes.Changed, entry.ChangeType);
Assert.Equal(InstanceId.Substring(0, 20), entry.InstanceId);
}
/// <summary>
/// Clean up any files that have been fully processed
/// </summary>
public virtual void CleanupProcessedFiles()
{
int filesDeleted = 0;
string[] statusFiles = Directory.GetFiles(_filePath, GetStatusFile("*"));
foreach (string statusFilePath in statusFiles)
{
try
{
// verify that the file has been fully processed
// if we're unable to get the last status or the file
// is not Processed, skip it
StatusFileEntry statusEntry = null;
if (!GetLastStatus(statusFilePath, out statusEntry) ||
statusEntry.State != ProcessingState.Processed)
{
continue;
}
// get all files starting with that file name. For example, for
// status file input.dat.status, this might return input.dat and
// input.dat.meta (if the file has other companion files)
string targetFileName = Path.GetFileNameWithoutExtension(statusFilePath);
string[] files = Directory.GetFiles(_filePath, targetFileName + "*");
// first delete the non status file(s)
foreach (string filePath in files)
{
if (IsStatusFile(filePath))
{
continue;
}
if (TryDelete(filePath))
{
filesDeleted++;
}
}
// then delete the status file
if (TryDelete(statusFilePath))
{
filesDeleted++;
}
}
catch
{
// ignore any delete failures
}
}
if (filesDeleted > 0)
{
_logger.LogDebug($"File Cleanup ({_filePath}): {filesDeleted} files deleted");
}
}
internal bool GetLastStatus(string statusFilePath, out StatusFileEntry statusEntry)
{
statusEntry = null;
if (!File.Exists(statusFilePath))
{
return(false);
}
using (Stream stream = File.OpenRead(statusFilePath))
{
statusEntry = GetLastStatus(stream);
}
return(statusEntry != null);
}
public async Task ProcessFileAsync_Failure_LeavesInProgressStatusFile()
{
string testFile = WriteTestFile("dat");
FunctionResult result = new FunctionResult(false);
mockExecutor.Setup(p => p.TryExecuteAsync(It.IsAny <TriggeredFunctionData>(), It.IsAny <CancellationToken>())).ReturnsAsync(result);
FileSystemEventArgs eventArgs = new FileSystemEventArgs(WatcherChangeTypes.Created, Path.GetDirectoryName(testFile), Path.GetFileName(testFile));
bool fileProcessedSuccessfully = await processor.ProcessFileAsync(eventArgs, CancellationToken.None);
Assert.False(fileProcessedSuccessfully);
Assert.True(File.Exists(testFile));
string statusFilePath = processor.GetStatusFile(testFile);
StatusFileEntry entry = processor.GetLastStatus(statusFilePath);
Assert.Equal(ProcessingState.Processing, entry.State);
}
public async Task ProcessFileAsync_AlreadyProcessing_ReturnsWithoutProcessing()
{
string testFile = WriteTestFile("dat");
// first take a lock on the status file
StatusFileEntry status = null;
using (StreamWriter statusFile = processor.AcquireStatusFileLock(testFile, WatcherChangeTypes.Created, out status))
{
// now attempt to process the file
FileSystemEventArgs eventArgs = new FileSystemEventArgs(WatcherChangeTypes.Created, Path.GetDirectoryName(testFile), Path.GetFileName(testFile));
bool fileProcessedSuccessfully = await processor.ProcessFileAsync(eventArgs, CancellationToken.None);
Assert.False(fileProcessedSuccessfully);
mockExecutor.Verify(p => p.TryExecuteAsync(It.IsAny <TriggeredFunctionData>(), It.IsAny <CancellationToken>()), Times.Never);
}
string statusFilePath = processor.GetStatusFile(testFile);
File.Delete(statusFilePath);
}
private void ProcessFiles()
{
// scan for any files that require processing (either new unprocessed files,
// or files that have failed previous processing)
string[] filesToProcess = Directory.GetFiles(_watchPath, _attribute.Filter)
.Where(p => _processor.ShouldProcessFile(p)).ToArray();
if (filesToProcess.Length > 0)
{
_trace.Verbose(string.Format("Found {0} file(s) at path '{1}' for ready processing", filesToProcess.Length, _watchPath));
}
foreach (string fileToProcess in filesToProcess)
{
WatcherChangeTypes changeType = WatcherChangeTypes.Created;
string statusFilePath = _processor.GetStatusFile(fileToProcess);
try
{
StatusFileEntry statusEntry = null;
if (_processor.GetLastStatus(statusFilePath, out statusEntry))
{
// if an in progress status file exists, we determine the ChangeType
// from the last entry (incomplete) in the file
changeType = statusEntry.ChangeType;
}
}
catch (IOException)
{
// if we get an exception reading the status file, it's
// likely because someone started processing and has it locked
continue;
}
string fileName = Path.GetFileName(fileToProcess);
FileSystemEventArgs args = new FileSystemEventArgs(changeType, _watchPath, fileName);
_workQueue.Post(args);
}
}
internal StatusFileEntry GetLastStatus(Stream statusFileStream)
{
StatusFileEntry statusEntry = null;
using (StreamReader reader = new StreamReader(statusFileStream, Encoding.UTF8, false, 1024, true))
{
string text = reader.ReadToEnd();
string[] fileLines = text.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries);
string lastLine = fileLines.LastOrDefault();
if (!string.IsNullOrEmpty(lastLine))
{
using (StringReader stringReader = new StringReader(lastLine))
{
statusEntry = (StatusFileEntry)_serializer.Deserialize(stringReader, typeof(StatusFileEntry));
}
}
}
statusFileStream.Seek(0, SeekOrigin.End);
return(statusEntry);
}
public void Cleanup_AutoDeleteOn_DeletesCompletedFiles()
{
FileTriggerAttribute attribute = new FileTriggerAttribute(attributeSubPath, "*.dat", autoDelete: true);
FileProcessor localProcessor = CreateTestProcessor(attribute);
// create a completed file set
string completedFile = WriteTestFile("dat");
string completedStatusFile = localProcessor.GetStatusFile(completedFile);
StatusFileEntry status = new StatusFileEntry
{
State = ProcessingState.Processing,
Timestamp = DateTime.UtcNow,
ChangeType = WatcherChangeTypes.Created,
InstanceId = "1"
};
StringWriter sw = new StringWriter();
_serializer.Serialize(sw, status);
sw.WriteLine();
status.State = ProcessingState.Processed;
status.Timestamp = status.Timestamp + TimeSpan.FromSeconds(15);
_serializer.Serialize(sw, status);
sw.WriteLine();
sw.Flush();
File.WriteAllText(completedStatusFile, sw.ToString());
// include an additional companion metadata file
string completedAdditionalFile = completedFile + ".metadata";
File.WriteAllText(completedAdditionalFile, "Data");
// write a file that SHOULDN'T be deleted
string dontDeleteFile = Path.ChangeExtension(completedFile, "json");
File.WriteAllText(dontDeleteFile, "Data");
// create an incomplete file set
string incompleteFile = WriteTestFile("dat");
string incompleteStatusFile = localProcessor.GetStatusFile(incompleteFile);
status = new StatusFileEntry
{
State = ProcessingState.Processing,
Timestamp = DateTime.UtcNow,
ChangeType = WatcherChangeTypes.Created,
InstanceId = "1"
};
sw = new StringWriter();
_serializer.Serialize(sw, status);
sw.WriteLine();
File.WriteAllText(incompleteStatusFile, sw.ToString());
localProcessor.Cleanup();
// expect the completed set to be deleted
Assert.False(File.Exists(completedFile));
Assert.False(File.Exists(completedAdditionalFile));
Assert.False(File.Exists(completedStatusFile));
Assert.True(File.Exists(dontDeleteFile));
// expect the incomplete set to remain
Assert.False(File.Exists(completedFile));
Assert.False(File.Exists(completedStatusFile));
}
public async Task ConcurrentListeners_ProcessFilesCorrectly(int concurrentListenerCount, int inputFileCount)
{
// mock out the executor so we can capture function invocations
Mock <ITriggeredFunctionExecutor> mockExecutor = new Mock <ITriggeredFunctionExecutor>(MockBehavior.Strict);
ConcurrentBag <string> processedFiles = new ConcurrentBag <string>();
FunctionResult result = new FunctionResult(true);
mockExecutor.Setup(p => p.TryExecuteAsync(It.IsAny <TriggeredFunctionData>(), It.IsAny <CancellationToken>()))
.Callback <TriggeredFunctionData, CancellationToken>(async(mockData, mockToken) =>
{
await Task.Delay(50);
FileSystemEventArgs fileEvent = mockData.TriggerValue as FileSystemEventArgs;
processedFiles.Add(fileEvent.Name);
})
.ReturnsAsync(result);
var options = new FilesOptions()
{
RootPath = rootPath
};
FileTriggerAttribute attribute = new FileTriggerAttribute(attributeSubPath, changeTypes: WatcherChangeTypes.Created | WatcherChangeTypes.Changed, filter: "*.dat");
// create a bunch of listeners and start them
CancellationTokenSource tokenSource = new CancellationTokenSource();
CancellationToken cancellationToken = tokenSource.Token;
List <Task> listenerStartupTasks = new List <Task>();
List <FileListener> listeners = new List <FileListener>();
for (int i = 0; i < concurrentListenerCount; i++)
{
FileListener listener = new FileListener(new OptionsWrapper <FilesOptions>(options), attribute, mockExecutor.Object, new TestLogger("Test"), new DefaultFileProcessorFactory());
listeners.Add(listener);
listenerStartupTasks.Add(listener.StartAsync(cancellationToken));
}
await Task.WhenAll(listenerStartupTasks);
// now start creating files
List <string> expectedFiles = new List <string>();
for (int i = 0; i < inputFileCount; i++)
{
string file = WriteTestFile();
await Task.Delay(50);
expectedFiles.Add(Path.GetFileName(file));
}
// wait for all files to be processed
await TestHelpers.Await(() =>
{
return(processedFiles.Count >= inputFileCount);
});
Assert.Equal(inputFileCount, processedFiles.Count);
// verify that each file was only processed once
Assert.True(expectedFiles.OrderBy(p => p).SequenceEqual(processedFiles.OrderBy(p => p)));
Assert.Equal(expectedFiles.Count * 2, Directory.GetFiles(testFileDir).Length);
// verify contents of each status file
FileProcessor processor = listeners[0].Processor;
foreach (string processedFile in processedFiles)
{
string statusFilePath = processor.GetStatusFile(Path.Combine(testFileDir, processedFile));
string[] statusLines = File.ReadAllLines(statusFilePath);
Assert.Equal(2, statusLines.Length);
StatusFileEntry statusEntry = JsonConvert.DeserializeObject <StatusFileEntry>(statusLines[0]);
Assert.Equal(ProcessingState.Processing, statusEntry.State);
Assert.Equal(WatcherChangeTypes.Created, statusEntry.ChangeType);
statusEntry = JsonConvert.DeserializeObject <StatusFileEntry>(statusLines[1]);
Assert.Equal(ProcessingState.Processed, statusEntry.State);
Assert.Equal(WatcherChangeTypes.Created, statusEntry.ChangeType);
}
// Now test concurrency handling for updates by updating some files
// and verifying the updates are only processed once
string[] filesToUpdate = processedFiles.Take(50).Select(p => Path.Combine(testFileDir, p)).ToArray();
string item;
while (!processedFiles.IsEmpty)
{
processedFiles.TryTake(out item);
}
await Task.Delay(1000);
foreach (string fileToUpdate in filesToUpdate)
{
await Task.Delay(50);
File.AppendAllText(fileToUpdate, "update");
}
// wait for all files to be processed
await TestHelpers.Await(() =>
{
return(processedFiles.Count >= filesToUpdate.Length);
});
Assert.Equal(filesToUpdate.Length, processedFiles.Count);
Assert.Equal(expectedFiles.Count * 2, Directory.GetFiles(testFileDir).Length);
// verify the status files are correct for each of the updated files
foreach (string updatedFile in filesToUpdate)
{
string statusFilePath = processor.GetStatusFile(updatedFile);
string[] statusLines = File.ReadAllLines(statusFilePath);
Assert.Equal(4, statusLines.Length);
StatusFileEntry statusEntry = JsonConvert.DeserializeObject <StatusFileEntry>(statusLines[0]);
Assert.Equal(ProcessingState.Processing, statusEntry.State);
Assert.Equal(WatcherChangeTypes.Created, statusEntry.ChangeType);
statusEntry = JsonConvert.DeserializeObject <StatusFileEntry>(statusLines[1]);
Assert.Equal(ProcessingState.Processed, statusEntry.State);
Assert.Equal(WatcherChangeTypes.Created, statusEntry.ChangeType);
statusEntry = JsonConvert.DeserializeObject <StatusFileEntry>(statusLines[2]);
Assert.Equal(ProcessingState.Processing, statusEntry.State);
Assert.Equal(WatcherChangeTypes.Changed, statusEntry.ChangeType);
statusEntry = JsonConvert.DeserializeObject <StatusFileEntry>(statusLines[3]);
Assert.Equal(ProcessingState.Processed, statusEntry.State);
Assert.Equal(WatcherChangeTypes.Changed, statusEntry.ChangeType);
}
// Now clean up all processed files
processor.CleanupProcessedFiles();
Assert.Empty(Directory.GetFiles(testFileDir));
foreach (FileListener listener in listeners)
{
listener.Dispose();
}
}
internal StreamWriter AcquireStatusFileLock(string filePath, WatcherChangeTypes changeType, out StatusFileEntry statusEntry)
{
Stream stream = null;
statusEntry = null;
try
{
// Attempt to create (or update) the companion status file and lock it. The status
// file is the mechanism for handling multi-instance concurrency.
string statusFilePath = GetStatusFile(filePath);
stream = File.Open(statusFilePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None);
// Once we've established the lock, we need to check to ensure that another instance
// hasn't already processed the file in the time between our getting the event and
// acquiring the lock.
statusEntry = GetLastStatus(stream);
if (statusEntry != null && statusEntry.State == ProcessingState.Processed)
{
// For file Create, we have no additional checks to perform. However for
// file Change, we need to also check the LastWrite value for the entry
// since there can be multiple Processed entries in the file over time.
if (changeType == WatcherChangeTypes.Created)
{
return(null);
}
else if (changeType == WatcherChangeTypes.Changed &&
File.GetLastWriteTimeUtc(filePath) == statusEntry.LastWrite)
{
return(null);
}
}
stream.Seek(0, SeekOrigin.End);
StreamWriter streamReader = new StreamWriter(stream);
streamReader.AutoFlush = true;
stream = null;
return(streamReader);
}
catch
{
return(null);
}
finally
{
if (stream != null)
{
stream.Dispose();
}
}
}
/// <summary>
/// Process the file indicated by the specified <see cref="FileSystemEventArgs"/>.
/// </summary>
/// <param name="eventArgs">The <see cref="FileSystemEventArgs"/> indicating the file to process.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> to use.</param>
/// <returns>
/// A <see cref="Task"/> that returns true if the file was processed successfully, false otherwise.
/// </returns>
public virtual async Task <bool> ProcessFileAsync(FileSystemEventArgs eventArgs, CancellationToken cancellationToken)
{
try
{
StatusFileEntry status = null;
string filePath = eventArgs.FullPath;
using (StreamWriter statusWriter = AcquireStatusFileLock(filePath, eventArgs.ChangeType, out status))
{
if (statusWriter == null)
{
return(false);
}
// We've acquired the lock. The current status might be either Failed
// or Processing (if processing failed before we were unable to update
// the file status to Failed)
int processCount = 0;
if (status != null)
{
processCount = status.ProcessCount;
}
while (processCount++ < MaxProcessCount)
{
FunctionResult result = null;
if (result != null)
{
TimeSpan delay = GetRetryInterval(result, processCount);
await Task.Delay(delay);
}
// write an entry indicating the file is being processed
status = new StatusFileEntry
{
State = ProcessingState.Processing,
Timestamp = DateTime.Now,
LastWrite = File.GetLastWriteTimeUtc(filePath),
ChangeType = eventArgs.ChangeType,
InstanceId = InstanceId,
ProcessCount = processCount
};
_serializer.Serialize(statusWriter, status);
statusWriter.WriteLine();
// invoke the job function
TriggeredFunctionData input = new TriggeredFunctionData
{
TriggerValue = eventArgs
};
result = await _executor.TryExecuteAsync(input, cancellationToken);
// write a status entry indicating the state of processing
status.State = result.Succeeded ? ProcessingState.Processed : ProcessingState.Failed;
status.Timestamp = DateTime.Now;
_serializer.Serialize(statusWriter, status);
statusWriter.WriteLine();
if (result.Succeeded)
{
return(true);
}
}
return(false);
}
}
catch
{
return(false);
}
}
