/// <summary>
/// Constructs a new instance
/// </summary>
/// <param name="options">The <see cref="FilesOptions"/></param>
/// <param name="attribute">The <see cref="FileTriggerAttribute"/></param>
/// <param name="executor">The function executor.</param>
/// <param name="logger">The <see cref="ILogger"/>.</param>
public FileProcessorFactoryContext(FilesOptions options, FileTriggerAttribute attribute, ITriggeredFunctionExecutor executor, ILogger logger)
{
Options = options ?? throw new ArgumentNullException(nameof(options));
Attribute = attribute ?? throw new ArgumentNullException(nameof(attribute));
Executor = executor ?? throw new ArgumentNullException(nameof(executor));
Logger = logger ?? throw new ArgumentNullException(nameof(logger));
}
public FileListener(FilesConfiguration config, FileTriggerAttribute attribute, ITriggeredFunctionExecutor triggerExecutor)
{
_config = config;
_attribute = attribute;
_triggerExecutor = triggerExecutor;
_cancellationTokenSource = new CancellationTokenSource();
_watchPath = Path.Combine(_config.RootPath, _attribute.GetNormalizedPath());
}
public FileProcessorTests()
{
rootPath = Path.GetTempPath();
combinedTestFilePath = Path.Combine(rootPath, attributeSubPath);
Directory.CreateDirectory(combinedTestFilePath);
DeleteTestFiles(combinedTestFilePath);
config = new FilesConfiguration()
{
RootPath = rootPath
};
FileTriggerAttribute attribute = new FileTriggerAttribute(attributeSubPath, "*.dat");
processor = CreateTestProcessor(attribute);
JsonSerializerSettings settings = new JsonSerializerSettings
{
DateFormatHandling = DateFormatHandling.IsoDateFormat,
};
_serializer = JsonSerializer.Create(settings);
Environment.SetEnvironmentVariable("WEBSITE_INSTANCE_ID", InstanceId);
}
/// <summary>
/// Constructs a new instance.
/// </summary>
/// <param name="context">The <see cref="FileProcessorFactoryContext"/> to use.</param>
public FileProcessor(FileProcessorFactoryContext context)
{
if (context == null)
{
throw new ArgumentNullException("context");
}
_options = context.Options;
_attribute = context.Attribute;
_executor = context.Executor;
_logger = context.Logger;
string attributePath = _attribute.GetRootPath();
_filePath = Path.Combine(_options.RootPath, attributePath);
JsonSerializerSettings settings = new JsonSerializerSettings
{
DateFormatHandling = DateFormatHandling.IsoDateFormat,
};
_serializer = JsonSerializer.Create(settings);
}
public async Task ProcessFileAsync_ChangeTypeCreate_Success()
{
FileTriggerAttribute attribute = new FileTriggerAttribute(attributeSubPath, "*.dat");
processor = CreateTestProcessor(attribute);
string testFile = WriteTestFile("dat");
FunctionResult result = new FunctionResult(true);
mockExecutor.Setup(p => p.TryExecuteAsync(It.IsAny<TriggeredFunctionData>(), It.IsAny<CancellationToken>())).ReturnsAsync(result);
string testFilePath = Path.GetDirectoryName(testFile);
string testFileName = Path.GetFileName(testFile);
FileSystemEventArgs eventArgs = new FileSystemEventArgs(WatcherChangeTypes.Created, testFilePath, testFileName);
bool fileProcessedSuccessfully = await processor.ProcessFileAsync(eventArgs, CancellationToken.None);
Assert.True(fileProcessedSuccessfully);
string expectedStatusFile = processor.GetStatusFile(testFile);
Assert.True(File.Exists(testFile));
Assert.True(File.Exists(expectedStatusFile));
string[] lines = File.ReadAllLines(expectedStatusFile);
Assert.Equal(2, lines.Length);
StatusFileEntry entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(lines[0]), typeof(StatusFileEntry));
Assert.Equal(ProcessingState.Processing, entry.State);
Assert.Equal(WatcherChangeTypes.Created, entry.ChangeType);
Assert.Equal(InstanceId.Substring(0, 20), entry.InstanceId);
entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(lines[1]), typeof(StatusFileEntry));
Assert.Equal(ProcessingState.Processed, entry.State);
Assert.Equal(WatcherChangeTypes.Created, entry.ChangeType);
Assert.Equal(InstanceId.Substring(0, 20), entry.InstanceId);
}
private FileProcessor CreateTestProcessor(FileTriggerAttribute attribute)
{
mockExecutor = new Mock<ITriggeredFunctionExecutor>(MockBehavior.Strict);
FileProcessorFactoryContext context = new FileProcessorFactoryContext(config, attribute, mockExecutor.Object, new TestTraceWriter());
return new FileProcessor(context);
}
public void Cleanup_AutoDeleteOn_DeletesCompletedFiles()
{
FileTriggerAttribute attribute = new FileTriggerAttribute(attributeSubPath, "*.dat", autoDelete: true);
FileProcessor localProcessor = CreateTestProcessor(attribute);
// create a completed file set
string completedFile = WriteTestFile("dat");
string completedStatusFile = localProcessor.GetStatusFile(completedFile);
StatusFileEntry status = new StatusFileEntry
{
State = ProcessingState.Processing,
Timestamp = DateTime.UtcNow,
ChangeType = WatcherChangeTypes.Created,
InstanceId = "1"
};
StringWriter sw = new StringWriter();
_serializer.Serialize(sw, status);
sw.WriteLine();
status.State = ProcessingState.Processed;
status.Timestamp = status.Timestamp + TimeSpan.FromSeconds(15);
_serializer.Serialize(sw, status);
sw.WriteLine();
sw.Flush();
File.WriteAllText(completedStatusFile, sw.ToString());
// include an additional companion metadata file
string completedAdditionalFile = completedFile + ".metadata";
File.WriteAllText(completedAdditionalFile, "Data");
// write a file that SHOULDN'T be deleted
string dontDeleteFile = Path.ChangeExtension(completedFile, "json");
File.WriteAllText(dontDeleteFile, "Data");
// create an incomplete file set
string incompleteFile = WriteTestFile("dat");
string incompleteStatusFile = localProcessor.GetStatusFile(incompleteFile);
status = new StatusFileEntry
{
State = ProcessingState.Processing,
Timestamp = DateTime.UtcNow,
ChangeType = WatcherChangeTypes.Created,
InstanceId = "1"
};
sw = new StringWriter();
_serializer.Serialize(sw, status);
sw.WriteLine();
File.WriteAllText(incompleteStatusFile, sw.ToString());
localProcessor.Cleanup();
// expect the completed set to be deleted
Assert.False(File.Exists(completedFile));
Assert.False(File.Exists(completedAdditionalFile));
Assert.False(File.Exists(completedStatusFile));
Assert.True(File.Exists(dontDeleteFile));
// expect the incomplete set to remain
Assert.False(File.Exists(completedFile));
Assert.False(File.Exists(completedStatusFile));
}
public async Task ConcurrentListeners_ProcessFilesCorrectly(int concurrentListenerCount, int inputFileCount)
{
// mock out the executor so we can capture function invocations
Mock<ITriggeredFunctionExecutor> mockExecutor = new Mock<ITriggeredFunctionExecutor>(MockBehavior.Strict);
ConcurrentBag<string> processedFiles = new ConcurrentBag<string>();
FunctionResult result = new FunctionResult(true);
mockExecutor.Setup(p => p.TryExecuteAsync(It.IsAny<TriggeredFunctionData>(), It.IsAny<CancellationToken>()))
.Callback<TriggeredFunctionData, CancellationToken>(async (mockData, mockToken) =>
{
await Task.Delay(50);
FileSystemEventArgs fileEvent = mockData.TriggerValue as FileSystemEventArgs;
processedFiles.Add(fileEvent.Name);
})
.ReturnsAsync(result);
FilesConfiguration config = new FilesConfiguration()
{
RootPath = rootPath
};
FileTriggerAttribute attribute = new FileTriggerAttribute(attributeSubPath, changeTypes: WatcherChangeTypes.Created | WatcherChangeTypes.Changed, filter: "*.dat");
// create a bunch of listeners and start them
CancellationTokenSource tokenSource = new CancellationTokenSource();
CancellationToken cancellationToken = tokenSource.Token;
List<Task> listenerStartupTasks = new List<Task>();
List<FileListener> listeners = new List<FileListener>();
for (int i = 0; i < concurrentListenerCount; i++)
{
FileListener listener = new FileListener(config, attribute, mockExecutor.Object, new TestTraceWriter());
listeners.Add(listener);
listenerStartupTasks.Add(listener.StartAsync(cancellationToken));
};
await Task.WhenAll(listenerStartupTasks);
// now start creating files
List<string> expectedFiles = new List<string>();
for (int i = 0; i < inputFileCount; i++)
{
string file = WriteTestFile();
await Task.Delay(50);
expectedFiles.Add(Path.GetFileName(file));
}
// wait for all files to be processed
await TestHelpers.Await(() =>
{
return processedFiles.Count >= inputFileCount;
});
Assert.Equal(inputFileCount, processedFiles.Count);
// verify that each file was only processed once
Assert.True(expectedFiles.OrderBy(p => p).SequenceEqual(processedFiles.OrderBy(p => p)));
Assert.Equal(expectedFiles.Count * 2, Directory.GetFiles(testFileDir).Length);
// verify contents of each status file
FileProcessor processor = listeners[0].Processor;
foreach (string processedFile in processedFiles)
{
string statusFilePath = processor.GetStatusFile(Path.Combine(testFileDir, processedFile));
string[] statusLines = File.ReadAllLines(statusFilePath);
Assert.Equal(2, statusLines.Length);
StatusFileEntry statusEntry = JsonConvert.DeserializeObject<StatusFileEntry>(statusLines[0]);
Assert.Equal(ProcessingState.Processing, statusEntry.State);
Assert.Equal(WatcherChangeTypes.Created, statusEntry.ChangeType);
statusEntry = JsonConvert.DeserializeObject<StatusFileEntry>(statusLines[1]);
Assert.Equal(ProcessingState.Processed, statusEntry.State);
Assert.Equal(WatcherChangeTypes.Created, statusEntry.ChangeType);
}
// Now test concurrency handling for updates by updating some files
// and verifying the updates are only processed once
string[] filesToUpdate = processedFiles.Take(50).Select(p => Path.Combine(testFileDir, p)).ToArray();
string item;
while (!processedFiles.IsEmpty)
{
processedFiles.TryTake(out item);
}
await Task.Delay(1000);
foreach (string fileToUpdate in filesToUpdate)
{
await Task.Delay(50);
File.AppendAllText(fileToUpdate, "update");
}
// wait for all files to be processed
await TestHelpers.Await(() =>
{
return processedFiles.Count >= filesToUpdate.Length;
});
Assert.Equal(filesToUpdate.Length, processedFiles.Count);
Assert.Equal(expectedFiles.Count * 2, Directory.GetFiles(testFileDir).Length);
// verify the status files are correct for each of the updated files
foreach (string updatedFile in filesToUpdate)
{
string statusFilePath = processor.GetStatusFile(updatedFile);
string[] statusLines = File.ReadAllLines(statusFilePath);
Assert.Equal(4, statusLines.Length);
StatusFileEntry statusEntry = JsonConvert.DeserializeObject<StatusFileEntry>(statusLines[0]);
Assert.Equal(ProcessingState.Processing, statusEntry.State);
Assert.Equal(WatcherChangeTypes.Created, statusEntry.ChangeType);
statusEntry = JsonConvert.DeserializeObject<StatusFileEntry>(statusLines[1]);
Assert.Equal(ProcessingState.Processed, statusEntry.State);
Assert.Equal(WatcherChangeTypes.Created, statusEntry.ChangeType);
statusEntry = JsonConvert.DeserializeObject<StatusFileEntry>(statusLines[2]);
Assert.Equal(ProcessingState.Processing, statusEntry.State);
Assert.Equal(WatcherChangeTypes.Changed, statusEntry.ChangeType);
statusEntry = JsonConvert.DeserializeObject<StatusFileEntry>(statusLines[3]);
Assert.Equal(ProcessingState.Processed, statusEntry.State);
Assert.Equal(WatcherChangeTypes.Changed, statusEntry.ChangeType);
}
// Now call purge to clean up all processed files
processor.CleanupProcessedFiles();
Assert.Equal(0, Directory.GetFiles(testFileDir).Length);
foreach (FileListener listener in listeners)
{
listener.Dispose();
}
}
