public async Task ProcessFileAsync_JobFunctionSucceeds() { string testFile = WriteTestFile("dat"); FunctionResult result = new FunctionResult(true); mockExecutor.Setup(p => p.TryExecuteAsync(It.IsAny <TriggeredFunctionData>(), It.IsAny <CancellationToken>())).ReturnsAsync(result); FileSystemEventArgs eventArgs = new FileSystemEventArgs(WatcherChangeTypes.Created, combinedTestFilePath, Path.GetFileName(testFile)); await processor.ProcessFileAsync(eventArgs, CancellationToken.None); Assert.Equal(2, Directory.GetFiles(combinedTestFilePath).Length); string expectedStatusFile = processor.GetStatusFile(testFile); Assert.True(File.Exists(testFile)); Assert.True(File.Exists(expectedStatusFile)); string[] statusLines = File.ReadAllLines(expectedStatusFile); Assert.Equal(2, statusLines.Length); StatusFileEntry entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(statusLines[0]), typeof(StatusFileEntry)); Assert.Equal(ProcessingState.Processing, entry.State); Assert.Equal(WatcherChangeTypes.Created, entry.ChangeType); Assert.Equal(processor.InstanceId, entry.InstanceId); entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(statusLines[1]), typeof(StatusFileEntry)); Assert.Equal(ProcessingState.Processed, entry.State); Assert.Equal(WatcherChangeTypes.Created, entry.ChangeType); Assert.Equal(processor.InstanceId, entry.InstanceId); }
/// <summary> /// Determines whether the specified file should be processed. /// </summary> /// <param name="filePath">The candidate file for processing.</param> /// <returns>True if the file should be processed, false otherwise.</returns> public virtual bool ShouldProcessFile(string filePath) { if (IsStatusFile(filePath)) { return(false); } string statusFilePath = GetStatusFile(filePath); if (!File.Exists(statusFilePath)) { return(true); } StatusFileEntry statusEntry = null; try { GetLastStatus(statusFilePath, out statusEntry); } catch (IOException) { // if we get an exception reading the status file, it's // likely because someone started processing and has it locked return(false); } return(statusEntry == null || (statusEntry.State != ProcessingState.Processed && statusEntry.ProcessCount < MaxProcessCount)); }
private void ProcessFiles() { // scan for any files that require processing (either new unprocessed files, // or files that have failed previous processing) IEnumerable <string> unprocessedFiles = Directory.GetFiles(_watchPath, _attribute.Filter) .Where(p => _processor.ShouldProcessFile(p)).ToArray(); foreach (string fileToProcess in unprocessedFiles) { WatcherChangeTypes changeType = WatcherChangeTypes.Created; string statusFilePath = _processor.GetStatusFile(fileToProcess); if (File.Exists(statusFilePath)) { // if an in progress status file exists, we determine the ChangeType // from the last entry (incomplete) in the file StatusFileEntry statusEntry = _processor.GetLastStatus(statusFilePath); if (statusEntry != null) { changeType = statusEntry.ChangeType; } } string fileName = Path.GetFileName(fileToProcess); FileSystemEventArgs args = new FileSystemEventArgs(changeType, _watchPath, fileName); _workQueue.Post(args); } }
public async Task ProcessFileAsync_ChangeTypeChange_Success() { FileTriggerAttribute attribute = new FileTriggerAttribute(AttributeSubPath, "*.dat"); processor = CreateTestProcessor(attribute); string testFile = WriteTestFile("dat"); FunctionResult result = new FunctionResult(true); mockExecutor.Setup(p => p.TryExecuteAsync(It.IsAny <TriggeredFunctionData>(), It.IsAny <CancellationToken>())).ReturnsAsync(result); // first process a Create event string testFilePath = Path.GetDirectoryName(testFile); string testFileName = Path.GetFileName(testFile); FileSystemEventArgs eventArgs = new FileSystemEventArgs(WatcherChangeTypes.Created, testFilePath, testFileName); bool fileProcessedSuccessfully = await processor.ProcessFileAsync(eventArgs, CancellationToken.None); Assert.True(fileProcessedSuccessfully); // Wait briefly so the changed time is slightly different than the created time await Task.Delay(10); // now process a Change event File.WriteAllText(testFile, "update"); eventArgs = new FileSystemEventArgs(WatcherChangeTypes.Changed, testFilePath, testFileName); fileProcessedSuccessfully = await processor.ProcessFileAsync(eventArgs, CancellationToken.None); Assert.True(fileProcessedSuccessfully); string expectedStatusFile = processor.GetStatusFile(testFile); Assert.True(File.Exists(testFile)); Assert.True(File.Exists(expectedStatusFile)); string[] lines = File.ReadAllLines(expectedStatusFile); Assert.Equal(4, lines.Length); StatusFileEntry entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(lines[0]), typeof(StatusFileEntry)); Assert.Equal(ProcessingState.Processing, entry.State); Assert.Equal(WatcherChangeTypes.Created, entry.ChangeType); Assert.Equal(InstanceId.Substring(0, 20), entry.InstanceId); entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(lines[1]), typeof(StatusFileEntry)); Assert.Equal(ProcessingState.Processed, entry.State); Assert.Equal(WatcherChangeTypes.Created, entry.ChangeType); Assert.Equal(InstanceId.Substring(0, 20), entry.InstanceId); entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(lines[2]), typeof(StatusFileEntry)); Assert.Equal(ProcessingState.Processing, entry.State); Assert.Equal(WatcherChangeTypes.Changed, entry.ChangeType); Assert.Equal(InstanceId.Substring(0, 20), entry.InstanceId); entry = (StatusFileEntry)_serializer.Deserialize(new StringReader(lines[3]), typeof(StatusFileEntry)); Assert.Equal(ProcessingState.Processed, entry.State); Assert.Equal(WatcherChangeTypes.Changed, entry.ChangeType); Assert.Equal(InstanceId.Substring(0, 20), entry.InstanceId); }
/// <summary> /// Clean up any files that have been fully processed /// </summary> public virtual void CleanupProcessedFiles() { int filesDeleted = 0; string[] statusFiles = Directory.GetFiles(_filePath, GetStatusFile("*")); foreach (string statusFilePath in statusFiles) { try { // verify that the file has been fully processed // if we're unable to get the last status or the file // is not Processed, skip it StatusFileEntry statusEntry = null; if (!GetLastStatus(statusFilePath, out statusEntry) || statusEntry.State != ProcessingState.Processed) { continue; } // get all files starting with that file name. For example, for // status file input.dat.status, this might return input.dat and // input.dat.meta (if the file has other companion files) string targetFileName = Path.GetFileNameWithoutExtension(statusFilePath); string[] files = Directory.GetFiles(_filePath, targetFileName + "*"); // first delete the non status file(s) foreach (string filePath in files) { if (IsStatusFile(filePath)) { continue; } if (TryDelete(filePath)) { filesDeleted++; } } // then delete the status file if (TryDelete(statusFilePath)) { filesDeleted++; } } catch { // ignore any delete failures } } if (filesDeleted > 0) { _logger.LogDebug($"File Cleanup ({_filePath}): {filesDeleted} files deleted"); } }
internal bool GetLastStatus(string statusFilePath, out StatusFileEntry statusEntry) { statusEntry = null; if (!File.Exists(statusFilePath)) { return(false); } using (Stream stream = File.OpenRead(statusFilePath)) { statusEntry = GetLastStatus(stream); } return(statusEntry != null); }
public async Task ProcessFileAsync_Failure_LeavesInProgressStatusFile() { string testFile = WriteTestFile("dat"); FunctionResult result = new FunctionResult(false); mockExecutor.Setup(p => p.TryExecuteAsync(It.IsAny <TriggeredFunctionData>(), It.IsAny <CancellationToken>())).ReturnsAsync(result); FileSystemEventArgs eventArgs = new FileSystemEventArgs(WatcherChangeTypes.Created, Path.GetDirectoryName(testFile), Path.GetFileName(testFile)); bool fileProcessedSuccessfully = await processor.ProcessFileAsync(eventArgs, CancellationToken.None); Assert.False(fileProcessedSuccessfully); Assert.True(File.Exists(testFile)); string statusFilePath = processor.GetStatusFile(testFile); StatusFileEntry entry = processor.GetLastStatus(statusFilePath); Assert.Equal(ProcessingState.Processing, entry.State); }
public async Task ProcessFileAsync_AlreadyProcessing_ReturnsWithoutProcessing() { string testFile = WriteTestFile("dat"); // first take a lock on the status file StatusFileEntry status = null; using (StreamWriter statusFile = processor.AcquireStatusFileLock(testFile, WatcherChangeTypes.Created, out status)) { // now attempt to process the file FileSystemEventArgs eventArgs = new FileSystemEventArgs(WatcherChangeTypes.Created, Path.GetDirectoryName(testFile), Path.GetFileName(testFile)); bool fileProcessedSuccessfully = await processor.ProcessFileAsync(eventArgs, CancellationToken.None); Assert.False(fileProcessedSuccessfully); mockExecutor.Verify(p => p.TryExecuteAsync(It.IsAny <TriggeredFunctionData>(), It.IsAny <CancellationToken>()), Times.Never); } string statusFilePath = processor.GetStatusFile(testFile); File.Delete(statusFilePath); }
private void ProcessFiles() { // scan for any files that require processing (either new unprocessed files, // or files that have failed previous processing) string[] filesToProcess = Directory.GetFiles(_watchPath, _attribute.Filter) .Where(p => _processor.ShouldProcessFile(p)).ToArray(); if (filesToProcess.Length > 0) { _trace.Verbose(string.Format("Found {0} file(s) at path '{1}' for ready processing", filesToProcess.Length, _watchPath)); } foreach (string fileToProcess in filesToProcess) { WatcherChangeTypes changeType = WatcherChangeTypes.Created; string statusFilePath = _processor.GetStatusFile(fileToProcess); try { StatusFileEntry statusEntry = null; if (_processor.GetLastStatus(statusFilePath, out statusEntry)) { // if an in progress status file exists, we determine the ChangeType // from the last entry (incomplete) in the file changeType = statusEntry.ChangeType; } } catch (IOException) { // if we get an exception reading the status file, it's // likely because someone started processing and has it locked continue; } string fileName = Path.GetFileName(fileToProcess); FileSystemEventArgs args = new FileSystemEventArgs(changeType, _watchPath, fileName); _workQueue.Post(args); } }
internal StatusFileEntry GetLastStatus(Stream statusFileStream) { StatusFileEntry statusEntry = null; using (StreamReader reader = new StreamReader(statusFileStream, Encoding.UTF8, false, 1024, true)) { string text = reader.ReadToEnd(); string[] fileLines = text.Split(new string[] { Environment.NewLine }, StringSplitOptions.RemoveEmptyEntries); string lastLine = fileLines.LastOrDefault(); if (!string.IsNullOrEmpty(lastLine)) { using (StringReader stringReader = new StringReader(lastLine)) { statusEntry = (StatusFileEntry)_serializer.Deserialize(stringReader, typeof(StatusFileEntry)); } } } statusFileStream.Seek(0, SeekOrigin.End); return(statusEntry); }
public void Cleanup_AutoDeleteOn_DeletesCompletedFiles() { FileTriggerAttribute attribute = new FileTriggerAttribute(attributeSubPath, "*.dat", autoDelete: true); FileProcessor localProcessor = CreateTestProcessor(attribute); // create a completed file set string completedFile = WriteTestFile("dat"); string completedStatusFile = localProcessor.GetStatusFile(completedFile); StatusFileEntry status = new StatusFileEntry { State = ProcessingState.Processing, Timestamp = DateTime.UtcNow, ChangeType = WatcherChangeTypes.Created, InstanceId = "1" }; StringWriter sw = new StringWriter(); _serializer.Serialize(sw, status); sw.WriteLine(); status.State = ProcessingState.Processed; status.Timestamp = status.Timestamp + TimeSpan.FromSeconds(15); _serializer.Serialize(sw, status); sw.WriteLine(); sw.Flush(); File.WriteAllText(completedStatusFile, sw.ToString()); // include an additional companion metadata file string completedAdditionalFile = completedFile + ".metadata"; File.WriteAllText(completedAdditionalFile, "Data"); // write a file that SHOULDN'T be deleted string dontDeleteFile = Path.ChangeExtension(completedFile, "json"); File.WriteAllText(dontDeleteFile, "Data"); // create an incomplete file set string incompleteFile = WriteTestFile("dat"); string incompleteStatusFile = localProcessor.GetStatusFile(incompleteFile); status = new StatusFileEntry { State = ProcessingState.Processing, Timestamp = DateTime.UtcNow, ChangeType = WatcherChangeTypes.Created, InstanceId = "1" }; sw = new StringWriter(); _serializer.Serialize(sw, status); sw.WriteLine(); File.WriteAllText(incompleteStatusFile, sw.ToString()); localProcessor.Cleanup(); // expect the completed set to be deleted Assert.False(File.Exists(completedFile)); Assert.False(File.Exists(completedAdditionalFile)); Assert.False(File.Exists(completedStatusFile)); Assert.True(File.Exists(dontDeleteFile)); // expect the incomplete set to remain Assert.False(File.Exists(completedFile)); Assert.False(File.Exists(completedStatusFile)); }
public async Task ConcurrentListeners_ProcessFilesCorrectly(int concurrentListenerCount, int inputFileCount) { // mock out the executor so we can capture function invocations Mock <ITriggeredFunctionExecutor> mockExecutor = new Mock <ITriggeredFunctionExecutor>(MockBehavior.Strict); ConcurrentBag <string> processedFiles = new ConcurrentBag <string>(); FunctionResult result = new FunctionResult(true); mockExecutor.Setup(p => p.TryExecuteAsync(It.IsAny <TriggeredFunctionData>(), It.IsAny <CancellationToken>())) .Callback <TriggeredFunctionData, CancellationToken>(async(mockData, mockToken) => { await Task.Delay(50); FileSystemEventArgs fileEvent = mockData.TriggerValue as FileSystemEventArgs; processedFiles.Add(fileEvent.Name); }) .ReturnsAsync(result); var options = new FilesOptions() { RootPath = rootPath }; FileTriggerAttribute attribute = new FileTriggerAttribute(attributeSubPath, changeTypes: WatcherChangeTypes.Created | WatcherChangeTypes.Changed, filter: "*.dat"); // create a bunch of listeners and start them CancellationTokenSource tokenSource = new CancellationTokenSource(); CancellationToken cancellationToken = tokenSource.Token; List <Task> listenerStartupTasks = new List <Task>(); List <FileListener> listeners = new List <FileListener>(); for (int i = 0; i < concurrentListenerCount; i++) { FileListener listener = new FileListener(new OptionsWrapper <FilesOptions>(options), attribute, mockExecutor.Object, new TestLogger("Test"), new DefaultFileProcessorFactory()); listeners.Add(listener); listenerStartupTasks.Add(listener.StartAsync(cancellationToken)); } await Task.WhenAll(listenerStartupTasks); // now start creating files List <string> expectedFiles = new List <string>(); for (int i = 0; i < inputFileCount; i++) { string file = WriteTestFile(); await Task.Delay(50); expectedFiles.Add(Path.GetFileName(file)); } // wait for all files to be processed await TestHelpers.Await(() => { return(processedFiles.Count >= inputFileCount); }); Assert.Equal(inputFileCount, processedFiles.Count); // verify that each file was only processed once Assert.True(expectedFiles.OrderBy(p => p).SequenceEqual(processedFiles.OrderBy(p => p))); Assert.Equal(expectedFiles.Count * 2, Directory.GetFiles(testFileDir).Length); // verify contents of each status file FileProcessor processor = listeners[0].Processor; foreach (string processedFile in processedFiles) { string statusFilePath = processor.GetStatusFile(Path.Combine(testFileDir, processedFile)); string[] statusLines = File.ReadAllLines(statusFilePath); Assert.Equal(2, statusLines.Length); StatusFileEntry statusEntry = JsonConvert.DeserializeObject <StatusFileEntry>(statusLines[0]); Assert.Equal(ProcessingState.Processing, statusEntry.State); Assert.Equal(WatcherChangeTypes.Created, statusEntry.ChangeType); statusEntry = JsonConvert.DeserializeObject <StatusFileEntry>(statusLines[1]); Assert.Equal(ProcessingState.Processed, statusEntry.State); Assert.Equal(WatcherChangeTypes.Created, statusEntry.ChangeType); } // Now test concurrency handling for updates by updating some files // and verifying the updates are only processed once string[] filesToUpdate = processedFiles.Take(50).Select(p => Path.Combine(testFileDir, p)).ToArray(); string item; while (!processedFiles.IsEmpty) { processedFiles.TryTake(out item); } await Task.Delay(1000); foreach (string fileToUpdate in filesToUpdate) { await Task.Delay(50); File.AppendAllText(fileToUpdate, "update"); } // wait for all files to be processed await TestHelpers.Await(() => { return(processedFiles.Count >= filesToUpdate.Length); }); Assert.Equal(filesToUpdate.Length, processedFiles.Count); Assert.Equal(expectedFiles.Count * 2, Directory.GetFiles(testFileDir).Length); // verify the status files are correct for each of the updated files foreach (string updatedFile in filesToUpdate) { string statusFilePath = processor.GetStatusFile(updatedFile); string[] statusLines = File.ReadAllLines(statusFilePath); Assert.Equal(4, statusLines.Length); StatusFileEntry statusEntry = JsonConvert.DeserializeObject <StatusFileEntry>(statusLines[0]); Assert.Equal(ProcessingState.Processing, statusEntry.State); Assert.Equal(WatcherChangeTypes.Created, statusEntry.ChangeType); statusEntry = JsonConvert.DeserializeObject <StatusFileEntry>(statusLines[1]); Assert.Equal(ProcessingState.Processed, statusEntry.State); Assert.Equal(WatcherChangeTypes.Created, statusEntry.ChangeType); statusEntry = JsonConvert.DeserializeObject <StatusFileEntry>(statusLines[2]); Assert.Equal(ProcessingState.Processing, statusEntry.State); Assert.Equal(WatcherChangeTypes.Changed, statusEntry.ChangeType); statusEntry = JsonConvert.DeserializeObject <StatusFileEntry>(statusLines[3]); Assert.Equal(ProcessingState.Processed, statusEntry.State); Assert.Equal(WatcherChangeTypes.Changed, statusEntry.ChangeType); } // Now clean up all processed files processor.CleanupProcessedFiles(); Assert.Empty(Directory.GetFiles(testFileDir)); foreach (FileListener listener in listeners) { listener.Dispose(); } }
internal StreamWriter AcquireStatusFileLock(string filePath, WatcherChangeTypes changeType, out StatusFileEntry statusEntry) { Stream stream = null; statusEntry = null; try { // Attempt to create (or update) the companion status file and lock it. The status // file is the mechanism for handling multi-instance concurrency. string statusFilePath = GetStatusFile(filePath); stream = File.Open(statusFilePath, FileMode.OpenOrCreate, FileAccess.ReadWrite, FileShare.None); // Once we've established the lock, we need to check to ensure that another instance // hasn't already processed the file in the time between our getting the event and // acquiring the lock. statusEntry = GetLastStatus(stream); if (statusEntry != null && statusEntry.State == ProcessingState.Processed) { // For file Create, we have no additional checks to perform. However for // file Change, we need to also check the LastWrite value for the entry // since there can be multiple Processed entries in the file over time. if (changeType == WatcherChangeTypes.Created) { return(null); } else if (changeType == WatcherChangeTypes.Changed && File.GetLastWriteTimeUtc(filePath) == statusEntry.LastWrite) { return(null); } } stream.Seek(0, SeekOrigin.End); StreamWriter streamReader = new StreamWriter(stream); streamReader.AutoFlush = true; stream = null; return(streamReader); } catch { return(null); } finally { if (stream != null) { stream.Dispose(); } } }
/// <summary> /// Process the file indicated by the specified <see cref="FileSystemEventArgs"/>. /// </summary> /// <param name="eventArgs">The <see cref="FileSystemEventArgs"/> indicating the file to process.</param> /// <param name="cancellationToken">The <see cref="CancellationToken"/> to use.</param> /// <returns> /// A <see cref="Task"/> that returns true if the file was processed successfully, false otherwise. /// </returns> public virtual async Task <bool> ProcessFileAsync(FileSystemEventArgs eventArgs, CancellationToken cancellationToken) { try { StatusFileEntry status = null; string filePath = eventArgs.FullPath; using (StreamWriter statusWriter = AcquireStatusFileLock(filePath, eventArgs.ChangeType, out status)) { if (statusWriter == null) { return(false); } // We've acquired the lock. The current status might be either Failed // or Processing (if processing failed before we were unable to update // the file status to Failed) int processCount = 0; if (status != null) { processCount = status.ProcessCount; } while (processCount++ < MaxProcessCount) { FunctionResult result = null; if (result != null) { TimeSpan delay = GetRetryInterval(result, processCount); await Task.Delay(delay); } // write an entry indicating the file is being processed status = new StatusFileEntry { State = ProcessingState.Processing, Timestamp = DateTime.Now, LastWrite = File.GetLastWriteTimeUtc(filePath), ChangeType = eventArgs.ChangeType, InstanceId = InstanceId, ProcessCount = processCount }; _serializer.Serialize(statusWriter, status); statusWriter.WriteLine(); // invoke the job function TriggeredFunctionData input = new TriggeredFunctionData { TriggerValue = eventArgs }; result = await _executor.TryExecuteAsync(input, cancellationToken); // write a status entry indicating the state of processing status.State = result.Succeeded ? ProcessingState.Processed : ProcessingState.Failed; status.Timestamp = DateTime.Now; _serializer.Serialize(statusWriter, status); statusWriter.WriteLine(); if (result.Succeeded) { return(true); } } return(false); } } catch { return(false); } }