// Triggers the processing event for the given file. private bool OnProcessing(string fullPath, bool alreadyProcessed) { FileProcessorEventArgs args; if ((object)Processing != null) { args = new FileProcessorEventArgs(fullPath, alreadyProcessed); Processing(this, args); return(args.Requeue); } return(false); }
// Attempts to processes the given file. // Returns true if the user requested to requeue the file. private void ProcessFile(FileProcessorEventArgs args) { string filePath = args.FullPath; // Requeue is requested by the user so // we must reset it before raising the event args.Requeue = false; if (m_disposed || !File.Exists(filePath)) { return; } OnProcessing(args); }
// Triggers the processing event for the given file. private void OnProcessing(FileProcessorEventArgs args) { Processing?.Invoke(this, args); }
// Runs an asynchronous loop to process the file. // Continues looping so long as the user continues requesting to requeue. private async Task RunProcessLoopAsync(string filePath, bool raisedByFileWatcher) { int retryCount = 0; int RetryCounter() => retryCount; FileProcessorEventArgs args = new FileProcessorEventArgs(filePath, raisedByFileWatcher, RetryCounter); // 8 * 250 ms = 2 sec (cumulative: 2 sec) const int FastRetryLimit = 8; const int FastRetryDelay = 250; // 13 * 1000 ms = 13 sec (cumulative: 15 sec) const int QuickRetryLimit = 13 + FastRetryLimit; const int QuickRetryDelay = 1000; // 9 * 5000 ms = 45 sec (cumulative: 60 sec) const int RelaxedRetryLimit = 9 + QuickRetryLimit; const int RelaxedRetryDelay = 5000; // After 60 seconds, continue with the slow retry delay const int SlowRetryDelay = 60000; int GetDelay() { if (retryCount < FastRetryLimit) { return(FastRetryDelay); } if (retryCount < QuickRetryLimit) { return(QuickRetryDelay); } if (retryCount < RelaxedRetryLimit) { return(RelaxedRetryDelay); } return(SlowRetryDelay); } using (m_requeueTokenSource.RetrieveToken(out var cancellationToken)) { while (true) { if (cancellationToken.IsCancellationRequested) { break; } int priority = (retryCount < RelaxedRetryLimit) ? 2 : 1; await m_processingThread.Join(priority); if (cancellationToken.IsCancellationRequested) { break; } ProcessFile(args); if (!args.Requeue) { break; } if (retryCount == 0) { Interlocked.Increment(ref m_requeuedFileCount); } retryCount++; int delay = GetDelay(); try { await Task.Delay(delay, cancellationToken); } catch (TaskCanceledException) { break; } } if (retryCount > 0) { Interlocked.Decrement(ref m_requeuedFileCount); } } }
// Called when the file processor has picked up a file in one of the watch // directories. This handler validates the file and processes it if able. private void FileProcessor_Processing(object sender, FileProcessorEventArgs fileProcessorEventArgs) { if (m_disposed) return; try { string filePath; string connectionString; SystemSettings systemSettings; filePath = fileProcessorEventArgs.FullPath; if (!FilePath.TryGetReadLockExclusive(filePath)) { fileProcessorEventArgs.Requeue = true; return; } connectionString = LoadSystemSettings(); systemSettings = new SystemSettings(connectionString); using (DbAdapterContainer dbAdapterContainer = new DbAdapterContainer(systemSettings.DbConnectionString, systemSettings.DbTimeout)) { try { ProcessFile( fileProcessorArgs: fileProcessorEventArgs, connectionString: connectionString, systemSettings: systemSettings, dbAdapterContainer: dbAdapterContainer); } catch (Exception ex) { // There may be a problem here where the outer exception's call stack // was overwritten by the call stack of the point where it was thrown ExceptionDispatchInfo exInfo = ExceptionDispatchInfo.Capture(ex); try { // Attempt to set the error flag on the file group FileInfoDataContext fileInfo = dbAdapterContainer.GetAdapter<FileInfoDataContext>(); FileWrapper fileWrapper = m_fileWrapperLookup.GetOrAdd(filePath, path => new FileWrapper(path)); FileGroup fileGroup = fileWrapper.GetFileGroup(fileInfo, systemSettings.XDATimeZoneInfo); fileGroup.ProcessingEndTime = fileGroup.ProcessingStartTime; fileGroup.Error = 1; fileInfo.SubmitChanges(); } catch (Exception fileGroupError) { // Log exceptions that occur when setting the error flag on the file group string message = $"Exception occurred setting error flag on file group: {fileGroupError.Message}"; OnProcessException(new Exception(message, fileGroupError)); } // Throw the original exception exInfo.Throw(); } } } catch (FileSkippedException) { // Do not wrap FileSkippedExceptions because // these only generate warning messages throw; } catch (Exception ex) { // Wrap all other exceptions to include the file path in the message string message = $"Exception occurred processing file \"{fileProcessorEventArgs.FullPath}\": {ex.Message}"; throw new Exception(message, ex); } finally { // Make sure to clean up file wrappers from // the lookup table to prevent memory leaks if (!fileProcessorEventArgs.Requeue) m_fileWrapperLookup.Remove(fileProcessorEventArgs.FullPath); } }
// Processes the file to determine if it can be parsed and kicks off the meter's processing thread. private void ProcessFile(FileProcessorEventArgs fileProcessorArgs, string connectionString, SystemSettings systemSettings, DbAdapterContainer dbAdapterContainer) { string filePath; string meterKey; FileInfoDataContext fileInfo; SystemInfoDataContext systemInfo; DataReader dataReader; DataReaderWrapper dataReaderWrapper; FileWrapper fileWrapper; int queuedFileCount; filePath = fileProcessorArgs.FullPath; fileInfo = dbAdapterContainer.GetAdapter<FileInfoDataContext>(); // Determine whether the file has already been // processed or needs to be processed again if (fileProcessorArgs.AlreadyProcessed) { DataFile dataFile = fileInfo.DataFiles .Where(file => file.FilePathHash == filePath.GetHashCode()) .Where(file => file.FilePath == filePath) .MaxBy(file => file.ID); // This will tell us whether the service was stopped in the middle // of processing the last time it attempted to process the file if ((object)dataFile != null && dataFile.FileGroup.ProcessingEndTime > DateTime.MinValue) { Log.Debug($"Skipped file \"{filePath}\" because it has already been processed."); return; } } // Get the data reader that will be used to parse the file systemInfo = dbAdapterContainer.GetAdapter<SystemInfoDataContext>(); dataReader = systemInfo.DataReaders .OrderBy(reader => reader.LoadOrder) .AsEnumerable() .FirstOrDefault(reader => FilePath.IsFilePatternMatch(reader.FilePattern, filePath, true)); if ((object)dataReader == null) { // Because the file processor is filtering files based on the DataReader file patterns, // this should only ever occur if the configuration changes during runtime UpdateFileProcessorFilter(systemSettings); throw new FileSkippedException($"Skipped file \"{filePath}\" because no data reader could be found to process the file."); } dataReaderWrapper = Wrap(dataReader); try { meterKey = null; // Determine whether the database contains configuration information for the meter that produced this file if ((object)dataReaderWrapper.DataObject.MeterDataSet != null) meterKey = GetMeterKey(filePath, systemSettings.FilePattern); // Apply connection string settings to the data reader ConnectionStringParser.ParseConnectionString(connectionString, dataReaderWrapper.DataObject); // Get the file wrapper from the lookup table fileWrapper = m_fileWrapperLookup.GetOrAdd(filePath, path => new FileWrapper(path)); // Determine whether the dataReader can parse the file if (!dataReaderWrapper.DataObject.CanParse(filePath, fileWrapper.GetMaxFileCreationTime())) { fileProcessorArgs.Requeue = true; dataReaderWrapper.Dispose(); return; } // Get the thread used to process this data GetThread(meterKey).Push(() => ParseFile(connectionString, systemSettings, filePath, meterKey, dataReaderWrapper, fileWrapper)); // Keep track of the number of operations in thread queues queuedFileCount = Interlocked.Increment(ref m_queuedFileCount); while (!m_stopped && !m_disposed && m_queuedFileCount >= systemSettings.MaxQueuedFileCount) Thread.Sleep(1000); } catch { // If an error occurs here, dispose of the data reader; // otherwise, the meter data thread will handle it dataReaderWrapper.Dispose(); throw; } }
// Triggers the processing event for the given file. private bool OnProcessing(string fullPath, bool alreadyProcessed) { FileProcessorEventArgs args; if ((object)Processing != null) { args = new FileProcessorEventArgs(fullPath, alreadyProcessed); Processing(this, args); return args.Requeue; } return false; }