// Called when the file processor has picked up a file in one of the watch
// directories. This handler validates the file and processes it if able.
private void FileProcessor_Processing(object sender, FileProcessorEventArgs fileProcessorEventArgs)
{
if (m_disposed)
return;
try
{
string filePath;
string connectionString;
SystemSettings systemSettings;
filePath = fileProcessorEventArgs.FullPath;
if (!FilePath.TryGetReadLockExclusive(filePath))
{
fileProcessorEventArgs.Requeue = true;
return;
}
connectionString = LoadSystemSettings();
systemSettings = new SystemSettings(connectionString);
using (DbAdapterContainer dbAdapterContainer = new DbAdapterContainer(systemSettings.DbConnectionString, systemSettings.DbTimeout))
{
try
{
ProcessFile(
fileProcessorArgs: fileProcessorEventArgs,
connectionString: connectionString,
systemSettings: systemSettings,
dbAdapterContainer: dbAdapterContainer);
}
catch (Exception ex)
{
// There may be a problem here where the outer exception's call stack
// was overwritten by the call stack of the point where it was thrown
ExceptionDispatchInfo exInfo = ExceptionDispatchInfo.Capture(ex);
try
{
// Attempt to set the error flag on the file group
FileInfoDataContext fileInfo = dbAdapterContainer.GetAdapter<FileInfoDataContext>();
FileWrapper fileWrapper = m_fileWrapperLookup.GetOrAdd(filePath, path => new FileWrapper(path));
FileGroup fileGroup = fileWrapper.GetFileGroup(fileInfo, systemSettings.XDATimeZoneInfo);
fileGroup.ProcessingEndTime = fileGroup.ProcessingStartTime;
fileGroup.Error = 1;
fileInfo.SubmitChanges();
}
catch (Exception fileGroupError)
{
// Log exceptions that occur when setting the error flag on the file group
string message = $"Exception occurred setting error flag on file group: {fileGroupError.Message}";
OnProcessException(new Exception(message, fileGroupError));
}
// Throw the original exception
exInfo.Throw();
}
}
}
catch (FileSkippedException)
{
// Do not wrap FileSkippedExceptions because
// these only generate warning messages
throw;
}
catch (Exception ex)
{
// Wrap all other exceptions to include the file path in the message
string message = $"Exception occurred processing file \"{fileProcessorEventArgs.FullPath}\": {ex.Message}";
throw new Exception(message, ex);
}
finally
{
// Make sure to clean up file wrappers from
// the lookup table to prevent memory leaks
if (!fileProcessorEventArgs.Requeue)
m_fileWrapperLookup.Remove(fileProcessorEventArgs.FullPath);
}
}
/// <summary>
/// Reloads system settings from the database.
/// </summary>
public void ReloadSystemSettings()
{
ConfigurationFile configurationFile;
CategorizedSettingsElementCollection category;
// Reload the configuration file
configurationFile = ConfigurationFile.Current;
configurationFile.Reload();
// Retrieve the connection string from the config file
category = configurationFile.Settings["systemSettings"];
category.Add("ConnectionString", "Data Source=localhost; Initial Catalog=openXDA; Integrated Security=SSPI", "Defines the connection to the openXDA database.");
m_dbConnectionString = category["ConnectionString"].Value;
// Load system settings from the database
m_systemSettings = new SystemSettings(LoadSystemSettings());
// Update the limit on the number of processing threads
if ((object)m_meterDataScheduler != null)
m_meterDataScheduler.MaxThreadCount = m_systemSettings.ProcessingThreadCount;
// Attempt to authenticate to configured file shares
foreach (FileShare fileShare in m_systemSettings.FileShareList)
{
if (!fileShare.TryAuthenticate())
OnProcessException(fileShare.AuthenticationException);
}
// Update the FileProcessor with the latest system settings
if ((object)m_fileProcessor != null)
{
m_fileProcessor.InternalBufferSize = m_systemSettings.FileWatcherBufferSize;
m_fileProcessor.EnumerationStrategy = m_systemSettings.FileWatcherEnumerationStrategy;
m_fileProcessor.MaxThreadCount = m_systemSettings.FileWatcherInternalThreadCount;
m_fileProcessor.MaxFragmentation = m_systemSettings.FileWatcherMaxFragmentation;
UpdateFileProcessorFilter(m_systemSettings);
foreach (string directory in m_fileProcessor.TrackedDirectories.ToList())
{
if (!m_systemSettings.WatchDirectoryList.Contains(directory, StringComparer.OrdinalIgnoreCase))
m_fileProcessor.RemoveTrackedDirectory(directory);
}
foreach (string directory in m_systemSettings.WatchDirectoryList)
m_fileProcessor.AddTrackedDirectory(directory);
}
}
// Determines whether the timestamps in the file extend beyond user-defined thresholds.
private static void ValidateFileTimestamps(string filePath, FileGroup fileGroup, SystemSettings systemSettings, FileInfoDataContext fileInfo)
{
DateTime now;
double timeDifference;
// Get the current time in XDA's time zone
now = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, systemSettings.XDATimeZoneInfo);
// Determine whether past timestamp validation is disabled
if (systemSettings.MinTimeOffset > 0.0D)
{
// Get the total number of hours between the current time and the start time of the data in the file
timeDifference = now.Subtract(fileGroup.DataStartTime).TotalHours;
// Determine whether the number of hours exceeds the threshold
if (timeDifference > systemSettings.MinTimeOffset)
throw new FileSkippedException($"Skipped file \"{filePath}\" because data start time '{fileGroup.DataStartTime}' is too old.");
}
// Determine whether future timestamp validation is disabled
if (systemSettings.MaxTimeOffset > 0.0D)
{
// Get the total number of hours between the current time and the end time of the data in the file
timeDifference = fileGroup.DataEndTime.Subtract(now).TotalHours;
// Determine whether the number of hours exceeds the threshold
if (timeDifference > systemSettings.MaxTimeOffset)
throw new FileSkippedException($"Skipped file \"{filePath}\" because data end time '{fileGroup.DataEndTime}' is too far in the future.");
}
}
// Updates the Filter property of the FileProcessor with the
// latest collection of filters from the DataReader table.
private void UpdateFileProcessorFilter(SystemSettings systemSettings)
{
SystemInfoDataContext systemInfo;
List<string> filterPatterns;
// Do not attempt to load filter patterns if file processor is not defined
if ((object)m_fileProcessor == null)
return;
// Get the list of file extensions to be processed by openXDA
using (DbAdapterContainer dbAdapterContainer = new DbAdapterContainer(systemSettings.DbConnectionString, systemSettings.DbTimeout))
{
systemInfo = dbAdapterContainer.GetAdapter<SystemInfoDataContext>();
filterPatterns = systemInfo.DataReaders
.Select(reader => reader.FilePattern)
.ToList();
}
m_fileProcessor.Filter = string.Join(Path.PathSeparator.ToString(), filterPatterns);
}
// Instantiates and executes data operations and data writers to process the meter data set.
private void ProcessMeterDataSet(MeterDataSet meterDataSet, SystemSettings systemSettings, DbAdapterContainer dbAdapterContainer)
{
SystemInfoDataContext systemInfo;
List<DataOperationWrapper> dataOperations = null;
List<DataWriterWrapper> dataWriters = null;
// Get the SystemInfoDataContext from the dbAdapterContainer
systemInfo = dbAdapterContainer.GetAdapter<SystemInfoDataContext>();
// Load the data operations from the database,
// in descending order so we can remove records while we iterate
dataOperations = systemInfo.DataOperations
.OrderByDescending(dataOperation => dataOperation.LoadOrder)
.Select(Wrap)
.Where(wrapper => (object)wrapper != null)
.ToList();
for (int i = dataOperations.Count - 1; i >= 0; i--)
{
try
{
Log.Debug($"Preparing data operation '{dataOperations[i].DataObject.GetType().Name}' for execution...");
// Load configuration parameters from the connection string into the data operation
ConnectionStringParser.ParseConnectionString(meterDataSet.ConnectionString, dataOperations[i].DataObject);
// Call the prepare method to allow the data operation to prepare any data it needs from the database
dataOperations[i].DataObject.Prepare(dbAdapterContainer);
Log.Debug($"Finished preparing data operation '{dataOperations[i].DataObject.GetType().Name}' for execution.");
}
catch (Exception ex)
{
// Log the error and remove the data operation from the list
string message = $"An error occurred while preparing data from meter '{meterDataSet.Meter.AssetKey}' for data operation of type '{dataOperations[i].DataObject.GetType().FullName}': {ex.Message}";
OnProcessException(new Exception(message, ex));
dataOperations[i].Dispose();
dataOperations.RemoveAt(i);
}
}
for (int i = dataOperations.Count - 1; i >= 0; i--)
{
try
{
Log.Debug($"Executing data operation '{dataOperations[i].DataObject.GetType().Name}'...");
// Call the execute method on the data operation to perform in-memory data transformations
dataOperations[i].DataObject.Execute(meterDataSet);
Log.Debug($"Finished execurting data operation '{dataOperations[i].DataObject.GetType().Name}'.");
}
catch (Exception ex)
{
// Log the error and skip to the next data operation
string message = $"An error occurred while executing data operation of type '{dataOperations[i].DataObject.GetType().FullName}' on data from meter '{meterDataSet.Meter.AssetKey}': {ex.Message}";
OnProcessException(new Exception(message, ex));
continue;
}
try
{
Log.Debug($"Loading data from data operation '{dataOperations[i].DataObject.GetType().Name}' into database...");
// Call the load method inside a transaction to load data into from the data operation into the database
using (TransactionScope transaction = new TransactionScope(TransactionScopeOption.Required, GetTransactionOptions()))
{
dataOperations[i].DataObject.Load(dbAdapterContainer);
transaction.Complete();
}
Log.Debug($"Finished loading data from data operation '{dataOperations[i].DataObject.GetType().Name}' into database.");
}
catch (Exception ex)
{
// Log the error and move on to the next data operation
string message = $"An error occurred while loading data from data operation of type '{dataOperations[i].DataObject.GetType().FullName}' for data from meter '{meterDataSet.Meter.AssetKey}': {ex.Message}";
OnProcessException(new Exception(message, ex));
}
}
// All data operations are complete, but we still need to clean up
for (int i = dataOperations.Count - 1; i >= 0; i--)
dataOperations[i].Dispose();
// Load the data writers from the database
dataWriters = systemInfo.DataWriters
.OrderBy(dataWriter => dataWriter.LoadOrder)
.Select(Wrap)
.Where(wrapper => (object)wrapper != null)
.ToList();
foreach (DataWriterWrapper dataWriter in dataWriters)
{
try
{
Log.Debug($"Writing results to external location with data writer '{dataWriter.DataObject.GetType().Name}'...");
// Load configuration parameters from the connection string into the data writer
ConnectionStringParser.ParseConnectionString(meterDataSet.ConnectionString, dataWriter.DataObject);
// Write the results to the data writer's destination by calling the WriteResults method
dataWriter.DataObject.WriteResults(dbAdapterContainer, meterDataSet);
Log.Debug($"Finished writing results with data writer '{dataWriter.DataObject.GetType().Name}'.");
}
catch (Exception ex)
{
// Log the error and move on to the next data writer
string message = $"An error occurred while writing data from meter '{meterDataSet.Meter.AssetKey}' using data writer of type '{dataWriter.DataObject.GetType().FullName}': {ex.Message}";
OnProcessException(new Exception(message, ex));
}
}
// All data writers are complete, but we still need to clean up
foreach (DataWriterWrapper dataWriter in dataWriters)
dataWriter.Dispose();
}
// Processes the file to determine if it can be parsed and kicks off the meter's processing thread.
private void ProcessFile(FileProcessorEventArgs fileProcessorArgs, string connectionString, SystemSettings systemSettings, DbAdapterContainer dbAdapterContainer)
{
string filePath;
string meterKey;
FileInfoDataContext fileInfo;
SystemInfoDataContext systemInfo;
DataReader dataReader;
DataReaderWrapper dataReaderWrapper;
FileWrapper fileWrapper;
int queuedFileCount;
filePath = fileProcessorArgs.FullPath;
fileInfo = dbAdapterContainer.GetAdapter<FileInfoDataContext>();
// Determine whether the file has already been
// processed or needs to be processed again
if (fileProcessorArgs.AlreadyProcessed)
{
DataFile dataFile = fileInfo.DataFiles
.Where(file => file.FilePathHash == filePath.GetHashCode())
.Where(file => file.FilePath == filePath)
.MaxBy(file => file.ID);
// This will tell us whether the service was stopped in the middle
// of processing the last time it attempted to process the file
if ((object)dataFile != null && dataFile.FileGroup.ProcessingEndTime > DateTime.MinValue)
{
Log.Debug($"Skipped file \"{filePath}\" because it has already been processed.");
return;
}
}
// Get the data reader that will be used to parse the file
systemInfo = dbAdapterContainer.GetAdapter<SystemInfoDataContext>();
dataReader = systemInfo.DataReaders
.OrderBy(reader => reader.LoadOrder)
.AsEnumerable()
.FirstOrDefault(reader => FilePath.IsFilePatternMatch(reader.FilePattern, filePath, true));
if ((object)dataReader == null)
{
// Because the file processor is filtering files based on the DataReader file patterns,
// this should only ever occur if the configuration changes during runtime
UpdateFileProcessorFilter(systemSettings);
throw new FileSkippedException($"Skipped file \"{filePath}\" because no data reader could be found to process the file.");
}
dataReaderWrapper = Wrap(dataReader);
try
{
meterKey = null;
// Determine whether the database contains configuration information for the meter that produced this file
if ((object)dataReaderWrapper.DataObject.MeterDataSet != null)
meterKey = GetMeterKey(filePath, systemSettings.FilePattern);
// Apply connection string settings to the data reader
ConnectionStringParser.ParseConnectionString(connectionString, dataReaderWrapper.DataObject);
// Get the file wrapper from the lookup table
fileWrapper = m_fileWrapperLookup.GetOrAdd(filePath, path => new FileWrapper(path));
// Determine whether the dataReader can parse the file
if (!dataReaderWrapper.DataObject.CanParse(filePath, fileWrapper.GetMaxFileCreationTime()))
{
fileProcessorArgs.Requeue = true;
dataReaderWrapper.Dispose();
return;
}
// Get the thread used to process this data
GetThread(meterKey).Push(() => ParseFile(connectionString, systemSettings, filePath, meterKey, dataReaderWrapper, fileWrapper));
// Keep track of the number of operations in thread queues
queuedFileCount = Interlocked.Increment(ref m_queuedFileCount);
while (!m_stopped && !m_disposed && m_queuedFileCount >= systemSettings.MaxQueuedFileCount)
Thread.Sleep(1000);
}
catch
{
// If an error occurs here, dispose of the data reader;
// otherwise, the meter data thread will handle it
dataReaderWrapper.Dispose();
throw;
}
}
// Validates the file before invoking the file processing handler.
// Improves file processor performance by executing the filter in
// parallel and also by bypassing the set of processed files.
private bool PrevalidateFile(string filePath)
{
try
{
string meterKey;
string connectionString;
SystemSettings systemSettings;
connectionString = LoadSystemSettings();
systemSettings = new SystemSettings(connectionString);
ValidateFileCreationTime(filePath, systemSettings.MaxFileCreationTimeOffset);
using (DbAdapterContainer dbAdapterContainer = new DbAdapterContainer(systemSettings.DbConnectionString, systemSettings.DbTimeout))
{
meterKey = GetMeterKey(filePath, systemSettings.FilePattern);
ValidateMeterKey(filePath, meterKey, dbAdapterContainer.GetAdapter<MeterInfoDataContext>());
}
return true;
}
catch (FileSkippedException ex)
{
// This method may be called if the file was deleted,
// in which case the user almost certainly doesn't care
// why it was skipped for processing and logging the
// error would only cause confusion
if (File.Exists(filePath))
Log.Warn(ex.Message);
return false;
}
}
// Parses the file on the meter's processing thread and kicks off processing of the meter data set.
private void ParseFile(string connectionString, SystemSettings systemSettings, string filePath, string meterKey, DataReaderWrapper dataReaderWrapper, FileWrapper fileWrapper)
{
FileGroup fileGroup = null;
MeterDataSet meterDataSet;
int queuedFileCount;
// Keep track of the number of operations in thread queues
queuedFileCount = Interlocked.Decrement(ref m_queuedFileCount);
if (m_stopped || m_disposed)
{
dataReaderWrapper.Dispose();
return;
}
using (dataReaderWrapper)
using (DbAdapterContainer dbAdapterContainer = new DbAdapterContainer(systemSettings.DbConnectionString, systemSettings.DbTimeout))
{
try
{
// Keep track of the meters and files currently being processed
if ((object)meterKey != null)
m_activeFiles[meterKey] = filePath;
ThreadContext.Properties["Meter"] = meterKey;
// Create the file group
fileGroup = fileWrapper.GetFileGroup(dbAdapterContainer.GetAdapter<FileInfoDataContext>(), systemSettings.XDATimeZoneInfo);
// Parse the file to turn it into a meter data set
OnStatusMessage($"Parsing data from file \"{filePath}\"...");
dataReaderWrapper.DataObject.Parse(filePath);
OnStatusMessage($"Finished parsing data from file \"{filePath}\".");
meterDataSet = dataReaderWrapper.DataObject.MeterDataSet;
// If the data reader does not return a data set,
// there is nothing left to do
if ((object)meterDataSet == null)
return;
// Data reader has finally outlived its usefulness
dataReaderWrapper.Dispose();
// Set file path, file group, connection string,
// and meter asset key for the meter data set
meterDataSet.FilePath = filePath;
meterDataSet.FileGroup = fileGroup;
meterDataSet.ConnectionString = connectionString;
meterDataSet.Meter.AssetKey = meterKey;
// Shift date/time values to the configured time zone and set the start and end time values on the file group
ShiftTime(meterDataSet, meterDataSet.Meter.GetTimeZoneInfo(systemSettings.DefaultMeterTimeZoneInfo), systemSettings.XDATimeZoneInfo);
SetDataTimeRange(meterDataSet, dbAdapterContainer.GetAdapter<FileInfoDataContext>());
// Determine whether the file duration is within a user-defined maximum tolerance
ValidateFileDuration(meterDataSet.FilePath, systemSettings.MaxFileDuration, meterDataSet.FileGroup);
// Determine whether the timestamps in the file extend beyond user-defined thresholds
ValidateFileTimestamps(meterDataSet.FilePath, meterDataSet.FileGroup, systemSettings, dbAdapterContainer.GetAdapter<FileInfoDataContext>());
// Process the meter data set
OnStatusMessage($"Processing meter data from file \"{filePath}\"...");
ProcessMeterDataSet(meterDataSet, systemSettings, dbAdapterContainer);
OnStatusMessage($"Finished processing data from file \"{filePath}\".");
}
catch (Exception ex)
{
// There seems to be a problem here where the outer exception's call stack
// was overwritten by the call stack of the point where it was thrown
ExceptionDispatchInfo exInfo = ExceptionDispatchInfo.Capture(ex);
try
{
// Attempt to set the error flag on the file group
if ((object)fileGroup != null)
fileGroup.Error = 1;
}
catch (Exception fileGroupError)
{
// Log any exceptions that occur when attempting to set the error flag on the file group
string message = $"Exception occurred setting error flag on file group: {fileGroupError.Message}";
OnProcessException(new Exception(message, fileGroupError));
}
// Throw the original exception
exInfo.Throw();
}
finally
{
if ((object)fileGroup != null)
{
try
{
// Attempt to set the processing end time of the file group
fileGroup.ProcessingEndTime = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, systemSettings.XDATimeZoneInfo);
dbAdapterContainer.GetAdapter<FileInfoDataContext>().SubmitChanges();
}
catch (Exception ex)
{
// Log any exceptions that occur when attempting to set processing end time on the file group
string message = $"Exception occurred setting processing end time on file group: {ex.Message}";
OnProcessException(new Exception(message, ex));
}
}
// Keep track of the meters and files currently being processed
if ((object)meterKey != null)
m_activeFiles.TryRemove(meterKey, out filePath);
ThreadContext.Properties.Remove("Meter");
}
}
}