// Determines whether configuration exists for the meter with the given asset key.
private static void ValidateMeterKey(SystemSettings systemSettings, string filePath, string meterKey)
{
UserAccount xdaUserAccount = new UserAccount();
xdaUserAccount.Name = systemSettings.XDAUsername;
xdaUserAccount.Password = systemSettings.XDAPassword;
// Determine whether there exists a meter whose asset key matches the given meterKey
if (WebAPIHub.GetRecordsWhere <Meter>(systemSettings.XDAAddress, $"AssetKey = '{meterKey}'", xdaUserAccount).Count() == 0)
{
throw new FileSkippedException($"Skipped file \"{filePath}\" because no meter configuration was found for meter {meterKey}.");
}
}
// Saves the given file group to the database using the given connection.
private void SaveFileGroup(FileGroup fileGroup, string xdaAddress, UserAccount userAccount)
{
fileGroup.ID = WebAPIHub.CreateRecord(xdaAddress, fileGroup, userAccount);
foreach (DataFile dataFile in fileGroup.DataFiles)
{
dataFile.FileGroupID = fileGroup.ID;
dataFile.ID = WebAPIHub.CreateRecord(xdaAddress, dataFile, userAccount);
FileBlob fileBlob = new FileBlob();
fileBlob.DataFileID = dataFile.ID;
fileBlob.Blob = File.ReadAllBytes(dataFile.FilePath);
WebAPIHub.CreateRecord(xdaAddress, fileBlob, userAccount);
}
}
// Updates the Filter property of the FileProcessor with the
// latest collection of filters from the DataReader table.
private void UpdateFileProcessorFilter(SystemSettings systemSettings)
{
// Do not attempt to load filter patterns if file processor is not defined
if ((object)m_fileProcessor == null)
{
return;
}
UserAccount userAccount = new UserAccount();
userAccount.Name = systemSettings.XDAUsername;
userAccount.Password = systemSettings.XDAPassword;
// Get the list of file extensions to be processed by openXDA
List <string> filterPatterns = WebAPIHub.GetRecords <DataReader>(systemSettings.XDAAddress, "all", userAccount)
.Select(reader => reader.FilePattern)
.ToList();
m_fileProcessor.Filter = string.Join(Path.PathSeparator.ToString(), filterPatterns);
}
// Loads system settings from the database.
private string LoadSystemSettings()
{
List <Setting> settingList = WebAPIHub.GetRecords <Setting>(m_xdaAddress, "all", m_xdaUserAccount)
.ToList();
foreach (IGrouping <string, Setting> grouping in settingList.GroupBy(setting => setting.Name))
{
if (grouping.Count() > 1)
{
Log.Warn($"Duplicate record for setting {grouping.Key} detected.");
}
}
// Convert the Setting table to a dictionary
Dictionary <string, string> settings = settingList
.DistinctBy(setting => setting.Name)
.ToDictionary(setting => setting.Name, setting => setting.Value, StringComparer.OrdinalIgnoreCase);
// Add the configuration files settings if they are not
// already explicitly specified in the Setting table
if (!settings.ContainsKey("XDAAddress"))
{
settings.Add("XDAAddress", m_xdaAddress);
}
if (!settings.ContainsKey("XDAUsername"))
{
settings.Add("XDAUsername", m_xdaUserAccount.Name);
}
if (!settings.ContainsKey("XDAPassword"))
{
settings.Add("XDAPassword", m_xdaUserAccount.Password);
}
// Convert dictionary to a connection string and return it
return(SystemSettings.ToConnectionString(settings));
}
// Called when the file processor has picked up a file in one of the watch
// directories. This handler validates the file and processes it if able.
private void FileProcessor_Processing(object sender, FileProcessorEventArgs fileProcessorEventArgs)
{
if (m_stopped || m_disposed)
{
return;
}
try
{
string filePath;
int priority;
filePath = fileProcessorEventArgs.FullPath;
priority = fileProcessorEventArgs.RaisedByFileWatcher
? FileWatcherPriority
: FileEnumerationPriority;
string connectionString = LoadSystemSettings();
SystemSettings systemSettings = new SystemSettings(connectionString);
UserAccount userAccount = new UserAccount();
userAccount.Name = systemSettings.XDAUsername;
userAccount.Password = systemSettings.XDAPassword;
// Determine whether the file has already been
// processed or needs to be processed again
if (fileProcessorEventArgs.AlreadyProcessed)
{
DataFile remoteDataFile = WebAPIHub.GetRecordsWhere <DataFile>(systemSettings.XDAAddress, $"FilePathHash = {filePath.GetHashCode()}", userAccount)
.Where(file => file.FilePath == filePath)
.MaxBy(file => file.ID);
if ((object)remoteDataFile != null)
{
FileGroup remoteFileGroup = WebAPIHub.GetRecordWhere <FileGroup>(systemSettings.XDAAddress, $"ID = {remoteDataFile.FileGroupID}", userAccount);
// This will tell us whether the service was stopped in the middle
// of processing the last time it attempted to process the file
if (remoteFileGroup.ProcessingEndTime > DateTime.MinValue)
{
// Explicitly use Log.Debug() so that the message does not appear on the remote console,
// but include a FileSkippedException so that the message gets routed to the skipped files log
FileSkippedException ex = new FileSkippedException($"Skipped file \"{filePath}\" because it has already been processed.");
Log.Debug(ex.Message, ex);
return;
}
}
}
byte[] buffer;
try
{
buffer = File.ReadAllBytes(filePath);
}
catch (IOException)
{
// Couldn't read from the file, likely because the
// process writing the file isn't finished writing
fileProcessorEventArgs.Requeue = true;
return;
}
Crc32 crc32 = new Crc32();
crc32.Update(buffer);
int crc = (int)crc32.Value;
if (systemSettings.SkipOnCRCHashMatch)
{
List <FileGroup> fileGroups = WebAPIHub.GetRecordsWhere <FileGroup>(systemSettings.XDAAddress, $"FileHash = {crc}", userAccount)
.ToList();
if (fileGroups.Any())
{
string fileGroupIDs = string.Join(",", fileGroups.Select(fg => fg.ID));
List <DataFile> dataFiles = WebAPIHub.GetRecordsWhere <DataFile>(systemSettings.XDAAddress, $"FileGroupID IN ({fileGroupIDs})", userAccount)
.ToList();
if (dataFiles.Any())
{
string dataFileIDs = string.Join(",", dataFiles.Select(dataFile => dataFile.ID));
List <FileBlob> fileBlobs = WebAPIHub.GetRecordsWhere <FileBlob>(systemSettings.XDAAddress, $"DataFileID IN ({dataFileIDs})", userAccount)
.ToList();
if (fileBlobs.Any(fileBlob => buffer.SequenceEqual(fileBlob.Blob)))
{
FileSkippedException ex = new FileSkippedException($"Skipped file \"{filePath}\" because it has already been processed.");
Log.Warn(ex.Message, ex);
return;
}
}
}
}
Log.Info($"Pushing file {filePath} into openXDA...");
FileGroup fileGroup = new FileGroup();
fileGroup.FileHash = crc;
fileGroup.DataFiles = CreateDataFiles(filePath, systemSettings.XDATimeZoneInfo);
SaveFileGroup(fileGroup, systemSettings.XDAAddress, userAccount);
string meterKey = GetMeterKey(filePath, systemSettings.FilePattern);
Meter meter = WebAPIHub.GetRecordsWhere <Meter>(systemSettings.XDAAddress, $"AssetKey = '{meterKey}'", userAccount).FirstOrDefault();
Dictionary <string, int> idObject = new Dictionary <string, int>();
idObject.Add("FileGroupID", fileGroup.ID);
idObject.Add("MeterID", meter.ID);
WebAPIHub.ProcessFileGroup(systemSettings.XDAAddress, JObject.FromObject(idObject), userAccount);
Log.Info($"Finished pushing file {filePath} into openXDA.");
}
catch (FileSkippedException)
{
// Do not wrap FileSkippedExceptions because
// these only generate warning messages
throw;
}
catch (Exception ex)
{
// Wrap all other exceptions to include the file path in the message
string message = $"Exception occurred processing file \"{fileProcessorEventArgs.FullPath}\": {ex.Message}";
throw new Exception(message, ex);
}
}
