protected override DateTime?GetValueWhenNotNull(DataReaderWrapper dataReader, int columnIndex)
{
return(dataReader.GetDateTime(columnIndex));
}
// Parses the file on the meter's processing thread and kicks off processing of the meter data set.
private void ParseFile(string connectionString, SystemSettings systemSettings, string filePath, string meterKey, DataReaderWrapper dataReaderWrapper, FileWrapper fileWrapper)
{
FileGroup fileGroup = null;
MeterDataSet meterDataSet;
int queuedFileCount;
// Keep track of the number of operations in thread queues
queuedFileCount = Interlocked.Decrement(ref m_queuedFileCount);
if (m_stopped || m_disposed)
{
dataReaderWrapper.Dispose();
return;
}
using (dataReaderWrapper)
using (DbAdapterContainer dbAdapterContainer = new DbAdapterContainer(systemSettings.DbConnectionString, systemSettings.DbTimeout))
{
try
{
// Keep track of the meters and files currently being processed
if ((object)meterKey != null)
m_activeFiles[meterKey] = filePath;
ThreadContext.Properties["Meter"] = meterKey;
// Create the file group
fileGroup = fileWrapper.GetFileGroup(dbAdapterContainer.GetAdapter<FileInfoDataContext>(), systemSettings.XDATimeZoneInfo);
// Parse the file to turn it into a meter data set
OnStatusMessage($"Parsing data from file \"{filePath}\"...");
dataReaderWrapper.DataObject.Parse(filePath);
OnStatusMessage($"Finished parsing data from file \"{filePath}\".");
meterDataSet = dataReaderWrapper.DataObject.MeterDataSet;
// If the data reader does not return a data set,
// there is nothing left to do
if ((object)meterDataSet == null)
return;
// Data reader has finally outlived its usefulness
dataReaderWrapper.Dispose();
// Set file path, file group, connection string,
// and meter asset key for the meter data set
meterDataSet.FilePath = filePath;
meterDataSet.FileGroup = fileGroup;
meterDataSet.ConnectionString = connectionString;
meterDataSet.Meter.AssetKey = meterKey;
// Shift date/time values to the configured time zone and set the start and end time values on the file group
ShiftTime(meterDataSet, meterDataSet.Meter.GetTimeZoneInfo(systemSettings.DefaultMeterTimeZoneInfo), systemSettings.XDATimeZoneInfo);
SetDataTimeRange(meterDataSet, dbAdapterContainer.GetAdapter<FileInfoDataContext>());
// Determine whether the file duration is within a user-defined maximum tolerance
ValidateFileDuration(meterDataSet.FilePath, systemSettings.MaxFileDuration, meterDataSet.FileGroup);
// Determine whether the timestamps in the file extend beyond user-defined thresholds
ValidateFileTimestamps(meterDataSet.FilePath, meterDataSet.FileGroup, systemSettings, dbAdapterContainer.GetAdapter<FileInfoDataContext>());
// Process the meter data set
OnStatusMessage($"Processing meter data from file \"{filePath}\"...");
ProcessMeterDataSet(meterDataSet, systemSettings, dbAdapterContainer);
OnStatusMessage($"Finished processing data from file \"{filePath}\".");
}
catch (Exception ex)
{
// There seems to be a problem here where the outer exception's call stack
// was overwritten by the call stack of the point where it was thrown
ExceptionDispatchInfo exInfo = ExceptionDispatchInfo.Capture(ex);
try
{
// Attempt to set the error flag on the file group
if ((object)fileGroup != null)
fileGroup.Error = 1;
}
catch (Exception fileGroupError)
{
// Log any exceptions that occur when attempting to set the error flag on the file group
string message = $"Exception occurred setting error flag on file group: {fileGroupError.Message}";
OnProcessException(new Exception(message, fileGroupError));
}
// Throw the original exception
exInfo.Throw();
}
finally
{
if ((object)fileGroup != null)
{
try
{
// Attempt to set the processing end time of the file group
fileGroup.ProcessingEndTime = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, systemSettings.XDATimeZoneInfo);
dbAdapterContainer.GetAdapter<FileInfoDataContext>().SubmitChanges();
}
catch (Exception ex)
{
// Log any exceptions that occur when attempting to set processing end time on the file group
string message = $"Exception occurred setting processing end time on file group: {ex.Message}";
OnProcessException(new Exception(message, ex));
}
}
// Keep track of the meters and files currently being processed
if ((object)meterKey != null)
m_activeFiles.TryRemove(meterKey, out filePath);
ThreadContext.Properties.Remove("Meter");
}
}
}
public override UInt64 GetValue(DataReaderWrapper dataReader, int columnIndex, Type targetType)
{
return(Convert.ToUInt64(dataReader.GetValue(columnIndex)));
}
protected override bool?GetValueWhenNotNull(DataReaderWrapper dataReader, int columnIndex)
{
return(Convert.ToBoolean(dataReader.GetValue(columnIndex)));
}
public override object GetValue(DataReaderWrapper dataReader, int columnIndex, Type targetType)
{
return(dataReader.GetValue(columnIndex));
}
private object ImportText()
{
var fileName = Project.Current.MapPath(FileName);
if (string.IsNullOrWhiteSpace(fileName) || !File.Exists(fileName))
{
throw new FileNotFoundException($"File '{fileName}' not found.");
}
if (Columns == null && NoHeaderRow)
{
throw new Exception($"{nameof(NoHeaderRow)} requires providing {nameof(Columns)} schema.");
}
var schema = new SeparatedValueSchema();
if (Columns != null)
{
foreach (var column in Columns)
{
var definition = CreateColumnDefinition(column);
schema.AddColumn(definition);
}
}
var options = new SeparatedValueOptions();
if (Separator != null)
{
options.Separator = Separator;
}
if (RecordSeparator != null)
{
options.RecordSeparator = RecordSeparator;
}
options.Quote = Quote;
options.IsFirstRecordSchema = !NoHeaderRow;
options.PreserveWhiteSpace = PreserveWhiteSpace;
if (!string.IsNullOrWhiteSpace(Culture))
{
options.FormatProvider = new CultureInfo(Culture);
}
var readerOptions = new FlatFileDataReaderOptions()
{
IsDBNullReturned = true,
IsNullStringAllowed = true
};
var reader = new StreamReader(File.OpenRead(fileName));
var csvReader = Columns != null ? new SeparatedValueReader(reader, schema, options) : new SeparatedValueReader(reader, options);
var dataReader = new FlatFileDataReader(csvReader, readerOptions);
var resultReader = new DataReaderWrapper(dataReader, new DataReaderWrapper.DataReaderWrapperParameters()
{
Columns = this.SelectColumns,
SkipColumns = this.SkipColumns,
IgnoreReaderErrors = this.IgnoreReaderErrors,
CloseAction = () =>
{
reader.Dispose();
}
});
if (AsDataReader)
{
return(resultReader);
}
else
{
var table = new DataTable("TextData");
table.Load(resultReader);
return(table);
}
}
protected override Single?GetValueWhenNotNull(DataReaderWrapper dataReader, int columnIndex)
{
return(Convert.ToSingle(dataReader.GetValue(columnIndex)));
}
protected override TEnum GetValueWhenNotNull(DataReaderWrapper dataReader, int columnIndex)
{
return((TEnum)Enum.ToObject(_enumType, dataReader.GetValue(columnIndex)));
}
public override TEnum GetValue(DataReaderWrapper dataReader, int columnIndex, Type targetType)
{
return((TEnum)Enum.ToObject(PropertyType, dataReader.GetValue(columnIndex)));
}
protected override Single?GetValueWhenNotNull(DataReaderWrapper dataReader, int columnIndex)
{
return(dataReader.GetFloat(columnIndex));
}
protected override Decimal?GetValueWhenNotNull(DataReaderWrapper dataReader, int columnIndex)
{
return(Convert.ToDecimal(dataReader.GetValue(columnIndex)));
}
public abstract TProperty GetValue(DataReaderWrapper dataReader, int columnIndex, Type targetType);
protected virtual TProperty GetValueWhenNotNull(DataReaderWrapper dataReader, int columnIndex)
{
return((TProperty)dataReader.GetValue(columnIndex));
}
protected override Char?GetValueWhenNotNull(DataReaderWrapper dataReader, int columnIndex)
{
return(dataReader.GetString(columnIndex)[0]);
}
public static TProperty GetValue(DataReaderWrapper dataReader, int columnIndex, Type targetType)
{
return(Handler.GetValue(dataReader, columnIndex, targetType));
}
