private List <MeterDataSet> LoadMeterDataSets(AdoDataConnection connection, FileGroup fileGroup)
{
List <MeterDataSet> meterDataSets = new List <MeterDataSet>();
IEnumerable <Event> eventTable = (new TableOperations <Event>(connection)).QueryRecordsWhere("FileGroupID = {0}", fileGroup.ID);
MeterDataSet meterDataSet;
DataGroup dataGroup;
foreach (IGrouping <int, Event> eventGroup in eventTable.GroupBy(evt => evt.MeterID))
{
meterDataSet = new MeterDataSet();
meterDataSet.Meter = (new TableOperations <Meter>(connection)).QueryRecordWhere("ID = {0}", eventGroup.Key);
meterDataSet.Meter.ConnectionFactory = () => new AdoDataConnection(connection.Connection, typeof(SqlDataAdapter), false);
foreach (Event evt in eventGroup)
{
dataGroup = new DataGroup();
dataGroup.FromData(meterDataSet.Meter, (new TableOperations <EventData>(connection)).QueryRecordWhere("ID = {0}", evt.EventDataID).TimeDomainData);
foreach (DataSeries dataSeries in dataGroup.DataSeries)
{
meterDataSet.DataSeries.Add(dataSeries);
}
}
meterDataSets.Add(meterDataSet);
}
return(meterDataSets);
}
private List <MeterDataSet> LoadMeterDataSets(DbAdapterContainer dbAdapterContainer, FileGroup fileGroup)
{
List <MeterDataSet> meterDataSets = new List <MeterDataSet>();
MeterInfoDataContext meterInfo = dbAdapterContainer.GetAdapter <MeterInfoDataContext>();
EventTableAdapter eventAdapter = dbAdapterContainer.GetAdapter <EventTableAdapter>();
EventDataTableAdapter eventDataAdapter = dbAdapterContainer.GetAdapter <EventDataTableAdapter>();
MeterData.EventDataTable eventTable = eventAdapter.GetDataByFileGroup(fileGroup.ID);
MeterDataSet meterDataSet;
DataGroup dataGroup;
foreach (IGrouping <int, MeterData.EventRow> eventGroup in eventTable.GroupBy(evt => evt.MeterID))
{
meterDataSet = new MeterDataSet();
meterDataSet.Meter = meterInfo.Meters.SingleOrDefault(meter => meter.ID == eventGroup.Key);
foreach (MeterData.EventRow evt in eventGroup)
{
dataGroup = new DataGroup();
dataGroup.FromData(meterDataSet.Meter, eventDataAdapter.GetTimeDomainData(evt.EventDataID));
foreach (DataSeries dataSeries in dataGroup.DataSeries)
{
meterDataSet.DataSeries.Add(dataSeries);
}
}
meterDataSets.Add(meterDataSet);
}
return(meterDataSets);
}
private void WriteResults(MeterDataSet meterDataSet, DataGroup dataGroup, VICycleDataGroup viCycleDataGroup, List <Fault> faults, string resultsFilePath)
{
XDocument resultsDocument;
XElement results;
string lineName;
lineName = meterDataSet.Meter.MeterLines
.Where(ml => ml.LineID == dataGroup.Line.ID)
.Select(ml => ml.LineName)
.FirstOrDefault() ?? dataGroup.Line.AssetKey;
results =
new XElement("results",
new XElement("meter", meterDataSet.Meter.Name),
new XElement("disturbanceFiles", GetPathElements(meterDataSet.FileGroup)),
new XElement("line",
new XElement("name", lineName),
new XElement("length", dataGroup.Line.Length.ToString(DoubleFormat))
),
new XElement("prefault",
new XElement("time", dataGroup.StartTime.ToString(DateTimeFormat)),
GetCycleElements(viCycleDataGroup, 0)
),
GetFaultElements(faults)
);
// Create the XML document
resultsDocument =
new XDocument(
new XElement("openFLE", results)
);
resultsDocument.Save(resultsFilePath);
}
public void Parse(string meterKey, string filePath)
{
string text = File.ReadAllText(filePath);
MeterDataSet = new MeterDataSet(meterKey, "systemSettings", filePath, text);
MeterDataSet.Type = DataSetType.Config;
}
private void ExecuteDataOperation(MeterDataSet meterDataSet, DbAdapterContainer dbAdapterContainer)
{
IDataOperation dataOperation = null;
try
{
// Create the data operation
dataOperation = new openEASSandBoxOperation();
// Provide system settings to the data operation
ConnectionStringParser.ParseConnectionString(meterDataSet.ConnectionString, dataOperation);
// Prepare for execution of the data operation
dataOperation.Prepare(dbAdapterContainer);
// Execute the data operation
dataOperation.Execute(meterDataSet);
// Load data from all data operations in a single transaction
using (TransactionScope transactionScope = new TransactionScope(TransactionScopeOption.Required, GetTransactionOptions()))
{
dataOperation.Load(dbAdapterContainer);
transactionScope.Complete();
}
}
finally
{
// ReSharper disable once SuspiciousTypeConversion.Global
if ((object)dataOperation != null)
{
TryDispose(dataOperation as IDisposable);
}
}
}
public void WriteResults(DbAdapterContainer dbAdapterContainer, MeterDataSet meterDataSet)
{
CycleDataResource cycleDataResource;
FaultDataResource faultDataResource;
DataGroup dataGroup;
VICycleDataGroup viCycleDataGroup;
FaultGroup faultGroup;
string rootFileName;
string fileName;
cycleDataResource = meterDataSet.GetResource(() => CycleDataResource.GetResource(meterDataSet, dbAdapterContainer));
faultDataResource = meterDataSet.GetResource(() => new FaultDataResource(dbAdapterContainer));
if (!Directory.Exists(m_resultsPath))
{
Directory.CreateDirectory(m_resultsPath);
}
for (int i = 0; i < cycleDataResource.DataGroups.Count; i++)
{
dataGroup = cycleDataResource.DataGroups[i];
if (faultDataResource.FaultLookup.TryGetValue(dataGroup, out faultGroup))
{
rootFileName = FilePath.GetFileNameWithoutExtension(meterDataSet.FilePath);
fileName = string.Format("{0},{1:000},Line{2}.xml", rootFileName, i, dataGroup.Line.AssetKey);
viCycleDataGroup = cycleDataResource.VICycleDataGroups[i];
WriteResults(meterDataSet, dataGroup, viCycleDataGroup, faultGroup.Faults, Path.Combine(m_resultsPath, fileName));
}
}
}
public Dictionary <string, string> ParseFields(MeterDataSet meterDataSet)
{
if (meterDataSet.Type != DataSetType.Config)
{
return(new Dictionary <string, string>());
}
Dictionary <string, string> result = new Dictionary <string, string>();
List <string> lines = meterDataSet.Text.Split('\n').ToList();
int i = 1;
foreach (string line in lines)
{
if (line.Contains('='))
{
List <string> parts = line.Split('=').ToList();
if (result.ContainsKey(parts[0]))
{
result.Add(parts[0] + "-" + i, string.Join("=", parts.Skip(1)));
}
else
{
result.Add(parts[0], string.Join("=", parts.Skip(1)));
}
}
i++;
}
return(result);
}
private bool HasComplianceMeter(MeterDataSet meterDataSet)
{
using (AdoDataConnection connection = new AdoDataConnection("systemSettings"))
{
TableOperations <ComplianceMeter> meterTbl = new TableOperations <ComplianceMeter>(connection);
return(meterTbl.QueryRecordCountWhere("MeterId = {0}", meterDataSet.Meter.ID) > 0);
}
}
public void WriteResults(DbAdapterContainer dbAdapterContainer, MeterDataSet meterDataSet)
{
Initialize(this);
foreach (EventRow evt in dbAdapterContainer.GetAdapter <EventTableAdapter>().GetDataByFileGroup(meterDataSet.FileGroup.ID))
{
if (GetEmailCount(dbAdapterContainer, evt.ID) == 0)
{
QueueEventID(evt.ID);
}
}
}
public void WriteResults(DbAdapterContainer dbAdapterContainer, MeterDataSet meterDataSet)
{
CycleDataResource cycleDataResource;
FaultDataResource faultDataResource;
DataGroup dataGroup;
FaultGroup faultGroup;
List <int> seriesIDs;
EventDataSet eventDataSet;
string rootFileName;
string fileName;
cycleDataResource = meterDataSet.GetResource(() => CycleDataResource.GetResource(meterDataSet, dbAdapterContainer));
faultDataResource = meterDataSet.GetResource(() => new FaultDataResource(dbAdapterContainer));
if (!Directory.Exists(m_resultsPath))
{
Directory.CreateDirectory(m_resultsPath);
}
for (int i = 0; i < cycleDataResource.DataGroups.Count; i++)
{
dataGroup = cycleDataResource.DataGroups[i];
if (faultDataResource.FaultLookup.TryGetValue(dataGroup, out faultGroup))
{
rootFileName = FilePath.GetFileNameWithoutExtension(meterDataSet.FilePath);
fileName = string.Format("{0},{1:000},Line{2}.dat", rootFileName, i, dataGroup.Line.AssetKey);
seriesIDs = dataGroup.DataSeries
.Select(series => series.SeriesInfo.ID)
.ToList();
eventDataSet = new EventDataSet()
{
ResultsPath = Path.Combine(m_resultsPath, fileName),
MeterDataSet = meterDataSet,
TimeZoneOffset = GetTimeZoneOffset(meterDataSet.Meter.TimeZone, dataGroup.StartTime),
DataGroup = dataGroup,
VICycleDataGroup = cycleDataResource.VICycleDataGroups[i],
Faults = faultGroup.Faults,
OutputChannels = dbAdapterContainer.GetAdapter <FaultLocationInfoDataContext>().OutputChannels.Where(channel => seriesIDs.Contains(channel.SeriesID)).ToList()
};
WriteResults(eventDataSet);
}
}
}
public void Initialize(MeterDataSet meterDataSet)
{
DataGroup dataGroup;
VICycleDataGroup viCycleDataGroup;
CycleDataResource cycleDataResource;
ConnectionStringParser.ParseConnectionString(meterDataSet.ConnectionString, this);
m_disturbances = new Dictionary <DataGroup, List <Disturbance> >();
cycleDataResource = meterDataSet.GetResource <CycleDataResource>();
for (int i = 0; i < cycleDataResource.DataGroups.Count; i++)
{
dataGroup = cycleDataResource.DataGroups[i];
viCycleDataGroup = cycleDataResource.VICycleDataGroups[i];
DetectDisturbances(dataGroup, viCycleDataGroup);
}
}
private void ProcessMeterData(MeterDataSet meterDataSet)
{
try
{
meterDataSet.ConnectionString = m_connectionString;
ExecuteDataOperation(meterDataSet);
}
catch (Exception ex)
{
try
{
OnHandleException(ex);
}
catch
{
// Ignore errors here as they are most likely
// related to the error we originally caught
}
}
}
public void Initialize(MeterDataSet meterDataSet)
{
ConnectionStringParser.ParseConnectionString(meterDataSet.ConnectionString, this);
m_disturbances = new Dictionary <DataGroup, List <Disturbance> >();
CycleDataResource cycleDataResource = meterDataSet.GetResource <CycleDataResource>();
int lineCount = meterDataSet.Meter.MeterLines.Count;
for (int i = 0; i < cycleDataResource.DataGroups.Count; i++)
{
DataGroup dataGroup = cycleDataResource.DataGroups[i];
VICycleDataGroup viCycleDataGroup = cycleDataResource.VICycleDataGroups[i];
Range <DateTime> eventDateRange = new Range <DateTime>(dataGroup.StartTime, dataGroup.EndTime);
if (lineCount == 1 && dataGroup.Disturbances.Count > 0)
{
ProcessReportedDisturbances(meterDataSet.Meter, dataGroup);
}
else
{
DetectDisturbances(dataGroup, viCycleDataGroup);
}
}
DataGroupsResource dataGroupsResource = meterDataSet.GetResource <DataGroupsResource>();
foreach (DataGroup dataGroup in dataGroupsResource.DataGroups)
{
if (dataGroup.DataSeries.Count > 0)
{
continue;
}
if (lineCount == 1 && dataGroup.Disturbances.Count > 0)
{
ProcessReportedDisturbances(meterDataSet.Meter, dataGroup);
}
}
}
public void Parse(string filePath)
{
m_pqdifReader.Parse(filePath);
using (AdoDataConnection connection = CreateDbConnection())
{
Func <DateTime, IDbDataParameter> toDateTime2 = dateTime => ToDateTime2(connection, dateTime);
TableOperations <Event> eventTable = new TableOperations <Event>(connection);
string meterKey = GetMeterKey(filePath, m_filePattern);
int meterID = connection.ExecuteScalar <int>("SELECT ID FROM Meter WHERE AssetKey = {0}", meterKey);
Predicate <DataSeries> isDuplicate = dataSeries =>
{
int samples = dataSeries.DataPoints.Count;
if (samples == 0)
{
return(false);
}
DateTime startTime = dataSeries.DataPoints[0].Time;
DateTime endTime = dataSeries.DataPoints[dataSeries.DataPoints.Count - 1].Time;
RecordRestriction recordRestriction =
new RecordRestriction("MeterID = {0}", meterID) &
new RecordRestriction("StartTime = {0}", toDateTime2(startTime)) &
new RecordRestriction("EndTime = {0}", toDateTime2(endTime)) &
new RecordRestriction("Samples = {0}", samples);
int eventCount = eventTable.QueryRecordCount(recordRestriction);
return(eventCount > 0);
};
MeterDataSet meterDataSet = m_pqdifReader.MeterDataSet;
meterDataSet.DataSeries.RemoveAll(isDuplicate);
meterDataSet.Digitals.RemoveAll(isDuplicate);
}
}
private void ProcessMeterData(MeterDataSet meterDataSet, DbAdapterContainer dbAdapterContainer)
{
try
{
meterDataSet.ConnectionString = m_connectionString;
ExecuteDataOperation(meterDataSet, dbAdapterContainer);
ExecuteDataWriters(meterDataSet, dbAdapterContainer);
}
catch (Exception ex)
{
try
{
OnHandleException(ex);
meterDataSet.FileGroup.Error = 1;
dbAdapterContainer.GetAdapter <FileInfoDataContext>().SubmitChanges();
}
catch
{
// Ignore errors here as they are most likely
// related to the error we originally caught
}
}
}
private void ExecuteDataWriters(MeterDataSet meterDataSet, DbAdapterContainer dbAdapterContainer)
{
IDataWriter dataWriter = null;
try
{
// Create the data writer
dataWriter = new openEASSandBoxWriter();
// Provide system settings to the data operation
ConnectionStringParser.ParseConnectionString(meterDataSet.ConnectionString, dataWriter);
// Prepare for execution of the data operation
dataWriter.WriteResults(dbAdapterContainer, meterDataSet);
}
finally
{
// ReSharper disable once SuspiciousTypeConversion.Global
if ((object)dataWriter != null)
{
TryDispose(dataWriter as IDisposable);
}
}
}
public void WriteResults(DbAdapterContainer dbAdapterContainer, MeterDataSet meterDataSet)
{
bool?faultDetectionResult;
bool faultValidationResult;
Initialize(this);
foreach (var faultGroup in dbAdapterContainer.GetAdapter <FaultGroupTableAdapter>().GetDataByFileGroup(meterDataSet.FileGroup.ID))
{
faultDetectionResult = !faultGroup.IsFaultDetectionLogicResultNull()
? Convert.ToBoolean(faultGroup.FaultDetectionLogicResult)
: (bool?)null;
faultValidationResult = Convert.ToBoolean(faultGroup.FaultValidationLogicResult);
if (faultDetectionResult == true || (m_faultLocationSettings.UseDefaultFaultDetectionLogic && faultValidationResult))
{
if (dbAdapterContainer.GetAdapter <EventFaultEmailTableAdapter>().GetFaultEmailCount(faultGroup.EventID) == 0)
{
QueueEventID(faultGroup.EventID);
}
}
}
}
private void ExecuteDataOperation(MeterDataSet meterDataSet)
{
IDataOperation dataOperation = null;
try
{
// Create the data operation
dataOperation = new openEASSandBoxOperation();
// Provide system settings to the data operation
ConnectionStringParser.ParseConnectionString(meterDataSet.ConnectionString, dataOperation);
// Execute the data operation
dataOperation.Execute(meterDataSet);
}
finally
{
// ReSharper disable once SuspiciousTypeConversion.Global
if ((object)dataOperation != null)
{
TryDispose(dataOperation as IDisposable);
}
}
}
public bool Execute(MeterDataSet meterDataSet)
{
if (meterDataSet.Type != DataSetType.AppTrace)
{
return(false);
}
using (AdoDataConnection connection = new AdoDataConnection("systemSettings"))
{
// get metadata for file
FileInfo fi = new FileInfo(meterDataSet.FilePath);
// read lines from file
string[] data = File.ReadAllLines(meterDataSet.FilePath);
// instantiate records object
List <AppTraceRecord> records = new List <AppTraceRecord>();
// retrieve last change for this file
AppTraceFileChanges lastChanges = new TableOperations <AppTraceFileChanges>(connection).QueryRecord("LastWriteTime DESC", new RecordRestriction("MeterID = {0} AND FileName = {1}", meterDataSet.Meter.ID, fi.Name));
// if record doesn't exist, use default
if (lastChanges == null)
{
lastChanges = new AppTraceFileChanges();
}
// parse each line
foreach (string line in data)
{
if (line == string.Empty)
{
continue;
}
string[] section = line.Split(new[] { ". " }, StringSplitOptions.RemoveEmptyEntries);
AppTraceRecord newRecord = new AppTraceRecord();
// date has 2 spaces if date is a single digit to keep specific column width
string regex = @"(\[\d+\/\d+\/\d+\s\d+:\d+:\d+\s[AP]M\])";
string[] results = Regex.Split(line, regex);
string format = "[M/d/yyyy h:mm:ss tt]";
if (results.Length <= 1)
{
regex = @"(\[\d+\/\d+\/\d+\])";
results = Regex.Split(line, regex);
format = "[M/d/yyyy]";
}
newRecord.Line = line;
newRecord.Time = DateTime.ParseExact(results[1], format, CultureInfo.InvariantCulture);
if (newRecord.Time > TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time")))
{
newRecord.Time = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time"));
newRecord.Line += ". MiMD Parsing Alarm: DFR time set in the future.\n";
}
newRecord.Description = results[2];
records.Add(newRecord);
}
// if no records, or if the last record is same or before record in database, stop. There was probably an error.
if (!records.Any() || records.Last().Time <= lastChanges.LastWriteTime)
{
return(false);
}
// instantiate new changes object
AppTraceFileChanges fileChanges = new AppTraceFileChanges();
List <string> alarmKeyWords = new List <string> {
"unsync<invalid(no signal)>", "[alarmon]", "sync loss", "chassis comm. error", "disk full", "master comm. error", "dsp board temperature", "analog fail", "pc health", "offline", "time in future"
};
IEnumerable <AppTraceRecord> newRecords = records.Where(x => x.Time > lastChanges.LastWriteTime);
// create new record
fileChanges.MeterID = meterDataSet.Meter.ID;
fileChanges.FileName = fi.Name;
fileChanges.FileSize = (int)(fi.Length / 1000);
fileChanges.LastWriteTime = records.Last().Time;
fileChanges.Span = (records.Last().Time - records.First().Time).Days;
fileChanges.NewRecords = newRecords.Count();
fileChanges.Alarms = newRecords.Where(x => alarmKeyWords.Contains(x.Line.ToLower())).Count();
if (records.Where(x => x.Line.ToLower().Contains("fault f")).Any())
{
fileChanges.LastFaultTime = records.Where(x => x.Line.ToLower().Contains("fault f")).Last().Time;
}
else
{
fileChanges.LastFaultTime = lastChanges.LastFaultTime;
}
fileChanges.FaultCount48hr = records.Where(x => x.Line.ToLower().Contains("fault f") && x.Time >= fileChanges.LastWriteTime.AddHours(-48)).Count();
// get html of new changes
DiffMatchPatch dmp = new DiffMatchPatch();
List <Diff> diff = dmp.DiffMain("", string.Join("\n", records.Where(x => x.Time > lastChanges.LastWriteTime).Select(x => x.Line)));
dmp.DiffCleanupSemantic(diff);
fileChanges.Html = dmp.DiffPrettyHtml(diff).Replace("¶", "");
// write new record to db
meterDataSet.DiagnosticAlarms = fileChanges.Alarms;
new TableOperations <AppTraceFileChanges>(connection).AddNewRecord(fileChanges);
return(true);
}
}
/// <summary>
/// Creates a new instance of the <see cref="PQubeTrendingDataCSVReader"/> class.
/// </summary>
public PQubeTrendingDataCSVReader()
{
m_meterDataSet = new MeterDataSet();
Channels = new Dictionary <string, int>();
}
/// <summary>
/// Creates a new instance of the <see cref="COMTRADEReader"/> class.
/// </summary>
public COMTRADEReader()
{
Settings = new COMTRADESettings();
MeterDataSet = new MeterDataSet();
}
/// <summary>
/// Creates a new instance of the <see cref="COMTRADEReader"/> class.
/// </summary>
public COMTRADEReader()
{
m_meterDataSet = new MeterDataSet();
}
private void WriteResults(MeterDataSet meterDataSet, DataGroup dataGroup, VICycleDataGroup viCycleDataGroup, List<Fault> faults, string resultsFilePath)
{
XDocument resultsDocument;
XElement results;
string lineName;
lineName = meterDataSet.Meter.MeterLines
.Where(ml => ml.LineID == dataGroup.Line.ID)
.Select(ml => ml.LineName)
.FirstOrDefault() ?? dataGroup.Line.AssetKey;
results =
new XElement("results",
new XElement("meter", meterDataSet.Meter.Name),
new XElement("disturbanceFiles", GetPathElements(meterDataSet.FileGroup)),
new XElement("line",
new XElement("name", lineName),
new XElement("length", dataGroup.Line.Length.ToString(DoubleFormat))
),
new XElement("prefault",
new XElement("time", dataGroup.StartTime.ToString(DateTimeFormat)),
GetCycleElements(viCycleDataGroup, 0)
),
GetFaultElements(faults)
);
// Create the XML document
resultsDocument =
new XDocument(
new XElement("openFLE", results)
);
resultsDocument.Save(resultsFilePath);
}
// Determines the start time and end time of the given data and sets the properties on the given file group.
private static void SetDataTimeRange(MeterDataSet meterDataSet, FileInfoDataContext fileInfo)
{
DateTime dataStartTime;
DateTime dataEndTime;
dataStartTime = meterDataSet.DataSeries
.Concat(meterDataSet.Digitals)
.Where(dataSeries => dataSeries.DataPoints.Any())
.Select(dataSeries => dataSeries.DataPoints.First().Time)
.DefaultIfEmpty()
.Min();
dataEndTime = meterDataSet.DataSeries
.Concat(meterDataSet.Digitals)
.Where(dataSeries => dataSeries.DataPoints.Any())
.Select(dataSeries => dataSeries.DataPoints.Last().Time)
.DefaultIfEmpty()
.Max();
if (dataStartTime != default(DateTime))
meterDataSet.FileGroup.DataStartTime = dataStartTime;
if (dataEndTime != default(DateTime))
meterDataSet.FileGroup.DataEndTime = dataEndTime;
fileInfo.SubmitChanges();
}
public bool Execute(MeterDataSet meterDataSet)
{
if (meterDataSet.Type != DataSetType.BENConfig)
{
return(false);
}
using (AdoDataConnection connection = new AdoDataConnection("systemSettings"))
{
FileInfo fi = new FileInfo(meterDataSet.FilePath);
DateTime lastWriteTime = DateTime.ParseExact(string.Join(",", fi.Name.Split(',').Take(2)), "yyMMdd,HHmmssfff", CultureInfo.InvariantCulture);
// see if there is already a record that matches this file
ConfigFileChanges configFileChanges = new TableOperations <ConfigFileChanges>(connection).QueryRecordWhere("MeterID = {0} AND FileName = {1} AND LastWriteTime = {2}", meterDataSet.Meter.ID, $"{meterDataSet.Meter.AssetKey}.cfg", lastWriteTime);
// if a record already exists for this file skip it. There was probably an error.
if (configFileChanges != null)
{
return(false);
}
configFileChanges = new ConfigFileChanges();
// create new record
configFileChanges.MeterID = meterDataSet.Meter.ID;
configFileChanges.FileName = $"{meterDataSet.Meter.AssetKey}.cfg";
configFileChanges.LastWriteTime = lastWriteTime;
configFileChanges.Text = meterDataSet.Text.Replace("\r", "");
string[] data = File.ReadAllLines(meterDataSet.FilePath);
int[] channelCounts = data[1].Split(',').Select(x => int.Parse(x.Replace("A", "").Replace("D", ""))).ToArray();
int totalChannels = channelCounts[0];
// get portion of cfg file that contains channel mappings
string relevantPortion = string.Join("\n", data.Take(2 + totalChannels));
// get the previous record for this file
ConfigFileChanges lastChanges = new TableOperations <ConfigFileChanges>(connection).QueryRecord("LastWriteTime DESC", new RecordRestriction("MeterID = {0} AND FileName = {1} AND LastWriteTime < {2}", meterDataSet.Meter.ID, $"{meterDataSet.Meter.AssetKey}.cfg", lastWriteTime));
// if there were no previous records for this file
if (lastChanges == null)
{
lastChanges = new ConfigFileChanges();
lastChanges.Text = configFileChanges.Text.Replace("\r", "");
// make diffs
DiffMatchPatch dmp = new DiffMatchPatch();
List <Diff> diff = dmp.DiffMain(relevantPortion, relevantPortion);
dmp.DiffCleanupSemantic(diff);
configFileChanges.Html = dmp.DiffPrettyHtml(diff).Replace("¶", "");
configFileChanges.Changes = 0;
}
else
{
string[] data2 = lastChanges.Text.Split('\n');
int[] channelCounts2 = data2[1].Split(',').Select(x => int.Parse(x.Replace("A", "").Replace("D", ""))).ToArray();
int totalChannels2 = channelCounts2[0];
// get portion of cfg file that contains channel mappings
string relevantPortion2 = string.Join("\n", data2.Take(2 + totalChannels2));
// make diffs
DiffMatchPatch dmp = new DiffMatchPatch();
List <Diff> diff = dmp.DiffMain(relevantPortion2, relevantPortion);
List <Patch> patch = dmp.PatchMake(relevantPortion2, relevantPortion);
dmp.DiffCleanupSemantic(diff);
configFileChanges.Html = dmp.DiffPrettyHtml(diff).Replace("¶", "");
configFileChanges.Changes = patch.Count;
if (patch.Count == 0)
{
return(false);
}
}
// write new record to db
meterDataSet.ConfigChanges = configFileChanges.Changes;
new TableOperations <ConfigFileChanges>(connection).AddNewRecord(configFileChanges);
return(true);
}
}
public void WriteResults(DbAdapterContainer dbAdapterContainer, MeterDataSet meterDataSet)
{
// Write results to an external data store
Log.InfoFormat("Results written to external data store.");
}
//Start this by Only looking at Config Files and Lines that have =
public bool Execute(MeterDataSet meterDataSet)
{
if (meterDataSet.Type != DataSetType.Config)
{
return(false);
}
if (!HasComplianceMeter(meterDataSet))
{
return(false);
}
using (AdoDataConnection connection = new AdoDataConnection("systemSettings"))
{
ComplianceMeter meter = new TableOperations <ComplianceMeter>(connection).QueryRecordWhere("MeterId = {0}", meterDataSet.Meter.ID);
//This means we need to activate the meter
if (!meter.Active)
{
meter.Active = true;
new TableOperations <ComplianceMeter>(connection).UpdateRecord(meter);
}
// Get appropriate Base Configurations
IEnumerable <BaseConfig> baseConfigsMeter = new TableOperations <BaseConfig>(connection).QueryRecordsWhere("MeterId = {0}", meter.ID);
//Filter by Pattern
baseConfigsMeter = baseConfigsMeter.Where(item => FilePath.IsFilePatternMatch(item.Pattern, meterDataSet.FilePath, true));
if (baseConfigsMeter.Count() == 0)
{
return(true);
}
// Pares File using Compliance Operations
// Instantiates the given data reader and wraps it in a disposable wrapper object.
Model.ComplianceOperation parser = new TableOperations <Model.ComplianceOperation>(connection).QueryRecords("LoadOrder")
.FirstOrDefault(reader => FilePath.IsFilePatternMatch(reader.FilePattern, meterDataSet.FilePath, true));
ComplianceParserWrapper wrapper;
try
{
Assembly assembly = Assembly.LoadFrom(parser.AssemblyName);
Type type = assembly.GetType(parser.TypeName);
wrapper = new ComplianceParserWrapper(parser.ID, type);
}
catch (Exception ex)
{
string message = $"Failed to create Compliance File parser of type {parser.TypeName}: {ex.Message}";
throw new TypeLoadException(message, ex);
}
// Parse ConfigFile into Dictionary of Fields
Dictionary <string, string> activeConfig = new Dictionary <string, string>();
try
{
activeConfig = wrapper.DataObject.ParseFields(meterDataSet);
}
catch (Exception ex)
{
string message = $"Failed to parse Config File for Compliance Fields {parser.TypeName}: {ex.Message}";
throw new TypeLoadException(message, ex);
}
// If Meter is not Reviewed just keep updating Base Config
if (!meter.Reviewed)
{
foreach (BaseConfig baseConfig in baseConfigsMeter)
{
UpdateBaseConfig(baseConfig, activeConfig);
}
return(true);
}
// Get a Bunch of AlarmStates
TableOperations <ComplianceState> complianceStateTbl = new TableOperations <ComplianceState>(connection);
ComplianceState resolved = complianceStateTbl.QueryRecordWhere("Description = 'In Compliance'");
ComplianceState noCompliance = complianceStateTbl.QueryRecordWhere("Description = 'Compliance Issue'");
ComplianceState acknowledged = complianceStateTbl.QueryRecordWhere("Description = 'Acknowledged'");
ComplianceState reviewed = complianceStateTbl.QueryRecordWhere("Description = 'Reviewed'");
TableOperations <ComplianceRecord> complianceRecordTbl = new TableOperations <ComplianceRecord>(connection);
TableOperations <ComplianceRecordView> complianceRecordViewTbl = new TableOperations <ComplianceRecordView>(connection);
TableOperations <ComplianceRecordField> recordFieldTbl = new TableOperations <ComplianceRecordField>(connection);
TableOperations <ComplianceField> complianceFieldTbl = new TableOperations <ComplianceField>(connection);
TableOperations <ComplianceAction> complianceActionTbl = new TableOperations <ComplianceAction>(connection);
TableOperations <ComplianceFieldValue> complianceFieldValueTbl = new TableOperations <ComplianceFieldValue>(connection);
TableOperations <ComplianceFieldValueView> complianceFieldValueViewTbl = new TableOperations <ComplianceFieldValueView>(connection);
//Walk through each BaseConfig separately
foreach (BaseConfig baseConfig in baseConfigsMeter)
{
// Get relevant Compliance Fields
List <ComplianceField> baseConfigfields = complianceFieldTbl.QueryRecordsWhere("BaseConfigId = {0}", baseConfig.ID).ToList();
if (baseConfigfields.Count() == 0)
{
continue;
}
List <ComplianceField> changingFields = baseConfigfields.Where(fld => {
ComplianceFieldValueView value = complianceFieldValueViewTbl.QueryRecordWhere("FieldId = {0}", fld.ID);
if (value == null)
{
return(!fld.Evaluate(activeConfig[fld.Name]));
}
return(activeConfig[fld.Name] != value.Value);
}).ToList();
if (changingFields.Count == 0)
{
continue;
}
//Clear so that -1 if Record is resolved
IEnumerable <IGrouping <int, ComplianceField> > recordGroups = changingFields.GroupBy(fld =>
{
ComplianceRecordView record = complianceRecordViewTbl.QueryRecordWhere("BaseConfigId = {0} AND ID IN (SELECT RecordID FROM ComplianceRecordFields WHERE FieldId = {1}) AND Status IN ({2},{3}, {4})", baseConfig.ID, fld.ID, acknowledged.ID, noCompliance.ID, reviewed.ID);
if (record == null)
{
return(-1);
}
return(record.ID);
}, fld => fld);
//Count those that exist and change Status
meterDataSet.ComplianceIssues += recordGroups.Count();
if (recordGroups.Any(g => g.Key == -1))
{
meterDataSet.ComplianceIssues--;
}
// Some Records can not be updated with failing Fields they need to be separate
List <ComplianceField> newFields = new List <ComplianceField>();
// Work Backwards to get associated Record for a Field
foreach (IGrouping <int, ComplianceField> group in recordGroups)
{
//we skip those that don't have a record
if (group.Key > -1)
{
// grab status
ComplianceRecordView view = complianceRecordViewTbl.QueryRecordWhere("Id = {0}", group.Key);
//Check if we can resolve everything including some fields that are previously resolved.
bool canResolve = complianceFieldValueViewTbl.QueryRecordsWhere("RecordID = {0}", group.Key).All(fld => {
string value;
if (!activeConfig.TryGetValue(fld.FieldName, out value))
{
value = fld.Value;
}
return(baseConfigfields.Find(field => field.ID == fld.FieldId).Evaluate(value));
});
if (canResolve)
{
//Create Action
complianceActionTbl.AddNewRecord(new ComplianceAction()
{
Note = "New File Downloaded",
UserAccount = "MiMD",
RecordId = group.Key,
StateId = resolved.ID,
Timestamp = DateTime.UtcNow
});
int resolvedId = connection.ExecuteScalar <int>("SELECT @@identity");
group.ToList().ForEach(item => complianceFieldValueTbl.AddNewRecord(new ComplianceFieldValue()
{
ActionId = resolvedId,
FieldId = item.ID,
Value = activeConfig[item.Name]
}));
}
else if (view.Status == noCompliance.ID)
{
//Create Action
complianceActionTbl.AddNewRecord(new ComplianceAction()
{
Note = "New File Downloaded",
UserAccount = "MiMD",
RecordId = group.Key,
StateId = null,
Timestamp = DateTime.UtcNow
});
int resolvedId = connection.ExecuteScalar <int>("SELECT @@identity");
group.ToList().ForEach(item => complianceFieldValueTbl.AddNewRecord(new ComplianceFieldValue()
{
ActionId = resolvedId,
FieldId = item.ID,
Value = activeConfig[item.Name]
}));
}
else
{
List <ComplianceField> nonresolving = group.ToList().Where(fld => {
string old = complianceFieldValueViewTbl.QueryRecordWhere("FieldId = {0}", fld.ID).Value;
return(!fld.Evaluate(activeConfig[fld.Name]) && fld.Evaluate(old));
}).ToList();
List <ComplianceField> resolving = group.ToList().Where(fld => {
string old = complianceFieldValueViewTbl.QueryRecordWhere("FieldId = {0}", fld.ID).Value;
return(!fld.Evaluate(old));
}).ToList();
if (resolving.Count > 0)
{
complianceActionTbl.AddNewRecord(new ComplianceAction()
{
Note = "New File Downloaded",
UserAccount = "MiMD",
RecordId = group.Key,
StateId = null,
Timestamp = DateTime.UtcNow
});
int resolvedId = connection.ExecuteScalar <int>("SELECT @@identity");
resolving.ForEach(item => complianceFieldValueTbl.AddNewRecord(new ComplianceFieldValue()
{
ActionId = resolvedId,
FieldId = item.ID,
Value = activeConfig[item.Name]
}));
}
newFields = newFields.Concat(nonresolving).ToList();
}
}
else
{
newFields = newFields.Concat(group.ToList()).ToList();
}
}
if (newFields.Count == 0)
{
continue;
}
// Add the new Compliance Issue to count if it is opened
meterDataSet.ComplianceIssues++;
complianceRecordTbl.AddNewRecord(new ComplianceRecord()
{
BaseConfigId = baseConfig.ID,
MeterId = meter.ID,
TimerOffset = 0
});
int recordId = connection.ExecuteScalar <int>("SELECT @@identity");
//Create Action
complianceActionTbl.AddNewRecord(new ComplianceAction()
{
Note = "New File Downloaded",
UserAccount = "MiMD",
RecordId = recordId,
StateId = noCompliance.ID,
Timestamp = DateTime.UtcNow
});
int actionId = connection.ExecuteScalar <int>("SELECT @@identity");
newFields.ForEach(item => recordFieldTbl.AddNewRecord(new ComplianceRecordField()
{
FieldId = item.ID,
RecordId = recordId
}));
newFields.ForEach(item => complianceFieldValueTbl.AddNewRecord(new ComplianceFieldValue()
{
ActionId = actionId,
FieldId = item.ID,
Value = activeConfig[item.Name]
}));
}
return(true);
}
}
private void ExecuteDataWriters(MeterDataSet meterDataSet, DbAdapterContainer dbAdapterContainer)
{
IDataWriter dataWriter = null;
try
{
// Create the data writer
dataWriter = new openEASSandBoxWriter();
// Provide system settings to the data operation
ConnectionStringParser.ParseConnectionString(meterDataSet.ConnectionString, dataWriter);
// Prepare for execution of the data operation
dataWriter.WriteResults(dbAdapterContainer, meterDataSet);
}
finally
{
// ReSharper disable once SuspiciousTypeConversion.Global
if ((object)dataWriter != null)
TryDispose(dataWriter as IDisposable);
}
}
public void Initialize(MeterDataSet meterDataSet, DbAdapterContainer dbAdapterContainer)
{
DataGroup dataGroup;
VICycleDataGroup viCycleDataGroup;
CycleDataResource cycleDataResource;
ConnectionStringParser.ParseConnectionString(meterDataSet.ConnectionString, this);
m_disturbances = new Dictionary<DataGroup, List<Disturbance>>();
cycleDataResource = CycleDataResource.GetResource(meterDataSet, dbAdapterContainer);
for (int i = 0; i < cycleDataResource.DataGroups.Count; i++)
{
dataGroup = cycleDataResource.DataGroups[i];
viCycleDataGroup = cycleDataResource.VICycleDataGroups[i];
DetectDisturbances(dataGroup, viCycleDataGroup);
}
}
private List<MeterDataSet> LoadMeterDataSets(DbAdapterContainer dbAdapterContainer, FileGroup fileGroup)
{
List<MeterDataSet> meterDataSets = new List<MeterDataSet>();
MeterInfoDataContext meterInfo = dbAdapterContainer.GetAdapter<MeterInfoDataContext>();
EventTableAdapter eventAdapter = dbAdapterContainer.GetAdapter<EventTableAdapter>();
EventDataTableAdapter eventDataAdapter = dbAdapterContainer.GetAdapter<EventDataTableAdapter>();
MeterData.EventDataTable eventTable = eventAdapter.GetDataByFileGroup(fileGroup.ID);
MeterDataSet meterDataSet;
DataGroup dataGroup;
foreach (IGrouping<int, MeterData.EventRow> eventGroup in eventTable.GroupBy(evt => evt.MeterID))
{
meterDataSet = new MeterDataSet();
meterDataSet.Meter = meterInfo.Meters.SingleOrDefault(meter => meter.ID == eventGroup.Key);
foreach (MeterData.EventRow evt in eventGroup)
{
dataGroup = new DataGroup();
dataGroup.FromData(meterDataSet.Meter, eventDataAdapter.GetTimeDomainData(evt.EventDataID));
foreach (DataSeries dataSeries in dataGroup.DataSeries)
meterDataSet.DataSeries.Add(dataSeries);
}
meterDataSets.Add(meterDataSet);
}
return meterDataSets;
}
public void WriteResults(DbAdapterContainer dbAdapterContainer, MeterDataSet meterDataSet)
{
CycleDataResource cycleDataResource;
FaultDataResource faultDataResource;
DataGroup dataGroup;
FaultGroup faultGroup;
List<int> seriesIDs;
EventDataSet eventDataSet;
string rootFileName;
string fileName;
cycleDataResource = meterDataSet.GetResource(() => CycleDataResource.GetResource(meterDataSet, dbAdapterContainer));
faultDataResource = meterDataSet.GetResource(() => new FaultDataResource(dbAdapterContainer));
if (!Directory.Exists(m_resultsPath))
Directory.CreateDirectory(m_resultsPath);
for (int i = 0; i < cycleDataResource.DataGroups.Count; i++)
{
dataGroup = cycleDataResource.DataGroups[i];
if (faultDataResource.FaultLookup.TryGetValue(dataGroup, out faultGroup))
{
rootFileName = FilePath.GetFileNameWithoutExtension(meterDataSet.FilePath);
fileName = string.Format("{0},{1:000},Line{2}.dat", rootFileName, i, dataGroup.Line.AssetKey);
seriesIDs = dataGroup.DataSeries
.Select(series => series.SeriesInfo.ID)
.ToList();
eventDataSet = new EventDataSet()
{
ResultsPath = Path.Combine(m_resultsPath, fileName),
MeterDataSet = meterDataSet,
TimeZoneOffset = GetTimeZoneOffset(meterDataSet.Meter.TimeZone, dataGroup.StartTime),
DataGroup = dataGroup,
VICycleDataGroup = cycleDataResource.VICycleDataGroups[i],
Faults = faultGroup.Faults,
OutputChannels = dbAdapterContainer.GetAdapter<FaultLocationInfoDataContext>().OutputChannels.Where(channel => seriesIDs.Contains(channel.SeriesID)).ToList()
};
WriteResults(eventDataSet);
}
}
}
private void ProcessMeterData(MeterDataSet meterDataSet, DbAdapterContainer dbAdapterContainer)
{
try
{
meterDataSet.ConnectionString = m_connectionString;
ExecuteDataOperation(meterDataSet, dbAdapterContainer);
ExecuteDataWriters(meterDataSet, dbAdapterContainer);
}
catch (Exception ex)
{
try
{
OnHandleException(ex);
meterDataSet.FileGroup.Error = 1;
dbAdapterContainer.GetAdapter<FileInfoDataContext>().SubmitChanges();
}
catch
{
// Ignore errors here as they are most likely
// related to the error we originally caught
}
}
}
/// <summary>
/// Creates a new instance of the <see cref="EMAXReader"/> class.
/// </summary>
public EMAXReader()
{
m_emaxSettings = new EMAXSettings();
m_meterDataSet = new MeterDataSet();
}
/// <summary>
/// Creates a new instance of the <see cref="IGridPQDIFReader"/> class.
/// </summary>
public IGridPQDIFReader()
{
m_meterDataSet = new MeterDataSet();
}
public void WriteResults(DbAdapterContainer dbAdapterContainer, MeterDataSet meterDataSet)
{
Initialize(this);
foreach (EventRow evt in dbAdapterContainer.GetAdapter<EventTableAdapter>().GetDataByFileGroup(meterDataSet.FileGroup.ID))
{
if (GetEmailCount(dbAdapterContainer, evt.ID) == 0)
QueueEventID(evt.ID);
}
}
// Instantiates and executes data operations and data writers to process the meter data set.
private void ProcessMeterDataSet(MeterDataSet meterDataSet, SystemSettings systemSettings, DbAdapterContainer dbAdapterContainer)
{
SystemInfoDataContext systemInfo;
List<DataOperationWrapper> dataOperations = null;
List<DataWriterWrapper> dataWriters = null;
// Get the SystemInfoDataContext from the dbAdapterContainer
systemInfo = dbAdapterContainer.GetAdapter<SystemInfoDataContext>();
// Load the data operations from the database,
// in descending order so we can remove records while we iterate
dataOperations = systemInfo.DataOperations
.OrderByDescending(dataOperation => dataOperation.LoadOrder)
.Select(Wrap)
.Where(wrapper => (object)wrapper != null)
.ToList();
for (int i = dataOperations.Count - 1; i >= 0; i--)
{
try
{
Log.Debug($"Preparing data operation '{dataOperations[i].DataObject.GetType().Name}' for execution...");
// Load configuration parameters from the connection string into the data operation
ConnectionStringParser.ParseConnectionString(meterDataSet.ConnectionString, dataOperations[i].DataObject);
// Call the prepare method to allow the data operation to prepare any data it needs from the database
dataOperations[i].DataObject.Prepare(dbAdapterContainer);
Log.Debug($"Finished preparing data operation '{dataOperations[i].DataObject.GetType().Name}' for execution.");
}
catch (Exception ex)
{
// Log the error and remove the data operation from the list
string message = $"An error occurred while preparing data from meter '{meterDataSet.Meter.AssetKey}' for data operation of type '{dataOperations[i].DataObject.GetType().FullName}': {ex.Message}";
OnProcessException(new Exception(message, ex));
dataOperations[i].Dispose();
dataOperations.RemoveAt(i);
}
}
for (int i = dataOperations.Count - 1; i >= 0; i--)
{
try
{
Log.Debug($"Executing data operation '{dataOperations[i].DataObject.GetType().Name}'...");
// Call the execute method on the data operation to perform in-memory data transformations
dataOperations[i].DataObject.Execute(meterDataSet);
Log.Debug($"Finished execurting data operation '{dataOperations[i].DataObject.GetType().Name}'.");
}
catch (Exception ex)
{
// Log the error and skip to the next data operation
string message = $"An error occurred while executing data operation of type '{dataOperations[i].DataObject.GetType().FullName}' on data from meter '{meterDataSet.Meter.AssetKey}': {ex.Message}";
OnProcessException(new Exception(message, ex));
continue;
}
try
{
Log.Debug($"Loading data from data operation '{dataOperations[i].DataObject.GetType().Name}' into database...");
// Call the load method inside a transaction to load data into from the data operation into the database
using (TransactionScope transaction = new TransactionScope(TransactionScopeOption.Required, GetTransactionOptions()))
{
dataOperations[i].DataObject.Load(dbAdapterContainer);
transaction.Complete();
}
Log.Debug($"Finished loading data from data operation '{dataOperations[i].DataObject.GetType().Name}' into database.");
}
catch (Exception ex)
{
// Log the error and move on to the next data operation
string message = $"An error occurred while loading data from data operation of type '{dataOperations[i].DataObject.GetType().FullName}' for data from meter '{meterDataSet.Meter.AssetKey}': {ex.Message}";
OnProcessException(new Exception(message, ex));
}
}
// All data operations are complete, but we still need to clean up
for (int i = dataOperations.Count - 1; i >= 0; i--)
dataOperations[i].Dispose();
// Load the data writers from the database
dataWriters = systemInfo.DataWriters
.OrderBy(dataWriter => dataWriter.LoadOrder)
.Select(Wrap)
.Where(wrapper => (object)wrapper != null)
.ToList();
foreach (DataWriterWrapper dataWriter in dataWriters)
{
try
{
Log.Debug($"Writing results to external location with data writer '{dataWriter.DataObject.GetType().Name}'...");
// Load configuration parameters from the connection string into the data writer
ConnectionStringParser.ParseConnectionString(meterDataSet.ConnectionString, dataWriter.DataObject);
// Write the results to the data writer's destination by calling the WriteResults method
dataWriter.DataObject.WriteResults(dbAdapterContainer, meterDataSet);
Log.Debug($"Finished writing results with data writer '{dataWriter.DataObject.GetType().Name}'.");
}
catch (Exception ex)
{
// Log the error and move on to the next data writer
string message = $"An error occurred while writing data from meter '{meterDataSet.Meter.AssetKey}' using data writer of type '{dataWriter.DataObject.GetType().FullName}': {ex.Message}";
OnProcessException(new Exception(message, ex));
}
}
// All data writers are complete, but we still need to clean up
foreach (DataWriterWrapper dataWriter in dataWriters)
dataWriter.Dispose();
}
private void ExecuteDataOperation(MeterDataSet meterDataSet, DbAdapterContainer dbAdapterContainer)
{
IDataOperation dataOperation = null;
try
{
// Create the data operation
dataOperation = new openEASSandBoxOperation();
// Provide system settings to the data operation
ConnectionStringParser.ParseConnectionString(meterDataSet.ConnectionString, dataOperation);
// Prepare for execution of the data operation
dataOperation.Prepare(dbAdapterContainer);
// Execute the data operation
dataOperation.Execute(meterDataSet);
// Load data from all data operations in a single transaction
using (TransactionScope transactionScope = new TransactionScope(TransactionScopeOption.Required, GetTransactionOptions()))
{
dataOperation.Load(dbAdapterContainer);
transactionScope.Complete();
}
}
finally
{
// ReSharper disable once SuspiciousTypeConversion.Global
if ((object)dataOperation != null)
TryDispose(dataOperation as IDisposable);
}
}
// Adjusts the timestamps in the given data sets to the time zone of XDA.
private static void ShiftTime(MeterDataSet meterDataSet, TimeZoneInfo meterTimeZone, TimeZoneInfo xdaTimeZone)
{
foreach (DataSeries dataSeries in meterDataSet.DataSeries)
{
foreach (DataPoint dataPoint in dataSeries.DataPoints)
{
if (dataPoint.Time.Kind != DateTimeKind.Unspecified)
dataPoint.Time = TimeZoneInfo.ConvertTimeToUtc(dataPoint.Time);
else
dataPoint.Time = TimeZoneInfo.ConvertTimeToUtc(dataPoint.Time, meterTimeZone);
dataPoint.Time = TimeZoneInfo.ConvertTimeFromUtc(dataPoint.Time, xdaTimeZone);
}
}
foreach (DataSeries dataSeries in meterDataSet.Digitals)
{
foreach (DataPoint dataPoint in dataSeries.DataPoints)
{
if (dataPoint.Time.Kind != DateTimeKind.Unspecified)
dataPoint.Time = TimeZoneInfo.ConvertTimeToUtc(dataPoint.Time);
else
dataPoint.Time = TimeZoneInfo.ConvertTimeToUtc(dataPoint.Time, meterTimeZone);
dataPoint.Time = TimeZoneInfo.ConvertTimeFromUtc(dataPoint.Time, xdaTimeZone);
}
}
}
/// <summary>
/// Creates a new instance of the <see cref="EMAXReader"/> class.
/// </summary>
public EMAXReader()
{
m_meterDataSet = new MeterDataSet();
}
public void WriteResults(DbAdapterContainer dbAdapterContainer, MeterDataSet meterDataSet)
{
CycleDataResource cycleDataResource;
FaultDataResource faultDataResource;
DataGroup dataGroup;
VICycleDataGroup viCycleDataGroup;
FaultGroup faultGroup;
string rootFileName;
string fileName;
cycleDataResource = meterDataSet.GetResource(() => CycleDataResource.GetResource(meterDataSet, dbAdapterContainer));
faultDataResource = meterDataSet.GetResource(() => new FaultDataResource(dbAdapterContainer));
if (!Directory.Exists(m_resultsPath))
Directory.CreateDirectory(m_resultsPath);
for (int i = 0; i < cycleDataResource.DataGroups.Count; i++)
{
dataGroup = cycleDataResource.DataGroups[i];
if (faultDataResource.FaultLookup.TryGetValue(dataGroup, out faultGroup))
{
rootFileName = FilePath.GetFileNameWithoutExtension(meterDataSet.FilePath);
fileName = string.Format("{0},{1:000},Line{2}.xml", rootFileName, i, dataGroup.Line.AssetKey);
viCycleDataGroup = cycleDataResource.VICycleDataGroups[i];
WriteResults(meterDataSet, dataGroup, viCycleDataGroup, faultGroup.Faults, Path.Combine(m_resultsPath, fileName));
}
}
}
public bool Execute(MeterDataSet meterDataSet)
{
if (meterDataSet.Type != DataSetType.EmaxEventHis)
{
return(false);
}
using (AdoDataConnection connection = new AdoDataConnection("systemSettings"))
{
// get metadata for file
FileInfo fi = new FileInfo(meterDataSet.FilePath);
// read lines from file
string[] data = File.ReadAllLines(meterDataSet.FilePath);
// instantiate records object
List <EMAXEventHisRecord> records = new List <EMAXEventHisRecord>();
// retrieve last change for this file
EmaxDiagnosticFileChanges lastChanges = new TableOperations <EmaxDiagnosticFileChanges>(connection).QueryRecord("LastWriteTime DESC", new RecordRestriction("MeterID = {0} AND FileName = {1}", meterDataSet.Meter.ID, fi.Name));
// if record doesn't exist, use default
if (lastChanges == null)
{
lastChanges = new EmaxDiagnosticFileChanges();
}
// parse each line
foreach (string line in data)
{
if (line == string.Empty)
{
continue;
}
string[] section = line.Split(new[] { ". " }, StringSplitOptions.RemoveEmptyEntries);
EMAXEventHisRecord newRecord = new EMAXEventHisRecord();
// date has 2 spaces if date is a single digit to keep specific column width
string format = "ddd MMM d HH:mm:ss yyyy";
if (section[0].Contains(" "))
{
format = "ddd MMM d HH:mm:ss yyyy";
}
newRecord.Line = line;
newRecord.Time = DateTime.ParseExact(section[0], format, CultureInfo.InvariantCulture);
if (newRecord.Time > TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time")))
{
newRecord.Time = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time"));
newRecord.Line += ". MiMD Parsing Alarm: DFR time set in the future.\n";
}
newRecord.Version = section[1];
records.Add(newRecord);
}
// if no records, or if the last record is same or before record in database, stop. There was probably an error.
if (!records.Any() || records.Last().Time <= lastChanges.LastWriteTime)
{
return(false);
}
List <EMAXEventHisRecord> newRecords = records.Where(x => x.Time > lastChanges.LastWriteTime).ToList();
// instantiate new changes object
EmaxDiagnosticFileChanges fileChanges = new EmaxDiagnosticFileChanges();
// create new record
fileChanges.MeterID = meterDataSet.Meter.ID;
fileChanges.FileName = fi.Name;
fileChanges.FileSize = (int)(fi.Length / 1000);
fileChanges.LastWriteTime = newRecords.Last().Time;
fileChanges.Span = (newRecords.Last().Time - newRecords.First().Time).Days;
fileChanges.NewRecords = newRecords.Count();
fileChanges.Alarms = 0;
for (int i = 0; i < newRecords.Count; ++i)
{
if (newRecords[i].Line.ToLower().Contains("operation alarm"))
{
}
else if (newRecords[i].Line.ToLower().Contains("offline. system offline"))
{
fileChanges.Alarms++;
}
else if (newRecords[i].Line.ToLower().Contains("buffer full"))
{
fileChanges.Alarms++;
}
else if (newRecords[i].Line.ToLower().Contains("error"))
{
fileChanges.Alarms++;
}
else if (newRecords[i].Line.ToLower().Contains("alarm"))
{
if (newRecords[i].Line.ToLower().Contains("time sync failed"))
{
if (newRecords.Where(x => x.Line.ToLower().Contains("system started") && newRecords[i].Time.Subtract(x.Time).TotalSeconds <= 60).Any())
{
}
else
{
fileChanges.Alarms++;
}
}
else
{
fileChanges.Alarms++;
}
}
}
// get html of new changes
DiffMatchPatch dmp = new DiffMatchPatch();
List <Diff> diff = dmp.DiffMain("", string.Join("\n", records.Where(x => x.Time > lastChanges.LastWriteTime).Select(x => x.Line)));
dmp.DiffCleanupSemantic(diff);
fileChanges.Html = dmp.DiffPrettyHtml(diff).Replace("¶", "");
// write new record to db
meterDataSet.DiagnosticAlarms = fileChanges.Alarms;
new TableOperations <EmaxDiagnosticFileChanges>(connection).AddNewRecord(fileChanges);
return(true);
}
}
/// <summary>
/// Creates a new instance of the <see cref="SELEVEReader"/> class.
/// </summary>
public SELEVEReader()
{
m_meterDataSet = new MeterDataSet();
}
public void Parse(string filePath)
{
Schema schema = Parser.Schema;
Meter meter = new Meter();
meter.MeterLocation = new MeterLocation();
meter.Channels = new List <Channel>();
meter.AssetKey = schema.DeviceID;
meter.Name = schema.DeviceID;
meter.ShortName = schema.DeviceID.Substring(0, Math.Min(schema.DeviceID.Length, 50));
MeterLocation meterLocation = meter.MeterLocation;
meterLocation.Meters = new List <Meter>()
{
meter
};
meterLocation.AssetKey = schema.StationName;
meterLocation.Name = schema.StationName;
meterLocation.ShortName = schema.StationName.Substring(0, Math.Min(schema.StationName.Length, 50));
meterLocation.Description = schema.StationName;
foreach (AnalogChannel analogChannel in schema.AnalogChannels)
{
Channel channel = ParseSeries(analogChannel);
channel.Meter = meter;
DataSeries dataSeries = new DataSeries();
dataSeries.SeriesInfo = channel.Series[0];
meter.Channels.Add(channel);
while (MeterDataSet.DataSeries.Count <= analogChannel.Index)
{
MeterDataSet.DataSeries.Add(new DataSeries());
}
MeterDataSet.DataSeries[analogChannel.Index] = dataSeries;
}
foreach (DigitalChannel digitalChannel in schema.DigitalChannels)
{
Channel channel = ParseSeries(digitalChannel);
channel.Meter = meter;
DataSeries dataSeries = new DataSeries();
dataSeries.SeriesInfo = channel.Series[0];
meter.Channels.Add(channel);
while (MeterDataSet.Digitals.Count <= digitalChannel.Index)
{
MeterDataSet.Digitals.Add(new DataSeries());
}
MeterDataSet.Digitals[digitalChannel.Index] = dataSeries;
}
try
{
while (Parser.ReadNext())
{
for (int i = 0; i < schema.AnalogChannels.Length; i++)
{
int seriesIndex = schema.AnalogChannels[i].Index;
string units = schema.AnalogChannels[i].Units.ToUpper();
double multiplier = (units.Contains("KA") || units.Contains("KV")) ? 1000.0D : 1.0D;
MeterDataSet.DataSeries[seriesIndex].DataPoints.Add(new DataPoint()
{
Time = Parser.Timestamp, Value = multiplier * Parser.PrimaryValues[i]
});
}
for (int i = 0; i < schema.DigitalChannels.Length; i++)
{
int valuesIndex = schema.TotalAnalogChannels + i;
int seriesIndex = schema.DigitalChannels[i].Index;
MeterDataSet.Digitals[seriesIndex].DataPoints.Add(new DataPoint()
{
Time = Parser.Timestamp, Value = Parser.Values[valuesIndex]
});
}
}
}
catch (InvalidOperationException ex)
{
Log.Warn(ex.Message, ex);
}
MeterDataSet.Meter = meter;
string infFilePath = Path.ChangeExtension(filePath, "inf");
if (File.Exists(infFilePath))
{
IniFile infFile = new IniFile(infFilePath);
INFDataSet infDataSet = new INFDataSet(infFile);
MeterDataSet.GetResource(() => new BreakerRestrikeResource(infDataSet));
}
}
/// <summary>
/// Creates a new instance of the <see cref="COMTRADEReader"/> class.
/// </summary>
public COMTRADEReader()
{
m_meterDataSet = new MeterDataSet();
}
public bool Execute(MeterDataSet meterDataSet)
{
if (meterDataSet.Type != DataSetType.AppStatus)
{
return(false);
}
using (AdoDataConnection connection = new AdoDataConnection("systemSettings"))
{
// get metadata for file
FileInfo fi = new FileInfo(meterDataSet.FilePath);
DateTime lastWriteTime = TimeZoneInfo.ConvertTimeFromUtc(fi.LastWriteTime.ToUniversalTime(), TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time"));
// read lines from file
string[] data = File.ReadAllLines(meterDataSet.FilePath);
// instantiate records object
AppStatusFileChanges newRecord = new AppStatusFileChanges();
// retrieve last change for this file
AppStatusFileChanges lastChanges = new TableOperations <AppStatusFileChanges>(connection).QueryRecord("LastWriteTime DESC", new RecordRestriction("MeterID = {0} AND FileName = {1}", meterDataSet.Meter.ID, fi.Name));
// if record doesn't exist, use default
if (lastChanges == null)
{
lastChanges = new AppStatusFileChanges();
}
// if lastChanges LastWriteTime equals new LastWriteTime return
if (lastChanges.LastWriteTime.ToString("MM/dd/yyyy HH:mm:ss") == lastWriteTime.ToString("MM/dd/yyyy HH:mm:ss"))
{
return(false);
}
newRecord.MeterID = meterDataSet.Meter.ID;
newRecord.FileName = fi.Name;
newRecord.LastWriteTime = lastWriteTime;
newRecord.FileSize = (int)(fi.Length / 1000);
newRecord.Text = meterDataSet.Text;
newRecord.Alarms = 0;
if (lastChanges.LastWriteTime > TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time")))
{
newRecord.LastWriteTime = TimeZoneInfo.ConvertTimeFromUtc(DateTime.UtcNow, TimeZoneInfo.FindSystemTimeZoneById("Central Standard Time"));
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: DFR time set in the future.\n";
}
// parse each line
foreach (string line in data)
{
// if line is empty go to the next line
if (line == string.Empty)
{
continue;
}
string[] section = line.Split(new[] { "=" }, StringSplitOptions.RemoveEmptyEntries);
if (section[0].ToLower() == "recorder")
{
newRecord.Version = section[1];
}
else if (section[0].ToLower() == "dfr")
{
newRecord.DFR = section[1];
if (newRecord.DFR.ToLower() != "online")
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: DFR not set to ONLINE.\n";
}
}
else if (section[0].ToLower() == "pc_time")
{
try
{
newRecord.PCTime = DateTime.ParseExact(section[1], "MM/dd/yyyy-HH:mm:ss", CultureInfo.InvariantCulture);
}
catch (Exception ex)
{
newRecord.PCTime = DateTime.MinValue;
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: Incorrect date format for PC_Time.\n";
}
}
else if (section[0].ToLower() == "time_mark_source")
{
newRecord.TimeMarkSource = section[1];
if (newRecord.TimeMarkSource.ToLower() == "irig-b")
{
}
else if (newRecord.TimeMarkSource.ToLower() == "pc")
{
}
else
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: TIME_MARK_SOURCE not set to IRIG-B.\n";
}
}
else if (section[0].ToLower() == "time_mark_time")
{
try
{
newRecord.TimeMarkTime = DateTime.ParseExact(section[1], "MM/dd/yyyy-HH:mm:ss.ffffff", CultureInfo.InvariantCulture);
if (newRecord.TimeMarkTime.Subtract(newRecord.PCTime).TotalSeconds > 2)
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: Time_Mark_Time and PC_Time difference greater than 2 seconds.";
}
}
catch (Exception ex)
{
newRecord.TimeMarkTime = DateTime.MinValue;
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: Incorrect date format for Time_Mark_Time.";
}
}
else if (section[0].ToLower() == "clock")
{
if (section[1].ToLower() == "sync(lock)")
{
}
else if (section[1].ToLower() == "unsync(unknown)" && newRecord.TimeMarkSource.ToLower() == "pc")
{
int count = connection.ExecuteScalar <int>(@"
with cte as
(SELECT TOP 1 * FROM AppStatusFileChanges
where meterid = {0}
order by LastWriteTime desc)
SELECT COUNT(*) FROM cte WHERE Text LIKE '%TIME_MARK_SOURCE=PC%' AND Text LIKE '%Clock=UNSYNC(unknown)%'
", meterDataSet.Meter.ID);
if (count > 0)
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: Time_Mark_Source Set to PC and Clock set to UNSYNC(unknown) in consecutive files.\n";
}
}
else
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: Clock not set to SYNC(lock).\n";
}
}
else if (section[0].ToLower() == "data_drive")
{
try
{
string[] values = section[1].Replace("GB", "").Split('/');
newRecord.DataDriveUsage = double.Parse(values[0]) / double.Parse(values[1]);
}
catch (Exception ex)
{
newRecord.DataDriveUsage = 0;
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: Incorrect format for Data_Drive.\n";
}
}
else if (section[0].ToLower() == "chassis_not_comm")
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: CHASSIS_NOT_COMM field present.\n";
}
else if (section[0].ToLower() == "timemark")
{
// sometimes this lines ends in a comma, remove it
string timeString = section[1];
IEnumerable <string> times = timeString.Split(',').Where(x => !Regex.IsMatch(x, "\\s*") && x != string.Empty);
bool flag = times.Distinct().Count() > 1;
if (flag)
{
int count = connection.RetrieveData(@"
with cte as
(SELECT TOP 14 * FROM AppStatusFileChanges
where meterid = {0}
order by LastWriteTime desc)
SELECT * FROM cte WHERE Text LIKE '%TimeMark values not equal%'
", meterDataSet.Meter.ID).Rows.Count;
newRecord.Text += "\nMiMD Parsing Warning: TimeMark values not equal.\n";
if (count == 14)
{
newRecord.Alarms += 1;
newRecord.Text += "\nMiMD Parsing Alarm: TimeMark values not equal in 15 consecutive files.\n";
}
}
}
else if (section[0].ToLower() == "dsp_board")
{
newRecord.DSPBoard = section[1];
}
else if (section[0].ToLower() == "dsp_revision")
{
newRecord.DSPRevision = section[1];
}
else if (section[0].ToLower().Contains("packet"))
{
newRecord.Packet = section[1];
}
else if (section[0].ToLower().Contains("recovery"))
{
newRecord.Recovery = section[1];
}
else if (section[0].ToLower() == "(>65c,c)")
{
newRecord.BoardTemp = section[1];
}
else if (section[0].ToLower() == "speedfan")
{
newRecord.SpeedFan = section[1].Replace("\"", "").Replace(" ", "");
}
else if (section[0].ToLower() == "alarmon" || section[0].ToLower() == "anfail")
{
newRecord.Alarms += 1;
}
}
// get html of new changes
DiffMatchPatch dmp = new DiffMatchPatch();
List <Diff> diff = dmp.DiffMain(lastChanges.Text ?? "", newRecord.Text);
dmp.DiffCleanupSemantic(diff);
newRecord.Html = dmp.DiffPrettyHtml(diff).Replace("¶", "");
// write new record to db
meterDataSet.DiagnosticAlarms = newRecord.Alarms;
new TableOperations <AppStatusFileChanges>(connection).AddNewRecord(newRecord);
return(true);
}
}
/// <summary>
/// Creates a new instance of the <see cref="PQDIFReader"/> class.
/// </summary>
public PQDIFReader()
{
m_meterDataSet = new MeterDataSet();
}
/// <summary>
/// Creates a new instance of the <see cref="SELEVEReader"/> class.
/// </summary>
public SELEVEReader()
{
m_meterDataSet = new MeterDataSet();
}
