public DataReader(Settings settings, Action <string> showMessage, LogPublisher log)
{
ShowMessage = showMessage;
Log = log;
m_settings = settings;
if (m_settings.ReadFromOpenHistorian)
{
m_startTime = new DateTime(m_settings.StartTime.Ticks, m_settings.UseUTCTime ? DateTimeKind.Utc : DateTimeKind.Local);
m_endTime = new DateTime(m_settings.EndTime.Ticks, m_settings.UseUTCTime ? DateTimeKind.Utc : DateTimeKind.Local);
// Convert all times to UTC since that's what the openHistorian uses.
m_startTime = m_startTime.ToUniversalTime();
m_endTime = m_endTime.ToUniversalTime();
m_timeRange = (m_endTime - m_startTime).TotalSeconds;
m_points = GetMetadata().Select(md => md.PointID);
m_historianClient = new SnapDBClient(m_settings.HostAddress, m_settings.DataPort, m_settings.InstanceName, m_startTime, m_endTime, m_settings.FrameRate, m_points);
m_dataBlock = new Dictionary <ulong, DataPoint>();
m_dataPoint = new DataPoint();
PointCount = m_points.Count();
m_readNext = ReadNextHistorianBlock;
if (!m_historianClient.ReadNext(m_dataPoint)) //Scan to first record in time range
{
throw new InvalidOperationException("No data for specified time range in openHistorian connection");
}
CurrentTimestamp = new DateTime((long)m_dataPoint.Timestamp);
}
else if (m_settings.ReadFromCsv)
{
m_readNext = ReadNextCsvBlock;
m_reader = new StreamReader(m_settings.CsvSourcePath);
string headerLine = m_reader.ReadLine();
string[] headers = headerLine.Split(',');
m_currentLine = 0;
IEnumerable <string> dataFile = File.ReadLines(m_settings.CsvSourcePath);
m_totalLines = dataFile.Count();
m_startTime = DateTime.Parse(dataFile.ElementAt(1).Split(',')[0]);
m_endTime = DateTime.Parse(dataFile.ElementAt(m_totalLines - 1).Split(',')[0]);
m_timeRange = (m_endTime - m_startTime).TotalSeconds;
m_indexToPointIDLookup = new ulong[headers.Length];
for (int i = 1; i < headers.Length; i++)
{
m_indexToPointIDLookup[i] = ParsePointID(headers[i].Trim());
}
PointCount = headers.Length - 1;
m_dataPoints = new DataPoint[PointCount];
Parallel.For(0, PointCount, i => m_dataPoints[i] = new DataPoint());
}
Read = m_readNext;
PercentComplete = 0;
}
// Internal Functions
private void ReadArchive(object state)
{
try
{
double timeRange = (m_settings.EndTime - m_settings.StartTime).TotalSeconds;
long receivedPoints = 0;
long processedDataBlocks = 0;
long duplicatePoints = 0;
Ticks operationTime;
Ticks operationStartTime;
DataPoint point = new DataPoint();
DateTime firstTimestamp = new DateTime(0L);
DateTime lastTimestamp = new DateTime(0L);
using (Algorithm algorithm = new Algorithm())
{
algorithm.ShowMessage = ShowUpdateMessage;
algorithm.MessageInterval = m_settings.MessageInterval;
algorithm.StartTime = m_settings.StartTime;
algorithm.EndTime = m_settings.EndTime;
algorithm.FrameRate = m_settings.FrameRate;
algorithm.TimeRange = timeRange;
algorithm.Log = m_log;
// Load historian meta-data
ShowUpdateMessage(">>> Loading source connection metadata...");
operationStartTime = DateTime.UtcNow.Ticks;
algorithm.Metadata = MetadataRecord.Query(m_settings.HostAddress, m_settings.MetadataPort, m_settings.MetadataTimeout);
operationTime = DateTime.UtcNow.Ticks - operationStartTime;
ShowUpdateMessage("*** Metadata Load Complete ***");
ShowUpdateMessage($"Total metadata load time {operationTime.ToElapsedTimeString(3)}...");
ShowUpdateMessage(">>> Processing filter expression for metadata...");
operationStartTime = DateTime.UtcNow.Ticks;
MeasurementKey[] inputKeys = AdapterBase.ParseInputMeasurementKeys(MetadataRecord.Metadata, false, textBoxPointList.Text, "MeasurementDetail");
List <ulong> pointIDList = inputKeys.Select(key => (ulong)key.ID).ToList();
operationTime = DateTime.UtcNow.Ticks - operationStartTime;
// Allow algorithm to augment (or even replace) point ID list as provided by user
algorithm.AugmentPointIDList(pointIDList);
ShowUpdateMessage($">>> Historian read will be for {pointIDList.Count:N0} points based on provided meta-data expression and algorithm augmentation.");
// Reduce metadata to filtered point list
ShowUpdateMessage($">>> Reducing metadata to the {pointIDList.Count:N0} defined points...");
List <MetadataRecord> records = new List <MetadataRecord>();
foreach (ulong pointID in pointIDList)
{
MetadataRecord record = algorithm.Metadata.FirstOrDefault(metadata => metadata.PointID == pointID);
if ((object)record != null)
{
records.Add(record);
}
}
algorithm.Metadata = records;
ShowUpdateMessage("*** Filter Expression Processing Complete ***");
ShowUpdateMessage($"Total filter expression processing time {operationTime.ToElapsedTimeString(3)}...");
ShowUpdateMessage(">>> Initializing algorithm...");
algorithm.Initialize();
ShowUpdateMessage(">>> Starting archive read...");
// Start historian data read
operationStartTime = DateTime.UtcNow.Ticks;
using (SnapDBClient historianClient = new SnapDBClient(m_settings.HostAddress, m_settings.DataPort, m_settings.InstanceName, m_settings.StartTime, m_settings.EndTime, m_settings.FrameRate, pointIDList))
{
// Scan to first record
if (!historianClient.ReadNext(point))
{
throw new InvalidOperationException("No data for specified time range in openHistorian connection!");
}
ulong currentTimestamp;
receivedPoints++;
while (!m_formClosing)
{
int timeComparison;
bool readSuccess = true;
// Create a new data block for current timestamp and load first/prior point
Dictionary <ulong, DataPoint> dataBlock = new Dictionary <ulong, DataPoint>
{
[point.PointID] = point.Clone()
};
currentTimestamp = point.Timestamp;
// Load remaining data for current timestamp
do
{
// Scan to next record
if (!historianClient.ReadNext(point))
{
readSuccess = false;
break;
}
receivedPoints++;
timeComparison = DataPoint.CompareTimestamps(point.Timestamp, currentTimestamp, m_settings.FrameRate);
if (timeComparison == 0)
{
// Timestamps are compared based on configured frame rate - if archived data rate is
// higher than configured frame rate, then data block will contain only latest values
if (dataBlock.ContainsKey(point.PointID))
{
duplicatePoints++;
}
dataBlock[point.PointID] = point.Clone();
}
}while (timeComparison == 0);
// Finished with data read
if (!readSuccess)
{
ShowUpdateMessage(">>> End of data in range encountered...");
break;
}
if (++processedDataBlocks % m_settings.MessageInterval == 0)
{
ShowUpdateMessage($"{Environment.NewLine}{receivedPoints:N0} points{(duplicatePoints > 0 ? $", which included {duplicatePoints:N0} duplicates," : "")} read so far averaging {receivedPoints / (DateTime.UtcNow.Ticks - operationStartTime).ToSeconds():N0} points per second.");
UpdateProgressBar((int)((1.0D - new Ticks(m_settings.EndTime.Ticks - (long)point.Timestamp).ToSeconds() / timeRange) * 100.0D));
}
try
{
lastTimestamp = new DateTime((long)currentTimestamp);
if (firstTimestamp.Ticks == 0L)
{
firstTimestamp = lastTimestamp;
}
// Analyze data block
algorithm.Execute(lastTimestamp, dataBlock.Values.ToArray());
}
catch (Exception ex)
{
ShowUpdateMessage($"ERROR: Algorithm exception: {ex.Message}");
m_log.Publish(MessageLevel.Error, "AlgorithmError", "Failed while processing data from the historian", exception: ex);
}
}
operationTime = DateTime.UtcNow.Ticks - operationStartTime;
if (m_formClosing)
{
ShowUpdateMessage("*** Historian Read Canceled ***");
UpdateProgressBar(0);
}
else
{
ShowUpdateMessage("*** Historian Read Complete ***");
UpdateProgressBar(100);
}
algorithm.Complete();
// Show some operational statistics
long expectedPoints = (long)(timeRange * m_settings.FrameRate * algorithm.Metadata.Count);
double dataCompleteness = Math.Round(receivedPoints / (double)expectedPoints * 100000.0D) / 1000.0D;
string overallSummary =
$"Total processing time {operationTime.ToElapsedTimeString(3)} at {receivedPoints / operationTime.ToSeconds():N0} points per second.{Environment.NewLine}" +
$"{Environment.NewLine}" +
$" Meta-data points: {algorithm.Metadata.Count}{Environment.NewLine}" +
$" Time-span covered: {timeRange:N0} seconds: {Ticks.FromSeconds(timeRange).ToElapsedTimeString(2)}{Environment.NewLine}" +
$" Processed timestamps: {processedDataBlocks:N0}{Environment.NewLine}" +
$" Expected points: {expectedPoints:N0} @ {m_settings.FrameRate:N0} samples per second{Environment.NewLine}" +
$" Received points: {receivedPoints:N0}{Environment.NewLine}" +
$" Duplicate points: {duplicatePoints:N0}{Environment.NewLine}" +
$" Data completeness: {dataCompleteness:N3}%{Environment.NewLine}" +
$" First timestamp with data: {firstTimestamp:yyyy-MM-dd HH:mm:ss.fff}{Environment.NewLine}" +
$" Last timestamp with data: {lastTimestamp:yyyy-MM-dd HH:mm:ss.fff}{Environment.NewLine}";
ShowUpdateMessage(overallSummary);
}
}
}
catch (Exception ex)
{
ShowUpdateMessage($"!!! Failure during historian read: {ex.Message}");
m_log.Publish(MessageLevel.Error, "HistorianDataRead", "Failed while reading data from the historian", exception: ex);
}
finally
{
SetGoButtonEnabledState(true);
}
}
