private void ExportData()
{
try
{
const int TargetBufferSize = 524288;
Dictionary <string, DeviceDetail> deviceMap = new Dictionary <string, DeviceDetail>(StringComparer.OrdinalIgnoreCase);
Dictionary <Guid, DeviceDetail> signalMap = new Dictionary <Guid, DeviceDetail>();
Dictionary <Guid, DeviceStats> deviceStats = new Dictionary <Guid, DeviceStats>();
string hostAddress = m_settings.HostAddress;
int port = m_settings.Port;
DateTime startTime = m_settings.StartTime;
DateTime endTime = m_settings.EndTime;
int frameRate = m_settings.FrameRate;
bool alignTimestamps = m_settings.AlignTimestamps;
bool missingAsNaN = m_settings.ExportMissingAsNaN;
bool fillMissingTimestamps = alignTimestamps && m_settings.FillInMissingTimestamps;
int messageInterval = m_settings.MessageInterval;
bool exportFilePerDataType = m_settings.ExportFilePerDataType;
string exportFileName = m_settings.ExportFileName;
double timeRange = (endTime - startTime).TotalSeconds;
int selectedDeviceCount = SelectedDeviceCount;
Ticks operationStartTime;
StringBuilder readBuffer = new StringBuilder(TargetBufferSize * 2);
ManualResetEventSlim bufferReady = new ManualResetEventSlim(false);
List <string> writeBuffer = new List <string>();
object writeBufferLock = new object();
TextWriter writer = null;
bool readComplete = false;
bool writeComplete = false;
long receivedPoints = 0L;
int exports = 0;
int totalExports;
Ticks[] subseconds = Ticks.SubsecondDistribution(frameRate);
long interval = subseconds.Length > 1 ? subseconds[1].Value : Ticks.PerSecond;
Dictionary <Guid, int> signalIDIndex = new Dictionary <Guid, int>();
double[] values = {};
int deviceCount = 0;
int measurementCount = 0;
long lastTimestamp = 0L;
long timestamp = 0L;
bool pastFirstRow = false;
void bufferValues()
{
// Write row values
readBuffer.Append(missingAsNaN ? string.Join(",", values) : string.Join(",", values.Select(val => double.IsNaN(val) ? "" : $"{val}")));
if (readBuffer.Length < TargetBufferSize)
{
return;
}
lock (writeBufferLock)
writeBuffer.Add(readBuffer.ToString());
readBuffer.Clear();
bufferReady.Set();
}
void writeData()
{
string[] localBuffer;
while (writeBuffer.Count > 0 || !writeComplete)
{
bufferReady.Wait();
bufferReady.Reset();
lock (writeBufferLock)
{
localBuffer = writeBuffer.ToArray();
writeBuffer.Clear();
}
foreach (string buffer in localBuffer)
{
writer?.Write(buffer);
}
}
}
string getHeaders(HashSet <string> signalTypes)
{
StringBuilder headers = new StringBuilder();
headers.AppendLine($"Data extraction from \"{hostAddress}:{port}\" exported on {DateTime.UtcNow.ToString(TimeTagBase.DefaultFormat)} UTC");
headers.AppendLine($"Export range: {startTime.ToString(TimeTagBase.DefaultFormat)} UTC to {endTime.ToString(TimeTagBase.DefaultFormat)} UTC");
headers.AppendLine($"Signal types: {string.Join(", ", signalTypes)}");
StringBuilder deviceRow = new StringBuilder();
StringBuilder measurementRow = new StringBuilder();
measurementRow.Append("\"Timestamp\"");
signalIDIndex.Clear();
foreach (DeviceDetail device in m_metadata.Devices.Where(d => d.Selected))
{
MeasurementDetail[] deviceMeasurements = m_metadata.Measurements.Where(m => string.Equals(m.DeviceName, device.Name, StringComparison.OrdinalIgnoreCase) && signalTypes.Contains(m.SignalAcronym)).ToArray();
if (deviceMeasurements.Length == 0)
{
continue;
}
deviceRow.Append($",{device.Name}");
deviceCount++;
for (int i = 0; i < deviceMeasurements.Length; i++)
{
MeasurementDetail measurement = deviceMeasurements[i];
if (i > 0)
{
deviceRow.Append(",");
}
measurementRow.Append($",{measurement.PointTag} [{measurement.SignalAcronym}]");
signalIDIndex.Add(measurement.SignalID, measurementCount++);
}
}
headers.AppendLine($"Device count: {deviceCount}");
headers.AppendLine($"Measurement count: {measurementCount}");
headers.AppendLine();
headers.AppendLine(deviceRow.ToString());
headers.Append(measurementRow);
values = new double[measurementCount];
for (int i = 0; i < values.Length; i++)
{
values[i] = double.NaN;
}
return(headers.ToString());
}
void handleNewMeasurements(ICollection <IMeasurement> measurements)
{
bool showMessage = receivedPoints + measurements.Count >= (receivedPoints / messageInterval + 1) * messageInterval;
receivedPoints += measurements.Count;
foreach (IMeasurement measurement in measurements)
{
if (signalMap.TryGetValue(measurement.Key.SignalID, out DeviceDetail device) && deviceStats.TryGetValue(device.UniqueID, out DeviceStats stats))
{
stats.Total++;
if (!measurement.ValueQualityIsGood())
{
stats.BadDataCount++;
}
if (!measurement.TimestampQualityIsGood())
{
stats.BadTimeCount++;
}
}
if (signalIDIndex.TryGetValue(measurement.ID, out int index))
{
if (alignTimestamps)
{
timestamp = Ticks.RoundToSubsecondDistribution(measurement.Timestamp, frameRate).Value;
}
else
{
timestamp = measurement.Timestamp;
}
// Start a new row for each encountered new timestamp
if (timestamp != lastTimestamp)
{
if (lastTimestamp > 0 && pastFirstRow)
{
bufferValues();
}
for (int i = 0; i < values.Length; i++)
{
values[i] = float.NaN;
}
// Handle any missing data rows
if (fillMissingTimestamps && lastTimestamp > 0 && timestamp > lastTimestamp)
{
long difference = timestamp - lastTimestamp;
if (difference > interval)
{
long interpolated = lastTimestamp;
for (long i = 1; i < difference / interval; i++)
{
interpolated = Ticks.RoundToSubsecondDistribution(interpolated + interval, frameRate).Value;
readBuffer.Append($"{Environment.NewLine}{new DateTime(interpolated, DateTimeKind.Utc).ToString(TimeTagBase.DefaultFormat)},");
bufferValues();
}
}
}
readBuffer.Append($"{Environment.NewLine}{new DateTime(timestamp, DateTimeKind.Utc).ToString(TimeTagBase.DefaultFormat)},");
lastTimestamp = timestamp;
pastFirstRow = true;
}
// Save value to its column
values[index] = measurement.AdjustedValue;
}
}
if (showMessage && measurements.Count > 0)
{
IMeasurement measurement = measurements.Last();
ShowUpdateMessage($"{Environment.NewLine}{receivedPoints:N0} points read so far averaging {receivedPoints / (DateTime.UtcNow.Ticks - operationStartTime).ToSeconds():N0} points per second.");
int exportFraction = (int)(100.0D * exports / totalExports);
double currentExportProgress = 1.0D - (endTime.Ticks - measurement.Timestamp).ToSeconds() / timeRange;
UpdateProgressBar(exportFraction + (int)(currentExportProgress / totalExports * 100.0D));
}
}
void flushBuffers()
{
// Flush last row
if (timestamp > 0)
{
bufferValues();
}
// Flush remaining buffers
if (readBuffer.Length > 0)
{
lock (writeBufferLock)
writeBuffer.Add(readBuffer.ToString());
}
}
void readCompleted()
{
readComplete = true;
if (++exports == totalExports)
{
ShowUpdateMessage($"Data read{(totalExports > 1 ? "s" : "" )} completed.");
}
else
{
ShowUpdateMessage($"{exports} of {totalExports} data reads completed.");
}
UpdateProgressBar((int)(100.0D * exports / totalExports));
}
void exportData(HashSet <string> signalTypes, string suffix = null)
{
string fileName;
if (string.IsNullOrEmpty(suffix))
{
fileName = exportFileName;
}
else
{
fileName = $"{FilePath.GetDirectoryName(exportFileName)}{FilePath.GetFileNameWithoutExtension(exportFileName)}_{suffix}{FilePath.GetExtension(exportFileName)}";
}
readComplete = false;
lastTimestamp = 0L;
deviceCount = 0;
measurementCount = 0;
pastFirstRow = false;
readBuffer.Clear();
writeBuffer.Clear();
if (File.Exists(fileName))
{
File.Delete(fileName);
}
using (writer = File.CreateText(fileName))
{
writer.Write(getHeaders(signalTypes));
writeComplete = false;
bufferReady.Reset();
Thread writeThread = new Thread(writeData);
writeThread.Start();
try
{
if (measurementCount > 0)
{
ShowUpdateMessage($"\nStarting data read for {string.Join(", ", signalTypes)} signal type{(signalTypes.Count > 1 ? "s" : "")}...\n");
using (new DataReceiver($"server={hostAddress}; port={port}; interface=0.0.0.0", GenerateFilterExpression(signalTypes, selectedDeviceCount), startTime, endTime + TimeSpan.FromTicks(interval))
{
NewMeasurementsCallback = handleNewMeasurements,
StatusMessageCallback = ShowUpdateMessage,
ProcessExceptionCallback = ex => ShowUpdateMessage($"Error: {ex.Message}"),
ReadCompletedCallback = readCompleted
})
{
while (!m_formClosing && !readComplete && m_exporting) //-V3063
{
Thread.Sleep(500);
}
}
flushBuffers();
}
}
finally
{
writeComplete = true;
bufferReady.Set();
}
writeThread.Join(5000);
}
}
// Start export operations
HashSet <string> selectedSignalTypes = new HashSet <string>(StringComparer.OrdinalIgnoreCase);
foreach (DeviceDetail device in m_metadata.Devices.Where(d => d.Selected))
{
deviceMap[device.Name] = device;
}
foreach (MeasurementDetail measurement in m_metadata.Measurements)
{
if (deviceMap.TryGetValue(measurement.DeviceName, out DeviceDetail device))
{
signalMap[measurement.SignalID] = device;
}
}
foreach (DeviceDetail device in m_metadata.Devices.Where(d => d.Selected))
{
deviceStats[device.UniqueID] = new DeviceStats {
Device = device
}
}
;
if (checkedListBoxDataTypes.CheckedItems.Count > 0)
{
selectedSignalTypes.UnionWith(checkedListBoxDataTypes.CheckedItems.Cast <string>());
}
else
{
selectedSignalTypes.UnionWith(checkedListBoxDataTypes.Items.Cast <string>());
}
operationStartTime = DateTime.UtcNow.Ticks;
if (exportFilePerDataType)
{
totalExports = selectedSignalTypes.Count;
foreach (string signalType in selectedSignalTypes)
{
exportData(new HashSet <string>(new[] { signalType }, StringComparer.OrdinalIgnoreCase), signalType);
}
}
else
{
totalExports = 1; //-V3137
exportData(selectedSignalTypes);
}
long expectedPoints = (long)(frameRate * timeRange);
// Show export summary information
foreach (DeviceStats stats in deviceStats.Values)
{
DeviceDetail device = stats.Device;
stats.MissingDataCount = expectedPoints - stats.Total;
double badData, badTime, missingData;
if (stats.Total == 0)
{
badTime = 0.0D;
badData = 0.0D;
missingData = 1.0D;
}
else
{
badData = stats.BadDataCount / (double)stats.Total;
badTime = stats.BadTimeCount / (double)stats.Total;
missingData = stats.MissingDataCount / (double)stats.Total;
if (badData < 0.0D)
{
badData = 0.0D;
}
if (badTime < 0.0D)
{
badTime = 0.0D;
}
if (missingData < 0.0D)
{
missingData = 0.0D;
}
}
ShowUpdateMessage($"Device \"{device.Name}\" bad data: {badData:0.00%}, bad time: {badTime:0.00%}, missing data: {missingData:0.00%}...");
}
Ticks operationTime = DateTime.UtcNow.Ticks - operationStartTime;
if (m_formClosing || !m_exporting)
{
ShowUpdateMessage("*** Data Export Canceled ***");
UpdateProgressBar(0);
}
else
{
ShowUpdateMessage("*** Data Export Complete ***");
UpdateProgressBar(100);
}
ShowUpdateMessage($"Total export processing time {operationTime.ToElapsedTimeString(3)} at {receivedPoints / operationTime.ToSeconds():N0} points per second.{Environment.NewLine}");
}
catch (Exception ex)
{
ShowUpdateMessage($"!!! Failure during historian read: {ex.Message}");
m_log.Publish(MessageLevel.Error, "HistorianDataRead", "Failed while reading data from the historian", exception: ex);
}
finally
{
SetButtonsEnabledState(true);
}
}
// Internal Functions
private void PreFilter()
{
//const int MaxPoints = 50;
try
{
double timeRange = (m_settings.EndTime - m_settings.StartTime).TotalSeconds;
Dictionary <string, DeviceDetail> deviceMap = new Dictionary <string, DeviceDetail>(StringComparer.OrdinalIgnoreCase);
Dictionary <Guid, DeviceDetail> signalMap = new Dictionary <Guid, DeviceDetail>();
Dictionary <Guid, DeviceStats> deviceStats = new Dictionary <Guid, DeviceStats>();
//Dictionary<Guid, Tuple<Ticks, List<double>, List<double>>> plotValues = new Dictionary<Guid, Tuple<Ticks, List<double>, List<double>>>();
bool readComplete = false;
long receivedPoints = 0L;
Ticks operationTime;
Ticks operationStartTime;
//double pointInterval = timeRange / MaxPoints;
void handleNewMeasurements(ICollection <IMeasurement> measurements)
{
bool showMessage = receivedPoints + measurements.Count >= (receivedPoints / m_settings.MessageInterval + 1) * m_settings.MessageInterval;
receivedPoints += measurements.Count;
foreach (IMeasurement measurement in measurements)
{
Guid signalID = measurement.ID;
if (signalMap.TryGetValue(signalID, out DeviceDetail device) && deviceStats.TryGetValue(device.UniqueID, out DeviceStats stats))
{
stats.Total++;
if (!measurement.ValueQualityIsGood())
{
stats.BadDataCount++;
}
if (!measurement.TimestampQualityIsGood())
{
stats.BadTimeCount++;
}
}
//Tuple<Ticks, List<double>, List<double>> plotData = plotValues.GetOrAdd(signalID, _ => new Tuple<Ticks, List<double>, List<double>>(measurement.Timestamp, new List<double>(new[] { (double)measurement.Timestamp }), new List<double>(new[] { measurement.AdjustedValue })));
//if ((measurement.Timestamp - plotData.Item1).ToSeconds() > pointInterval)
//{
// plotData.Item2.Add(measurement.Timestamp);
// plotData.Item3.Add(measurement.AdjustedValue);
// plotValues[signalID] = new Tuple<Ticks, List<double>, List<double>>(measurement.Timestamp, plotData.Item2, plotData.Item3);
//}
}
if (showMessage && measurements.Count > 0)
{
IMeasurement measurement = measurements.Last();
ShowUpdateMessage($"{Environment.NewLine}{receivedPoints:N0} points read so far averaging {receivedPoints / (DateTime.UtcNow.Ticks - operationStartTime).ToSeconds():N0} points per second.");
UpdateProgressBar((int)((1.0D - new Ticks(m_settings.EndTime.Ticks - (long)measurement.Timestamp).ToSeconds() / timeRange) * 100.0D));
}
}
void readCompleted()
{
readComplete = true;
ShowUpdateMessage("Data read completed.");
//foreach (KeyValuePair<Guid, Tuple<Ticks, List<double>, List<double>>> plotData in plotValues)
// m_graphData.PlotLine(plotData.Value.Item2, plotData.Value.Item3);
}
operationStartTime = DateTime.UtcNow.Ticks;
foreach (DeviceDetail device in m_metadata.Devices.Where(d => d.Selected))
{
deviceMap[device.Name] = device;
}
foreach (MeasurementDetail measurement in m_metadata.Measurements)
{
if (deviceMap.TryGetValue(measurement.DeviceName, out DeviceDetail device))
{
signalMap[measurement.SignalID] = device;
}
}
foreach (DeviceDetail device in m_metadata.Devices.Where(d => d.Selected))
{
deviceStats[device.UniqueID] = new DeviceStats {
Device = device
}
}
;
//m_graphData.ClearPlots();
using (new DataReceiver($"server={m_settings.HostAddress}; port={m_settings.Port}; interface=0.0.0.0", m_settings.FilterExpression, m_settings.StartTime, m_settings.EndTime)
{
NewMeasurementsCallback = handleNewMeasurements,
StatusMessageCallback = ShowUpdateMessage,
ProcessExceptionCallback = ex => ShowUpdateMessage($"Error: {ex.Message}"),
ReadCompletedCallback = readCompleted
})
{
while (!m_formClosing && !readComplete && m_prefiltering)
{
Thread.Sleep(500);
}
}
long expectedPoints = (long)(m_settings.FrameRate * timeRange);
foreach (DeviceStats stats in deviceStats.Values)
{
DeviceDetail device = stats.Device;
stats.MissingDataCount = expectedPoints - stats.Total;
double badData, badTime, missingData;
if (stats.Total == 0)
{
badTime = 0.0D;
badData = 0.0D;
missingData = 1.0D;
}
else
{
badData = stats.BadDataCount / (double)stats.Total;
badTime = stats.BadTimeCount / (double)stats.Total;
missingData = stats.MissingDataCount / (double)stats.Total;
if (badData < 0.0D)
{
badData = 0.0D;
}
if (badTime < 0.0D)
{
badTime = 0.0D;
}
if (missingData < 0.0D)
{
missingData = 0.0D;
}
}
if (stats.BadDataCount / (double)stats.Total * 100.0D > m_settings.AcceptableBadData)
{
device.Selected = false;
ShowUpdateMessage($"Device \"{device.Name}\" unselected - too much bad data: {badData:0.00%}...");
}
else if (stats.BadTimeCount / (double)stats.Total * 100.0D > m_settings.AcceptableBadTime)
{
device.Selected = false;
ShowUpdateMessage($"Device \"{device.Name}\" unselected - too much bad data with bad time: {badTime:0.00%}...");
}
else if (stats.MissingDataCount / (double)stats.Total * 100.0D > m_settings.AcceptableMissingData)
{
device.Selected = false;
ShowUpdateMessage($"Device \"{device.Name}\" unselected - too much missing data: {missingData:0.00%}...");
}
}
RefreshSelectedCount();
RefreshDevicesDataGrid();
operationTime = DateTime.UtcNow.Ticks - operationStartTime;
if (m_formClosing || !m_prefiltering)
{
ShowUpdateMessage("*** Data Pre-filter Canceled ***");
UpdateProgressBar(0);
}
else
{
ShowUpdateMessage("*** Data Pre-filter Complete ***");
UpdateProgressBar(100);
}
ShowUpdateMessage($"Total pre-filter processing time {operationTime.ToElapsedTimeString(3)} at {receivedPoints / operationTime.ToSeconds():N0} points per second.{Environment.NewLine}");
}
catch (Exception ex)
{
ShowUpdateMessage($"!!! Failure during historian read: {ex.Message}");
m_log.Publish(MessageLevel.Error, "HistorianDataRead", "Failed while reading data from the historian", exception: ex);
}
finally
{
SetButtonsEnabledState(true);
}
}