예제 #1
0
파일: MainForm.cs 프로젝트: sotaria/gsf
        private void ExportData()
        {
            try
            {
                const int TargetBufferSize = 524288;

                Dictionary <string, DeviceDetail> deviceMap   = new Dictionary <string, DeviceDetail>(StringComparer.OrdinalIgnoreCase);
                Dictionary <Guid, DeviceDetail>   signalMap   = new Dictionary <Guid, DeviceDetail>();
                Dictionary <Guid, DeviceStats>    deviceStats = new Dictionary <Guid, DeviceStats>();

                string   hostAddress           = m_settings.HostAddress;
                int      port                  = m_settings.Port;
                DateTime startTime             = m_settings.StartTime;
                DateTime endTime               = m_settings.EndTime;
                int      frameRate             = m_settings.FrameRate;
                bool     alignTimestamps       = m_settings.AlignTimestamps;
                bool     missingAsNaN          = m_settings.ExportMissingAsNaN;
                bool     fillMissingTimestamps = alignTimestamps && m_settings.FillInMissingTimestamps;
                int      messageInterval       = m_settings.MessageInterval;
                bool     exportFilePerDataType = m_settings.ExportFilePerDataType;
                string   exportFileName        = m_settings.ExportFileName;
                double   timeRange             = (endTime - startTime).TotalSeconds;
                int      selectedDeviceCount   = SelectedDeviceCount;

                Ticks                operationStartTime;
                StringBuilder        readBuffer      = new StringBuilder(TargetBufferSize * 2);
                ManualResetEventSlim bufferReady     = new ManualResetEventSlim(false);
                List <string>        writeBuffer     = new List <string>();
                object               writeBufferLock = new object();

                TextWriter writer         = null;
                bool       readComplete   = false;
                bool       writeComplete  = false;
                long       receivedPoints = 0L;
                int        exports        = 0;
                int        totalExports;

                Ticks[] subseconds = Ticks.SubsecondDistribution(frameRate);
                long    interval   = subseconds.Length > 1 ? subseconds[1].Value : Ticks.PerSecond;

                Dictionary <Guid, int> signalIDIndex = new Dictionary <Guid, int>();
                double[] values           = {};
                int      deviceCount      = 0;
                int      measurementCount = 0;
                long     lastTimestamp    = 0L;
                long     timestamp        = 0L;
                bool     pastFirstRow     = false;

                void bufferValues()
                {
                    // Write row values
                    readBuffer.Append(missingAsNaN ? string.Join(",", values) : string.Join(",", values.Select(val => double.IsNaN(val) ? "" : $"{val}")));

                    if (readBuffer.Length < TargetBufferSize)
                    {
                        return;
                    }

                    lock (writeBufferLock)
                        writeBuffer.Add(readBuffer.ToString());

                    readBuffer.Clear();
                    bufferReady.Set();
                }

                void writeData()
                {
                    string[] localBuffer;

                    while (writeBuffer.Count > 0 || !writeComplete)
                    {
                        bufferReady.Wait();
                        bufferReady.Reset();

                        lock (writeBufferLock)
                        {
                            localBuffer = writeBuffer.ToArray();
                            writeBuffer.Clear();
                        }

                        foreach (string buffer in localBuffer)
                        {
                            writer?.Write(buffer);
                        }
                    }
                }

                string getHeaders(HashSet <string> signalTypes)
                {
                    StringBuilder headers = new StringBuilder();

                    headers.AppendLine($"Data extraction from \"{hostAddress}:{port}\" exported on {DateTime.UtcNow.ToString(TimeTagBase.DefaultFormat)} UTC");
                    headers.AppendLine($"Export range: {startTime.ToString(TimeTagBase.DefaultFormat)} UTC to {endTime.ToString(TimeTagBase.DefaultFormat)} UTC");
                    headers.AppendLine($"Signal types: {string.Join(", ", signalTypes)}");

                    StringBuilder deviceRow      = new StringBuilder();
                    StringBuilder measurementRow = new StringBuilder();

                    measurementRow.Append("\"Timestamp\"");
                    signalIDIndex.Clear();

                    foreach (DeviceDetail device in m_metadata.Devices.Where(d => d.Selected))
                    {
                        MeasurementDetail[] deviceMeasurements = m_metadata.Measurements.Where(m => string.Equals(m.DeviceName, device.Name, StringComparison.OrdinalIgnoreCase) && signalTypes.Contains(m.SignalAcronym)).ToArray();

                        if (deviceMeasurements.Length == 0)
                        {
                            continue;
                        }

                        deviceRow.Append($",{device.Name}");
                        deviceCount++;

                        for (int i = 0; i < deviceMeasurements.Length; i++)
                        {
                            MeasurementDetail measurement = deviceMeasurements[i];

                            if (i > 0)
                            {
                                deviceRow.Append(",");
                            }

                            measurementRow.Append($",{measurement.PointTag} [{measurement.SignalAcronym}]");
                            signalIDIndex.Add(measurement.SignalID, measurementCount++);
                        }
                    }

                    headers.AppendLine($"Device count: {deviceCount}");
                    headers.AppendLine($"Measurement count: {measurementCount}");
                    headers.AppendLine();

                    headers.AppendLine(deviceRow.ToString());
                    headers.Append(measurementRow);

                    values = new double[measurementCount];

                    for (int i = 0; i < values.Length; i++)
                    {
                        values[i] = double.NaN;
                    }

                    return(headers.ToString());
                }

                void handleNewMeasurements(ICollection <IMeasurement> measurements)
                {
                    bool showMessage = receivedPoints + measurements.Count >= (receivedPoints / messageInterval + 1) * messageInterval;

                    receivedPoints += measurements.Count;

                    foreach (IMeasurement measurement in measurements)
                    {
                        if (signalMap.TryGetValue(measurement.Key.SignalID, out DeviceDetail device) && deviceStats.TryGetValue(device.UniqueID, out DeviceStats stats))
                        {
                            stats.Total++;

                            if (!measurement.ValueQualityIsGood())
                            {
                                stats.BadDataCount++;
                            }

                            if (!measurement.TimestampQualityIsGood())
                            {
                                stats.BadTimeCount++;
                            }
                        }

                        if (signalIDIndex.TryGetValue(measurement.ID, out int index))
                        {
                            if (alignTimestamps)
                            {
                                timestamp = Ticks.RoundToSubsecondDistribution(measurement.Timestamp, frameRate).Value;
                            }
                            else
                            {
                                timestamp = measurement.Timestamp;
                            }

                            // Start a new row for each encountered new timestamp
                            if (timestamp != lastTimestamp)
                            {
                                if (lastTimestamp > 0 && pastFirstRow)
                                {
                                    bufferValues();
                                }

                                for (int i = 0; i < values.Length; i++)
                                {
                                    values[i] = float.NaN;
                                }

                                // Handle any missing data rows
                                if (fillMissingTimestamps && lastTimestamp > 0 && timestamp > lastTimestamp)
                                {
                                    long difference = timestamp - lastTimestamp;

                                    if (difference > interval)
                                    {
                                        long interpolated = lastTimestamp;

                                        for (long i = 1; i < difference / interval; i++)
                                        {
                                            interpolated = Ticks.RoundToSubsecondDistribution(interpolated + interval, frameRate).Value;
                                            readBuffer.Append($"{Environment.NewLine}{new DateTime(interpolated, DateTimeKind.Utc).ToString(TimeTagBase.DefaultFormat)},");
                                            bufferValues();
                                        }
                                    }
                                }

                                readBuffer.Append($"{Environment.NewLine}{new DateTime(timestamp, DateTimeKind.Utc).ToString(TimeTagBase.DefaultFormat)},");
                                lastTimestamp = timestamp;
                                pastFirstRow  = true;
                            }

                            // Save value to its column
                            values[index] = measurement.AdjustedValue;
                        }
                    }

                    if (showMessage && measurements.Count > 0)
                    {
                        IMeasurement measurement = measurements.Last();
                        ShowUpdateMessage($"{Environment.NewLine}{receivedPoints:N0} points read so far averaging {receivedPoints / (DateTime.UtcNow.Ticks - operationStartTime).ToSeconds():N0} points per second.");

                        int    exportFraction        = (int)(100.0D * exports / totalExports);
                        double currentExportProgress = 1.0D - (endTime.Ticks - measurement.Timestamp).ToSeconds() / timeRange;

                        UpdateProgressBar(exportFraction + (int)(currentExportProgress / totalExports * 100.0D));
                    }
                }

                void flushBuffers()
                {
                    // Flush last row
                    if (timestamp > 0)
                    {
                        bufferValues();
                    }

                    // Flush remaining buffers
                    if (readBuffer.Length > 0)
                    {
                        lock (writeBufferLock)
                            writeBuffer.Add(readBuffer.ToString());
                    }
                }

                void readCompleted()
                {
                    readComplete = true;

                    if (++exports == totalExports)
                    {
                        ShowUpdateMessage($"Data read{(totalExports > 1 ? "s" : "" )} completed.");
                    }
                    else
                    {
                        ShowUpdateMessage($"{exports} of {totalExports} data reads completed.");
                    }

                    UpdateProgressBar((int)(100.0D * exports / totalExports));
                }

                void exportData(HashSet <string> signalTypes, string suffix = null)
                {
                    string fileName;

                    if (string.IsNullOrEmpty(suffix))
                    {
                        fileName = exportFileName;
                    }
                    else
                    {
                        fileName = $"{FilePath.GetDirectoryName(exportFileName)}{FilePath.GetFileNameWithoutExtension(exportFileName)}_{suffix}{FilePath.GetExtension(exportFileName)}";
                    }

                    readComplete     = false;
                    lastTimestamp    = 0L;
                    deviceCount      = 0;
                    measurementCount = 0;
                    pastFirstRow     = false;
                    readBuffer.Clear();
                    writeBuffer.Clear();

                    if (File.Exists(fileName))
                    {
                        File.Delete(fileName);
                    }

                    using (writer = File.CreateText(fileName))
                    {
                        writer.Write(getHeaders(signalTypes));
                        writeComplete = false;
                        bufferReady.Reset();

                        Thread writeThread = new Thread(writeData);
                        writeThread.Start();

                        try
                        {
                            if (measurementCount > 0)
                            {
                                ShowUpdateMessage($"\nStarting data read for {string.Join(", ", signalTypes)} signal type{(signalTypes.Count > 1 ? "s" : "")}...\n");

                                using (new DataReceiver($"server={hostAddress}; port={port}; interface=0.0.0.0", GenerateFilterExpression(signalTypes, selectedDeviceCount), startTime, endTime + TimeSpan.FromTicks(interval))
                                {
                                    NewMeasurementsCallback = handleNewMeasurements,
                                    StatusMessageCallback = ShowUpdateMessage,
                                    ProcessExceptionCallback = ex => ShowUpdateMessage($"Error: {ex.Message}"),
                                    ReadCompletedCallback = readCompleted
                                })
                                {
                                    while (!m_formClosing && !readComplete && m_exporting) //-V3063
                                    {
                                        Thread.Sleep(500);
                                    }
                                }

                                flushBuffers();
                            }
                        }
                        finally
                        {
                            writeComplete = true;
                            bufferReady.Set();
                        }

                        writeThread.Join(5000);
                    }
                }

                // Start export operations
                HashSet <string> selectedSignalTypes = new HashSet <string>(StringComparer.OrdinalIgnoreCase);

                foreach (DeviceDetail device in m_metadata.Devices.Where(d => d.Selected))
                {
                    deviceMap[device.Name] = device;
                }

                foreach (MeasurementDetail measurement in m_metadata.Measurements)
                {
                    if (deviceMap.TryGetValue(measurement.DeviceName, out DeviceDetail device))
                    {
                        signalMap[measurement.SignalID] = device;
                    }
                }

                foreach (DeviceDetail device in m_metadata.Devices.Where(d => d.Selected))
                {
                    deviceStats[device.UniqueID] = new DeviceStats {
                        Device = device
                    }
                }
                ;

                if (checkedListBoxDataTypes.CheckedItems.Count > 0)
                {
                    selectedSignalTypes.UnionWith(checkedListBoxDataTypes.CheckedItems.Cast <string>());
                }
                else
                {
                    selectedSignalTypes.UnionWith(checkedListBoxDataTypes.Items.Cast <string>());
                }

                operationStartTime = DateTime.UtcNow.Ticks;

                if (exportFilePerDataType)
                {
                    totalExports = selectedSignalTypes.Count;

                    foreach (string signalType in selectedSignalTypes)
                    {
                        exportData(new HashSet <string>(new[] { signalType }, StringComparer.OrdinalIgnoreCase), signalType);
                    }
                }
                else
                {
                    totalExports = 1; //-V3137
                    exportData(selectedSignalTypes);
                }

                long expectedPoints = (long)(frameRate * timeRange);

                // Show export summary information
                foreach (DeviceStats stats in deviceStats.Values)
                {
                    DeviceDetail device = stats.Device;
                    stats.MissingDataCount = expectedPoints - stats.Total;
                    double badData, badTime, missingData;

                    if (stats.Total == 0)
                    {
                        badTime     = 0.0D;
                        badData     = 0.0D;
                        missingData = 1.0D;
                    }
                    else
                    {
                        badData     = stats.BadDataCount / (double)stats.Total;
                        badTime     = stats.BadTimeCount / (double)stats.Total;
                        missingData = stats.MissingDataCount / (double)stats.Total;

                        if (badData < 0.0D)
                        {
                            badData = 0.0D;
                        }

                        if (badTime < 0.0D)
                        {
                            badTime = 0.0D;
                        }

                        if (missingData < 0.0D)
                        {
                            missingData = 0.0D;
                        }
                    }

                    ShowUpdateMessage($"Device \"{device.Name}\" bad data: {badData:0.00%}, bad time: {badTime:0.00%}, missing data: {missingData:0.00%}...");
                }

                Ticks operationTime = DateTime.UtcNow.Ticks - operationStartTime;

                if (m_formClosing || !m_exporting)
                {
                    ShowUpdateMessage("*** Data Export Canceled ***");
                    UpdateProgressBar(0);
                }
                else
                {
                    ShowUpdateMessage("*** Data Export Complete ***");
                    UpdateProgressBar(100);
                }

                ShowUpdateMessage($"Total export processing time {operationTime.ToElapsedTimeString(3)} at {receivedPoints / operationTime.ToSeconds():N0} points per second.{Environment.NewLine}");
            }
            catch (Exception ex)
            {
                ShowUpdateMessage($"!!! Failure during historian read: {ex.Message}");
                m_log.Publish(MessageLevel.Error, "HistorianDataRead", "Failed while reading data from the historian", exception: ex);
            }
            finally
            {
                SetButtonsEnabledState(true);
            }
        }
예제 #2
0
파일: MainForm.cs 프로젝트: sotaria/gsf
        // Internal Functions

        private void PreFilter()
        {
            //const int MaxPoints = 50;

            try
            {
                double timeRange = (m_settings.EndTime - m_settings.StartTime).TotalSeconds;
                Dictionary <string, DeviceDetail> deviceMap   = new Dictionary <string, DeviceDetail>(StringComparer.OrdinalIgnoreCase);
                Dictionary <Guid, DeviceDetail>   signalMap   = new Dictionary <Guid, DeviceDetail>();
                Dictionary <Guid, DeviceStats>    deviceStats = new Dictionary <Guid, DeviceStats>();
                //Dictionary<Guid, Tuple<Ticks, List<double>, List<double>>> plotValues = new Dictionary<Guid, Tuple<Ticks, List<double>, List<double>>>();
                bool  readComplete   = false;
                long  receivedPoints = 0L;
                Ticks operationTime;
                Ticks operationStartTime;
                //double pointInterval = timeRange / MaxPoints;

                void handleNewMeasurements(ICollection <IMeasurement> measurements)
                {
                    bool showMessage = receivedPoints + measurements.Count >= (receivedPoints / m_settings.MessageInterval + 1) * m_settings.MessageInterval;

                    receivedPoints += measurements.Count;

                    foreach (IMeasurement measurement in measurements)
                    {
                        Guid signalID = measurement.ID;

                        if (signalMap.TryGetValue(signalID, out DeviceDetail device) && deviceStats.TryGetValue(device.UniqueID, out DeviceStats stats))
                        {
                            stats.Total++;

                            if (!measurement.ValueQualityIsGood())
                            {
                                stats.BadDataCount++;
                            }

                            if (!measurement.TimestampQualityIsGood())
                            {
                                stats.BadTimeCount++;
                            }
                        }

                        //Tuple<Ticks, List<double>, List<double>> plotData = plotValues.GetOrAdd(signalID, _ => new Tuple<Ticks, List<double>, List<double>>(measurement.Timestamp, new List<double>(new[] { (double)measurement.Timestamp }), new List<double>(new[] { measurement.AdjustedValue })));

                        //if ((measurement.Timestamp - plotData.Item1).ToSeconds() > pointInterval)
                        //{
                        //    plotData.Item2.Add(measurement.Timestamp);
                        //    plotData.Item3.Add(measurement.AdjustedValue);
                        //    plotValues[signalID] = new Tuple<Ticks, List<double>, List<double>>(measurement.Timestamp, plotData.Item2, plotData.Item3);
                        //}
                    }

                    if (showMessage && measurements.Count > 0)
                    {
                        IMeasurement measurement = measurements.Last();
                        ShowUpdateMessage($"{Environment.NewLine}{receivedPoints:N0} points read so far averaging {receivedPoints / (DateTime.UtcNow.Ticks - operationStartTime).ToSeconds():N0} points per second.");
                        UpdateProgressBar((int)((1.0D - new Ticks(m_settings.EndTime.Ticks - (long)measurement.Timestamp).ToSeconds() / timeRange) * 100.0D));
                    }
                }

                void readCompleted()
                {
                    readComplete = true;
                    ShowUpdateMessage("Data read completed.");

                    //foreach (KeyValuePair<Guid, Tuple<Ticks, List<double>, List<double>>> plotData in plotValues)
                    //    m_graphData.PlotLine(plotData.Value.Item2, plotData.Value.Item3);
                }

                operationStartTime = DateTime.UtcNow.Ticks;

                foreach (DeviceDetail device in m_metadata.Devices.Where(d => d.Selected))
                {
                    deviceMap[device.Name] = device;
                }

                foreach (MeasurementDetail measurement in m_metadata.Measurements)
                {
                    if (deviceMap.TryGetValue(measurement.DeviceName, out DeviceDetail device))
                    {
                        signalMap[measurement.SignalID] = device;
                    }
                }

                foreach (DeviceDetail device in m_metadata.Devices.Where(d => d.Selected))
                {
                    deviceStats[device.UniqueID] = new DeviceStats {
                        Device = device
                    }
                }
                ;

                //m_graphData.ClearPlots();

                using (new DataReceiver($"server={m_settings.HostAddress}; port={m_settings.Port}; interface=0.0.0.0", m_settings.FilterExpression, m_settings.StartTime, m_settings.EndTime)
                {
                    NewMeasurementsCallback = handleNewMeasurements,
                    StatusMessageCallback = ShowUpdateMessage,
                    ProcessExceptionCallback = ex => ShowUpdateMessage($"Error: {ex.Message}"),
                    ReadCompletedCallback = readCompleted
                })
                {
                    while (!m_formClosing && !readComplete && m_prefiltering)
                    {
                        Thread.Sleep(500);
                    }
                }

                long expectedPoints = (long)(m_settings.FrameRate * timeRange);

                foreach (DeviceStats stats in deviceStats.Values)
                {
                    DeviceDetail device = stats.Device;
                    stats.MissingDataCount = expectedPoints - stats.Total;
                    double badData, badTime, missingData;

                    if (stats.Total == 0)
                    {
                        badTime     = 0.0D;
                        badData     = 0.0D;
                        missingData = 1.0D;
                    }
                    else
                    {
                        badData     = stats.BadDataCount / (double)stats.Total;
                        badTime     = stats.BadTimeCount / (double)stats.Total;
                        missingData = stats.MissingDataCount / (double)stats.Total;

                        if (badData < 0.0D)
                        {
                            badData = 0.0D;
                        }

                        if (badTime < 0.0D)
                        {
                            badTime = 0.0D;
                        }

                        if (missingData < 0.0D)
                        {
                            missingData = 0.0D;
                        }
                    }

                    if (stats.BadDataCount / (double)stats.Total * 100.0D > m_settings.AcceptableBadData)
                    {
                        device.Selected = false;
                        ShowUpdateMessage($"Device \"{device.Name}\" unselected - too much bad data: {badData:0.00%}...");
                    }
                    else if (stats.BadTimeCount / (double)stats.Total * 100.0D > m_settings.AcceptableBadTime)
                    {
                        device.Selected = false;
                        ShowUpdateMessage($"Device \"{device.Name}\" unselected - too much bad data with bad time: {badTime:0.00%}...");
                    }
                    else if (stats.MissingDataCount / (double)stats.Total * 100.0D > m_settings.AcceptableMissingData)
                    {
                        device.Selected = false;
                        ShowUpdateMessage($"Device \"{device.Name}\" unselected - too much missing data: {missingData:0.00%}...");
                    }
                }

                RefreshSelectedCount();
                RefreshDevicesDataGrid();

                operationTime = DateTime.UtcNow.Ticks - operationStartTime;

                if (m_formClosing || !m_prefiltering)
                {
                    ShowUpdateMessage("*** Data Pre-filter Canceled ***");
                    UpdateProgressBar(0);
                }
                else
                {
                    ShowUpdateMessage("*** Data Pre-filter Complete ***");
                    UpdateProgressBar(100);
                }

                ShowUpdateMessage($"Total pre-filter processing time {operationTime.ToElapsedTimeString(3)} at {receivedPoints / operationTime.ToSeconds():N0} points per second.{Environment.NewLine}");
            }
            catch (Exception ex)
            {
                ShowUpdateMessage($"!!! Failure during historian read: {ex.Message}");
                m_log.Publish(MessageLevel.Error, "HistorianDataRead", "Failed while reading data from the historian", exception: ex);
            }
            finally
            {
                SetButtonsEnabledState(true);
            }
        }