Represents time-series data stored in ArchiveFile.
Inheritance: IDataPoint
Exemple #1
0
        /// <summary>
        /// Compares the current <see cref="ArchiveDataPoint"/> object to <paramref name="obj"/>.
        /// </summary>
        /// <param name="obj">Object against which the current <see cref="ArchiveDataPoint"/> object is to be compared.</param>
        /// <returns>
        /// Negative value if the current <see cref="ArchiveDataPoint"/> object is less than <paramref name="obj"/>,
        /// Zero if the current <see cref="ArchiveDataPoint"/> object is equal to <paramref name="obj"/>,
        /// Positive value if the current <see cref="ArchiveDataPoint"/> object is greater than <paramref name="obj"/>.
        /// </returns>
        public virtual int CompareTo(object obj)
        {
            ArchiveDataPoint other = obj as ArchiveDataPoint;

            if (other == null)
            {
                return(1);
            }
            else
            {
                int result = m_historianID.CompareTo(other.HistorianID);
                if (result != 0)
                {
                    return(result);
                }
                else
                {
                    return(m_time.CompareTo(other.Time));
                }
            }
        }
Exemple #2
0
        /// <summary>
        /// Reads existing <see cref="ArchiveDataPoint"/>s from the <see cref="ArchiveDataBlock"/>.
        /// </summary>
        /// <returns>Returns <see cref="ArchiveDataPoint"/>s from the <see cref="ArchiveDataBlock"/>.</returns>
        public IEnumerable <IDataPoint> Read()
        {
            ArchiveDataPoint dataPoint;

            lock (m_parent.FileData)
            {
                // We'll start reading from where the data block begins.
                m_parent.FileData.Seek(Location, SeekOrigin.Begin);

                for (int i = 0; i < Capacity; i++)
                {
                    // Read the data in the block.
                    m_lastActivityTime = DateTime.UtcNow;
                    m_parent.FileData.Read(m_readBuffer, 0, m_readBuffer.Length);

                    // Attempt to parse archive data point
                    try
                    {
                        dataPoint = new ArchiveDataPoint(m_historianID, m_readBuffer, 0, m_readBuffer.Length);
                    }
                    catch (Exception ex)
                    {
                        dataPoint = null;
                        OnDataReadException(ex);
                    }

                    if (dataPoint != null && !dataPoint.IsEmpty)
                    {
                        // There is data - use it.
                        m_writeCursor = m_parent.FileData.Position;
                        yield return(dataPoint);
                    }
                    else
                    {
                        // Data is empty - stop reading.
                        yield break;
                    }
                }
            }
        }
        /// <summary>
        /// Reads existing <see cref="ArchiveDataPoint"/>s from the <see cref="ArchiveDataBlock"/>.
        /// </summary>
        /// <returns>Returns <see cref="ArchiveDataPoint"/>s from the <see cref="ArchiveDataBlock"/>.</returns>
        public IEnumerable<IDataPoint> Read()
        {
            ArchiveDataPoint dataPoint;

            lock (m_parent.FileData)
            {
                // We'll start reading from where the data block begins.
                m_parent.FileData.Seek(Location, SeekOrigin.Begin);

                for (int i = 0; i < Capacity; i++)
                {
                    // Read the data in the block.
                    m_lastActivityTime = DateTime.UtcNow;
                    m_parent.FileData.Read(m_readBuffer, 0, m_readBuffer.Length);

                    // Attempt to parse archive data point
                    try
                    {
                        dataPoint = new ArchiveDataPoint(m_historianID, m_readBuffer, 0, m_readBuffer.Length);
                    }
                    catch (Exception ex)
                    {
                        dataPoint = null;
                        OnDataReadException(ex);
                    }

                    if (dataPoint != null && !dataPoint.IsEmpty)
                    {
                        // There is data - use it.
                        m_writeCursor = m_parent.FileData.Position;
                        yield return dataPoint;
                    }
                    else
                    {
                        // Data is empty - stop reading.
                        yield break;
                    }
                }
            }
        }
Exemple #4
0
        private void WriteToCurrentArchiveFile(IDataPoint[] items)
        {
            Dictionary<int, List<IDataPoint>> sortedDataPoints = new Dictionary<int, List<IDataPoint>>();

            // First we'll separate all point data by ID.
            for (int i = 0; i < items.Length; i++)
            {
                if (!sortedDataPoints.ContainsKey(items[i].HistorianID))
                    sortedDataPoints.Add(items[i].HistorianID, new List<IDataPoint>());

                sortedDataPoints[items[i].HistorianID].Add(items[i]);
            }

            IntercomRecord system = m_intercomFile.Read(1);

            foreach (int pointID in sortedDataPoints.Keys)
            {
                // Initialize local variables.
                StateRecord state = m_stateFile.Read(pointID);
                MetadataRecord metadata = m_metadataFile.Read(pointID);
                IDataPoint dataPoint;

                for (int i = 0; i < sortedDataPoints[pointID].Count; i++)
                {
                    dataPoint = sortedDataPoints[pointID][i];

                    // Ensure that the received data is to be archived.
                    if ((object)state == null || (object)metadata == null || !metadata.GeneralFlags.Enabled)
                    {
                        OnOrphanDataReceived(dataPoint);
                        continue;
                    }

                    // Ensure that data is not far out in to the future.
                    if (dataPoint.Time > DateTime.UtcNow.AddMinutes(m_leadTimeTolerance))
                    {
                        OnFutureDataReceived(dataPoint);
                        continue;
                    }

                    // Perform quality check if data quality is not set.
                    if ((int)dataPoint.Quality == 31)
                    {
                        // Note: Here we're checking if the Quality is 31 instead of -1 because the quality value is stored
                        // in the first 5 bits (QualityMask = 31) of Flags in the point data. Initially when the Quality is
                        // set to -1, all the bits Flags (a 32-bit integer) are set to 1. And therefore, when we get the
                        // Quality, which is a masked value of Flags, we get 31 and not -1.
                        switch (metadata.GeneralFlags.DataType)
                        {
                            case DataType.Analog:
                                if (dataPoint.Value >= metadata.AnalogFields.HighRange)
                                    dataPoint.Quality = Quality.UnreasonableHigh;
                                else if (dataPoint.Value >= metadata.AnalogFields.HighAlarm)
                                    dataPoint.Quality = Quality.ValueAboveHiHiAlarm;
                                else if (dataPoint.Value >= metadata.AnalogFields.HighWarning)
                                    dataPoint.Quality = Quality.ValueAboveHiAlarm;
                                else if (dataPoint.Value <= metadata.AnalogFields.LowRange)
                                    dataPoint.Quality = Quality.UnreasonableLow;
                                else if (dataPoint.Value <= metadata.AnalogFields.LowAlarm)
                                    dataPoint.Quality = Quality.ValueBelowLoLoAlarm;
                                else if (dataPoint.Value <= metadata.AnalogFields.LowWarning)
                                    dataPoint.Quality = Quality.ValueBelowLoAlarm;
                                else
                                    dataPoint.Quality = Quality.Good;
                                break;
                            case DataType.Digital:
                                if ((int)dataPoint.Value == metadata.DigitalFields.AlarmState)
                                    dataPoint.Quality = Quality.LogicalAlarm;
                                else
                                    dataPoint.Quality = Quality.Good;
                                break;
                        }
                    }

                    // Update information about the latest data point received.
                    if (dataPoint.Time.CompareTo(system.LatestDataTime) > 0)
                    {
                        system.LatestDataID = dataPoint.HistorianID;
                        system.LatestDataTime = dataPoint.Time;
                        m_intercomFile.Write(1, system);
                    }

                    // Check for data that out-of-sequence based on it's time.
                    if (dataPoint.Time.CompareTo(state.PreviousData.Time) <= 0)
                    {
                        if (dataPoint.Time == state.PreviousData.Time)
                        {
                            // Discard data that is an exact duplicate of data in line for archival.
                            if (dataPoint.Value == state.PreviousData.Value && dataPoint.Quality == state.PreviousData.Quality)
                                continue;
                        }
                        else
                        {
                            // Queue out-of-sequence data for processing if it is not be discarded.
                            if (!m_discardOutOfSequenceData)
                                m_outOfSequenceDataQueue.Add(dataPoint);

                            OnOutOfSequenceDataReceived(dataPoint);
                            continue;
                        }
                    }

                    // [BEGIN]   Data compression
                    bool archiveData = false;
                    bool calculateSlopes = false;
                    float compressionLimit = metadata.AnalogFields.CompressionLimit;

                    // Set the compression limit to a very low number for digital points.
                    if (metadata.GeneralFlags.DataType == DataType.Digital)
                        compressionLimit = 0.000000001f;

                    state.CurrentData = new StateRecordDataPoint(dataPoint);

                    if (state.ArchivedData.IsEmpty)
                    {
                        // This is the first time data is received.
                        state.CurrentData = new StateRecordDataPoint(-1);
                        archiveData = true;
                    }
                    else if (state.PreviousData.IsEmpty)
                    {
                        // This is the second time data is received.
                        calculateSlopes = true;
                    }
                    else
                    {
                        // Process quality-based alarming if enabled.
                        if (metadata.GeneralFlags.AlarmEnabled)
                        {
                            if (metadata.AlarmFlags.Value.CheckBits(BitExtensions.BitVal((int)state.CurrentData.Quality)))
                            {
                                // Current data quality warrants alarming based on the alarming settings.
                                decimal delay = 0;
                                switch (metadata.GeneralFlags.DataType)
                                {
                                    case DataType.Analog:
                                        delay = (decimal)metadata.AnalogFields.AlarmDelay;
                                        break;
                                    case DataType.Digital:
                                        delay = (decimal)metadata.DigitalFields.AlarmDelay;
                                        break;
                                }

                                // Dispatch the alarm immediately or after a given time based on settings.
                                if (delay > 0)
                                {
                                    // Wait before dispatching alarm.
                                    decimal first;
                                    if (m_delayedAlarmProcessing.TryGetValue(dataPoint.HistorianID, out first))
                                    {
                                        if (state.CurrentData.Time.Value - first > delay)
                                        {
                                            // Wait is now over, dispatch the alarm.
                                            m_delayedAlarmProcessing.Remove(dataPoint.HistorianID);
                                            OnProcessAlarmNotification(state);
                                        }
                                    }
                                    else
                                    {
                                        m_delayedAlarmProcessing.Add(state.HistorianID, state.CurrentData.Time.Value);
                                    }
                                }
                                else
                                {
                                    // Dispatch the alarm immediately.
                                    OnProcessAlarmNotification(state);
                                }
                            }
                            else
                            {
                                m_delayedAlarmProcessing.Remove(dataPoint.HistorianID);
                            }
                        }

                        if (m_compressData)
                        {
                            // Data is to be compressed.
                            if (metadata.CompressionMinTime > 0 && state.CurrentData.Time.Value - state.ArchivedData.Time.Value < metadata.CompressionMinTime)
                            {
                                // CompressionMinTime is in effect.
                                archiveData = false;
                                calculateSlopes = false;
                            }
                            else if (state.CurrentData.Quality != state.ArchivedData.Quality || state.CurrentData.Quality != state.PreviousData.Quality || (metadata.CompressionMaxTime > 0 && state.PreviousData.Time.Value - state.ArchivedData.Time.Value > metadata.CompressionMaxTime))
                            {
                                // Quality changed or CompressionMaxTime is exceeded.
                                dataPoint = new ArchiveDataPoint(state.PreviousData);
                                archiveData = true;
                                calculateSlopes = true;
                            }
                            else
                            {
                                // Perform a compression test.
                                double slope1;
                                double slope2;
                                double currentSlope;

                                slope1 = (state.CurrentData.Value - (state.ArchivedData.Value + compressionLimit)) / (double)(state.CurrentData.Time.Value - state.ArchivedData.Time.Value);
                                slope2 = (state.CurrentData.Value - (state.ArchivedData.Value - compressionLimit)) / (double)(state.CurrentData.Time.Value - state.ArchivedData.Time.Value);
                                currentSlope = (state.CurrentData.Value - state.ArchivedData.Value) / (double)(state.CurrentData.Time.Value - state.ArchivedData.Time.Value);

                                if (slope1 >= state.Slope1)
                                    state.Slope1 = slope1;

                                if (slope2 <= state.Slope2)
                                    state.Slope2 = slope2;

                                if (currentSlope <= state.Slope1 || currentSlope >= state.Slope2)
                                {
                                    dataPoint = new ArchiveDataPoint(state.PreviousData);
                                    archiveData = true;
                                    calculateSlopes = true;
                                }
                            }
                        }
                        else
                        {
                            // Data is not to be compressed.
                            dataPoint = new ArchiveDataPoint(state.PreviousData);
                            archiveData = true;
                        }
                    }
                    // [END]     Data compression

                    // [BEGIN]   Data archival
                    m_fat.DataPointsReceived++;

                    if (archiveData)
                    {
                        if (dataPoint.Time.CompareTo(m_fat.FileStartTime) >= 0)
                        {
                            // Data belongs to this file.
                            ArchiveDataBlock dataBlock;
                            lock (m_dataBlocks)
                            {
                                dataBlock = m_dataBlocks[dataPoint.HistorianID - 1];
                            }

                            if ((object)dataBlock == null || dataBlock.SlotsAvailable == 0)
                            {
                                // Need to find a data block for writting the data.
                                if ((object)dataBlock != null)
                                    state.ActiveDataBlockIndex = -1;

                                if (state.ActiveDataBlockIndex >= 0)
                                {
                                    // Retrieve previously used data block.
                                    dataBlock = m_fat.RequestDataBlock(dataPoint.HistorianID, dataPoint.Time, state.ActiveDataBlockIndex);
                                }
                                else
                                {
                                    // Time to request a brand new data block.
                                    dataBlock = m_fat.RequestDataBlock(dataPoint.HistorianID, dataPoint.Time, system.DataBlocksUsed);
                                }

                                if ((object)dataBlock != null)
                                {
                                    // Update the total number of data blocks used.
                                    if (dataBlock.SlotsUsed == 0 && system.DataBlocksUsed == dataBlock.Index)
                                    {
                                        system.DataBlocksUsed++;
                                        m_intercomFile.Write(1, system);
                                    }

                                    // Update the active data block index information.
                                    state.ActiveDataBlockIndex = dataBlock.Index;
                                }

                                // Keep in-memory reference to the data block for consecutive writes.
                                lock (m_dataBlocks)
                                {
                                    m_dataBlocks[dataPoint.HistorianID - 1] = dataBlock;
                                }

                                // Kick-off the rollover preparation when its threshold is reached.
                                if (Statistics.FileUsage >= m_rolloverPreparationThreshold && !File.Exists(StandbyArchiveFileName) && !m_rolloverPreparationThread.IsAlive)
                                {
                                    m_rolloverPreparationThread = new Thread(PrepareForRollover);
                                    m_rolloverPreparationThread.Priority = ThreadPriority.Lowest;
                                    m_rolloverPreparationThread.Start();
                                }
                            }

                            if ((object)dataBlock != null)
                            {
                                // Write data to the data block.
                                dataBlock.Write(dataPoint);
                                m_fat.DataPointsArchived++;
                            }
                            else
                            {
                                OnFileFull();   // Current file is full.

                                m_fat.DataPointsReceived--;
                                while (true)
                                {
                                    Rollover(); // Rollover current file.
                                    if (m_rolloverWaitHandle.WaitOne(1, false))
                                        break;  // Rollover is successful.
                                }

                                i--;                                // Process current data point again.
                                system = m_intercomFile.Read(1);    // Re-read modified intercom record.
                                continue;
                            }
                        }
                        else
                        {
                            // Data is historic.
                            m_fat.DataPointsReceived--;
                            m_historicDataQueue.Add(dataPoint);
                            OnHistoricDataReceived(dataPoint);
                        }

                        state.ArchivedData = new StateRecordDataPoint(dataPoint);
                    }

                    if (calculateSlopes)
                    {
                        if (state.CurrentData.Time.Value != state.ArchivedData.Time.Value)
                        {
                            state.Slope1 = (state.CurrentData.Value - (state.ArchivedData.Value + compressionLimit)) / (double)(state.CurrentData.Time.Value - state.ArchivedData.Time.Value);
                            state.Slope2 = (state.CurrentData.Value - (state.ArchivedData.Value - compressionLimit)) / (double)(state.CurrentData.Time.Value - state.ArchivedData.Time.Value);
                        }
                        else
                        {
                            state.Slope1 = 0;
                            state.Slope2 = 0;
                        }
                    }

                    state.PreviousData = state.CurrentData;

                    // Write state information to the file.
                    m_stateFile.Write(state.HistorianID, state);
                    // [END]     Data archival
                }
            }
        }
        private IEnumerable<IDataPoint> ReadDataStream(TreeStream<HistorianKey, HistorianValue> stream)
        {
            HistorianKey key = new HistorianKey();
            HistorianValue value = new HistorianValue();

            List<ArchiveDataPoint> queriedData = new List<ArchiveDataPoint>();
            ArchiveDataPoint point;
            MeasurementStateFlags stateFlags;

            while (stream.Read(key, value))
            {
                point = new ArchiveDataPoint((int)key.PointID);
                point.Time = new TimeTag(new DateTime((long)key.Timestamp));
                point.Value = BitConvert.ToSingle(value.Value1);

                stateFlags = (MeasurementStateFlags)value.Value3;

                if ((stateFlags & MeasurementStateFlags.BadData) == 0)
                {
                    if ((stateFlags & MeasurementStateFlags.BadTime) == 0)
                        point.Quality = Quality.Good;
                    else
                        point.Quality = Quality.Old;
                }
                else
                {
                    point.Quality = Quality.SuspectData;
                }

                queriedData.Add(point);
            }

            return queriedData;
        }
Exemple #6
0
        private bool ValidateOutputFormat()
        {
            bool valid = false;
            try
            {
                object[] args = new object[OutputPlainTextDataFormat.Text.Split('{').Where(value => !string.IsNullOrWhiteSpace(value)).Select(value => int.Parse(value.Split(':')[0])).Max() + 1];
                ArchiveDataPoint sample = new ArchiveDataPoint(1);

                for (int i = 0; i < args.Length; i++)
                {
                    args[i] = sample;
                }

                Encoding.ASCII.GetBytes(string.Format(OutputPlainTextDataFormat.Text, args));

                valid = true;
            }
            catch (Exception ex)
            {
                StringBuilder sb = new StringBuilder();
                sb.AppendLine("Invalid Data Output Format: ");
                sb.AppendLine(ex.Message);
                sb.AppendLine();
                sb.AppendLine("A valid format, for example, is:");
                sb.AppendLine("{0:Source}:{1:ID},{2:Name},{3:Synonym1},{4:Time},{5:UnixTime},{6:Value},{7:Quality},{8:Description}");
                MessageBox.Show(sb.ToString(), "Invalid Output Data Format", MessageBoxButtons.OK);
                OutputPlainTextDataFormat.Focus();
            }

            return valid;
        }