/// <summary>
        /// Processes the rollover of this file.
        /// </summary>
        private void m_rolloverTask_Running(object sender, EventArgs <ScheduledTaskRunningReason> e)
        {
            //the nature of how the ScheduledTask works
            //gaurentees that this function will not be called concurrently

            //The worker can be disposed either via the Stop() method or
            //the Dispose() method.  If via the dispose method, then
            //don't do any cleanup.
            if (m_disposed && e.Argument == ScheduledTaskRunningReason.Disposing)
            {
                Log.Publish(MessageLevel.Info, "Rollover thread is Disposing");

                m_waitForEmptyActiveQueue.Dispose();
                return;
            }

            lock (m_syncRoot)
            {
                int count = m_activeQueue.Count;
                if (count == 0)
                {
                    m_waitForEmptyActiveQueue.Set();
                    return;
                }

                //Swap active and processing.
                SortedPointBuffer <TKey, TValue> swap = m_activeQueue;
                m_activeQueue               = m_processingQueue;
                m_processingQueue           = swap;
                m_activeQueue.IsReadingMode = false; //Should do nothing, but just to be sure.

                m_waitForEmptyActiveQueue.Set();
                m_currentTransactionIdRollingOver = m_latestTransactionId;
                m_currentlyRollingOverFullQueue   = m_processingQueue.IsFull;
            }

            //ToDo: The current inner loop for inserting random data is the sorting process here.
            //ToDo:  If the current speed isn't fast enough, this can be multithreaded to improve
            //ToDo:  the insert performance. However, at this time, the added complexity is
            //ToDo:  not worth it since write speeds are already blazing fast.
            try
            {
                m_processingQueue.IsReadingMode = true; //Very CPU intensive. This does a sort on the incoming measurements. Profiling shows that about 33% of the time is spent sorting elements.
                PrebufferRolloverArgs <TKey, TValue> args = new PrebufferRolloverArgs <TKey, TValue>(m_processingQueue, m_currentTransactionIdRollingOver);
                m_onRollover(args);
                m_processingQueue.IsReadingMode = false; //Clears the queue
            }
            catch (Exception ex)
            {
                Log.Publish(MessageLevel.Critical, "Rollover process unhandled exception", "The rollover process threw an unhandled exception. There is likely data loss that will result from this exception", null, ex);
            }
            m_currentlyRollingOverFullQueue = false;
        }
Example #2
0
        /// <summary>
        /// Appends this data to this stage. Also queues up for deletion if necessary.
        /// </summary>
        /// <param name="args">arguments handed to this class from either the
        /// PrestageWriter or another StageWriter of a previous generation</param>
        /// <remarks>
        /// This method must be called in a single threaded manner.
        /// </remarks>
        public void AppendData(PrebufferRolloverArgs <TKey, TValue> args)
        {
            if (m_stopped)
            {
                Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost");
                return;
            }
            if (m_disposed)
            {
                Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost");
                return;
            }

            SortedTreeFile file = SortedTreeFile.CreateInMemory(4096);
            SortedTreeTable <TKey, TValue> table = file.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);

            using (SortedTreeTableEditor <TKey, TValue> edit = table.BeginEdit())
            {
                edit.AddPoints(args.Stream);
                edit.Commit();
            }

            bool shouldWait = false;

            //If there is data to write then write it to the current archive.
            lock (m_syncRoot)
            {
                if (m_stopped)
                {
                    Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost");
                    table.Dispose();
                    return;
                }
                if (m_disposed)
                {
                    Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost");
                    table.Dispose();
                    return;
                }

                using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
                {
                    edit.Add(table);
                }
                m_pendingTables1.Add(table);

                if (m_pendingTables1.Count == 10)
                {
                    using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables1.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
                    {
                        SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096);
                        SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
                        using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit())
                        {
                            edit.AddPoints(reader);
                            edit.Commit();
                        }

                        using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
                        {
                            //Add the newly created file.
                            edit.Add(table1);

                            foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables1)
                            {
                                edit.TryRemoveAndDelete(table2.ArchiveId);
                            }
                        }

                        m_pendingTables2.Add(table1);
                        m_pendingTables1.Clear();
                    }
                }

                if (m_pendingTables2.Count == 10)
                {
                    using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables2.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
                    {
                        SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096);
                        SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
                        using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit())
                        {
                            edit.AddPoints(reader);
                            edit.Commit();
                        }

                        using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
                        {
                            //Add the newly created file.
                            edit.Add(table1);

                            foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables2)
                            {
                                edit.TryRemoveAndDelete(table2.ArchiveId);
                            }
                        }

                        m_pendingTables3.Add(table1);
                        m_pendingTables2.Clear();
                    }
                }

                m_lastCommitedSequenceNumber.Value = args.TransactionId;

                long currentSizeMb = (m_pendingTables1.Sum(x => x.BaseFile.ArchiveSize) + m_pendingTables2.Sum(x => x.BaseFile.ArchiveSize)) >> 20;
                if (currentSizeMb > m_settings.MaximumAllowedMb)
                {
                    shouldWait = true;
                    m_rolloverTask.Start();
                    m_rolloverComplete.Reset();
                }
                else if (currentSizeMb > m_settings.RolloverSizeMb)
                {
                    m_rolloverTask.Start();
                }
                else
                {
                    m_rolloverTask.Start(m_settings.RolloverInterval);
                }
            }

            if (SequenceNumberCommitted != null)
            {
                SequenceNumberCommitted(args.TransactionId);
            }

            if (shouldWait)
            {
                Log.Publish(MessageLevel.NA, MessageFlags.PerformanceIssue, "Queue is full", "Rollover task is taking a long time. A long pause on the inputs is about to occur.");
                m_rolloverComplete.WaitOne();
            }
        }