/// <summary>
/// Processes the rollover of this file.
/// </summary>
private void m_rolloverTask_Running(object sender, EventArgs <ScheduledTaskRunningReason> e)
{
//the nature of how the ScheduledTask works
//gaurentees that this function will not be called concurrently
//The worker can be disposed either via the Stop() method or
//the Dispose() method. If via the dispose method, then
//don't do any cleanup.
if (m_disposed && e.Argument == ScheduledTaskRunningReason.Disposing)
{
Log.Publish(MessageLevel.Info, "Rollover thread is Disposing");
m_waitForEmptyActiveQueue.Dispose();
return;
}
lock (m_syncRoot)
{
int count = m_activeQueue.Count;
if (count == 0)
{
m_waitForEmptyActiveQueue.Set();
return;
}
//Swap active and processing.
SortedPointBuffer <TKey, TValue> swap = m_activeQueue;
m_activeQueue = m_processingQueue;
m_processingQueue = swap;
m_activeQueue.IsReadingMode = false; //Should do nothing, but just to be sure.
m_waitForEmptyActiveQueue.Set();
m_currentTransactionIdRollingOver = m_latestTransactionId;
m_currentlyRollingOverFullQueue = m_processingQueue.IsFull;
}
//ToDo: The current inner loop for inserting random data is the sorting process here.
//ToDo: If the current speed isn't fast enough, this can be multithreaded to improve
//ToDo: the insert performance. However, at this time, the added complexity is
//ToDo: not worth it since write speeds are already blazing fast.
try
{
m_processingQueue.IsReadingMode = true; //Very CPU intensive. This does a sort on the incoming measurements. Profiling shows that about 33% of the time is spent sorting elements.
PrebufferRolloverArgs <TKey, TValue> args = new PrebufferRolloverArgs <TKey, TValue>(m_processingQueue, m_currentTransactionIdRollingOver);
m_onRollover(args);
m_processingQueue.IsReadingMode = false; //Clears the queue
}
catch (Exception ex)
{
Log.Publish(MessageLevel.Critical, "Rollover process unhandled exception", "The rollover process threw an unhandled exception. There is likely data loss that will result from this exception", null, ex);
}
m_currentlyRollingOverFullQueue = false;
}
/// <summary>
/// Appends this data to this stage. Also queues up for deletion if necessary.
/// </summary>
/// <param name="args">arguments handed to this class from either the
/// PrestageWriter or another StageWriter of a previous generation</param>
/// <remarks>
/// This method must be called in a single threaded manner.
/// </remarks>
public void AppendData(PrebufferRolloverArgs <TKey, TValue> args)
{
if (m_stopped)
{
Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost");
return;
}
if (m_disposed)
{
Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost");
return;
}
SortedTreeFile file = SortedTreeFile.CreateInMemory(4096);
SortedTreeTable <TKey, TValue> table = file.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
using (SortedTreeTableEditor <TKey, TValue> edit = table.BeginEdit())
{
edit.AddPoints(args.Stream);
edit.Commit();
}
bool shouldWait = false;
//If there is data to write then write it to the current archive.
lock (m_syncRoot)
{
if (m_stopped)
{
Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost");
table.Dispose();
return;
}
if (m_disposed)
{
Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost");
table.Dispose();
return;
}
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
edit.Add(table);
}
m_pendingTables1.Add(table);
if (m_pendingTables1.Count == 10)
{
using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables1.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
{
SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096);
SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit())
{
edit.AddPoints(reader);
edit.Commit();
}
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
//Add the newly created file.
edit.Add(table1);
foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables1)
{
edit.TryRemoveAndDelete(table2.ArchiveId);
}
}
m_pendingTables2.Add(table1);
m_pendingTables1.Clear();
}
}
if (m_pendingTables2.Count == 10)
{
using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables2.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
{
SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096);
SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit())
{
edit.AddPoints(reader);
edit.Commit();
}
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
//Add the newly created file.
edit.Add(table1);
foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables2)
{
edit.TryRemoveAndDelete(table2.ArchiveId);
}
}
m_pendingTables3.Add(table1);
m_pendingTables2.Clear();
}
}
m_lastCommitedSequenceNumber.Value = args.TransactionId;
long currentSizeMb = (m_pendingTables1.Sum(x => x.BaseFile.ArchiveSize) + m_pendingTables2.Sum(x => x.BaseFile.ArchiveSize)) >> 20;
if (currentSizeMb > m_settings.MaximumAllowedMb)
{
shouldWait = true;
m_rolloverTask.Start();
m_rolloverComplete.Reset();
}
else if (currentSizeMb > m_settings.RolloverSizeMb)
{
m_rolloverTask.Start();
}
else
{
m_rolloverTask.Start(m_settings.RolloverInterval);
}
}
if (SequenceNumberCommitted != null)
{
SequenceNumberCommitted(args.TransactionId);
}
if (shouldWait)
{
Log.Publish(MessageLevel.NA, MessageFlags.PerformanceIssue, "Queue is full", "Rollover task is taking a long time. A long pause on the inputs is about to occur.");
m_rolloverComplete.WaitOne();
}
}