public void Test(int count)
{
var lst = new List <SortedTreeTable <HistorianKey, HistorianValue> >();
for (int x = 0; x < count; x++)
{
lst.Add(CreateTable());
}
using (var reader = new UnionTreeStream <HistorianKey, HistorianValue>(lst.Select(x => new ArchiveTreeStreamWrapper <HistorianKey, HistorianValue>(x)), true))
{
var key = new HistorianKey();
var value = new HistorianValue();
Stopwatch sw = new Stopwatch();
sw.Start();
while (reader.Read(key, value))
{
;
}
sw.Stop();
System.Console.Write("{0}\t{1}\t{2}", count, sw.Elapsed.TotalSeconds, sw.Elapsed.TotalSeconds / count);
System.Console.WriteLine();
}
lst.ForEach(x => x.Dispose());
}
void Sort()
{
var lst = new List <TreeStream <TKey, TValue> >();
foreach (var buffer in m_buffers)
{
if (buffer.Count > 0)
{
buffer.IsReadingMode = true;
lst.Add(buffer);
}
}
m_reader = new UnionTreeStream <TKey, TValue>(lst, false);
}
void Clear()
{
SetEos(false);
m_currentBuffer = 0;
m_reader = null;
foreach (var buffer in m_buffers)
{
try
{
buffer.IsReadingMode = false;
}
catch (Exception)
{
System.Console.Write("Error");
throw;
}
}
}
public static void CreateNonSequential(string pendingFileName, string completeFileName, int blockSize, Action <Guid> archiveIdCallback, EncodingDefinition treeNodeType, TreeStream <TKey, TValue> treeStream, params Guid[] flags)
{
SortedPointBuffer <TKey, TValue> m_queue;
m_queue = new SortedPointBuffer <TKey, TValue>(100000, true);
m_queue.IsReadingMode = false;
TKey key = new TKey();
TValue value = new TValue();
List <SortedTreeTable <TKey, TValue> > pendingFiles = new List <SortedTreeTable <TKey, TValue> >();
try
{
while (treeStream.Read(key, value))
{
if (m_queue.IsFull)
{
pendingFiles.Add(CreateMemoryFile(treeNodeType, m_queue));
}
m_queue.TryEnqueue(key, value);
}
if (m_queue.Count > 0)
{
pendingFiles.Add(CreateMemoryFile(treeNodeType, m_queue));
}
using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(pendingFiles.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), false))
{
Create(pendingFileName, completeFileName, blockSize, archiveIdCallback, treeNodeType, reader, flags);
}
}
finally
{
pendingFiles.ForEach(x => x.Dispose());
}
}
public void Test(int count)
{
var lst = new List<SortedTreeTable<HistorianKey, HistorianValue>>();
for (int x = 0; x < count; x++)
{
lst.Add(CreateTable());
}
using (var reader = new UnionTreeStream<HistorianKey, HistorianValue>(lst.Select(x => new ArchiveTreeStreamWrapper<HistorianKey, HistorianValue>(x)), true))
{
var key = new HistorianKey();
var value = new HistorianValue();
Stopwatch sw = new Stopwatch();
sw.Start();
while (reader.Read(key, value))
;
sw.Stop();
System.Console.Write("{0}\t{1}\t{2}", count, sw.Elapsed.TotalSeconds, sw.Elapsed.TotalSeconds / count);
System.Console.WriteLine();
}
lst.ForEach(x => x.Dispose());
}
private void RolloverTask_Running(object sender, EventArgs <ScheduledTaskRunningReason> e)
{
//The worker can be disposed either via the Stop() method or
//the Dispose() method. If via the dispose method, then
//don't do any cleanup.
if (m_disposed && e.Argument == ScheduledTaskRunningReason.Disposing)
{
Log.Publish(MessageLevel.Info, "Rollover thread is Disposing");
m_rolloverComplete.Dispose();
return;
}
List <SortedTreeTable <TKey, TValue> > pendingTables1;
List <SortedTreeTable <TKey, TValue> > pendingTables2;
List <SortedTreeTable <TKey, TValue> > pendingTables3;
long sequenceNumber;
lock (m_syncRoot)
{
pendingTables1 = m_pendingTables1;
pendingTables2 = m_pendingTables2;
pendingTables3 = m_pendingTables3;
sequenceNumber = m_lastCommitedSequenceNumber;
m_pendingTables1 = new List <SortedTreeTable <TKey, TValue> >();
m_pendingTables2 = new List <SortedTreeTable <TKey, TValue> >();
m_pendingTables3 = new List <SortedTreeTable <TKey, TValue> >();
m_rolloverComplete.Set();
}
TKey startKey = new TKey();
TKey endKey = new TKey();
startKey.SetMax();
endKey.SetMin();
Log.Publish(MessageLevel.Info, "Pending Tables Report", "Pending Tables V1: " + pendingTables1.Count + " V2: " + pendingTables2.Count + " V3: " + pendingTables3.Count);
List <ArchiveTableSummary <TKey, TValue> > summaryTables = new List <ArchiveTableSummary <TKey, TValue> >();
foreach (SortedTreeTable <TKey, TValue> table in pendingTables1)
{
ArchiveTableSummary <TKey, TValue> summary = new ArchiveTableSummary <TKey, TValue>(table);
if (!summary.IsEmpty)
{
summaryTables.Add(summary);
if (startKey.IsGreaterThan(summary.FirstKey))
{
summary.FirstKey.CopyTo(startKey);
}
if (endKey.IsLessThan(summary.LastKey))
{
summary.LastKey.CopyTo(endKey);
}
}
}
foreach (SortedTreeTable <TKey, TValue> table in pendingTables2)
{
ArchiveTableSummary <TKey, TValue> summary = new ArchiveTableSummary <TKey, TValue>(table);
if (!summary.IsEmpty)
{
summaryTables.Add(summary);
if (startKey.IsGreaterThan(summary.FirstKey))
{
summary.FirstKey.CopyTo(startKey);
}
if (endKey.IsLessThan(summary.LastKey))
{
summary.LastKey.CopyTo(endKey);
}
}
}
foreach (SortedTreeTable <TKey, TValue> table in pendingTables3)
{
ArchiveTableSummary <TKey, TValue> summary = new ArchiveTableSummary <TKey, TValue>(table);
if (!summary.IsEmpty)
{
summaryTables.Add(summary);
if (startKey.IsGreaterThan(summary.FirstKey))
{
summary.FirstKey.CopyTo(startKey);
}
if (endKey.IsLessThan(summary.LastKey))
{
summary.LastKey.CopyTo(endKey);
}
}
}
long size = summaryTables.Sum(x => x.SortedTreeTable.BaseFile.ArchiveSize);
if (summaryTables.Count > 0)
{
using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(summaryTables.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
{
SortedTreeTable <TKey, TValue> newTable = m_createNextStageFile.CreateArchiveFile(startKey, endKey, size, reader, null);
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
//Add the newly created file.
edit.Add(newTable);
foreach (SortedTreeTable <TKey, TValue> table in pendingTables1)
{
edit.TryRemoveAndDelete(table.ArchiveId);
}
foreach (SortedTreeTable <TKey, TValue> table in pendingTables2)
{
edit.TryRemoveAndDelete(table.ArchiveId);
}
foreach (SortedTreeTable <TKey, TValue> table in pendingTables3)
{
edit.TryRemoveAndDelete(table.ArchiveId);
}
}
}
}
m_lastRolledOverSequenceNumber.Value = sequenceNumber;
if (RolloverComplete != null)
{
RolloverComplete(sequenceNumber);
}
}
/// <summary>
/// Appends this data to this stage. Also queues up for deletion if necessary.
/// </summary>
/// <param name="args">arguments handed to this class from either the
/// PrestageWriter or another StageWriter of a previous generation</param>
/// <remarks>
/// This method must be called in a single threaded manner.
/// </remarks>
public void AppendData(PrebufferRolloverArgs <TKey, TValue> args)
{
if (m_stopped)
{
Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost");
return;
}
if (m_disposed)
{
Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost");
return;
}
SortedTreeFile file = SortedTreeFile.CreateInMemory(4096);
SortedTreeTable <TKey, TValue> table = file.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
using (SortedTreeTableEditor <TKey, TValue> edit = table.BeginEdit())
{
edit.AddPoints(args.Stream);
edit.Commit();
}
bool shouldWait = false;
//If there is data to write then write it to the current archive.
lock (m_syncRoot)
{
if (m_stopped)
{
Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost");
table.Dispose();
return;
}
if (m_disposed)
{
Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost");
table.Dispose();
return;
}
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
edit.Add(table);
}
m_pendingTables1.Add(table);
if (m_pendingTables1.Count == 10)
{
using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables1.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
{
SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096);
SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit())
{
edit.AddPoints(reader);
edit.Commit();
}
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
//Add the newly created file.
edit.Add(table1);
foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables1)
{
edit.TryRemoveAndDelete(table2.ArchiveId);
}
}
m_pendingTables2.Add(table1);
m_pendingTables1.Clear();
}
}
if (m_pendingTables2.Count == 10)
{
using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables2.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
{
SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096);
SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit())
{
edit.AddPoints(reader);
edit.Commit();
}
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
//Add the newly created file.
edit.Add(table1);
foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables2)
{
edit.TryRemoveAndDelete(table2.ArchiveId);
}
}
m_pendingTables3.Add(table1);
m_pendingTables2.Clear();
}
}
m_lastCommitedSequenceNumber.Value = args.TransactionId;
long currentSizeMb = (m_pendingTables1.Sum(x => x.BaseFile.ArchiveSize) + m_pendingTables2.Sum(x => x.BaseFile.ArchiveSize)) >> 20;
if (currentSizeMb > m_settings.MaximumAllowedMb)
{
shouldWait = true;
m_rolloverTask.Start();
m_rolloverComplete.Reset();
}
else if (currentSizeMb > m_settings.RolloverSizeMb)
{
m_rolloverTask.Start();
}
else
{
m_rolloverTask.Start(m_settings.RolloverInterval);
}
}
if (SequenceNumberCommitted != null)
{
SequenceNumberCommitted(args.TransactionId);
}
if (shouldWait)
{
Log.Publish(MessageLevel.NA, MessageFlags.PerformanceIssue, "Queue is full", "Rollover task is taking a long time. A long pause on the inputs is about to occur.");
m_rolloverComplete.WaitOne();
}
}
