/// <summary>
/// Creates a <see cref="ArchiveTreeStreamWrapper{TKey,TValue}"/>
/// </summary>
/// <param name="table">The table to wrap.</param>
public ArchiveTreeStreamWrapper(ArchiveTableSummary <TKey, TValue> table)
{
m_table = table;
m_snapshot = m_table.ActiveSnapshotInfo.CreateReadSnapshot();
m_scanner = m_snapshot.GetTreeScanner();
m_scanner.SeekToStart();
}
/// <summary>
/// Creates the table reader.
/// </summary>
/// <param name="index"></param>
/// <param name="table"></param>
public BufferedArchiveStream(int index, ArchiveTableSummary <TKey, TValue> table)
{
Index = index;
m_table = table;
m_snapshot = m_table.ActiveSnapshotInfo.CreateReadSnapshot();
Scanner = m_snapshot.GetTreeScanner();
}
private void OnExecute(object sender, EventArgs <ScheduledTaskRunningReason> e)
{
//The worker can be disposed either via the Stop() method or
//the Dispose() method. If via the dispose method, then
//don't do any cleanup.
if (m_disposed && e.Argument == ScheduledTaskRunningReason.Disposing)
{
return;
}
//go ahead and schedule the next rollover since nothing
//will happen until this function exits anyway.
//if the task is disposing, the following line does nothing.
m_rolloverTask.Start(m_settings.ExecuteTimer);
lock (m_syncRoot)
{
if (m_disposed)
{
return;
}
using (ArchiveListSnapshot <TKey, TValue> resource = m_archiveList.CreateNewClientResources())
{
resource.UpdateSnapshot();
List <ArchiveTableSummary <TKey, TValue> > list = new List <ArchiveTableSummary <TKey, TValue> >();
List <Guid> listIds = new List <Guid>();
for (int x = 0; x < resource.Tables.Length; x++)
{
ArchiveTableSummary <TKey, TValue> table = resource.Tables[x];
if (table.SortedTreeTable.BaseFile.Snapshot.Header.Flags.Contains(m_settings.MatchFlag) && table.SortedTreeTable.BaseFile.Snapshot.Header.Flags.Contains(FileFlags.IntermediateFile))
{
list.Add(table);
listIds.Add(table.FileId);
}
else
{
resource.Tables[x] = null;
}
}
bool shouldRollover = list.Count >= m_settings.CombineOnFileCount;
long size = 0;
for (int x = 0; x < list.Count; x++)
{
size += list[x].SortedTreeTable.BaseFile.ArchiveSize;
if (size > m_settings.CombineOnFileSize)
{
if (x != list.Count - 1)//If not the last entry
{
list.RemoveRange(x + 1, list.Count - x - 1);
}
break;
}
}
if (size > m_settings.CombineOnFileSize)
{
shouldRollover = true;
}
if (shouldRollover)
{
TKey startKey = new TKey();
TKey endKey = new TKey();
startKey.SetMax();
endKey.SetMin();
foreach (Guid fileId in listIds)
{
ArchiveTableSummary <TKey, TValue> table = resource.TryGetFile(fileId);
if (table is null)
{
throw new Exception("File not found");
}
if (!table.IsEmpty)
{
if (startKey.IsGreaterThan(table.FirstKey))
{
table.FirstKey.CopyTo(startKey);
}
if (endKey.IsLessThan(table.LastKey))
{
table.LastKey.CopyTo(endKey);
}
}
}
RolloverLogFile logFile = null;
Action <Guid> createLog = (x) =>
{
logFile = m_rolloverLog.Create(listIds, x);
};
using (UnionReader <TKey, TValue> reader = new UnionReader <TKey, TValue>(list))
{
SortedTreeTable <TKey, TValue> dest = m_createNextStageFile.CreateArchiveFile(startKey, endKey, size, reader, createLog);
resource.Dispose();
using (ArchiveListEditor <TKey, TValue> edit = m_archiveList.AcquireEditLock())
{
//Add the newly created file.
edit.Add(dest);
foreach (ArchiveTableSummary <TKey, TValue> table in list)
{
edit.TryRemoveAndDelete(table.FileId);
}
}
}
if (logFile != null)
{
logFile.Delete();
}
}
resource.Dispose();
}
m_rolloverComplete.Set();
}
}
public SequentialReaderStream(ArchiveList <TKey, TValue> archiveList,
SortedTreeEngineReaderOptions readerOptions = null,
SeekFilterBase <TKey> keySeekFilter = null,
MatchFilterBase <TKey, TValue> keyMatchFilter = null,
WorkerThreadSynchronization workerThreadSynchronization = null)
{
if (readerOptions is null)
{
readerOptions = SortedTreeEngineReaderOptions.Default;
}
if (keySeekFilter is null)
{
keySeekFilter = new SeekFilterUniverse <TKey>();
}
if (keyMatchFilter is null)
{
keyMatchFilter = new MatchFilterUniverse <TKey, TValue>();
}
if (workerThreadSynchronization is null)
{
m_ownsWorkerThreadSynchronization = true;
workerThreadSynchronization = new WorkerThreadSynchronization();
}
m_workerThreadSynchronization = workerThreadSynchronization;
m_pointCount = 0;
m_keySeekFilter = keySeekFilter;
m_keyMatchFilter = keyMatchFilter;
m_keyMatchIsUniverse = m_keyMatchFilter as MatchFilterUniverse <TKey, TValue> != null;
if (readerOptions.Timeout.Ticks > 0)
{
m_timeout = new TimeoutOperation();
m_timeout.RegisterTimeout(readerOptions.Timeout, () => m_timedOut = true);
}
m_snapshot = archiveList.CreateNewClientResources();
m_snapshot.UpdateSnapshot();
m_tablesOrigList = new List <BufferedArchiveStream <TKey, TValue> >();
for (int x = 0; x < m_snapshot.Tables.Count(); x++)
{
ArchiveTableSummary <TKey, TValue> table = m_snapshot.Tables[x];
if (table != null)
{
if (table.Contains(keySeekFilter.StartOfRange, keySeekFilter.EndOfRange))
{
try
{
m_tablesOrigList.Add(new BufferedArchiveStream <TKey, TValue>(x, table));
}
catch (Exception e)
{
//ToDo: Make sure firstkey.tostring doesn't ever throw an exception.
StringBuilder sb = new StringBuilder();
sb.AppendLine($"Archive ID {table.FileId}");
sb.AppendLine($"First Key {table.FirstKey.ToString()}");
sb.AppendLine($"Last Key {table.LastKey.ToString()}");
sb.AppendLine($"File Size {table.SortedTreeTable.BaseFile.ArchiveSize}");
sb.AppendLine($"File Name {table.SortedTreeTable.BaseFile.FilePath}");
Log.Publish(MessageLevel.Error, "Error while reading file", sb.ToString(), null, e);
}
}
else
{
m_snapshot.Tables[x] = null;
}
}
}
m_sortedArchiveStreams = new CustomSortHelper <BufferedArchiveStream <TKey, TValue> >(m_tablesOrigList, IsLessThan);
m_keySeekFilter.Reset();
if (m_keySeekFilter.NextWindow())
{
SeekToKey(m_keySeekFilter.StartOfFrame);
}
else
{
Dispose();
}
}
private void RolloverTask_Running(object sender, EventArgs <ScheduledTaskRunningReason> e)
{
//The worker can be disposed either via the Stop() method or
//the Dispose() method. If via the dispose method, then
//don't do any cleanup.
if (m_disposed && e.Argument == ScheduledTaskRunningReason.Disposing)
{
Log.Publish(MessageLevel.Info, "Rollover thread is Disposing");
m_rolloverComplete.Dispose();
return;
}
List <SortedTreeTable <TKey, TValue> > pendingTables1;
List <SortedTreeTable <TKey, TValue> > pendingTables2;
List <SortedTreeTable <TKey, TValue> > pendingTables3;
long sequenceNumber;
lock (m_syncRoot)
{
pendingTables1 = m_pendingTables1;
pendingTables2 = m_pendingTables2;
pendingTables3 = m_pendingTables3;
sequenceNumber = m_lastCommitedSequenceNumber;
m_pendingTables1 = new List <SortedTreeTable <TKey, TValue> >();
m_pendingTables2 = new List <SortedTreeTable <TKey, TValue> >();
m_pendingTables3 = new List <SortedTreeTable <TKey, TValue> >();
m_rolloverComplete.Set();
}
TKey startKey = new TKey();
TKey endKey = new TKey();
startKey.SetMax();
endKey.SetMin();
Log.Publish(MessageLevel.Info, "Pending Tables Report", "Pending Tables V1: " + pendingTables1.Count + " V2: " + pendingTables2.Count + " V3: " + pendingTables3.Count);
List <ArchiveTableSummary <TKey, TValue> > summaryTables = new List <ArchiveTableSummary <TKey, TValue> >();
foreach (SortedTreeTable <TKey, TValue> table in pendingTables1)
{
ArchiveTableSummary <TKey, TValue> summary = new ArchiveTableSummary <TKey, TValue>(table);
if (!summary.IsEmpty)
{
summaryTables.Add(summary);
if (startKey.IsGreaterThan(summary.FirstKey))
{
summary.FirstKey.CopyTo(startKey);
}
if (endKey.IsLessThan(summary.LastKey))
{
summary.LastKey.CopyTo(endKey);
}
}
}
foreach (SortedTreeTable <TKey, TValue> table in pendingTables2)
{
ArchiveTableSummary <TKey, TValue> summary = new ArchiveTableSummary <TKey, TValue>(table);
if (!summary.IsEmpty)
{
summaryTables.Add(summary);
if (startKey.IsGreaterThan(summary.FirstKey))
{
summary.FirstKey.CopyTo(startKey);
}
if (endKey.IsLessThan(summary.LastKey))
{
summary.LastKey.CopyTo(endKey);
}
}
}
foreach (SortedTreeTable <TKey, TValue> table in pendingTables3)
{
ArchiveTableSummary <TKey, TValue> summary = new ArchiveTableSummary <TKey, TValue>(table);
if (!summary.IsEmpty)
{
summaryTables.Add(summary);
if (startKey.IsGreaterThan(summary.FirstKey))
{
summary.FirstKey.CopyTo(startKey);
}
if (endKey.IsLessThan(summary.LastKey))
{
summary.LastKey.CopyTo(endKey);
}
}
}
long size = summaryTables.Sum(x => x.SortedTreeTable.BaseFile.ArchiveSize);
if (summaryTables.Count > 0)
{
using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(summaryTables.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
{
SortedTreeTable <TKey, TValue> newTable = m_createNextStageFile.CreateArchiveFile(startKey, endKey, size, reader, null);
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
//Add the newly created file.
edit.Add(newTable);
foreach (SortedTreeTable <TKey, TValue> table in pendingTables1)
{
edit.TryRemoveAndDelete(table.ArchiveId);
}
foreach (SortedTreeTable <TKey, TValue> table in pendingTables2)
{
edit.TryRemoveAndDelete(table.ArchiveId);
}
foreach (SortedTreeTable <TKey, TValue> table in pendingTables3)
{
edit.TryRemoveAndDelete(table.ArchiveId);
}
}
}
}
m_lastRolledOverSequenceNumber.Value = sequenceNumber;
if (RolloverComplete != null)
{
RolloverComplete(sequenceNumber);
}
}
public SequentialReaderStream(ArchiveList <TKey, TValue> archiveList,
SortedTreeEngineReaderOptions readerOptions = null,
SeekFilterBase <TKey> keySeekFilter = null,
MatchFilterBase <TKey, TValue> keyMatchFilter = null,
WorkerThreadSynchronization workerThreadSynchronization = null)
{
if (readerOptions == null)
{
readerOptions = SortedTreeEngineReaderOptions.Default;
}
if (keySeekFilter == null)
{
keySeekFilter = new SeekFilterUniverse <TKey>();
}
if (keyMatchFilter == null)
{
keyMatchFilter = new MatchFilterUniverse <TKey, TValue>();
}
if (workerThreadSynchronization == null)
{
m_ownsWorkerThreadSynchronization = true;
workerThreadSynchronization = new WorkerThreadSynchronization();
}
m_workerThreadSynchronization = workerThreadSynchronization;
m_pointCount = 0;
m_keySeekFilter = keySeekFilter;
m_keyMatchFilter = keyMatchFilter;
m_keyMatchIsUniverse = (m_keyMatchFilter as MatchFilterUniverse <TKey, TValue>) != null;
if (readerOptions.Timeout.Ticks > 0)
{
m_timeout = new TimeoutOperation();
m_timeout.RegisterTimeout(readerOptions.Timeout, () => m_timedOut = true);
}
m_snapshot = archiveList.CreateNewClientResources();
m_snapshot.UpdateSnapshot();
m_tablesOrigList = new List <BufferedArchiveStream <TKey, TValue> >();
for (int x = 0; x < m_snapshot.Tables.Count(); x++)
{
ArchiveTableSummary <TKey, TValue> table = m_snapshot.Tables[x];
if (table != null)
{
if (table.Contains(keySeekFilter.StartOfRange, keySeekFilter.EndOfRange))
{
m_tablesOrigList.Add(new BufferedArchiveStream <TKey, TValue>(x, table));
}
else
{
m_snapshot.Tables[x] = null;
}
}
}
m_sortedArchiveStreams = new CustomSortHelper <BufferedArchiveStream <TKey, TValue> >(m_tablesOrigList, IsLessThan);
m_keySeekFilter.Reset();
if (m_keySeekFilter.NextWindow())
{
SeekToKey(m_keySeekFilter.StartOfFrame);
}
else
{
Dispose();
}
}
