public void BenchmarkOneFile()
{
MemoryPoolTest.TestMemoryLeak();
const int Max = 1000000;
ArchiveList <HistorianKey, HistorianValue> list = new ArchiveList <HistorianKey, HistorianValue>(null);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
SortedTreeTable <HistorianKey, HistorianValue> table1 = CreateTable();
AddData(table1, 100, 100, Max);
using (ArchiveListEditor <HistorianKey, HistorianValue> editor = list.AcquireEditLock())
{
editor.Add(table1);
}
SequentialReaderStream <HistorianKey, HistorianValue> sequencer = new SequentialReaderStream <HistorianKey, HistorianValue>(list);
DebugStopwatch sw = new DebugStopwatch();
double sec = sw.TimeEvent(() =>
{
SequentialReaderStream <HistorianKey, HistorianValue> scanner = sequencer;
while (scanner.Read(key, value))
{
}
});
System.Console.WriteLine(Max / sec / 1000000);
table1.Dispose();
MemoryPoolTest.TestMemoryLeak();
}
public void BenchmarkRawFile()
{
MemoryPoolTest.TestMemoryLeak();
const int Max = 1000000;
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
SortedTreeTable <HistorianKey, HistorianValue> master = CreateTable();
AddData(master, 100, 100, Max);
DebugStopwatch sw = new DebugStopwatch();
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> masterRead = master.BeginRead())
{
double sec = sw.TimeEvent(() =>
{
SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = masterRead.GetTreeScanner();
scanner.SeekToStart();
while (scanner.Read(key, value))
{
}
});
System.Console.WriteLine(Max / sec / 1000000);
}
master.Dispose();
MemoryPoolTest.TestMemoryLeak();
}
public void TestOneFile()
{
HistorianKey key1 = new HistorianKey();
HistorianKey key2 = new HistorianKey();
HistorianValue value1 = new HistorianValue();
HistorianValue value2 = new HistorianValue();
Logger.Console.Verbose = VerboseLevel.All;
MemoryPoolTest.TestMemoryLeak();
ArchiveList <HistorianKey, HistorianValue> list = new ArchiveList <HistorianKey, HistorianValue>(null);
SortedTreeTable <HistorianKey, HistorianValue> master = CreateTable();
SortedTreeTable <HistorianKey, HistorianValue> table1 = CreateTable();
AddData(master, 100, 100, 100);
AddData(table1, 100, 100, 100);
using (ArchiveListEditor <HistorianKey, HistorianValue> editor = list.AcquireEditLock())
{
editor.Add(table1);
}
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> masterRead = master.BeginRead())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> masterScan = masterRead.GetTreeScanner();
masterScan.SeekToStart();
TreeStreamSequential <HistorianKey, HistorianValue> masterScanSequential = masterScan.TestSequential();
using (SequentialReaderStream <HistorianKey, HistorianValue> sequencer = new SequentialReaderStream <HistorianKey, HistorianValue>(list))
{
TreeStreamSequential <HistorianKey, HistorianValue> scanner = sequencer.TestSequential();
int count = 0;
while (scanner.Read(key1, value1))
{
count++;
if (!masterScanSequential.Read(key2, value2))
{
throw new Exception();
}
if (!key1.IsEqualTo(key2))
{
throw new Exception();
}
if (!value1.IsEqualTo(value2))
{
throw new Exception();
}
}
if (masterScan.Read(key2, value2))
{
throw new Exception();
}
}
}
list.Dispose();
master.Dispose();
MemoryPoolTest.TestMemoryLeak();
}
/// <summary>
/// Appends this data to this stage. Also queues up for deletion if necessary.
/// </summary>
/// <param name="args">arguments handed to this class from either the
/// PrestageWriter or another StageWriter of a previous generation</param>
/// <remarks>
/// This method must be called in a single threaded manner.
/// </remarks>
public void AppendData(PrebufferRolloverArgs <TKey, TValue> args)
{
if (m_stopped)
{
Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost");
return;
}
if (m_disposed)
{
Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost");
return;
}
SortedTreeFile file = SortedTreeFile.CreateInMemory(4096);
SortedTreeTable <TKey, TValue> table = file.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
using (SortedTreeTableEditor <TKey, TValue> edit = table.BeginEdit())
{
edit.AddPoints(args.Stream);
edit.Commit();
}
bool shouldWait = false;
//If there is data to write then write it to the current archive.
lock (m_syncRoot)
{
if (m_stopped)
{
Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost");
table.Dispose();
return;
}
if (m_disposed)
{
Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost");
table.Dispose();
return;
}
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
edit.Add(table);
}
m_pendingTables1.Add(table);
if (m_pendingTables1.Count == 10)
{
using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables1.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
{
SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096);
SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit())
{
edit.AddPoints(reader);
edit.Commit();
}
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
//Add the newly created file.
edit.Add(table1);
foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables1)
{
edit.TryRemoveAndDelete(table2.ArchiveId);
}
}
m_pendingTables2.Add(table1);
m_pendingTables1.Clear();
}
}
if (m_pendingTables2.Count == 10)
{
using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables2.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
{
SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096);
SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit())
{
edit.AddPoints(reader);
edit.Commit();
}
using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
{
//Add the newly created file.
edit.Add(table1);
foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables2)
{
edit.TryRemoveAndDelete(table2.ArchiveId);
}
}
m_pendingTables3.Add(table1);
m_pendingTables2.Clear();
}
}
m_lastCommitedSequenceNumber.Value = args.TransactionId;
long currentSizeMb = (m_pendingTables1.Sum(x => x.BaseFile.ArchiveSize) + m_pendingTables2.Sum(x => x.BaseFile.ArchiveSize)) >> 20;
if (currentSizeMb > m_settings.MaximumAllowedMb)
{
shouldWait = true;
m_rolloverTask.Start();
m_rolloverComplete.Reset();
}
else if (currentSizeMb > m_settings.RolloverSizeMb)
{
m_rolloverTask.Start();
}
else
{
m_rolloverTask.Start(m_settings.RolloverInterval);
}
}
if (SequenceNumberCommitted != null)
{
SequenceNumberCommitted(args.TransactionId);
}
if (shouldWait)
{
Log.Publish(MessageLevel.NA, MessageFlags.PerformanceIssue, "Queue is full", "Rollover task is taking a long time. A long pause on the inputs is about to occur.");
m_rolloverComplete.WaitOne();
}
}
/// <summary>
/// Loads the specified files into the archive list.
/// </summary>
/// <param name="archiveFiles"></param>
public override void LoadFiles(IEnumerable <string> archiveFiles)
{
if (m_disposed)
{
throw new Exception("Object is disposing");
}
List <SortedTreeTable <TKey, TValue> > loadedFiles = new List <SortedTreeTable <TKey, TValue> >();
foreach (string file in archiveFiles)
{
try
{
SortedTreeFile sortedTreeFile = SortedTreeFile.OpenFile(file, isReadOnly: true);
SortedTreeTable <TKey, TValue> table = sortedTreeFile.OpenTable <TKey, TValue>();
if (table is null)
{
sortedTreeFile.Dispose();
//archiveFile.Delete(); // TODO: Consider the consequences of deleting a file.
}
else
{
if (m_listLog.ShouldBeDeleted(table.ArchiveId))
{
Log.Publish(MessageLevel.Warning, "File being deleted", "The supplied file is being deleted because it was part of a previous rollover that completed but the server crashed before it was properly deleted." + file);
table.BaseFile.Delete();
}
else
{
loadedFiles.Add(table);
}
}
Log.Publish(MessageLevel.Info, "Loading Files", "Successfully opened: " + file);
}
catch (Exception ex)
{
Log.Publish(MessageLevel.Warning, "Loading Files", "Skipping Failed File: " + file, null, ex);
}
}
using ArchiveListEditor <TKey, TValue> edit = AcquireEditLock();
if (m_disposed)
{
loadedFiles.ForEach(table => table.Dispose());
throw new Exception("Object is disposing");
}
foreach (SortedTreeTable <TKey, TValue> file in loadedFiles)
{
try
{
edit.Add(file);
}
catch (Exception ex)
{
Log.Publish(MessageLevel.Warning, "Attaching File", "File already attached: " + file.ArchiveId, file.BaseFile.FilePath, ex);
file.BaseFile.Dispose();
}
}
}
private void BtnFindCorruptFiles_Click(object sender, EventArgs e)
{
DataTable resultsTable = new DataTable();
resultsTable.Columns.Add("File", typeof(string));
resultsTable.Columns.Add("ID", typeof(string));
StringBuilder sb = new StringBuilder();
using (OpenFileDialog dlg = new OpenFileDialog())
{
dlg.Multiselect = true;
dlg.Filter = "Open Historian 2.0 File|*.d2";
if (dlg.ShowDialog() == DialogResult.OK)
{
foreach (string fileName in dlg.FileNames)
{
sb.Clear();
try
{
using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, true))
{
sb.AppendFormat("ID: {0} ", file.Snapshot.Header.ArchiveId);
sb.AppendFormat("Commit Number: {0} ", file.Snapshot.Header.SnapshotSequenceNumber);
SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>();
if (table is null)
{
sb.Append("ERROR - No Historian Table ");
}
else
{
try
{
if ((long)table.FirstKey.Timestamp < DateTime.MinValue.Ticks || (long)table.FirstKey.Timestamp > DateTime.MaxValue.Ticks)
{
sb.Append("Start Time: Invalid ");
}
else
{
sb.AppendFormat("Start Time: {0} ", table.FirstKey.TimestampAsDate.ToString());
}
if ((long)table.LastKey.Timestamp < DateTime.MinValue.Ticks || (long)table.LastKey.Timestamp > DateTime.MaxValue.Ticks)
{
sb.Append("End Time: Invalid ");
}
else
{
sb.AppendFormat("End Time: {0} ", table.LastKey.TimestampAsDate.ToString());
}
}
finally
{
table.Dispose();
}
}
}
}
catch (Exception ex)
{
sb.AppendFormat("File Header Corrupt: {0} ", ex.ToString());
throw;
}
resultsTable.Rows.Add(fileName, sb.ToString());
}
FrmDisplayFileMetaData win = new FrmDisplayFileMetaData(resultsTable);
win.Show();
}
}
}
