public void BenchmarkOneFile() { MemoryPoolTest.TestMemoryLeak(); const int Max = 1000000; ArchiveList <HistorianKey, HistorianValue> list = new ArchiveList <HistorianKey, HistorianValue>(null); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); SortedTreeTable <HistorianKey, HistorianValue> table1 = CreateTable(); AddData(table1, 100, 100, Max); using (ArchiveListEditor <HistorianKey, HistorianValue> editor = list.AcquireEditLock()) { editor.Add(table1); } SequentialReaderStream <HistorianKey, HistorianValue> sequencer = new SequentialReaderStream <HistorianKey, HistorianValue>(list); DebugStopwatch sw = new DebugStopwatch(); double sec = sw.TimeEvent(() => { SequentialReaderStream <HistorianKey, HistorianValue> scanner = sequencer; while (scanner.Read(key, value)) { } }); System.Console.WriteLine(Max / sec / 1000000); table1.Dispose(); MemoryPoolTest.TestMemoryLeak(); }
public void BenchmarkRawFile() { MemoryPoolTest.TestMemoryLeak(); const int Max = 1000000; HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); SortedTreeTable <HistorianKey, HistorianValue> master = CreateTable(); AddData(master, 100, 100, Max); DebugStopwatch sw = new DebugStopwatch(); using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> masterRead = master.BeginRead()) { double sec = sw.TimeEvent(() => { SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = masterRead.GetTreeScanner(); scanner.SeekToStart(); while (scanner.Read(key, value)) { } }); System.Console.WriteLine(Max / sec / 1000000); } master.Dispose(); MemoryPoolTest.TestMemoryLeak(); }
public void TestOneFile() { HistorianKey key1 = new HistorianKey(); HistorianKey key2 = new HistorianKey(); HistorianValue value1 = new HistorianValue(); HistorianValue value2 = new HistorianValue(); Logger.Console.Verbose = VerboseLevel.All; MemoryPoolTest.TestMemoryLeak(); ArchiveList <HistorianKey, HistorianValue> list = new ArchiveList <HistorianKey, HistorianValue>(null); SortedTreeTable <HistorianKey, HistorianValue> master = CreateTable(); SortedTreeTable <HistorianKey, HistorianValue> table1 = CreateTable(); AddData(master, 100, 100, 100); AddData(table1, 100, 100, 100); using (ArchiveListEditor <HistorianKey, HistorianValue> editor = list.AcquireEditLock()) { editor.Add(table1); } using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> masterRead = master.BeginRead()) { SortedTreeScannerBase <HistorianKey, HistorianValue> masterScan = masterRead.GetTreeScanner(); masterScan.SeekToStart(); TreeStreamSequential <HistorianKey, HistorianValue> masterScanSequential = masterScan.TestSequential(); using (SequentialReaderStream <HistorianKey, HistorianValue> sequencer = new SequentialReaderStream <HistorianKey, HistorianValue>(list)) { TreeStreamSequential <HistorianKey, HistorianValue> scanner = sequencer.TestSequential(); int count = 0; while (scanner.Read(key1, value1)) { count++; if (!masterScanSequential.Read(key2, value2)) { throw new Exception(); } if (!key1.IsEqualTo(key2)) { throw new Exception(); } if (!value1.IsEqualTo(value2)) { throw new Exception(); } } if (masterScan.Read(key2, value2)) { throw new Exception(); } } } list.Dispose(); master.Dispose(); MemoryPoolTest.TestMemoryLeak(); }
/// <summary> /// Appends this data to this stage. Also queues up for deletion if necessary. /// </summary> /// <param name="args">arguments handed to this class from either the /// PrestageWriter or another StageWriter of a previous generation</param> /// <remarks> /// This method must be called in a single threaded manner. /// </remarks> public void AppendData(PrebufferRolloverArgs <TKey, TValue> args) { if (m_stopped) { Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost"); return; } if (m_disposed) { Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost"); return; } SortedTreeFile file = SortedTreeFile.CreateInMemory(4096); SortedTreeTable <TKey, TValue> table = file.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod); using (SortedTreeTableEditor <TKey, TValue> edit = table.BeginEdit()) { edit.AddPoints(args.Stream); edit.Commit(); } bool shouldWait = false; //If there is data to write then write it to the current archive. lock (m_syncRoot) { if (m_stopped) { Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost"); table.Dispose(); return; } if (m_disposed) { Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost"); table.Dispose(); return; } using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock()) { edit.Add(table); } m_pendingTables1.Add(table); if (m_pendingTables1.Count == 10) { using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables1.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true)) { SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096); SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod); using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit()) { edit.AddPoints(reader); edit.Commit(); } using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock()) { //Add the newly created file. edit.Add(table1); foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables1) { edit.TryRemoveAndDelete(table2.ArchiveId); } } m_pendingTables2.Add(table1); m_pendingTables1.Clear(); } } if (m_pendingTables2.Count == 10) { using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables2.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true)) { SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096); SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod); using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit()) { edit.AddPoints(reader); edit.Commit(); } using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock()) { //Add the newly created file. edit.Add(table1); foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables2) { edit.TryRemoveAndDelete(table2.ArchiveId); } } m_pendingTables3.Add(table1); m_pendingTables2.Clear(); } } m_lastCommitedSequenceNumber.Value = args.TransactionId; long currentSizeMb = (m_pendingTables1.Sum(x => x.BaseFile.ArchiveSize) + m_pendingTables2.Sum(x => x.BaseFile.ArchiveSize)) >> 20; if (currentSizeMb > m_settings.MaximumAllowedMb) { shouldWait = true; m_rolloverTask.Start(); m_rolloverComplete.Reset(); } else if (currentSizeMb > m_settings.RolloverSizeMb) { m_rolloverTask.Start(); } else { m_rolloverTask.Start(m_settings.RolloverInterval); } } if (SequenceNumberCommitted != null) { SequenceNumberCommitted(args.TransactionId); } if (shouldWait) { Log.Publish(MessageLevel.NA, MessageFlags.PerformanceIssue, "Queue is full", "Rollover task is taking a long time. A long pause on the inputs is about to occur."); m_rolloverComplete.WaitOne(); } }
/// <summary> /// Loads the specified files into the archive list. /// </summary> /// <param name="archiveFiles"></param> public override void LoadFiles(IEnumerable <string> archiveFiles) { if (m_disposed) { throw new Exception("Object is disposing"); } List <SortedTreeTable <TKey, TValue> > loadedFiles = new List <SortedTreeTable <TKey, TValue> >(); foreach (string file in archiveFiles) { try { SortedTreeFile sortedTreeFile = SortedTreeFile.OpenFile(file, isReadOnly: true); SortedTreeTable <TKey, TValue> table = sortedTreeFile.OpenTable <TKey, TValue>(); if (table is null) { sortedTreeFile.Dispose(); //archiveFile.Delete(); // TODO: Consider the consequences of deleting a file. } else { if (m_listLog.ShouldBeDeleted(table.ArchiveId)) { Log.Publish(MessageLevel.Warning, "File being deleted", "The supplied file is being deleted because it was part of a previous rollover that completed but the server crashed before it was properly deleted." + file); table.BaseFile.Delete(); } else { loadedFiles.Add(table); } } Log.Publish(MessageLevel.Info, "Loading Files", "Successfully opened: " + file); } catch (Exception ex) { Log.Publish(MessageLevel.Warning, "Loading Files", "Skipping Failed File: " + file, null, ex); } } using ArchiveListEditor <TKey, TValue> edit = AcquireEditLock(); if (m_disposed) { loadedFiles.ForEach(table => table.Dispose()); throw new Exception("Object is disposing"); } foreach (SortedTreeTable <TKey, TValue> file in loadedFiles) { try { edit.Add(file); } catch (Exception ex) { Log.Publish(MessageLevel.Warning, "Attaching File", "File already attached: " + file.ArchiveId, file.BaseFile.FilePath, ex); file.BaseFile.Dispose(); } } }
private void BtnFindCorruptFiles_Click(object sender, EventArgs e) { DataTable resultsTable = new DataTable(); resultsTable.Columns.Add("File", typeof(string)); resultsTable.Columns.Add("ID", typeof(string)); StringBuilder sb = new StringBuilder(); using (OpenFileDialog dlg = new OpenFileDialog()) { dlg.Multiselect = true; dlg.Filter = "Open Historian 2.0 File|*.d2"; if (dlg.ShowDialog() == DialogResult.OK) { foreach (string fileName in dlg.FileNames) { sb.Clear(); try { using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, true)) { sb.AppendFormat("ID: {0} ", file.Snapshot.Header.ArchiveId); sb.AppendFormat("Commit Number: {0} ", file.Snapshot.Header.SnapshotSequenceNumber); SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>(); if (table is null) { sb.Append("ERROR - No Historian Table "); } else { try { if ((long)table.FirstKey.Timestamp < DateTime.MinValue.Ticks || (long)table.FirstKey.Timestamp > DateTime.MaxValue.Ticks) { sb.Append("Start Time: Invalid "); } else { sb.AppendFormat("Start Time: {0} ", table.FirstKey.TimestampAsDate.ToString()); } if ((long)table.LastKey.Timestamp < DateTime.MinValue.Ticks || (long)table.LastKey.Timestamp > DateTime.MaxValue.Ticks) { sb.Append("End Time: Invalid "); } else { sb.AppendFormat("End Time: {0} ", table.LastKey.TimestampAsDate.ToString()); } } finally { table.Dispose(); } } } } catch (Exception ex) { sb.AppendFormat("File Header Corrupt: {0} ", ex.ToString()); throw; } resultsTable.Rows.Add(fileName, sb.ToString()); } FrmDisplayFileMetaData win = new FrmDisplayFileMetaData(resultsTable); win.Show(); } } }