public void TestBulkRolloverFile() { Stats.LookupKeys = 0; DiskIoSession.ReadCount = 0; DiskIoSession.WriteCount = 0; Stats.ChecksumCount = 0; DiskIoSession.Lookups = 0; DiskIoSession.CachedLookups = 0; long cnt; Stopwatch sw = new Stopwatch(); sw.Start(); //using (SortedTreeTable<HistorianKey, HistorianValue> af = SortedTreeFile.CreateInMemory(4096).OpenOrCreateTable<HistorianKey, HistorianValue>(SortedTree.FixedSizeNode)) using (SortedTreeTable <HistorianKey, HistorianValue> af = SortedTreeFile.CreateInMemory(blockSize: 4096).OpenOrCreateTable <HistorianKey, HistorianValue>(HistorianFileEncodingDefinition.TypeGuid)) { using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = af.BeginEdit()) { edit.AddPoints(new PointStreamSequentialPoints(1, 20000000)); edit.Commit(); } sw.Stop(); cnt = af.Count(); System.Console.WriteLine(cnt); } System.Console.WriteLine((float)(20 / sw.Elapsed.TotalSeconds)); }
public void TestRollover() { int count = 1000000; Stopwatch sw = new Stopwatch(); using (SortedTreeFile af = SortedTreeFile.CreateInMemory()) using (SortedTreeTable <AmiKey, AmiKey> table = af.OpenOrCreateTable <AmiKey, AmiKey>(EncodingDefinition.FixedSizeCombinedEncoding)) { using (SortedTreeTableEditor <AmiKey, AmiKey> edit = table.BeginEdit()) { sw.Start(); AmiKey key = new AmiKey(); AmiKey value = new AmiKey(); for (int x = 0; x < count; x++) { key.Timestamp = (uint)x; edit.AddPoint(key, value); } edit.Commit(); sw.Stop(); Console.WriteLine(af.ArchiveSize / 1024.0 / 1024.0); Console.WriteLine(count / sw.Elapsed.TotalSeconds / 1000000); } using (SortedTreeFile af2 = SortedTreeFile.CreateInMemory()) using (SortedTreeTable <AmiKey, AmiKey> table2 = af2.OpenOrCreateTable <AmiKey, AmiKey>(new EncodingDefinition(EncodingDefinition.FixedSizeCombinedEncoding.KeyValueEncodingMethod, EncodingDefinition.FixedSizeCombinedEncoding.KeyValueEncodingMethod))) using (SortedTreeTableEditor <AmiKey, AmiKey> edit = table2.BeginEdit()) { using (SortedTreeTableReadSnapshot <AmiKey, AmiKey> read = table.BeginRead()) using (SortedTreeScannerBase <AmiKey, AmiKey> scan = read.GetTreeScanner()) { scan.SeekToStart(); edit.AddPoints(scan); } } Console.WriteLine(count); } }
private TestResults Test(int pageSize) { Stats.LookupKeys = 0; DiskIoSession.ReadCount = 0; DiskIoSession.WriteCount = 0; Stats.ChecksumCount = 0; DiskIoSession.Lookups = 0; DiskIoSession.CachedLookups = 0; long cnt; Stopwatch sw = new Stopwatch(); Stopwatch sw2 = new Stopwatch(); sw.Start(); using (SortedTreeTable <HistorianKey, HistorianValue> af = SortedTreeFile.CreateInMemory(blockSize: pageSize).OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding)) { using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = af.BeginEdit()) { for (int x = 0; x < 100; x++) { edit.AddPoints(new PointStreamSequential(x * 10000, 10000)); } edit.Commit(); } sw.Stop(); sw2.Start(); cnt = af.Count(); sw2.Stop(); } return(new TestResults() { Count = cnt, PageSize = pageSize, RateWrite = (float)(1 / sw.Elapsed.TotalSeconds), RateRead = (float)(1 / sw2.Elapsed.TotalSeconds), ReadCount = DiskIoSession.ReadCount, WriteCount = DiskIoSession.WriteCount, ChecksumCount = Stats.ChecksumCount, Lookups = DiskIoSession.Lookups, CachedLookups = DiskIoSession.CachedLookups }); }
/// <summary> /// Appends this data to this stage. Also queues up for deletion if necessary. /// </summary> /// <param name="args">arguments handed to this class from either the /// PrestageWriter or another StageWriter of a previous generation</param> /// <remarks> /// This method must be called in a single threaded manner. /// </remarks> public void AppendData(PrebufferRolloverArgs <TKey, TValue> args) { if (m_stopped) { Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost"); return; } if (m_disposed) { Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost"); return; } SortedTreeFile file = SortedTreeFile.CreateInMemory(4096); SortedTreeTable <TKey, TValue> table = file.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod); using (SortedTreeTableEditor <TKey, TValue> edit = table.BeginEdit()) { edit.AddPoints(args.Stream); edit.Commit(); } bool shouldWait = false; //If there is data to write then write it to the current archive. lock (m_syncRoot) { if (m_stopped) { Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost"); table.Dispose(); return; } if (m_disposed) { Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost"); table.Dispose(); return; } using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock()) { edit.Add(table); } m_pendingTables1.Add(table); if (m_pendingTables1.Count == 10) { using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables1.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true)) { SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096); SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod); using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit()) { edit.AddPoints(reader); edit.Commit(); } using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock()) { //Add the newly created file. edit.Add(table1); foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables1) { edit.TryRemoveAndDelete(table2.ArchiveId); } } m_pendingTables2.Add(table1); m_pendingTables1.Clear(); } } if (m_pendingTables2.Count == 10) { using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables2.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true)) { SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096); SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod); using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit()) { edit.AddPoints(reader); edit.Commit(); } using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock()) { //Add the newly created file. edit.Add(table1); foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables2) { edit.TryRemoveAndDelete(table2.ArchiveId); } } m_pendingTables3.Add(table1); m_pendingTables2.Clear(); } } m_lastCommitedSequenceNumber.Value = args.TransactionId; long currentSizeMb = (m_pendingTables1.Sum(x => x.BaseFile.ArchiveSize) + m_pendingTables2.Sum(x => x.BaseFile.ArchiveSize)) >> 20; if (currentSizeMb > m_settings.MaximumAllowedMb) { shouldWait = true; m_rolloverTask.Start(); m_rolloverComplete.Reset(); } else if (currentSizeMb > m_settings.RolloverSizeMb) { m_rolloverTask.Start(); } else { m_rolloverTask.Start(m_settings.RolloverInterval); } } if (SequenceNumberCommitted != null) { SequenceNumberCommitted(args.TransactionId); } if (shouldWait) { Log.Publish(MessageLevel.NA, MessageFlags.PerformanceIssue, "Queue is full", "Rollover task is taking a long time. A long pause on the inputs is about to occur."); m_rolloverComplete.WaitOne(); } }