void AddDataTerminal(SortedTreeTable <HistorianKey, HistorianValue> table, ulong pointID, DateTime startTime, TimeSpan stepTime, ulong startValue, ulong stepValue, int count)
        {
            using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = table.BeginEdit())
            {
                HistorianKey   key   = new HistorianKey();
                HistorianValue value = new HistorianValue();
                key.SetMin();
                ulong t = (ulong)startTime.Ticks;
                ulong v = startValue;

                while (count > 0)
                {
                    count--;
                    key.Timestamp = t;
                    key.PointID   = pointID;
                    value.Value1  = v;

                    edit.AddPoint(key, value);
                    t += (ulong)stepTime.Ticks;
                    v += stepValue;
                }

                edit.Commit();
            }
        }
Esempio n. 2
0
        private TestResults TestRandom(int pageSize, uint count)
        {
            //StringBuilder sb = new StringBuilder();
            Random         R     = new Random(1);
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            DiskIoSession.ReadCount     = 0;
            DiskIoSession.WriteCount    = 0;
            Stats.ChecksumCount         = 0;
            DiskIoSession.Lookups       = 0;
            DiskIoSession.CachedLookups = 0;

            Stopwatch sw = new Stopwatch();

            sw.Start();
            using (SortedTreeFile af = SortedTreeFile.CreateInMemory(blockSize: pageSize))
                using (SortedTreeTable <HistorianKey, HistorianValue> af2 = af.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                {
                    uint pointPairs = count / 5000;
                    for (uint i = 0; i < pointPairs; i++)
                    {
                        uint max = i * 5000 + 5000;
                        using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = af2.BeginEdit())
                        {
                            for (ulong x = i * 5000; x < max; x++)
                            {
                                key.Timestamp = (uint)R.Next();
                                key.PointID   = 2 * x;
                                value.Value3  = 3 * x;
                                value.Value1  = 4 * x;
                                //if ((x % 100) == 0)
                                //    sb.AppendLine(x + "," + DiskIoSession.ReadCount + "," + DiskIoSession.WriteCount);
                                //if (x == 1000)
                                //    DiskIoSession.BreakOnIO = true;
                                edit.AddPoint(key, value);
                                //edit.AddPoint(uint.MaxValue - x, 2 * x, 3 * x, 4 * x);
                            }
                            edit.Commit();
                        }
                    }
                    //long cnt = af.Count();
                }

            sw.Stop();

            //File.WriteAllText(@"C:\temp\" + pageSize + ".csv",sb.ToString());


            return(new TestResults()
            {
                PageSize = pageSize,
                Rate = (float)(count / sw.Elapsed.TotalSeconds / 1000000),
                ReadCount = DiskIoSession.ReadCount,
                WriteCount = DiskIoSession.WriteCount,
                ChecksumCount = Stats.ChecksumCount,
                Lookups = DiskIoSession.Lookups,
                CachedLookups = DiskIoSession.CachedLookups
            });
        }
        public void TestSmall()
        {
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            using (SortedTreeFile af = SortedTreeFile.CreateInMemory())
                using (SortedTreeTable <HistorianKey, HistorianValue> file = af.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                {
                    using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = file.BeginEdit())
                    {
                        for (int x = 0; x < 10000000; x++)
                        {
                            key.Timestamp = (ulong)x;
                            edit.AddPoint(key, value);
                        }
                        edit.Commit();
                    }

                    using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> read = file.BeginRead())
                        using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = read.GetTreeScanner())
                        {
                            int count = 0;
                            scan.SeekToStart();
                            while (scan.Read(key, value))
                            {
                                count++;
                            }
                            System.Console.WriteLine(count.ToString());
                        }
                }
        }
Esempio n. 4
0
        public void TestBulkRolloverFile()
        {
            Stats.LookupKeys            = 0;
            DiskIoSession.ReadCount     = 0;
            DiskIoSession.WriteCount    = 0;
            Stats.ChecksumCount         = 0;
            DiskIoSession.Lookups       = 0;
            DiskIoSession.CachedLookups = 0;
            long      cnt;
            Stopwatch sw = new Stopwatch();

            sw.Start();
            //using (SortedTreeTable<HistorianKey, HistorianValue> af = SortedTreeFile.CreateInMemory(4096).OpenOrCreateTable<HistorianKey, HistorianValue>(SortedTree.FixedSizeNode))
            using (SortedTreeTable <HistorianKey, HistorianValue> af = SortedTreeFile.CreateInMemory(blockSize: 4096).OpenOrCreateTable <HistorianKey, HistorianValue>(HistorianFileEncodingDefinition.TypeGuid))
            {
                using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = af.BeginEdit())
                {
                    edit.AddPoints(new PointStreamSequentialPoints(1, 20000000));
                    edit.Commit();
                }
                sw.Stop();

                cnt = af.Count();
                System.Console.WriteLine(cnt);
            }

            System.Console.WriteLine((float)(20 / sw.Elapsed.TotalSeconds));
        }
        public void WriteFile()
        {
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            if (File.Exists("c:\\temp\\ArchiveTestFileBig.d2"))
            {
                File.Delete("c:\\temp\\ArchiveTestFileBig.d2");
            }
            //using (var af = ArchiveFile.CreateInMemory(CompressionMethod.TimeSeriesEncoded))
            using (SortedTreeFile af = SortedTreeFile.CreateFile("c:\\temp\\ArchiveTestFileBig.d2"))
                using (SortedTreeTable <HistorianKey, HistorianValue> af2 = af.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                {
                    Random r = new Random(3);

                    for (ulong v1 = 1; v1 < 36; v1++)
                    {
                        using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = af2.BeginEdit())
                        {
                            for (ulong v2 = 1; v2 < 86000; v2++)
                            {
                                key.Timestamp = v1 * 2342523;
                                key.PointID   = v2;
                                value.Value1  = (ulong)r.Next();
                                value.Value3  = 0;

                                edit.AddPoint(key, value);
                            }
                            edit.Commit();
                        }
                        af2.Count();
                    }
                    af2.Count();
                }
        }
Esempio n. 6
0
        public void RollbackChangesToAFile()
        {
            string         fileName = @"C:\Temp\ArchiveFile.d2";
            HistorianKey   key      = new HistorianKey();
            HistorianValue value    = new HistorianValue();

            using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: false))
                using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                {
                    using (SortedTreeTableEditor <HistorianKey, HistorianValue> editor = table.BeginEdit())
                    {
                        key.TimestampAsDate = DateTime.Now.AddDays(-1);
                        key.PointID         = 234;
                        value.AsString      = "Add Me";
                        editor.AddPoint(key, value);
                        editor.Commit();
                    }

                    using (SortedTreeTableEditor <HistorianKey, HistorianValue> editor = table.BeginEdit())
                    {
                        key.Timestamp  = 31;
                        value.AsString = "But Not Me";
                        editor.AddPoint(key, value);
                        editor.Rollback(); //These changes will not be written to the disk
                    }
                }
            ReadDataFromAFile();
        }
Esempio n. 7
0
        public void ReadFromAFileWhileWritingToIt()
        {
            HistorianKey   key      = new HistorianKey();
            HistorianValue value    = new HistorianValue();
            string         fileName = @"C:\Temp\ArchiveFile.d2";

            using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: false))
                using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
                {
                    using (SortedTreeTableEditor <HistorianKey, HistorianValue> writer = table.BeginEdit())
                        using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead())
                        {
                            SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = reader.GetTreeScanner();
                            scanner.SeekToStart();
                            while (scanner.Read(key, value))
                            {
                                key.Timestamp++;
                                value.Value1++;
                                writer.AddPoint(key, value);
                            }

                            writer.Commit();
                        }
                }
            ReadDataFromAFile();
        }
Esempio n. 8
0
        private void TestFile(int pageSize, string fileName)
        {
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            key.Timestamp = 1;

            if (File.Exists(fileName))
            {
                File.Delete(fileName);
            }
            Stopwatch sw = new Stopwatch();

            sw.Start();
            using (SortedTreeTable <HistorianKey, HistorianValue> af = SortedTreeFile.CreateFile(fileName, blockSize: pageSize).OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = af.BeginEdit())
                {
                    for (uint x = 0; x < 1000000; x++)
                    {
                        key.PointID = x;
                        edit.AddPoint(key, value);
                    }
                    edit.Commit();
                }
            sw.Stop();
            System.Console.WriteLine("Size: " + pageSize + " Rate: " + (1 / sw.Elapsed.TotalSeconds).ToString());
        }
Esempio n. 9
0
        public void EnduranceTest()
        {
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            using (SortedTreeTable <HistorianKey, HistorianValue> target = SortedTreeFile.CreateInMemory().OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
            {
                for (uint x = 0; x < 100; x++)
                {
                    using (SortedTreeTableEditor <HistorianKey, HistorianValue> fileEditor = target.BeginEdit())
                    {
                        for (int y = 0; y < 10; y++)
                        {
                            key.Timestamp = x;
                            key.PointID   = x;
                            value.Value1  = x;
                            value.Value3  = x;
                            fileEditor.AddPoint(key, value);
                            x++;
                        }
                        fileEditor.Commit();
                    }
                    Assert.AreEqual(target.FirstKey.Timestamp, 0);
                    Assert.AreEqual(target.LastKey.Timestamp, x - 1);
                }
            }
        }
Esempio n. 10
0
        public void TestCompressed()
        {
            int       count = 10000000;
            Stopwatch sw    = new Stopwatch();

            using (SortedTreeFile af = SortedTreeFile.CreateInMemory())
                using (SortedTreeTable <AmiKey, AmiValue> table = af.OpenOrCreateTable <AmiKey, AmiValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                {
                    using (SortedTreeTableEditor <AmiKey, AmiValue> edit = table.BeginEdit())
                    {
                        sw.Start();
                        AmiKey   key   = new AmiKey();
                        AmiValue value = new AmiValue();

                        for (int x = 0; x < count; x++)
                        {
                            key.Timestamp = (uint)x;
                            edit.AddPoint(key, value);
                        }

                        edit.Commit();
                        sw.Stop();

                        Console.WriteLine(af.ArchiveSize / 1024.0 / 1024.0);
                        Console.WriteLine(count / sw.Elapsed.TotalSeconds / 1000000);
                    }
                }
        }
Esempio n. 11
0
 public void AddPointTest()
 {
     using (SortedTreeTable <HistorianKey, HistorianValue> target = SortedTreeFile.CreateInMemory().OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
     {
         using (SortedTreeTableEditor <HistorianKey, HistorianValue> fileEditor = target.BeginEdit())
         {
             fileEditor.AddPoint(new HistorianKey(), new HistorianValue());
             fileEditor.Commit();
         }
     }
 }
        void AddData(SortedTreeTable <HistorianKey, HistorianValue> table, ulong start, ulong step, ulong count)
        {
            using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = table.BeginEdit())
            {
                HistorianKey   key   = new HistorianKey();
                HistorianValue value = new HistorianValue();

                for (ulong v = start; v < start + step * count; v += step)
                {
                    key.SetMin();
                    key.PointID = v;
                    edit.AddPoint(key, value);
                }
                edit.Commit();
            }
        }
Esempio n. 13
0
        public void CreateSnapshotTest()
        {
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            key.Timestamp = 1;
            key.PointID   = 2;
            value.Value1  = 3;
            value.Value2  = 4;
            using (SortedTreeTable <HistorianKey, HistorianValue> target = SortedTreeFile.CreateInMemory().OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
            {
                ulong date    = 1;
                ulong pointId = 2;
                ulong value1  = 3;
                ulong value2  = 4;
                SortedTreeTableSnapshotInfo <HistorianKey, HistorianValue> snap1;
                using (SortedTreeTableEditor <HistorianKey, HistorianValue> fileEditor = target.BeginEdit())
                {
                    fileEditor.AddPoint(key, value);
                    key.Timestamp++;
                    fileEditor.AddPoint(key, value);
                    snap1 = target.AcquireReadSnapshot();
                    fileEditor.Commit();
                }
                SortedTreeTableSnapshotInfo <HistorianKey, HistorianValue> snap2 = target.AcquireReadSnapshot();

                using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> instance = snap1.CreateReadSnapshot())
                {
                    SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = instance.GetTreeScanner();
                    scanner.SeekToStart();
                    Assert.AreEqual(false, scanner.Read(key, value));
                }
                using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> instance = snap2.CreateReadSnapshot())
                {
                    SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = instance.GetTreeScanner();
                    scanner.SeekToStart();
                    Assert.AreEqual(true, scanner.Read(key, value));
                    Assert.AreEqual(1uL, key.Timestamp);
                    Assert.AreEqual(2uL, key.PointID);
                    Assert.AreEqual(3uL, value.Value1);
                    Assert.AreEqual(4uL, value.Value2);
                }
                Assert.AreEqual(1uL, target.FirstKey.Timestamp);
                Assert.AreEqual(2uL, target.LastKey.Timestamp);
            }
        }
Esempio n. 14
0
        public void AppendDataToAnExistingFile()
        {
            string         fileName = @"C:\Temp\ArchiveFile.d2";
            HistorianKey   key      = new HistorianKey();
            HistorianValue value    = new HistorianValue();

            using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: false))
                using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
                    using (SortedTreeTableEditor <HistorianKey, HistorianValue> editor = table.BeginEdit())
                    {
                        key.TimestampAsDate = DateTime.Now;
                        key.PointID         = 2;
                        value.AsString      = "Test Append";
                        editor.AddPoint(key, value);
                        editor.Commit();
                    }
            ReadDataFromAFile();
        }
Esempio n. 15
0
        public void TestRollover()
        {
            int       count = 1000000;
            Stopwatch sw    = new Stopwatch();

            using (SortedTreeFile af = SortedTreeFile.CreateInMemory())
                using (SortedTreeTable <AmiKey, AmiKey> table = af.OpenOrCreateTable <AmiKey, AmiKey>(EncodingDefinition.FixedSizeCombinedEncoding))
                {
                    using (SortedTreeTableEditor <AmiKey, AmiKey> edit = table.BeginEdit())
                    {
                        sw.Start();
                        AmiKey key   = new AmiKey();
                        AmiKey value = new AmiKey();

                        for (int x = 0; x < count; x++)
                        {
                            key.Timestamp = (uint)x;
                            edit.AddPoint(key, value);
                        }

                        edit.Commit();
                        sw.Stop();

                        Console.WriteLine(af.ArchiveSize / 1024.0 / 1024.0);
                        Console.WriteLine(count / sw.Elapsed.TotalSeconds / 1000000);
                    }

                    using (SortedTreeFile af2 = SortedTreeFile.CreateInMemory())
                        using (SortedTreeTable <AmiKey, AmiKey> table2 = af2.OpenOrCreateTable <AmiKey, AmiKey>(new EncodingDefinition(EncodingDefinition.FixedSizeCombinedEncoding.KeyValueEncodingMethod, EncodingDefinition.FixedSizeCombinedEncoding.KeyValueEncodingMethod)))
                            using (SortedTreeTableEditor <AmiKey, AmiKey> edit = table2.BeginEdit())
                            {
                                using (SortedTreeTableReadSnapshot <AmiKey, AmiKey> read = table.BeginRead())
                                    using (SortedTreeScannerBase <AmiKey, AmiKey> scan = read.GetTreeScanner())
                                    {
                                        scan.SeekToStart();
                                        edit.AddPoints(scan);
                                    }
                            }

                    Console.WriteLine(count);
                }
        }
Esempio n. 16
0
        private TestResults Test(int pageSize)
        {
            Stats.LookupKeys            = 0;
            DiskIoSession.ReadCount     = 0;
            DiskIoSession.WriteCount    = 0;
            Stats.ChecksumCount         = 0;
            DiskIoSession.Lookups       = 0;
            DiskIoSession.CachedLookups = 0;
            long      cnt;
            Stopwatch sw  = new Stopwatch();
            Stopwatch sw2 = new Stopwatch();

            sw.Start();
            using (SortedTreeTable <HistorianKey, HistorianValue> af = SortedTreeFile.CreateInMemory(blockSize: pageSize).OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
            {
                using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = af.BeginEdit())
                {
                    for (int x = 0; x < 100; x++)
                    {
                        edit.AddPoints(new PointStreamSequential(x * 10000, 10000));
                    }
                    edit.Commit();
                }
                sw.Stop();
                sw2.Start();
                cnt = af.Count();
                sw2.Stop();
            }
            return(new TestResults()
            {
                Count = cnt,
                PageSize = pageSize,
                RateWrite = (float)(1 / sw.Elapsed.TotalSeconds),
                RateRead = (float)(1 / sw2.Elapsed.TotalSeconds),
                ReadCount = DiskIoSession.ReadCount,
                WriteCount = DiskIoSession.WriteCount,
                ChecksumCount = Stats.ChecksumCount,
                Lookups = DiskIoSession.Lookups,
                CachedLookups = DiskIoSession.CachedLookups
            });
        }
 void AddData(SortedTreeTable <HistorianKey, HistorianValue> table, DateTime startTime, TimeSpan stepTime, int countTime, ulong startPoint, ulong stepPoint, ulong countPoint)
 {
     using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = table.BeginEdit())
     {
         HistorianKey   key   = new HistorianKey();
         HistorianValue value = new HistorianValue();
         key.SetMin();
         ulong stepTimeTicks = (ulong)stepTime.Ticks;
         ulong stopTime      = (ulong)(startTime.Ticks + countTime * stepTime.Ticks);
         for (ulong t = (ulong)startTime.Ticks; t < stopTime; t += stepTimeTicks)
         {
             for (ulong v = startPoint; v < startPoint + stepPoint * countPoint; v += stepPoint)
             {
                 key.Timestamp = t;
                 key.PointID   = v;
                 edit.AddPoint(key, value);
             }
         }
         edit.Commit();
     }
 }
Esempio n. 18
0
        public void WriteDataToAFile()
        {
            Logger.Console.Verbose = VerboseLevel.All;
            string fileName = @"C:\Temp\ArchiveFile.d2";

            if (File.Exists(fileName))
            {
                File.Delete(fileName);
            }
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();

            using (SortedTreeFile file = SortedTreeFile.CreateFile(fileName))
                using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
                    using (SortedTreeTableEditor <HistorianKey, HistorianValue> editor = table.BeginEdit())
                    {
                        key.TimestampAsDate = DateTime.Now;
                        key.PointID         = 1;
                        value.AsString      = "Test Write";
                        editor.AddPoint(key, value);
                        editor.Commit();
                    }
        }
Esempio n. 19
0
        SortedTreeTable <HistorianKey, HistorianValue> CreateTable()
        {
            Random         r     = new Random(seed++);
            HistorianKey   key   = new HistorianKey();
            HistorianValue value = new HistorianValue();
            SortedTreeFile file  = SortedTreeFile.CreateInMemory();
            SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding);

            using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = table.BeginEdit())
            {
                for (int x = 0; x < 1000; x++)
                {
                    key.Timestamp   = (ulong)r.Next();
                    key.PointID     = (ulong)r.Next();
                    key.EntryNumber = (ulong)r.Next();
                    edit.AddPoint(key, value);
                }
                edit.Commit();
            }



            return(table);
        }
Esempio n. 20
0
        /// <summary>
        /// Appends this data to this stage. Also queues up for deletion if necessary.
        /// </summary>
        /// <param name="args">arguments handed to this class from either the
        /// PrestageWriter or another StageWriter of a previous generation</param>
        /// <remarks>
        /// This method must be called in a single threaded manner.
        /// </remarks>
        public void AppendData(PrebufferRolloverArgs <TKey, TValue> args)
        {
            if (m_stopped)
            {
                Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost");
                return;
            }
            if (m_disposed)
            {
                Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost");
                return;
            }

            SortedTreeFile file = SortedTreeFile.CreateInMemory(4096);
            SortedTreeTable <TKey, TValue> table = file.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);

            using (SortedTreeTableEditor <TKey, TValue> edit = table.BeginEdit())
            {
                edit.AddPoints(args.Stream);
                edit.Commit();
            }

            bool shouldWait = false;

            //If there is data to write then write it to the current archive.
            lock (m_syncRoot)
            {
                if (m_stopped)
                {
                    Log.Publish(MessageLevel.Info, "No new points can be added. Point queue has been stopped. Data in rollover will be lost");
                    table.Dispose();
                    return;
                }
                if (m_disposed)
                {
                    Log.Publish(MessageLevel.Info, "First stage writer has been disposed. Data in rollover will be lost");
                    table.Dispose();
                    return;
                }

                using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
                {
                    edit.Add(table);
                }
                m_pendingTables1.Add(table);

                if (m_pendingTables1.Count == 10)
                {
                    using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables1.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
                    {
                        SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096);
                        SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
                        using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit())
                        {
                            edit.AddPoints(reader);
                            edit.Commit();
                        }

                        using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
                        {
                            //Add the newly created file.
                            edit.Add(table1);

                            foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables1)
                            {
                                edit.TryRemoveAndDelete(table2.ArchiveId);
                            }
                        }

                        m_pendingTables2.Add(table1);
                        m_pendingTables1.Clear();
                    }
                }

                if (m_pendingTables2.Count == 10)
                {
                    using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(m_pendingTables2.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), true))
                    {
                        SortedTreeFile file1 = SortedTreeFile.CreateInMemory(4096);
                        SortedTreeTable <TKey, TValue> table1 = file1.OpenOrCreateTable <TKey, TValue>(m_settings.EncodingMethod);
                        using (SortedTreeTableEditor <TKey, TValue> edit = table1.BeginEdit())
                        {
                            edit.AddPoints(reader);
                            edit.Commit();
                        }

                        using (ArchiveListEditor <TKey, TValue> edit = m_list.AcquireEditLock())
                        {
                            //Add the newly created file.
                            edit.Add(table1);

                            foreach (SortedTreeTable <TKey, TValue> table2 in m_pendingTables2)
                            {
                                edit.TryRemoveAndDelete(table2.ArchiveId);
                            }
                        }

                        m_pendingTables3.Add(table1);
                        m_pendingTables2.Clear();
                    }
                }

                m_lastCommitedSequenceNumber.Value = args.TransactionId;

                long currentSizeMb = (m_pendingTables1.Sum(x => x.BaseFile.ArchiveSize) + m_pendingTables2.Sum(x => x.BaseFile.ArchiveSize)) >> 20;
                if (currentSizeMb > m_settings.MaximumAllowedMb)
                {
                    shouldWait = true;
                    m_rolloverTask.Start();
                    m_rolloverComplete.Reset();
                }
                else if (currentSizeMb > m_settings.RolloverSizeMb)
                {
                    m_rolloverTask.Start();
                }
                else
                {
                    m_rolloverTask.Start(m_settings.RolloverInterval);
                }
            }

            if (SequenceNumberCommitted != null)
            {
                SequenceNumberCommitted(args.TransactionId);
            }

            if (shouldWait)
            {
                Log.Publish(MessageLevel.NA, MessageFlags.PerformanceIssue, "Queue is full", "Rollover task is taking a long time. A long pause on the inputs is about to occur.");
                m_rolloverComplete.WaitOne();
            }
        }