public void Test() { const int MaxCount = 1000; Stopwatch sw = new Stopwatch(); SortedPointBuffer <HistorianKey, HistorianValue> buffer = new SortedPointBuffer <HistorianKey, HistorianValue>(MaxCount, true); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); Random r = new Random(1); for (int x = 0; x < MaxCount; x++) { key.Timestamp = (ulong)r.Next(); key.PointID = (ulong)x; buffer.TryEnqueue(key, value); } sw.Start(); buffer.IsReadingMode = true; sw.Stop(); System.Console.WriteLine(sw.ElapsedMilliseconds); System.Console.WriteLine(MaxCount / sw.Elapsed.TotalSeconds / 1000000); for (int x = 0; x < MaxCount; x++) { buffer.ReadSorted(x, key, value); System.Console.WriteLine(key.Timestamp.ToString() + "\t" + key.PointID.ToString()); } }
public void BenchmarkOld2(int pointCount) { SortedPointBuffer<HistorianKey, HistorianValue> points = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); for (int x = 0; x < pointCount; x++) { key.PointID = (ulong)x; points.TryEnqueue(key, value); } points.IsReadingMode = true; var sw = new Stopwatch(); sw.Start(); using (var bs = new BinaryStream(true)) { var st = SortedTree<HistorianKey, HistorianValue>.Create(bs, 4096, HistorianFileEncodingDefinition.TypeGuid); st.AddRange(points); //SequentialSortedTreeWriter<HistorianKey, HistorianValue>.Create(bs, 4096, SortedTree.FixedSizeNode, points); } sw.Stop(); System.Console.WriteLine("Points {0}: {1}MPPS", pointCount, (pointCount / sw.Elapsed.TotalSeconds / 1000000).ToString("0.0")); }
public void BenchmarkRandomData(int pointCount) { Stopwatch sw = new Stopwatch(); SortedPointBuffer <HistorianKey, HistorianValue> buffer = new SortedPointBuffer <HistorianKey, HistorianValue>(pointCount, true); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); List <double> times = new List <double>(); for (int cnt = 0; cnt < 10; cnt++) { Random r = new Random(1); buffer.IsReadingMode = false; for (int x = 0; x < pointCount; x++) { key.Timestamp = (ulong)r.Next(); key.PointID = (ulong)x; buffer.TryEnqueue(key, value); } sw.Restart(); buffer.IsReadingMode = true; sw.Stop(); times.Add(sw.Elapsed.TotalSeconds); } times.Sort(); System.Console.WriteLine("{0} points {1}ms {2} Million/second ", pointCount, times[5] * 1000, pointCount / times[5] / 1000000); }
/// <summary> /// Creates a prestage writer. /// </summary> /// <param name="settings">The settings to use for this prebuffer writer</param> /// <param name="onRollover">delegate to call when a file is done with this stage.</param> public PrebufferWriter(PrebufferWriterSettings settings, Action <PrebufferRolloverArgs <TKey, TValue> > onRollover) : base(MessageClass.Framework) { if (settings is null) { throw new ArgumentNullException("settings"); } if (onRollover is null) { throw new ArgumentNullException("onRollover"); } m_settings = settings.CloneReadonly(); m_settings.Validate(); m_performanceLog = Log.RegisterEvent(MessageLevel.Info, MessageFlags.PerformanceIssue, "Queue is full", 0, MessageRate.PerSecond(1), 1); m_currentlyRollingOverFullQueue = false; m_latestTransactionId.Value = 0; m_syncRoot = new object(); m_activeQueue = new SortedPointBuffer <TKey, TValue>(m_settings.MaximumPointCount, true); m_processingQueue = new SortedPointBuffer <TKey, TValue>(m_settings.MaximumPointCount, true); m_activeQueue.IsReadingMode = false; m_processingQueue.IsReadingMode = false; m_onRollover = onRollover; m_waitForEmptyActiveQueue = new SafeManualResetEvent(false); m_rolloverTask = new ScheduledTask(ThreadingMode.DedicatedForeground, ThreadPriority.AboveNormal); m_rolloverTask.Running += m_rolloverTask_Running; m_rolloverTask.UnhandledException += OnProcessException; }
public void BenchmarkRandomData(int pointCount) { Stopwatch sw = new Stopwatch(); var buffer = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true); var key = new HistorianKey(); var value = new HistorianValue(); List<double> times = new List<double>(); for (int cnt = 0; cnt < 10; cnt++) { Random r = new Random(1); buffer.IsReadingMode = false; for (int x = 0; x < pointCount; x++) { key.Timestamp = (ulong)r.Next(); key.PointID = (ulong)x; buffer.TryEnqueue(key, value); } sw.Restart(); buffer.IsReadingMode = true; sw.Stop(); times.Add(sw.Elapsed.TotalSeconds); } times.Sort(); System.Console.WriteLine("{0} points {1}ms {2} Million/second ", pointCount, times[5] * 1000, pointCount / times[5] / 1000000); }
public void Test() { const int MaxCount = 1000; Stopwatch sw = new Stopwatch(); var buffer = new SortedPointBuffer<HistorianKey, HistorianValue>(MaxCount, true); var key = new HistorianKey(); var value = new HistorianValue(); Random r = new Random(1); for (int x = 0; x < MaxCount; x++) { key.Timestamp = (ulong)r.Next(); key.PointID = (ulong)x; buffer.TryEnqueue(key, value); } sw.Start(); buffer.IsReadingMode = true; sw.Stop(); System.Console.WriteLine(sw.ElapsedMilliseconds); System.Console.WriteLine(MaxCount / sw.Elapsed.TotalSeconds / 1000000); for (int x = 0; x < MaxCount; x++) { buffer.ReadSorted(x, key, value); System.Console.WriteLine(key.Timestamp.ToString() + "\t" + key.PointID.ToString()); } }
public void BenchmarkOld(int pointCount) { SortedPointBuffer <HistorianKey, HistorianValue> points = new SortedPointBuffer <HistorianKey, HistorianValue>(pointCount, true); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); for (int x = 0; x < pointCount; x++) { key.PointID = (ulong)x; points.TryEnqueue(key, value); } points.IsReadingMode = true; Stopwatch sw = new Stopwatch(); sw.Start(); using (BinaryStream bs = new BinaryStream(true)) { SortedTree <HistorianKey, HistorianValue> st = SortedTree <HistorianKey, HistorianValue> .Create(bs, 4096, HistorianFileEncodingDefinition.TypeGuid); st.TryAddRange(points); //SequentialSortedTreeWriter<HistorianKey, HistorianValue>.Create(bs, 4096, SortedTree.FixedSizeNode, points); } sw.Stop(); System.Console.WriteLine("Points {0}: {1}MPPS", pointCount, (pointCount / sw.Elapsed.TotalSeconds / 1000000).ToString("0.0")); }
public void CountIO() { Test(1000, false); int pointCount = 10000000; SortedPointBuffer<HistorianKey, HistorianValue> points = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); for (int x = 0; x < pointCount; x++) { key.PointID = (ulong)x; points.TryEnqueue(key, value); } points.IsReadingMode = true; File.Delete(@"C:\Temp\fileTemp.~d2i"); File.Delete(@"C:\Temp\fileTemp.d2i"); Stopwatch sw = new Stopwatch(); sw.Start(); SortedTreeFileSimpleWriter<HistorianKey, HistorianValue>.Create(@"C:\Temp\fileTemp.~d2i", @"C:\Temp\fileTemp.d2i", 4096, null, EncodingDefinition.FixedSizeCombinedEncoding, points); sw.Stop(); System.Console.WriteLine(SimplifiedSubFileStreamIoSession.ReadBlockCount); System.Console.WriteLine(SimplifiedSubFileStreamIoSession.WriteBlockCount); System.Console.WriteLine(sw.Elapsed.TotalSeconds.ToString()); }
public void CountIO() { Test(1000, false); int pointCount = 10000000; SortedPointBuffer <HistorianKey, HistorianValue> points = new SortedPointBuffer <HistorianKey, HistorianValue>(pointCount, true); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); for (int x = 0; x < pointCount; x++) { key.PointID = (ulong)x; points.TryEnqueue(key, value); } points.IsReadingMode = true; File.Delete(@"C:\Temp\fileTemp.~d2i"); File.Delete(@"C:\Temp\fileTemp.d2i"); Stopwatch sw = new Stopwatch(); sw.Start(); SortedTreeFileSimpleWriter <HistorianKey, HistorianValue> .Create(@"C:\Temp\fileTemp.~d2i", @"C:\Temp\fileTemp.d2i", 4096, null, EncodingDefinition.FixedSizeCombinedEncoding, points); sw.Stop(); System.Console.WriteLine(SimplifiedSubFileStreamIoSession.ReadBlockCount); System.Console.WriteLine(SimplifiedSubFileStreamIoSession.WriteBlockCount); System.Console.WriteLine(sw.Elapsed.TotalSeconds.ToString()); }
/// <summary> /// Create a large sorted point buffer /// </summary> /// <param name="capacity"></param> /// <param name="multiplier"></param> public LargeSortedPointBuffer(int capacity, int multiplier) { m_capacity = capacity; m_buffers = new SortedPointBuffer <TKey, TValue> [multiplier]; m_currentBuffer = 0; for (int x = 0; x < m_buffers.Length; x++) { m_buffers[x] = new SortedPointBuffer <TKey, TValue>(capacity, true); } }
public static long ConvertVersion1FileHandleDuplicates(string oldFileName, string newFileName, EncodingDefinition compressionMethod, out long readTime, out long sortTime, out long writeTime) { if (!File.Exists(oldFileName)) { throw new ArgumentException("Old file does not exist", "oldFileName"); } if (File.Exists(newFileName)) { throw new ArgumentException("New file already exists", "newFileName"); } HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); long startTime; int count; // Derived SortedPointBuffer class increments EntryNumbers instead of removing duplicates SortedPointBuffer points; startTime = DateTime.UtcNow.Ticks; using (OldHistorianReader archiveFile = new OldHistorianReader()) { archiveFile.Open(oldFileName); count = archiveFile.PointsArchived; points = new SortedPointBuffer(count, false); foreach (OldHistorianReader.DataPoint point in archiveFile.Read()) { key.Timestamp = (ulong)point.Timestamp.Ticks; key.PointID = (ulong)point.PointID; value.Value1 = BitConvert.ToUInt64(point.Value); value.Value3 = (ulong)point.Flags; points.TryEnqueue(key, value); } } readTime = DateTime.UtcNow.Ticks - startTime; startTime = DateTime.UtcNow.Ticks; points.IsReadingMode = true; sortTime = DateTime.UtcNow.Ticks - startTime; startTime = DateTime.UtcNow.Ticks; SortedTreeFileSimpleWriter <HistorianKey, HistorianValue> .Create(Path.Combine(FilePath.GetDirectoryName(newFileName), FilePath.GetFileNameWithoutExtension(newFileName) + ".~d2i"), newFileName, 4096, null, compressionMethod, points); writeTime = DateTime.UtcNow.Ticks - startTime; return(count); }
/// <summary> /// Processes the rollover of this file. /// </summary> private void m_rolloverTask_Running(object sender, EventArgs <ScheduledTaskRunningReason> e) { //the nature of how the ScheduledTask works //gaurentees that this function will not be called concurrently //The worker can be disposed either via the Stop() method or //the Dispose() method. If via the dispose method, then //don't do any cleanup. if (m_disposed && e.Argument == ScheduledTaskRunningReason.Disposing) { Log.Publish(MessageLevel.Info, "Rollover thread is Disposing"); m_waitForEmptyActiveQueue.Dispose(); return; } lock (m_syncRoot) { int count = m_activeQueue.Count; if (count == 0) { m_waitForEmptyActiveQueue.Set(); return; } //Swap active and processing. SortedPointBuffer <TKey, TValue> swap = m_activeQueue; m_activeQueue = m_processingQueue; m_processingQueue = swap; m_activeQueue.IsReadingMode = false; //Should do nothing, but just to be sure. m_waitForEmptyActiveQueue.Set(); m_currentTransactionIdRollingOver = m_latestTransactionId; m_currentlyRollingOverFullQueue = m_processingQueue.IsFull; } //ToDo: The current inner loop for inserting random data is the sorting process here. //ToDo: If the current speed isn't fast enough, this can be multithreaded to improve //ToDo: the insert performance. However, at this time, the added complexity is //ToDo: not worth it since write speeds are already blazing fast. try { m_processingQueue.IsReadingMode = true; //Very CPU intensive. This does a sort on the incoming measurements. Profiling shows that about 33% of the time is spent sorting elements. PrebufferRolloverArgs <TKey, TValue> args = new PrebufferRolloverArgs <TKey, TValue>(m_processingQueue, m_currentTransactionIdRollingOver); m_onRollover(args); m_processingQueue.IsReadingMode = false; //Clears the queue } catch (Exception ex) { Log.Publish(MessageLevel.Critical, "Rollover process unhandled exception", "The rollover process threw an unhandled exception. There is likely data loss that will result from this exception", null, ex); } m_currentlyRollingOverFullQueue = false; }
public void TestNonSequential(int pointCount, bool verify) { SortedPointBuffer <HistorianKey, HistorianValue> points = new SortedPointBuffer <HistorianKey, HistorianValue>(pointCount, true); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); for (int x = 0; x < pointCount; x++) { key.PointID = (ulong)x; points.TryEnqueue(key, value); } points.IsReadingMode = true; File.Delete(@"C:\Temp\fileTemp.~d2i"); File.Delete(@"C:\Temp\fileTemp.d2i"); SortedTreeFileSimpleWriter <HistorianKey, HistorianValue> .CreateNonSequential(@"C:\Temp\fileTemp.~d2i", @"C:\Temp\fileTemp.d2i", 4096, null, EncodingDefinition.FixedSizeCombinedEncoding, points); if (!verify) { return; } using (SortedTreeFile file = SortedTreeFile.OpenFile(@"C:\Temp\fileTemp.d2i", true)) using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>()) using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> read = table.AcquireReadSnapshot().CreateReadSnapshot()) using (SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = read.GetTreeScanner()) { scanner.SeekToStart(); int cnt = 0; while (scanner.Read(key, value)) { if (key.PointID != (ulong)cnt) { throw new Exception(); } cnt++; } if (cnt != pointCount) { throw new Exception(); } } }
public void TestOld() { Test(1000, false); int pointCount = 10000000; SortedPointBuffer<HistorianKey, HistorianValue> points = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); for (int x = 0; x < pointCount; x++) { key.PointID = (ulong)x; points.TryEnqueue(key, value); } points.IsReadingMode = true; File.Delete(@"C:\Temp\fileTemp.~d2i"); File.Delete(@"C:\Temp\fileTemp.d2i"); Stopwatch sw = new Stopwatch(); sw.Start(); using (var file = SortedTreeFile.CreateFile(@"C:\Temp\fileTemp.~d2i")) using (var table = file.OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding)) { using (var edit = table.BeginEdit()) { edit.AddPoints(points); edit.Commit(); } } //SortedTreeFileSimpleWriter<HistorianKey, HistorianValue>.Create(@"C:\Temp\fileTemp.~d2i", @"C:\Temp\fileTemp.d2i", 4096, SortedTree.FixedSizeNode, points); sw.Stop(); System.Console.WriteLine(SimplifiedSubFileStreamIoSession.ReadBlockCount); System.Console.WriteLine(SimplifiedSubFileStreamIoSession.WriteBlockCount); System.Console.WriteLine(sw.Elapsed.TotalSeconds.ToString()); }
public void TestOld() { Test(1000, false); int pointCount = 10000000; SortedPointBuffer <HistorianKey, HistorianValue> points = new SortedPointBuffer <HistorianKey, HistorianValue>(pointCount, true); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); for (int x = 0; x < pointCount; x++) { key.PointID = (ulong)x; points.TryEnqueue(key, value); } points.IsReadingMode = true; File.Delete(@"C:\Temp\fileTemp.~d2i"); File.Delete(@"C:\Temp\fileTemp.d2i"); Stopwatch sw = new Stopwatch(); sw.Start(); using (SortedTreeFile file = SortedTreeFile.CreateFile(@"C:\Temp\fileTemp.~d2i")) using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding)) { using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = table.BeginEdit()) { edit.AddPoints(points); edit.Commit(); } } //SortedTreeFileSimpleWriter<HistorianKey, HistorianValue>.Create(@"C:\Temp\fileTemp.~d2i", @"C:\Temp\fileTemp.d2i", 4096, SortedTree.FixedSizeNode, points); sw.Stop(); System.Console.WriteLine(SimplifiedSubFileStreamIoSession.ReadBlockCount); System.Console.WriteLine(SimplifiedSubFileStreamIoSession.WriteBlockCount); System.Console.WriteLine(sw.Elapsed.TotalSeconds.ToString()); }
public void Test(int pointCount) { SortedPointBuffer <HistorianKey, HistorianValue> points = new SortedPointBuffer <HistorianKey, HistorianValue>(pointCount, true); Random r = new Random(1); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); for (int x = 0; x < pointCount; x++) { key.PointID = (ulong)r.Next(); key.Timestamp = (ulong)r.Next(); value.Value1 = key.PointID; points.TryEnqueue(key, value); } points.IsReadingMode = true; using (BinaryStream bs = new BinaryStream(true)) { //var tree = new SequentialSortedTreeWriter<HistorianKey, HistorianValue>(bs, 256, SortedTree.FixedSizeNode); //SequentialSortedTreeWriter<HistorianKey, HistorianValue>.Create(bs, 512, CreateTsCombinedEncoding.TypeGuid, points); SequentialSortedTreeWriter <HistorianKey, HistorianValue> .Create(bs, 512, EncodingDefinition.FixedSizeCombinedEncoding, points); SortedTree <HistorianKey, HistorianValue> sts = SortedTree <HistorianKey, HistorianValue> .Open(bs); r = new Random(1); for (int x = 0; x < pointCount; x++) { key.PointID = (ulong)r.Next(); key.Timestamp = (ulong)r.Next(); sts.Get(key, value); if (value.Value1 != key.PointID) { throw new Exception(); } } } }
public static void CreateNonSequential(string pendingFileName, string completeFileName, int blockSize, Action <Guid> archiveIdCallback, EncodingDefinition treeNodeType, TreeStream <TKey, TValue> treeStream, params Guid[] flags) { SortedPointBuffer <TKey, TValue> m_queue; m_queue = new SortedPointBuffer <TKey, TValue>(100000, true); m_queue.IsReadingMode = false; TKey key = new TKey(); TValue value = new TValue(); List <SortedTreeTable <TKey, TValue> > pendingFiles = new List <SortedTreeTable <TKey, TValue> >(); try { while (treeStream.Read(key, value)) { if (m_queue.IsFull) { pendingFiles.Add(CreateMemoryFile(treeNodeType, m_queue)); } m_queue.TryEnqueue(key, value); } if (m_queue.Count > 0) { pendingFiles.Add(CreateMemoryFile(treeNodeType, m_queue)); } using (UnionTreeStream <TKey, TValue> reader = new UnionTreeStream <TKey, TValue>(pendingFiles.Select(x => new ArchiveTreeStreamWrapper <TKey, TValue>(x)), false)) { Create(pendingFileName, completeFileName, blockSize, archiveIdCallback, treeNodeType, reader, flags); } } finally { pendingFiles.ForEach(x => x.Dispose()); } }
private static SortedTreeTable <TKey, TValue> CreateMemoryFile(EncodingDefinition treeNodeType, SortedPointBuffer <TKey, TValue> buffer) { buffer.IsReadingMode = true; SortedTreeFile file = SortedTreeFile.CreateInMemory(4096); SortedTreeTable <TKey, TValue> table = file.OpenOrCreateTable <TKey, TValue>(treeNodeType); using (SortedTreeTableEditor <TKey, TValue> edit = table.BeginEdit()) { edit.AddPoints(buffer); edit.Commit(); } buffer.IsReadingMode = false; return(table); }
public void Test(int pointCount) { SortedPointBuffer<HistorianKey, HistorianValue> points = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true); var r = new Random(1); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); for (int x = 0; x < pointCount; x++) { key.PointID = (ulong)r.Next(); key.Timestamp = (ulong)r.Next(); value.Value1 = key.PointID; points.TryEnqueue(key, value); } points.IsReadingMode = true; using (var bs = new BinaryStream(true)) { //var tree = new SequentialSortedTreeWriter<HistorianKey, HistorianValue>(bs, 256, SortedTree.FixedSizeNode); //SequentialSortedTreeWriter<HistorianKey, HistorianValue>.Create(bs, 512, CreateTsCombinedEncoding.TypeGuid, points); SequentialSortedTreeWriter<HistorianKey, HistorianValue>.Create(bs, 512, EncodingDefinition.FixedSizeCombinedEncoding, points); var sts = SortedTree<HistorianKey, HistorianValue>.Open(bs); r = new Random(1); for (int x = 0; x < pointCount; x++) { key.PointID = (ulong)r.Next(); key.Timestamp = (ulong)r.Next(); sts.Get(key, value); if (value.Value1 != key.PointID) throw new Exception(); } } }
public void TestNonSequential(int pointCount, bool verify) { SortedPointBuffer<HistorianKey, HistorianValue> points = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); for (int x = 0; x < pointCount; x++) { key.PointID = (ulong)x; points.TryEnqueue(key, value); } points.IsReadingMode = true; File.Delete(@"C:\Temp\fileTemp.~d2i"); File.Delete(@"C:\Temp\fileTemp.d2i"); SortedTreeFileSimpleWriter<HistorianKey, HistorianValue>.CreateNonSequential(@"C:\Temp\fileTemp.~d2i", @"C:\Temp\fileTemp.d2i", 4096, null, EncodingDefinition.FixedSizeCombinedEncoding, points); if (!verify) return; using (var file = SortedTreeFile.OpenFile(@"C:\Temp\fileTemp.d2i", true)) using (var table = file.OpenTable<HistorianKey, HistorianValue>()) using (var read = table.AcquireReadSnapshot().CreateReadSnapshot()) using (var scanner = read.GetTreeScanner()) { scanner.SeekToStart(); int cnt = 0; while (scanner.Read(key, value)) { if (key.PointID != (ulong)cnt) throw new Exception(); cnt++; } if (cnt != pointCount) throw new Exception(); } }