public void TestFile() { // FilePath "C:\\Temp\\Historian\\635287587300536177-Stage1-d559e63e-d938-46a9-8d57-268f7c8ba194.d2" string //635329017197429979-Stage1-38887e11-4097-4937-b269-ce4037157691.d2 //using (var file = SortedTreeFile.OpenFile(@"C:\Temp\Historian\635287587300536177-Stage1-d559e63e-d938-46a9-8d57-268f7c8ba194.d2", true)) using (var file = SortedTreeFile.OpenFile(@"C:\Archive\635329017197429979-Stage1-38887e11-4097-4937-b269-ce4037157691.d2", true)) //using (var file = SortedTreeFile.OpenFile(@"C:\Temp\Historian\635255664136496199-Stage2-6e758046-b2af-40ff-ae4e-85cd0c0e4501.d2", true)) using (var table = file.OpenTable <HistorianKey, HistorianValue>()) using (var reader = table.BeginRead()) { var scanner = reader.GetTreeScanner(); scanner.SeekToStart(); scanner.TestSequential().Count(); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); int x = 0; scanner.Peek(key, value); while (scanner.Read(key, value) && x < 10000) { System.Console.WriteLine(key.PointID); x++; scanner.Peek(key, value); } scanner.Count(); } }
public void GetDifference() { StringBuilder sb = new StringBuilder(); sb.AppendLine("Bucket Number, Count"); using (SortedTreeFile file = SortedTreeFile.OpenFile(@"C:\Unison\GPA\Codeplex\openHistorian\Main\Build\Output\Release\Applications\openHistorian\Archive\635293583194231435-Stage2-0ef36dcc-4264-498f-b194-01b2043a9231.d2", true)) using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>()) using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead()) using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = reader.GetTreeScanner()) { HistorianKey key1 = new HistorianKey(); HistorianKey key2 = new HistorianKey(); HistorianValue value = new HistorianValue(); int count = 0; scan.SeekToStart(); while (scan.Read(key1, value)) { count++; } int[] bucket = new int[130]; scan.SeekToStart(); while (true) { if (!scan.Read(key1, value)) { break; } if (key1.Timestamp == key2.Timestamp) { int diff = Math.Abs((int)(key1.PointID - key2.PointID)); diff = Math.Min(129, diff); bucket[diff]++; } if (!scan.Read(key2, value)) { break; } if (key1.Timestamp == key2.Timestamp) { int diff = Math.Abs((int)(key1.PointID - key2.PointID)); diff = Math.Min(129, diff); bucket[diff]++; } } for (uint x = 0; x < bucket.Length; x++) { sb.AppendLine(x.ToString() + "," + (bucket[x] / (double)count * 100.0).ToString("0.00")); } } Console.WriteLine(sb.ToString()); }
private void StartStream(object args) { HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); using (HistorianClient client = new HistorianClient("127.0.0.1", 54996)) using (ClientDatabaseBase <HistorianKey, HistorianValue> database = client.GetDatabase <HistorianKey, HistorianValue>(string.Empty)) { using (SortedTreeTable <HistorianKey, HistorianValue> file = SortedTreeFile.OpenFile(@"H:\OGE 2009.d2", isReadOnly: true).OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding)) { using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> read = file.BeginRead()) { SortedTreeScannerBase <HistorianKey, HistorianValue> scan = read.GetTreeScanner(); scan.SeekToStart(); long count = 0; while (scan.Read(key, value)) { count++; database.Write(key, value); if ((count % 10) == 1) { Thread.Sleep(1); } } } } } }
public void RollbackChangesToAFile() { string fileName = @"C:\Temp\ArchiveFile.d2"; HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: false)) using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding)) { using (SortedTreeTableEditor <HistorianKey, HistorianValue> editor = table.BeginEdit()) { key.TimestampAsDate = DateTime.Now.AddDays(-1); key.PointID = 234; value.AsString = "Add Me"; editor.AddPoint(key, value); editor.Commit(); } using (SortedTreeTableEditor <HistorianKey, HistorianValue> editor = table.BeginEdit()) { key.Timestamp = 31; value.AsString = "But Not Me"; editor.AddPoint(key, value); editor.Rollback(); //These changes will not be written to the disk } } ReadDataFromAFile(); }
public void GetBits() { HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); StringBuilder sb = new StringBuilder(); sb.AppendLine("Higher Bits, Bucket Number, Count, FloatValue"); using (SortedTreeFile file = SortedTreeFile.OpenFile(@"C:\Archive\635184227258021940-Stage2-8b835d6a-8299-45bb-9624-d4a470e4abe1.d2", true)) using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>()) using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead()) using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = reader.GetTreeScanner()) { int count = 0; scan.SeekToStart(); while (scan.Read(key, value)) { count++; } for (int x = 1; x < 24; x++) { scan.SeekToStart(); int[] bucket = MeasureBits(scan, x); Write(sb, bucket, x, count); } } Console.WriteLine(sb.ToString()); }
public void ReadDataFromAFile() { string fileName = @"c:\temp\Tulsa Bank 1 LTC 1.d2"; DateTime start = DateTime.Parse("4/17/2013 10:38 AM"); DateTime stop = DateTime.Parse("4/17/2013 10:38 AM"); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); using (var file = SortedTreeFile.OpenFile(fileName, isReadOnly: true)) using (var table = file.OpenTable <HistorianKey, HistorianValue>()) using (var snapshot = table.BeginRead()) { var scanner = snapshot.GetTreeScanner(); var seekKey = new HistorianKey(); seekKey.TimestampAsDate = start; seekKey.PointID = 3142023; scanner.SeekToKey(seekKey); while (scanner.Read(key, value) && key.TimestampAsDate <= stop) { Console.WriteLine("{0}, {1}, {2}", key.TimestampAsDate.ToString(), key.PointID, value.AsString); } } }
public void ReadFromAFileWhileWritingToIt() { HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); string fileName = @"C:\Temp\ArchiveFile.d2"; using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: false)) using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>()) { using (SortedTreeTableEditor <HistorianKey, HistorianValue> writer = table.BeginEdit()) using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead()) { SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = reader.GetTreeScanner(); scanner.SeekToStart(); while (scanner.Read(key, value)) { key.Timestamp++; value.Value1++; writer.AddPoint(key, value); } writer.Commit(); } } ReadDataFromAFile(); }
/// <summary> /// Creates a new <see cref="SortedTreeTable{TKey,TValue}"/> based on the settings passed to this class. /// Once created, it is up to he caller to make sure that this class is properly disposed of. /// </summary> /// <param name="startKey">the first key in the archive file</param> /// <param name="endKey">the last key in the archive file</param> /// <param name="estimatedSize">The estimated size of the file. -1 to ignore this feature and write to the first available directory.</param> /// <param name="archiveIdCallback">the archiveId to assign to the new file.</param> /// <returns></returns> public SortedTreeTable <TKey, TValue> CreateArchiveFile(TKey startKey, TKey endKey, long estimatedSize, TreeStream <TKey, TValue> data, Action <Guid> archiveIdCallback) { SimplifiedArchiveInitializerSettings settings = m_settings; string pendingFile = CreateArchiveName(GetPathWithEnoughSpace(estimatedSize), startKey, endKey); string finalFile = Path.ChangeExtension(pendingFile, settings.FinalExtension); SortedTreeFileSimpleWriter <TKey, TValue> .Create(pendingFile, finalFile, 4096, archiveIdCallback, settings.EncodingMethod, data, settings.Flags.ToArray()); return(SortedTreeFile.OpenFile(finalFile, true).OpenTable <TKey, TValue>()); }
public void Test() { HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); using (SortedTreeFile file = SortedTreeFile.OpenFile(@"C:\Unison\GPA\Codeplex\openHistorian\Main\Build\Output\Release\Applications\openHistorian\Archive\635293583194231435-Stage2-0ef36dcc-4264-498f-b194-01b2043a9231.d2", true)) using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>()) using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead()) using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = reader.GetTreeScanner()) { scan.SeekToStart(); while (scan.Read(key, value)) { ; } } }
public void AppendDataToAnExistingFile() { string fileName = @"C:\Temp\ArchiveFile.d2"; HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: false)) using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>()) using (SortedTreeTableEditor <HistorianKey, HistorianValue> editor = table.BeginEdit()) { key.TimestampAsDate = DateTime.Now; key.PointID = 2; value.AsString = "Test Append"; editor.AddPoint(key, value); editor.Commit(); } ReadDataFromAFile(); }
public void ConcurrentReadingFromAFile() { string fileName = @"C:\Temp\ArchiveFile.d2"; HistorianKey key1 = new HistorianKey(); HistorianKey key2 = new HistorianKey(); HistorianValue value1 = new HistorianValue(); HistorianValue value2 = new HistorianValue(); using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: true)) using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>()) { SortedTreeTableSnapshotInfo <HistorianKey, HistorianValue> snapshotInfo = table.AcquireReadSnapshot(); using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader1 = snapshotInfo.CreateReadSnapshot()) using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader2 = snapshotInfo.CreateReadSnapshot()) { SortedTreeScannerBase <HistorianKey, HistorianValue> scanner1 = reader1.GetTreeScanner(); SortedTreeScannerBase <HistorianKey, HistorianValue> scanner2 = reader2.GetTreeScanner(); scanner1.SeekToStart(); scanner2.SeekToStart(); bool scanner1Read = scanner1.Read(key1, value1); bool scanner2Read = scanner2.Read(key2, value2); while (scanner1Read && scanner2Read) { Assert.AreEqual(key1.Timestamp, key2.Timestamp); Assert.AreEqual(key1.PointID, key2.PointID); Assert.AreEqual(key1.EntryNumber, key2.EntryNumber); Assert.AreEqual(value1.Value1, value2.Value1); Assert.AreEqual(value1.Value2, value2.Value2); Assert.AreEqual(value1.Value3, value2.Value3); scanner1Read = scanner1.Read(key1, value1); scanner2Read = scanner2.Read(key2, value2); } } } ReadDataFromAFile(); }
public void ReadDataFromAFile() { string fileName = @"C:\Temp\ArchiveFile.d2"; HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: true)) using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>()) using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> snapshot = table.BeginRead()) { SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = snapshot.GetTreeScanner(); scanner.SeekToStart(); while (scanner.Read(key, value)) { Console.WriteLine("{0}, {1}, {2}", key.TimestampAsDate.ToString(), key.PointID, value.AsString); } } }
public void TestReadDataFromArchive() { DebugStopwatch sw = new DebugStopwatch(); HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); string path = Directory.GetFiles(@"c:\temp\Scada\", "*.d2")[0]; double time; double count = 0; using (SortedTreeFile file = SortedTreeFile.OpenFile(path, true)) { var table = file.OpenTable <HistorianKey, HistorianValue>(); time = sw.TimeEvent(() => { count = 0; using (var scan = table.BeginRead()) { var t = scan.GetTreeScanner(); t.SeekToStart(); while (t.Read(key, value)) { count++; } } }); } Console.WriteLine((count / 1000000 / time).ToString() + " Million PPS"); //Console.WriteLine("KeyMethodsBase calls"); //for (int x = 0; x < 23; x++) //{ // Console.WriteLine(TreeKeyMethodsBase<HistorianKey>.CallMethods[x] + "\t" + ((TreeKeyMethodsBase<HistorianKey>.Method)(x)).ToString()); //} //Console.WriteLine("ValueMethodsBase calls"); //for (int x = 0; x < 5; x++) //{ // Console.WriteLine(TreeValueMethodsBase<HistorianValue>.CallMethods[x] + "\t" + ((TreeValueMethodsBase<HistorianValue>.Method)(x)).ToString()); //} }
public void ReadFile() { using (SortedTreeFile af = SortedTreeFile.OpenFile("c:\\temp\\ArchiveTestFileBig.d2", isReadOnly: true)) using (SortedTreeTable <HistorianKey, HistorianValue> af2 = af.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding)) { HistorianKey key = new HistorianKey(); HistorianValue value = new HistorianValue(); Random r = new Random(3); SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = af2.AcquireReadSnapshot().CreateReadSnapshot().GetTreeScanner(); scanner.SeekToStart(); for (ulong v1 = 1; v1 < 36; v1++) { for (ulong v2 = 1; v2 < 86000; v2++) { Assert.IsTrue(scanner.Read(key, value)); Assert.AreEqual(key.Timestamp, v1 * 2342523); Assert.AreEqual(key.PointID, v2); Assert.AreEqual(value.Value3, 0ul); Assert.AreEqual(value.Value1, (ulong)r.Next()); } } } }
/// <summary> /// Loads the specified files into the archive list. /// </summary> /// <param name="archiveFiles"></param> public override void LoadFiles(IEnumerable <string> archiveFiles) { if (m_disposed) { throw new Exception("Object is disposing"); } var loadedFiles = new List <SortedTreeTable <TKey, TValue> >(); foreach (string file in archiveFiles) { try { SortedTreeFile sortedTreeFile = SortedTreeFile.OpenFile(file, isReadOnly: true); var table = sortedTreeFile.OpenTable <TKey, TValue>(); if (table == null) { sortedTreeFile.Dispose(); //archiveFile.Delete(); //ToDo: Consider the consequences of deleting a file. } else { if (m_listLog.ShouldBeDeleted(table.ArchiveId)) { Log.Publish(MessageLevel.Warning, "File being deleted", "The supplied file is being deleted because it was part of a previous rollover that completed but the server crashed before it was properly deleted." + file); table.BaseFile.Delete(); } else { loadedFiles.Add(table); } } Log.Publish(MessageLevel.Info, "Loading Files", "Successfully opened: " + file); } catch (Exception ex) { Log.Publish(MessageLevel.Warning, "Loading Files", "Skipping Failed File: " + file, null, ex); } } using (var edit = AcquireEditLock()) { if (m_disposed) { loadedFiles.ForEach(x => x.Dispose()); throw new Exception("Object is disposing"); } foreach (var file in loadedFiles) { try { edit.Add(file); } catch (Exception ex) { Log.Publish(MessageLevel.Warning, "Attaching File", "File already attached: " + file.ArchiveId, file.BaseFile.FilePath, ex); file.BaseFile.Dispose(); } } } }
private void BtnFindCorruptFiles_Click(object sender, EventArgs e) { DataTable resultsTable = new DataTable(); resultsTable.Columns.Add("File", typeof(string)); resultsTable.Columns.Add("ID", typeof(string)); StringBuilder sb = new StringBuilder(); using (var dlg = new OpenFileDialog()) { dlg.Multiselect = true; dlg.Filter = "Open Historian 2.0 File|*.d2"; if (dlg.ShowDialog() == DialogResult.OK) { foreach (var fileName in dlg.FileNames) { sb.Clear(); try { using (var file = SortedTreeFile.OpenFile(fileName, true)) { sb.AppendFormat("ID: {0} ", file.Snapshot.Header.ArchiveId); sb.AppendFormat("Commit Number: {0} ", file.Snapshot.Header.SnapshotSequenceNumber); var table = file.OpenTable <HistorianKey, HistorianValue>(); if (table == null) { sb.Append("ERROR - No Historian Table "); } else { try { if ((long)table.FirstKey.Timestamp < DateTime.MinValue.Ticks || (long)table.FirstKey.Timestamp > DateTime.MaxValue.Ticks) { sb.Append("Start Time: Invalid "); } else { sb.AppendFormat("Start Time: {0} ", table.FirstKey.TimestampAsDate.ToString()); } if ((long)table.LastKey.Timestamp < DateTime.MinValue.Ticks || (long)table.LastKey.Timestamp > DateTime.MaxValue.Ticks) { sb.Append("End Time: Invalid "); } else { sb.AppendFormat("End Time: {0} ", table.LastKey.TimestampAsDate.ToString()); } } finally { table.Dispose(); } } } } catch (Exception ex) { sb.AppendFormat("File Header Corrupt: {0} ", ex.ToString()); throw; } resultsTable.Rows.Add(fileName, sb.ToString()); } var win = new FrmDisplayFileMetaData(resultsTable); win.Show(); } } }