public void GetBits()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
StringBuilder sb = new StringBuilder();
sb.AppendLine("Higher Bits, Bucket Number, Count, FloatValue");
using (SortedTreeFile file = SortedTreeFile.OpenFile(@"C:\Archive\635184227258021940-Stage2-8b835d6a-8299-45bb-9624-d4a470e4abe1.d2", true))
using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead())
using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = reader.GetTreeScanner())
{
int count = 0;
scan.SeekToStart();
while (scan.Read(key, value))
{
count++;
}
for (int x = 1; x < 24; x++)
{
scan.SeekToStart();
int[] bucket = MeasureBits(scan, x);
Write(sb, bucket, x, count);
}
}
Console.WriteLine(sb.ToString());
}
public void GetDifference()
{
StringBuilder sb = new StringBuilder();
sb.AppendLine("Bucket Number, Count");
using (SortedTreeFile file = SortedTreeFile.OpenFile(@"C:\Unison\GPA\Codeplex\openHistorian\Main\Build\Output\Release\Applications\openHistorian\Archive\635293583194231435-Stage2-0ef36dcc-4264-498f-b194-01b2043a9231.d2", true))
using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead())
using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = reader.GetTreeScanner())
{
HistorianKey key1 = new HistorianKey();
HistorianKey key2 = new HistorianKey();
HistorianValue value = new HistorianValue();
int count = 0;
scan.SeekToStart();
while (scan.Read(key1, value))
{
count++;
}
int[] bucket = new int[130];
scan.SeekToStart();
while (true)
{
if (!scan.Read(key1, value))
{
break;
}
if (key1.Timestamp == key2.Timestamp)
{
int diff = Math.Abs((int)(key1.PointID - key2.PointID));
diff = Math.Min(129, diff);
bucket[diff]++;
}
if (!scan.Read(key2, value))
{
break;
}
if (key1.Timestamp == key2.Timestamp)
{
int diff = Math.Abs((int)(key1.PointID - key2.PointID));
diff = Math.Min(129, diff);
bucket[diff]++;
}
}
for (uint x = 0; x < bucket.Length; x++)
{
sb.AppendLine(x.ToString() + "," + (bucket[x] / (double)count * 100.0).ToString("0.00"));
}
}
Console.WriteLine(sb.ToString());
}
/// <summary>
/// Creates a <see cref="ArchiveTreeStreamWrapper{TKey,TValue}"/>
/// </summary>
/// <param name="table">The table to wrap.</param>
public ArchiveTreeStreamWrapper(ArchiveTableSummary <TKey, TValue> table)
{
m_table = table;
m_snapshot = m_table.ActiveSnapshotInfo.CreateReadSnapshot();
m_scanner = m_snapshot.GetTreeScanner();
m_scanner.SeekToStart();
}
public void BenchmarkRawFile()
{
MemoryPoolTest.TestMemoryLeak();
const int Max = 1000000;
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
SortedTreeTable <HistorianKey, HistorianValue> master = CreateTable();
AddData(master, 100, 100, Max);
DebugStopwatch sw = new DebugStopwatch();
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> masterRead = master.BeginRead())
{
double sec = sw.TimeEvent(() =>
{
SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = masterRead.GetTreeScanner();
scanner.SeekToStart();
while (scanner.Read(key, value))
{
}
});
System.Console.WriteLine(Max / sec / 1000000);
}
master.Dispose();
MemoryPoolTest.TestMemoryLeak();
}
public void TestOneFile()
{
HistorianKey key1 = new HistorianKey();
HistorianKey key2 = new HistorianKey();
HistorianValue value1 = new HistorianValue();
HistorianValue value2 = new HistorianValue();
Logger.Console.Verbose = VerboseLevel.All;
MemoryPoolTest.TestMemoryLeak();
ArchiveList <HistorianKey, HistorianValue> list = new ArchiveList <HistorianKey, HistorianValue>(null);
SortedTreeTable <HistorianKey, HistorianValue> master = CreateTable();
SortedTreeTable <HistorianKey, HistorianValue> table1 = CreateTable();
AddData(master, 100, 100, 100);
AddData(table1, 100, 100, 100);
using (ArchiveListEditor <HistorianKey, HistorianValue> editor = list.AcquireEditLock())
{
editor.Add(table1);
}
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> masterRead = master.BeginRead())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> masterScan = masterRead.GetTreeScanner();
masterScan.SeekToStart();
TreeStreamSequential <HistorianKey, HistorianValue> masterScanSequential = masterScan.TestSequential();
using (SequentialReaderStream <HistorianKey, HistorianValue> sequencer = new SequentialReaderStream <HistorianKey, HistorianValue>(list))
{
TreeStreamSequential <HistorianKey, HistorianValue> scanner = sequencer.TestSequential();
int count = 0;
while (scanner.Read(key1, value1))
{
count++;
if (!masterScanSequential.Read(key2, value2))
{
throw new Exception();
}
if (!key1.IsEqualTo(key2))
{
throw new Exception();
}
if (!value1.IsEqualTo(value2))
{
throw new Exception();
}
}
if (masterScan.Read(key2, value2))
{
throw new Exception();
}
}
}
list.Dispose();
master.Dispose();
MemoryPoolTest.TestMemoryLeak();
}
public void TestSmall()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (SortedTreeFile af = SortedTreeFile.CreateInMemory())
using (SortedTreeTable <HistorianKey, HistorianValue> file = af.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = file.BeginEdit())
{
for (int x = 0; x < 10000000; x++)
{
key.Timestamp = (ulong)x;
edit.AddPoint(key, value);
}
edit.Commit();
}
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> read = file.BeginRead())
using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = read.GetTreeScanner())
{
int count = 0;
scan.SeekToStart();
while (scan.Read(key, value))
{
count++;
}
System.Console.WriteLine(count.ToString());
}
}
}
public void ReadFromAFileWhileWritingToIt()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
string fileName = @"C:\Temp\ArchiveFile.d2";
using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: false))
using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
{
using (SortedTreeTableEditor <HistorianKey, HistorianValue> writer = table.BeginEdit())
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = reader.GetTreeScanner();
scanner.SeekToStart();
while (scanner.Read(key, value))
{
key.Timestamp++;
value.Value1++;
writer.AddPoint(key, value);
}
writer.Commit();
}
}
ReadDataFromAFile();
}
private void StartStream(object args)
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (HistorianClient client = new HistorianClient("127.0.0.1", 54996))
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = client.GetDatabase <HistorianKey, HistorianValue>(string.Empty))
{
using (SortedTreeTable <HistorianKey, HistorianValue> file = SortedTreeFile.OpenFile(@"H:\OGE 2009.d2", isReadOnly: true).OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> read = file.BeginRead())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> scan = read.GetTreeScanner();
scan.SeekToStart();
long count = 0;
while (scan.Read(key, value))
{
count++;
database.Write(key, value);
if ((count % 10) == 1)
{
Thread.Sleep(1);
}
}
}
}
}
}
public void TestFile()
{
// FilePath "C:\\Temp\\Historian\\635287587300536177-Stage1-d559e63e-d938-46a9-8d57-268f7c8ba194.d2" string
//635329017197429979-Stage1-38887e11-4097-4937-b269-ce4037157691.d2
//using (var file = SortedTreeFile.OpenFile(@"C:\Temp\Historian\635287587300536177-Stage1-d559e63e-d938-46a9-8d57-268f7c8ba194.d2", true))
using (SortedTreeFile file = SortedTreeFile.OpenFile(@"C:\Archive\635329017197429979-Stage1-38887e11-4097-4937-b269-ce4037157691.d2", true))
//using (var file = SortedTreeFile.OpenFile(@"C:\Temp\Historian\635255664136496199-Stage2-6e758046-b2af-40ff-ae4e-85cd0c0e4501.d2", true))
using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = reader.GetTreeScanner();
scanner.SeekToStart();
scanner.TestSequential().Count();
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
int x = 0;
scanner.Peek(key, value);
while (scanner.Read(key, value) && x < 10000)
{
System.Console.WriteLine(key.PointID);
x++;
scanner.Peek(key, value);
}
scanner.Count();
}
}
public static long Count(this SortedTreeTable <HistorianKey, HistorianValue> stream)
{
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> read = stream.BeginRead())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> scan = read.GetTreeScanner();
scan.SeekToStart();
return(scan.Count());
}
}
public void Test()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (SortedTreeFile file = SortedTreeFile.OpenFile(@"C:\Unison\GPA\Codeplex\openHistorian\Main\Build\Output\Release\Applications\openHistorian\Archive\635293583194231435-Stage2-0ef36dcc-4264-498f-b194-01b2043a9231.d2", true))
using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead())
using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = reader.GetTreeScanner())
{
scan.SeekToStart();
while (scan.Read(key, value))
{
;
}
}
}
public void CreateSnapshotTest()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
key.Timestamp = 1;
key.PointID = 2;
value.Value1 = 3;
value.Value2 = 4;
using (SortedTreeTable <HistorianKey, HistorianValue> target = SortedTreeFile.CreateInMemory().OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
ulong date = 1;
ulong pointId = 2;
ulong value1 = 3;
ulong value2 = 4;
SortedTreeTableSnapshotInfo <HistorianKey, HistorianValue> snap1;
using (SortedTreeTableEditor <HistorianKey, HistorianValue> fileEditor = target.BeginEdit())
{
fileEditor.AddPoint(key, value);
key.Timestamp++;
fileEditor.AddPoint(key, value);
snap1 = target.AcquireReadSnapshot();
fileEditor.Commit();
}
SortedTreeTableSnapshotInfo <HistorianKey, HistorianValue> snap2 = target.AcquireReadSnapshot();
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> instance = snap1.CreateReadSnapshot())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = instance.GetTreeScanner();
scanner.SeekToStart();
Assert.AreEqual(false, scanner.Read(key, value));
}
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> instance = snap2.CreateReadSnapshot())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = instance.GetTreeScanner();
scanner.SeekToStart();
Assert.AreEqual(true, scanner.Read(key, value));
Assert.AreEqual(1uL, key.Timestamp);
Assert.AreEqual(2uL, key.PointID);
Assert.AreEqual(3uL, value.Value1);
Assert.AreEqual(4uL, value.Value2);
}
Assert.AreEqual(1uL, target.FirstKey.Timestamp);
Assert.AreEqual(2uL, target.LastKey.Timestamp);
}
}
public void TestNonSequential(int pointCount, bool verify)
{
SortedPointBuffer <HistorianKey, HistorianValue> points = new SortedPointBuffer <HistorianKey, HistorianValue>(pointCount, true);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
for (int x = 0; x < pointCount; x++)
{
key.PointID = (ulong)x;
points.TryEnqueue(key, value);
}
points.IsReadingMode = true;
File.Delete(@"C:\Temp\fileTemp.~d2i");
File.Delete(@"C:\Temp\fileTemp.d2i");
SortedTreeFileSimpleWriter <HistorianKey, HistorianValue> .CreateNonSequential(@"C:\Temp\fileTemp.~d2i", @"C:\Temp\fileTemp.d2i", 4096, null, EncodingDefinition.FixedSizeCombinedEncoding, points);
if (!verify)
{
return;
}
using (SortedTreeFile file = SortedTreeFile.OpenFile(@"C:\Temp\fileTemp.d2i", true))
using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> read = table.AcquireReadSnapshot().CreateReadSnapshot())
using (SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = read.GetTreeScanner())
{
scanner.SeekToStart();
int cnt = 0;
while (scanner.Read(key, value))
{
if (key.PointID != (ulong)cnt)
{
throw new Exception();
}
cnt++;
}
if (cnt != pointCount)
{
throw new Exception();
}
}
}
public void ConcurrentReadingFromAFile()
{
string fileName = @"C:\Temp\ArchiveFile.d2";
HistorianKey key1 = new HistorianKey();
HistorianKey key2 = new HistorianKey();
HistorianValue value1 = new HistorianValue();
HistorianValue value2 = new HistorianValue();
using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: true))
using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
{
SortedTreeTableSnapshotInfo <HistorianKey, HistorianValue> snapshotInfo = table.AcquireReadSnapshot();
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader1 = snapshotInfo.CreateReadSnapshot())
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader2 = snapshotInfo.CreateReadSnapshot())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> scanner1 = reader1.GetTreeScanner();
SortedTreeScannerBase <HistorianKey, HistorianValue> scanner2 = reader2.GetTreeScanner();
scanner1.SeekToStart();
scanner2.SeekToStart();
bool scanner1Read = scanner1.Read(key1, value1);
bool scanner2Read = scanner2.Read(key2, value2);
while (scanner1Read && scanner2Read)
{
Assert.AreEqual(key1.Timestamp, key2.Timestamp);
Assert.AreEqual(key1.PointID, key2.PointID);
Assert.AreEqual(key1.EntryNumber, key2.EntryNumber);
Assert.AreEqual(value1.Value1, value2.Value1);
Assert.AreEqual(value1.Value2, value2.Value2);
Assert.AreEqual(value1.Value3, value2.Value3);
scanner1Read = scanner1.Read(key1, value1);
scanner2Read = scanner2.Read(key2, value2);
}
}
}
ReadDataFromAFile();
}
public void TestRollover()
{
int count = 1000000;
Stopwatch sw = new Stopwatch();
using (SortedTreeFile af = SortedTreeFile.CreateInMemory())
using (SortedTreeTable <AmiKey, AmiKey> table = af.OpenOrCreateTable <AmiKey, AmiKey>(EncodingDefinition.FixedSizeCombinedEncoding))
{
using (SortedTreeTableEditor <AmiKey, AmiKey> edit = table.BeginEdit())
{
sw.Start();
AmiKey key = new AmiKey();
AmiKey value = new AmiKey();
for (int x = 0; x < count; x++)
{
key.Timestamp = (uint)x;
edit.AddPoint(key, value);
}
edit.Commit();
sw.Stop();
Console.WriteLine(af.ArchiveSize / 1024.0 / 1024.0);
Console.WriteLine(count / sw.Elapsed.TotalSeconds / 1000000);
}
using (SortedTreeFile af2 = SortedTreeFile.CreateInMemory())
using (SortedTreeTable <AmiKey, AmiKey> table2 = af2.OpenOrCreateTable <AmiKey, AmiKey>(new EncodingDefinition(EncodingDefinition.FixedSizeCombinedEncoding.KeyValueEncodingMethod, EncodingDefinition.FixedSizeCombinedEncoding.KeyValueEncodingMethod)))
using (SortedTreeTableEditor <AmiKey, AmiKey> edit = table2.BeginEdit())
{
using (SortedTreeTableReadSnapshot <AmiKey, AmiKey> read = table.BeginRead())
using (SortedTreeScannerBase <AmiKey, AmiKey> scan = read.GetTreeScanner())
{
scan.SeekToStart();
edit.AddPoints(scan);
}
}
Console.WriteLine(count);
}
}
public void ReadDataFromAFile()
{
string fileName = @"C:\Temp\ArchiveFile.d2";
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: true))
using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> snapshot = table.BeginRead())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = snapshot.GetTreeScanner();
scanner.SeekToStart();
while (scanner.Read(key, value))
{
Console.WriteLine("{0}, {1}, {2}",
key.TimestampAsDate.ToString(), key.PointID, value.AsString);
}
}
}
public void TestReadDataFromArchive()
{
DebugStopwatch sw = new DebugStopwatch();
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
string path = Directory.GetFiles(@"c:\temp\Scada\", "*.d2")[0];
double time;
double count = 0;
using (SortedTreeFile file = SortedTreeFile.OpenFile(path, true))
{
SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>();
time = sw.TimeEvent(() =>
{
count = 0;
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> scan = table.BeginRead())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> t = scan.GetTreeScanner();
t.SeekToStart();
while (t.Read(key, value))
{
count++;
}
}
});
}
Console.WriteLine((count / 1000000 / time).ToString() + " Million PPS");
//Console.WriteLine("KeyMethodsBase calls");
//for (int x = 0; x < 23; x++)
//{
// Console.WriteLine(TreeKeyMethodsBase<HistorianKey>.CallMethods[x] + "\t" + ((TreeKeyMethodsBase<HistorianKey>.Method)(x)).ToString());
//}
//Console.WriteLine("ValueMethodsBase calls");
//for (int x = 0; x < 5; x++)
//{
// Console.WriteLine(TreeValueMethodsBase<HistorianValue>.CallMethods[x] + "\t" + ((TreeValueMethodsBase<HistorianValue>.Method)(x)).ToString());
//}
}
public void ReadFile()
{
using (SortedTreeFile af = SortedTreeFile.OpenFile("c:\\temp\\ArchiveTestFileBig.d2", isReadOnly: true))
using (SortedTreeTable <HistorianKey, HistorianValue> af2 = af.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
Random r = new Random(3);
SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = af2.AcquireReadSnapshot().CreateReadSnapshot().GetTreeScanner();
scanner.SeekToStart();
for (ulong v1 = 1; v1 < 36; v1++)
{
for (ulong v2 = 1; v2 < 86000; v2++)
{
Assert.IsTrue(scanner.Read(key, value));
Assert.AreEqual(key.Timestamp, v1 * 2342523);
Assert.AreEqual(key.PointID, v2);
Assert.AreEqual(value.Value3, 0ul);
Assert.AreEqual(value.Value1, (ulong)r.Next());
}
}
}
}