void ScannerThread()
{
int threadId = Interlocked.Increment(ref ThreadNumber);
try
{
//DateTime start = DateTime.FromBinary(Convert.ToDateTime("2/1/2014").Date.Ticks + Convert.ToDateTime("6:00:00PM").TimeOfDay.Ticks).ToUniversalTime();
while (!StopReading)
{
Stopwatch sw = new Stopwatch();
using (var client = new HistorianClient("127.0.0.1", 12345))
using (var database = client.GetDatabase <HistorianKey, HistorianValue>(String.Empty))
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
sw.Start();
var scan = database.Read(0, ulong.MaxValue, new ulong[] { 65, 953, 5562 });
while (scan.Read(key, value))
{
;
}
sw.Stop();
}
//Console.WriteLine("Thread: " + threadId.ToString() + " " + "Run Number: " + myId.ToString() + " " + (pointCount / sw.Elapsed.TotalSeconds / 1000000).ToString());
}
}
catch (Exception ex)
{
//Console.WriteLine(ex.ToString());
}
Console.WriteLine("Thread: " + threadId.ToString() + " Quit");
}
private void TestFile(int pageSize, string fileName)
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
key.Timestamp = 1;
if (File.Exists(fileName))
{
File.Delete(fileName);
}
Stopwatch sw = new Stopwatch();
sw.Start();
using (SortedTreeTable <HistorianKey, HistorianValue> af = SortedTreeFile.CreateFile(fileName, blockSize: pageSize).OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = af.BeginEdit())
{
for (uint x = 0; x < 1000000; x++)
{
key.PointID = x;
edit.AddPoint(key, value);
}
edit.Commit();
}
sw.Stop();
System.Console.WriteLine("Size: " + pageSize + " Rate: " + (1 / sw.Elapsed.TotalSeconds).ToString());
}
public void RollbackChangesToAFile()
{
string fileName = @"C:\Temp\ArchiveFile.d2";
var key = new HistorianKey();
var value = new HistorianValue();
using (var file = SortedTreeFile.OpenFile(fileName, isReadOnly: false))
using (var table = file.OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
using (var editor = table.BeginEdit())
{
key.TimestampAsDate = DateTime.Now.AddDays(-1);
key.PointID = 234;
value.AsString = "Add Me";
editor.AddPoint(key, value);
editor.Commit();
}
using (var editor = table.BeginEdit())
{
key.Timestamp = 31;
value.AsString = "But Not Me";
editor.AddPoint(key, value);
editor.Rollback(); //These changes will not be written to the disk
}
}
ReadDataFromAFile();
}
public void TestReadPoints()
{
Stopwatch sw = new Stopwatch();
int pointCount = 0;
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("PPA", @"C:\Program Files\openHistorian\Archive\", true);
using (HistorianServer server = new HistorianServer(settings))
{
using (HistorianClient client = new HistorianClient("127.0.0.1", 12345))
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = client.GetDatabase <HistorianKey, HistorianValue>(String.Empty))
{
TreeStream <HistorianKey, HistorianValue> stream = database.Read(0, (ulong)DateTime.MaxValue.Ticks, new ulong[] { 1 });
while (stream.Read(key, value))
{
;
}
sw.Start();
stream = database.Read(0, (ulong)DateTime.MaxValue.Ticks, new ulong[] { 65, 953, 5562 });
while (stream.Read(key, value))
{
pointCount++;
}
sw.Stop();
}
}
Console.WriteLine(pointCount);
Console.WriteLine(sw.Elapsed.TotalSeconds.ToString());
}
public void TestSmall()
{
var key = new HistorianKey();
var value = new HistorianValue();
using (var af = SortedTreeFile.CreateInMemory())
using (var file = af.OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
using (var edit = file.BeginEdit())
{
for (int x = 0; x < 10000000; x++)
{
key.Timestamp = (ulong)x;
edit.AddPoint(key, value);
}
edit.Commit();
}
using (var read = file.BeginRead())
using (var scan = read.GetTreeScanner())
{
int count = 0;
scan.SeekToStart();
while (scan.Read(key,value))
{
count++;
}
System.Console.WriteLine(count.ToString());
}
}
}
public void ReadDataFromAFile()
{
string fileName = @"c:\temp\Tulsa Bank 1 LTC 1.d2";
DateTime start = DateTime.Parse("4/17/2013 10:38 AM");
DateTime stop = DateTime.Parse("4/17/2013 10:38 AM");
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (var file = SortedTreeFile.OpenFile(fileName, isReadOnly: true))
using (var table = file.OpenTable <HistorianKey, HistorianValue>())
using (var snapshot = table.BeginRead())
{
var scanner = snapshot.GetTreeScanner();
var seekKey = new HistorianKey();
seekKey.TimestampAsDate = start;
seekKey.PointID = 3142023;
scanner.SeekToKey(seekKey);
while (scanner.Read(key, value) && key.TimestampAsDate <= stop)
{
Console.WriteLine("{0}, {1}, {2}",
key.TimestampAsDate.ToString(), key.PointID, value.AsString);
}
}
}
public void CountIO()
{
Test(1000, false);
int pointCount = 10000000;
SortedPointBuffer <HistorianKey, HistorianValue> points = new SortedPointBuffer <HistorianKey, HistorianValue>(pointCount, true);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
for (int x = 0; x < pointCount; x++)
{
key.PointID = (ulong)x;
points.TryEnqueue(key, value);
}
points.IsReadingMode = true;
File.Delete(@"C:\Temp\fileTemp.~d2i");
File.Delete(@"C:\Temp\fileTemp.d2i");
Stopwatch sw = new Stopwatch();
sw.Start();
SortedTreeFileSimpleWriter <HistorianKey, HistorianValue> .Create(@"C:\Temp\fileTemp.~d2i", @"C:\Temp\fileTemp.d2i", 4096, null, EncodingDefinition.FixedSizeCombinedEncoding, points);
sw.Stop();
System.Console.WriteLine(SimplifiedSubFileStreamIoSession.ReadBlockCount);
System.Console.WriteLine(SimplifiedSubFileStreamIoSession.WriteBlockCount);
System.Console.WriteLine(sw.Elapsed.TotalSeconds.ToString());
}
public void WriteFile()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
if (File.Exists("c:\\temp\\ArchiveTestFileBig.d2"))
{
File.Delete("c:\\temp\\ArchiveTestFileBig.d2");
}
//using (var af = ArchiveFile.CreateInMemory(CompressionMethod.TimeSeriesEncoded))
using (SortedTreeFile af = SortedTreeFile.CreateFile("c:\\temp\\ArchiveTestFileBig.d2"))
using (SortedTreeTable <HistorianKey, HistorianValue> af2 = af.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
Random r = new Random(3);
for (ulong v1 = 1; v1 < 36; v1++)
{
using (var edit = af2.BeginEdit())
{
for (ulong v2 = 1; v2 < 86000; v2++)
{
key.Timestamp = v1 * 2342523;
key.PointID = v2;
value.Value1 = (ulong)r.Next();
value.Value3 = 0;
edit.AddPoint(key, value);
}
edit.Commit();
}
af2.Count();
}
af2.Count();
}
}
public void EnduranceTest()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (SortedTreeTable <HistorianKey, HistorianValue> target = SortedTreeFile.CreateInMemory().OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
for (uint x = 0; x < 100; x++)
{
using (SortedTreeTableEditor <HistorianKey, HistorianValue> fileEditor = target.BeginEdit())
{
for (int y = 0; y < 10; y++)
{
key.Timestamp = x;
key.PointID = x;
value.Value1 = x;
value.Value3 = x;
fileEditor.AddPoint(key, value);
x++;
}
fileEditor.Commit();
}
Assert.AreEqual(target.FirstKey.Timestamp, 0);
Assert.AreEqual(target.LastKey.Timestamp, x - 1);
}
}
}
private void StartStream(object args)
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (HistorianClient client = new HistorianClient("127.0.0.1", 54996))
using (ClientDatabaseBase<HistorianKey, HistorianValue> database = client.GetDatabase<HistorianKey, HistorianValue>(string.Empty))
{
using (SortedTreeTable<HistorianKey, HistorianValue> file = SortedTreeFile.OpenFile(@"H:\OGE 2009.d2", isReadOnly: true).OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
using (SortedTreeTableReadSnapshot<HistorianKey, HistorianValue> read = file.BeginRead())
{
SortedTreeScannerBase<HistorianKey, HistorianValue> scan = read.GetTreeScanner();
scan.SeekToStart();
long count = 0;
while (scan.Read(key, value))
{
count++;
database.Write(key, value);
if ((count % 10) == 1)
Thread.Sleep(1);
}
}
}
}
}
/// <summary>
/// Initializes a new instance of the <see cref="RemoteOutputAdapter"/> class.
/// </summary>
public RemoteOutputAdapter()
{
m_port = LocalOutputAdapter.DefaultPort;
m_server = DefaultServer;
m_key = new HistorianKey();
m_value = new HistorianValue();
}
public void TestSmall()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (SortedTreeFile af = SortedTreeFile.CreateInMemory())
using (SortedTreeTable <HistorianKey, HistorianValue> file = af.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = file.BeginEdit())
{
for (int x = 0; x < 10000000; x++)
{
key.Timestamp = (ulong)x;
edit.AddPoint(key, value);
}
edit.Commit();
}
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> read = file.BeginRead())
using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = read.GetTreeScanner())
{
int count = 0;
scan.SeekToStart();
while (scan.Read(key, value))
{
count++;
}
System.Console.WriteLine(count.ToString());
}
}
}
public void Test()
{
const int MaxCount = 1000;
Stopwatch sw = new Stopwatch();
SortedPointBuffer <HistorianKey, HistorianValue> buffer = new SortedPointBuffer <HistorianKey, HistorianValue>(MaxCount, true);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
Random r = new Random(1);
for (int x = 0; x < MaxCount; x++)
{
key.Timestamp = (ulong)r.Next();
key.PointID = (ulong)x;
buffer.TryEnqueue(key, value);
}
sw.Start();
buffer.IsReadingMode = true;
sw.Stop();
System.Console.WriteLine(sw.ElapsedMilliseconds);
System.Console.WriteLine(MaxCount / sw.Elapsed.TotalSeconds / 1000000);
for (int x = 0; x < MaxCount; x++)
{
buffer.ReadSorted(x, key, value);
System.Console.WriteLine(key.Timestamp.ToString() + "\t" + key.PointID.ToString());
}
}
public void Test()
{
const int MaxCount = 1000;
Stopwatch sw = new Stopwatch();
var buffer = new SortedPointBuffer<HistorianKey, HistorianValue>(MaxCount, true);
var key = new HistorianKey();
var value = new HistorianValue();
Random r = new Random(1);
for (int x = 0; x < MaxCount; x++)
{
key.Timestamp = (ulong)r.Next();
key.PointID = (ulong)x;
buffer.TryEnqueue(key, value);
}
sw.Start();
buffer.IsReadingMode = true;
sw.Stop();
System.Console.WriteLine(sw.ElapsedMilliseconds);
System.Console.WriteLine(MaxCount / sw.Elapsed.TotalSeconds / 1000000);
for (int x = 0; x < MaxCount; x++)
{
buffer.ReadSorted(x, key, value);
System.Console.WriteLine(key.Timestamp.ToString() + "\t" + key.PointID.ToString());
}
}
public void BenchmarkRandomData(int pointCount)
{
Stopwatch sw = new Stopwatch();
var buffer = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true);
var key = new HistorianKey();
var value = new HistorianValue();
List<double> times = new List<double>();
for (int cnt = 0; cnt < 10; cnt++)
{
Random r = new Random(1);
buffer.IsReadingMode = false;
for (int x = 0; x < pointCount; x++)
{
key.Timestamp = (ulong)r.Next();
key.PointID = (ulong)x;
buffer.TryEnqueue(key, value);
}
sw.Restart();
buffer.IsReadingMode = true;
sw.Stop();
times.Add(sw.Elapsed.TotalSeconds);
}
times.Sort();
System.Console.WriteLine("{0} points {1}ms {2} Million/second ", pointCount, times[5] * 1000, pointCount / times[5] / 1000000);
}
public void GetDifference()
{
StringBuilder sb = new StringBuilder();
sb.AppendLine("Bucket Number, Count");
using (SortedTreeFile file = SortedTreeFile.OpenFile(@"C:\Unison\GPA\Codeplex\openHistorian\Main\Build\Output\Release\Applications\openHistorian\Archive\635293583194231435-Stage2-0ef36dcc-4264-498f-b194-01b2043a9231.d2", true))
using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead())
using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = reader.GetTreeScanner())
{
HistorianKey key1 = new HistorianKey();
HistorianKey key2 = new HistorianKey();
HistorianValue value = new HistorianValue();
int count = 0;
scan.SeekToStart();
while (scan.Read(key1, value))
{
count++;
}
int[] bucket = new int[130];
scan.SeekToStart();
while (true)
{
if (!scan.Read(key1, value))
{
break;
}
if (key1.Timestamp == key2.Timestamp)
{
int diff = Math.Abs((int)(key1.PointID - key2.PointID));
diff = Math.Min(129, diff);
bucket[diff]++;
}
if (!scan.Read(key2, value))
{
break;
}
if (key1.Timestamp == key2.Timestamp)
{
int diff = Math.Abs((int)(key1.PointID - key2.PointID));
diff = Math.Min(129, diff);
bucket[diff]++;
}
}
for (uint x = 0; x < bucket.Length; x++)
{
sb.AppendLine(x.ToString() + "," + (bucket[x] / (double)count * 100.0).ToString("0.00"));
}
}
Console.WriteLine(sb.ToString());
}
private IEnumerable <TrendingDataPoint> Read(IEnumerable <ulong> measurementIDs, DateTime startTime, DateTime stopTime)
{
SeekFilterBase <HistorianKey> timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, stopTime);
MatchFilterBase <HistorianKey, HistorianValue> pointFilter = null;
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
if ((object)measurementIDs != null)
{
pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(measurementIDs);
}
// Start stream reader for the provided time window and selected points
using (TreeStream <HistorianKey, HistorianValue> stream = m_database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter))
{
m_treeStreams.Add(stream);
while (stream.Read(key, value))
{
yield return(new TrendingDataPoint()
{
ChannelID = (int)key.PointID.HighDoubleWord(),
SeriesID = (SeriesID)(int)key.PointID.LowDoubleWord(),
Timestamp = key.TimestampAsDate,
Value = value.AsSingle
});
}
}
}
public void TestFile()
{
// FilePath "C:\\Temp\\Historian\\635287587300536177-Stage1-d559e63e-d938-46a9-8d57-268f7c8ba194.d2" string
//635329017197429979-Stage1-38887e11-4097-4937-b269-ce4037157691.d2
//using (var file = SortedTreeFile.OpenFile(@"C:\Temp\Historian\635287587300536177-Stage1-d559e63e-d938-46a9-8d57-268f7c8ba194.d2", true))
using (var file = SortedTreeFile.OpenFile(@"C:\Archive\635329017197429979-Stage1-38887e11-4097-4937-b269-ce4037157691.d2", true))
//using (var file = SortedTreeFile.OpenFile(@"C:\Temp\Historian\635255664136496199-Stage2-6e758046-b2af-40ff-ae4e-85cd0c0e4501.d2", true))
using (var table = file.OpenTable <HistorianKey, HistorianValue>())
using (var reader = table.BeginRead())
{
var scanner = reader.GetTreeScanner();
scanner.SeekToStart();
scanner.TestSequential().Count();
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
int x = 0;
scanner.Peek(key, value);
while (scanner.Read(key, value) && x < 10000)
{
System.Console.WriteLine(key.PointID);
x++;
scanner.Peek(key, value);
}
scanner.Count();
}
}
public void TestFile()
{
// FilePath "C:\\Temp\\Historian\\635287587300536177-Stage1-d559e63e-d938-46a9-8d57-268f7c8ba194.d2" string
//635329017197429979-Stage1-38887e11-4097-4937-b269-ce4037157691.d2
//using (var file = SortedTreeFile.OpenFile(@"C:\Temp\Historian\635287587300536177-Stage1-d559e63e-d938-46a9-8d57-268f7c8ba194.d2", true))
using (var file = SortedTreeFile.OpenFile(@"C:\Archive\635329017197429979-Stage1-38887e11-4097-4937-b269-ce4037157691.d2", true))
//using (var file = SortedTreeFile.OpenFile(@"C:\Temp\Historian\635255664136496199-Stage2-6e758046-b2af-40ff-ae4e-85cd0c0e4501.d2", true))
using (var table = file.OpenTable<HistorianKey, HistorianValue>())
using (var reader = table.BeginRead())
{
var scanner = reader.GetTreeScanner();
scanner.SeekToStart();
scanner.TestSequential().Count();
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
int x = 0;
scanner.Peek(key, value);
while (scanner.Read(key, value) && x < 10000)
{
System.Console.WriteLine(key.PointID);
x++;
scanner.Peek(key, value);
}
scanner.Count();
}
}
public void ReadFromAFileWhileWritingToIt()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
string fileName = @"C:\Temp\ArchiveFile.d2";
using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: false))
using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
{
using (SortedTreeTableEditor <HistorianKey, HistorianValue> writer = table.BeginEdit())
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = reader.GetTreeScanner();
scanner.SeekToStart();
while (scanner.Read(key, value))
{
key.Timestamp++;
value.Value1++;
writer.AddPoint(key, value);
}
writer.Commit();
}
}
ReadDataFromAFile();
}
/// <summary>
/// Initializes a new instance of the <see cref="RemoteOutputAdapter"/> class.
/// </summary>
public RemoteOutputAdapter()
{
m_port = LocalOutputAdapter.DefaultPort;
m_server = DefaultServer;
m_key = new HistorianKey();
m_value = new HistorianValue();
}
void AddDataTerminal(SortedTreeTable <HistorianKey, HistorianValue> table, ulong pointID, DateTime startTime, TimeSpan stepTime, ulong startValue, ulong stepValue, int count)
{
using (SortedTreeTableEditor <HistorianKey, HistorianValue> edit = table.BeginEdit())
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
key.SetMin();
ulong t = (ulong)startTime.Ticks;
ulong v = startValue;
while (count > 0)
{
count--;
key.Timestamp = t;
key.PointID = pointID;
value.Value1 = v;
edit.AddPoint(key, value);
t += (ulong)stepTime.Ticks;
v += stepValue;
}
edit.Commit();
}
}
public void BenchmarkRawFile()
{
MemoryPoolTest.TestMemoryLeak();
const int Max = 1000000;
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
SortedTreeTable <HistorianKey, HistorianValue> master = CreateTable();
AddData(master, 100, 100, Max);
DebugStopwatch sw = new DebugStopwatch();
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> masterRead = master.BeginRead())
{
double sec = sw.TimeEvent(() =>
{
SortedTreeScannerBase <HistorianKey, HistorianValue> scanner = masterRead.GetTreeScanner();
scanner.SeekToStart();
while (scanner.Read(key, value))
{
}
});
System.Console.WriteLine(Max / sec / 1000000);
}
master.Dispose();
MemoryPoolTest.TestMemoryLeak();
}
public void EnduranceTest()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (SortedTreeTable<HistorianKey, HistorianValue> target = SortedTreeFile.CreateInMemory().OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
for (uint x = 0; x < 100; x++)
{
using (var fileEditor = target.BeginEdit())
{
for (int y = 0; y < 10; y++)
{
key.Timestamp = x;
key.PointID = x;
value.Value1 = x;
value.Value3 = x;
fileEditor.AddPoint(key, value);
x++;
}
fileEditor.Commit();
}
Assert.AreEqual(target.FirstKey.Timestamp, (ulong)0);
Assert.AreEqual(target.LastKey.Timestamp, (ulong)(x - 1));
}
}
}
public void TestOneFile()
{
HistorianKey key1 = new HistorianKey();
HistorianKey key2 = new HistorianKey();
HistorianValue value1 = new HistorianValue();
HistorianValue value2 = new HistorianValue();
Logger.Console.Verbose = VerboseLevel.All;
MemoryPoolTest.TestMemoryLeak();
ArchiveList <HistorianKey, HistorianValue> list = new ArchiveList <HistorianKey, HistorianValue>(null);
SortedTreeTable <HistorianKey, HistorianValue> master = CreateTable();
SortedTreeTable <HistorianKey, HistorianValue> table1 = CreateTable();
AddData(master, 100, 100, 100);
AddData(table1, 100, 100, 100);
using (ArchiveListEditor <HistorianKey, HistorianValue> editor = list.AcquireEditLock())
{
editor.Add(table1);
}
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> masterRead = master.BeginRead())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> masterScan = masterRead.GetTreeScanner();
masterScan.SeekToStart();
TreeStreamSequential <HistorianKey, HistorianValue> masterScanSequential = masterScan.TestSequential();
using (SequentialReaderStream <HistorianKey, HistorianValue> sequencer = new SequentialReaderStream <HistorianKey, HistorianValue>(list))
{
TreeStreamSequential <HistorianKey, HistorianValue> scanner = sequencer.TestSequential();
int count = 0;
while (scanner.Read(key1, value1))
{
count++;
if (!masterScanSequential.Read(key2, value2))
{
throw new Exception();
}
if (!key1.IsEqualTo(key2))
{
throw new Exception();
}
if (!value1.IsEqualTo(value2))
{
throw new Exception();
}
}
if (masterScan.Read(key2, value2))
{
throw new Exception();
}
}
}
list.Dispose();
master.Dispose();
MemoryPoolTest.TestMemoryLeak();
}
public void GetBits()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
StringBuilder sb = new StringBuilder();
sb.AppendLine("Higher Bits, Bucket Number, Count, FloatValue");
using (SortedTreeFile file = SortedTreeFile.OpenFile(@"C:\Archive\635184227258021940-Stage2-8b835d6a-8299-45bb-9624-d4a470e4abe1.d2", true))
using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenTable <HistorianKey, HistorianValue>())
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> reader = table.BeginRead())
using (SortedTreeScannerBase <HistorianKey, HistorianValue> scan = reader.GetTreeScanner())
{
int count = 0;
scan.SeekToStart();
while (scan.Read(key, value))
{
count++;
}
for (int x = 1; x < 24; x++)
{
scan.SeekToStart();
int[] bucket = MeasureBits(scan, x);
Write(sb, bucket, x, count);
}
}
Console.WriteLine(sb.ToString());
}
/// <summary>
/// Gets concentrated frames from the provided stream
/// </summary>
/// <param name="stream">the database to use</param>
/// <returns></returns>
public static SortedList<DateTime, FrameData> GetFrames(this TreeStream<HistorianKey, HistorianValue> stream)
{
SortedList<DateTime, FrameDataConstructor> results = new SortedList<DateTime, FrameDataConstructor>();
ulong lastTime = ulong.MinValue;
FrameDataConstructor lastFrame = null;
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
while (stream.Read(key, value))
{
if (lastFrame == null || key.Timestamp != lastTime)
{
lastTime = key.Timestamp;
DateTime timestamp = new DateTime((long)lastTime);
if (!results.TryGetValue(timestamp, out lastFrame))
{
lastFrame = new FrameDataConstructor();
results.Add(timestamp, lastFrame);
}
}
lastFrame.PointId.Add(key.PointID);
lastFrame.Values.Add(value.ToStruct());
}
List<FrameData> data = new List<FrameData>(results.Count);
data.AddRange(results.Values.Select(x => x.ToFrameData()));
return SortedListConstructor.Create(results.Keys, data);
}
public void BenchmarkOld(int pointCount)
{
SortedPointBuffer <HistorianKey, HistorianValue> points = new SortedPointBuffer <HistorianKey, HistorianValue>(pointCount, true);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
for (int x = 0; x < pointCount; x++)
{
key.PointID = (ulong)x;
points.TryEnqueue(key, value);
}
points.IsReadingMode = true;
Stopwatch sw = new Stopwatch();
sw.Start();
using (BinaryStream bs = new BinaryStream(true))
{
SortedTree <HistorianKey, HistorianValue> st = SortedTree <HistorianKey, HistorianValue> .Create(bs, 4096, HistorianFileEncodingDefinition.TypeGuid);
st.TryAddRange(points);
//SequentialSortedTreeWriter<HistorianKey, HistorianValue>.Create(bs, 4096, SortedTree.FixedSizeNode, points);
}
sw.Stop();
System.Console.WriteLine("Points {0}: {1}MPPS", pointCount, (pointCount / sw.Elapsed.TotalSeconds / 1000000).ToString("0.0"));
}
public void BenchmarkOld2(int pointCount)
{
SortedPointBuffer<HistorianKey, HistorianValue> points = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
for (int x = 0; x < pointCount; x++)
{
key.PointID = (ulong)x;
points.TryEnqueue(key, value);
}
points.IsReadingMode = true;
var sw = new Stopwatch();
sw.Start();
using (var bs = new BinaryStream(true))
{
var st = SortedTree<HistorianKey, HistorianValue>.Create(bs, 4096, HistorianFileEncodingDefinition.TypeGuid);
st.AddRange(points);
//SequentialSortedTreeWriter<HistorianKey, HistorianValue>.Create(bs, 4096, SortedTree.FixedSizeNode, points);
}
sw.Stop();
System.Console.WriteLine("Points {0}: {1}MPPS", pointCount, (pointCount / sw.Elapsed.TotalSeconds / 1000000).ToString("0.0"));
}
/// <summary>
/// Gets concentrated frames from the provided stream
/// </summary>
/// <param name="stream">the database to use</param>
/// <returns></returns>
public static SortedList <DateTime, FrameData> GetFrames(this TreeStream <HistorianKey, HistorianValue> stream)
{
SortedList <DateTime, FrameDataConstructor> results = new SortedList <DateTime, FrameDataConstructor>();
ulong lastTime = ulong.MinValue;
FrameDataConstructor lastFrame = null;
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
while (stream.Read(key, value))
{
if (lastFrame == null || key.Timestamp != lastTime)
{
lastTime = key.Timestamp;
DateTime timestamp = new DateTime((long)lastTime);
if (!results.TryGetValue(timestamp, out lastFrame))
{
lastFrame = new FrameDataConstructor();
results.Add(timestamp, lastFrame);
}
}
lastFrame.PointId.Add(key.PointID);
lastFrame.Values.Add(value.ToStruct());
}
List <FrameData> data = new List <FrameData>(results.Count);
data.AddRange(results.Values.Select(x => x.ToFrameData()));
return(SortedListConstructor.Create(results.Keys, data));
}
public void WriteFile()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
if (File.Exists("c:\\temp\\ArchiveTestFileBig.d2"))
File.Delete("c:\\temp\\ArchiveTestFileBig.d2");
//using (var af = ArchiveFile.CreateInMemory(CompressionMethod.TimeSeriesEncoded))
using (SortedTreeFile af = SortedTreeFile.CreateFile("c:\\temp\\ArchiveTestFileBig.d2"))
using (SortedTreeTable<HistorianKey, HistorianValue> af2 = af.OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
Random r = new Random(3);
for (ulong v1 = 1; v1 < 36; v1++)
{
using (var edit = af2.BeginEdit())
{
for (ulong v2 = 1; v2 < 86000; v2++)
{
key.Timestamp = v1 * 2342523;
key.PointID = v2;
value.Value1 = (ulong)r.Next();
value.Value3 = 0;
edit.AddPoint(key, value);
}
edit.Commit();
}
af2.Count();
}
af2.Count();
}
}
public void BenchmarkRandomData(int pointCount)
{
Stopwatch sw = new Stopwatch();
SortedPointBuffer <HistorianKey, HistorianValue> buffer = new SortedPointBuffer <HistorianKey, HistorianValue>(pointCount, true);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
List <double> times = new List <double>();
for (int cnt = 0; cnt < 10; cnt++)
{
Random r = new Random(1);
buffer.IsReadingMode = false;
for (int x = 0; x < pointCount; x++)
{
key.Timestamp = (ulong)r.Next();
key.PointID = (ulong)x;
buffer.TryEnqueue(key, value);
}
sw.Restart();
buffer.IsReadingMode = true;
sw.Stop();
times.Add(sw.Elapsed.TotalSeconds);
}
times.Sort();
System.Console.WriteLine("{0} points {1}ms {2} Million/second ", pointCount, times[5] * 1000, pointCount / times[5] / 1000000);
}
public void GetBits()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
StringBuilder sb = new StringBuilder();
sb.AppendLine("Higher Bits, Bucket Number, Count, FloatValue");
using (var file = SortedTreeFile.OpenFile(@"C:\Archive\635184227258021940-Stage2-8b835d6a-8299-45bb-9624-d4a470e4abe1.d2", true))
using (var table = file.OpenTable<HistorianKey, HistorianValue>())
using (var reader = table.BeginRead())
using (var scan = reader.GetTreeScanner())
{
int count = 0;
scan.SeekToStart();
while (scan.Read(key,value))
count++;
for (int x = 1; x < 24; x++)
{
scan.SeekToStart();
var bucket = MeasureBits(scan, x);
Write(sb, bucket, x, count);
}
}
Console.WriteLine(sb.ToString());
}
private void StartStream(object args)
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (HistorianClient client = new HistorianClient("127.0.0.1", 54996))
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = client.GetDatabase <HistorianKey, HistorianValue>(string.Empty))
{
using (SortedTreeTable <HistorianKey, HistorianValue> file = SortedTreeFile.OpenFile(@"H:\OGE 2009.d2", isReadOnly: true).OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
using (SortedTreeTableReadSnapshot <HistorianKey, HistorianValue> read = file.BeginRead())
{
SortedTreeScannerBase <HistorianKey, HistorianValue> scan = read.GetTreeScanner();
scan.SeekToStart();
long count = 0;
while (scan.Read(key, value))
{
count++;
database.Write(key, value);
if ((count % 10) == 1)
{
Thread.Sleep(1);
}
}
}
}
}
}
public void BenchmarkTwoFiles()
{
MemoryPoolTest.TestMemoryLeak();
const int Max = 1000000;
ArchiveList <HistorianKey, HistorianValue> list = new ArchiveList <HistorianKey, HistorianValue>(null);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
SortedTreeTable <HistorianKey, HistorianValue> table1 = CreateTable();
SortedTreeTable <HistorianKey, HistorianValue> table2 = CreateTable();
AddData(table1, 100, 100, Max / 2);
AddData(table2, 101, 100, Max / 2);
using (ArchiveListEditor <HistorianKey, HistorianValue> editor = list.AcquireEditLock())
{
editor.Add(table1);
editor.Add(table2);
}
SequentialReaderStream <HistorianKey, HistorianValue> sequencer = new SequentialReaderStream <HistorianKey, HistorianValue>(list);
DebugStopwatch sw = new DebugStopwatch();
double sec = sw.TimeEvent(() =>
{
SequentialReaderStream <HistorianKey, HistorianValue> scanner = sequencer;
while (scanner.Read(key, value))
{
}
});
System.Console.WriteLine(Max / sec / 1000000);
list.Dispose();
MemoryPoolTest.TestMemoryLeak();
}
public void RollbackChangesToAFile()
{
string fileName = @"C:\Temp\ArchiveFile.d2";
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (SortedTreeFile file = SortedTreeFile.OpenFile(fileName, isReadOnly: false))
using (SortedTreeTable <HistorianKey, HistorianValue> table = file.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
using (SortedTreeTableEditor <HistorianKey, HistorianValue> editor = table.BeginEdit())
{
key.TimestampAsDate = DateTime.Now.AddDays(-1);
key.PointID = 234;
value.AsString = "Add Me";
editor.AddPoint(key, value);
editor.Commit();
}
using (SortedTreeTableEditor <HistorianKey, HistorianValue> editor = table.BeginEdit())
{
key.Timestamp = 31;
value.AsString = "But Not Me";
editor.AddPoint(key, value);
editor.Rollback(); //These changes will not be written to the disk
}
}
ReadDataFromAFile();
}
public void CreateScadaDatabase()
{
Array.ForEach(Directory.GetFiles(@"c:\temp\Scada\", "*.d2", SearchOption.AllDirectories), File.Delete);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("DB", @"c:\temp\Scada\", true);
using (HistorianServer server = new HistorianServer(settings, 12345))
{
using (HistorianClient client = new HistorianClient("127.0.0.1", 12345))
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = client.GetDatabase <HistorianKey, HistorianValue>("DB"))
{
for (ulong x = 0; x < 1000; x++)
{
key.Timestamp = x;
database.Write(key, value);
}
database.HardCommit();
System.Threading.Thread.Sleep(1200);
}
}
}
/// <summary>
/// Read historian data from server.
/// </summary>
/// <param name="connection">openHistorian connection.</param>
/// <param name="startTime">Start time of query.</param>
/// <param name="stopTime">Stop time of query.</param>
/// <param name="measurementIDs">Array of measurement IDs to query - or <c>null</c> for all available points.</param>
/// <param name="resolution">Resolution for data query.</param>
/// <returns>Enumeration of <see cref="IMeasurement"/> values read for time range.</returns>
/// <remarks>
/// <example>
/// <code>
/// using (var connection = new Connection("127.0.0.1", "PPA"))
/// foreach(var measurement in GetHistorianData(connection, DateTime.UtcNow.AddMinutes(-1.0D), DateTime.UtcNow))
/// Console.WriteLine("{0}:{1} @ {2} = {3}, quality: {4}", measurement.Key.Source, measurement.Key.ID, measurement.Timestamp, measurement.Value, measurement.StateFlags);
/// </code>
/// </example>
/// </remarks>
public static IEnumerable <IMeasurement> GetHistorianData(Connection connection, DateTime startTime, DateTime stopTime, IEnumerable <ulong> measurementIDs = null, Resolution resolution = Resolution.Full)
{
SeekFilterBase <HistorianKey> timeFilter;
MatchFilterBase <HistorianKey, HistorianValue> pointFilter = null;
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
// Set data scan resolution
if (resolution == Resolution.Full)
{
timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, stopTime);
}
else
{
TimeSpan resolutionInterval = resolution.GetInterval();
BaselineTimeInterval interval = BaselineTimeInterval.Second;
if (resolutionInterval.Ticks < Ticks.PerMinute)
{
interval = BaselineTimeInterval.Second;
}
else if (resolutionInterval.Ticks < Ticks.PerHour)
{
interval = BaselineTimeInterval.Minute;
}
else if (resolutionInterval.Ticks == Ticks.PerHour)
{
interval = BaselineTimeInterval.Hour;
}
startTime = startTime.BaselinedTimestamp(interval);
stopTime = stopTime.BaselinedTimestamp(interval);
timeFilter = TimestampSeekFilter.CreateFromIntervalData <HistorianKey>(startTime, stopTime, resolutionInterval, new TimeSpan(TimeSpan.TicksPerMillisecond));
}
// Setup point ID selections
if (measurementIDs != null)
{
pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(measurementIDs);
}
// Start stream reader for the provided time window and selected points
using (Database database = connection.OpenDatabase())
{
TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter);
while (stream.Read(key, value))
{
yield return new Measurement
{
Metadata = MeasurementKey.LookUpOrCreate(connection.InstanceName, (uint)key.PointID).Metadata,
Timestamp = key.TimestampAsDate,
Value = value.AsSingle,
StateFlags = (MeasurementStateFlags)value.Value3
}
}
;
}
}
public void Test(int count)
{
var lst = new List <SortedTreeTable <HistorianKey, HistorianValue> >();
for (int x = 0; x < count; x++)
{
lst.Add(CreateTable());
}
using (var reader = new UnionTreeStream <HistorianKey, HistorianValue>(lst.Select(x => new ArchiveTreeStreamWrapper <HistorianKey, HistorianValue>(x)), true))
{
var key = new HistorianKey();
var value = new HistorianValue();
Stopwatch sw = new Stopwatch();
sw.Start();
while (reader.Read(key, value))
{
;
}
sw.Stop();
System.Console.Write("{0}\t{1}\t{2}", count, sw.Elapsed.TotalSeconds, sw.Elapsed.TotalSeconds / count);
System.Console.WriteLine();
}
lst.ForEach(x => x.Dispose());
}
private TestResults TestRandom(int pageSize, uint count)
{
//StringBuilder sb = new StringBuilder();
Random R = new Random(1);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
DiskIoSession.ReadCount = 0;
DiskIoSession.WriteCount = 0;
Stats.ChecksumCount = 0;
DiskIoSession.Lookups = 0;
DiskIoSession.CachedLookups = 0;
Stopwatch sw = new Stopwatch();
sw.Start();
using (SortedTreeFile af = SortedTreeFile.CreateInMemory(blockSize: pageSize))
using (SortedTreeTable <HistorianKey, HistorianValue> af2 = af.OpenOrCreateTable <HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
uint pointPairs = count / 5000;
for (uint i = 0; i < pointPairs; i++)
{
uint max = i * 5000 + 5000;
using (var edit = af2.BeginEdit())
{
for (ulong x = i * 5000; x < max; x++)
{
key.Timestamp = (uint)R.Next();
key.PointID = 2 * x;
value.Value3 = 3 * x;
value.Value1 = 4 * x;
//if ((x % 100) == 0)
// sb.AppendLine(x + "," + DiskIoSession.ReadCount + "," + DiskIoSession.WriteCount);
//if (x == 1000)
// DiskIoSession.BreakOnIO = true;
edit.AddPoint(key, value);
//edit.AddPoint(uint.MaxValue - x, 2 * x, 3 * x, 4 * x);
}
edit.Commit();
}
}
//long cnt = af.Count();
}
sw.Stop();
//File.WriteAllText(@"C:\temp\" + pageSize + ".csv",sb.ToString());
return(new TestResults()
{
PageSize = pageSize,
Rate = (float)(count / sw.Elapsed.TotalSeconds / 1000000),
ReadCount = DiskIoSession.ReadCount,
WriteCount = DiskIoSession.WriteCount,
ChecksumCount = Stats.ChecksumCount,
Lookups = DiskIoSession.Lookups,
CachedLookups = DiskIoSession.CachedLookups
});
}
public void CountIO()
{
Test(1000, false);
int pointCount = 10000000;
SortedPointBuffer<HistorianKey, HistorianValue> points = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
for (int x = 0; x < pointCount; x++)
{
key.PointID = (ulong)x;
points.TryEnqueue(key, value);
}
points.IsReadingMode = true;
File.Delete(@"C:\Temp\fileTemp.~d2i");
File.Delete(@"C:\Temp\fileTemp.d2i");
Stopwatch sw = new Stopwatch();
sw.Start();
SortedTreeFileSimpleWriter<HistorianKey, HistorianValue>.Create(@"C:\Temp\fileTemp.~d2i", @"C:\Temp\fileTemp.d2i", 4096, null, EncodingDefinition.FixedSizeCombinedEncoding, points);
sw.Stop();
System.Console.WriteLine(SimplifiedSubFileStreamIoSession.ReadBlockCount);
System.Console.WriteLine(SimplifiedSubFileStreamIoSession.WriteBlockCount);
System.Console.WriteLine(sw.Elapsed.TotalSeconds.ToString());
}
SortedTreeTable<HistorianKey, HistorianValue> CreateTable()
{
var r = new Random(seed++);
var key = new HistorianKey();
var value = new HistorianValue();
var file = SortedTreeFile.CreateInMemory();
var table = file.OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding);
using (var edit = table.BeginEdit())
{
for (int x = 0; x < 1000; x++)
{
key.Timestamp = (ulong)r.Next();
key.PointID = (ulong)r.Next();
key.EntryNumber = (ulong)r.Next();
edit.AddPoint(key, value);
}
edit.Commit();
}
return table;
}
public SnapDBClient(SnapDBEngine engine, string instanceName)
{
m_client = SnapClient.Connect(engine.ServerHost);
m_database = m_client.GetDatabase <HistorianKey, HistorianValue>(instanceName);
m_key = new HistorianKey();
m_value = new HistorianValue();
m_lastKey = new HistorianKey();
}
public void TestTwoIdenticalFiles()
{
MemoryPoolTest.TestMemoryLeak();
ArchiveList <HistorianKey, HistorianValue> list = new ArchiveList <HistorianKey, HistorianValue>(null);
HistorianKey key1 = new HistorianKey();
HistorianKey key2 = new HistorianKey();
HistorianValue value1 = new HistorianValue();
HistorianValue value2 = new HistorianValue();
var master = CreateTable();
var table1 = CreateTable();
var table2 = CreateTable();
AddData(master, 100, 100, 100);
AddData(table1, 100, 100, 100);
AddData(table2, 100, 100, 100);
using (var editor = list.AcquireEditLock())
{
editor.Add(table1);
editor.Add(table2);
}
using (var masterRead = master.BeginRead())
{
var masterScan = masterRead.GetTreeScanner();
masterScan.SeekToStart();
var masterScanSequential = masterScan.TestSequential();
using (var sequencer = new SequentialReaderStream <HistorianKey, HistorianValue>(list))
{
var scanner = sequencer.TestSequential();
while (scanner.Read(key1, value1))
{
if (!masterScanSequential.Read(key2, value2))
{
throw new Exception();
}
if (!key1.IsEqualTo(key2))
{
throw new Exception();
}
if (!value1.IsEqualTo(value2))
{
throw new Exception();
}
}
if (masterScan.Read(key2, value2))
{
throw new Exception();
}
}
}
list.Dispose();
master.Dispose();
MemoryPoolTest.TestMemoryLeak();
}
public SnapDBClient(string hostAddress, int port, string instanceName, ulong startTime, ulong endTime, int frameRate, IEnumerable<ulong> pointIDs)
{
m_client = new HistorianClient(hostAddress, port);
m_database = m_client.GetDatabase<HistorianKey, HistorianValue>(instanceName);
m_key = new HistorianKey();
m_value = new HistorianValue();
SeekFilterBase<HistorianKey> timeFilter = TimestampSeekFilter.CreateFromRange<HistorianKey>(DataPoint.RoundTimestamp(startTime, frameRate), DataPoint.RoundTimestamp(endTime, frameRate));
MatchFilterBase<HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList<HistorianKey, HistorianValue>(pointIDs);
m_stream = m_database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter);
}
public void TestTwoFiles()
{
MemoryPoolTest.TestMemoryLeak();
ArchiveList<HistorianKey, HistorianValue> list = new ArchiveList<HistorianKey, HistorianValue>(null);
HistorianKey key1 = new HistorianKey();
HistorianKey key2 = new HistorianKey();
HistorianValue value1 = new HistorianValue();
HistorianValue value2 = new HistorianValue();
var master = CreateTable();
var table1 = CreateTable();
var table2 = CreateTable();
AddData(master, 100, 100, 100);
AddData(table1, 100, 100, 100);
AddData(master, 101, 100, 100);
AddData(table2, 101, 100, 100);
using (var editor = list.AcquireEditLock())
{
editor.Add(table1);
editor.Add(table2);
}
using (var masterRead = master.BeginRead())
{
var masterScan = masterRead.GetTreeScanner();
masterScan.SeekToStart();
var masterScanSequential = masterScan.TestSequential();
using (var sequencer = new SequentialReaderStream<HistorianKey, HistorianValue>(list))
{
var scanner = sequencer.TestSequential();
while (scanner.Read(key1, value1))
{
if (!masterScanSequential.Read(key2, value2))
throw new Exception();
if (!key1.IsEqualTo(key2))
throw new Exception();
if (!value1.IsEqualTo(value2))
throw new Exception();
}
if (masterScan.Read(key2, value2))
throw new Exception();
}
}
master.Dispose();
list.Dispose();
MemoryPoolTest.TestMemoryLeak();
}
public int[] MeasureBits(TreeStream<HistorianKey, HistorianValue> stream, int higherBits)
{
HistorianKey hkey = new HistorianKey();
HistorianValue hvalue = new HistorianValue();
int[] bucket = new int[1 << higherBits];
int shiftBits = 32 - higherBits;
while (stream.Read(hkey,hvalue))
{
uint value = (uint)hvalue.Value1 >> shiftBits;
bucket[value]++;
}
return bucket;
}
public void Test()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (var file = SortedTreeFile.OpenFile(@"C:\Unison\GPA\Codeplex\openHistorian\Main\Build\Output\Release\Applications\openHistorian\Archive\635293583194231435-Stage2-0ef36dcc-4264-498f-b194-01b2043a9231.d2", true))
using (var table = file.OpenTable<HistorianKey, HistorianValue>())
using (var reader = table.BeginRead())
using (var scan = reader.GetTreeScanner())
{
scan.SeekToStart();
while (scan.Read(key,value))
;
}
}
/// <summary>
/// Queries the provided signals within a the provided time window [Inclusive]
/// </summary>
/// <param name="database"></param>
/// <param name="startTime">the lower bound of the time</param>
/// <param name="endTime">the upper bound of the time. [Inclusive]</param>
/// <param name="signals">an IEnumerable of all of the signals to query as part of the results set.</param>
/// <returns></returns>
public static Dictionary<ulong, RawSignalTimeValue> GetRawSignals(this ClientDatabaseBase<HistorianKey, HistorianValue> database, DateTime startTime, DateTime endTime, IEnumerable<ulong> signals)
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
Dictionary<ulong, RawSignalTimeValue> results = signals.ToDictionary((x) => x, (x) => new RawSignalTimeValue());
TreeStream<HistorianKey, HistorianValue> stream = database.Read((ulong)startTime.Ticks, (ulong)endTime.Ticks, signals);
while (stream.Read(key, value))
{
results[key.PointID].Signals.Add(key.TimestampAsDate, value.ToStruct());
}
return results;
}
public void AppendDataToAnExistingFile()
{
string fileName = @"C:\Temp\ArchiveFile.d2";
var key = new HistorianKey();
var value = new HistorianValue();
using (var file = SortedTreeFile.OpenFile(fileName, isReadOnly: false))
using (var table = file.OpenTable<HistorianKey, HistorianValue>())
using (var editor = table.BeginEdit())
{
key.TimestampAsDate = DateTime.Now;
key.PointID = 2;
value.AsString = "Test Append";
editor.AddPoint(key, value);
editor.Commit();
}
ReadDataFromAFile();
}
public void TestReadData()
{
var key = new HistorianKey();
var value = new HistorianValue();
var settings = new HistorianServerDatabaseConfig("DB", @"c:\temp\Scada\", true);
using (HistorianServer server = new HistorianServer(settings, 12345))
{
using (var client = new HistorianClient("127.0.0.1", 12345))
using (var database = client.GetDatabase<HistorianKey, HistorianValue>("DB"))
{
var stream = database.Read(0, 1000);
while (stream.Read(key, value))
Console.WriteLine(key.Timestamp);
}
}
}
public void WriteData()
{
Logger.Console.Verbose = VerboseLevel.All;
using (var server = CreateServer())
{
using (var client = SnapClient.Connect(server))
using (var db = client.GetDatabase<HistorianKey, HistorianValue>("PPA"))
{
var key = new HistorianKey();
var value = new HistorianValue();
key.TimestampAsDate = DateTime.Now;
key.PointID = LittleEndian.ToUInt64(Guid.NewGuid().ToByteArray(), 0);
db.Write(key, value);
}
}
}
public void CreateSnapshotTest()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
key.Timestamp = 1;
key.PointID = 2;
value.Value1 = 3;
value.Value2 = 4;
using (SortedTreeTable<HistorianKey, HistorianValue> target = SortedTreeFile.CreateInMemory().OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
ulong date = 1;
ulong pointId = 2;
ulong value1 = 3;
ulong value2 = 4;
SortedTreeTableSnapshotInfo<HistorianKey, HistorianValue> snap1;
using (var fileEditor = target.BeginEdit())
{
fileEditor.AddPoint(key, value);
key.Timestamp++;
fileEditor.AddPoint(key, value);
snap1 = target.AcquireReadSnapshot();
fileEditor.Commit();
}
SortedTreeTableSnapshotInfo<HistorianKey, HistorianValue> snap2 = target.AcquireReadSnapshot();
using (SortedTreeTableReadSnapshot<HistorianKey, HistorianValue> instance = snap1.CreateReadSnapshot())
{
SortedTreeScannerBase<HistorianKey, HistorianValue> scanner = instance.GetTreeScanner();
scanner.SeekToStart();
Assert.AreEqual(false, scanner.Read(key, value));
}
using (SortedTreeTableReadSnapshot<HistorianKey, HistorianValue> instance = snap2.CreateReadSnapshot())
{
SortedTreeScannerBase<HistorianKey, HistorianValue> scanner = instance.GetTreeScanner();
scanner.SeekToStart();
Assert.AreEqual(true, scanner.Read(key, value));
Assert.AreEqual(1uL, key.Timestamp);
Assert.AreEqual(2uL, key.PointID);
Assert.AreEqual(3uL, value.Value1);
Assert.AreEqual(4uL, value.Value2);
}
Assert.AreEqual(1uL, target.FirstKey.Timestamp);
Assert.AreEqual(2uL, target.LastKey.Timestamp);
}
}
public void TestReadData()
{
using (var server = new HistorianServer(new HistorianServerDatabaseConfig("DB", @"c:\temp\Scada\", false), 1234))
{
using (var client = SnapClient.Connect(server.Host))
{
var database = client.GetDatabase<HistorianKey, HistorianValue>("DB");
var stream = database.Read(10, 800 - 1);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
while (stream.Read(key, value))
{
Console.WriteLine(key.Timestamp);
}
}
}
}
public void BenchmarkWriteSpeed()
{
DebugStopwatch sw = new DebugStopwatch();
double time;
double count = 0;
using (SortedTreeFile file = SortedTreeFile.CreateInMemory())
{
var table = file.OpenOrCreateTable<HistorianKey, HistorianValue>(HistorianFileEncodingDefinition.TypeGuid);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
time = sw.TimeEvent(() =>
{
//TreeKeyMethodsBase<HistorianKey>.ClearStats();
//TreeValueMethodsBase<HistorianKey>.ClearStats();
count = 0;
using (var scan = table.BeginEdit())
{
for (uint x = 0; x < 10000000; x++)
{
key.PointID = x;
scan.AddPoint(key, value);
count++;
}
scan.Rollback();
}
});
}
Console.WriteLine((count / 1000000 / time).ToString() + " Million PPS");
//Console.WriteLine("KeyMethodsBase calls");
//for (int x = 0; x < 23; x++)
//{
// Console.WriteLine(TreeKeyMethodsBase<HistorianKey>.CallMethods[x] + "\t" + ((TreeKeyMethodsBase<HistorianKey>.Method)(x)).ToString());
//}
//Console.WriteLine("ValueMethodsBase calls");
//for (int x = 0; x < 5; x++)
//{
// Console.WriteLine(TreeValueMethodsBase<HistorianValue>.CallMethods[x] + "\t" + ((TreeValueMethodsBase<HistorianValue>.Method)(x)).ToString());
//}
}
public void VerifyDB()
{
//Logger.ReportToConsole(VerboseLevel.All ^ VerboseLevel.DebugLow);
//Logger.ConsoleSubscriber.AddIgnored(Logger.LookupType("GSF.SortedTreeStore"));
Globals.MemoryPool.SetMaximumBufferSize(1000 * 1024 * 1024);
Globals.MemoryPool.SetTargetUtilizationLevel(TargetUtilizationLevels.Low);
var settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\benchmark\\", true);
using (var engine = new SnapServer(settings))
using (var client = SnapClient.Connect(engine))
using (var db = client.GetDatabase<HistorianKey, HistorianValue>("DB"))
using (var scan = db.Read(null, null, null))
{
var key = new HistorianKey();
var value = new HistorianValue();
Stopwatch sw = new Stopwatch();
sw.Start();
for (int x = 0; x < PointsToArchive; x++)
{
if (!scan.Read(key, value))
throw new Exception("Missing points");
if (key.PointID != (ulong)x)
throw new Exception("Corrupt");
if (key.Timestamp != 0)
throw new Exception("Corrupt");
if (key.EntryNumber != 0)
throw new Exception("Corrupt");
if (value.Value1 != 0)
throw new Exception("Corrupt");
if (value.Value1 != 0)
throw new Exception("Corrupt");
if (value.Value1 != 0)
throw new Exception("Corrupt");
}
double totalTime = sw.Elapsed.TotalSeconds;
Console.WriteLine("Completed read test in {0:#,##0.00} seconds at {1:#,##0.00} points per second", totalTime, PointsToArchive / totalTime);
if (scan.Read(key, value))
throw new Exception("too many points");
}
}
public void WriteDataToAFile()
{
Logger.Console.Verbose = VerboseLevel.All;
string fileName = @"C:\Temp\ArchiveFile.d2";
if (File.Exists(fileName))
File.Delete(fileName);
var key = new HistorianKey();
var value = new HistorianValue();
using (var file = SortedTreeFile.CreateFile(fileName))
using (var table = file.OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
using (var editor = table.BeginEdit())
{
key.TimestampAsDate = DateTime.Now;
key.PointID = 1;
value.AsString = "Test Write";
editor.AddPoint(key, value);
editor.Commit();
}
}
public void TestOld()
{
Test(1000, false);
int pointCount = 10000000;
SortedPointBuffer<HistorianKey, HistorianValue> points = new SortedPointBuffer<HistorianKey, HistorianValue>(pointCount, true);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
for (int x = 0; x < pointCount; x++)
{
key.PointID = (ulong)x;
points.TryEnqueue(key, value);
}
points.IsReadingMode = true;
File.Delete(@"C:\Temp\fileTemp.~d2i");
File.Delete(@"C:\Temp\fileTemp.d2i");
Stopwatch sw = new Stopwatch();
sw.Start();
using (var file = SortedTreeFile.CreateFile(@"C:\Temp\fileTemp.~d2i"))
using (var table = file.OpenOrCreateTable<HistorianKey, HistorianValue>(EncodingDefinition.FixedSizeCombinedEncoding))
{
using (var edit = table.BeginEdit())
{
edit.AddPoints(points);
edit.Commit();
}
}
//SortedTreeFileSimpleWriter<HistorianKey, HistorianValue>.Create(@"C:\Temp\fileTemp.~d2i", @"C:\Temp\fileTemp.d2i", 4096, SortedTree.FixedSizeNode, points);
sw.Stop();
System.Console.WriteLine(SimplifiedSubFileStreamIoSession.ReadBlockCount);
System.Console.WriteLine(SimplifiedSubFileStreamIoSession.WriteBlockCount);
System.Console.WriteLine(sw.Elapsed.TotalSeconds.ToString());
}
public void ReadDataFromAFile()
{
string fileName = @"C:\Temp\ArchiveFile.d2";
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (var file = SortedTreeFile.OpenFile(fileName, isReadOnly: true))
using (var table = file.OpenTable<HistorianKey, HistorianValue>())
using (var snapshot = table.BeginRead())
{
var scanner = snapshot.GetTreeScanner();
scanner.SeekToStart();
while (scanner.Read(key,value))
{
Console.WriteLine("{0}, {1}, {2}",
key.TimestampAsDate.ToString(), key.PointID, value.AsString);
}
}
}
private void BuildListOfAllPoints()
{
HashSet<ulong> keys = new HashSet<ulong>();
var client = SnapClient.Connect(m_archiveFile.Host);
var db = client.GetDatabase<HistorianKey, HistorianValue>("");
TreeStream<HistorianKey, HistorianValue> scanner = db.Read(0, ulong.MaxValue);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
while (scanner.Read(key, value))
{
keys.Add(key.PointID);
}
List<ulong> AllKeys = keys.ToList();
AllKeys.Sort();
chkAllPoints.Items.Clear();
AllKeys.ForEach((x) => chkAllPoints.Items.Add(x));
db.Dispose();
client.Dispose();
}
