public void Test1()
{
Logger.Console.Verbose = VerboseLevel.All;
var netStream = new NetworkStreamSimulator();
var dbcfg = new HistorianServerDatabaseConfig("DB", @"C:\Archive", true);
var server = new HistorianServer(dbcfg);
var auth = new SecureStreamServer<SocketUserPermissions>();
auth.SetDefaultUser(true, new SocketUserPermissions()
{
CanRead = true,
CanWrite = true,
IsAdmin = true
});
var netServer = new SnapStreamingServer(auth, netStream.ServerStream, server.Host);
ThreadPool.QueueUserWorkItem(ProcessClient, netServer);
var client = new SnapStreamingClient(netStream.ClientStream, new SecureStreamClientDefault(), true);
var db = client.GetDatabase("DB");
client.Dispose();
server.Dispose();
}
public void SendAllPoints()
{
Stats.Clear();
long points;
StopReading = false;
var settings = new HistorianServerDatabaseConfig("PPA", @"C:\Program Files\openHistorian\Archive\", true);
using (HistorianServer server = new HistorianServer(settings))
{
Thread.Sleep(1000);
for (int x = 1; x < 30; x++)
{
StartReader();
Thread.Sleep(1000);
if (x == 1)
Thread.Sleep(5000);
Interlocked.Exchange(ref Stats.PointsReturned, 0);
Thread.Sleep(1000);
long v = Interlocked.Read(ref Stats.PointsReturned);
Console.WriteLine("Clients: " + x.ToString() + " points " + v.ToString());
}
StopReading = true;
Thread.Sleep(2000);
}
Thread.Sleep(2000);
}
private void BtnStart_Click(object sender, EventArgs e)
{
GSF.Globals.MemoryPool.SetMaximumBufferSize(long.Parse(TxtMaxMB.Text) * 1024 * 1024);
var settings = new HistorianServerDatabaseConfig(txtDbName.Text, TxtArchivePath.Text, true);
m_server = new HistorianServer(settings, int.Parse(TxtLocalPort.Text));
BtnStart.Enabled = false;
AllocConsole();
Logger.Console.Verbose = VerboseLevel.High;
}
public void Start(string[] paths)
{
m_database = "PPA";
var settings = new HistorianServerDatabaseConfig(m_database, null, false);
settings.ImportPaths.AddRange(paths);
m_localServer = new HistorianServer(settings);
var query = new HistorianQuery(SnapClient.Connect(m_localServer.Host));
m_updateFramework.Start(query);
m_updateFramework.Mode = ExecutionMode.Manual;
m_updateFramework.Enabled = true;
}
private void btnOpenFile_Click(object sender, EventArgs e)
{
using (OpenFileDialog dlgOpen = new OpenFileDialog())
{
dlgOpen.Filter = "openHistorian 2.0 file|*.d2";
if (dlgOpen.ShowDialog() == DialogResult.OK)
{
var db = new HistorianServerDatabaseConfig("", "", false);
db.ImportPaths.AddRange(dlgOpen.FileNames);
m_archiveFile = new HistorianServer(db);
}
}
BuildListOfAllPoints();
}
public void TestReadData()
{
var key = new HistorianKey();
var value = new HistorianValue();
var settings = new HistorianServerDatabaseConfig("DB", @"c:\temp\Scada\", true);
using (HistorianServer server = new HistorianServer(settings, 12345))
{
using (var client = new HistorianClient("127.0.0.1", 12345))
using (var database = client.GetDatabase<HistorianKey, HistorianValue>("DB"))
{
var stream = database.Read(0, 1000);
while (stream.Read(key, value))
Console.WriteLine(key.Timestamp);
}
}
}
public void VerifyDB()
{
//Logger.ReportToConsole(VerboseLevel.All ^ VerboseLevel.DebugLow);
//Logger.ConsoleSubscriber.AddIgnored(Logger.LookupType("GSF.SortedTreeStore"));
Globals.MemoryPool.SetMaximumBufferSize(1000 * 1024 * 1024);
Globals.MemoryPool.SetTargetUtilizationLevel(TargetUtilizationLevels.Low);
var settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\benchmark\\", true);
using (var engine = new SnapServer(settings))
using (var client = SnapClient.Connect(engine))
using (var db = client.GetDatabase<HistorianKey, HistorianValue>("DB"))
using (var scan = db.Read(null, null, null))
{
var key = new HistorianKey();
var value = new HistorianValue();
Stopwatch sw = new Stopwatch();
sw.Start();
for (int x = 0; x < PointsToArchive; x++)
{
if (!scan.Read(key, value))
throw new Exception("Missing points");
if (key.PointID != (ulong)x)
throw new Exception("Corrupt");
if (key.Timestamp != 0)
throw new Exception("Corrupt");
if (key.EntryNumber != 0)
throw new Exception("Corrupt");
if (value.Value1 != 0)
throw new Exception("Corrupt");
if (value.Value1 != 0)
throw new Exception("Corrupt");
if (value.Value1 != 0)
throw new Exception("Corrupt");
}
double totalTime = sw.Elapsed.TotalSeconds;
Console.WriteLine("Completed read test in {0:#,##0.00} seconds at {1:#,##0.00} points per second", totalTime, PointsToArchive / totalTime);
if (scan.Read(key, value))
throw new Exception("too many points");
}
}
public void CreateAllDatabases()
{
Logger.Console.Verbose = VerboseLevel.All;
Array.ForEach(Directory.GetFiles(@"c:\temp\Scada\", "*.d2", SearchOption.AllDirectories), File.Delete);
Array.ForEach(Directory.GetFiles(@"c:\temp\Synchrophasor\", "*.d2", SearchOption.AllDirectories), File.Delete);
var config1 = new HistorianServerDatabaseConfig("Scada", @"c:\temp\Scada\", true);
var config2 = new HistorianServerDatabaseConfig("Synchrophasor", @"c:\temp\Synchrophasor\", true);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
using (HistorianServer server = new HistorianServer())
{
server.AddDatabase(config1);
server.AddDatabase(config2);
using (var client = SnapClient.Connect(server.Host))
{
var database = client.GetDatabase<HistorianKey, HistorianValue>("Scada");
for (ulong x = 0; x < 10000; x++)
{
key.Timestamp = x;
database.Write(key, value);
}
database.HardCommit();
database = client.GetDatabase<HistorianKey, HistorianValue>("Synchrophasor");
for (ulong x = 0; x < 10000; x++)
{
key.Timestamp = x;
database.Write(key, value);
}
database.HardCommit();
}
}
}
public void TestReadData()
{
var config1 = new HistorianServerDatabaseConfig("Scada", @"c:\temp\Scada\", true);
var config2 = new HistorianServerDatabaseConfig("Synchrophasor", @"c:\temp\Synchrophasor\", true);
using (HistorianServer server = new HistorianServer())
{
server.AddDatabase(config1);
server.AddDatabase(config2);
using (var client = SnapClient.Connect(server.Host))
{
var database = client.GetDatabase<HistorianKey,HistorianValue>("Scada");
TreeStream<HistorianKey, HistorianValue> stream = database.Read(0, 100);
stream.Dispose();
database = client.GetDatabase<HistorianKey, HistorianValue>("Synchrophasor");
stream = database.Read(0, 100);
stream.Dispose();
}
}
}
public void CreateScadaDatabase()
{
Logger.Console.Verbose = VerboseLevel.All;
Array.ForEach(Directory.GetFiles(@"c:\temp\Scada\", "*.d2", SearchOption.AllDirectories), File.Delete);
var key = new HistorianKey();
var value = new HistorianValue();
var settings = new HistorianServerDatabaseConfig("DB", @"c:\temp\Scada\", true);
using (var server = new HistorianServer(settings))
using (var client = SnapClient.Connect(server.Host))
{
var database = client.GetDatabase<HistorianKey, HistorianValue>("db");
for (ulong x = 0; x < 1000; x++)
{
key.Timestamp = x;
database.Write(key, value);
}
database.HardCommit();
}
}
public void TestRemoteAdapter()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
var settings = new HistorianServerDatabaseConfig("PPA", @"c:\temp\historian\", true);
using (HistorianServer server = new HistorianServer(settings))
using (var client = SnapClient.Connect(server.Host))
{
using (HistorianInputQueue queue = new HistorianInputQueue(() => client.GetDatabase<HistorianKey, HistorianValue>(string.Empty)))
{
for (uint x = 0; x < 100000; x++)
{
key.PointID = x;
queue.Enqueue(key, value);
}
Thread.Sleep(100);
}
Thread.Sleep(100);
}
//Thread.Sleep(100);
}
public static void ReadAllPoints()
{
Stopwatch sw = new Stopwatch();
int pointCount = 0;
var settings = new HistorianServerDatabaseConfig("PPA", @"C:\Program Files\openHistorian\Archive\", true);
using (HistorianServer server = new HistorianServer(settings))
{
DateTime start = DateTime.FromBinary(Convert.ToDateTime("2/1/2014").Date.Ticks + Convert.ToDateTime("6:00:00PM").TimeOfDay.Ticks).ToUniversalTime();
using (var client = new HistorianClient("127.0.0.1", 12345))
using (var database = client.GetDatabase<HistorianKey, HistorianValue>(String.Empty))
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
sw.Start();
var scan = database.Read((ulong)start.Ticks, ulong.MaxValue);
while (scan.Read(key, value) && pointCount < 10000000)
pointCount++;
sw.Stop();
//sw.Start();
//using (var frameReader = database.GetPointStream(DateTime.MinValue, DateTime.MaxValue))
//{
// while (frameReader.Read())
// ;
//}
//sw.Stop();
}
}
Console.WriteLine(pointCount);
Console.WriteLine(sw.Elapsed.TotalSeconds.ToString());
Console.WriteLine((pointCount / sw.Elapsed.TotalSeconds / 1000000).ToString());
}
public void CreateScadaDatabase()
{
Array.ForEach(Directory.GetFiles(@"c:\temp\Scada\", "*.d2", SearchOption.AllDirectories), File.Delete);
var key = new HistorianKey();
var value = new HistorianValue();
var settings = new HistorianServerDatabaseConfig("DB", @"c:\temp\Scada\", true);
using (var server = new HistorianServer(settings, 12345))
{
using (var client = new HistorianClient("127.0.0.1", 12345))
using (var database = client.GetDatabase<HistorianKey, HistorianValue>("DB"))
{
for (ulong x = 0; x < 1000; x++)
{
key.Timestamp = x;
database.Write(key, value);
}
database.HardCommit();
System.Threading.Thread.Sleep(1200);
}
}
}
/// <summary> /// Adds the supplied database to this server. /// </summary> /// <param name="database"></param> public void AddDatabase(HistorianServerDatabaseConfig database) => Host.AddDatabase(database);
public HistorianServer(HistorianServerDatabaseConfig database, int?port = null, string networkInterfaceIP = null)
: this(port, networkInterfaceIP)
{
AddDatabase(database);
}
/// <summary>
/// Initializes this <see cref="LocalOutputAdapter"/>.
/// </summary>
public override void Initialize()
{
base.Initialize();
//const string errorMessage = "{0} is missing from Settings - Example: instanceName=default; ArchiveDirectories={{c:\\Archive1\\;d:\\Backups2\\}}; dataChannel={{port=9591; interface=0.0.0.0}}";
Dictionary<string, string> settings = Settings;
string setting;
// Validate settings.
if (!settings.TryGetValue("instanceName", out m_instanceName) || string.IsNullOrWhiteSpace(m_instanceName))
m_instanceName = Name;
// Track instance in static dictionary
Instances[InstanceName] = this;
if (!settings.TryGetValue("WorkingDirectory", out setting) || string.IsNullOrEmpty(setting))
setting = "Archive";
WorkingDirectory = setting;
if (settings.TryGetValue("ArchiveDirectories", out setting))
ArchiveDirectories = setting;
if (settings.TryGetValue("AttachedPaths", out setting))
AttachedPaths = setting;
if (!settings.TryGetValue("DataChannel", out m_dataChannel))
m_dataChannel = DefaultDataChannel;
double targetFileSize;
if (!settings.TryGetValue("TargetFileSize", out setting) || !double.TryParse(setting, out targetFileSize))
targetFileSize = DefaultTargetFileSize;
if (targetFileSize < 0.1D || targetFileSize > SI2.Tera)
targetFileSize = DefaultTargetFileSize;
if (!settings.TryGetValue("MaximumArchiveDays", out setting) || !int.TryParse(setting, out m_maximumArchiveDays))
m_maximumArchiveDays = DefaultMaximumArchiveDays;
if (!settings.TryGetValue("DirectoryNamingMode", out setting) || !Enum.TryParse(setting, true, out m_directoryNamingMode))
DirectoryNamingMode = DefaultDirectoryNamingMode;
// Handle advanced settings - there are hidden but available from manual entry into connection string
int stagingCount, diskFlushInterval, cacheFlushInterval;
if (!settings.TryGetValue("StagingCount", out setting) || !int.TryParse(setting, out stagingCount))
stagingCount = 3;
if (!settings.TryGetValue("DiskFlushInterval", out setting) || !int.TryParse(setting, out diskFlushInterval))
diskFlushInterval = 10000;
if (!settings.TryGetValue("CacheFlushInterval", out setting) || !int.TryParse(setting, out cacheFlushInterval))
cacheFlushInterval = 100;
// Establish archive information for this historian instance
m_archiveInfo = new HistorianServerDatabaseConfig(InstanceName, WorkingDirectory, true);
if ((object)m_archiveDirectories != null)
m_archiveInfo.FinalWritePaths.AddRange(m_archiveDirectories);
if ((object)m_attachedPaths != null)
m_archiveInfo.ImportPaths.AddRange(m_attachedPaths);
m_archiveInfo.TargetFileSize = (long)(targetFileSize * SI.Giga);
m_archiveInfo.DirectoryMethod = DirectoryNamingMode;
m_archiveInfo.StagingCount = stagingCount;
m_archiveInfo.DiskFlushInterval = diskFlushInterval;
m_archiveInfo.CacheFlushInterval = cacheFlushInterval;
// Provide web service support
m_dataServices = new DataServices();
m_dataServices.AdapterCreated += DataServices_AdapterCreated;
m_dataServices.AdapterLoaded += DataServices_AdapterLoaded;
m_dataServices.AdapterUnloaded += DataServices_AdapterUnloaded;
m_dataServices.AdapterLoadException += AdapterLoader_AdapterLoadException;
// Provide archive replication support
m_replicationProviders = new ReplicationProviders();
m_replicationProviders.AdapterCreated += ReplicationProviders_AdapterCreated;
m_replicationProviders.AdapterLoaded += ReplicationProviders_AdapterLoaded;
m_replicationProviders.AdapterUnloaded += ReplicationProviders_AdapterUnloaded;
m_replicationProviders.AdapterLoadException += AdapterLoader_AdapterLoadException;
if (MaximumArchiveDays > 0)
{
m_dailyTimer = new Timer(Time.SecondsPerDay * 1000.0D);
m_dailyTimer.AutoReset = true;
m_dailyTimer.Elapsed += m_dailyTimer_Elapsed;
m_dailyTimer.Enabled = true;
}
}
public SnapServer CreateServer()
{
var settings = new HistorianServerDatabaseConfig("PPA", @"C:\Temp\Synchrophasor", true);
var server = new SnapServer(settings);
return server;
}
public HistorianServer(HistorianServerDatabaseConfig database, int?port = null)
: this(port)
{
AddDatabase(database);
}
public void TestRollover()
{
Logger.Console.Verbose = VerboseLevel.All;
Globals.MemoryPool.SetMaximumBufferSize(4000 * 1024 * 1024L);
foreach (var file in Directory.GetFiles("c:\\temp\\Test\\", "*.*", SearchOption.AllDirectories))
File.Delete(file);
PointCount = 0;
var settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\Test\\Main\\", true);
settings.FinalWritePaths.Add("c:\\temp\\Test\\Rollover\\");
ulong time = (ulong)DateTime.Now.Ticks;
using (var engine = new SnapServer(settings))
using (var client = SnapClient.Connect(engine))
using (var db = client.GetDatabase<HistorianKey, HistorianValue>("DB"))
{
Thread.Sleep(100);
var key = new HistorianKey();
var value = new HistorianValue();
for (int x = 0; x < 100000000; x++)
{
if (x % 100 == 0)
Thread.Sleep(10);
key.Timestamp = (ulong)time;
time += TimeSpan.TicksPerMinute;
db.Write(key, value);
}
}
GC.Collect();
GC.WaitForPendingFinalizers();
Thread.Sleep(100);
}
public void TestReadData()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
var settings = new HistorianServerDatabaseConfig("PPA", @"c:\temp\Scada\", true);
using (HistorianServer server = new HistorianServer(settings))
{
double count = 0;
DebugStopwatch sw = new DebugStopwatch();
double time = sw.TimeEvent(() =>
{
count = 0;
using (HistorianClient client = new HistorianClient("127.0.0.1", 12345))
using (ClientDatabaseBase<HistorianKey, HistorianValue> database = client.GetDatabase<HistorianKey, HistorianValue>(String.Empty))
{
//IHistorianDatabase<HistorianKey, HistorianValue> database = server.GetDefaultDatabase();//.GetDatabase();
//TreeStream<HistorianKey, HistorianValue> stream = reader.Read(0, ulong.MaxValue, new ulong[] { 2 });
TreeStream<HistorianKey, HistorianValue> stream = database.Read(0, ulong.MaxValue);
while (stream.Read(key,value))
{
count++;
}
}
});
Console.WriteLine((count / 1000000 / time).ToString() + " Million PPS");
}
//Console.WriteLine("KeyMethodsBase calls");
//for (int x = 0; x < 23; x++)
//{
// Console.WriteLine(TreeKeyMethodsBase<HistorianKey>.CallMethods[x] + "\t" + ((TreeKeyMethodsBase<HistorianKey>.Method)(x)).ToString());
//}
//Console.WriteLine("ValueMethodsBase calls");
//for (int x = 0; x < 5; x++)
//{
// Console.WriteLine(TreeValueMethodsBase<HistorianValue>.CallMethods[x] + "\t" + ((TreeValueMethodsBase<HistorianValue>.Method)(x)).ToString());
//}
//for (int x = 0; x < 15; x++)
//{
// Console.WriteLine(BinaryStreamBase.CallMethods[x] + "\t" + ((BinaryStreamBase.Method)(x)).ToString());
//}
}
public HistorianServer(HistorianServerDatabaseConfig database, int? port = null)
: this(port)
{
AddDatabase(database);
}
public HistorianHost()
{
Directory.GetFiles(@"G:\HistorianData\", "*.d2").ToList().ForEach(File.Delete);
var settings = new HistorianServerDatabaseConfig("DB", @"G:\HistorianData\", true);
m_server = new HistorianServer(settings);
}
public void TestWriteServer()
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
Logger.Console.Verbose = VerboseLevel.All;
Logger.FileWriter.SetPath(@"C:\Temp\", VerboseLevel.All);
var netStream = new NetworkStreamSimulator();
var dbcfg = new HistorianServerDatabaseConfig("DB", @"C:\Temp\Scada", true);
var server = new HistorianServer(dbcfg);
var auth = new SecureStreamServer<SocketUserPermissions>();
auth.SetDefaultUser(true, new SocketUserPermissions()
{
CanRead = true,
CanWrite = true,
IsAdmin = true
});
var netServer = new SnapStreamingServer(auth, netStream.ServerStream, server.Host);
ThreadPool.QueueUserWorkItem(ProcessClient, netServer);
var client = new SnapStreamingClient(netStream.ClientStream, new SecureStreamClientDefault(), false);
var db = client.GetDatabase<HistorianKey, HistorianValue>("DB");
for (uint x = 0; x < 1000; x++)
{
key.Timestamp = x;
db.Write(key, value);
break;
}
client.Dispose();
server.Dispose();
}
/// <summary>
/// Adds the supplied database to this server.
/// </summary>
/// <param name="database"></param>
public void AddDatabase(HistorianServerDatabaseConfig database)
{
m_host.AddDatabase(database);
}
public void TestWriteSpeed()
{
//Logger.ReportToConsole(VerboseLevel.All ^ VerboseLevel.DebugLow);
//Logger.SetLoggingPath("c:\\temp\\");
Globals.MemoryPool.SetMaximumBufferSize(4000 * 1024 * 1024L);
//Thread th = new Thread(WriteSpeed);
//th.IsBackground = true;
//th.Start();
//Quit = false;
foreach (var file in Directory.GetFiles("c:\\temp\\benchmark\\", "*.*", SearchOption.AllDirectories))
File.Delete(file);
//PointCount = 0;
var settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\benchmark\\", true);
using (var engine = new SnapServer(settings))
using (var client = SnapClient.Connect(engine))
using (var db = client.GetDatabase<HistorianKey, HistorianValue>("DB"))
{
Thread.Sleep(100);
var key = new HistorianKey();
var value = new HistorianValue();
Stopwatch sw = new Stopwatch();
sw.Start();
for (int x = 0; x < PointsToArchive; x++)
{
key.PointID = (ulong)x;
//PointCount = x;
db.Write(key, value);
}
double totalTime = sw.Elapsed.TotalSeconds;
Console.WriteLine("Completed write test in {0:#,##0.00} seconds at {1:#,##0.00} points per second", totalTime, PointsToArchive / totalTime);
}
//Quit = true;
//th.Join();
//Console.WriteLine("Time (sec)\tPoints");
//foreach (var kvp in PointSamples)
//{
// Console.WriteLine(kvp.Key.ToString() + "\t" + kvp.Value.ToString());
//}
GC.Collect();
GC.WaitForPendingFinalizers();
Thread.Sleep(100);
}
public void TestWriteSpeedRandom()
{
Logger.Console.Verbose = VerboseLevel.All;
Random r = new Random(1);
Thread th = new Thread(WriteSpeed);
th.IsBackground = true;
th.Start();
Quit = false;
foreach (var file in Directory.GetFiles("c:\\temp\\benchmark\\"))
File.Delete(file);
PointCount = 0;
var settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\benchmark\\", true);
using (var engine = new SnapServer(settings))
using (var client = SnapClient.Connect(engine))
using (var db = client.GetDatabase<HistorianKey, HistorianValue>("DB"))
{
Thread.Sleep(100);
var key = new HistorianKey();
var value = new HistorianValue();
for (int x = 0; x < 10000000; x++)
{
key.Timestamp = (ulong)r.Next();
key.PointID = (ulong)x;
PointCount = x;
db.Write(key, value);
}
}
Quit = true;
th.Join();
Console.WriteLine("Time (sec)\tPoints");
foreach (var kvp in PointSamples)
{
Console.WriteLine(kvp.Key.ToString() + "\t" + kvp.Value.ToString());
}
}
public static void TestReadPoints2()
{
int pointCount = 0;
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
var settings = new HistorianServerDatabaseConfig("PPA", @"C:\Program Files\openHistorian\Archive\", true);
using (HistorianServer server = new HistorianServer(settings))
{
Stopwatch sw = new Stopwatch();
sw.Start();
using (var client = new HistorianClient("127.0.0.1", 12345))
using (var database = client.GetDatabase<HistorianKey, HistorianValue>(String.Empty))
{
var stream = database.Read(0, (ulong)DateTime.MaxValue.Ticks, new ulong[] { 65, 953, 5562 });
while (stream.Read(key, value))
pointCount++;
}
sw.Stop();
//MessageBox.Show(sw.Elapsed.TotalSeconds.ToString());
}
}