public SnapServer CreateServer()
{
var settings = new HistorianServerDatabaseConfig("PPA", @"C:\Temp\Synchrophasor", true);
var server = new SnapServer(settings);
return(server);
}
/// <summary>
/// Releases the unmanaged resources used by the <see cref="SnapSocketListener"/> object and optionally releases the managed resources.
/// </summary>
/// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param>
protected override void Dispose(bool disposing)
{
if (!m_disposed)
{
try
{
// This will be done regardless of whether the object is finalized or disposed.
if (disposing)
{
m_isRunning = false;
if (m_listener != null)
{
m_listener.Stop();
}
m_server = null;
lock (m_clients)
{
foreach (SnapNetworkServer client in m_clients)
{
client.Dispose();
}
m_clients.Clear();
}
// This will be done only when the object is disposed by calling Dispose().
}
}
finally
{
m_disposed = true; // Prevent duplicate dispose.
base.Dispose(disposing); // Call base class Dispose().
}
}
}
/// <summary>
/// Read historian data from server.
/// </summary>
/// <param name="instanceName">Historian instance name.</param>
/// <param name="startTime">Start time of query.</param>
/// <param name="stopTime">Stop time of query.</param>
/// <param name="measurementIDs">Measurement IDs to query - or <c>null</c> for all available points.</param>
/// <param name="resolution">Resolution for data query.</param>
/// <param name="seriesLimit">Maximum number of points per series.</param>
/// <param name="forceLimit">Flag that determines if series limit should be strictly enforced.</param>
/// <param name="timestampType">Type of timestamps.</param>
/// <returns>Enumeration of <see cref="TrendValue"/> instances read for time range.</returns>
public IEnumerable <TrendValue> GetHistorianData(string instanceName, DateTime startTime, DateTime stopTime, ulong[] measurementIDs, Resolution resolution, int seriesLimit, bool forceLimit, TimestampType timestampType = TimestampType.UnixMilliseconds)
{
// Cancel any running operation
CancellationToken cancellationToken = new CancellationToken();
Interlocked.Exchange(ref m_readCancellationToken, cancellationToken)?.Cancel();
SnapServer server = GetServer(instanceName)?.Host;
IEnumerable <TrendValue> values = TrendValueAPI.GetHistorianData(server, instanceName, startTime, stopTime, measurementIDs, resolution, seriesLimit, forceLimit, cancellationToken);
switch (timestampType)
{
case TimestampType.Ticks:
return(values.Select(value =>
{
value.Timestamp = value.Timestamp * 10000.0D + 621355968000000000.0D;
return value;
}));
case TimestampType.UnixSeconds:
return(values.Select(value =>
{
value.Timestamp = value.Timestamp / 1000.0D;
return value;
}));
default:
return(values);
}
}
public void TestArchiveWriter()
{
Random r = new Random(3);
_ = new EncodingDefinition(EncodingDefinition.FixedSizeIndividualGuid, EncodingDefinition.FixedSizeIndividualGuid);
AdvancedServerDatabaseConfig <AmiKey, AmiKey> config = new AdvancedServerDatabaseConfig <AmiKey, AmiKey>("KV2CPQ", "C:\\Temp\\AMI", true);
using (SnapServer server = new SnapServer(config))
{
using (SnapClient client = SnapClient.Connect(server))
using (ClientDatabaseBase <AmiKey, AmiKey> db = client.GetDatabase <AmiKey, AmiKey>("KV2CPQ"))
{
int count = 10000000;
Stopwatch sw = new Stopwatch();
sw.Start();
AmiKey key = new AmiKey();
AmiKey value = new AmiKey();
for (int x = count; x >= 0; x--)
{
key.Timestamp = (ulong)r.Next();
key.TableId = r.Next();
db.Write(key, value);
}
sw.Stop();
Console.WriteLine(count / sw.Elapsed.TotalSeconds / 1000000);
Console.WriteLine(count);
}
}
}
/// <summary>
/// Creates a <see cref="SnapStreamingServer"/>
/// </summary>
protected void Initialize(SecureStreamServer <SocketUserPermissions> authentication, Stream stream, SnapServer server, bool requireSsl)
{
RequireSSL = requireSsl;
m_rawStream = stream;
m_authentication = authentication;
m_server = server;
}
protected override List <TimeSeriesValues> QueryTimeSeriesValues(DateTime startTime, DateTime stopTime, int maxDataPoints, Dictionary <ulong, string> targetMap, CancellationToken cancellationToken)
{
Dictionary <ulong, TimeSeriesValues> queriedTimeSeriesValues = new Dictionary <ulong, TimeSeriesValues>();
if (targetMap.Count > 0)
{
SnapServer server = GetAdapterInstance(InstanceName)?.Server?.Host;
if ((object)server != null)
{
ulong[] measurementIDs = targetMap.Keys.ToArray();
Resolution resolution = TrendValueAPI.EstimatePlotResolution(InstanceName, startTime, stopTime, measurementIDs);
using (SnapClient connection = SnapClient.Connect(server))
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(InstanceName))
{
foreach (TrendValue trendValue in TrendValueAPI.GetHistorianData(database, startTime, stopTime, measurementIDs, resolution, maxDataPoints, false, (CompatibleCancellationToken)cancellationToken))
{
queriedTimeSeriesValues.GetOrAdd((ulong)trendValue.ID, id => new TimeSeriesValues {
target = targetMap[id], datapoints = new List <double[]>()
})
.datapoints.Add(new[] { trendValue.Value, trendValue.Timestamp });
}
}
}
}
return(queriedTimeSeriesValues.Values.ToList());
}
public SnapNetworkServer(SecureStreamServer<SocketUserPermissions> authentication, TcpClient client, SnapServer server, bool requireSsl = false)
{
m_client = client;
m_rawStream = new NetworkStream(m_client.Client);
m_client.Client.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.NoDelay, true);
Initialize(authentication, m_rawStream, server, requireSsl);
}
/// <summary>
/// Creates a <see cref="SnapSocketListener"/>
/// </summary>
/// <param name="settings"></param>
/// <param name="server"></param>
/// <param name="parent"></param>
public SnapSocketListener(SnapSocketListenerSettings settings, SnapServer server)
: base(MessageClass.Framework)
{
if ((object)server == null)
{
throw new ArgumentNullException("server");
}
if ((object)settings == null)
{
throw new ArgumentNullException("settings");
}
m_server = server;
m_settings = settings.CloneReadonly();
m_settings.Validate();
m_authenticator = new SecureStreamServer <SocketUserPermissions>();
if (settings.DefaultUserCanRead || settings.DefaultUserCanWrite || settings.DefaultUserIsAdmin)
{
m_authenticator.SetDefaultUser(true, new SocketUserPermissions()
{
CanRead = settings.DefaultUserCanRead,
CanWrite = settings.DefaultUserCanWrite,
IsAdmin = settings.DefaultUserIsAdmin
});
}
foreach (var user in settings.Users)
{
m_authenticator.AddUserIntegratedSecurity(user, new SocketUserPermissions()
{
CanRead = true,
CanWrite = true,
IsAdmin = true
});
}
// TODO: Shouldn't we use GSF.Communications async library here for scalability? If not, why not?
// TODO: I think async communication classes could pass NetworkBinaryStream to a handler like ProcessClient...
// TODO: Communications library may need a simple modification... Check with S. Wills for thoughts here...
m_isRunning = true;
m_listener = new TcpListener(m_settings.LocalEndPoint);
//m_listener.Server.DualMode = true;
m_listener.Start();
//var socket = m_listener.AcceptSocketAsync();
//socket.ContinueWith(ProcessDataRequests);
//socket.Start();
Log.Publish(MessageLevel.Info, "Constructor Called", "Listening on " + m_settings.LocalEndPoint.ToString());
Thread th = new Thread(ProcessDataRequests);
th.IsBackground = true;
th.Start();
}
public void TestWriteSpeed()
{
//Logger.ReportToConsole(VerboseLevel.All ^ VerboseLevel.DebugLow);
//Logger.SetLoggingPath("c:\\temp\\");
Globals.MemoryPool.SetMaximumBufferSize(4000 * 1024 * 1024L);
//Thread th = new Thread(WriteSpeed);
//th.IsBackground = true;
//th.Start();
//Quit = false;
foreach (string file in Directory.GetFiles("c:\\temp\\benchmark\\", "*.*", SearchOption.AllDirectories))
{
File.Delete(file);
}
//PointCount = 0;
HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\benchmark\\", true);
using (SnapServer engine = new SnapServer(settings))
using (SnapClient client = SnapClient.Connect(engine))
using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("DB"))
{
Thread.Sleep(100);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
Stopwatch sw = new Stopwatch();
sw.Start();
for (int x = 0; x < PointsToArchive; x++)
{
key.PointID = (ulong)x;
//PointCount = x;
db.Write(key, value);
}
double totalTime = sw.Elapsed.TotalSeconds;
Console.WriteLine("Completed write test in {0:#,##0.00} seconds at {1:#,##0.00} points per second", totalTime, PointsToArchive / totalTime);
}
//Quit = true;
//th.Join();
//Console.WriteLine("Time (sec)\tPoints");
//foreach (var kvp in PointSamples)
//{
// Console.WriteLine(kvp.Key.ToString() + "\t" + kvp.Value.ToString());
//}
GC.Collect();
GC.WaitForPendingFinalizers();
Thread.Sleep(100);
}
public void CreateHistorian()
{
Logger.Console.Verbose = VerboseLevel.All;
using (SnapServer server = CreateServer())
{
}
GC.Collect();
GC.WaitForPendingFinalizers();
Thread.Sleep(1000);
}
/// <summary>
/// Creates a new <see cref="ReportHistorianReader"/>.
/// </summary>
/// <param name="server">Snapserver to connect to <see cref="SnapServer"/>.</param>
/// <param name="instanceName">Name of the instance to connect to.</param>
/// <param name="startTime">Starttime.</param>
/// <param name="endTime">Endtime.</param>
/// <param name="frameRate">SamplingRate of the signal.</param>
/// <param name="pointIDs">PointIDs to be collected.</param>
public ReportHistorianReader(SnapServer server, string instanceName, DateTime startTime, DateTime endTime, int frameRate, IEnumerable <ulong> pointIDs)
{
m_client = SnapClient.Connect(server);
m_database = m_client.GetDatabase <HistorianKey, HistorianValue>(instanceName);
m_key = new HistorianKey();
m_value = new HistorianValue();
SeekFilterBase <HistorianKey> timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(DataPoint.RoundTimestamp(startTime, frameRate), DataPoint.RoundTimestamp(endTime, frameRate));
MatchFilterBase <HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(pointIDs);
m_stream = m_database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter);
}
public void AttachFile()
{
Logger.Console.Verbose = VerboseLevel.All;
using (SnapServer server = CreateServer())
{
using (SnapClient client = SnapClient.Connect(server))
{
client.GetDatabase("PPA").AttachFilesOrPaths(new string[] { @"C:\Temp\Synchrophasor\Dir\File2.d2" });
}
}
}
/// <summary>
/// Creates a <see cref="SnapSocketListener"/>
/// </summary>
/// <param name="settings"></param>
/// <param name="server"></param>
/// <param name="parent"></param>
public SnapSocketListener(SnapSocketListenerSettings settings, SnapServer server)
: base(MessageClass.Framework)
{
if ((object)server == null)
{
throw new ArgumentNullException(nameof(server));
}
if ((object)settings == null)
{
throw new ArgumentNullException(nameof(settings));
}
m_server = server;
m_settings = settings.CloneReadonly();
m_settings.Validate();
m_authenticator = new SecureStreamServer <SocketUserPermissions>();
if (settings.DefaultUserCanRead || settings.DefaultUserCanWrite || settings.DefaultUserIsAdmin)
{
m_authenticator.SetDefaultUser(true, new SocketUserPermissions
{
CanRead = settings.DefaultUserCanRead,
CanWrite = settings.DefaultUserCanWrite,
IsAdmin = settings.DefaultUserIsAdmin
});
}
foreach (string user in settings.Users)
{
m_authenticator.AddUserIntegratedSecurity(user, new SocketUserPermissions
{
CanRead = true,
CanWrite = true,
IsAdmin = true
});
}
m_isRunning = true;
m_listener = new TcpListener(m_settings.LocalEndPoint);
m_listener.Server.DualMode = m_settings.LocalEndPoint.AddressFamily == AddressFamily.InterNetworkV6;
m_listener.Start();
Log.Publish(MessageLevel.Info, "Constructor Called", $"Listening on {m_settings.LocalEndPoint}");
new Thread(ProcessDataRequests)
{
IsBackground = true
}.Start();
}
public void ReadData()
{
Logger.Console.Verbose = VerboseLevel.All;
using (SnapServer server = CreateServer())
{
using (SnapClient client = SnapClient.Connect(server))
using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("PPA"))
using (TreeStream <HistorianKey, HistorianValue> stream = db.Read(null, null, null))
{
Console.WriteLine(stream.Count());
}
}
}
/// <summary>
/// Creates a <see cref="SnapSocketListener"/>
/// </summary>
/// <param name="settings"></param>
/// <param name="server"></param>
/// <param name="parent"></param>
public SnapSocketListener(SnapSocketListenerSettings settings, SnapServer server)
: base(MessageClass.Framework)
{
if ((object)server == null)
throw new ArgumentNullException("server");
if ((object)settings == null)
throw new ArgumentNullException("settings");
m_server = server;
m_settings = settings.CloneReadonly();
m_settings.Validate();
m_authenticator = new SecureStreamServer<SocketUserPermissions>();
if (settings.DefaultUserCanRead || settings.DefaultUserCanWrite || settings.DefaultUserIsAdmin)
{
m_authenticator.SetDefaultUser(true, new SocketUserPermissions()
{
CanRead = settings.DefaultUserCanRead,
CanWrite = settings.DefaultUserCanWrite,
IsAdmin = settings.DefaultUserIsAdmin
});
}
foreach (var user in settings.Users)
{
m_authenticator.AddUserIntegratedSecurity(user, new SocketUserPermissions()
{
CanRead = true,
CanWrite = true,
IsAdmin = true
});
}
// TODO: Shouldn't we use GSF.Communications async library here for scalability? If not, why not?
// TODO: I think async communication classes could pass NetworkBinaryStream to a handler like ProcessClient...
// TODO: Communications library may need a simple modification... Check with S. Wills for thoughts here...
m_isRunning = true;
m_listener = new TcpListener(m_settings.LocalEndPoint);
//m_listener.Server.DualMode = true;
m_listener.Start();
//var socket = m_listener.AcceptSocketAsync();
//socket.ContinueWith(ProcessDataRequests);
//socket.Start();
Log.Publish(MessageLevel.Info, "Constructor Called", "Listening on " + m_settings.LocalEndPoint.ToString());
Thread th = new Thread(ProcessDataRequests);
th.IsBackground = true;
th.Start();
}
public void WriteData()
{
Logger.Console.Verbose = VerboseLevel.All;
using (SnapServer server = CreateServer())
{
using (SnapClient client = SnapClient.Connect(server))
using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("PPA"))
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
key.TimestampAsDate = DateTime.Now;
key.PointID = LittleEndian.ToUInt64(Guid.NewGuid().ToByteArray(), 0);
db.Write(key, value);
}
}
}
public void GetAllFiles()
{
Logger.Console.Verbose = VerboseLevel.All;
using (SnapServer server = CreateServer())
{
using (SnapClient client = SnapClient.Connect(server))
using (ClientDatabaseBase db = client.GetDatabase("PPA"))
{
foreach (ArchiveDetails f in db.GetAllAttachedFiles())
{
Console.WriteLine("{0}MB {1} TO {2}; ID:{3} Name: {4}",
(f.FileSize / 1024d / 1024d).ToString("0.0"),
f.FirstKey, f.LastKey, f.Id, f.FileName);
}
}
}
}
/// <summary>
/// Creates a new <see cref="HistorianServer"/> instance.
/// </summary>
public HistorianServer(int?port)
{
var server = new ServerSettings();
if (port.HasValue)
{
var settings = new SnapSocketListenerSettings()
{
LocalTcpPort = port.Value
};
settings.DefaultUserCanRead = true;
settings.DefaultUserCanWrite = true;
settings.DefaultUserIsAdmin = true;
server.Listeners.Add(settings);
}
// Maintain a member level list of all established archive database engines
m_host = new SnapServer(server);
}
public void TestWriteSpeedRandom()
{
Logger.Console.Verbose = VerboseLevel.All;
Random r = new Random(1);
Thread th = new Thread(WriteSpeed);
th.IsBackground = true;
th.Start();
Quit = false;
foreach (string file in Directory.GetFiles("c:\\temp\\benchmark\\"))
{
File.Delete(file);
}
PointCount = 0;
HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\benchmark\\", true);
using (SnapServer engine = new SnapServer(settings))
using (SnapClient client = SnapClient.Connect(engine))
using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("DB"))
{
Thread.Sleep(100);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
for (int x = 0; x < 10000000; x++)
{
key.Timestamp = (ulong)r.Next();
key.PointID = (ulong)x;
PointCount = x;
db.Write(key, value);
}
}
Quit = true;
th.Join();
Console.WriteLine("Time (sec)\tPoints");
foreach (KeyValuePair <double, int> kvp in PointSamples)
{
Console.WriteLine(kvp.Key.ToString() + "\t" + kvp.Value.ToString());
}
}
public void TestRollover()
{
Logger.Console.Verbose = VerboseLevel.All;
Globals.MemoryPool.SetMaximumBufferSize(4000 * 1024 * 1024L);
foreach (string file in Directory.GetFiles("c:\\temp\\Test\\", "*.*", SearchOption.AllDirectories))
{
File.Delete(file);
}
PointCount = 0;
HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\Test\\Main\\", true);
settings.FinalWritePaths.Add("c:\\temp\\Test\\Rollover\\");
ulong time = (ulong)DateTime.Now.Ticks;
using (SnapServer engine = new SnapServer(settings))
using (SnapClient client = SnapClient.Connect(engine))
using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("DB"))
{
Thread.Sleep(100);
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
for (int x = 0; x < 100000000; x++)
{
if (x % 100 == 0)
{
Thread.Sleep(10);
}
key.Timestamp = time;
time += TimeSpan.TicksPerMinute;
db.Write(key, value);
}
}
GC.Collect();
GC.WaitForPendingFinalizers();
Thread.Sleep(100);
}
public void DetatchFiles()
{
Logger.Console.Verbose = VerboseLevel.All;
using (SnapServer server = CreateServer())
{
using (SnapClient client = SnapClient.Connect(server))
using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("PPA"))
{
using (TreeStream <HistorianKey, HistorianValue> stream = db.Read(null, null, null))
{
Console.WriteLine(stream.Count());
}
db.DetatchFiles(db.GetAllAttachedFiles().Select(x => x.Id).ToList());
using (TreeStream <HistorianKey, HistorianValue> stream = db.Read(null, null, null))
{
Console.WriteLine(stream.Count());
}
}
}
}
/// <summary>
/// Creates a new <see cref="HistorianServer"/> instance.
/// </summary>
public HistorianServer(int?port, string networkInterfaceIP = null)
{
ServerSettings server = new ServerSettings();
if (port.HasValue || !string.IsNullOrWhiteSpace(networkInterfaceIP))
{
SnapSocketListenerSettings settings = new SnapSocketListenerSettings
{
LocalTcpPort = port ?? SnapSocketListenerSettings.DefaultNetworkPort,
LocalIpAddress = networkInterfaceIP,
DefaultUserCanRead = true,
DefaultUserCanWrite = true,
DefaultUserIsAdmin = true
};
server.Listeners.Add(settings);
}
// Maintain a member level list of all established archive database engines
Host = new SnapServer(server);
}
/// <summary>
/// Releases the unmanaged resources used by the <see cref="SnapSocketListener"/> object and optionally releases the managed resources.
/// </summary>
/// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param>
protected override void Dispose(bool disposing)
{
if (!m_disposed)
{
try
{
if (!disposing)
{
return;
}
m_isRunning = false;
if (m_listener != null)
{
m_listener.Stop();
}
m_server = null;
lock (m_clients)
{
foreach (SnapNetworkServer client in m_clients)
{
client.Dispose();
}
m_clients.Clear();
}
}
finally
{
m_disposed = true; // Prevent duplicate dispose.
base.Dispose(disposing); // Call base class Dispose().
}
}
}
/// <summary>
/// Read historian data from server.
/// </summary>
/// <param name="server">The server to use for the query.</param>
/// <param name="instanceName">Name of the archive to be queried.</param>
/// <param name="startTime">Start time of query.</param>
/// <param name="stopTime">Stop time of query.</param>
/// <param name="measurementIDs">Measurement IDs to query - or <c>null</c> for all available points.</param>
/// <param name="resolution">Resolution for data query.</param>
/// <param name="seriesLimit">Maximum number of points per series.</param>
/// <param name="forceLimit">Flag that determines if series limit should be strictly enforced.</param>
/// <param name="cancellationToken">Cancellation token for query.</param>
/// <returns>Enumeration of <see cref="TrendValue"/> instances read for time range.</returns>
public static IEnumerable <TrendValue> GetHistorianData(SnapServer server, string instanceName, DateTime startTime, DateTime stopTime, ulong[] measurementIDs, Resolution resolution, int seriesLimit, bool forceLimit, ICancellationToken cancellationToken = null)
{
if (cancellationToken == null)
{
cancellationToken = new CancellationToken();
}
if (server == null)
{
yield break;
}
// Setting series limit to zero requests full resolution data, which overrides provided parameter
if (seriesLimit < 1)
{
resolution = Resolution.Full;
}
TimeSpan resolutionInterval = resolution.GetInterval();
SeekFilterBase <HistorianKey> timeFilter;
MatchFilterBase <HistorianKey, HistorianValue> pointFilter = null;
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
// Set data scan resolution
if (resolution == Resolution.Full)
{
timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, stopTime);
}
else
{
BaselineTimeInterval interval = BaselineTimeInterval.Second;
if (resolutionInterval.Ticks < Ticks.PerMinute)
{
interval = BaselineTimeInterval.Second;
}
else if (resolutionInterval.Ticks < Ticks.PerHour)
{
interval = BaselineTimeInterval.Minute;
}
else if (resolutionInterval.Ticks == Ticks.PerHour)
{
interval = BaselineTimeInterval.Hour;
}
startTime = startTime.BaselinedTimestamp(interval);
stopTime = stopTime.BaselinedTimestamp(interval);
timeFilter = TimestampSeekFilter.CreateFromIntervalData <HistorianKey>(startTime, stopTime, resolutionInterval, new TimeSpan(TimeSpan.TicksPerMillisecond));
}
Dictionary <ulong, DataRow> metadata = null;
using (SnapClient connection = SnapClient.Connect(server))
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(instanceName))
{
if (database == null)
{
yield break;
}
if (LocalOutputAdapter.Instances.TryGetValue(database.Info?.DatabaseName ?? DefaultInstanceName, out LocalOutputAdapter historianAdapter))
{
metadata = historianAdapter?.Measurements;
}
if (metadata == null)
{
yield break;
}
// Setup point ID selections
if (measurementIDs != null)
{
pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(measurementIDs);
}
else
{
measurementIDs = metadata.Keys.ToArray();
}
// Start stream reader for the provided time window and selected points
Dictionary <ulong, long> pointCounts = new Dictionary <ulong, long>(measurementIDs.Length);
Dictionary <ulong, long> intervals = new Dictionary <ulong, long>(measurementIDs.Length);
Dictionary <ulong, ulong> lastTimes = new Dictionary <ulong, ulong>(measurementIDs.Length);
double range = (stopTime - startTime).TotalSeconds;
ulong pointID, timestamp, resolutionSpan = (ulong)resolutionInterval.Ticks, baseTicks = (ulong)UnixTimeTag.BaseTicks.Value;
long pointCount;
if (resolutionSpan <= 1UL)
{
resolutionSpan = Ticks.PerSecond;
}
if (seriesLimit < 1)
{
seriesLimit = 1;
}
// Estimate total measurement counts per point so decimation intervals for each series can be calculated
foreach (ulong measurementID in measurementIDs)
{
if (resolution == Resolution.Full)
{
pointCounts[measurementID] = metadata.TryGetValue(measurementID, out DataRow row) ? (long)(int.Parse(row["FramesPerSecond"].ToString()) * range) : 2;
}
else
{
pointCounts[measurementID] = (long)(range / resolutionInterval.TotalSeconds.NotZero(1.0D));
}
}
foreach (ulong measurementID in pointCounts.Keys)
{
intervals[measurementID] = (pointCounts[measurementID] / seriesLimit).NotZero(1L);
}
using (TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter))
{
while (stream.Read(key, value) && !cancellationToken.IsCancelled)
{
pointID = key.PointID;
timestamp = key.Timestamp;
pointCount = pointCounts[pointID];
if (pointCount++ % intervals[pointID] == 0 || !forceLimit && timestamp - lastTimes.GetOrAdd(pointID, 0UL) > resolutionSpan)
{
yield return new TrendValue
{
ID = (long)pointID,
Timestamp = (timestamp - baseTicks) / (double)Ticks.PerMillisecond,
Value = value.AsSingle
}
}
;
pointCounts[pointID] = pointCount;
lastTimes[pointID] = timestamp;
}
}
}
}
}
/// <summary>
/// Starts a query that will read data source values, given a set of point IDs and targets, over a time range.
/// </summary>
/// <param name="startTime">Start-time for query.</param>
/// <param name="stopTime">Stop-time for query.</param>
/// <param name="interval">Interval from Grafana request.</param>
/// <param name="includePeaks">Flag that determines if decimated data should include min/max interval peaks over provided time range.</param>
/// <param name="targetMap">Set of IDs with associated targets to query.</param>
/// <returns>Queried data source data in terms of value and time.</returns>
protected override IEnumerable <DataSourceValue> QueryDataSourceValues(DateTime startTime, DateTime stopTime, string interval, bool includePeaks, Dictionary <ulong, string> targetMap)
{
SnapServer server = GetAdapterInstance(InstanceName)?.Server?.Host;
if (server == null)
{
yield break;
}
using (SnapClient connection = SnapClient.Connect(server))
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(InstanceName))
{
if (database == null)
{
yield break;
}
if (!TryParseInterval(interval, out TimeSpan resolutionInterval))
{
Resolution resolution = TrendValueAPI.EstimatePlotResolution(InstanceName, startTime, stopTime, targetMap.Keys);
resolutionInterval = resolution.GetInterval();
}
BaselineTimeInterval timeInterval = BaselineTimeInterval.Second;
if (resolutionInterval.Ticks < Ticks.PerMinute)
{
timeInterval = BaselineTimeInterval.Second;
}
else if (resolutionInterval.Ticks < Ticks.PerHour)
{
timeInterval = BaselineTimeInterval.Minute;
}
else if (resolutionInterval.Ticks == Ticks.PerHour)
{
timeInterval = BaselineTimeInterval.Hour;
}
startTime = startTime.BaselinedTimestamp(timeInterval);
stopTime = stopTime.BaselinedTimestamp(timeInterval);
if (startTime == stopTime)
{
stopTime = stopTime.AddSeconds(1.0D);
}
SeekFilterBase <HistorianKey> timeFilter;
// Set timestamp filter resolution
if (includePeaks || resolutionInterval == TimeSpan.Zero)
{
// Full resolution query
timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, stopTime);
}
else
{
// Interval query
timeFilter = TimestampSeekFilter.CreateFromIntervalData <HistorianKey>(startTime, stopTime, resolutionInterval, new TimeSpan(TimeSpan.TicksPerMillisecond));
}
// Setup point ID selections
MatchFilterBase <HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(targetMap.Keys);
Dictionary <ulong, ulong> lastTimes = new Dictionary <ulong, ulong>(targetMap.Count);
Dictionary <ulong, Peak> peaks = new Dictionary <ulong, Peak>(targetMap.Count);
ulong resolutionSpan = (ulong)resolutionInterval.Ticks;
if (includePeaks)
{
resolutionSpan *= 2UL;
}
// Start stream reader for the provided time window and selected points
using (TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter))
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
Peak peak = Peak.Default;
while (stream.Read(key, value))
{
ulong pointID = key.PointID;
ulong timestamp = key.Timestamp;
float pointValue = value.AsSingle;
if (includePeaks)
{
peak = peaks.GetOrAdd(pointID, _ => new Peak());
peak.Set(pointValue, timestamp);
}
if (resolutionSpan > 0UL && timestamp - lastTimes.GetOrAdd(pointID, 0UL) < resolutionSpan)
{
continue;
}
// New value is ready for publication
string target = targetMap[pointID];
MeasurementStateFlags flags = (MeasurementStateFlags)value.Value3;
if (includePeaks)
{
if (peak.MinTimestamp > 0UL)
{
yield return(new DataSourceValue
{
Target = target,
Value = peak.Min,
Time = (peak.MinTimestamp - m_baseTicks) / (double)Ticks.PerMillisecond,
Flags = flags
});
}
if (peak.MaxTimestamp != peak.MinTimestamp)
{
yield return(new DataSourceValue
{
Target = target,
Value = peak.Max,
Time = (peak.MaxTimestamp - m_baseTicks) / (double)Ticks.PerMillisecond,
Flags = flags
});
}
peak.Reset();
}
else
{
yield return(new DataSourceValue
{
Target = target,
Value = pointValue,
Time = (timestamp - m_baseTicks) / (double)Ticks.PerMillisecond,
Flags = flags
});
}
lastTimes[pointID] = timestamp;
}
}
}
}
/// <summary>
/// Starts a query that will read data source values, given a set of point IDs and targets, over a time range.
/// </summary>
/// <param name="startTime">Start-time for query.</param>
/// <param name="stopTime">Stop-time for query.</param>
/// <param name="interval">Interval from Grafana request.</param>
/// <param name="decimate">Flag that determines if data should be decimated over provided time range.</param>
/// <param name="targetMap">Set of IDs with associated targets to query.</param>
/// <returns>Queried data source data in terms of value and time.</returns>
protected override IEnumerable <DataSourceValue> QueryDataSourceValues(DateTime startTime, DateTime stopTime, string interval, bool decimate, Dictionary <ulong, string> targetMap)
{
SnapServer server = GetAdapterInstance(InstanceName)?.Server?.Host;
if ((object)server == null)
{
yield break;
}
using (SnapClient connection = SnapClient.Connect(server))
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(InstanceName))
{
if ((object)database == null)
{
yield break;
}
Resolution resolution = TrendValueAPI.EstimatePlotResolution(InstanceName, startTime, stopTime, targetMap.Keys);
SeekFilterBase <HistorianKey> timeFilter;
// Set data scan resolution
if (!decimate || resolution == Resolution.Full)
{
timeFilter = TimestampSeekFilter.CreateFromRange <HistorianKey>(startTime, stopTime);
}
else
{
TimeSpan resolutionInterval = resolution.GetInterval();
BaselineTimeInterval timeInterval = BaselineTimeInterval.Second;
if (resolutionInterval.Ticks < Ticks.PerMinute)
{
timeInterval = BaselineTimeInterval.Second;
}
else if (resolutionInterval.Ticks < Ticks.PerHour)
{
timeInterval = BaselineTimeInterval.Minute;
}
else if (resolutionInterval.Ticks == Ticks.PerHour)
{
timeInterval = BaselineTimeInterval.Hour;
}
startTime = startTime.BaselinedTimestamp(timeInterval);
stopTime = stopTime.BaselinedTimestamp(timeInterval);
int milliseconds = 1;
try
{
ConfigurationFile configFile = ConfigurationFile.Open(AppDomain.CurrentDomain.SetupInformation.ConfigurationFile);
CategorizedSettingsSection categorizedSettings = configFile.Settings;
CategorizedSettingsElementCollection systemSettings = categorizedSettings["systemSettings"];
string val = systemSettings["HistoryTolerance"].Value;
}
catch { } // something went wrong, so just use original default
timeFilter = TimestampSeekFilter.CreateFromIntervalData <HistorianKey>(startTime, stopTime, resolutionInterval, new TimeSpan(TimeSpan.TicksPerMillisecond * milliseconds));
}
// Setup point ID selections
MatchFilterBase <HistorianKey, HistorianValue> pointFilter = PointIdMatchFilter.CreateFromList <HistorianKey, HistorianValue>(targetMap.Keys);
// Start stream reader for the provided time window and selected points
using (TreeStream <HistorianKey, HistorianValue> stream = database.Read(SortedTreeEngineReaderOptions.Default, timeFilter, pointFilter))
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
while (stream.Read(key, value))
{
yield return(new DataSourceValue
{
Target = targetMap[key.PointID],
Time = (key.Timestamp - m_baseTicks) / (double)Ticks.PerMillisecond,
Value = value.AsSingle,
Flags = (MeasurementStateFlags)value.Value3
});
}
}
}
}
/// <summary>
/// Releases the unmanaged resources used by the <see cref="SnapSocketListener"/> object and optionally releases the managed resources.
/// </summary>
/// <param name="disposing">true to release both managed and unmanaged resources; false to release only unmanaged resources.</param>
protected override void Dispose(bool disposing)
{
if (!m_disposed)
{
try
{
// This will be done regardless of whether the object is finalized or disposed.
if (disposing)
{
m_isRunning = false;
if (m_listener != null)
m_listener.Stop();
m_server = null;
lock (m_clients)
{
foreach (SnapNetworkServer client in m_clients)
client.Dispose();
m_clients.Clear();
}
// This will be done only when the object is disposed by calling Dispose().
}
}
finally
{
m_disposed = true; // Prevent duplicate dispose.
base.Dispose(disposing); // Call base class Dispose().
}
}
}
/// <summary>
/// Begins a new historian write operation.
/// </summary>
/// <param name="instanceName">Historian instance name.</param>
/// <param name="values">Enumeration of <see cref="TrendValue"/> instances to write.</param>
/// <param name="totalValues">Total values to write, if known in advance.</param>
/// <param name="timestampType">Type of timestamps.</param>
/// <returns>New operational state handle.</returns>
public uint BeginHistorianWrite(string instanceName, IEnumerable <TrendValue> values, long totalValues = 0, TimestampType timestampType = TimestampType.UnixMilliseconds)
{
HistorianWriteOperationState operationState = new HistorianWriteOperationState
{
Total = totalValues
};
uint operationHandle = Random.UInt32;
while (!m_historianWriteOperationStates.TryAdd(operationHandle, operationState))
{
operationHandle = Random.UInt32;
}
new Thread(() =>
{
operationState.StartTime = DateTime.UtcNow.Ticks;
try
{
SnapServer server = GetServer(instanceName)?.Host;
if (server == null)
{
throw new InvalidOperationException($"Server is null for instance [{instanceName}].");
}
using (SnapClient connection = SnapClient.Connect(server))
using (ClientDatabaseBase <HistorianKey, HistorianValue> database = connection.GetDatabase <HistorianKey, HistorianValue>(instanceName))
{
if (database == null)
{
throw new InvalidOperationException($"Database is null for instance [{instanceName}].");
}
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
foreach (TrendValue trendValue in values)
{
key.PointID = (ulong)trendValue.ID;
switch (timestampType)
{
case TimestampType.Ticks:
key.Timestamp = (ulong)trendValue.Timestamp;
break;
case TimestampType.UnixSeconds:
key.Timestamp = (ulong)trendValue.Timestamp * 10000000UL + 621355968000000000UL;
break;
case TimestampType.UnixMilliseconds:
key.Timestamp = (ulong)trendValue.Timestamp * 10000UL + 621355968000000000UL;
break;
}
value.AsSingle = (float)trendValue.Value;
database.Write(key, value);
operationState.Progress++;
if (operationState.CancellationToken.IsCancelled)
{
break;
}
}
operationState.Completed = !operationState.CancellationToken.IsCancelled;
}
}
catch (Exception ex)
{
operationState.Failed = true;
operationState.FailedReason = ex.Message;
}
// Schedule operation handle to be removed
CancelHistorianWrite(operationHandle);
operationState.StopTime = DateTime.UtcNow.Ticks;
})
{
IsBackground = true
}
.Start();
return(operationHandle);
}
public SnapNetworkServer(SecureStreamServer <SocketUserPermissions> authentication, TcpClient client, SnapServer server, bool requireSsl = false)
{
m_client = client;
m_rawStream = new NetworkStream(m_client.Client);
m_client.Client.SetSocketOption(SocketOptionLevel.Tcp, SocketOptionName.NoDelay, true);
Initialize(authentication, m_rawStream, server, requireSsl);
}
/// <summary>
/// Reads Data From the Historian and returns condensed DataPoints.
/// </summary>
/// <param name="start">Starttime.</param>
/// <param name="end">EndTime.</param>
/// <param name="measurements">Measurements to be read.</param>
/// <param name="threshold">Threshhold used to determine number of alerts.</param>
/// <param name="cancelationToken">Cancleation Token for the operation.</param>
/// <param name="progress"> Progress Tracker <see cref="IProgress{T}"/>.</param>
public IEnumerable <CondensedDataPoint> ReadCondensed(DateTime start, DateTime end, IEnumerable <ActiveMeasurement> measurements, double threshold, CancellationToken cancelationToken, IProgress <ulong> progress)
{
// Enumerate measurements once
ActiveMeasurement[] activeMeasurements = measurements.ToArray();
List <CondensedDataPoint> result = new List <CondensedDataPoint>(activeMeasurements.Length);
SnapServer server = GetServer(m_instance)?.Host;
// start by separating all framerates
foreach (int frameRate in activeMeasurements.Select(item => item.FramesPerSecond.GetValueOrDefault()).Distinct())
{
ulong[] pointIDs = activeMeasurements.Where(item => item.FramesPerSecond == frameRate).Select(item => item.PointID).ToArray();
Dictionary <ulong, CondensedDataPoint> frameRateResult = new Dictionary <ulong, CondensedDataPoint>();
foreach (ulong key in pointIDs)
{
frameRateResult.Add(key, CondensedDataPoint.EmptyPoint(key));
if (cancelationToken.IsCancellationRequested)
{
return(result);
}
}
using (ReportHistorianReader reader = new ReportHistorianReader(server, m_instance, start, end, frameRate, pointIDs))
{
DataPoint point = new DataPoint();
// this.mProgress.Report(this.m_prevProgress);
// Scan to first record
if (!reader.ReadNext(point))
{
throw new InvalidOperationException("No data for specified time range in openHistorian connection!");
}
ulong currentTimeStamp = point.Timestamp;
while (reader.ReadNext(point))
{
if (currentTimeStamp < point.Timestamp)
{
progress.Report(point.Timestamp);
}
if (cancelationToken.IsCancellationRequested)
{
return(result);
}
if (!float.IsNaN(point.ValueAsSingle))
{
if (frameRateResult.TryGetValue(point.PointID, out CondensedDataPoint dataPoint) && dataPoint != null)
{
if (point.ValueAsSingle > dataPoint.Max)
{
dataPoint.Max = point.ValueAsSingle;
}
if (point.ValueAsSingle < dataPoint.Min)
{
dataPoint.Min = point.ValueAsSingle;
}
dataPoint.Sum += point.ValueAsSingle;
dataPoint.SqrSum += point.ValueAsSingle * point.ValueAsSingle;
dataPoint.TotalPoints++;
if (point.ValueAsSingle > threshold)
{
dataPoint.Alert++;
}
}
}
currentTimeStamp = point.Timestamp;
}
}
result.AddRange(frameRateResult.Where(item => item.Value.TotalPoints > 0).Select(item => { item.Value.PointID = item.Key; return(item.Value); }));
if (cancelationToken.IsCancellationRequested)
{
return(result);
}
}
return(result);
}
/// <summary>
/// Creates a new <see cref="HistorianServer"/>
/// </summary>
public HistorianServer()
{
Host = new SnapServer();
}
public void VerifyDB()
{
//Logger.ReportToConsole(VerboseLevel.All ^ VerboseLevel.DebugLow);
//Logger.ConsoleSubscriber.AddIgnored(Logger.LookupType("GSF.SortedTreeStore"));
Globals.MemoryPool.SetMaximumBufferSize(1000 * 1024 * 1024);
Globals.MemoryPool.SetTargetUtilizationLevel(TargetUtilizationLevels.Low);
HistorianServerDatabaseConfig settings = new HistorianServerDatabaseConfig("DB", "c:\\temp\\benchmark\\", true);
using (SnapServer engine = new SnapServer(settings))
using (SnapClient client = SnapClient.Connect(engine))
using (ClientDatabaseBase <HistorianKey, HistorianValue> db = client.GetDatabase <HistorianKey, HistorianValue>("DB"))
using (TreeStream <HistorianKey, HistorianValue> scan = db.Read(null, null, null))
{
HistorianKey key = new HistorianKey();
HistorianValue value = new HistorianValue();
Stopwatch sw = new Stopwatch();
sw.Start();
for (int x = 0; x < PointsToArchive; x++)
{
if (!scan.Read(key, value))
{
throw new Exception("Missing points");
}
if (key.PointID != (ulong)x)
{
throw new Exception("Corrupt");
}
if (key.Timestamp != 0)
{
throw new Exception("Corrupt");
}
if (key.EntryNumber != 0)
{
throw new Exception("Corrupt");
}
if (value.Value1 != 0)
{
throw new Exception("Corrupt");
}
if (value.Value1 != 0)
{
throw new Exception("Corrupt");
}
if (value.Value1 != 0)
{
throw new Exception("Corrupt");
}
}
double totalTime = sw.Elapsed.TotalSeconds;
Console.WriteLine("Completed read test in {0:#,##0.00} seconds at {1:#,##0.00} points per second", totalTime, PointsToArchive / totalTime);
if (scan.Read(key, value))
{
throw new Exception("too many points");
}
}
}
public SnapStreamingServer(SecureStreamServer <SocketUserPermissions> authentication, Stream stream, SnapServer server, bool requireSsl = false)
: base(MessageClass.Framework)
{
Initialize(authentication, stream, server, requireSsl);
}
